d57c78bc71
This is how I did it: 1. `git clone https://github.com/ooni/probe-engine internal/engine` 2. ``` (cd internal/engine && git describe --tags) v0.23.0 ``` 3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod` 4. `rm -rf internal/.git internal/engine/go.{mod,sum}` 5. `git add internal/engine` 6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;` 7. `go build ./...` (passes) 8. `go test -race ./...` (temporary failure on RiseupVPN) 9. `go mod tidy` 10. this commit message Once this piece of work is done, we can build a new version of `ooniprobe` that is using `internal/engine` directly. We need to do more work to ensure all the other functionality in `probe-engine` (e.g. making mobile packages) are still WAI. Part of https://github.com/ooni/probe/issues/1335
105 lines
2.7 KiB
Go
105 lines
2.7 KiB
Go
// Package resources contains code to download resources.
|
|
package resources
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
|
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
|
)
|
|
|
|
// Client is a client for fetching resources.
|
|
type Client struct {
|
|
// HTTPClient is the HTTP client to use.
|
|
HTTPClient *http.Client
|
|
|
|
// Logger is the logger to use.
|
|
Logger model.Logger
|
|
|
|
// OSMkdirAll allows testing os.MkdirAll failures.
|
|
OSMkdirAll func(path string, perm os.FileMode) error
|
|
|
|
// UserAgent is the user agent to use.
|
|
UserAgent string
|
|
|
|
// WorkDir is the directory where to save resources.
|
|
WorkDir string
|
|
}
|
|
|
|
// Ensure ensures that resources are downloaded and current.
|
|
func (c *Client) Ensure(ctx context.Context) error {
|
|
mkdirall := c.OSMkdirAll
|
|
if mkdirall == nil {
|
|
mkdirall = os.MkdirAll
|
|
}
|
|
if err := mkdirall(c.WorkDir, 0700); err != nil {
|
|
return err
|
|
}
|
|
for name, resource := range All {
|
|
if err := c.EnsureForSingleResource(
|
|
ctx, name, resource, func(real, expected string) bool {
|
|
return real == expected
|
|
},
|
|
gzip.NewReader, ioutil.ReadAll,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// EnsureForSingleResource ensures that a single resource
|
|
// is downloaded and is current.
|
|
func (c *Client) EnsureForSingleResource(
|
|
ctx context.Context, name string, resource ResourceInfo,
|
|
equal func(real, expected string) bool,
|
|
gzipNewReader func(r io.Reader) (*gzip.Reader, error),
|
|
ioutilReadAll func(r io.Reader) ([]byte, error),
|
|
) error {
|
|
fullpath := filepath.Join(c.WorkDir, name)
|
|
data, err := ioutil.ReadFile(fullpath)
|
|
if err == nil {
|
|
sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
|
|
if equal(sha256sum, resource.SHA256) {
|
|
return nil
|
|
}
|
|
c.Logger.Debugf("resources: %s is outdated", fullpath)
|
|
} else {
|
|
c.Logger.Debugf("resources: can't read %s: %s", fullpath, err.Error())
|
|
}
|
|
data, err = (httpx.Client{
|
|
BaseURL: BaseURL,
|
|
HTTPClient: c.HTTPClient,
|
|
Logger: c.Logger,
|
|
UserAgent: c.UserAgent,
|
|
}).FetchResourceAndVerify(ctx, resource.URLPath, resource.GzSHA256)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c.Logger.Debugf("resources: uncompress %s", fullpath)
|
|
gzreader, err := gzipNewReader(bytes.NewReader(data))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer gzreader.Close() // we already have a sha256 for it
|
|
data, err = ioutilReadAll(gzreader) // small file
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
|
|
if equal(sha256sum, resource.SHA256) == false {
|
|
return fmt.Errorf("resources: %s sha256 mismatch", fullpath)
|
|
}
|
|
c.Logger.Debugf("resources: overwrite %s", fullpath)
|
|
return ioutil.WriteFile(fullpath, data, 0600)
|
|
}
|