feat: use go1.16 and resources embedding (#235)
* feat: use go1.16 embedding for resources We want to embed everything that can be easily embedded. We should, at a minimum, replace the downloading of resources and bindata. Ref: https://github.com/ooni/probe/issues/1367. * fix: get rid of bindata and use go embed instead * fix: start unbreaking some automatic tests * fix: fetch resources as part of the mobile build * fix: convert more stuff to go1.16 I still expect many breakages, but we'll fix them. * fix: make the windows CI green * fix: get resources before running QA * fix: go1.16 uses modules by default * hopefully fix all other outstanding issues * fix(QA/telegram.py): add another DC IP address * Apply suggestions from code review
This commit is contained in:
Simone Basso
GitHub
@@ -1,3 +0,0 @@
|
||||
# Package github.com/ooni/probe-engine/resources
|
||||
|
||||
This package contains code to download OONI resources.
|
||||
@@ -0,0 +1,3 @@
|
||||
// Package resources contains info on resources. See also
|
||||
// the resourcesmanager package.
|
||||
package resources
|
||||
@@ -1,2 +0,0 @@
|
||||
/asn.mmdb.gz
|
||||
/country.mmdb.gz
|
||||
@@ -1,104 +0,0 @@
|
||||
// Package resources contains code to download resources.
|
||||
package resources
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
)
|
||||
|
||||
// Client is a client for fetching resources.
|
||||
type Client struct {
|
||||
// HTTPClient is the HTTP client to use.
|
||||
HTTPClient *http.Client
|
||||
|
||||
// Logger is the logger to use.
|
||||
Logger model.Logger
|
||||
|
||||
// OSMkdirAll allows testing os.MkdirAll failures.
|
||||
OSMkdirAll func(path string, perm os.FileMode) error
|
||||
|
||||
// UserAgent is the user agent to use.
|
||||
UserAgent string
|
||||
|
||||
// WorkDir is the directory where to save resources.
|
||||
WorkDir string
|
||||
}
|
||||
|
||||
// Ensure ensures that resources are downloaded and current.
|
||||
func (c *Client) Ensure(ctx context.Context) error {
|
||||
mkdirall := c.OSMkdirAll
|
||||
if mkdirall == nil {
|
||||
mkdirall = os.MkdirAll
|
||||
}
|
||||
if err := mkdirall(c.WorkDir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
for name, resource := range All {
|
||||
if err := c.EnsureForSingleResource(
|
||||
ctx, name, resource, func(real, expected string) bool {
|
||||
return real == expected
|
||||
},
|
||||
gzip.NewReader, ioutil.ReadAll,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureForSingleResource ensures that a single resource
|
||||
// is downloaded and is current.
|
||||
func (c *Client) EnsureForSingleResource(
|
||||
ctx context.Context, name string, resource ResourceInfo,
|
||||
equal func(real, expected string) bool,
|
||||
gzipNewReader func(r io.Reader) (*gzip.Reader, error),
|
||||
ioutilReadAll func(r io.Reader) ([]byte, error),
|
||||
) error {
|
||||
fullpath := filepath.Join(c.WorkDir, name)
|
||||
data, err := ioutil.ReadFile(fullpath)
|
||||
if err == nil {
|
||||
sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
|
||||
if equal(sha256sum, resource.SHA256) {
|
||||
return nil
|
||||
}
|
||||
c.Logger.Debugf("resources: %s is outdated", fullpath)
|
||||
} else {
|
||||
c.Logger.Debugf("resources: can't read %s: %s", fullpath, err.Error())
|
||||
}
|
||||
data, err = (httpx.Client{
|
||||
BaseURL: BaseURL,
|
||||
HTTPClient: c.HTTPClient,
|
||||
Logger: c.Logger,
|
||||
UserAgent: c.UserAgent,
|
||||
}).FetchResourceAndVerify(ctx, resource.URLPath, resource.GzSHA256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Logger.Debugf("resources: uncompress %s", fullpath)
|
||||
gzreader, err := gzipNewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gzreader.Close() // we already have a sha256 for it
|
||||
data, err = ioutilReadAll(gzreader) // small file
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
|
||||
if equal(sha256sum, resource.SHA256) == false {
|
||||
return fmt.Errorf("resources: %s sha256 mismatch", fullpath)
|
||||
}
|
||||
c.Logger.Debugf("resources: overwrite %s", fullpath)
|
||||
return ioutil.WriteFile(fullpath, data, 0600)
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
package resources_test
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/resources"
|
||||
)
|
||||
|
||||
func TestEnsureMkdirAllFailure(t *testing.T) {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
expected := errors.New("mocked error")
|
||||
client := resources.Client{
|
||||
HTTPClient: http.DefaultClient,
|
||||
Logger: log.Log,
|
||||
OSMkdirAll: func(string, os.FileMode) error {
|
||||
return expected
|
||||
},
|
||||
UserAgent: "ooniprobe-engine/0.1.0",
|
||||
WorkDir: "/foobar",
|
||||
}
|
||||
err := client.Ensure(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsure(t *testing.T) {
|
||||
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := resources.Client{
|
||||
HTTPClient: http.DefaultClient,
|
||||
Logger: log.Log,
|
||||
UserAgent: "ooniprobe-engine/0.1.0",
|
||||
WorkDir: tempdir,
|
||||
}
|
||||
err = client.Ensure(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// the second round should be idempotent
|
||||
err = client.Ensure(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureFailure(t *testing.T) {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := resources.Client{
|
||||
HTTPClient: http.DefaultClient,
|
||||
Logger: log.Log,
|
||||
UserAgent: "ooniprobe-engine/0.1.0",
|
||||
WorkDir: tempdir,
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err = client.Ensure(ctx)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureFailAllComparisons(t *testing.T) {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := resources.Client{
|
||||
HTTPClient: http.DefaultClient,
|
||||
Logger: log.Log,
|
||||
UserAgent: "ooniprobe-engine/0.1.0",
|
||||
WorkDir: tempdir,
|
||||
}
|
||||
// run once to download the resource once
|
||||
err = client.EnsureForSingleResource(
|
||||
context.Background(), "ca-bundle.pem", resources.ResourceInfo{
|
||||
URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
|
||||
GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
|
||||
SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
|
||||
}, func(left, right string) bool {
|
||||
return left == right
|
||||
},
|
||||
gzip.NewReader, ioutil.ReadAll,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// re-run with broken comparison operator so that we should
|
||||
// first redownload and then fail for invalid SHA256.
|
||||
err = client.EnsureForSingleResource(
|
||||
context.Background(), "ca-bundle.pem", resources.ResourceInfo{
|
||||
URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
|
||||
GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
|
||||
SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
|
||||
}, func(left, right string) bool {
|
||||
return false // comparison for equality always fails
|
||||
},
|
||||
gzip.NewReader, ioutil.ReadAll,
|
||||
)
|
||||
if err == nil || !strings.HasSuffix(err.Error(), "sha256 mismatch") {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureFailGzipNewReader(t *testing.T) {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := resources.Client{
|
||||
HTTPClient: http.DefaultClient,
|
||||
Logger: log.Log,
|
||||
UserAgent: "ooniprobe-engine/0.1.0",
|
||||
WorkDir: tempdir,
|
||||
}
|
||||
expected := errors.New("mocked error")
|
||||
err = client.EnsureForSingleResource(
|
||||
context.Background(), "ca-bundle.pem", resources.ResourceInfo{
|
||||
URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
|
||||
GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
|
||||
SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
|
||||
}, func(left, right string) bool {
|
||||
return left == right
|
||||
},
|
||||
func(r io.Reader) (*gzip.Reader, error) {
|
||||
return nil, expected
|
||||
},
|
||||
ioutil.ReadAll,
|
||||
)
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureFailIoUtilReadAll(t *testing.T) {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := resources.Client{
|
||||
HTTPClient: http.DefaultClient,
|
||||
Logger: log.Log,
|
||||
UserAgent: "ooniprobe-engine/0.1.0",
|
||||
WorkDir: tempdir,
|
||||
}
|
||||
expected := errors.New("mocked error")
|
||||
err = client.EnsureForSingleResource(
|
||||
context.Background(), "ca-bundle.pem", resources.ResourceInfo{
|
||||
URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
|
||||
GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
|
||||
SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
|
||||
}, func(left, right string) bool {
|
||||
return left == right
|
||||
},
|
||||
gzip.NewReader, func(r io.Reader) ([]byte, error) {
|
||||
return nil, expected
|
||||
},
|
||||
)
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user