feat: use go1.16 and resources embedding (#235)

* feat: use go1.16 embedding for resources

We want to embed everything that can be easily embedded. We should, at a
minimum, replace the downloading of resources and bindata.

Ref: https://github.com/ooni/probe/issues/1367.

* fix: get rid of bindata and use go embed instead

* fix: start unbreaking some automatic tests

* fix: fetch resources as part of the mobile build

* fix: convert more stuff to go1.16

I still expect many breakages, but we'll fix them.

* fix: make the windows CI green

* fix: get resources before running QA

* fix: go1.16 uses modules by default

* hopefully fix all other outstanding issues

* fix(QA/telegram.py): add another DC IP address

* Apply suggestions from code review

internal/engine/geolocate/mmdblookup_test.go

@@ -1,12 +1,9 @@
 package geolocate
 
 import (
-	"context"
-	"net/http"
 	"testing"
 
-	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+	"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
 )
 
 const (
@@ -16,13 +13,8 @@ const (
 )
 
 func maybeFetchResources(t *testing.T) {
-	c := &resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    "../testdata/",
-	}
-	if err := c.Ensure(context.Background()); err != nil {
+	c := &resourcesmanager.CopyWorker{DestDir: "../testdata/"}
+	if err := c.Ensure(); err != nil {
 		t.Fatal(err)
 	}
 }

internal/engine/netx/archival/archival_test.go

@@ -10,14 +10,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/apex/log"
 	"github.com/google/go-cmp/cmp"
 	"github.com/gorilla/websocket"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+	"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
 )
 
 func TestNewTCPConnectList(t *testing.T) {
@@ -286,12 +285,7 @@ func TestNewRequestList(t *testing.T) {
 }
 
 func TestNewDNSQueriesList(t *testing.T) {
-	err := (&resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "miniooni/0.1.0-dev",
-		WorkDir:    "../../testdata",
-	}).Ensure(context.Background())
+	err := (&resourcesmanager.CopyWorker{DestDir: "../../testdata"}).Ensure()
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/resources/README.md

@@ -1,3 +0,0 @@
-# Package github.com/ooni/probe-engine/resources
-
-This package contains code to download OONI resources.

internal/engine/resources/doc.go

@@ -0,0 +1,3 @@
+// Package resources contains info on resources. See also
+// the resourcesmanager package.
+package resources

internal/engine/resources/resources.go

@@ -1,104 +0,0 @@
-// Package resources contains code to download resources.
-package resources
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"crypto/sha256"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-
-	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
-	"github.com/ooni/probe-cli/v3/internal/engine/model"
-)
-
-// Client is a client for fetching resources.
-type Client struct {
-	// HTTPClient is the HTTP client to use.
-	HTTPClient *http.Client
-
-	// Logger is the logger to use.
-	Logger model.Logger
-
-	// OSMkdirAll allows testing os.MkdirAll failures.
-	OSMkdirAll func(path string, perm os.FileMode) error
-
-	// UserAgent is the user agent to use.
-	UserAgent string
-
-	// WorkDir is the directory where to save resources.
-	WorkDir string
-}
-
-// Ensure ensures that resources are downloaded and current.
-func (c *Client) Ensure(ctx context.Context) error {
-	mkdirall := c.OSMkdirAll
-	if mkdirall == nil {
-		mkdirall = os.MkdirAll
-	}
-	if err := mkdirall(c.WorkDir, 0700); err != nil {
-		return err
-	}
-	for name, resource := range All {
-		if err := c.EnsureForSingleResource(
-			ctx, name, resource, func(real, expected string) bool {
-				return real == expected
-			},
-			gzip.NewReader, ioutil.ReadAll,
-		); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// EnsureForSingleResource ensures that a single resource
-// is downloaded and is current.
-func (c *Client) EnsureForSingleResource(
-	ctx context.Context, name string, resource ResourceInfo,
-	equal func(real, expected string) bool,
-	gzipNewReader func(r io.Reader) (*gzip.Reader, error),
-	ioutilReadAll func(r io.Reader) ([]byte, error),
-) error {
-	fullpath := filepath.Join(c.WorkDir, name)
-	data, err := ioutil.ReadFile(fullpath)
-	if err == nil {
-		sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
-		if equal(sha256sum, resource.SHA256) {
-			return nil
-		}
-		c.Logger.Debugf("resources: %s is outdated", fullpath)
-	} else {
-		c.Logger.Debugf("resources: can't read %s: %s", fullpath, err.Error())
-	}
-	data, err = (httpx.Client{
-		BaseURL:    BaseURL,
-		HTTPClient: c.HTTPClient,
-		Logger:     c.Logger,
-		UserAgent:  c.UserAgent,
-	}).FetchResourceAndVerify(ctx, resource.URLPath, resource.GzSHA256)
-	if err != nil {
-		return err
-	}
-	c.Logger.Debugf("resources: uncompress %s", fullpath)
-	gzreader, err := gzipNewReader(bytes.NewReader(data))
-	if err != nil {
-		return err
-	}
-	defer gzreader.Close()              // we already have a sha256 for it
-	data, err = ioutilReadAll(gzreader) // small file
-	if err != nil {
-		return err
-	}
-	sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
-	if equal(sha256sum, resource.SHA256) == false {
-		return fmt.Errorf("resources: %s sha256 mismatch", fullpath)
-	}
-	c.Logger.Debugf("resources: overwrite %s", fullpath)
-	return ioutil.WriteFile(fullpath, data, 0600)
-}

internal/engine/resources/resources_test.go

@@ -1,180 +0,0 @@
-package resources_test
-
-import (
-	"compress/gzip"
-	"context"
-	"errors"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
-)
-
-func TestEnsureMkdirAllFailure(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	expected := errors.New("mocked error")
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		OSMkdirAll: func(string, os.FileMode) error {
-			return expected
-		},
-		UserAgent: "ooniprobe-engine/0.1.0",
-		WorkDir:   "/foobar",
-	}
-	err := client.Ensure(context.Background())
-	if !errors.Is(err, expected) {
-		t.Fatal("not the error we expected")
-	}
-}
-
-func TestEnsure(t *testing.T) {
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	err = client.Ensure(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-	// the second round should be idempotent
-	err = client.Ensure(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestEnsureFailure(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	err = client.Ensure(ctx)
-	if !errors.Is(err, context.Canceled) {
-		t.Fatal("not the error we expected")
-	}
-}
-
-func TestEnsureFailAllComparisons(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	// run once to download the resource once
-	err = client.EnsureForSingleResource(
-		context.Background(), "ca-bundle.pem", resources.ResourceInfo{
-			URLPath:  "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
-			GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
-			SHA256:   "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
-		}, func(left, right string) bool {
-			return left == right
-		},
-		gzip.NewReader, ioutil.ReadAll,
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// re-run with broken comparison operator so that we should
-	// first redownload and then fail for invalid SHA256.
-	err = client.EnsureForSingleResource(
-		context.Background(), "ca-bundle.pem", resources.ResourceInfo{
-			URLPath:  "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
-			GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
-			SHA256:   "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
-		}, func(left, right string) bool {
-			return false // comparison for equality always fails
-		},
-		gzip.NewReader, ioutil.ReadAll,
-	)
-	if err == nil || !strings.HasSuffix(err.Error(), "sha256 mismatch") {
-		t.Fatal("not the error we expected")
-	}
-}
-
-func TestEnsureFailGzipNewReader(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	expected := errors.New("mocked error")
-	err = client.EnsureForSingleResource(
-		context.Background(), "ca-bundle.pem", resources.ResourceInfo{
-			URLPath:  "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
-			GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
-			SHA256:   "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
-		}, func(left, right string) bool {
-			return left == right
-		},
-		func(r io.Reader) (*gzip.Reader, error) {
-			return nil, expected
-		},
-		ioutil.ReadAll,
-	)
-	if !errors.Is(err, expected) {
-		t.Fatal("not the error we expected")
-	}
-}
-
-func TestEnsureFailIoUtilReadAll(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	expected := errors.New("mocked error")
-	err = client.EnsureForSingleResource(
-		context.Background(), "ca-bundle.pem", resources.ResourceInfo{
-			URLPath:  "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
-			GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
-			SHA256:   "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
-		}, func(left, right string) bool {
-			return left == right
-		},
-		gzip.NewReader, func(r io.Reader) ([]byte, error) {
-			return nil, expected
-		},
-	)
-	if !errors.Is(err, expected) {
-		t.Fatal("not the error we expected")
-	}
-}

internal/engine/resourcesmanager/.gitignore

@@ -1,2 +1,3 @@
 /asn.mmdb.gz
 /country.mmdb.gz
+/testdata

internal/engine/resourcesmanager/resourcesmanager.go

@@ -0,0 +1,149 @@
+// Package resourcesmanager contains the resources manager.
+package resourcesmanager
+
+import (
+	"compress/gzip"
+	"crypto/sha256"
+	"embed"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+)
+
+// Errors returned by this package.
+var (
+	ErrDestDirEmpty   = errors.New("resources: DestDir is empty")
+	ErrSHA256Mismatch = errors.New("resources: sha256 mismatch")
+)
+
+// CopyWorker ensures that resources are current. You always need to set
+// the DestDir attribute. All the rest is optional.
+type CopyWorker struct {
+	DestDir   string                                                     // mandatory
+	Different func(left, right string) bool                              // optional
+	Equal     func(left, right string) bool                              // optional
+	MkdirAll  func(path string, perm os.FileMode) error                  // optional
+	NewReader func(r io.Reader) (io.ReadCloser, error)                   // optional
+	Open      func(path string) (fs.File, error)                         // optional
+	ReadAll   func(r io.Reader) ([]byte, error)                          // optional
+	ReadFile  func(filename string) ([]byte, error)                      // optional
+	WriteFile func(filename string, data []byte, perm fs.FileMode) error // optional
+}
+
+//go:embed *.mmdb.gz
+var efs embed.FS
+
+func (cw *CopyWorker) mkdirAll(path string, perm os.FileMode) error {
+	if cw.MkdirAll != nil {
+		return cw.MkdirAll(path, perm)
+	}
+	return os.MkdirAll(path, perm)
+}
+
+// Ensure ensures that the resources on disk are current.
+func (cw *CopyWorker) Ensure() error {
+	if cw.DestDir == "" {
+		return ErrDestDirEmpty
+	}
+	if err := cw.mkdirAll(cw.DestDir, 0700); err != nil {
+		return err
+	}
+	for name, resource := range resources.All {
+		if err := cw.ensureFor(name, &resource); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (cw *CopyWorker) readFile(path string) ([]byte, error) {
+	if cw.ReadFile != nil {
+		return cw.ReadFile(path)
+	}
+	return ioutil.ReadFile(path)
+}
+
+func (cw *CopyWorker) equal(left, right string) bool {
+	if cw.Equal != nil {
+		return cw.Equal(left, right)
+	}
+	return left == right
+}
+
+func (cw *CopyWorker) different(left, right string) bool {
+	if cw.Different != nil {
+		return cw.Different(left, right)
+	}
+	return left != right
+}
+
+func (cw *CopyWorker) open(path string) (fs.File, error) {
+	if cw.Open != nil {
+		return cw.Open(path)
+	}
+	return efs.Open(path)
+}
+
+func (cw *CopyWorker) newReader(r io.Reader) (io.ReadCloser, error) {
+	if cw.NewReader != nil {
+		return cw.NewReader(r)
+	}
+	return gzip.NewReader(r)
+}
+
+func (cw *CopyWorker) readAll(r io.Reader) ([]byte, error) {
+	if cw.ReadAll != nil {
+		return cw.ReadAll(r)
+	}
+	return ioutil.ReadAll(r)
+}
+
+func (cw *CopyWorker) writeFile(filename string, data []byte, perm fs.FileMode) error {
+	if cw.WriteFile != nil {
+		return cw.WriteFile(filename, data, perm)
+	}
+	return ioutil.WriteFile(filename, data, perm)
+}
+
+func (cw *CopyWorker) sha256sum(data []byte) string {
+	return fmt.Sprintf("%x", sha256.Sum256(data))
+}
+
+func (cw *CopyWorker) allGood(rpath string, resource *resources.ResourceInfo) bool {
+	data, err := cw.readFile(rpath)
+	if err != nil {
+		return false
+	}
+	return cw.equal(cw.sha256sum(data), resource.SHA256)
+}
+
+func (cw *CopyWorker) ensureFor(name string, resource *resources.ResourceInfo) error {
+	rpath := filepath.Join(cw.DestDir, name)
+	if cw.allGood(rpath, resource) {
+		return nil
+	}
+	filep, err := cw.open(name + ".gz")
+	if err != nil {
+		return err
+	}
+	defer filep.Close()
+	gzfilep, err := cw.newReader(filep)
+	if err != nil {
+		return err
+	}
+	defer gzfilep.Close()
+	data, err := cw.readAll(gzfilep)
+	if err != nil {
+		return err
+	}
+	if cw.different(cw.sha256sum(data), resource.SHA256) {
+		return ErrSHA256Mismatch
+	}
+	return cw.writeFile(rpath, data, 0600)
+}

internal/engine/resourcesmanager/resourcesmanager_test.go

@@ -0,0 +1,142 @@
+package resourcesmanager
+
+import (
+	"errors"
+	"io"
+	"io/fs"
+	"os"
+	"testing"
+)
+
+func TestAllGood(t *testing.T) {
+	// make sure we start from scratch
+	if err := os.RemoveAll("testdata"); err != nil {
+		t.Fatal(err)
+	}
+	// first iteration should copy the resources
+	cw := &CopyWorker{DestDir: "testdata"}
+	if err := cw.Ensure(); err != nil {
+		t.Fatal(err)
+	}
+	// second iteration should just ensure they're there
+	if err := cw.Ensure(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEmptyDestDir(t *testing.T) {
+	cw := &CopyWorker{DestDir: ""}
+	if err := cw.Ensure(); !errors.Is(err, ErrDestDirEmpty) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestMkdirAllFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestOpenFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		ReadFile: func(path string) ([]byte, error) {
+			return []byte(`fake`), nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		Open: func(path string) (fs.File, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestNewReaderFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		NewReader: func(r io.Reader) (io.ReadCloser, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestReadAllFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		ReadAll: func(r io.Reader) ([]byte, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestSHA256Mismatch(t *testing.T) {
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		Different: func(left, right string) bool {
+			return true
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, ErrSHA256Mismatch) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestWriteFileFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		WriteFile: func(filename string, data []byte, perm fs.FileMode) error {
+			return errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}

internal/engine/session.go

@@ -22,6 +22,7 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
 	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
 	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+	"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
 	"github.com/ooni/probe-cli/v3/internal/version"
 )
 
@@ -414,12 +415,7 @@ func (s *Session) UserAgent() (useragent string) {
 
 // MaybeUpdateResources updates the resources if needed.
 func (s *Session) MaybeUpdateResources(ctx context.Context) error {
-	return (&resources.Client{
-		HTTPClient: s.DefaultHTTPClient(),
-		Logger:     s.logger,
-		UserAgent:  s.UserAgent(),
-		WorkDir:    s.assetsDir,
-	}).Ensure(ctx)
+	return (&resourcesmanager.CopyWorker{DestDir: s.assetsDir}).Ensure()
 }
 
 func (s *Session) getAvailableProbeServices() []model.Service {