feat: use go1.16 and resources embedding (#235)

* feat: use go1.16 embedding for resources We want to embed everything that can be easily embedded. We should, at a minimum, replace the downloading of resources and bindata. Ref: https://github.com/ooni/probe/issues/1367. * fix: get rid of bindata and use go embed instead * fix: start unbreaking some automatic tests * fix: fetch resources as part of the mobile build * fix: convert more stuff to go1.16 I still expect many breakages, but we'll fix them. * fix: make the windows CI green * fix: get resources before running QA * fix: go1.16 uses modules by default * hopefully fix all other outstanding issues * fix(QA/telegram.py): add another DC IP address * Apply suggestions from code review
2021-03-02 12:08:24 +01:00
parent c33af04e29
commit 322394fe63
51 changed files with 441 additions and 780 deletions
@@ -0,0 +1,3 @@
+/asn.mmdb.gz
+/country.mmdb.gz
+/testdata
@@ -0,0 +1,149 @@
+// Package resourcesmanager contains the resources manager.
+package resourcesmanager
+
+import (
+	"compress/gzip"
+	"crypto/sha256"
+	"embed"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+)
+
+// Errors returned by this package.
+var (
+	ErrDestDirEmpty   = errors.New("resources: DestDir is empty")
+	ErrSHA256Mismatch = errors.New("resources: sha256 mismatch")
+)
+
+// CopyWorker ensures that resources are current. You always need to set
+// the DestDir attribute. All the rest is optional.
+type CopyWorker struct {
+	DestDir   string                                                     // mandatory
+	Different func(left, right string) bool                              // optional
+	Equal     func(left, right string) bool                              // optional
+	MkdirAll  func(path string, perm os.FileMode) error                  // optional
+	NewReader func(r io.Reader) (io.ReadCloser, error)                   // optional
+	Open      func(path string) (fs.File, error)                         // optional
+	ReadAll   func(r io.Reader) ([]byte, error)                          // optional
+	ReadFile  func(filename string) ([]byte, error)                      // optional
+	WriteFile func(filename string, data []byte, perm fs.FileMode) error // optional
+}
+
+//go:embed *.mmdb.gz
+var efs embed.FS
+
+func (cw *CopyWorker) mkdirAll(path string, perm os.FileMode) error {
+	if cw.MkdirAll != nil {
+		return cw.MkdirAll(path, perm)
+	}
+	return os.MkdirAll(path, perm)
+}
+
+// Ensure ensures that the resources on disk are current.
+func (cw *CopyWorker) Ensure() error {
+	if cw.DestDir == "" {
+		return ErrDestDirEmpty
+	}
+	if err := cw.mkdirAll(cw.DestDir, 0700); err != nil {
+		return err
+	}
+	for name, resource := range resources.All {
+		if err := cw.ensureFor(name, &resource); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (cw *CopyWorker) readFile(path string) ([]byte, error) {
+	if cw.ReadFile != nil {
+		return cw.ReadFile(path)
+	}
+	return ioutil.ReadFile(path)
+}
+
+func (cw *CopyWorker) equal(left, right string) bool {
+	if cw.Equal != nil {
+		return cw.Equal(left, right)
+	}
+	return left == right
+}
+
+func (cw *CopyWorker) different(left, right string) bool {
+	if cw.Different != nil {
+		return cw.Different(left, right)
+	}
+	return left != right
+}
+
+func (cw *CopyWorker) open(path string) (fs.File, error) {
+	if cw.Open != nil {
+		return cw.Open(path)
+	}
+	return efs.Open(path)
+}
+
+func (cw *CopyWorker) newReader(r io.Reader) (io.ReadCloser, error) {
+	if cw.NewReader != nil {
+		return cw.NewReader(r)
+	}
+	return gzip.NewReader(r)
+}
+
+func (cw *CopyWorker) readAll(r io.Reader) ([]byte, error) {
+	if cw.ReadAll != nil {
+		return cw.ReadAll(r)
+	}
+	return ioutil.ReadAll(r)
+}
+
+func (cw *CopyWorker) writeFile(filename string, data []byte, perm fs.FileMode) error {
+	if cw.WriteFile != nil {
+		return cw.WriteFile(filename, data, perm)
+	}
+	return ioutil.WriteFile(filename, data, perm)
+}
+
+func (cw *CopyWorker) sha256sum(data []byte) string {
+	return fmt.Sprintf("%x", sha256.Sum256(data))
+}
+
+func (cw *CopyWorker) allGood(rpath string, resource *resources.ResourceInfo) bool {
+	data, err := cw.readFile(rpath)
+	if err != nil {
+		return false
+	}
+	return cw.equal(cw.sha256sum(data), resource.SHA256)
+}
+
+func (cw *CopyWorker) ensureFor(name string, resource *resources.ResourceInfo) error {
+	rpath := filepath.Join(cw.DestDir, name)
+	if cw.allGood(rpath, resource) {
+		return nil
+	}
+	filep, err := cw.open(name + ".gz")
+	if err != nil {
+		return err
+	}
+	defer filep.Close()
+	gzfilep, err := cw.newReader(filep)
+	if err != nil {
+		return err
+	}
+	defer gzfilep.Close()
+	data, err := cw.readAll(gzfilep)
+	if err != nil {
+		return err
+	}
+	if cw.different(cw.sha256sum(data), resource.SHA256) {
+		return ErrSHA256Mismatch
+	}
+	return cw.writeFile(rpath, data, 0600)
+}
@@ -0,0 +1,142 @@
+package resourcesmanager
+
+import (
+	"errors"
+	"io"
+	"io/fs"
+	"os"
+	"testing"
+)
+
+func TestAllGood(t *testing.T) {
+	// make sure we start from scratch
+	if err := os.RemoveAll("testdata"); err != nil {
+		t.Fatal(err)
+	}
+	// first iteration should copy the resources
+	cw := &CopyWorker{DestDir: "testdata"}
+	if err := cw.Ensure(); err != nil {
+		t.Fatal(err)
+	}
+	// second iteration should just ensure they're there
+	if err := cw.Ensure(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEmptyDestDir(t *testing.T) {
+	cw := &CopyWorker{DestDir: ""}
+	if err := cw.Ensure(); !errors.Is(err, ErrDestDirEmpty) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestMkdirAllFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestOpenFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		ReadFile: func(path string) ([]byte, error) {
+			return []byte(`fake`), nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		Open: func(path string) (fs.File, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestNewReaderFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		NewReader: func(r io.Reader) (io.ReadCloser, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestReadAllFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		ReadAll: func(r io.Reader) ([]byte, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestSHA256Mismatch(t *testing.T) {
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		Different: func(left, right string) bool {
+			return true
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, ErrSHA256Mismatch) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestWriteFileFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		WriteFile: func(filename string, data []byte, perm fs.FileMode) error {
+			return errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}