feat: use go1.16 and resources embedding (#235)

* feat: use go1.16 embedding for resources

We want to embed everything that can be easily embedded. We should, at a
minimum, replace the downloading of resources and bindata.

Ref: https://github.com/ooni/probe/issues/1367.

* fix: get rid of bindata and use go embed instead

* fix: start unbreaking some automatic tests

* fix: fetch resources as part of the mobile build

* fix: convert more stuff to go1.16

I still expect many breakages, but we'll fix them.

* fix: make the windows CI green

* fix: get resources before running QA

* fix: go1.16 uses modules by default

* hopefully fix all other outstanding issues

* fix(QA/telegram.py): add another DC IP address

* Apply suggestions from code review

.github/workflows/alltests.yml

@@ -11,6 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
+      - run: go run ./internal/cmd/getresources
       - run: go test -race -tags shaping ./...

.github/workflows/android.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: brew install --cask android-sdk
       - run: echo y | sdkmanager --install "platforms;android-29"

.github/workflows/bindata.yml

@@ -1,22 +0,0 @@
-# Make sure we can embed bindata
-name: bindata
-on:
-  push:
-  schedule:
-    - cron: "14 17 * * 3"
-jobs:
-  test:
-    runs-on: "${{ matrix.os }}"
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        go: ["1.14"]
-    steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: ${{ matrix.go }}
-      - uses: actions/checkout@v2
-      - run: ./updatebindata.sh
-      - run: go mod tidy  # revert changes caused by installing bindata
-      - run: git diff --exit-code  # if this fails, run ./updatebindata.sh locally and push

.github/workflows/coverage.yml

@@ -9,12 +9,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ "1.14" ]
+        go: [ "1.16" ]
     steps:
       - uses: actions/setup-go@v1
         with:
           go-version: "${{ matrix.go }}"
       - uses: actions/checkout@v2
+      - run: go run ./internal/cmd/getresources
       - run: go test -short -race -tags shaping -coverprofile=probe-cli.cov ./...
       - uses: shogo82148/actions-goveralls@v1
         with:

.github/workflows/cross.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: sudo apt update
       - run: sudo apt install --yes mingw-w64
@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: brew install mingw-w64
       - run: ./build.sh windows

.github/workflows/debian.yml

@@ -17,7 +17,7 @@ jobs:
           sudo service docker restart
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64
       - run: sudo apt-get update -q

.github/workflows/e2eminiooni.yml

@@ -13,6 +13,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./E2E/miniooni.bash

.github/workflows/generate.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: go generate ./...

.github/workflows/ios.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./build-ios.bash
       - run: ./publish-ios.bash

.github/workflows/jafar.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: go build -v ./internal/cmd/jafar
       - run: sudo ./testjafar.bash

.github/workflows/linux.yml

@@ -14,7 +14,7 @@ jobs:
           sudo service docker restart
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux
       - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe

.github/workflows/macos.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./build.sh darwin
       - run: ./smoketest.sh ./CLI/darwin/amd64/ooniprobe

.github/workflows/miniooni.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
 
       - run: ./build-miniooni.sh linux

.github/workflows/qafbmessenger.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "fbmessenger"

.github/workflows/qahhfm.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "hhfm"

.github/workflows/qahirl.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "hirl"

.github/workflows/qatelegram.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "telegram"

.github/workflows/qawebconnectivity.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "webconnectivity"

.github/workflows/qawhatsapp.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "whatsapp"

.github/workflows/shorttests.yml

@@ -9,10 +9,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ "1.14", "1.15" ]
+        go: [ "1.16" ]
     steps:
       - uses: actions/setup-go@v1
         with:
           go-version: "${{ matrix.go }}"
       - uses: actions/checkout@v2
+      - run: go run ./internal/cmd/getresources
       - run: go test -short -race -tags shaping ./...

.github/workflows/windows.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: bash.exe ./build.sh windows_amd64  # TODO(bassosimone): make windows_386 work
       - run: bash.exe ./smoketest.sh ./CLI/windows/amd64/ooniprobe.exe

E2E/miniooni.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -e
+go run ./internal/cmd/getresources
 go build -v ./internal/cmd/miniooni
 probeservices=()
 probeservices+=( "https://ps1.ooni.io" )

QA/Dockerfile

@@ -1,2 +1,2 @@
-FROM golang:1.14-alpine
+FROM golang:1.16-alpine
 RUN apk add go git musl-dev iptables tmux bind-tools curl sudo python3

QA/pyrun.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 set -ex
 export GOPATH=/jafar/QA/GOPATH GOCACHE=/jafar/QA/GOCACHE GO111MODULE=on
+go run ./internal/cmd/getresources
 go build -v ./internal/cmd/miniooni
 go build -v ./internal/cmd/jafar
 sudo ./QA/$1.py ./miniooni

QA/telegram.py

@@ -26,6 +26,7 @@ ALL_POP_IPS = (
     "149.154.175.100",
     "149.154.167.91",
     "149.154.171.5",
+    "95.161.76.100",
 )
 
 

Readme.md

@@ -23,8 +23,16 @@ Every top-level directory contains an explanatory README file.
 
 ## Development setup
 
-Be sure you have golang >= 1.14 and a C compiler (when developing for Windows, you
-need Mingw-w64 installed). The most basic build command is:
+Be sure you have golang >= 1.16 and a C compiler (when developing for Windows, you
+need Mingw-w64 installed).
+
+You need to download assets first using:
+
+```bash
+go run ./internal/cmd/getresources
+```
+
+Then you can build using:
 
 ```bash
 go build -v ./cmd/ooniprobe
@@ -32,16 +40,6 @@ go build -v ./cmd/ooniprobe
 
 This will generate a binary called `ooniprobe` in the current directory.
 
-## Update bundled assets
-
-To update bundled assets use:
-
-```bash
-./updatebindata.sh
-```
-
-Then commit the changes.
-
 ## Android bindings
 
 ```bash

build-android.bash

@@ -47,9 +47,8 @@ fi
 topdir=$(cd $(dirname $0) && pwd -P)
 set -x
 export PATH=$(go env GOPATH)/bin:$PATH
-export GO111MODULE=off
 go get -u golang.org/x/mobile/cmd/gomobile
 gomobile init
-export GO111MODULE=on
 output=MOBILE/android/oonimkall.aar
+go run ./internal/cmd/getresources
 gomobile bind -target=android -o $output -ldflags="-s -w" ./pkg/oonimkall

build-ios.bash

@@ -3,9 +3,8 @@ set -e
 topdir=$(cd $(dirname $0) && pwd -P)
 set -x
 export PATH=$(go env GOPATH)/bin:$PATH
-export GO111MODULE=off
 go get -u golang.org/x/mobile/cmd/gomobile
 gomobile init
-export GO111MODULE=on
 output=MOBILE/ios/oonimkall.framework
+go run ./internal/cmd/getresources
 gomobile bind -target=ios -o $output -ldflags="-s -w" ./pkg/oonimkall

build-miniooni.sh

@@ -2,10 +2,12 @@
 set -e
 case $1 in
   macos|darwin)
+    go run ./internal/cmd/getresources
     export GOOS=darwin GOARCH=amd64
     go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./internal/cmd/miniooni
     echo "Binary ready at ./CLI/darwin/amd64/miniooni";;
   linux)
+    go run ./internal/cmd/getresources
     export GOOS=linux GOARCH=386
     go build -o ./CLI/linux/386 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni
     echo "Binary ready at ./CLI/linux/386/miniooni"
@@ -19,6 +21,7 @@ case $1 in
     go build -o ./CLI/linux/arm64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni
     echo "Binary ready at ./CLI/linux/arm64/miniooni";;
   windows)
+    go run ./internal/cmd/getresources
     export GOOS=windows GOARCH=386
     go build -o ./CLI/windows/386 -ldflags="-s -w" ./internal/cmd/miniooni
     echo "Binary ready at ./CLI/windows/386/miniooni.exe"

build.sh

@@ -12,6 +12,7 @@ case $1 in
     ;;
 
   windows_amd64)
+    go run ./internal/cmd/getresources
     # Note! This assumes we've installed the mingw-w64 compiler.
     GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \
       go build -ldflags='-s -w' ./cmd/ooniprobe
@@ -22,6 +23,7 @@ case $1 in
     ;;
 
   windows_386)
+    go run ./internal/cmd/getresources
     # Note! This assumes we've installed the mingw-w64 compiler.
     GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc \
       go build -ldflags='-s -w' ./cmd/ooniprobe
@@ -38,15 +40,17 @@ case $1 in
     ;;
 
   linux_amd64)
-    docker pull --platform linux/amd64 golang:1.14-alpine
-    docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.14-alpine ./build.sh _alpine
+    go run ./internal/cmd/getresources
+    docker pull --platform linux/amd64 golang:1.16-alpine
+    docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine
     tar -cvzf ooniprobe_${v}_linux_amd64.tar.gz LICENSE.md Readme.md ooniprobe
     mv ooniprobe ./CLI/linux/amd64/
     ;;
 
   linux_386)
-    docker pull --platform linux/386 golang:1.14-alpine
-    docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.14-alpine ./build.sh _alpine
+    go run ./internal/cmd/getresources
+    docker pull --platform linux/386 golang:1.16-alpine
+    docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine
     tar -cvzf ooniprobe_${v}_linux_386.tar.gz LICENSE.md Readme.md ooniprobe
     mv ooniprobe ./CLI/linux/386/
     ;;
@@ -60,6 +64,7 @@ case $1 in
 
   macos|darwin)
     set -x
+    go run ./internal/cmd/getresources
     # Note! The following line _assumes_ you have a working C compiler. If you
     # have Xcode command line tools installed, you are fine.
     go build -ldflags='-s -w' ./cmd/ooniprobe
@@ -79,7 +84,7 @@ case $1 in
     set +x
     echo "Usage: $0 darwin|linux|macos|windows|release"
     echo ""
-    echo "You need a C compiler and Go >= 1.14. The C compiler must be a"
+    echo "You need a C compiler and Go >= 1.16. The C compiler must be a"
     echo "UNIX like compiler like GCC, Clang, Mingw-w64."
     echo ""
     echo "To build a static Linux binary, we use Docker and Alpine. We currently"

cmd/ooniprobe/internal/database/database.go

@@ -2,21 +2,45 @@ package database
 
 import (
 	"database/sql"
+	"embed"
+	"io/ioutil"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/bindata"
 	migrate "github.com/rubenv/sql-migrate"
 	"upper.io/db.v3/lib/sqlbuilder"
 	"upper.io/db.v3/sqlite"
 )
 
+//go:embed migrations/*.sql
+var efs embed.FS
+
+func readAsset(path string) ([]byte, error) {
+	filep, err := efs.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.ReadAll(filep)
+}
+
+func readAssetDir(path string) ([]string, error) {
+	var out []string
+	lst, err := efs.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+	for _, e := range lst {
+		out = append(out, e.Name())
+	}
+	return out, nil
+}
+
 // RunMigrations runs the database migrations
 func RunMigrations(db *sql.DB) error {
 	log.Debugf("running migrations")
 	migrations := &migrate.AssetMigrationSource{
-		Asset:    bindata.Asset,
-		AssetDir: bindata.AssetDir,
-		Dir:      "data/migrations",
+		Asset:    readAsset,
+		AssetDir: readAssetDir,
+		Dir:      "migrations",
 	}
 	n, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up)
 	if err != nil {

cmd/ooniprobe/internal/ooni/ooni.go

@@ -1,6 +1,7 @@
 package ooni
 
 import (
+	_ "embed" // because we embed a file
 	"io/ioutil"
 	"os"
 	"os/signal"
@@ -8,7 +9,6 @@ import (
 	"syscall"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/bindata"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex"
@@ -239,6 +239,9 @@ func MaybeInitializeHome(home string) error {
 	return nil
 }
 
+//go:embed default-config.json
+var defaultConfig []byte
+
 // InitDefaultConfig reads the config from common locations or creates it if
 // missing.
 func InitDefaultConfig(home string) (*config.Config, error) {
@@ -252,12 +255,7 @@ func InitDefaultConfig(home string) (*config.Config, error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			log.Debugf("writing default config to %s", configPath)
-			var data []byte
-			data, err = bindata.Asset("data/default-config.json")
-			if err != nil {
-				return nil, err
-			}
-			if err = ioutil.WriteFile(configPath, data, 0644); err != nil {
+			if err = ioutil.WriteFile(configPath, defaultConfig, 0644); err != nil {
 				return nil, err
 			}
 			// If the user did the informed consent procedure in

data/README.md

@@ -1,3 +0,0 @@
-# Directory github.com/ooni/probe-cli/data
-
-This directory contains data that is embedded into builds.

go.mod

@@ -1,6 +1,6 @@
 module github.com/ooni/probe-cli/v3
 
-go 1.14
+go 1.16
 
 require (
 	git.torproject.org/pluggable-transports/goptlib.git v1.1.0

internal/cmd/getresources/getresources.go

@@ -0,0 +1,51 @@
+// Command getresources downloads the resources
+package main
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"path/filepath"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+)
+
+func main() {
+	for name, ri := range resources.All {
+		if err := getit(name, &ri); err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func getit(name string, ri *resources.ResourceInfo) error {
+	workDir := filepath.Join("internal", "engine", "resourcesmanager")
+	URL, err := url.Parse(resources.BaseURL)
+	if err != nil {
+		return err
+	}
+	URL.Path = ri.URLPath
+	log.Println("fetching", URL.String())
+	resp, err := http.Get(URL.String())
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return errors.New("http request failed")
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	checksum := fmt.Sprintf("%x", sha256.Sum256(data))
+	if checksum != ri.GzSHA256 {
+		return errors.New("sha256 mismatch")
+	}
+	fullpath := filepath.Join(workDir, name+".gz")
+	return ioutil.WriteFile(fullpath, data, 0644)
+}

internal/cmd/jafar/README.md

@@ -8,7 +8,7 @@ any system but it really on works on Linux.
 
 ## Building
 
-We use Go >= 1.14. Jafar also needs the C library headers,
+We use Go >= 1.16. Jafar also needs the C library headers,
 iptables installed, and root permissions.
 
 With Linux Alpine edge, you can compile Jafar with:

internal/engine/geolocate/mmdblookup_test.go

@@ -1,12 +1,9 @@
 package geolocate
 
 import (
-	"context"
-	"net/http"
 	"testing"
 
-	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+	"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
 )
 
 const (
@@ -16,13 +13,8 @@ const (
 )
 
 func maybeFetchResources(t *testing.T) {
-	c := &resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    "../testdata/",
-	}
-	if err := c.Ensure(context.Background()); err != nil {
+	c := &resourcesmanager.CopyWorker{DestDir: "../testdata/"}
+	if err := c.Ensure(); err != nil {
 		t.Fatal(err)
 	}
 }

internal/engine/netx/archival/archival_test.go

@@ -10,14 +10,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/apex/log"
 	"github.com/google/go-cmp/cmp"
 	"github.com/gorilla/websocket"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+	"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
 )
 
 func TestNewTCPConnectList(t *testing.T) {
@@ -286,12 +285,7 @@ func TestNewRequestList(t *testing.T) {
 }
 
 func TestNewDNSQueriesList(t *testing.T) {
-	err := (&resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "miniooni/0.1.0-dev",
-		WorkDir:    "../../testdata",
-	}).Ensure(context.Background())
+	err := (&resourcesmanager.CopyWorker{DestDir: "../../testdata"}).Ensure()
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/resources/README.md

@@ -1,3 +0,0 @@
-# Package github.com/ooni/probe-engine/resources
-
-This package contains code to download OONI resources.

internal/engine/resources/doc.go

@@ -0,0 +1,3 @@
+// Package resources contains info on resources. See also
+// the resourcesmanager package.
+package resources

internal/engine/resources/resources.go

@@ -1,104 +0,0 @@
-// Package resources contains code to download resources.
-package resources
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"crypto/sha256"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-
-	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
-	"github.com/ooni/probe-cli/v3/internal/engine/model"
-)
-
-// Client is a client for fetching resources.
-type Client struct {
-	// HTTPClient is the HTTP client to use.
-	HTTPClient *http.Client
-
-	// Logger is the logger to use.
-	Logger model.Logger
-
-	// OSMkdirAll allows testing os.MkdirAll failures.
-	OSMkdirAll func(path string, perm os.FileMode) error
-
-	// UserAgent is the user agent to use.
-	UserAgent string
-
-	// WorkDir is the directory where to save resources.
-	WorkDir string
-}
-
-// Ensure ensures that resources are downloaded and current.
-func (c *Client) Ensure(ctx context.Context) error {
-	mkdirall := c.OSMkdirAll
-	if mkdirall == nil {
-		mkdirall = os.MkdirAll
-	}
-	if err := mkdirall(c.WorkDir, 0700); err != nil {
-		return err
-	}
-	for name, resource := range All {
-		if err := c.EnsureForSingleResource(
-			ctx, name, resource, func(real, expected string) bool {
-				return real == expected
-			},
-			gzip.NewReader, ioutil.ReadAll,
-		); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// EnsureForSingleResource ensures that a single resource
-// is downloaded and is current.
-func (c *Client) EnsureForSingleResource(
-	ctx context.Context, name string, resource ResourceInfo,
-	equal func(real, expected string) bool,
-	gzipNewReader func(r io.Reader) (*gzip.Reader, error),
-	ioutilReadAll func(r io.Reader) ([]byte, error),
-) error {
-	fullpath := filepath.Join(c.WorkDir, name)
-	data, err := ioutil.ReadFile(fullpath)
-	if err == nil {
-		sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
-		if equal(sha256sum, resource.SHA256) {
-			return nil
-		}
-		c.Logger.Debugf("resources: %s is outdated", fullpath)
-	} else {
-		c.Logger.Debugf("resources: can't read %s: %s", fullpath, err.Error())
-	}
-	data, err = (httpx.Client{
-		BaseURL:    BaseURL,
-		HTTPClient: c.HTTPClient,
-		Logger:     c.Logger,
-		UserAgent:  c.UserAgent,
-	}).FetchResourceAndVerify(ctx, resource.URLPath, resource.GzSHA256)
-	if err != nil {
-		return err
-	}
-	c.Logger.Debugf("resources: uncompress %s", fullpath)
-	gzreader, err := gzipNewReader(bytes.NewReader(data))
-	if err != nil {
-		return err
-	}
-	defer gzreader.Close()              // we already have a sha256 for it
-	data, err = ioutilReadAll(gzreader) // small file
-	if err != nil {
-		return err
-	}
-	sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
-	if equal(sha256sum, resource.SHA256) == false {
-		return fmt.Errorf("resources: %s sha256 mismatch", fullpath)
-	}
-	c.Logger.Debugf("resources: overwrite %s", fullpath)
-	return ioutil.WriteFile(fullpath, data, 0600)
-}

internal/engine/resources/resources_test.go

@@ -1,180 +0,0 @@
-package resources_test
-
-import (
-	"compress/gzip"
-	"context"
-	"errors"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
-)
-
-func TestEnsureMkdirAllFailure(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	expected := errors.New("mocked error")
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		OSMkdirAll: func(string, os.FileMode) error {
-			return expected
-		},
-		UserAgent: "ooniprobe-engine/0.1.0",
-		WorkDir:   "/foobar",
-	}
-	err := client.Ensure(context.Background())
-	if !errors.Is(err, expected) {
-		t.Fatal("not the error we expected")
-	}
-}
-
-func TestEnsure(t *testing.T) {
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	err = client.Ensure(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-	// the second round should be idempotent
-	err = client.Ensure(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestEnsureFailure(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	err = client.Ensure(ctx)
-	if !errors.Is(err, context.Canceled) {
-		t.Fatal("not the error we expected")
-	}
-}
-
-func TestEnsureFailAllComparisons(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	// run once to download the resource once
-	err = client.EnsureForSingleResource(
-		context.Background(), "ca-bundle.pem", resources.ResourceInfo{
-			URLPath:  "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
-			GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
-			SHA256:   "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
-		}, func(left, right string) bool {
-			return left == right
-		},
-		gzip.NewReader, ioutil.ReadAll,
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// re-run with broken comparison operator so that we should
-	// first redownload and then fail for invalid SHA256.
-	err = client.EnsureForSingleResource(
-		context.Background(), "ca-bundle.pem", resources.ResourceInfo{
-			URLPath:  "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
-			GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
-			SHA256:   "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
-		}, func(left, right string) bool {
-			return false // comparison for equality always fails
-		},
-		gzip.NewReader, ioutil.ReadAll,
-	)
-	if err == nil || !strings.HasSuffix(err.Error(), "sha256 mismatch") {
-		t.Fatal("not the error we expected")
-	}
-}
-
-func TestEnsureFailGzipNewReader(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	expected := errors.New("mocked error")
-	err = client.EnsureForSingleResource(
-		context.Background(), "ca-bundle.pem", resources.ResourceInfo{
-			URLPath:  "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
-			GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
-			SHA256:   "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
-		}, func(left, right string) bool {
-			return left == right
-		},
-		func(r io.Reader) (*gzip.Reader, error) {
-			return nil, expected
-		},
-		ioutil.ReadAll,
-	)
-	if !errors.Is(err, expected) {
-		t.Fatal("not the error we expected")
-	}
-}
-
-func TestEnsureFailIoUtilReadAll(t *testing.T) {
-	log.SetLevel(log.DebugLevel)
-	tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := resources.Client{
-		HTTPClient: http.DefaultClient,
-		Logger:     log.Log,
-		UserAgent:  "ooniprobe-engine/0.1.0",
-		WorkDir:    tempdir,
-	}
-	expected := errors.New("mocked error")
-	err = client.EnsureForSingleResource(
-		context.Background(), "ca-bundle.pem", resources.ResourceInfo{
-			URLPath:  "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
-			GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
-			SHA256:   "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
-		}, func(left, right string) bool {
-			return left == right
-		},
-		gzip.NewReader, func(r io.Reader) ([]byte, error) {
-			return nil, expected
-		},
-	)
-	if !errors.Is(err, expected) {
-		t.Fatal("not the error we expected")
-	}
-}

internal/engine/resourcesmanager/.gitignore

@@ -1,2 +1,3 @@
 /asn.mmdb.gz
 /country.mmdb.gz
+/testdata

internal/engine/resourcesmanager/resourcesmanager.go

@@ -0,0 +1,149 @@
+// Package resourcesmanager contains the resources manager.
+package resourcesmanager
+
+import (
+	"compress/gzip"
+	"crypto/sha256"
+	"embed"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+)
+
+// Errors returned by this package.
+var (
+	ErrDestDirEmpty   = errors.New("resources: DestDir is empty")
+	ErrSHA256Mismatch = errors.New("resources: sha256 mismatch")
+)
+
+// CopyWorker ensures that resources are current. You always need to set
+// the DestDir attribute. All the rest is optional.
+type CopyWorker struct {
+	DestDir   string                                                     // mandatory
+	Different func(left, right string) bool                              // optional
+	Equal     func(left, right string) bool                              // optional
+	MkdirAll  func(path string, perm os.FileMode) error                  // optional
+	NewReader func(r io.Reader) (io.ReadCloser, error)                   // optional
+	Open      func(path string) (fs.File, error)                         // optional
+	ReadAll   func(r io.Reader) ([]byte, error)                          // optional
+	ReadFile  func(filename string) ([]byte, error)                      // optional
+	WriteFile func(filename string, data []byte, perm fs.FileMode) error // optional
+}
+
+//go:embed *.mmdb.gz
+var efs embed.FS
+
+func (cw *CopyWorker) mkdirAll(path string, perm os.FileMode) error {
+	if cw.MkdirAll != nil {
+		return cw.MkdirAll(path, perm)
+	}
+	return os.MkdirAll(path, perm)
+}
+
+// Ensure ensures that the resources on disk are current.
+func (cw *CopyWorker) Ensure() error {
+	if cw.DestDir == "" {
+		return ErrDestDirEmpty
+	}
+	if err := cw.mkdirAll(cw.DestDir, 0700); err != nil {
+		return err
+	}
+	for name, resource := range resources.All {
+		if err := cw.ensureFor(name, &resource); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (cw *CopyWorker) readFile(path string) ([]byte, error) {
+	if cw.ReadFile != nil {
+		return cw.ReadFile(path)
+	}
+	return ioutil.ReadFile(path)
+}
+
+func (cw *CopyWorker) equal(left, right string) bool {
+	if cw.Equal != nil {
+		return cw.Equal(left, right)
+	}
+	return left == right
+}
+
+func (cw *CopyWorker) different(left, right string) bool {
+	if cw.Different != nil {
+		return cw.Different(left, right)
+	}
+	return left != right
+}
+
+func (cw *CopyWorker) open(path string) (fs.File, error) {
+	if cw.Open != nil {
+		return cw.Open(path)
+	}
+	return efs.Open(path)
+}
+
+func (cw *CopyWorker) newReader(r io.Reader) (io.ReadCloser, error) {
+	if cw.NewReader != nil {
+		return cw.NewReader(r)
+	}
+	return gzip.NewReader(r)
+}
+
+func (cw *CopyWorker) readAll(r io.Reader) ([]byte, error) {
+	if cw.ReadAll != nil {
+		return cw.ReadAll(r)
+	}
+	return ioutil.ReadAll(r)
+}
+
+func (cw *CopyWorker) writeFile(filename string, data []byte, perm fs.FileMode) error {
+	if cw.WriteFile != nil {
+		return cw.WriteFile(filename, data, perm)
+	}
+	return ioutil.WriteFile(filename, data, perm)
+}
+
+func (cw *CopyWorker) sha256sum(data []byte) string {
+	return fmt.Sprintf("%x", sha256.Sum256(data))
+}
+
+func (cw *CopyWorker) allGood(rpath string, resource *resources.ResourceInfo) bool {
+	data, err := cw.readFile(rpath)
+	if err != nil {
+		return false
+	}
+	return cw.equal(cw.sha256sum(data), resource.SHA256)
+}
+
+func (cw *CopyWorker) ensureFor(name string, resource *resources.ResourceInfo) error {
+	rpath := filepath.Join(cw.DestDir, name)
+	if cw.allGood(rpath, resource) {
+		return nil
+	}
+	filep, err := cw.open(name + ".gz")
+	if err != nil {
+		return err
+	}
+	defer filep.Close()
+	gzfilep, err := cw.newReader(filep)
+	if err != nil {
+		return err
+	}
+	defer gzfilep.Close()
+	data, err := cw.readAll(gzfilep)
+	if err != nil {
+		return err
+	}
+	if cw.different(cw.sha256sum(data), resource.SHA256) {
+		return ErrSHA256Mismatch
+	}
+	return cw.writeFile(rpath, data, 0600)
+}

internal/engine/resourcesmanager/resourcesmanager_test.go

@@ -0,0 +1,142 @@
+package resourcesmanager
+
+import (
+	"errors"
+	"io"
+	"io/fs"
+	"os"
+	"testing"
+)
+
+func TestAllGood(t *testing.T) {
+	// make sure we start from scratch
+	if err := os.RemoveAll("testdata"); err != nil {
+		t.Fatal(err)
+	}
+	// first iteration should copy the resources
+	cw := &CopyWorker{DestDir: "testdata"}
+	if err := cw.Ensure(); err != nil {
+		t.Fatal(err)
+	}
+	// second iteration should just ensure they're there
+	if err := cw.Ensure(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEmptyDestDir(t *testing.T) {
+	cw := &CopyWorker{DestDir: ""}
+	if err := cw.Ensure(); !errors.Is(err, ErrDestDirEmpty) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestMkdirAllFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestOpenFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		ReadFile: func(path string) ([]byte, error) {
+			return []byte(`fake`), nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		Open: func(path string) (fs.File, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestNewReaderFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		NewReader: func(r io.Reader) (io.ReadCloser, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestReadAllFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		ReadAll: func(r io.Reader) ([]byte, error) {
+			return nil, errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestSHA256Mismatch(t *testing.T) {
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		Different: func(left, right string) bool {
+			return true
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, ErrSHA256Mismatch) {
+		t.Fatal("not the error we expected", err)
+	}
+}
+
+func TestWriteFileFailure(t *testing.T) {
+	errMocked := errors.New("mocked error")
+	cw := &CopyWorker{
+		DestDir: "testdata",
+		MkdirAll: func(path string, perm os.FileMode) error {
+			return nil
+		},
+		Equal: func(left, right string) bool {
+			return false
+		},
+		WriteFile: func(filename string, data []byte, perm fs.FileMode) error {
+			return errMocked
+		},
+	}
+	if err := cw.Ensure(); !errors.Is(err, errMocked) {
+		t.Fatal("not the error we expected", err)
+	}
+}

internal/engine/session.go

@@ -22,6 +22,7 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
 	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
 	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+	"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
 	"github.com/ooni/probe-cli/v3/internal/version"
 )
 
@@ -414,12 +415,7 @@ func (s *Session) UserAgent() (useragent string) {
 
 // MaybeUpdateResources updates the resources if needed.
 func (s *Session) MaybeUpdateResources(ctx context.Context) error {
-	return (&resources.Client{
-		HTTPClient: s.DefaultHTTPClient(),
-		Logger:     s.logger,
-		UserAgent:  s.UserAgent(),
-		WorkDir:    s.assetsDir,
-	}).Ensure(ctx)
+	return (&resourcesmanager.CopyWorker{DestDir: s.assetsDir}).Ensure()
 }
 
 func (s *Session) getAvailableProbeServices() []model.Service {

updatebindata.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-set -ex
-go get -u github.com/shuLhan/go-bindata/...
-gobindata=`go env GOPATH`/bin/go-bindata
-version=`$gobindata -version | grep ^go-bindata | cut -d ' ' -f2`
-if [ "$version" != "4.0.0" ]; then
-  echo "FATAL: unexpected go-bindata version" 1>&2
-  exit 1
-fi
-$gobindata -nometadata -o cmd/ooniprobe/internal/bindata/bindata.go -pkg bindata data/...