refactor: redesign how we import assets (#260)

* fix(pkg.go.dev): import a subpackage containing the assets

We're trying to fix this issue that pkg.go.dev does not build.

Thanks to @hellais for this very neat idea! Let's keep our
fingers crossed and see whether it fixes!

* feat: use embedded geoip databases

Closes https://github.com/ooni/probe/issues/1372.

Work done as part of https://github.com/ooni/probe/issues/1369.

* fix(assetsx): add tests

* feat: simplify and just vendor uncompressed DBs

* remove tests that seems not necessary anymore

* fix: run go mod tidy

* Address https://github.com/ooni/probe-cli/pull/260/files#r605181364

* rewrite a test in a better way

* fix: gently cleanup the legacy assetsdir

Do not remove the whole directory with brute force. Just zap the
files whose name we know. Then attempt to delete the legacy directory
as well. If not empty, just fail. This is fine because it means the
user has stored other files inside the directory.

* fix: create .miniooni if missing

internal/cmd/getresources/getresources.go

@@ -2,50 +2,13 @@
 package main
 
 import (
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io/ioutil"
 	"log"
-	"net/http"
-	"net/url"
-	"path/filepath"
-
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
+	"time"
 )
 
 func main() {
-	for name, ri := range resources.All {
-		if err := getit(name, &ri); err != nil {
-			log.Fatal(err)
-		}
-	}
-}
-
-func getit(name string, ri *resources.ResourceInfo) error {
-	workDir := filepath.Join("internal", "engine", "resourcesmanager")
-	URL, err := url.Parse(resources.BaseURL)
-	if err != nil {
-		return err
-	}
-	URL.Path = ri.URLPath
-	log.Println("fetching", URL.String())
-	resp, err := http.Get(URL.String())
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != 200 {
-		return errors.New("http request failed")
-	}
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	checksum := fmt.Sprintf("%x", sha256.Sum256(data))
-	if checksum != ri.GzSHA256 {
-		return errors.New("sha256 mismatch")
-	}
-	fullpath := filepath.Join(workDir, name+".gz")
-	return ioutil.WriteFile(fullpath, data, 0644)
+	log.Printf("This command is no longer needed. We will keep it into")
+	log.Printf("the repository until the end of 2021, so you have\n")
+	log.Printf("time to adjust your build workflow.\n")
+	time.Sleep(5 * time.Second)
 }

internal/cmd/miniooni/libminiooni.go

@@ -18,6 +18,7 @@ import (
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/engine"
 	"github.com/ooni/probe-cli/v3/internal/engine/humanizex"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/assetsdir"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
 	"github.com/ooni/probe-cli/v3/internal/version"
@@ -303,9 +304,18 @@ func MainWithConfiguration(experimentName string, currentOptions Options) {
 	homeDir := gethomedir(currentOptions.HomeDir)
 	fatalIfFalse(homeDir != "", "home directory is empty")
 	miniooniDir := path.Join(homeDir, ".miniooni")
+	err = os.MkdirAll(miniooniDir, 0700)
+	fatalOnError(err, "cannot create $HOME/.miniooni directory")
+
+	// We cleanup the assets files used by versions of ooniprobe
+	// older than v3.9.0, where we started embedding the assets
+	// into the binary and use that directly. This cleanup doesn't
+	// remove the whole directory but only known files inside it
+	// and then the directory itself, if empty. We explicitly discard
+	// the return value as it does not matter to us here.
 	assetsDir := path.Join(miniooniDir, "assets")
-	err = os.MkdirAll(assetsDir, 0700)
-	fatalOnError(err, "cannot create assets directory")
+	_, _ = assetsdir.Cleanup(assetsDir)
+
 	log.Debugf("miniooni state directory: %s", miniooniDir)
 
 	consentFile := path.Join(miniooniDir, "informed")
@@ -324,7 +334,6 @@ func MainWithConfiguration(experimentName string, currentOptions Options) {
 	fatalOnError(err, "cannot create kvstore2 directory")
 
 	config := engine.SessionConfig{
-		AssetsDir:       assetsDir,
 		KVStore:         kvstore,
 		Logger:          logger,
 		ProxyURL:        proxyURL,