refactor: redesign how we import assets (#260)

* fix(pkg.go.dev): import a subpackage containing the assets

We're trying to fix this issue that pkg.go.dev does not build.

Thanks to @hellais for this very neat idea! Let's keep our
fingers crossed and see whether it fixes!

* feat: use embedded geoip databases

Closes https://github.com/ooni/probe/issues/1372.

Work done as part of https://github.com/ooni/probe/issues/1369.

* fix(assetsx): add tests

* feat: simplify and just vendor uncompressed DBs

* remove tests that seems not necessary anymore

* fix: run go mod tidy

* Address https://github.com/ooni/probe-cli/pull/260/files#r605181364

* rewrite a test in a better way

* fix: gently cleanup the legacy assetsdir

Do not remove the whole directory with brute force. Just zap the
files whose name we know. Then attempt to delete the legacy directory
as well. If not empty, just fail. This is fine because it means the
user has stored other files inside the directory.

* fix: create .miniooni if missing

pkg/oonimkall/internal/tasks/runner.go

@@ -75,7 +75,6 @@ func (r *Runner) newsession(logger *ChanLogger) (*engine.Session, error) {
 		return nil, err
 	}
 	config := engine.SessionConfig{
-		AssetsDir:       r.settings.AssetsDir,
 		KVStore:         kvstore,
 		Logger:          logger,
 		SoftwareName:    r.settings.Options.SoftwareName,

pkg/oonimkall/session.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/ooni/probe-cli/v3/internal/engine"
 	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/assetsdir"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
@@ -54,6 +55,9 @@ type ExperimentCallbacks interface {
 type SessionConfig struct {
 	// AssetsDir is the mandatory directory where to store assets
 	// required by a Session, e.g. MaxMind DB files.
+	//
+	// This field is currently deprecated and unused. We will
+	// remove it when we'll bump the major number.
 	AssetsDir string
 
 	// Logger is the optional logger that will receive all the
@@ -116,6 +120,15 @@ func NewSession(config *SessionConfig) (*Session, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	// We cleanup the assets files used by versions of ooniprobe
+	// older than v3.9.0, where we started embedding the assets
+	// into the binary and use that directly. This cleanup doesn't
+	// remove the whole directory but only known files inside it
+	// and then the directory itself, if empty. We explicitly discard
+	// the return value as it does not matter to us here.
+	_, _ = assetsdir.Cleanup(config.AssetsDir)
+
 	var availableps []model.Service
 	if config.ProbeServicesURL != "" {
 		availableps = append(availableps, model.Service{
@@ -124,7 +137,6 @@ func NewSession(config *SessionConfig) (*Session, error) {
 		})
 	}
 	engineConfig := engine.SessionConfig{
-		AssetsDir:              config.AssetsDir,
 		AvailableProbeServices: availableps,
 		KVStore:                kvstore,
 		Logger:                 newLogger(config.Logger, config.Verbose),
@@ -212,15 +224,10 @@ type GeolocateResults struct {
 	Org string
 }
 
-// MaybeUpdateResources ensures that resources are up to date. This function
-// could perform network activity when we need to update resources.
-//
-// This function locks the session until it's done. That is, no other operation
-// can be performed as long as this function is pending.
+// MaybeUpdateResources is a legacy stub. It does nothing. We will
+// remove it when we're ready to bump the major number.
 func (sess *Session) MaybeUpdateResources(ctx *Context) error {
-	sess.mtx.Lock()
-	defer sess.mtx.Unlock()
-	return sess.sessp.MaybeUpdateResources(ctx.ctx)
+	return nil
 }
 
 // Geolocate performs a geolocate operation and returns the results.

pkg/oonimkall/session_integration_test.go

@@ -47,25 +47,9 @@ func TestNewSessionWithInvalidStateDir(t *testing.T) {
 	}
 }
 
-func TestNewSessionWithMissingSoftwareName(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skip test in short mode")
-	}
-	sess, err := oonimkall.NewSession(&oonimkall.SessionConfig{
-		StateDir: "../testdata/oonimkall/state",
-	})
-	if err == nil || err.Error() != "AssetsDir is empty" {
-		t.Fatal("not the error we expected")
-	}
-	if sess != nil {
-		t.Fatal("expected a nil Session here")
-	}
-}
-
 func TestMaybeUpdateResourcesWithCancelledContext(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skip test in short mode")
-	}
+	// Note that MaybeUpdateResources is now a deprecated stub that
+	// does nothing. We will remove it when we bump major.
 	dir, err := ioutil.TempDir("", "xx")
 	if err != nil {
 		t.Fatal(err)

pkg/oonimkall/task_integration_test.go

@@ -172,38 +172,6 @@ func TestEmptyStateDir(t *testing.T) {
 	}
 }
 
-func TestEmptyAssetsDir(t *testing.T) {
-	task, err := oonimkall.StartTask(`{
-		"log_level": "DEBUG",
-		"name": "Example",
-		"options": {
-			"software_name": "oonimkall-test",
-			"software_version": "0.1.0"
-		},
-		"state_dir": "../testdata/oonimkall/state",
-		"version": 1
-	}`)
-	if err != nil {
-		t.Fatal(err)
-	}
-	var seen bool
-	for !task.IsDone() {
-		eventstr := task.WaitForNextEvent()
-		var event eventlike
-		if err := json.Unmarshal([]byte(eventstr), &event); err != nil {
-			t.Fatal(err)
-		}
-		if event.Key == "failure.startup" {
-			if strings.Contains(eventstr, "AssetsDir is empty") {
-				seen = true
-			}
-		}
-	}
-	if !seen {
-		t.Fatal("did not see failure.startup")
-	}
-}
-
 func TestUnknownExperiment(t *testing.T) {
 	task, err := oonimkall.StartTask(`{
 		"assets_dir": "../testdata/oonimkall/assets",