ooni/probe-cli v3.9.0

-----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEEc4h3qmyCnyakMcX0gLaRJ3cz2VsFAmBmEZgACgkQgLaRJ3cz
 2VvP/RAAgoyC5dahE1c6DBlcWi58mzXMhoIFvhwXynozo3f+BoX99E2iSX5FuMQl
 f7OaFjJP3VHTXosupIKAW2gMvBPDnWQaHWmTORrxO+3rc1sayTRfj/49z+vJFvUU
 CmNQPQig2MEri4ArjktH/UTFUv20kjRYUocmEwvKEaijAlVWpVIzjaH31sYHQi0K
 RnYjrbCqq3g3BghP5eQ7njBQmgMkKgdcsO6zMuH3Z9nwsxyT6fLIa5TA8YckAzJp
 I2CXcpEj9JNJ44jPqcC3wxafufc2QUlrSQMCc1+kt8EMxHObYtxqq4Z3mIwvC41W
 WPjf4/ihwAUkyQTdqHVnJ20ZlOPq0IQEJbj6sHLsDcyZ8CD7YxNVnjvoc1i77lrl
 LXjiMLd6ARAiJoi2NfmWwnAgF/f32YCnwO21xHX44s8ISZC2V6g0IRl4eMAr4kEm
 8ESvgjZ/LVW4hfiJREMaxZ0kzJQjT0ORrWyCUe6jJmRgFoTbbW3K40QLVHqPxuEW
 wfxTzLwU/NFDZGUFFAfWXgvbDdAcYQMf9JyrNRNgAIi2IY+7KhFEZ/zMN7zy9ugJ
 sz5ryj6mKmke49u25kX0dm0sKlzVNyIUDsqbc9XiJ18zPv2CvY67BlTiA+uVCYq2
 UoEB9dV7uAYyAsuIqNZORTNdqquPEACFMl7CJthB2Ei+eqY1B+M=
 =HvUU
 -----END PGP SIGNATURE-----

Merge tag 'v3.9.0' into mobile-staging

ooni/probe-cli v3.9.0

.github/workflows/alltests.yml

@@ -13,5 +13,4 @@ jobs:
         with:
           go-version: "1.16"
       - uses: actions/checkout@v2
-      - run: go run ./internal/cmd/getresources
       - run: go test -race -tags shaping ./...

.github/workflows/coverage.yml

@@ -15,7 +15,6 @@ jobs:
         with:
           go-version: "${{ matrix.go }}"
       - uses: actions/checkout@v2
-      - run: go run ./internal/cmd/getresources
       - run: go test -short -race -tags shaping -coverprofile=probe-cli.cov ./...
       - uses: shogo82148/actions-goveralls@v1
         with:

.github/workflows/shorttests.yml

@@ -15,5 +15,4 @@ jobs:
         with:
           go-version: "${{ matrix.go }}"
       - uses: actions/checkout@v2
-      - run: go run ./internal/cmd/getresources
       - run: go test -short -race -tags shaping ./...

E2E/miniooni.bash

@@ -1,6 +1,5 @@
 #!/bin/bash
 set -e
-go run ./internal/cmd/getresources
 go build -v ./internal/cmd/miniooni
 probeservices=()
 probeservices+=( "https://ps1.ooni.io" )

QA/pyrun.sh

@@ -1,7 +1,6 @@
 #!/bin/sh
 set -ex
 export GOPATH=/jafar/QA/GOPATH GOCACHE=/jafar/QA/GOCACHE GO111MODULE=on
-go run ./internal/cmd/getresources
 go build -v ./internal/cmd/miniooni
 go build -v ./internal/cmd/jafar
 sudo ./QA/$1.py ./miniooni

Readme.md

@@ -24,15 +24,7 @@ Every top-level directory contains an explanatory README file.
 ## Development setup
 
 Be sure you have golang >= 1.16 and a C compiler (when developing for Windows, you
-need Mingw-w64 installed).
+need Mingw-w64 installed). You can build using:
-
-You need to download assets first using:
-
-```bash
-go run ./internal/cmd/getresources
-```
-
-Then you can build using:
 
 ```bash
 go build -v ./cmd/ooniprobe
@@ -80,10 +72,6 @@ go get -u -v ./... && go mod tidy
 
 ## Releasing
 
-1. update binary data as described above;
+Create an issue according to [the routine release template](
-
+https://github.com/ooni/probe/blob/master/.github/ISSUE_TEMPLATE/routine-sprint-releases.md)
-2. update `internal/version/version.go`;
+and perform any item inside the check-list.
-
-3. make sure you have updated dependencies;
-
-4. run `./build.sh release` and follow instructions.

build-android.bash

@@ -28,5 +28,4 @@ export PATH=$(go env GOPATH)/bin:$PATH
 go get -u golang.org/x/mobile/cmd/gomobile
 gomobile init
 output=MOBILE/android/oonimkall.aar
-go run ./internal/cmd/getresources
 gomobile bind -target=android -o $output -ldflags="-s -w" ./pkg/oonimkall

build-ios.bash

@@ -6,5 +6,4 @@ export PATH=$(go env GOPATH)/bin:$PATH
 go get -u golang.org/x/mobile/cmd/gomobile
 gomobile init
 output=MOBILE/ios/oonimkall.framework
-go run ./internal/cmd/getresources
 gomobile bind -target=ios -o $output -ldflags="-s -w" ./pkg/oonimkall

build-miniooni.sh

@@ -2,12 +2,10 @@
 set -e
 case $1 in
   macos|darwin)
-    go run ./internal/cmd/getresources
     export GOOS=darwin GOARCH=amd64
     go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./internal/cmd/miniooni
     echo "Binary ready at ./CLI/darwin/amd64/miniooni";;
   linux)
-    go run ./internal/cmd/getresources
     export GOOS=linux GOARCH=386
     go build -o ./CLI/linux/386 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni
     echo "Binary ready at ./CLI/linux/386/miniooni"
@@ -21,7 +19,6 @@ case $1 in
     go build -o ./CLI/linux/arm64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni
     echo "Binary ready at ./CLI/linux/arm64/miniooni";;
   windows)
-    go run ./internal/cmd/getresources
     export GOOS=windows GOARCH=386
     go build -o ./CLI/windows/386 -ldflags="-s -w" ./internal/cmd/miniooni
     echo "Binary ready at ./CLI/windows/386/miniooni.exe"

build.sh

@@ -12,7 +12,6 @@ case $1 in
     ;;
 
   windows_amd64)
-    go run ./internal/cmd/getresources
     # Note! This assumes we've installed the mingw-w64 compiler.
     GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \
       go build -ldflags='-s -w' ./cmd/ooniprobe
@@ -23,7 +22,6 @@ case $1 in
     ;;
 
   windows_386)
-    go run ./internal/cmd/getresources
     # Note! This assumes we've installed the mingw-w64 compiler.
     GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc \
       go build -ldflags='-s -w' ./cmd/ooniprobe
@@ -40,7 +38,6 @@ case $1 in
     ;;
 
   linux_amd64)
-    go run ./internal/cmd/getresources
     docker pull --platform linux/amd64 golang:1.16-alpine
     docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine
     tar -cvzf ooniprobe_${v}_linux_amd64.tar.gz LICENSE.md Readme.md ooniprobe
@@ -48,7 +45,6 @@ case $1 in
     ;;
 
   linux_386)
-    go run ./internal/cmd/getresources
     docker pull --platform linux/386 golang:1.16-alpine
     docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine
     tar -cvzf ooniprobe_${v}_linux_386.tar.gz LICENSE.md Readme.md ooniprobe
@@ -64,7 +60,6 @@ case $1 in
 
   macos|darwin)
     set -x
-    go run ./internal/cmd/getresources
     # Note! The following line _assumes_ you have a working C compiler. If you
     # have Xcode command line tools installed, you are fine.
     go build -ldflags='-s -w' ./cmd/ooniprobe

cmd/ooniprobe/internal/cli/run/run.go

@@ -1,8 +1,6 @@
 package run
 
 import (
-	"runtime"
-
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/fatih/color"
@@ -28,19 +26,23 @@ func init() {
 			log.WithError(err).Error("failed to perform onboarding")
 			return err
 		}
-		if *noCollector == true {
+		if *noCollector {
 			probe.Config().Sharing.UploadResults = false
 		}
 		return nil
 	})
 
-	functionalRun := func(pred func(name string, gr nettests.Group) bool) error {
+	functionalRun := func(runType string, pred func(name string, gr nettests.Group) bool) error {
 		for name, group := range nettests.All {
-			if pred(name, group) != true {
+			if !pred(name, group) {
 				continue
 			}
 			log.Infof("Running %s tests", color.BlueString(name))
-			conf := nettests.RunGroupConfig{GroupName: name, Probe: probe}
+			conf := nettests.RunGroupConfig{
+				GroupName: name,
+				Probe:     probe,
+				RunType:   runType,
+			}
 			if err := nettests.RunGroup(conf); err != nil {
 				log.WithError(err).Errorf("failed to run %s", name)
 			}
@@ -50,7 +52,7 @@ func init() {
 
 	genRunWithGroupName := func(targetName string) func(*kingpin.ParseContext) error {
 		return func(*kingpin.ParseContext) error {
-			return functionalRun(func(groupName string, gr nettests.Group) bool {
+			return functionalRun("manual", func(groupName string, gr nettests.Group) bool {
 				return groupName == targetName
 			})
 		}
@@ -66,6 +68,7 @@ func init() {
 			Probe:      probe,
 			InputFiles: *inputFile,
 			Inputs:     *input,
+			RunType:    "manual",
 		})
 	})
 
@@ -77,22 +80,14 @@ func init() {
 
 	unattendedCmd := cmd.Command("unattended", "")
 	unattendedCmd.Action(func(_ *kingpin.ParseContext) error {
-		// Until we have enabled the check-in API we're called every
+		return functionalRun("timed", func(name string, gr nettests.Group) bool {
-		// hour on darwin and we need to self throttle.
+			return gr.UnattendedOK
-		// TODO(bassosimone): switch to check-in and remove this hack.
-		switch runtime.GOOS {
-		case "darwin", "windows":
-			const veryFew = 10
-			probe.Config().Nettests.WebsitesURLLimit = veryFew
-		}
-		return functionalRun(func(name string, gr nettests.Group) bool {
-			return gr.UnattendedOK == true
 		})
 	})
 
 	allCmd := cmd.Command("all", "").Default()
 	allCmd.Action(func(_ *kingpin.ParseContext) error {
-		return functionalRun(func(name string, gr nettests.Group) bool {
+		return functionalRun("manual", func(name string, gr nettests.Group) bool {
 			return true
 		})
 	})

cmd/ooniprobe/internal/config/settings.go

@@ -1,38 +1,5 @@
 package config
 
-var websiteCategories = []string{
-	"ALDR",
-	"ANON",
-	"COMM",
-	"COMT",
-	"CTRL",
-	"CULTR",
-	"DATE",
-	"ECON",
-	"ENV",
-	"FILE",
-	"GAME",
-	"GMB",
-	"GOVT",
-	"GRP",
-	"HACK",
-	"HATE",
-	"HOST",
-	"HUMR",
-	"IGO",
-	"LGBT",
-	"MILX",
-	"MMED",
-	"NEWS",
-	"POLR",
-	"PORN",
-	"PROV",
-	"PUBH",
-	"REL",
-	"SRCH",
-	"XED",
-}
-
 // Sharing settings
 type Sharing struct {
 	UploadResults bool `json:"upload_results"`
@@ -45,6 +12,7 @@ type Advanced struct {
 
 // Nettests related settings
 type Nettests struct {
+	WebsitesMaxRuntime           int64    `json:"websites_max_runtime"`
 	WebsitesURLLimit             int64    `json:"websites_url_limit"`
 	WebsitesEnabledCategoryCodes []string `json:"websites_enabled_category_codes"`
 }

cmd/ooniprobe/internal/config/testdata/valid-config.json

@@ -5,7 +5,7 @@
     "upload_results": true
   },
   "nettests": {
-    "websites_url_limit": 0
+    "websites_max_runtime": 0
   },
   "advanced": {
     "send_crash_reports": true

cmd/ooniprobe/internal/log/handlers/batch/batch.go

@@ -9,8 +9,17 @@ import (
 	"github.com/apex/log"
 )
 
-// Default handler outputting to stderr.
+// Default handler outputting to stdout. We want to emit the batch
-var Default = New(os.Stderr)
+// output on the standard output, for two reasons:
+//
+// 1. because third party libraries MAY log on the stderr and
+// their logs are most likely not JSON;
+//
+// 2. because this enables piping to `jq` or other tools in
+// a much more natural way than when emitting on stderr.
+//
+// See https://github.com/ooni/probe/issues/1384.
+var Default = New(os.Stdout)
 
 // Handler implementation.
 type Handler struct {

cmd/ooniprobe/internal/log/handlers/cli/cli.go

@@ -14,7 +14,7 @@ import (
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
 )
 
-// Default handler outputting to stderr.
+// Default handler outputting to stdout.
 var Default = New(os.Stdout)
 
 // start time.

cmd/ooniprobe/internal/nettests/groups.go

@@ -37,6 +37,7 @@ var All = map[string]Group{
 			FacebookMessenger{},
 			Telegram{},
 			WhatsApp{},
+			Signal{},
 		},
 		UnattendedOK: true,
 	},
@@ -54,7 +55,6 @@ var All = map[string]Group{
 		Nettests: []Nettest{
 			DNSCheck{},
 			STUNReachability{},
-			Signal{},
 		},
 	},
 }

cmd/ooniprobe/internal/nettests/nettests.go

@@ -53,6 +53,10 @@ type Controller struct {
 	// using the command line using the --input flag.
 	Inputs []string
 
+	// RunType contains the run_type hint for the CheckIn API. If
+	// not set, the underlying code defaults to "timed".
+	RunType string
+
 	// numInputs is the total number of inputs
 	numInputs int
 
@@ -111,10 +115,20 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err
 		}
 	}
 
-	c.ntStartTime = time.Now()
+	maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second
+	if c.RunType == "timed" && maxRuntime > 0 {
+		log.Debug("disabling maxRuntime when running in the background")
+		maxRuntime = 0
+	}
+	start := time.Now()
+	c.ntStartTime = start
 	for idx, input := range inputs {
-		if c.Probe.IsTerminated() == true {
+		if c.Probe.IsTerminated() {
-			log.Debug("isTerminated == true, breaking the input loop")
+			log.Info("user requested us to terminate using Ctrl-C")
+			break
+		}
+		if maxRuntime > 0 && time.Since(start) > maxRuntime {
+			log.Info("exceeded maximum runtime")
 			break
 		}
 		c.curInputIdx = idx // allow for precise progress
@@ -166,7 +180,7 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err
 			}
 		}
 		// We only save the measurement to disk if we failed to upload the measurement
-		if saveToDisk == true {
+		if saveToDisk {
 			if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil {
 				return errors.Wrap(err, "failed to save measurement on disk")
 			}
@@ -198,6 +212,18 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err
 
 // OnProgress should be called when a new progress event is available.
 func (c *Controller) OnProgress(perc float64, msg string) {
+	// when we have maxRuntime, honor it
+	maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second
+	if c.RunType == "manual" && maxRuntime > 0 {
+		elapsed := time.Since(c.ntStartTime)
+		perc = float64(elapsed) / float64(maxRuntime)
+		eta := maxRuntime.Seconds() - elapsed.Seconds()
+		log.Debugf("OnProgress: %f - %s", perc, msg)
+		key := fmt.Sprintf("%T", c.nt)
+		output.Progress(key, perc, eta, msg)
+		return
+	}
+	// otherwise estimate the ETA
 	log.Debugf("OnProgress: %f - %s", perc, msg)
 	var eta float64
 	eta = -1.0
@@ -207,7 +233,7 @@ func (c *Controller) OnProgress(perc float64, msg string) {
 		step := 1.0 / float64(c.numInputs)
 		perc = floor + perc*step
 		if c.curInputIdx > 0 {
-			eta = (time.Now().Sub(c.ntStartTime).Seconds() / float64(c.curInputIdx)) * float64(c.numInputs-c.curInputIdx)
+			eta = (time.Since(c.ntStartTime).Seconds() / float64(c.curInputIdx)) * float64(c.numInputs-c.curInputIdx)
 		}
 	}
 	if c.ntCount > 0 {

cmd/ooniprobe/internal/nettests/run.go

@@ -1,6 +1,9 @@
 package nettests
 
 import (
+	"sync"
+	"time"
+
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
@@ -10,14 +13,46 @@ import (
 // RunGroupConfig contains the settings for running a nettest group.
 type RunGroupConfig struct {
 	GroupName  string
-	Probe      *ooni.Probe
 	InputFiles []string
 	Inputs     []string
+	Probe      *ooni.Probe
+	RunType    string // hint for check-in API
 }
 
+const websitesURLLimitRemoved = `WARNING: CONFIGURATION CHANGE REQUIRED:
+
+* Since ooniprobe 3.9.0, websites_url_limit has been replaced
+  by websites_max_runtime in the configuration
+
+* To silence this warning either set websites_url_limit to zero or
+  replace it with websites_max_runtime
+
+* For the rest of 2021, we will automatically convert websites_url_limit
+  to websites_max_runtime (if the latter is not already set)
+
+* We will consider that each URL in websites_url_limit takes five
+  seconds to run and thus calculate websites_max_runtime
+
+* Since 2022, we will start silently ignoring websites_url_limit
+`
+
+var deprecationWarningOnce sync.Once
+
 // RunGroup runs a group of nettests according to the specified config.
 func RunGroup(config RunGroupConfig) error {
-	if config.Probe.IsTerminated() == true {
+	if config.Probe.Config().Nettests.WebsitesURLLimit > 0 {
+		if config.Probe.Config().Nettests.WebsitesMaxRuntime <= 0 {
+			limit := config.Probe.Config().Nettests.WebsitesURLLimit
+			maxRuntime := 5 * limit
+			config.Probe.Config().Nettests.WebsitesMaxRuntime = maxRuntime
+		}
+		deprecationWarningOnce.Do(func() {
+			log.Warn(websitesURLLimitRemoved)
+			time.Sleep(30 * time.Second)
+		})
+	}
+
+	if config.Probe.IsTerminated() {
 		log.Debugf("context is terminated, stopping runNettestGroup early")
 		return nil
 	}
@@ -61,7 +96,7 @@ func RunGroup(config RunGroupConfig) error {
 	config.Probe.ListenForSignals()
 	config.Probe.MaybeListenForStdinClosed()
 	for i, nt := range group.Nettests {
-		if config.Probe.IsTerminated() == true {
+		if config.Probe.IsTerminated() {
 			log.Debugf("context is terminated, stopping group.Nettests early")
 			break
 		}
@@ -69,6 +104,7 @@ func RunGroup(config RunGroupConfig) error {
 		ctl := NewController(nt, config.Probe, result, sess)
 		ctl.InputFiles = config.InputFiles
 		ctl.Inputs = config.Inputs
+		ctl.RunType = config.RunType
 		ctl.SetNettestIndex(i, len(group.Nettests))
 		if err = nt.Run(ctl); err != nil {
 			log.WithError(err).Errorf("Failed to run %s", group.Label)

cmd/ooniprobe/internal/nettests/web_connectivity.go

@@ -6,23 +6,59 @@ import (
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	engine "github.com/ooni/probe-cli/v3/internal/engine"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 
-func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
+// preventMistakes makes the code more robust with respect to any possible
-	inputloader := engine.NewInputLoader(engine.InputLoaderConfig{
+// integration issue where the backend returns to us URLs that don't
-		InputPolicy:   engine.InputOrQueryTestLists,
+// belong to the category codes we requested.
-		Session:       ctl.Session,
+func preventMistakes(input []model.URLInfo, categories []string) (output []model.URLInfo) {
-		SourceFiles:   ctl.InputFiles,
+	if len(categories) <= 0 {
-		StaticInputs:  ctl.Inputs,
+		return input
-		URLCategories: categories,
+	}
-		URLLimit:      limit,
+	for _, entry := range input {
-	})
+		var found bool
+		for _, cat := range categories {
+			if entry.CategoryCode == cat {
+				found = true
+				break
+			}
+		}
+		if !found {
+			log.Warnf("URL %+v not in %+v; skipping", entry, categories)
+			continue
+		}
+		output = append(output, entry)
+	}
+	return
+}
+
+func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64, error) {
+	inputloader := &engine.InputLoader{
+		CheckInConfig: &model.CheckInConfig{
+			// Setting Charging and OnWiFi to true causes the CheckIn
+			// API to return to us as much URL as possible with the
+			// given RunType hint.
+			Charging: true,
+			OnWiFi:   true,
+			RunType:  ctl.RunType,
+			WebConnectivity: model.CheckInConfigWebConnectivity{
+				CategoryCodes: categories,
+			},
+		},
+		InputPolicy:  engine.InputOrQueryBackend,
+		Session:      ctl.Session,
+		SourceFiles:  ctl.InputFiles,
+		StaticInputs: ctl.Inputs,
+	}
+	log.Infof("Calling CheckIn API with %s runType", ctl.RunType)
 	testlist, err := inputloader.Load(context.Background())
-	var urls []string
-	urlIDMap := make(map[int64]int64)
 	if err != nil {
 		return nil, nil, err
 	}
+	testlist = preventMistakes(testlist, categories)
+	var urls []string
+	urlIDMap := make(map[int64]int64)
 	for idx, url := range testlist {
 		log.Debugf("Going over URL %d", idx)
 		urlID, err := database.CreateOrUpdateURL(
@@ -40,13 +76,12 @@ func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, ma
 }
 
 // WebConnectivity test implementation
-type WebConnectivity struct {
+type WebConnectivity struct{}
-}
 
 // Run starts the test
 func (n WebConnectivity) Run(ctl *Controller) error {
 	log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
-	urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesURLLimit, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
+	urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
 	if err != nil {
 		return err
 	}

cmd/ooniprobe/internal/nettests/web_connectivity_test.go

@@ -0,0 +1,77 @@
+package nettests
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestPreventMistakesWithCategories(t *testing.T) {
+	input := []model.URLInfo{{
+		CategoryCode: "NEWS",
+		URL:          "https://repubblica.it/",
+		CountryCode:  "IT",
+	}, {
+		CategoryCode: "HACK",
+		URL:          "https://2600.com",
+		CountryCode:  "XX",
+	}, {
+		CategoryCode: "FILE",
+		URL:          "https://addons.mozilla.org/",
+		CountryCode:  "XX",
+	}}
+	desired := []model.URLInfo{{
+		CategoryCode: "NEWS",
+		URL:          "https://repubblica.it/",
+		CountryCode:  "IT",
+	}, {
+		CategoryCode: "FILE",
+		URL:          "https://addons.mozilla.org/",
+		CountryCode:  "XX",
+	}}
+	output := preventMistakes(input, []string{"NEWS", "FILE"})
+	if diff := cmp.Diff(desired, output); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestPreventMistakesWithoutCategoriesAndNil(t *testing.T) {
+	input := []model.URLInfo{{
+		CategoryCode: "NEWS",
+		URL:          "https://repubblica.it/",
+		CountryCode:  "IT",
+	}, {
+		CategoryCode: "HACK",
+		URL:          "https://2600.com",
+		CountryCode:  "XX",
+	}, {
+		CategoryCode: "FILE",
+		URL:          "https://addons.mozilla.org/",
+		CountryCode:  "XX",
+	}}
+	output := preventMistakes(input, nil)
+	if diff := cmp.Diff(input, output); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestPreventMistakesWithoutCategoriesAndEmpty(t *testing.T) {
+	input := []model.URLInfo{{
+		CategoryCode: "NEWS",
+		URL:          "https://repubblica.it/",
+		CountryCode:  "IT",
+	}, {
+		CategoryCode: "HACK",
+		URL:          "https://2600.com",
+		CountryCode:  "XX",
+	}, {
+		CategoryCode: "FILE",
+		URL:          "https://addons.mozilla.org/",
+		CountryCode:  "XX",
+	}}
+	output := preventMistakes(input, []string{})
+	if diff := cmp.Diff(input, output); diff != "" {
+		t.Fatal(diff)
+	}
+}

cmd/ooniprobe/internal/ooni/default-config.json

@@ -5,7 +5,7 @@
     "upload_results": true
   },
   "nettests": {
-    "websites_url_limit": 0
+    "websites_max_runtime": 0
   },
   "advanced": {
     "send_crash_reports": true

cmd/ooniprobe/internal/ooni/ooni.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
 	engine "github.com/ooni/probe-cli/v3/internal/engine"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/assetsdir"
 	"github.com/pkg/errors"
 	"upper.io/db.v3/lib/sqlbuilder"
 )
@@ -177,6 +178,14 @@ func (p *Probe) Init(softwareName, softwareVersion string) error {
 	}
 	p.db = db
 
+	// We cleanup the assets files used by versions of ooniprobe
+	// older than v3.9.0, where we started embedding the assets
+	// into the binary and use that directly. This cleanup doesn't
+	// remove the whole directory but only known files inside it
+	// and then the directory itself, if empty. We explicitly discard
+	// the return value as it does not matter to us here.
+	_, _ = assetsdir.Cleanup(utils.AssetsDir(p.home))
+
 	tempDir, err := ioutil.TempDir("", "ooni")
 	if err != nil {
 		return errors.Wrap(err, "creating TempDir")
@@ -199,7 +208,6 @@ func (p *Probe) NewSession() (*engine.Session, error) {
 		return nil, errors.Wrap(err, "creating engine's kvstore")
 	}
 	return engine.NewSession(engine.SessionConfig{
-		AssetsDir:       utils.AssetsDir(p.home),
 		KVStore:         kvstore,
 		Logger:          enginex.Logger,
 		SoftwareName:    p.softwareName,

cmd/ooniprobe/internal/utils/utils.go

@@ -39,7 +39,7 @@ func EscapeAwareRuneCountInString(s string) int {
 	return n
 }
 
-// RightPadd adds right padding in from of a string
+// RightPad adds right padding in from of a string
 func RightPad(str string, length int) string {
 	c := length - EscapeAwareRuneCountInString(str)
 	if c < 0 {

cmd/ooniprobe/testdata/testing-config.json

@@ -5,7 +5,7 @@
     "upload_results": true
   },
   "nettests": {
-    "websites_url_limit": 10
+    "websites_max_runtime": 15
   },
   "advanced": {
     "send_crash_reports": true

debian/ooniprobe.conf.disabled

@@ -6,7 +6,7 @@
     "upload_results": true
   },
   "nettests": {
-    "websites_url_limit": 0,
+    "websites_max_runtime": 0,
     "websites_enabled_category_codes": null
   },
   "advanced": {

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/cretz/bine v0.1.0
 	github.com/dchest/siphash v1.2.2 // indirect
 	github.com/fatih/color v1.10.0
-	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/google/go-cmp v0.5.2
 	github.com/google/martian/v3 v3.1.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
@@ -19,29 +18,27 @@ require (
 	github.com/gorilla/websocket v1.4.2
 	github.com/hexops/gotextdiff v1.0.3
 	github.com/iancoleman/strcase v0.1.3
-	github.com/lucas-clemente/quic-go v0.19.3
+	github.com/lucas-clemente/quic-go v0.20.0
-	github.com/marten-seemann/qtls-go1-15 v0.1.2 // indirect
 	github.com/mattn/go-colorable v0.1.8
 	github.com/mattn/go-sqlite3 v1.14.6 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
-	github.com/miekg/dns v1.1.40
+	github.com/miekg/dns v1.1.41
 	github.com/montanaflynn/stats v0.6.5
-	github.com/ooni/psiphon v0.5.0
+	github.com/ooni/probe-assets v0.0.0-20210401100648-90ed7b6dff90
+	github.com/ooni/psiphon v0.6.0
 	github.com/oschwald/geoip2-golang v1.5.0
 	github.com/pborman/getopt/v2 v2.1.0
 	github.com/pion/stun v0.3.5
 	github.com/pkg/errors v0.9.1
-	github.com/rogpeppe/go-internal v1.7.0
+	github.com/rogpeppe/go-internal v1.8.0
 	github.com/rubenv/sql-migrate v0.0.0-20210215143335-f84234893558
 	github.com/sirupsen/logrus v1.7.0 // indirect
 	gitlab.com/yawning/obfs4.git v0.0.0-20201217005658-f638c33f6c6f
-	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
-	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
+	golang.org/x/net v0.0.0-20210331060903-cb1fcc7394e5
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44
-	golang.org/x/sys v0.0.0-20210303074136-134d130e1a04
 	golang.org/x/text v0.3.5 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/AlecAivazis/survey.v1 v1.8.8
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	upper.io/db.v3 v3.8.0+incompatible

go.sum

@@ -139,13 +139,12 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -154,10 +153,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -165,7 +161,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
@@ -262,19 +257,18 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4=
+github.com/lucas-clemente/quic-go v0.20.0 h1:FSU3YN5VnLafHR27Ejs1r1CYMS7XMyIVDzRewkDLNBw=
-github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8=
+github.com/lucas-clemente/quic-go v0.20.0/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
-github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
+github.com/marten-seemann/qtls-go1-15 v0.1.4 h1:RehYMOyRW8hPVEja1KBVsFVNSm35Jj9Mvs5yNoZZ28A=
-github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
+github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
-github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
+github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJGYmwfwKQN2gU=
-github.com/marten-seemann/qtls-go1-15 v0.1.2 h1:KLXnVazsIS+EhrEqXqg0NyQZ2rwxkaSaNxMFc1krFIA=
+github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
-github.com/marten-seemann/qtls-go1-15 v0.1.2/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -298,8 +292,8 @@ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQ
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.40 h1:pyyPFfGMnciYUk/mXpKkVmeMQjfXqt3FAJ2hy7tPiLA=
+github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
-github.com/miekg/dns v1.1.40/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -341,8 +335,10 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/ooni/psiphon v0.5.0 h1:DAKZ66tmh4r6uop4yfSiQDObv7yNGb0j1jY+xeft6h4=
+github.com/ooni/probe-assets v0.0.0-20210401100648-90ed7b6dff90 h1:G7urihGBD88p5iU1bg7cFAqX/38qIPZH03wCTmyUAXI=
-github.com/ooni/psiphon v0.5.0/go.mod h1:i1v6JweJtxDKaI0i1aEw2/Fr/CUi5BoQ75GYz5KmKwU=
+github.com/ooni/probe-assets v0.0.0-20210401100648-90ed7b6dff90/go.mod h1:N0PyNM3aadlYDDCFXAPzs54HC54+MZA/4/xnCtd9EAo=
+github.com/ooni/psiphon v0.6.0 h1:TWXFSlXBOFUwfGblLKaLXSoI7UL6dreoPXHW2uQTHlc=
+github.com/ooni/psiphon v0.6.0/go.mod h1:i1v6JweJtxDKaI0i1aEw2/Fr/CUi5BoQ75GYz5KmKwU=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
@@ -368,6 +364,7 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg=
 github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -402,8 +399,8 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.7.0 h1:3qqXGV8nn7GJT65debw77Dzrx9sfWYgP0DDo7xcMFRk=
+github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
-github.com/rogpeppe/go-internal v1.7.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/rubenv/sql-migrate v0.0.0-20210215143335-f84234893558 h1:o8N+eY3HGAzZ+5sXNdcbCVOHW3NOksmKeEOuygusmr8=
 github.com/rubenv/sql-migrate v0.0.0-20210215143335-f84234893558/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -521,8 +518,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -532,6 +529,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -553,11 +551,11 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210331060903-cb1fcc7394e5 h1:zuP3axpB9rV3xH0EA7n3/gCrNPZm2SRl0l4mVH2BRj4=
+golang.org/x/net v0.0.0-20210331060903-cb1fcc7394e5/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -569,8 +567,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -593,7 +591,6 @@ golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -604,12 +601,12 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 h1:cEhElsAv9LUt9ZUUocxzWe05oFLVd+AA2nstydTeI8g=
+golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -634,7 +631,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -660,7 +657,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -672,17 +668,12 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/AlecAivazis/survey.v1 v1.8.8 h1:5UtTowJZTz1j7NxVzDGKTz6Lm9IWm8DDF6b7a2wq9VY=
 gopkg.in/AlecAivazis/survey.v1 v1.8.8/go.mod h1:CaHjv79TCgAvXMSFJSVgonHXYWxnhzI3eoHtnX5UgUo=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -719,8 +710,6 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=

internal/cmd/getresources/getresources.go

@@ -2,50 +2,13 @@
 package main
 
 import (
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io/ioutil"
 	"log"
-	"net/http"
+	"time"
-	"net/url"
-	"path/filepath"
-
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
 )
 
 func main() {
-	for name, ri := range resources.All {
+	log.Printf("This command is no longer needed. We will keep it into")
-		if err := getit(name, &ri); err != nil {
+	log.Printf("the repository until the end of 2021, so you have\n")
-			log.Fatal(err)
+	log.Printf("time to adjust your build workflow.\n")
-		}
+	time.Sleep(5 * time.Second)
-	}
-}
-
-func getit(name string, ri *resources.ResourceInfo) error {
-	workDir := filepath.Join("internal", "engine", "resourcesmanager")
-	URL, err := url.Parse(resources.BaseURL)
-	if err != nil {
-		return err
-	}
-	URL.Path = ri.URLPath
-	log.Println("fetching", URL.String())
-	resp, err := http.Get(URL.String())
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != 200 {
-		return errors.New("http request failed")
-	}
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	checksum := fmt.Sprintf("%x", sha256.Sum256(data))
-	if checksum != ri.GzSHA256 {
-		return errors.New("sha256 mismatch")
-	}
-	fullpath := filepath.Join(workDir, name+".gz")
-	return ioutil.WriteFile(fullpath, data, 0644)
 }

internal/cmd/miniooni/README.md

@@ -3,6 +3,7 @@
 This directory contains the source code of a simple CLI client that we
 use for research as well as for running QA scripts. We designed this tool
 to have a CLI similar to MK and OONI Probe v2.x to ease running Jafar
-scripts that check whether these tools behave similarly.
+scripts that check whether these tools behave similarly. Perfect backwards
-
+compatibility was not a design goal for miniooni. Rather, we aimed to
-See also libminiooni.
+have as little conflict as possible, such that we can run side-by-side
+QA checks.

internal/cmd/miniooni/libminiooni.go

@@ -1,16 +1,4 @@
-// Package libminiooni implements the cmd/miniooni CLI. Miniooni is our
+package main
-// experimental client used for research and QA testing.
-//
-// This CLI has CLI options that do not conflict with Measurement Kit
-// v0.10.x CLI options. There are some options conflict with the legacy
-// OONI Probe CLI options. Perfect backwards compatibility is not a
-// design goal for miniooni. Rather, we aim to have as little conflict
-// as possible such that we can run side by side QA checks.
-//
-// We extracted this package from cmd/miniooni to allow us to further
-// integrate the miniooni CLI into other binaries (see for example the
-// code at github.com/bassosimone/aladdin).
-package libminiooni
 
 import (
 	"context"
@@ -30,6 +18,7 @@ import (
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/engine"
 	"github.com/ooni/probe-cli/v3/internal/engine/humanizex"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/assetsdir"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
 	"github.com/ooni/probe-cli/v3/internal/version"
@@ -44,6 +33,7 @@ type Options struct {
 	Inputs           []string
 	InputFilePaths   []string
 	Limit            int64
+	MaxRuntime       int64
 	NoJSON           bool
 	NoCollector      bool
 	ProbeServicesURL string
@@ -93,6 +83,10 @@ func init() {
 		&globalOptions.Limit, "limit", 0,
 		"Limit the number of URLs tested by Web Connectivity", "N",
 	)
+	getopt.FlagLong(
+		&globalOptions.MaxRuntime, "max-runtime", 0,
+		"Maximum runtime in seconds when looping over a list of inputs (zero means infinite)", "N",
+	)
 	getopt.FlagLong(
 		&globalOptions.NoJSON, "no-json", 'N', "Disable writing to disk",
 	)
@@ -140,10 +134,6 @@ func init() {
 	)
 }
 
-func fatalWithString(msg string) {
-	panic(msg)
-}
-
 func fatalIfFalse(cond bool, msg string) {
 	if !cond {
 		panic(msg)
@@ -274,12 +264,23 @@ func maybeWriteConsentFile(yes bool, filepath string) (err error) {
 	return
 }
 
+// limitRemoved is the text printed when the user uses --limit
+const limitRemoved = `USAGE CHANGE: The --limit option has been removed in favor of
+the --max-runtime option. Please, update your script to use --max-runtime
+instead of --limit. The argument to --max-runtime is the maximum number
+of seconds after which to stop running Web Connectivity.
+
+This error message will be removed after 2021-11-01.
+`
+
 // MainWithConfiguration is the miniooni main with a specific configuration
 // represented by the experiment name and the current options.
 //
 // This function will panic in case of a fatal error. It is up to you that
 // integrate this function to either handle the panic of ignore it.
 func MainWithConfiguration(experimentName string, currentOptions Options) {
+	fatalIfFalse(currentOptions.Limit == 0, limitRemoved)
+
 	ctx := context.Background()
 
 	extraOptions := mustMakeMap(currentOptions.ExtraOptions)
@@ -303,9 +304,18 @@ func MainWithConfiguration(experimentName string, currentOptions Options) {
 	homeDir := gethomedir(currentOptions.HomeDir)
 	fatalIfFalse(homeDir != "", "home directory is empty")
 	miniooniDir := path.Join(homeDir, ".miniooni")
+	err = os.MkdirAll(miniooniDir, 0700)
+	fatalOnError(err, "cannot create $HOME/.miniooni directory")
+
+	// We cleanup the assets files used by versions of ooniprobe
+	// older than v3.9.0, where we started embedding the assets
+	// into the binary and use that directly. This cleanup doesn't
+	// remove the whole directory but only known files inside it
+	// and then the directory itself, if empty. We explicitly discard
+	// the return value as it does not matter to us here.
 	assetsDir := path.Join(miniooniDir, "assets")
-	err = os.MkdirAll(assetsDir, 0700)
+	_, _ = assetsdir.Cleanup(assetsDir)
-	fatalOnError(err, "cannot create assets directory")
+
 	log.Debugf("miniooni state directory: %s", miniooniDir)
 
 	consentFile := path.Join(miniooniDir, "informed")
@@ -324,7 +334,6 @@ func MainWithConfiguration(experimentName string, currentOptions Options) {
 	fatalOnError(err, "cannot create kvstore2 directory")
 
 	config := engine.SessionConfig{
-		AssetsDir:       assetsDir,
 		KVStore:         kvstore,
 		Logger:          logger,
 		ProxyURL:        proxyURL,
@@ -370,13 +379,17 @@ func MainWithConfiguration(experimentName string, currentOptions Options) {
 	builder, err := sess.NewExperimentBuilder(experimentName)
 	fatalOnError(err, "cannot create experiment builder")
 
-	inputLoader := engine.NewInputLoader(engine.InputLoaderConfig{
+	inputLoader := &engine.InputLoader{
+		CheckInConfig: &model.CheckInConfig{
+			RunType:  "manual",
+			OnWiFi:   true, // meaning: not on 4G
+			Charging: true,
+		},
+		InputPolicy:  builder.InputPolicy(),
 		StaticInputs: currentOptions.Inputs,
 		SourceFiles:  currentOptions.InputFilePaths,
-		InputPolicy:  builder.InputPolicy(),
 		Session:      sess,
-		URLLimit:     currentOptions.Limit,
+	}
-	})
 	inputs, err := inputLoader.Load(context.Background())
 	fatalOnError(err, "cannot load inputs")
 
@@ -399,29 +412,30 @@ func MainWithConfiguration(experimentName string, currentOptions Options) {
 	}()
 
 	submitter, err := engine.NewSubmitter(ctx, engine.SubmitterConfig{
-		Enabled: currentOptions.NoCollector == false,
+		Enabled: !currentOptions.NoCollector,
 		Session: sess,
 		Logger:  log.Log,
 	})
 	fatalOnError(err, "cannot create submitter")
 
 	saver, err := engine.NewSaver(engine.SaverConfig{
-		Enabled:    currentOptions.NoJSON == false,
+		Enabled:    !currentOptions.NoJSON,
 		Experiment: experiment,
 		FilePath:   currentOptions.ReportFile,
 		Logger:     log.Log,
 	})
 	fatalOnError(err, "cannot create saver")
 
-	inputProcessor := engine.InputProcessor{
+	inputProcessor := &engine.InputProcessor{
 		Annotations: annotations,
 		Experiment: &experimentWrapper{
 			child: engine.NewInputProcessorExperimentWrapper(experiment),
 			total: len(inputs),
 		},
-		Inputs:  inputs,
+		Inputs:     inputs,
-		Options: currentOptions.ExtraOptions,
+		MaxRuntime: time.Duration(currentOptions.MaxRuntime) * time.Second,
-		Saver:   engine.NewInputProcessorSaverWrapper(saver),
+		Options:    currentOptions.ExtraOptions,
+		Saver:      engine.NewInputProcessorSaverWrapper(saver),
 		Submitter: submitterWrapper{
 			child: engine.NewInputProcessorSubmitterWrapper(submitter),
 		},

internal/cmd/miniooni/libminiooni_test.go

@@ -0,0 +1,12 @@
+package main
+
+import "testing"
+
+func TestSimple(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	MainWithConfiguration("example", Options{
+		Yes: true,
+	})
+}

internal/cmd/miniooni/main.go

@@ -1,14 +1,10 @@
 // Command miniooni is a simple binary for research and QA purposes
 // with a CLI interface similar to MK and OONI Probe v2.x.
-//
-// See also libminiooni, which is where we implement this CLI.
 package main
 
 import (
 	"fmt"
 	"os"
-
-	"github.com/ooni/probe-cli/v3/internal/libminiooni"
 )
 
 func main() {
@@ -17,5 +13,5 @@ func main() {
 			fmt.Fprintf(os.Stderr, "%s", s)
 		}
 	}()
-	libminiooni.Main()
+	Main()
 }

internal/cmd/oohelper/oohelper.go

@@ -24,9 +24,21 @@ var (
 	target      = flag.String("target", "", "Target URL for the test helper")
 )
 
+func newhttpclient() *http.Client {
+	// Use a nonstandard resolver, which is enough to work around the
+	// puzzling https://github.com/ooni/probe/issues/1409 issue.
+	childResolver, err := netx.NewDNSClient(
+		netx.Config{Logger: log.Log}, "dot://8.8.8.8:853")
+	runtimex.PanicOnError(err, "netx.NewDNSClient should not fail here")
+	txp := netx.NewHTTPTransport(netx.Config{
+		BaseResolver: childResolver,
+		Logger:       log.Log,
+	})
+	return &http.Client{Transport: txp}
+}
+
 func init() {
-	txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
+	httpClient = newhttpclient()
-	httpClient = &http.Client{Transport: txp}
 	resolver = netx.NewResolver(netx.Config{Logger: log.Log})
 }
 

internal/engine/allexperiments.go

@@ -152,7 +152,7 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
 				))
 			},
 			config:      &httphostheader.Config{},
-			inputPolicy: InputOrQueryTestLists,
+			inputPolicy: InputOrQueryBackend,
 		}
 	},
 
@@ -237,7 +237,7 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
 				))
 			},
 			config:      &sniblocking.Config{},
-			inputPolicy: InputOrQueryTestLists,
+			inputPolicy: InputOrQueryBackend,
 		}
 	},
 
@@ -273,7 +273,7 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
 				))
 			},
 			config:      &tlstool.Config{},
-			inputPolicy: InputOrQueryTestLists,
+			inputPolicy: InputOrQueryBackend,
 		}
 	},
 
@@ -309,7 +309,7 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
 				))
 			},
 			config:      &webconnectivity.Config{},
-			inputPolicy: InputOrQueryTestLists,
+			inputPolicy: InputOrQueryBackend,
 		}
 	},
 

internal/engine/experiment.go

@@ -1,4 +1,4 @@
-// Package engine contains the engine API
+// Package engine contains the engine API.
 package engine
 
 import (
@@ -7,7 +7,6 @@ import (
 	"errors"
 	"net/http"
 	"os"
-	"strconv"
 	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
@@ -17,7 +16,6 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
 	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
-	"github.com/ooni/probe-cli/v3/internal/engine/resources"
 	"github.com/ooni/probe-cli/v3/internal/version"
 )
 
@@ -168,7 +166,6 @@ func (e *Experiment) newMeasurement(input string) *model.Measurement {
 		TestStartTime:             e.testStartTime,
 		TestVersion:               e.testVersion,
 	}
-	m.AddAnnotation("assets_version", strconv.FormatInt(resources.Version, 10))
 	m.AddAnnotation("engine_name", "ooniprobe-engine")
 	m.AddAnnotation("engine_version", version.Version)
 	m.AddAnnotation("platform", platform.Name())
@@ -188,10 +185,7 @@ func (e *Experiment) OpenReportContext(ctx context.Context) error {
 			Counter:      e.byteCounter,
 		},
 	}
-	if e.session.selectedProbeService == nil {
+	client, err := e.session.NewProbeServicesClient(ctx)
-		return errors.New("no probe services selected")
-	}
-	client, err := probeservices.NewClient(e.session, *e.session.selectedProbeService)
 	if err != nil {
 		e.session.logger.Debugf("%+v", err)
 		return err

internal/engine/experiment/dnscheck/dnscheck.go

@@ -169,7 +169,7 @@ func (m *Measurer) Run(
 		ResolveSaver: evsaver,
 	})
 	addrs, err := m.lookupHost(ctx, URL.Hostname(), resolver)
-	queries := archival.NewDNSQueriesList(begin, evsaver.Read(), sess.ASNDatabasePath())
+	queries := archival.NewDNSQueriesList(begin, evsaver.Read())
 	tk.BootstrapFailure = archival.NewFailure(err)
 	if len(queries) > 0 {
 		// We get no queries in case we are resolving an IP address, since

internal/engine/experiment/fbmessenger/fbmessenger_test.go

@@ -222,7 +222,6 @@ func TestComputeEndpointStatsDNSIsLying(t *testing.T) {
 
 func newsession(t *testing.T) model.ExperimentSession {
 	sess, err := engine.NewSession(engine.SessionConfig{
-		AssetsDir: "../../testdata",
 		AvailableProbeServices: []model.Service{{
 			Address: "https://ams-pg-test.ooni.org",
 			Type:    "https",

internal/engine/experiment/hhfm/hhfm_test.go

@@ -559,7 +559,6 @@ func TestTransactCannotReadBody(t *testing.T) {
 
 func newsession(t *testing.T) model.ExperimentSession {
 	sess, err := engine.NewSession(engine.SessionConfig{
-		AssetsDir: "../../testdata",
 		AvailableProbeServices: []model.Service{{
 			Address: "https://ams-pg-test.ooni.org",
 			Type:    "https",

internal/engine/experiment/riseupvpn/riseupvpn.go

@@ -9,7 +9,6 @@ import (
 	"errors"
 	"time"
 
-	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx"
@@ -18,7 +17,7 @@ import (
 
 const (
 	testName      = "riseupvpn"
-	testVersion   = "0.1.0"
+	testVersion   = "0.2.0"
 	eipServiceURL = "https://api.black.riseup.net:443/3/config/eip-service.json"
 	providerURL   = "https://riseup.net/provider.json"
 	geoServiceURL = "https://api.black.riseup.net:9001/json"
@@ -66,6 +65,7 @@ type TestKeys struct {
 	APIStatus       string              `json:"api_status"`
 	CACertStatus    bool                `json:"ca_cert_status"`
 	FailingGateways []GatewayConnection `json:"failing_gateways"`
+	TransportStatus map[string]string   `json:"transport_status"`
 }
 
 // NewTestKeys creates new riseupvpn TestKeys.
@@ -75,6 +75,7 @@ func NewTestKeys() *TestKeys {
 		APIStatus:       "ok",
 		CACertStatus:    true,
 		FailingGateways: nil,
+		TransportStatus: nil,
 	}
 }
 
@@ -96,6 +97,7 @@ func (tk *TestKeys) UpdateProviderAPITestKeys(v urlgetter.MultiOutput) {
 }
 
 // AddGatewayConnectTestKeys updates the TestKeys using the given MultiOutput result of gateway connectivity testing.
+// Sets TransportStatus to "ok" if any successful TCP connection could be made
 func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transportType string) {
 	tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
 	tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
@@ -108,6 +110,29 @@ func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transport
 	return
 }
 
+func (tk *TestKeys) updateTransportStatus(openvpnGatewayCount int, obfs4GatewayCount int) {
+	failingOpenvpnGateways, failingObfs4Gateways := 0, 0
+	for _, gw := range tk.FailingGateways {
+		if gw.TransportType == "openvpn" {
+			failingOpenvpnGateways++
+		} else if gw.TransportType == "obfs4" {
+			failingObfs4Gateways++
+		}
+	}
+
+	if failingOpenvpnGateways < openvpnGatewayCount {
+		tk.TransportStatus["openvpn"] = "ok"
+	} else {
+		tk.TransportStatus["openvpn"] = "blocked"
+	}
+
+	if failingObfs4Gateways < obfs4GatewayCount {
+		tk.TransportStatus["obfs4"] = "ok"
+	} else {
+		tk.TransportStatus["obfs4"] = "blocked"
+	}
+}
+
 func newGatewayConnection(tcpConnect archival.TCPConnectEntry, transportType string) *GatewayConnection {
 	return &GatewayConnection{
 		IP:            tcpConnect.IP,
@@ -160,30 +185,32 @@ func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
 	urlgetter.RegisterExtensions(measurement)
 
 	caTarget := "https://black.riseup.net/ca.crt"
-	caGetter := urlgetter.Getter{
-		Config:  m.Config.Config,
-		Session: sess,
-		Target:  caTarget,
-	}
-	log.Info("Getting CA certificate; please be patient...")
-	tk, err := caGetter.Get(ctx)
-	testkeys.AddCACertFetchTestKeys(tk)
-
-	if err != nil {
-		log.Error("Getting CA certificate failed. Aborting test.")
-		return nil
-	}
-
 	certPool := netx.NewDefaultCertPool()
-	if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok {
+
-		testkeys.CACertStatus = false
+	multi := urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
-		testkeys.APIStatus = "blocked"
+	inputs := []urlgetter.MultiInput{
-		errorValue := "invalid_ca"
+		{Target: caTarget, Config: urlgetter.Config{
-		testkeys.APIFailure = &errorValue
+			Method:          "GET",
-		return nil
+			FailOnHTTPError: true,
+		}},
+	}
+	for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) {
+		tk := entry.TestKeys
+		testkeys.AddCACertFetchTestKeys(tk)
+		if tk.Failure != nil {
+			return nil
+		}
+
+		if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok {
+			testkeys.CACertStatus = false
+			testkeys.APIStatus = "blocked"
+			errorValue := "invalid_ca"
+			testkeys.APIFailure = &errorValue
+			return nil
+		}
 	}
 
-	inputs := []urlgetter.MultiInput{
+	inputs = []urlgetter.MultiInput{
 
 		// Here we need to provide the method explicitly. See
 		// https://github.com/ooni/probe-engine/issues/827.
@@ -203,30 +230,34 @@ func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
 			FailOnHTTPError: true,
 		}},
 	}
-	multi := urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
+	multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
 
-	for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) {
+	for entry := range multi.CollectOverall(ctx, inputs, 1, 50, "riseupvpn", callbacks) {
 		testkeys.UpdateProviderAPITestKeys(entry)
 	}
 
 	// test gateways now
+	testkeys.TransportStatus = map[string]string{}
 	gateways := parseGateways(testkeys)
 	openvpnEndpoints := generateMultiInputs(gateways, "openvpn")
 	obfs4Endpoints := generateMultiInputs(gateways, "obfs4")
-	overallCount := len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints)
+	overallCount := 1 + len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints)
 
 	// measure openvpn in parallel
 	multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
-	for entry := range multi.CollectOverall(ctx, openvpnEndpoints, len(inputs), overallCount, "riseupvpn", callbacks) {
+	for entry := range multi.CollectOverall(ctx, openvpnEndpoints, 1+len(inputs), overallCount, "riseupvpn", callbacks) {
 		testkeys.AddGatewayConnectTestKeys(entry, "openvpn")
 	}
 
 	// measure obfs4 in parallel
 	multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
-	for entry := range multi.CollectOverall(ctx, obfs4Endpoints, len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) {
+	for entry := range multi.CollectOverall(ctx, obfs4Endpoints, 1+len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) {
 		testkeys.AddGatewayConnectTestKeys(entry, "obfs4")
 	}
 
+	// set transport status based on gateway test results
+	testkeys.updateTransportStatus(len(openvpnEndpoints), len(obfs4Endpoints))
+
 	return nil
 }
 
@@ -287,10 +318,11 @@ func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
 // Note that this structure is part of the ABI contract with probe-cli
 // therefore we should be careful when changing it.
 type SummaryKeys struct {
-	APIBlocked      bool `json:"api_blocked"`
+	APIBlocked      bool              `json:"api_blocked"`
-	ValidCACert     bool `json:"valid_ca_cert"`
+	ValidCACert     bool              `json:"valid_ca_cert"`
-	FailingGateways int  `json:"failing_gateways"`
+	FailingGateways int               `json:"failing_gateways"`
-	IsAnomaly       bool `json:"-"`
+	TransportStatus map[string]string `json:"transport_status"`
+	IsAnomaly       bool              `json:"-"`
 }
 
 // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
@@ -303,7 +335,8 @@ func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, e
 	sk.APIBlocked = tk.APIStatus != "ok"
 	sk.ValidCACert = tk.CACertStatus
 	sk.FailingGateways = len(tk.FailingGateways)
+	sk.TransportStatus = tk.TransportStatus
 	sk.IsAnomaly = (sk.APIBlocked == true || tk.CACertStatus == false ||
-		sk.FailingGateways != 0)
+		tk.TransportStatus["openvpn"] == "blocked" || tk.TransportStatus["obfs4"] == "blocked")
 	return sk, nil
 }

internal/engine/experiment/riseupvpn/riseupvpn_test.go

@@ -2,15 +2,14 @@ package riseupvpn_test
 
 import (
 	"context"
-	"crypto/tls"
+	"encoding/json"
-	"crypto/x509"
-	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
-	"math/rand"
+	"strconv"
-	"net/http"
+	"strings"
 	"testing"
-	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
 
 	"github.com/apex/log"
 	"github.com/google/go-cmp/cmp"
@@ -19,36 +18,204 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
-	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
 )
 
+const (
+	provider = `{
+		"api_uri": "https://api.black.riseup.net:443",
+		"api_version": "3",
+		"ca_cert_fingerprint": "SHA256: a5244308a1374709a9afce95e3ae47c1b44bc2398c0a70ccbf8b3a8a97f29494",
+		"ca_cert_uri": "https://black.riseup.net/ca.crt",
+		"default_language": "en",
+		"description": {
+		  "en": "Riseup is a non-profit collective in Seattle that provides online communication tools for people and groups working toward liberatory social change."
+		},
+		"domain": "riseup.net",
+		"enrollment_policy": "closed",
+		"languages": [
+		  "en"
+		],
+		"name": {
+		  "en": "Riseup Networks"
+		},
+		"service": {
+		  "allow_anonymous": true,
+		  "allow_free": true,
+		  "allow_limited_bandwidth": false,
+		  "allow_paid": false,
+		  "allow_registration": false,
+		  "allow_unlimited_bandwidth": true,
+		  "bandwidth_limit": 102400,
+		  "default_service_level": 1,
+		  "levels": {
+			"1": {
+			  "description": "Please donate.",
+			  "name": "free"
+			}
+		  }
+		},
+		"services": [
+		  "openvpn"
+		]
+	  }`
+	eipservice = `{
+		"gateways": [
+		  {
+			"capabilities": {
+			  "adblock": false,
+			  "filter_dns": false,
+			  "limited": false,
+			  "transport":[
+				{
+				  "type":"openvpn",
+				  "protocols":[
+					"tcp"
+				  ],
+				  "ports":[
+					"443"
+				  ]
+				}
+			  ],
+			  "user_ips": false
+			},
+			"host": "test1.riseup.net",
+			"ip_address": "123.456.123.456",
+			"location": "paris"
+		  },
+		  {
+			"capabilities": {
+			  "adblock": false,
+			  "filter_dns": false,
+			  "limited": false,
+			  "transport":[
+				{
+				  "type":"obfs4",
+				  "protocols":[
+					"tcp"
+				  ],
+				  "ports":[
+					"23042"
+				  ],
+				  "options": {
+					"cert": "XXXXXXXXXXXXXXXXXXXXXXXXX",
+					"iatMode": "0"
+				  }
+				},              
+				{
+				  "type":"openvpn",
+				  "protocols":[
+					"tcp"
+				  ],
+				  "ports":[
+					"443"
+				  ]
+				}
+			  ],
+			  "user_ips": false
+			},
+			"host": "test2.riseup.net",
+			"ip_address": "234.345.234.345",
+			"location": "seattle"
+		  }
+		],
+		"locations": {
+		  "paris": {
+			"country_code": "FR",
+			"hemisphere": "N",
+			"name": "Paris",
+			"timezone": "+2"
+		  },
+		  "seattle": {
+			"country_code": "US",
+			"hemisphere": "N",
+			"name": "Seattle",
+			"timezone": "-7"
+		  }
+		},
+		"openvpn_configuration": {
+		  "auth": "SHA1",
+		  "cipher": "AES-128-CBC",
+		  "keepalive": "10 30",
+		  "tls-cipher": "DHE-RSA-AES128-SHA",
+		  "tun-ipv6": true
+		},
+		"serial": 3,
+		"version": 3
+	  }`
+	geoservice = `{"ip":"51.15.0.88","cc":"NL","city":"Haarlem","lat":52.381,"lon":4.6275,"gateways":["test1.riseup.net","test2.riseup.net"]}`
+	cacert     = `-----BEGIN CERTIFICATE-----
+MIIFjTCCA3WgAwIBAgIBATANBgkqhkiG9w0BAQ0FADBZMRgwFgYDVQQKDA9SaXNl
+dXAgTmV0d29ya3MxGzAZBgNVBAsMEmh0dHBzOi8vcmlzZXVwLm5ldDEgMB4GA1UE
+AwwXUmlzZXVwIE5ldHdvcmtzIFJvb3QgQ0EwHhcNMTQwNDI4MDAwMDAwWhcNMjQw
+NDI4MDAwMDAwWjBZMRgwFgYDVQQKDA9SaXNldXAgTmV0d29ya3MxGzAZBgNVBAsM
+Emh0dHBzOi8vcmlzZXVwLm5ldDEgMB4GA1UEAwwXUmlzZXVwIE5ldHdvcmtzIFJv
+b3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC76J4ciMJ8Sg0m
+TP7DF2DT9zNe0Csk4myoMFC57rfJeqsAlJCv1XMzBmXrw8wq/9z7XHv6n/0sWU7a
+7cF2hLR33ktjwODlx7vorU39/lXLndo492ZBhXQtG1INMShyv+nlmzO6GT7ESfNE
+LliFitEzwIegpMqxCIHXFuobGSCWF4N0qLHkq/SYUMoOJ96O3hmPSl1kFDRMtWXY
+iw1SEKjUvpyDJpVs3NGxeLCaA7bAWhDY5s5Yb2fA1o8ICAqhowurowJpW7n5ZuLK
+5VNTlNy6nZpkjt1QycYvNycffyPOFm/Q/RKDlvnorJIrihPkyniV3YY5cGgP+Qkx
+HUOT0uLA6LHtzfiyaOqkXwc4b0ZcQD5Vbf6Prd20Ppt6ei0zazkUPwxld3hgyw58
+m/4UIjG3PInWTNf293GngK2Bnz8Qx9e/6TueMSAn/3JBLem56E0WtmbLVjvko+LF
+PM5xA+m0BmuSJtrD1MUCXMhqYTtiOvgLBlUm5zkNxALzG+cXB28k6XikXt6MRG7q
+hzIPG38zwkooM55yy5i1YfcIi5NjMH6A+t4IJxxwb67MSb6UFOwg5kFokdONZcwj
+shczHdG9gLKSBIvrKa03Nd3W2dF9hMbRu//STcQxOailDBQCnXXfAATj9pYzdY4k
+ha8VCAREGAKTDAex9oXf1yRuktES4QIDAQABo2AwXjAdBgNVHQ4EFgQUC4tdmLVu
+f9hwfK4AGliaet5KkcgwDgYDVR0PAQH/BAQDAgIEMAwGA1UdEwQFMAMBAf8wHwYD
+VR0jBBgwFoAUC4tdmLVuf9hwfK4AGliaet5KkcgwDQYJKoZIhvcNAQENBQADggIB
+AGzL+GRnYu99zFoy0bXJKOGCF5XUXP/3gIXPRDqQf5g7Cu/jYMID9dB3No4Zmf7v
+qHjiSXiS8jx1j/6/Luk6PpFbT7QYm4QLs1f4BlfZOti2KE8r7KRDPIecUsUXW6P/
+3GJAVYH/+7OjA39za9AieM7+H5BELGccGrM5wfl7JeEz8in+V2ZWDzHQO4hMkiTQ
+4ZckuaL201F68YpiItBNnJ9N5nHr1MRiGyApHmLXY/wvlrOpclh95qn+lG6/2jk7
+3AmihLOKYMlPwPakJg4PYczm3icFLgTpjV5sq2md9bRyAg3oPGfAuWHmKj2Ikqch
+Td5CHKGxEEWbGUWEMP0s1A/JHWiCbDigc4Cfxhy56CWG4q0tYtnc2GMw8OAUO6Wf
+Xu5pYKNkzKSEtT/MrNJt44tTZWbKV/Pi/N2Fx36my7TgTUj7g3xcE9eF4JV2H/sg
+tsK3pwE0FEqGnT4qMFbixQmc8bGyuakr23wjMvfO7eZUxBuWYR2SkcP26sozF9PF
+tGhbZHQVGZUTVPyvwahMUEhbPGVerOW0IYpxkm0x/eaWdTc4vPpf/rIlgbAjarnJ
+UN9SaWRlWKSdP4haujnzCoJbM7dU9bjvlGZNyXEekgeT0W2qFeGGp+yyUWw8tNsp
+0BuC1b7uW/bBn/xKm319wXVDvBgZgcktMolak39V7DVO
+-----END CERTIFICATE-----`
+
+	eipserviceurl = "https://api.black.riseup.net:443/3/config/eip-service.json"
+	providerurl   = "https://riseup.net/provider.json"
+	geoserviceurl = "https://api.black.riseup.net:9001/json"
+	cacerturl     = "https://black.riseup.net/ca.crt"
+	openvpnurl1   = "tcpconnect://234.345.234.345:443"
+	openvpnurl2   = "tcpconnect://123.456.123.456:443"
+	obfs4url1     = "tcpconnect://234.345.234.345:23042"
+)
+
+var RequestResponse = map[string]string{
+	eipserviceurl: eipservice,
+	providerurl:   provider,
+	geoserviceurl: geoservice,
+	cacerturl:     cacert,
+	openvpnurl1:   "",
+	openvpnurl2:   "",
+	obfs4url1:     "",
+}
+
 func TestNewExperimentMeasurer(t *testing.T) {
 	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
 	if measurer.ExperimentName() != "riseupvpn" {
 		t.Fatal("unexpected name")
 	}
-	if measurer.ExperimentVersion() != "0.1.0" {
+	if measurer.ExperimentVersion() != "0.2.0" {
 		t.Fatal("unexpected version")
 	}
 }
 
 func TestGood(t *testing.T) {
-	if testing.Short() {
+	measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
-		t.Skip("skip test in short mode")
+		cacerturl:     true,
-	}
+		eipserviceurl: true,
-	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+		providerurl:   true,
-	measurement := new(model.Measurement)
+		geoserviceurl: true,
-	err := measurer.Run(
+		openvpnurl1:   true,
-		context.Background(),
+		openvpnurl2:   true,
-		&mockable.Session{
+		obfs4url1:     true,
-			MockableLogger: log.Log,
+	}))
-		},
+
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
 	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
 	if tk.Agent != "" {
 		t.Fatal("unexpected Agent: " + tk.Agent)
@@ -59,21 +226,6 @@ func TestGood(t *testing.T) {
 	if tk.Failure != nil {
 		t.Fatal("unexpected Failure")
 	}
-	if len(tk.NetworkEvents) <= 0 {
-		t.Fatal("no NetworkEvents?!")
-	}
-	if len(tk.Queries) <= 0 {
-		t.Fatal("no Queries?!")
-	}
-	if len(tk.Requests) <= 0 {
-		t.Fatal("no Requests?!")
-	}
-	if len(tk.TCPConnect) <= 0 {
-		t.Fatal("no TCPConnect?!")
-	}
-	if len(tk.TLSHandshakes) <= 0 {
-		t.Fatal("no TLSHandshakes?!")
-	}
 	if tk.APIFailure != nil {
 		t.Fatal("unexpected ApiFailure")
 	}
@@ -86,6 +238,12 @@ func TestGood(t *testing.T) {
 	if tk.FailingGateways != nil {
 		t.Fatal("unexpected FailingGateways value")
 	}
+	if tk.TransportStatus == nil {
+		t.Fatal("unexpected nil TransportStatus struct ")
+	}
+	if tk.TransportStatus["openvpn"] != "ok" {
+		t.Fatal("unexpected openvpn transport status")
+	}
 }
 
 // TestUpdateWithMixedResults tests if one operation failed
@@ -132,13 +290,39 @@ func TestUpdateWithMixedResults(t *testing.T) {
 	if *tk.APIFailure != errorx.FailureEOFError {
 		t.Fatal("invalid ApiFailure")
 	}
+	if tk.FailingGateways != nil {
+		t.Fatal("invalid FailingGateways")
+	}
+	if tk.TransportStatus != nil {
+		t.Fatal("invalid TransportStatus")
+	}
 }
 
-func TestFailureCaCertFetch(t *testing.T) {
+func TestInvalidCaCert(t *testing.T) {
-	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	requestResponseMap := map[string]string{
+		eipserviceurl: eipservice,
+		providerurl:   provider,
+		geoserviceurl: geoservice,
+		cacerturl:     "invalid",
+		openvpnurl1:   "",
+		openvpnurl2:   "",
+		obfs4url1:     "",
+	}
+	measurer := riseupvpn.Measurer{
+		Config: riseupvpn.Config{},
+		Getter: generateMockGetter(requestResponseMap, map[string]bool{
+			cacerturl:     true,
+			eipserviceurl: true,
+			providerurl:   true,
+			geoserviceurl: true,
+			openvpnurl1:   false,
+			openvpnurl2:   true,
+			obfs4url1:     true,
+		}),
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
-	// we're cancelling immediately so that the CA Cert fetch fails
+	defer cancel()
-	cancel()
 
 	sess := &mockable.Session{MockableLogger: log.Log}
 	measurement := new(model.Measurement)
@@ -147,6 +331,26 @@ func TestFailureCaCertFetch(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus == true {
+		t.Fatal("unexpected CaCertStatus")
+	}
+	if tk.APIStatus != "blocked" {
+		t.Fatal("ApiStatus should be blocked")
+	}
+}
+
+func TestFailureCaCertFetch(t *testing.T) {
+	measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
+		cacerturl:     false,
+		eipserviceurl: true,
+		providerurl:   true,
+		geoserviceurl: true,
+		openvpnurl1:   true,
+		openvpnurl2:   true,
+		obfs4url1:     true,
+	}))
+
 	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
 	if tk.CACertStatus != false {
 		t.Fatal("invalid CACertStatus ")
@@ -164,21 +368,15 @@ func TestFailureCaCertFetch(t *testing.T) {
 }
 
 func TestFailureEipServiceBlocked(t *testing.T) {
-	if testing.Short() {
+	measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
-		t.Skip("skip test in short mode")
+		cacerturl:     true,
-	}
+		eipserviceurl: false,
-	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+		providerurl:   true,
-	ctx, cancel := context.WithCancel(context.Background())
+		geoserviceurl: true,
-	defer cancel()
+		openvpnurl1:   true,
-	selfcensor.Enable(`{"PoisonSystemDNS":{"api.black.riseup.net":["NXDOMAIN"]}}`)
+		openvpnurl2:   true,
-
+		obfs4url1:     true,
-	sess := &mockable.Session{MockableLogger: log.Log}
+	}))
-	measurement := new(model.Measurement)
-	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
-	if err != nil {
-		t.Fatal(err)
-	}
 	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
 	if tk.CACertStatus != true {
 		t.Fatal("invalid CACertStatus ")
@@ -202,21 +400,15 @@ func TestFailureEipServiceBlocked(t *testing.T) {
 }
 
 func TestFailureProviderUrlBlocked(t *testing.T) {
-	if testing.Short() {
+	measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
-		t.Skip("skip test in short mode")
+		cacerturl:     true,
-	}
+		eipserviceurl: true,
-	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+		providerurl:   false,
-	ctx, cancel := context.WithCancel(context.Background())
+		geoserviceurl: true,
-	defer cancel()
+		openvpnurl1:   true,
-	selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.70:443":"REJECT"}}`)
+		openvpnurl2:   true,
-
+		obfs4url1:     true,
-	sess := &mockable.Session{MockableLogger: log.Log}
+	}))
-	measurement := new(model.Measurement)
-	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
-	if err != nil {
-		t.Fatal(err)
-	}
 	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
 
 	for _, entry := range tk.Requests {
@@ -240,21 +432,15 @@ func TestFailureProviderUrlBlocked(t *testing.T) {
 }
 
 func TestFailureGeoIpServiceBlocked(t *testing.T) {
-	if testing.Short() {
+	measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
-		t.Skip("skip test in short mode")
+		cacerturl:     true,
-	}
+		eipserviceurl: true,
-	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+		providerurl:   true,
-	ctx, cancel := context.WithCancel(context.Background())
+		geoserviceurl: false,
-	defer cancel()
+		openvpnurl1:   true,
-	selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.107:9001":"REJECT"}}`)
+		openvpnurl2:   true,
-
+		obfs4url1:     true,
-	sess := &mockable.Session{MockableLogger: log.Log}
+	}))
-	measurement := new(model.Measurement)
-	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
-	if err != nil {
-		t.Fatal(err)
-	}
 	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
 	if tk.CACertStatus != true {
 		t.Fatal("invalid CACertStatus ")
@@ -277,138 +463,16 @@ func TestFailureGeoIpServiceBlocked(t *testing.T) {
 	}
 }
 
-func TestFailureGateway(t *testing.T) {
+func TestFailureGateway1(t *testing.T) {
-	if testing.Short() {
+	measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
-		t.Skip("skip test in short mode")
+		cacerturl:     true,
-	}
+		eipserviceurl: true,
-	var testCases = [...]string{"openvpn", "obfs4"}
+		providerurl:   true,
-	eipService, err := fetchEipService()
+		geoserviceurl: true,
-	if err != nil {
+		openvpnurl1:   false,
-		t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error())
+		openvpnurl2:   true,
-		t.SkipNow()
+		obfs4url1:     true,
-	}
+	}))
-	for _, tc := range testCases {
-		t.Run(fmt.Sprintf("testing censored transport %s", tc), func(t *testing.T) {
-			censoredGateway, err := selfCensorRandomGateway(eipService, tc)
-			if err == nil {
-				censorString := `{"BlockedEndpoints":{"` + censoredGateway.IP + `:` + censoredGateway.Port + `":"REJECT"}}`
-				selfcensor.Enable(censorString)
-			} else {
-				t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error())
-				t.SkipNow()
-			}
-
-			// - run measurement
-			runGatewayTest(t, censoredGateway)
-		})
-	}
-}
-
-type SelfCensoredGateway struct {
-	IP   string
-	Port string
-}
-
-func fetchEipService() (*riseupvpn.EipService, error) {
-	// - fetch client cert and add to certpool
-	caFetchClient := &http.Client{
-		Timeout: time.Second * 30,
-	}
-
-	caCertResponse, err := caFetchClient.Get("https://black.riseup.net/ca.crt")
-	if err != nil {
-		return nil, err
-	}
-
-	var bodyString string
-
-	if caCertResponse.StatusCode != http.StatusOK {
-		return nil, errors.New("unexpected HTTP response code")
-	}
-	bodyBytes, err := ioutil.ReadAll(caCertResponse.Body)
-	defer caCertResponse.Body.Close()
-
-	if err != nil {
-		return nil, err
-	}
-	bodyString = string(bodyBytes)
-
-	certs := x509.NewCertPool()
-	certs.AppendCertsFromPEM([]byte(bodyString))
-
-	// - fetch and parse eip-service.json
-	client := &http.Client{
-		Timeout: time.Second * 30,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				RootCAs: certs,
-			},
-		},
-	}
-
-	eipResponse, err := client.Get("https://api.black.riseup.net/3/config/eip-service.json")
-	if err != nil {
-		return nil, err
-	}
-	if eipResponse.StatusCode != http.StatusOK {
-		return nil, errors.New("Unexpected HTTP response code")
-	}
-
-	bodyBytes, err = ioutil.ReadAll(eipResponse.Body)
-	defer eipResponse.Body.Close()
-	if err != nil {
-		return nil, err
-	}
-	bodyString = string(bodyBytes)
-
-	eipService, err := riseupvpn.DecodeEIP3(bodyString)
-	if err != nil {
-		return nil, err
-	}
-	return eipService, nil
-}
-
-func selfCensorRandomGateway(eipService *riseupvpn.EipService, transportType string) (*SelfCensoredGateway, error) {
-
-	// - self censor random gateway
-	gateways := eipService.Gateways
-	if gateways == nil || len(gateways) == 0 {
-		return nil, errors.New("No gateways found")
-	}
-
-	var selfcensoredGateways []SelfCensoredGateway
-	for _, gateway := range gateways {
-		for _, transport := range gateway.Capabilities.Transport {
-			if transport.Type == transportType {
-				selfcensoredGateways = append(selfcensoredGateways, SelfCensoredGateway{IP: gateway.IPAddress, Port: transport.Ports[0]})
-			}
-		}
-	}
-
-	if len(selfcensoredGateways) == 0 {
-		return nil, errors.New("transport " + transportType + " doesn't seem to be supported.")
-	}
-
-	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
-	min := 0
-	max := len(selfcensoredGateways) - 1
-	randomIndex := rnd.Intn(max-min+1) + min
-	return &selfcensoredGateways[randomIndex], nil
-
-}
-
-func runGatewayTest(t *testing.T, censoredGateway *SelfCensoredGateway) {
-	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	sess := &mockable.Session{MockableLogger: log.Log}
-	measurement := new(model.Measurement)
-	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
-	if err != nil {
-		t.Fatal(err)
-	}
 	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
 	if tk.CACertStatus != true {
 		t.Fatal("invalid CACertStatus ")
@@ -418,18 +482,125 @@ func runGatewayTest(t *testing.T, censoredGateway *SelfCensoredGateway) {
 		t.Fatal("unexpected amount of failing gateways")
 	}
 
-	entry := tk.FailingGateways[0]
+	gw := tk.FailingGateways[0]
-	if entry.IP != censoredGateway.IP || fmt.Sprint(entry.Port) != censoredGateway.Port {
+	if gw.IP != "234.345.234.345" {
-		t.Fatal("unexpected failed gateway configuration")
+		t.Fatal("invalid failed gateway ip: " + fmt.Sprint(gw.IP))
+	}
+	if gw.Port != 443 {
+		t.Fatal("invalid failed gateway port: " + fmt.Sprint(gw.Port))
+	}
+	if gw.TransportType != "openvpn" {
+		t.Fatal("invalid failed transport type: " + fmt.Sprint(gw.TransportType))
 	}
 
 	if tk.APIStatus == "blocked" {
-		t.Fatal("invalid ApiStatus", tk.APIStatus)
+		t.Fatal("invalid ApiStatus")
 	}
 
 	if tk.APIFailure != nil {
 		t.Fatal("ApiFailure should be null")
 	}
+
+	if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] == "blocked" {
+		t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
+	}
+
+	if tk.TransportStatus == nil || tk.TransportStatus["obfs4"] == "blocked" {
+		t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
+	}
+}
+
+func TestFailureTransport(t *testing.T) {
+	measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
+		cacerturl:     true,
+		eipserviceurl: true,
+		providerurl:   true,
+		geoserviceurl: true,
+		openvpnurl1:   false,
+		openvpnurl2:   false,
+		obfs4url1:     false,
+	}))
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+
+	if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] != "blocked" {
+		t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
+	}
+
+	if tk.TransportStatus == nil || tk.TransportStatus["obfs4"] != "blocked" {
+		t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
+	}
+}
+
+func TestMissingTransport(t *testing.T) {
+	eipService, err := riseupvpn.DecodeEIP3(eipservice)
+	if err != nil {
+		t.Fatal("Preconditions for the test are not met.")
+	}
+
+	//remove obfs4 capability from 2. gateway so that our
+	//mock provider supports only openvpn
+	index := -1
+	transports := eipService.Gateways[1].Capabilities.Transport
+	for i, transport := range transports {
+		if transport.Type == "obfs4" {
+			index = i
+			break
+		}
+	}
+	if index == -1 {
+		t.Fatal("Preconditions for the test are not met. Default eipservice string should contain obfs4 transport.")
+	}
+
+	transports[index] = transports[len(transports)-1]
+	transports = transports[:len(transports)-1]
+	eipService.Gateways[1].Capabilities.Transport = transports
+	eipservicejson, err := json.Marshal(eipservice)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	requestResponseMap := map[string]string{
+		eipserviceurl: string(eipservicejson),
+		providerurl:   provider,
+		geoserviceurl: geoservice,
+		cacerturl:     cacert,
+		openvpnurl1:   "",
+		openvpnurl2:   "",
+		obfs4url1:     "",
+	}
+
+	measurer := riseupvpn.Measurer{
+		Config: riseupvpn.Config{},
+		Getter: generateMockGetter(requestResponseMap, map[string]bool{
+			cacerturl:     true,
+			eipserviceurl: true,
+			providerurl:   true,
+			geoserviceurl: true,
+			openvpnurl1:   true,
+			openvpnurl2:   true,
+			obfs4url1:     false,
+		}),
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err = measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] != "blocked" {
+		t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
+	}
+
+	if _, found := tk.TransportStatus["obfs"]; found {
+		t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
+	}
+
 }
 
 func TestSummaryKeysInvalidType(t *testing.T) {
@@ -450,21 +621,27 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) {
 			APIStatus:       "blocked",
 			CACertStatus:    true,
 			FailingGateways: nil,
+			TransportStatus: nil,
 		},
 		sk: riseupvpn.SummaryKeys{
-			APIBlocked:  true,
+			APIBlocked:      true,
-			ValidCACert: true,
+			ValidCACert:     true,
-			IsAnomaly:   true,
+			IsAnomaly:       true,
+			TransportStatus: nil,
+			FailingGateways: 0,
 		},
 	}, {
 		tk: riseupvpn.TestKeys{
 			APIStatus:       "ok",
 			CACertStatus:    false,
 			FailingGateways: nil,
+			TransportStatus: nil,
 		},
 		sk: riseupvpn.SummaryKeys{
-			ValidCACert: false,
+			ValidCACert:     false,
-			IsAnomaly:   true,
+			IsAnomaly:       true,
+			FailingGateways: 0,
+			TransportStatus: nil,
 		},
 	}, {
 		tk: riseupvpn.TestKeys{
@@ -475,13 +652,39 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) {
 				Port:          443,
 				TransportType: "obfs4",
 			}},
+			TransportStatus: map[string]string{
+				"obfs4":   "blocked",
+				"openvpn": "ok",
+			},
 		},
 		sk: riseupvpn.SummaryKeys{
 			FailingGateways: 1,
 			IsAnomaly:       true,
 			ValidCACert:     true,
+			TransportStatus: map[string]string{
+				"obfs4":   "blocked",
+				"openvpn": "ok",
+			},
 		},
-	}}
+	}, {
+		tk: riseupvpn.TestKeys{
+			APIStatus:       "ok",
+			CACertStatus:    true,
+			FailingGateways: nil,
+			TransportStatus: map[string]string{
+				"openvpn": "ok",
+			},
+		},
+		sk: riseupvpn.SummaryKeys{
+			ValidCACert:     true,
+			IsAnomaly:       false,
+			FailingGateways: 0,
+			TransportStatus: map[string]string{
+				"openvpn": "ok",
+			},
+		},
+	},
+	}
 	for idx, tt := range tests {
 		t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
 			m := &riseupvpn.Measurer{}
@@ -498,3 +701,95 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) {
 		})
 	}
 }
+
+func generateMockGetter(requestResponse map[string]string, responseStatus map[string]bool) urlgetter.MultiGetter {
+	return func(ctx context.Context, g urlgetter.Getter) (urlgetter.TestKeys, error) {
+		url := g.Target
+		responseBody, foundRequest := requestResponse[url]
+		isSuccessStatus, foundStatus := responseStatus[url]
+		if !foundRequest || !foundStatus {
+			return urlgetter.DefaultMultiGetter(ctx, g)
+		}
+
+		var failure *string
+		var failedOperation *string
+		isBlocked := !isSuccessStatus
+		var responseStatus int64 = 200
+
+		if isBlocked {
+			responseBody = ""
+			eofError := io.EOF.Error()
+			failure = &eofError
+			connectOperation := errorx.ConnectOperation
+			failedOperation = &connectOperation
+			responseStatus = 0
+		}
+
+		tcpConnect := archival.TCPConnectEntry{
+			// use some dummy IP/Port combination for URLs, we don't do DNS resolution
+			IP:   "123.456.234.123",
+			Port: 443,
+			Status: archival.TCPConnectStatus{
+				Success: isSuccessStatus,
+				Blocked: &isBlocked,
+				Failure: failure,
+			},
+		}
+		if strings.Contains(url, "tcpconnect://") {
+			ipPort := strings.Split(strings.Split(url, "//")[1], ":")
+			port, err := strconv.ParseInt(ipPort[1], 10, 32)
+			if err == nil {
+				tcpConnect.IP = ipPort[0]
+				tcpConnect.Port = int(port)
+			}
+		}
+
+		tk := urlgetter.TestKeys{
+			Failure:            failure,
+			FailedOperation:    failedOperation,
+			HTTPResponseStatus: responseStatus,
+			HTTPResponseBody:   responseBody,
+			Requests: []archival.RequestEntry{{
+				Failure: failure,
+				Request: archival.HTTPRequest{
+					URL:             url,
+					Body:            archival.MaybeBinaryValue{},
+					BodyIsTruncated: false,
+				},
+				Response: archival.HTTPResponse{
+					Body: archival.HTTPBody{
+						Value: responseBody,
+					},
+					BodyIsTruncated: false,
+				}},
+			},
+			TCPConnect: []archival.TCPConnectEntry{tcpConnect},
+		}
+		return tk, nil
+	}
+}
+func generateDefaultMockGetter(responseStatuses map[string]bool) urlgetter.MultiGetter {
+	return generateMockGetter(RequestResponse, responseStatuses)
+}
+
+func runDefaultMockTest(t *testing.T, multiGetter urlgetter.MultiGetter) *model.Measurement {
+	measurer := riseupvpn.Measurer{
+		Config: riseupvpn.Config{},
+		Getter: multiGetter,
+	}
+
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	return measurement
+}

internal/engine/experiment/stunreachability/stunreachability.go

@@ -107,7 +107,7 @@ func (tk *TestKeys) run(
 		tk.NetworkEvents, archival.NewNetworkEventsList(begin, events)...,
 	)
 	tk.Queries = append(
-		tk.Queries, archival.NewDNSQueriesList(begin, events, sess.ASNDatabasePath())...,
+		tk.Queries, archival.NewDNSQueriesList(begin, events)...,
 	)
 	return err
 }

internal/engine/experiment/tlstool/internal/splitter.go

@@ -29,7 +29,6 @@ func SNISplitter(input []byte, sni []byte) (output [][]byte) {
 	}
 	if len(buf) > 0 {
 		output = append(output, buf)
-		buf = nil
 	}
 	output = append(output, input[idx+len(sni):])
 	return

internal/engine/experiment/urlgetter/getter.go

@@ -58,10 +58,7 @@ func (g Getter) Get(ctx context.Context) (TestKeys, error) {
 	tk.FailedOperation = archival.NewFailedOperation(err)
 	tk.Failure = archival.NewFailure(err)
 	events := saver.Read()
-	tk.Queries = append(
+	tk.Queries = append(tk.Queries, archival.NewDNSQueriesList(g.Begin, events)...)
-		tk.Queries, archival.NewDNSQueriesList(
-			g.Begin, events, g.Session.ASNDatabasePath())...,
-	)
 	tk.NetworkEvents = append(
 		tk.NetworkEvents, archival.NewNetworkEventsList(g.Begin, events)...,
 	)

internal/engine/experiment/webconnectivity/connects.go

@@ -3,6 +3,7 @@ package webconnectivity
 import (
 	"context"
 	"net/url"
+	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
@@ -10,6 +11,7 @@ import (
 
 // ConnectsConfig contains the config for Connects
 type ConnectsConfig struct {
+	Begin         time.Time
 	Session       model.ExperimentSession
 	TargetURL     *url.URL
 	URLGetterURLs []string
@@ -28,7 +30,7 @@ type ConnectsResult struct {
 // check whether the resolved endpoints are reachable.
 func Connects(ctx context.Context, config ConnectsConfig) (out ConnectsResult) {
 	out.AllKeys = []urlgetter.TestKeys{}
-	multi := urlgetter.Multi{Session: config.Session}
+	multi := urlgetter.Multi{Begin: config.Begin, Session: config.Session}
 	inputs := []urlgetter.MultiInput{}
 	for _, url := range config.URLGetterURLs {
 		inputs = append(inputs, urlgetter.MultiInput{

internal/engine/experiment/webconnectivity/control.go

@@ -78,7 +78,7 @@ func (dns *ControlDNSResult) FillASNs(sess model.ExperimentSession) {
 	for _, ip := range dns.Addrs {
 		// TODO(bassosimone): this would be more efficient if we'd open just
 		// once the database and then reuse it for every address.
-		asn, _, _ := geolocate.LookupASN(sess.ASNDatabasePath(), ip)
+		asn, _, _ := geolocate.LookupASN(ip)
 		dns.ASNs = append(dns.ASNs, int64(asn))
 	}
 }

internal/engine/experiment/webconnectivity/control_test.go

@@ -16,15 +16,6 @@ func TestFillASNsEmpty(t *testing.T) {
 	}
 }
 
-func TestFillASNsNoDatabase(t *testing.T) {
-	dns := new(webconnectivity.ControlDNSResult)
-	dns.Addrs = []string{"8.8.8.8", "1.1.1.1"}
-	dns.FillASNs(new(mockable.Session))
-	if diff := cmp.Diff(dns.ASNs, []int64{0, 0}); diff != "" {
-		t.Fatal(diff)
-	}
-}
-
 func TestFillASNsSuccess(t *testing.T) {
 	sess := newsession(t, false)
 	dns := new(webconnectivity.ControlDNSResult)

internal/engine/experiment/webconnectivity/dnslookup.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/url"
 	"sort"
+	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
@@ -12,6 +13,7 @@ import (
 
 // DNSLookupConfig contains settings for the DNS lookup.
 type DNSLookupConfig struct {
+	Begin   time.Time
 	Session model.ExperimentSession
 	URL     *url.URL
 }
@@ -27,7 +29,8 @@ type DNSLookupResult struct {
 func DNSLookup(ctx context.Context, config DNSLookupConfig) (out DNSLookupResult) {
 	target := fmt.Sprintf("dnslookup://%s", config.URL.Hostname())
 	config.Session.Logger().Infof("%s...", target)
-	result, err := urlgetter.Getter{Session: config.Session, Target: target}.Get(ctx)
+	result, err := urlgetter.Getter{
+		Begin: config.Begin, Session: config.Session, Target: target}.Get(ctx)
 	out.Addrs = make(map[string]int64)
 	for _, query := range result.Queries {
 		for _, answer := range query.Answers {

internal/engine/experiment/webconnectivity/httpget.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
@@ -14,6 +15,7 @@ import (
 // HTTPGetConfig contains the config for HTTPGet
 type HTTPGetConfig struct {
 	Addresses []string
+	Begin     time.Time
 	Session   model.ExperimentSession
 	TargetURL *url.URL
 }
@@ -54,6 +56,7 @@ func HTTPGet(ctx context.Context, config HTTPGetConfig) (out HTTPGetResult) {
 	config.Session.Logger().Infof("GET %s...", target)
 	domain := config.TargetURL.Hostname()
 	result, err := urlgetter.Getter{
+		Begin: config.Begin,
 		Config: urlgetter.Config{
 			DNSCache: HTTPGetMakeDNSCache(domain, addresses),
 		},

internal/engine/experiment/webconnectivity/webconnectivity.go

@@ -19,7 +19,7 @@ import (
 
 const (
 	testName    = "web_connectivity"
-	testVersion = "0.3.0"
+	testVersion = "0.4.0"
 )
 
 // Config contains the experiment config.
@@ -32,6 +32,15 @@ type TestKeys struct {
 	Retries        *int64  `json:"retries"`    // unused
 	SOCKSProxy     *string `json:"socksproxy"` // unused
 
+	// For now mostly TCP/TLS "connect" experiment but we are
+	// considering adding more events. An open question is
+	// currently how to properly tag these events so that it
+	// is rather obvious where they come from.
+	//
+	// See https://github.com/ooni/probe/issues/1413.
+	NetworkEvents []archival.NetworkEvent `json:"network_events"`
+	TLSHandshakes []archival.TLSHandshake `json:"tls_handshakes"`
+
 	// DNS experiment
 	Queries              []archival.DNSQueryEntry `json:"queries"`
 	DNSExperimentFailure *string                  `json:"dns_experiment_failure"`
@@ -42,7 +51,7 @@ type TestKeys struct {
 	ControlRequest ControlRequest  `json:"-"`
 	Control        ControlResponse `json:"control"`
 
-	// TCP connect experiment
+	// TCP/TLS "connect" experiment
 	TCPConnect          []archival.TCPConnectEntry `json:"tcp_connect"`
 	TCPConnectSuccesses int                        `json:"-"`
 	TCPConnectAttempts  int                        `json:"-"`
@@ -90,6 +99,19 @@ var (
 	ErrUnsupportedInput = errors.New("unsupported input scheme")
 )
 
+// Tags describing the section of this experiment in which
+// the data has been collected.
+const (
+	// DNSExperimentTag is a tag indicating the DNS experiment.
+	DNSExperimentTag = "dns_experiment"
+
+	// TCPTLSExperimentTag is a tag indicating the connect experiment.
+	TCPTLSExperimentTag = "tcptls_experiment"
+
+	// HTTPExperimentTag is a tag indicating the HTTP experiment.
+	HTTPExperimentTag = "http_experiment"
+)
+
 // Run implements ExperimentMeasurer.Run.
 func (m Measurer) Run(
 	ctx context.Context,
@@ -129,7 +151,9 @@ func (m Measurer) Run(
 		"backend": testhelper,
 	}
 	// 2. perform the DNS lookup step
-	dnsResult := DNSLookup(ctx, DNSLookupConfig{Session: sess, URL: URL})
+	dnsResult := DNSLookup(ctx, DNSLookupConfig{
+		Begin:   measurement.MeasurementStartTimeSaved,
+		Session: sess, URL: URL})
 	tk.Queries = append(tk.Queries, dnsResult.TestKeys.Queries...)
 	tk.DNSExperimentFailure = dnsResult.Failure
 	epnts := NewEndpoints(URL, dnsResult.Addresses())
@@ -152,7 +176,13 @@ func (m Measurer) Run(
 	sess.Logger().Infof("DNS analysis result: %+v", internal.StringPointerToString(
 		tk.DNSAnalysisResult.DNSConsistency))
 	// 5. perform TCP/TLS connects
+	//
+	// TODO(bassosimone): here we should also follow the IP addresses
+	// returned by the control experiment.
+	//
+	// See https://github.com/ooni/probe/issues/1414
 	connectsResult := Connects(ctx, ConnectsConfig{
+		Begin:         measurement.MeasurementStartTimeSaved,
 		Session:       sess,
 		TargetURL:     URL,
 		URLGetterURLs: epnts.URLs(),
@@ -164,12 +194,21 @@ func (m Measurer) Run(
 		// sad that we're storing analysis result inside the measurement
 		tk.TCPConnect = append(tk.TCPConnect, ComputeTCPBlocking(
 			tcpkeys.TCPConnect, tk.Control.TCPConnect)...)
+		for _, ev := range tcpkeys.NetworkEvents {
+			ev.Tags = []string{TCPTLSExperimentTag}
+			tk.NetworkEvents = append(tk.NetworkEvents, ev)
+		}
+		for _, ev := range tcpkeys.TLSHandshakes {
+			ev.Tags = []string{TCPTLSExperimentTag}
+			tk.TLSHandshakes = append(tk.TLSHandshakes, ev)
+		}
 	}
 	tk.TCPConnectAttempts = connectsResult.Total
 	tk.TCPConnectSuccesses = connectsResult.Successes
 	// 6. perform HTTP/HTTPS measurement
 	httpResult := HTTPGet(ctx, HTTPGetConfig{
 		Addresses: dnsResult.Addresses(),
+		Begin:     measurement.MeasurementStartTimeSaved,
 		Session:   sess,
 		TargetURL: URL,
 	})

internal/engine/experiment/webconnectivity/webconnectivity_test.go

@@ -21,7 +21,7 @@ func TestNewExperimentMeasurer(t *testing.T) {
 	if measurer.ExperimentName() != "web_connectivity" {
 		t.Fatal("unexpected name")
 	}
-	if measurer.ExperimentVersion() != "0.3.0" {
+	if measurer.ExperimentVersion() != "0.4.0" {
 		t.Fatal("unexpected version")
 	}
 }
@@ -202,7 +202,6 @@ func TestMeasureWithNoAvailableTestHelpers(t *testing.T) {
 
 func newsession(t *testing.T, lookupBackends bool) model.ExperimentSession {
 	sess, err := engine.NewSession(engine.SessionConfig{
-		AssetsDir: "../../testdata",
 		AvailableProbeServices: []model.Service{{
 			Address: "https://ams-pg-test.ooni.org",
 			Type:    "https",

internal/engine/experiment_integration_test.go

@@ -142,7 +142,7 @@ func TestNeedsInput(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if builder.InputPolicy() != InputOrQueryTestLists {
+	if builder.InputPolicy() != InputOrQueryBackend {
 		t.Fatal("web_connectivity certainly needs input")
 	}
 }

internal/engine/experimentbuilder.go

@@ -16,12 +16,12 @@ import (
 type InputPolicy string
 
 const (
-	// InputOrQueryTestLists indicates that the experiment requires
+	// InputOrQueryBackend indicates that the experiment requires
 	// external input to run and that this kind of input is URLs
 	// from the citizenlab/test-lists repository. If this input
 	// not provided to the experiment, then the code that runs the
 	// experiment is supposed to fetch from URLs from OONI's backends.
-	InputOrQueryTestLists = InputPolicy("or_query_test_lists")
+	InputOrQueryBackend = InputPolicy("or_query_backend")
 
 	// InputStrictlyRequired indicates that the experiment
 	// requires input and we currently don't have an API for
@@ -193,7 +193,7 @@ func canonicalizeExperimentName(name string) string {
 }
 
 func newExperimentBuilder(session *Session, name string) (*ExperimentBuilder, error) {
-	factory, _ := experimentsByName[canonicalizeExperimentName(name)]
+	factory := experimentsByName[canonicalizeExperimentName(name)]
 	if factory == nil {
 		return nil, fmt.Errorf("no such experiment: %s", name)
 	}

internal/engine/geolocate/geolocate.go

@@ -3,7 +3,6 @@ package geolocate
 
 import (
 	"context"
-	"errors"
 	"fmt"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
@@ -43,12 +42,6 @@ var (
 	DefaultResolverASNString = fmt.Sprintf("AS%d", DefaultResolverASN)
 )
 
-var (
-	// ErrMissingResourcesManager indicates that no resources
-	// manager has been configured inside of Config.
-	ErrMissingResourcesManager = errors.New("geolocate: ResourcesManager is nil")
-)
-
 // Logger is the definition of Logger used by this package.
 type Logger interface {
 	Debug(msg string)
@@ -96,30 +89,17 @@ type probeIPLookupper interface {
 }
 
 type asnLookupper interface {
-	LookupASN(path string, ip string) (asn uint, network string, err error)
+	LookupASN(ip string) (asn uint, network string, err error)
 }
 
 type countryLookupper interface {
-	LookupCC(path string, ip string) (cc string, err error)
+	LookupCC(ip string) (cc string, err error)
 }
 
 type resolverIPLookupper interface {
 	LookupResolverIP(ctx context.Context) (addr string, err error)
 }
 
-// ResourcesManager manages the required resources.
-type ResourcesManager interface {
-	// ASNDatabasePath returns the path of the ASN database.
-	ASNDatabasePath() string
-
-	// CountryDatabasePath returns the path of the country database.
-	CountryDatabasePath() string
-
-	// MaybeUpdateResources ensures that the required resources
-	// have been downloaded and are current.
-	MaybeUpdateResources(ctx context.Context) error
-}
-
 // Resolver is a DNS resolver.
 type Resolver interface {
 	LookupHost(ctx context.Context, domain string) ([]string, error)
@@ -142,10 +122,6 @@ type Config struct {
 	// use a logger that discards all messages.
 	Logger Logger
 
-	// ResourcesManager is the mandatory resources manager. If not
-	// set, we will not be able to perform any lookup.
-	ResourcesManager ResourcesManager
-
 	// UserAgent is the user agent to use. If not set, then
 	// we will use a default user agent.
 	UserAgent string
@@ -162,9 +138,6 @@ func NewTask(config Config) (*Task, error) {
 	if config.Logger == nil {
 		config.Logger = model.DiscardLogger
 	}
-	if config.ResourcesManager == nil {
-		return nil, ErrMissingResourcesManager
-	}
 	if config.UserAgent == "" {
 		config.UserAgent = fmt.Sprintf("ooniprobe-engine/%s", version.Version)
 	}
@@ -183,7 +156,6 @@ func NewTask(config Config) (*Task, error) {
 		probeASNLookupper:    mmdbLookupper{},
 		resolverASNLookupper: mmdbLookupper{},
 		resolverIPLookupper:  resolverLookupClient{},
-		resourcesManager:     config.ResourcesManager,
 	}, nil
 }
 
@@ -196,7 +168,6 @@ type Task struct {
 	probeASNLookupper    asnLookupper
 	resolverASNLookupper asnLookupper
 	resolverIPLookupper  resolverIPLookupper
-	resourcesManager     ResourcesManager
 }
 
 // Run runs the task.
@@ -211,23 +182,18 @@ func (op Task) Run(ctx context.Context) (*Results, error) {
 		ResolverIP:          DefaultResolverIP,
 		ResolverNetworkName: DefaultResolverNetworkName,
 	}
-	if err := op.resourcesManager.MaybeUpdateResources(ctx); err != nil {
-		return out, fmt.Errorf("MaybeUpdateResource failed: %w", err)
-	}
 	ip, err := op.probeIPLookupper.LookupProbeIP(ctx)
 	if err != nil {
 		return out, fmt.Errorf("lookupProbeIP failed: %w", err)
 	}
 	out.ProbeIP = ip
-	asn, networkName, err := op.probeASNLookupper.LookupASN(
+	asn, networkName, err := op.probeASNLookupper.LookupASN(out.ProbeIP)
-		op.resourcesManager.ASNDatabasePath(), out.ProbeIP)
 	if err != nil {
 		return out, fmt.Errorf("lookupASN failed: %w", err)
 	}
 	out.ASN = asn
 	out.NetworkName = networkName
-	cc, err := op.countryLookupper.LookupCC(
+	cc, err := op.countryLookupper.LookupCC(out.ProbeIP)
-		op.resourcesManager.CountryDatabasePath(), out.ProbeIP)
 	if err != nil {
 		return out, fmt.Errorf("lookupProbeCC failed: %w", err)
 	}
@@ -244,7 +210,7 @@ func (op Task) Run(ctx context.Context) (*Results, error) {
 		}
 		out.ResolverIP = resolverIP
 		resolverASN, resolverNetworkName, err := op.resolverASNLookupper.LookupASN(
-			op.resourcesManager.ASNDatabasePath(), out.ResolverIP,
+			out.ResolverIP,
 		)
 		if err != nil {
 			return out, nil

internal/engine/geolocate/geolocate_test.go

@@ -6,57 +6,6 @@ import (
 	"testing"
 )
 
-type taskResourcesManager struct {
-	asnDatabasePath     string
-	countryDatabasePath string
-	err                 error
-}
-
-func (c taskResourcesManager) ASNDatabasePath() string {
-	return c.asnDatabasePath
-}
-
-func (c taskResourcesManager) CountryDatabasePath() string {
-	return c.countryDatabasePath
-}
-
-func (c taskResourcesManager) MaybeUpdateResources(ctx context.Context) error {
-	return c.err
-}
-
-func TestLocationLookupCannotUpdateResources(t *testing.T) {
-	expected := errors.New("mocked error")
-	op := Task{
-		resourcesManager: taskResourcesManager{err: expected},
-	}
-	ctx := context.Background()
-	out, err := op.Run(ctx)
-	if !errors.Is(err, expected) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out.ASN != DefaultProbeASN {
-		t.Fatal("invalid ASN value")
-	}
-	if out.CountryCode != DefaultProbeCC {
-		t.Fatal("invalid CountryCode value")
-	}
-	if out.NetworkName != DefaultProbeNetworkName {
-		t.Fatal("invalid NetworkName value")
-	}
-	if out.ProbeIP != DefaultProbeIP {
-		t.Fatal("invalid ProbeIP value")
-	}
-	if out.ResolverASN != DefaultResolverASN {
-		t.Fatal("invalid ResolverASN value")
-	}
-	if out.ResolverIP != DefaultResolverIP {
-		t.Fatal("invalid ResolverIP value")
-	}
-	if out.ResolverNetworkName != DefaultResolverNetworkName {
-		t.Fatal("invalid ResolverNetworkName value")
-	}
-}
-
 type taskProbeIPLookupper struct {
 	ip  string
 	err error
@@ -69,7 +18,6 @@ func (c taskProbeIPLookupper) LookupProbeIP(ctx context.Context) (string, error)
 func TestLocationLookupCannotLookupProbeIP(t *testing.T) {
 	expected := errors.New("mocked error")
 	op := Task{
-		resourcesManager: taskResourcesManager{},
 		probeIPLookupper: taskProbeIPLookupper{err: expected},
 	}
 	ctx := context.Background()
@@ -106,14 +54,13 @@ type taskASNLookupper struct {
 	name string
 }
 
-func (c taskASNLookupper) LookupASN(path string, ip string) (uint, string, error) {
+func (c taskASNLookupper) LookupASN(ip string) (uint, string, error) {
 	return c.asn, c.name, c.err
 }
 
 func TestLocationLookupCannotLookupProbeASN(t *testing.T) {
 	expected := errors.New("mocked error")
 	op := Task{
-		resourcesManager:  taskResourcesManager{},
 		probeIPLookupper:  taskProbeIPLookupper{ip: "1.2.3.4"},
 		probeASNLookupper: taskASNLookupper{err: expected},
 	}
@@ -150,14 +97,13 @@ type taskCCLookupper struct {
 	cc  string
 }
 
-func (c taskCCLookupper) LookupCC(path string, ip string) (string, error) {
+func (c taskCCLookupper) LookupCC(ip string) (string, error) {
 	return c.cc, c.err
 }
 
 func TestLocationLookupCannotLookupProbeCC(t *testing.T) {
 	expected := errors.New("mocked error")
 	op := Task{
-		resourcesManager:  taskResourcesManager{},
 		probeIPLookupper:  taskProbeIPLookupper{ip: "1.2.3.4"},
 		probeASNLookupper: taskASNLookupper{asn: 1234, name: "1234.com"},
 		countryLookupper:  taskCCLookupper{cc: "US", err: expected},
@@ -202,7 +148,6 @@ func (c taskResolverIPLookupper) LookupResolverIP(ctx context.Context) (string,
 func TestLocationLookupCannotLookupResolverIP(t *testing.T) {
 	expected := errors.New("mocked error")
 	op := Task{
-		resourcesManager:     taskResourcesManager{},
 		probeIPLookupper:     taskProbeIPLookupper{ip: "1.2.3.4"},
 		probeASNLookupper:    taskASNLookupper{asn: 1234, name: "1234.com"},
 		countryLookupper:     taskCCLookupper{cc: "IT"},
@@ -243,7 +188,6 @@ func TestLocationLookupCannotLookupResolverIP(t *testing.T) {
 func TestLocationLookupCannotLookupResolverNetworkName(t *testing.T) {
 	expected := errors.New("mocked error")
 	op := Task{
-		resourcesManager:     taskResourcesManager{},
 		probeIPLookupper:     taskProbeIPLookupper{ip: "1.2.3.4"},
 		probeASNLookupper:    taskASNLookupper{asn: 1234, name: "1234.com"},
 		countryLookupper:     taskCCLookupper{cc: "IT"},
@@ -284,7 +228,6 @@ func TestLocationLookupCannotLookupResolverNetworkName(t *testing.T) {
 
 func TestLocationLookupSuccessWithResolverLookup(t *testing.T) {
 	op := Task{
-		resourcesManager:     taskResourcesManager{},
 		probeIPLookupper:     taskProbeIPLookupper{ip: "1.2.3.4"},
 		probeASNLookupper:    taskASNLookupper{asn: 1234, name: "1234.com"},
 		countryLookupper:     taskCCLookupper{cc: "IT"},
@@ -325,7 +268,6 @@ func TestLocationLookupSuccessWithResolverLookup(t *testing.T) {
 
 func TestLocationLookupSuccessWithoutResolverLookup(t *testing.T) {
 	op := Task{
-		resourcesManager:     taskResourcesManager{},
 		probeIPLookupper:     taskProbeIPLookupper{ip: "1.2.3.4"},
 		probeASNLookupper:    taskASNLookupper{asn: 1234, name: "1234.com"},
 		countryLookupper:     taskCCLookupper{cc: "IT"},
@@ -364,13 +306,8 @@ func TestLocationLookupSuccessWithoutResolverLookup(t *testing.T) {
 }
 
 func TestSmoke(t *testing.T) {
-	maybeFetchResources(t)
 	config := Config{
 		EnableResolverLookup: true,
-		ResourcesManager: taskResourcesManager{
-			asnDatabasePath:     asnDBPath,
-			countryDatabasePath: countryDBPath,
-		},
 	}
 	task := Must(NewTask(config))
 	result, err := task.Run(context.Background())
@@ -384,16 +321,6 @@ func TestSmoke(t *testing.T) {
 	// value is okay for all codepaths.
 }
 
-func TestNewTaskWithNoResourcesManager(t *testing.T) {
-	task, err := NewTask(Config{})
-	if !errors.Is(err, ErrMissingResourcesManager) {
-		t.Fatal("not the error we expected")
-	}
-	if task != nil {
-		t.Fatal("expected nil task here")
-	}
-}
-
 func TestASNStringWorks(t *testing.T) {
 	r := Results{ASN: 1234}
 	if r.ASNString() != "AS1234" {

internal/engine/geolocate/ipconfig_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net"
 	"net/http"
+	"os"
 	"testing"
 
 	"github.com/apex/log"
@@ -11,6 +12,11 @@ import (
 )
 
 func TestIPLookupWorksUsingIPConfig(t *testing.T) {
+	if os.Getenv("CI") == "true" {
+		// See https://github.com/ooni/probe-cli/pull/259/checks?check_run_id=2166066881#step:5:123
+		// as well as https://github.com/ooni/probe/issues/1418.
+		t.Skip("This test does not work with GitHub Actions")
+	}
 	ip, err := ipConfigIPLookup(
 		context.Background(),
 		http.DefaultClient,

internal/engine/geolocate/iplookup.go

@@ -7,7 +7,6 @@ import (
 	"math/rand"
 	"net"
 	"net/http"
-	"sync"
 	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/multierror"
@@ -61,8 +60,6 @@ var (
 			fn:   ubuntuIPLookup,
 		},
 	}
-
-	once sync.Once
 )
 
 type ipLookupClient struct {

internal/engine/geolocate/mmdblookup.go

@@ -3,14 +3,15 @@ package geolocate
 import (
 	"net"
 
+	"github.com/ooni/probe-assets/assets"
 	"github.com/oschwald/geoip2-golang"
 )
 
 type mmdbLookupper struct{}
 
-func (mmdbLookupper) LookupASN(path, ip string) (asn uint, org string, err error) {
+func (mmdbLookupper) LookupASN(ip string) (asn uint, org string, err error) {
 	asn, org = DefaultProbeASN, DefaultProbeNetworkName
-	db, err := geoip2.Open(path)
+	db, err := geoip2.FromBytes(assets.ASNDatabaseData())
 	if err != nil {
 		return
 	}
@@ -28,13 +29,13 @@ func (mmdbLookupper) LookupASN(path, ip string) (asn uint, org string, err error
 
 // LookupASN returns the ASN and the organization associated with the
 // given ip using the ASN database at path.
-func LookupASN(path, ip string) (asn uint, org string, err error) {
+func LookupASN(ip string) (asn uint, org string, err error) {
-	return (mmdbLookupper{}).LookupASN(path, ip)
+	return (mmdbLookupper{}).LookupASN(ip)
 }
 
-func (mmdbLookupper) LookupCC(path, ip string) (cc string, err error) {
+func (mmdbLookupper) LookupCC(ip string) (cc string, err error) {
 	cc = DefaultProbeCC
-	db, err := geoip2.Open(path)
+	db, err := geoip2.FromBytes(assets.CountryDatabaseData())
 	if err != nil {
 		return
 	}

internal/engine/geolocate/mmdblookup_test.go

@@ -1,27 +1,11 @@
 package geolocate
 
-import (
+import "testing"
-	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
+const ipAddr = "35.204.49.125"
-)
-
-const (
-	asnDBPath     = "../testdata/asn.mmdb"
-	countryDBPath = "../testdata/country.mmdb"
-	ipAddr        = "35.204.49.125"
-)
-
-func maybeFetchResources(t *testing.T) {
-	c := &resourcesmanager.CopyWorker{DestDir: "../testdata/"}
-	if err := c.Ensure(); err != nil {
-		t.Fatal(err)
-	}
-}
 
 func TestLookupASN(t *testing.T) {
-	maybeFetchResources(t)
+	asn, org, err := LookupASN(ipAddr)
-	asn, org, err := LookupASN(asnDBPath, ipAddr)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -33,23 +17,8 @@ func TestLookupASN(t *testing.T) {
 	}
 }
 
-func TestLookupASNInvalidFile(t *testing.T) {
-	maybeFetchResources(t)
-	asn, org, err := LookupASN("/nonexistent", ipAddr)
-	if err == nil {
-		t.Fatal("expected an error here")
-	}
-	if asn != DefaultProbeASN {
-		t.Fatal("expected a zero ASN")
-	}
-	if org != DefaultProbeNetworkName {
-		t.Fatal("expected an empty org")
-	}
-}
-
 func TestLookupASNInvalidIP(t *testing.T) {
-	maybeFetchResources(t)
+	asn, org, err := LookupASN("xxx")
-	asn, org, err := LookupASN(asnDBPath, "xxx")
 	if err == nil {
 		t.Fatal("expected an error here")
 	}
@@ -62,8 +31,7 @@ func TestLookupASNInvalidIP(t *testing.T) {
 }
 
 func TestLookupCC(t *testing.T) {
-	maybeFetchResources(t)
+	cc, err := (mmdbLookupper{}).LookupCC(ipAddr)
-	cc, err := (mmdbLookupper{}).LookupCC(countryDBPath, ipAddr)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -72,20 +40,8 @@ func TestLookupCC(t *testing.T) {
 	}
 }
 
-func TestLookupCCInvalidFile(t *testing.T) {
-	maybeFetchResources(t)
-	cc, err := (mmdbLookupper{}).LookupCC("/nonexistent", ipAddr)
-	if err == nil {
-		t.Fatal("expected an error here")
-	}
-	if cc != DefaultProbeCC {
-		t.Fatal("expected an empty cc")
-	}
-}
-
 func TestLookupCCInvalidIP(t *testing.T) {
-	maybeFetchResources(t)
+	cc, err := (mmdbLookupper{}).LookupCC("xxx")
-	cc, err := (mmdbLookupper{}).LookupCC(asnDBPath, "xxx")
 	if err == nil {
 		t.Fatal("expected an error here")
 	}

internal/engine/httpheader/useragent.go

@@ -3,8 +3,8 @@ package httpheader
 
 // UserAgent returns the User-Agent header used for measuring.
 func UserAgent() string {
-	// 8.0% as of Mar 3, 2021 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
+	// 11.4% as of Mar 31, 2021 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
-	const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"
+	const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36"
 	return ua
 }
 

internal/engine/inputloader.go

@@ -5,28 +5,33 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/fsx"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 
-// The following errors are returned by the InputLoader.
+// These errors are returned by the InputLoader.
 var (
-	ErrNoInputExpected   = errors.New("we did not expect any input")
+	ErrNoURLsReturned    = errors.New("no URLs returned")
-	ErrInputRequired     = errors.New("no input provided")
 	ErrDetectedEmptyFile = errors.New("file did not contain any input")
+	ErrInputRequired     = errors.New("no input provided")
+	ErrNoInputExpected   = errors.New("we did not expect any input")
 )
 
-// InputLoaderSession is the session according to an InputLoader.
+// InputLoaderSession is the session according to an InputLoader. We
+// introduce this abstraction because it helps us with testing.
 type InputLoaderSession interface {
-	MaybeLookupLocationContext(ctx context.Context) error
+	CheckIn(ctx context.Context,
-	NewOrchestraClient(ctx context.Context) (model.ExperimentOrchestraClient, error)
+		config *model.CheckInConfig) (*model.CheckInInfo, error)
-	ProbeCC() string
 }
 
 // InputLoader loads input according to the specified policy
-// from the specified sources and OONI services. The behaviour
+// either from command line and input files or from OONI services. The
-// depends on the input policy as described below.
+// behaviour depends on the input policy as described below.
+//
+// You MUST NOT change any public field of this structure when
+// in use, because that MAY lead to data races.
 //
 // InputNone
 //
@@ -40,74 +45,52 @@ type InputLoaderSession interface {
 // input, we return it. Otherwise we return a single, empty entry
 // that causes experiments that don't require input to run once.
 //
-// InputOrQueryTestLists
+// InputOrQueryBackend
 //
 // We gather input from StaticInput and SourceFiles. If there is
 // input, we return it. Otherwise, we use OONI's probe services
-// to gather input using the test lists API.
+// to gather input using the best API for the task.
 //
 // InputStrictlyRequired
 //
-// Like InputOrQueryTestLists but, if there is no input, it's an
+// We gather input from StaticInput and SourceFiles. If there is
-// user error and we just abort running the experiment.
+// input, we return it. Otherwise, we return an error.
-type InputLoader interface {
+type InputLoader struct {
-	// Load attempts to load input using the specified input loader. We will
+	// CheckInConfig contains options for the CheckIn API. If
-	// return a list of URLs because this is the only input we support.
+	// not set, then we'll create a default config. If set but
-	Load(ctx context.Context) ([]model.URLInfo, error)
+	// there are fields inside it that are not set, then we
-}
+	// will set them to a default value.
+	CheckInConfig *model.CheckInConfig
+
+	// InputPolicy specifies the input policy for the
+	// current experiment. We will not load any input if
+	// the policy says we should not. You MUST fill in
+	// this field.
+	InputPolicy InputPolicy
+
+	// Session is the current measurement session. You
+	// MUST fill in this field.
+	Session InputLoaderSession
 
-// InputLoaderConfig contains config for InputLoader.
-type InputLoaderConfig struct {
 	// StaticInputs contains optional input to be added
 	// to the resulting input list if possible.
 	StaticInputs []string
 
 	// SourceFiles contains optional files to read input
 	// from. Each file should contain a single input string
-	// per line. We will fail if any file is unreadable.
+	// per line. We will fail if any file is unreadable
+	// as well as if any file is empty.
 	SourceFiles []string
-
-	// InputPolicy specifies the input policy for the
-	// current experiment. We will not load any input if
-	// the policy says we should not.
-	InputPolicy InputPolicy
-
-	// Session is the current measurement session.
-	Session InputLoaderSession
-
-	// URLLimit is the optional limit on the number of URLs
-	// that probe services should return to us.
-	URLLimit int64
-
-	// URLCategories limits the categories of URLs that
-	// probe services should return to us.
-	URLCategories []string
 }
 
-// NewInputLoader creates a new InputLoader.
-func NewInputLoader(config InputLoaderConfig) InputLoader {
-	// TODO(bassosimone): the current implementation stems from a
-	// simple refactoring from a previous implementation where
-	// we weren't using interfaces. Because now we're using interfaces,
-	// there is the opportunity to select behaviour here depending
-	// on the specified policy rather than later inside Load.
-	return inputLoader{InputLoaderConfig: config}
-}
-
-type inputLoader struct {
-	InputLoaderConfig
-}
-
-var _ InputLoader = inputLoader{}
-
 // Load attempts to load input using the specified input loader. We will
 // return a list of URLs because this is the only input we support.
-func (il inputLoader) Load(ctx context.Context) ([]model.URLInfo, error) {
+func (il *InputLoader) Load(ctx context.Context) ([]model.URLInfo, error) {
 	switch il.InputPolicy {
 	case InputOptional:
 		return il.loadOptional()
-	case InputOrQueryTestLists:
+	case InputOrQueryBackend:
-		return il.loadOrQueryTestList(ctx)
+		return il.loadOrQueryBackend(ctx)
 	case InputStrictlyRequired:
 		return il.loadStrictlyRequired(ctx)
 	default:
@@ -115,22 +98,27 @@ func (il inputLoader) Load(ctx context.Context) ([]model.URLInfo, error) {
 	}
 }
 
-func (il inputLoader) loadNone() ([]model.URLInfo, error) {
+// loadNone implements the InputNone policy.
+func (il *InputLoader) loadNone() ([]model.URLInfo, error) {
 	if len(il.StaticInputs) > 0 || len(il.SourceFiles) > 0 {
 		return nil, ErrNoInputExpected
 	}
+	// Note that we need to return a single empty entry.
 	return []model.URLInfo{{}}, nil
 }
 
-func (il inputLoader) loadOptional() ([]model.URLInfo, error) {
+// loadOptional implements the InputOptional policy.
+func (il *InputLoader) loadOptional() ([]model.URLInfo, error) {
 	inputs, err := il.loadLocal()
 	if err == nil && len(inputs) <= 0 {
+		// Note that we need to return a single empty entry.
 		inputs = []model.URLInfo{{}}
 	}
 	return inputs, err
 }
 
-func (il inputLoader) loadStrictlyRequired(ctx context.Context) ([]model.URLInfo, error) {
+// loadStrictlyRequired implements the InputStrictlyRequired policy.
+func (il *InputLoader) loadStrictlyRequired(ctx context.Context) ([]model.URLInfo, error) {
 	inputs, err := il.loadLocal()
 	if err != nil || len(inputs) > 0 {
 		return inputs, err
@@ -138,15 +126,17 @@ func (il inputLoader) loadStrictlyRequired(ctx context.Context) ([]model.URLInfo
 	return nil, ErrInputRequired
 }
 
-func (il inputLoader) loadOrQueryTestList(ctx context.Context) ([]model.URLInfo, error) {
+// loadOrQueryBackend implements the InputOrQueryBackend policy.
+func (il *InputLoader) loadOrQueryBackend(ctx context.Context) ([]model.URLInfo, error) {
 	inputs, err := il.loadLocal()
 	if err != nil || len(inputs) > 0 {
 		return inputs, err
 	}
-	return il.loadRemote(loadRemoteConfig{ctx: ctx, session: il.Session})
+	return il.loadRemote(ctx)
 }
 
-func (il inputLoader) loadLocal() ([]model.URLInfo, error) {
+// loadLocal loads inputs from StaticInputs and SourceFiles.
+func (il *InputLoader) loadLocal() ([]model.URLInfo, error) {
 	inputs := []model.URLInfo{}
 	for _, input := range il.StaticInputs {
 		inputs = append(inputs, model.URLInfo{URL: input})
@@ -165,7 +155,12 @@ func (il inputLoader) loadLocal() ([]model.URLInfo, error) {
 	return inputs, nil
 }
 
-func (il inputLoader) readfile(filepath string, open func(string) (fsx.File, error)) ([]model.URLInfo, error) {
+// inputLoaderOpenFn is the type of the function to open a file.
+type inputLoaderOpenFn func(filepath string) (fs.File, error)
+
+// readfile reads inputs from the specified file. The open argument should be
+// compatible with stdlib's fs.Open and helps us with unit testing.
+func (il *InputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]model.URLInfo, error) {
 	inputs := []model.URLInfo{}
 	filep, err := open(filepath)
 	if err != nil {
@@ -188,22 +183,22 @@ func (il inputLoader) readfile(filepath string, open func(string) (fsx.File, err
 	return inputs, nil
 }
 
-type loadRemoteConfig struct {
+// loadRemote loads inputs from a remote source.
-	ctx     context.Context
+func (il *InputLoader) loadRemote(ctx context.Context) ([]model.URLInfo, error) {
-	session InputLoaderSession
+	config := il.CheckInConfig
-}
+	if config == nil {
-
+		// Note: Session.CheckIn documentation says it will fill in
-func (il inputLoader) loadRemote(conf loadRemoteConfig) ([]model.URLInfo, error) {
+		// any field with a required value with a reasonable default
-	if err := conf.session.MaybeLookupLocationContext(conf.ctx); err != nil {
+		// if such value is missing. So, here we just need to be
-		return nil, err
+		// concerned about NOT passing it a NULL pointer.
+		config = &model.CheckInConfig{}
 	}
-	client, err := conf.session.NewOrchestraClient(conf.ctx)
+	reply, err := il.Session.CheckIn(ctx, config)
 	if err != nil {
 		return nil, err
 	}
-	return client.FetchURLList(conf.ctx, model.URLListConfig{
+	if reply.WebConnectivity == nil || len(reply.WebConnectivity.URLs) <= 0 {
-		CountryCode: conf.session.ProbeCC(),
+		return nil, ErrNoURLsReturned
-		Limit:       il.URLLimit,
+	}
-		Categories:  il.URLCategories,
+	return reply.WebConnectivity.URLs, nil
-	})
 }

internal/engine/inputloader_integration_test.go

@@ -1,316 +0,0 @@
-package engine_test
-
-import (
-	"context"
-	"errors"
-	"syscall"
-	"testing"
-
-	"github.com/apex/log"
-	"github.com/google/go-cmp/cmp"
-	engine "github.com/ooni/probe-cli/v3/internal/engine"
-	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
-	"github.com/ooni/probe-cli/v3/internal/engine/model"
-)
-
-func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		StaticInputs: []string{"https://www.google.com/"},
-		InputPolicy:  engine.InputNone,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if !errors.Is(err, engine.ErrNoInputExpected) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		SourceFiles: []string{
-			"testdata/inputloader1.txt",
-			"testdata/inputloader2.txt",
-		},
-		InputPolicy: engine.InputNone,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if !errors.Is(err, engine.ErrNoInputExpected) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputNoneWithBothInputs(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		StaticInputs: []string{"https://www.google.com/"},
-		SourceFiles: []string{
-			"testdata/inputloader1.txt",
-			"testdata/inputloader2.txt",
-		},
-		InputPolicy: engine.InputNone,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if !errors.Is(err, engine.ErrNoInputExpected) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputNoneWithNoInput(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		InputPolicy: engine.InputNone,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(out) != 1 || out[0].URL != "" {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputOptionalWithNoInput(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		InputPolicy: engine.InputOptional,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(out) != 1 || out[0].URL != "" {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputOptionalWithInput(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		StaticInputs: []string{"https://www.google.com/"},
-		SourceFiles: []string{
-			"testdata/inputloader1.txt",
-			"testdata/inputloader2.txt",
-		},
-		InputPolicy: engine.InputOptional,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(out) != 5 {
-		t.Fatal("not the output length we expected")
-	}
-	expect := []model.URLInfo{
-		{URL: "https://www.google.com/"},
-		{URL: "https://www.x.org/"},
-		{URL: "https://www.slashdot.org/"},
-		{URL: "https://abc.xyz/"},
-		{URL: "https://run.ooni.io/"},
-	}
-	if diff := cmp.Diff(out, expect); diff != "" {
-		t.Fatal(diff)
-	}
-}
-
-func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		StaticInputs: []string{"https://www.google.com/"},
-		SourceFiles: []string{
-			"testdata/inputloader1.txt",
-			"/nonexistent",
-			"testdata/inputloader2.txt",
-		},
-		InputPolicy: engine.InputOptional,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if !errors.Is(err, syscall.ENOENT) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		StaticInputs: []string{"https://www.google.com/"},
-		SourceFiles: []string{
-			"testdata/inputloader1.txt",
-			"testdata/inputloader2.txt",
-		},
-		InputPolicy: engine.InputStrictlyRequired,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(out) != 5 {
-		t.Fatal("not the output length we expected")
-	}
-	expect := []model.URLInfo{
-		{URL: "https://www.google.com/"},
-		{URL: "https://www.x.org/"},
-		{URL: "https://www.slashdot.org/"},
-		{URL: "https://abc.xyz/"},
-		{URL: "https://run.ooni.io/"},
-	}
-	if diff := cmp.Diff(out, expect); diff != "" {
-		t.Fatal(diff)
-	}
-}
-
-func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		InputPolicy: engine.InputStrictlyRequired,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if !errors.Is(err, engine.ErrInputRequired) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		InputPolicy: engine.InputStrictlyRequired,
-		SourceFiles: []string{
-			"testdata/inputloader1.txt",
-			"testdata/inputloader3.txt", // we want it before inputloader2.txt
-			"testdata/inputloader2.txt",
-		},
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if !errors.Is(err, engine.ErrDetectedEmptyFile) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputOrQueryTestListsWithInput(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		StaticInputs: []string{"https://www.google.com/"},
-		SourceFiles: []string{
-			"testdata/inputloader1.txt",
-			"testdata/inputloader2.txt",
-		},
-		InputPolicy: engine.InputOrQueryTestLists,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(out) != 5 {
-		t.Fatal("not the output length we expected")
-	}
-	expect := []model.URLInfo{
-		{URL: "https://www.google.com/"},
-		{URL: "https://www.x.org/"},
-		{URL: "https://www.slashdot.org/"},
-		{URL: "https://abc.xyz/"},
-		{URL: "https://run.ooni.io/"},
-	}
-	if diff := cmp.Diff(out, expect); diff != "" {
-		t.Fatal(diff)
-	}
-}
-
-func TestInputLoaderInputOrQueryTestListsWithNoInputAndCancelledContext(t *testing.T) {
-	sess, err := engine.NewSession(engine.SessionConfig{
-		AssetsDir:       "testdata",
-		KVStore:         kvstore.NewMemoryKeyValueStore(),
-		Logger:          log.Log,
-		SoftwareName:    "miniooni",
-		SoftwareVersion: "0.1.0-dev",
-		TempDir:         "testdata",
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer sess.Close()
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		InputPolicy: engine.InputOrQueryTestLists,
-		Session:     sess,
-	})
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel() // fail immediately
-	out, err := il.Load(ctx)
-	if !errors.Is(err, context.Canceled) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("not the output we expected")
-	}
-}
-
-func TestInputLoaderInputOrQueryTestListsWithNoInput(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skip test in short mode")
-	}
-	sess, err := engine.NewSession(engine.SessionConfig{
-		AvailableProbeServices: []model.Service{{
-			Address: "https://ams-pg-test.ooni.org/",
-			Type:    "https",
-		}},
-		AssetsDir:       "testdata",
-		KVStore:         kvstore.NewMemoryKeyValueStore(),
-		Logger:          log.Log,
-		SoftwareName:    "miniooni",
-		SoftwareVersion: "0.1.0-dev",
-		TempDir:         "testdata",
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer sess.Close()
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		InputPolicy: engine.InputOrQueryTestLists,
-		Session:     sess,
-		URLLimit:    30,
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(out) < 10 {
-		t.Fatal("not the output length we expected")
-	}
-}
-
-func TestInputLoaderInputOrQueryTestListsWithEmptyFile(t *testing.T) {
-	il := engine.NewInputLoader(engine.InputLoaderConfig{
-		InputPolicy: engine.InputOrQueryTestLists,
-		SourceFiles: []string{
-			"testdata/inputloader1.txt",
-			"testdata/inputloader3.txt", // we want it before inputloader2.txt
-			"testdata/inputloader2.txt",
-		},
-	})
-	ctx := context.Background()
-	out, err := il.Load(ctx)
-	if !errors.Is(err, engine.ErrDetectedEmptyFile) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("not the output we expected")
-	}
-}

internal/engine/inputloader_network_test.go

@@ -0,0 +1,45 @@
+package engine_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/apex/log"
+	engine "github.com/ooni/probe-cli/v3/internal/engine"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	sess, err := engine.NewSession(engine.SessionConfig{
+		AvailableProbeServices: []model.Service{{
+			Address: "https://ams-pg-test.ooni.org/",
+			Type:    "https",
+		}},
+		KVStore:         kvstore.NewMemoryKeyValueStore(),
+		Logger:          log.Log,
+		SoftwareName:    "miniooni",
+		SoftwareVersion: "0.1.0-dev",
+		TempDir:         "testdata",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer sess.Close()
+	il := &engine.InputLoader{
+		InputPolicy: engine.InputOrQueryBackend,
+		Session:     sess,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out) < 10 {
+		// check-in SHOULD return AT LEAST 20 URLs at a time.
+		t.Fatal("not the output length we expected")
+	}
+}

internal/engine/inputloader_test.go

@@ -4,17 +4,286 @@ import (
 	"context"
 	"errors"
 	"io"
+	"io/fs"
 	"os"
 	"syscall"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/fsx"
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 
+func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) {
+	il := &InputLoader{
+		StaticInputs: []string{"https://www.google.com/"},
+		InputPolicy:  InputNone,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if !errors.Is(err, ErrNoInputExpected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) {
+	il := &InputLoader{
+		SourceFiles: []string{
+			"testdata/inputloader1.txt",
+			"testdata/inputloader2.txt",
+		},
+		InputPolicy: InputNone,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if !errors.Is(err, ErrNoInputExpected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputNoneWithBothInputs(t *testing.T) {
+	il := &InputLoader{
+		StaticInputs: []string{"https://www.google.com/"},
+		SourceFiles: []string{
+			"testdata/inputloader1.txt",
+			"testdata/inputloader2.txt",
+		},
+		InputPolicy: InputNone,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if !errors.Is(err, ErrNoInputExpected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputNoneWithNoInput(t *testing.T) {
+	il := &InputLoader{
+		InputPolicy: InputNone,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out) != 1 || out[0].URL != "" {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputOptionalWithNoInput(t *testing.T) {
+	il := &InputLoader{
+		InputPolicy: InputOptional,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out) != 1 || out[0].URL != "" {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputOptionalWithInput(t *testing.T) {
+	il := &InputLoader{
+		StaticInputs: []string{"https://www.google.com/"},
+		SourceFiles: []string{
+			"testdata/inputloader1.txt",
+			"testdata/inputloader2.txt",
+		},
+		InputPolicy: InputOptional,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out) != 5 {
+		t.Fatal("not the output length we expected")
+	}
+	expect := []model.URLInfo{
+		{URL: "https://www.google.com/"},
+		{URL: "https://www.x.org/"},
+		{URL: "https://www.slashdot.org/"},
+		{URL: "https://abc.xyz/"},
+		{URL: "https://run.ooni.io/"},
+	}
+	if diff := cmp.Diff(out, expect); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) {
+	il := &InputLoader{
+		StaticInputs: []string{"https://www.google.com/"},
+		SourceFiles: []string{
+			"testdata/inputloader1.txt",
+			"/nonexistent",
+			"testdata/inputloader2.txt",
+		},
+		InputPolicy: InputOptional,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if !errors.Is(err, syscall.ENOENT) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) {
+	il := &InputLoader{
+		StaticInputs: []string{"https://www.google.com/"},
+		SourceFiles: []string{
+			"testdata/inputloader1.txt",
+			"testdata/inputloader2.txt",
+		},
+		InputPolicy: InputStrictlyRequired,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out) != 5 {
+		t.Fatal("not the output length we expected")
+	}
+	expect := []model.URLInfo{
+		{URL: "https://www.google.com/"},
+		{URL: "https://www.x.org/"},
+		{URL: "https://www.slashdot.org/"},
+		{URL: "https://abc.xyz/"},
+		{URL: "https://run.ooni.io/"},
+	}
+	if diff := cmp.Diff(out, expect); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) {
+	il := &InputLoader{
+		InputPolicy: InputStrictlyRequired,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if !errors.Is(err, ErrInputRequired) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) {
+	il := &InputLoader{
+		InputPolicy: InputStrictlyRequired,
+		SourceFiles: []string{
+			"testdata/inputloader1.txt",
+			"testdata/inputloader3.txt", // we want it before inputloader2.txt
+			"testdata/inputloader2.txt",
+		},
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if !errors.Is(err, ErrDetectedEmptyFile) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) {
+	il := &InputLoader{
+		StaticInputs: []string{"https://www.google.com/"},
+		SourceFiles: []string{
+			"testdata/inputloader1.txt",
+			"testdata/inputloader2.txt",
+		},
+		InputPolicy: InputOrQueryBackend,
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(out) != 5 {
+		t.Fatal("not the output length we expected")
+	}
+	expect := []model.URLInfo{
+		{URL: "https://www.google.com/"},
+		{URL: "https://www.x.org/"},
+		{URL: "https://www.slashdot.org/"},
+		{URL: "https://abc.xyz/"},
+		{URL: "https://run.ooni.io/"},
+	}
+	if diff := cmp.Diff(out, expect); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing.T) {
+	sess, err := NewSession(SessionConfig{
+		KVStore:         kvstore.NewMemoryKeyValueStore(),
+		Logger:          log.Log,
+		SoftwareName:    "miniooni",
+		SoftwareVersion: "0.1.0-dev",
+		TempDir:         "testdata",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer sess.Close()
+	il := &InputLoader{
+		InputPolicy: InputOrQueryBackend,
+		Session:     sess,
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	out, err := il.Load(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("not the output we expected")
+	}
+}
+
+func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) {
+	il := &InputLoader{
+		InputPolicy: InputOrQueryBackend,
+		SourceFiles: []string{
+			"testdata/inputloader1.txt",
+			"testdata/inputloader3.txt", // we want it before inputloader2.txt
+			"testdata/inputloader2.txt",
+		},
+	}
+	ctx := context.Background()
+	out, err := il.Load(ctx)
+	if !errors.Is(err, ErrDetectedEmptyFile) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("not the output we expected")
+	}
+}
+
 type InputLoaderBrokenFS struct{}
 
-func (InputLoaderBrokenFS) Open(filepath string) (fsx.File, error) {
+func (InputLoaderBrokenFS) Open(filepath string) (fs.File, error) {
 	return InputLoaderBrokenFile{}, nil
 }
 
@@ -33,7 +302,7 @@ func (InputLoaderBrokenFile) Close() error {
 }
 
 func TestInputLoaderReadfileScannerFailure(t *testing.T) {
-	il := inputLoader{}
+	il := &InputLoader{}
 	out, err := il.readfile("", InputLoaderBrokenFS{}.Open)
 	if !errors.Is(err, syscall.EFAULT) {
 		t.Fatal("not the error we expected")
@@ -43,64 +312,34 @@ func TestInputLoaderReadfileScannerFailure(t *testing.T) {
 	}
 }
 
-type InputLoaderBrokenSession struct {
+// InputLoaderMockableSession is a mockable session
-	OrchestraClient model.ExperimentOrchestraClient
+// used by InputLoader tests.
-	Error           error
+type InputLoaderMockableSession struct {
+	// Output contains the output of CheckIn. It should
+	// be nil when Error is not-nil.
+	Output *model.CheckInInfo
+
+	// Error is the error to be returned by CheckIn. It
+	// should be nil when Output is not-nil.
+	Error error
 }
 
-func (InputLoaderBrokenSession) MaybeLookupLocationContext(ctx context.Context) error {
+// CheckIn implements InputLoaderSession.CheckIn.
-	return nil
+func (sess *InputLoaderMockableSession) CheckIn(
-}
+	ctx context.Context, config *model.CheckInConfig) (*model.CheckInInfo, error) {
-
+	if sess.Output == nil && sess.Error == nil {
-func (ilbs InputLoaderBrokenSession) NewOrchestraClient(ctx context.Context) (model.ExperimentOrchestraClient, error) {
+		return nil, errors.New("both Output and Error are nil")
-	if ilbs.OrchestraClient != nil {
-		return ilbs.OrchestraClient, nil
 	}
-	return nil, io.EOF
+	return sess.Output, sess.Error
 }
 
-func (InputLoaderBrokenSession) ProbeCC() string {
+func TestInputLoaderCheckInFailure(t *testing.T) {
-	return "IT"
+	il := &InputLoader{
-}
+		Session: &InputLoaderMockableSession{
-
+			Error: io.EOF,
-func TestInputLoaderNewOrchestraClientFailure(t *testing.T) {
-	il := inputLoader{}
-	lrc := loadRemoteConfig{
-		ctx:     context.Background(),
-		session: InputLoaderBrokenSession{},
-	}
-	out, err := il.loadRemote(lrc)
-	if !errors.Is(err, io.EOF) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if out != nil {
-		t.Fatal("expected nil output here")
-	}
-}
-
-type InputLoaderBrokenOrchestraClient struct{}
-
-func (InputLoaderBrokenOrchestraClient) FetchPsiphonConfig(ctx context.Context) ([]byte, error) {
-	return nil, io.EOF
-}
-
-func (InputLoaderBrokenOrchestraClient) FetchTorTargets(ctx context.Context, cc string) (map[string]model.TorTarget, error) {
-	return nil, io.EOF
-}
-
-func (InputLoaderBrokenOrchestraClient) FetchURLList(ctx context.Context, config model.URLListConfig) ([]model.URLInfo, error) {
-	return nil, io.EOF
-}
-
-func TestInputLoaderFetchURLListFailure(t *testing.T) {
-	il := inputLoader{}
-	lrc := loadRemoteConfig{
-		ctx: context.Background(),
-		session: InputLoaderBrokenSession{
-			OrchestraClient: InputLoaderBrokenOrchestraClient{},
 		},
 	}
-	out, err := il.loadRemote(lrc)
+	out, err := il.loadRemote(context.Background())
 	if !errors.Is(err, io.EOF) {
 		t.Fatalf("not the error we expected: %+v", err)
 	}
@@ -108,3 +347,63 @@ func TestInputLoaderFetchURLListFailure(t *testing.T) {
 		t.Fatal("expected nil output here")
 	}
 }
+
+func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) {
+	il := &InputLoader{
+		Session: &InputLoaderMockableSession{
+			Output: &model.CheckInInfo{},
+		},
+	}
+	out, err := il.loadRemote(context.Background())
+	if !errors.Is(err, ErrNoURLsReturned) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("expected nil output here")
+	}
+}
+
+func TestInputLoaderCheckInSuccessWithNoURLs(t *testing.T) {
+	il := &InputLoader{
+		Session: &InputLoaderMockableSession{
+			Output: &model.CheckInInfo{
+				WebConnectivity: &model.CheckInInfoWebConnectivity{},
+			},
+		},
+	}
+	out, err := il.loadRemote(context.Background())
+	if !errors.Is(err, ErrNoURLsReturned) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if out != nil {
+		t.Fatal("expected nil output here")
+	}
+}
+
+func TestInputLoaderCheckInSuccessWithSomeURLs(t *testing.T) {
+	expect := []model.URLInfo{{
+		CategoryCode: "NEWS",
+		CountryCode:  "IT",
+		URL:          "https://repubblica.it",
+	}, {
+		CategoryCode: "NEWS",
+		CountryCode:  "IT",
+		URL:          "https://corriere.it",
+	}}
+	il := &InputLoader{
+		Session: &InputLoaderMockableSession{
+			Output: &model.CheckInInfo{
+				WebConnectivity: &model.CheckInInfoWebConnectivity{
+					URLs: expect,
+				},
+			},
+		},
+	}
+	out, err := il.loadRemote(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(expect, out); diff != "" {
+		t.Fatal(diff)
+	}
+}

internal/engine/inputprocessor.go

@@ -2,6 +2,8 @@ package engine
 
 import (
 	"context"
+	"sync/atomic"
+	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
@@ -50,6 +52,12 @@ type InputProcessor struct {
 	// Inputs is the list of inputs to measure.
 	Inputs []model.URLInfo
 
+	// MaxRuntime is the optional maximum runtime
+	// when looping over a list of inputs (e.g. when
+	// running Web Connectivity). Zero means that
+	// there will be no MaxRuntime limit.
+	MaxRuntime time.Duration
+
 	// Options contains command line options for this experiment.
 	Options []string
 
@@ -60,6 +68,11 @@ type InputProcessor struct {
 	// Submitter is the code that will submit measurements
 	// to the OONI collector.
 	Submitter InputProcessorSubmitterWrapper
+
+	// terminatedByMaxRuntime is an internal atomic variabile
+	// incremented when we're terminated by MaxRuntime. We
+	// only use this variable when testing.
+	terminatedByMaxRuntime int32
 }
 
 // InputProcessorSaverWrapper is InputProcessor's
@@ -115,8 +128,13 @@ func (ipsw inputProcessorSubmitterWrapper) Submit(
 // is always causing us to break out of the loop. The user
 // though is free to choose different policies by configuring
 // the Experiment, Submitter, and Saver fields properly.
-func (ip InputProcessor) Run(ctx context.Context) error {
+func (ip *InputProcessor) Run(ctx context.Context) error {
+	start := time.Now()
 	for idx, url := range ip.Inputs {
+		if ip.MaxRuntime > 0 && time.Since(start) > ip.MaxRuntime {
+			atomic.AddInt32(&ip.terminatedByMaxRuntime, 1)
+			return nil
+		}
 		input := url.URL
 		meas, err := ip.Experiment.MeasureWithContext(ctx, idx, input)
 		if err != nil {

internal/engine/inputprocessor_test.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 
 type FakeInputProcessorExperiment struct {
-	Err error
+	SleepTime time.Duration
-	M   []*model.Measurement
+	Err       error
+	M         []*model.Measurement
 }
 
 func (fipe *FakeInputProcessorExperiment) MeasureWithContext(
@@ -18,6 +20,9 @@ func (fipe *FakeInputProcessorExperiment) MeasureWithContext(
 	if fipe.Err != nil {
 		return nil, fipe.Err
 	}
+	if fipe.SleepTime > 0 {
+		time.Sleep(fipe.SleepTime)
+	}
 	m := new(model.Measurement)
 	// Here we add annotations to ensure that the input processor
 	// is MERGING annotations as opposed to overwriting them.
@@ -30,7 +35,7 @@ func (fipe *FakeInputProcessorExperiment) MeasureWithContext(
 
 func TestInputProcessorMeasurementFailed(t *testing.T) {
 	expected := errors.New("mocked error")
-	ip := InputProcessor{
+	ip := &InputProcessor{
 		Experiment: NewInputProcessorExperimentWrapper(
 			&FakeInputProcessorExperiment{Err: expected},
 		),
@@ -58,7 +63,7 @@ func (fips *FakeInputProcessorSubmitter) Submit(
 func TestInputProcessorSubmissionFailed(t *testing.T) {
 	fipe := &FakeInputProcessorExperiment{}
 	expected := errors.New("mocked error")
-	ip := InputProcessor{
+	ip := &InputProcessor{
 		Annotations: map[string]string{
 			"foo": "bar",
 		},
@@ -108,7 +113,7 @@ func (fips *FakeInputProcessorSaver) SaveMeasurement(m *model.Measurement) error
 
 func TestInputProcessorSaveOnDiskFailed(t *testing.T) {
 	expected := errors.New("mocked error")
-	ip := InputProcessor{
+	ip := &InputProcessor{
 		Experiment: NewInputProcessorExperimentWrapper(
 			&FakeInputProcessorExperiment{},
 		),
@@ -133,7 +138,7 @@ func TestInputProcessorGood(t *testing.T) {
 	fipe := &FakeInputProcessorExperiment{}
 	saver := &FakeInputProcessorSaver{Err: nil}
 	submitter := &FakeInputProcessorSubmitter{Err: nil}
-	ip := InputProcessor{
+	ip := &InputProcessor{
 		Experiment: NewInputProcessorExperimentWrapper(fipe),
 		Inputs: []model.URLInfo{{
 			URL: "https://www.kernel.org/",
@@ -148,6 +153,9 @@ func TestInputProcessorGood(t *testing.T) {
 	if err := ip.Run(ctx); err != nil {
 		t.Fatal(err)
 	}
+	if ip.terminatedByMaxRuntime > 0 {
+		t.Fatal("terminated by max runtime!?")
+	}
 	if len(fipe.M) != 2 || len(saver.M) != 2 || len(submitter.M) != 2 {
 		t.Fatal("not all measurements saved")
 	}
@@ -164,3 +172,30 @@ func TestInputProcessorGood(t *testing.T) {
 		t.Fatal("invalid saver.M[1].Input")
 	}
 }
+
+func TestInputProcessorMaxRuntime(t *testing.T) {
+	fipe := &FakeInputProcessorExperiment{
+		SleepTime: 50 * time.Millisecond,
+	}
+	saver := &FakeInputProcessorSaver{Err: nil}
+	submitter := &FakeInputProcessorSubmitter{Err: nil}
+	ip := &InputProcessor{
+		Experiment: NewInputProcessorExperimentWrapper(fipe),
+		Inputs: []model.URLInfo{{
+			URL: "https://www.kernel.org/",
+		}, {
+			URL: "https://www.slashdot.org/",
+		}},
+		MaxRuntime: 1 * time.Nanosecond,
+		Options:    []string{"fake=true"},
+		Saver:      NewInputProcessorSaverWrapper(saver),
+		Submitter:  NewInputProcessorSubmitterWrapper(submitter),
+	}
+	ctx := context.Background()
+	if err := ip.Run(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if ip.terminatedByMaxRuntime <= 0 {
+		t.Fatal("not terminated by max runtime")
+	}
+}

internal/engine/internal/fsx/fsx.go

@@ -3,31 +3,18 @@ package fsx
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"syscall"
 )
 
-// File is a generic file. This interface is taken from the draft
-// iofs golang design. We'll use fs.File when available.
-type File interface {
-	Stat() (os.FileInfo, error)
-	Read([]byte) (int, error)
-	Close() error
-}
-
-// FS is a generic file system. Like File, it's adapted from
-// the draft iofs golang design document.
-type FS interface {
-	Open(name string) (File, error)
-}
-
 // Open is a wrapper for os.Open that ensures that we're opening a file.
-func Open(pathname string) (File, error) {
+func Open(pathname string) (fs.File, error) {
 	return OpenWithFS(filesystem{}, pathname)
 }
 
 // OpenWithFS is like Open but with explicit file system argument.
-func OpenWithFS(fs FS, pathname string) (File, error) {
+func OpenWithFS(fs fs.FS, pathname string) (fs.File, error) {
 	file, err := fs.Open(pathname)
 	if err != nil {
 		return nil, err
@@ -39,13 +26,16 @@ func OpenWithFS(fs FS, pathname string) (File, error) {
 	}
 	if info.IsDir() {
 		file.Close()
-		return nil, fmt.Errorf("input path points to a directory: %w", syscall.EISDIR)
+		return nil, fmt.Errorf(
+			"input path points to a directory: %w", syscall.EISDIR)
 	}
 	return file, nil
 }
 
+// filesystem is a private implementation of fs.FS.
 type filesystem struct{}
 
-func (filesystem) Open(pathname string) (File, error) {
+// Open implements fs.FS.Open.
+func (filesystem) Open(pathname string) (fs.File, error) {
 	return os.Open(pathname)
 }

internal/engine/internal/fsx/fsx_test.go

@@ -2,6 +2,7 @@ package fsx_test
 
 import (
 	"errors"
+	"io/fs"
 	"os"
 	"sync/atomic"
 	"syscall"
@@ -26,8 +27,8 @@ func (FailingStatFile) Stat() (os.FileInfo, error) {
 	return nil, errStatFailed
 }
 
-func (fs FailingStatFS) Open(pathname string) (fsx.File, error) {
+func (f FailingStatFS) Open(pathname string) (fs.File, error) {
-	return FailingStatFile{CloseCount: fs.CloseCount}, nil
+	return FailingStatFile(f), nil
 }
 
 func (fs FailingStatFile) Close() error {

internal/engine/internal/mockable/mockable.go

@@ -18,7 +18,6 @@ import (
 
 // Session allows to mock sessions.
 type Session struct {
-	MockableASNDatabasePath      string
 	MockableTestHelpers          map[string][]model.Service
 	MockableHTTPClient           *http.Client
 	MockableLogger               model.Logger
@@ -39,11 +38,6 @@ type Session struct {
 	MockableUserAgent            string
 }
 
-// ASNDatabasePath implements ExperimentSession.ASNDatabasePath
-func (sess *Session) ASNDatabasePath() string {
-	return sess.MockableASNDatabasePath
-}
-
 // GetTestHelpersByName implements ExperimentSession.GetTestHelpersByName
 func (sess *Session) GetTestHelpersByName(name string) ([]model.Service, bool) {
 	services, okay := sess.MockableTestHelpers[name]
@@ -162,6 +156,8 @@ var _ torx.Session = &Session{}
 // ExperimentOrchestraClient is the experiment's view of
 // a client for querying the OONI orchestra.
 type ExperimentOrchestraClient struct {
+	MockableCheckInInfo              *model.CheckInInfo
+	MockableCheckInErr               error
 	MockableFetchPsiphonConfigResult []byte
 	MockableFetchPsiphonConfigErr    error
 	MockableFetchTorTargetsResult    map[string]model.TorTarget
@@ -170,6 +166,12 @@ type ExperimentOrchestraClient struct {
 	MockableFetchURLListErr          error
 }
 
+// CheckIn implements ExperimentOrchestraClient.CheckIn.
+func (c ExperimentOrchestraClient) CheckIn(
+	ctx context.Context, config model.CheckInConfig) (*model.CheckInInfo, error) {
+	return c.MockableCheckInInfo, c.MockableCheckInErr
+}
+
 // FetchPsiphonConfig implements ExperimentOrchestraClient.FetchPsiphonConfig
 func (c ExperimentOrchestraClient) FetchPsiphonConfig(
 	ctx context.Context) ([]byte, error) {

internal/engine/internal/psiphonx/psiphonx_test.go

@@ -17,7 +17,6 @@ func TestStartWithCancelledContext(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 	sess, err := engine.NewSession(engine.SessionConfig{
-		AssetsDir:       "../../testdata",
 		Logger:          log.Log,
 		SoftwareName:    "ooniprobe-engine",
 		SoftwareVersion: "0.0.1",
@@ -39,7 +38,6 @@ func TestStartStop(t *testing.T) {
 		t.Skip("skip test in short mode")
 	}
 	sess, err := engine.NewSession(engine.SessionConfig{
-		AssetsDir:       "../../testdata",
 		Logger:          log.Log,
 		SoftwareName:    "ooniprobe-engine",
 		SoftwareVersion: "0.0.1",

internal/engine/internal/sessionresolver/resolvermaker.go

@@ -21,7 +21,7 @@ const systemResolverURL = "system:///"
 
 // allmakers contains all the makers in a list. We use the http3
 // prefix to indicate we wanna use http3. The code will translate
-// this to https and set the proper next options.
+// this to https and set the proper netx options.
 var allmakers = []*resolvermaker{{
 	url: "https://cloudflare-dns.com/dns-query",
 }, {
@@ -97,7 +97,7 @@ func (r *Resolver) newresolver(URL string) (childResolver, error) {
 func (r *Resolver) getresolver(URL string) (childResolver, error) {
 	defer r.mu.Unlock()
 	r.mu.Lock()
-	if re, found := r.res[URL]; found == true {
+	if re, found := r.res[URL]; found {
 		return re, nil // already created
 	}
 	re, err := r.newresolver(URL)

internal/engine/internal/sessionresolver/sessionresolver.go

@@ -17,6 +17,10 @@
 // but it will of course be the most popular resolver if anything else
 // is failing us. (We will still occasionally probe for other working
 // resolvers and increase their score on success.)
+//
+// We also support a socks5 proxy. When such a proxy is configured,
+// the code WILL skip http3 resolvers AS WELL AS the system
+// resolver, in an attempt to avoid leaking your queries.
 package sessionresolver
 
 import (
@@ -34,18 +38,60 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
 )
 
-// Resolver is the session resolver. You should create an instance of
+// Resolver is the session resolver. Resolver will try to use
-// this structure and use it in session.go.
+// a bunch of DoT/DoH resolvers before falling back to the
+// system resolver. The relative priorities of the resolver
+// are stored onto the KVStore such that we can remember them
+// and therefore we can generally give preference to underlying
+// DoT/DoH resolvers that work better.
+//
+// You MUST NOT modify public fields of this structure once it
+// has been created, because that MAY lead to data races.
+//
+// You should create an instance of this structure and use
+// it in internal/engine/session.go.
 type Resolver struct {
-	ByteCounter    *bytecounter.Counter // optional
+	// ByteCounter is the optional byte counter. It will count
-	KVStore        KVStore              // optional
+	// the bytes used by any child resolver except for the
-	Logger         Logger               // optional
+	// system resolver, whose bytes ARE NOT counted. If this
-	ProxyURL       *url.URL             // optional
+	// field is not set, then we won't count the bytes.
-	codec          codec
+	ByteCounter *bytecounter.Counter
+
+	// KVStore is the optional key-value store where you
+	// want us to write statistics about which resolver is
+	// working better in your network. If this field is
+	// not set, then we'll use a in-memory store.
+	KVStore KVStore
+
+	// Logger is the optional logger you want us to use
+	// to emit log messages.
+	Logger Logger
+
+	// ProxyURL is the optional URL of the socks5 proxy
+	// we should be using. If not set, then we WON'T use
+	// any proxy. If set, then we WON'T use any http3
+	// based resolvers and we WON'T use the system resolver.
+	ProxyURL *url.URL
+
+	// codec is the optional codec to use. If not set, we
+	// will construct a default codec.
+	codec codec
+
+	// dnsClientMaker is the optional dnsclientmaker to
+	// use. If not set, we will use the default.
 	dnsClientMaker dnsclientmaker
-	mu             sync.Mutex
+
-	once           sync.Once
+	// mu provides synchronisation of internal fields.
-	res            map[string]childResolver
+	mu sync.Mutex
+
+	// once ensures that CloseIdleConnection is
+	// run just once.
+	once sync.Once
+
+	// res maps a URL to a child resolver. We will
+	// construct child resolvers just once and we
+	// will track them into this field.
+	res map[string]childResolver
 }
 
 // CloseIdleConnections closes the idle connections, if any. This
@@ -73,7 +119,8 @@ func (r *Resolver) LookupHost(ctx context.Context, hostname string) ([]string, e
 	defer r.writestate(state)
 	me := multierror.New(ErrLookupHost)
 	for _, e := range state {
-		if r.shouldSkipWithProxy(e) {
+		if r.ProxyURL != nil && r.shouldSkipWithProxy(e) {
+			r.logger().Infof("sessionresolver: skipping with proxy: %+v", e)
 			continue // we cannot proxy this URL so ignore it
 		}
 		addrs, err := r.lookupHost(ctx, e, hostname)

internal/engine/internal/sessionresolver/sessionresolver_test.go

@@ -294,7 +294,7 @@ func TestResolverWorksWithProxy(t *testing.T) {
 	<-done
 	// check results
 	if !errors.Is(err, ErrLookupHost) {
-		t.Fatal("not the error we expected")
+		t.Fatal("not the error we expected", err)
 	}
 	if addrs != nil {
 		t.Fatal("expected nil addrs")

internal/engine/legacy/assetsdir/assetsdir.go

@@ -0,0 +1,62 @@
+// Package assetsdir contains code to cleanup the assets dir. We removed
+// the assetsdir in the 3.9.0 development cycle.
+package assetsdir
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+)
+
+// ErrEmptyDir indicates that you passed to Cleanup an empty dir.
+var ErrEmptyDir = errors.New("empty assets directory")
+
+// Result is the result of a Cleanup run.
+type Result struct {
+	// ASNDatabaseErr is the error of deleting the
+	// file containing the old ASN database.
+	ASNDatabaseErr error
+
+	// CABundleErr is the error of deleting the file
+	// containing the old CA bundle.
+	CABundleErr error
+
+	// CountryDatabaseErr is the error of deleting the
+	// file containing the old country database.
+	CountryDatabaseErr error
+
+	// RmdirErr is the error of deleting the supposedly
+	// empty directory that contained assets.
+	RmdirErr error
+}
+
+// Cleanup removes data from the assetsdir. This function will
+// try to delete the known assets inside of dir. It then also
+// tries to delete the directory. If the directory is not empty,
+// this operation will fail. That means the user has put some
+// extra data in there and we don't want to remove it.
+//
+// Returns the Result of cleaning up the assets on success and
+// an error on failure. The only cause of error is passing to
+// this function an empty directory. The Result data structure
+// contains the result of each individual remove operation.
+func Cleanup(dir string) (*Result, error) {
+	return fcleanup(dir, os.Remove)
+}
+
+// fcleanup is a version of Cleanup where we can mock the real function
+// used for removing files and dirs, so we can write unit tests.
+func fcleanup(dir string, remove func(name string) error) (*Result, error) {
+	if dir == "" {
+		return nil, ErrEmptyDir
+	}
+	r := &Result{}
+	asndb := filepath.Join(dir, "asn.mmdb")
+	r.ASNDatabaseErr = os.Remove(asndb)
+	cabundle := filepath.Join(dir, "ca-bundle.pem")
+	r.CABundleErr = os.Remove(cabundle)
+	countrydb := filepath.Join(dir, "country.mmdb")
+	r.CountryDatabaseErr = os.Remove(countrydb)
+	r.RmdirErr = os.Remove(dir)
+	return r, nil
+}

internal/engine/legacy/assetsdir/assetsdir_test.go

@@ -0,0 +1,40 @@
+package assetsdir
+
+import (
+	"errors"
+	"strings"
+	"testing"
+)
+
+func TestCleanupNormalUsage(t *testing.T) {
+	result, err := Cleanup("testdata")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// we expect a bunch of ENOENT because the directory does not exist.
+	isExpectedErr := func(err error) bool {
+		return err != nil && strings.HasSuffix(err.Error(), "no such file or directory")
+	}
+	if !isExpectedErr(result.ASNDatabaseErr) {
+		t.Fatal("unexpected error", result.ASNDatabaseErr)
+	}
+	if !isExpectedErr(result.CABundleErr) {
+		t.Fatal("unexpected error", result.CABundleErr)
+	}
+	if !isExpectedErr(result.CountryDatabaseErr) {
+		t.Fatal("unexpected error", result.CountryDatabaseErr)
+	}
+	if !isExpectedErr(result.RmdirErr) {
+		t.Fatal("unexpected error", result.RmdirErr)
+	}
+}
+
+func TestCleanupWithEmptyInput(t *testing.T) {
+	result, err := Cleanup("")
+	if !errors.Is(err, ErrEmptyDir) {
+		t.Fatal("unexpected error", err)
+	}
+	if result != nil {
+		t.Fatal("expected nil result")
+	}
+}

internal/engine/legacy/netx/dialer_test.go

@@ -101,6 +101,9 @@ func TestDialerSetCABundleWAI(t *testing.T) {
 func TestDialerForceSpecificSNI(t *testing.T) {
 	dialer := netx.NewDialer()
 	err := dialer.ForceSpecificSNI("www.facebook.com")
+	if err != nil {
+		t.Fatal(err)
+	}
 	conn, err := dialer.DialTLS("tcp", "www.google.com:443")
 	if err == nil {
 		t.Fatal("expected an error here")

internal/engine/legacy/netx/modelx/modelx_test.go

@@ -56,7 +56,7 @@ func TestMeasurementRootWithMeasurementRootPanic(t *testing.T) {
 		}
 	}()
 	ctx := context.Background()
-	ctx = WithMeasurementRoot(ctx, nil)
+	_ = WithMeasurementRoot(ctx, nil)
 }
 
 func TestErrWrapperPublicAPI(t *testing.T) {

internal/engine/model/experiment.go

@@ -8,14 +8,22 @@ import (
 // ExperimentOrchestraClient is the experiment's view of
 // a client for querying the OONI orchestra API.
 type ExperimentOrchestraClient interface {
+	// CheckIn calls the check-in API.
+	CheckIn(ctx context.Context, config CheckInConfig) (*CheckInInfo, error)
+
+	// FetchPsiphonConfig returns psiphon config from the API.
 	FetchPsiphonConfig(ctx context.Context) ([]byte, error)
+
+	// FetchTorTargets returns tor targets from the API.
 	FetchTorTargets(ctx context.Context, cc string) (map[string]TorTarget, error)
+
+	// FetchURLList returns URLs from the API.
+	// This method is deprecated and will be removed soon.
 	FetchURLList(ctx context.Context, config URLListConfig) ([]URLInfo, error)
 }
 
 // ExperimentSession is the experiment's view of a session.
 type ExperimentSession interface {
-	ASNDatabasePath() string
 	GetTestHelpersByName(name string) ([]Service, bool)
 	DefaultHTTPClient() *http.Client
 	Logger() Logger

internal/engine/netx/archival/archival.go

@@ -397,7 +397,7 @@ type DNSQueryEntry struct {
 type dnsQueryType string
 
 // NewDNSQueriesList returns a list of DNS queries.
-func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []DNSQueryEntry {
+func NewDNSQueriesList(begin time.Time, events []trace.Event) []DNSQueryEntry {
 	// TODO(bassosimone): add support for CNAME lookups.
 	var out []DNSQueryEntry
 	for _, ev := range events {
@@ -409,7 +409,7 @@ func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []D
 			for _, addr := range ev.Addresses {
 				if qtype.ipoftype(addr) {
 					entry.Answers = append(
-						entry.Answers, qtype.makeanswerentry(addr, dbpath))
+						entry.Answers, qtype.makeanswerentry(addr))
 				}
 			}
 			if len(entry.Answers) <= 0 && ev.Err == nil {
@@ -431,16 +431,16 @@ func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []D
 func (qtype dnsQueryType) ipoftype(addr string) bool {
 	switch qtype {
 	case "A":
-		return strings.Contains(addr, ":") == false
+		return !strings.Contains(addr, ":")
 	case "AAAA":
-		return strings.Contains(addr, ":") == true
+		return strings.Contains(addr, ":")
 	}
 	return false
 }
 
-func (qtype dnsQueryType) makeanswerentry(addr string, dbpath string) DNSAnswerEntry {
+func (qtype dnsQueryType) makeanswerentry(addr string) DNSAnswerEntry {
 	answer := DNSAnswerEntry{AnswerType: string(qtype)}
-	asn, org, _ := geolocate.LookupASN(dbpath, addr)
+	asn, org, _ := geolocate.LookupASN(addr)
 	answer.ASN = int64(asn)
 	answer.ASOrgName = org
 	switch qtype {
@@ -463,17 +463,20 @@ func (qtype dnsQueryType) makequeryentry(begin time.Time, ev trace.Event) DNSQue
 	}
 }
 
-// NetworkEvent is a network event.
+// NetworkEvent is a network event. It contains all the possible fields
+// and most fields are optional. They are only added when it makes sense
+// for them to be there _and_ we have data to show.
 type NetworkEvent struct {
-	Address       string  `json:"address,omitempty"`
+	Address       string   `json:"address,omitempty"`
-	ConnID        int64   `json:"conn_id,omitempty"`
+	ConnID        int64    `json:"conn_id,omitempty"`
-	DialID        int64   `json:"dial_id,omitempty"`
+	DialID        int64    `json:"dial_id,omitempty"`
-	Failure       *string `json:"failure"`
+	Failure       *string  `json:"failure"`
-	NumBytes      int64   `json:"num_bytes,omitempty"`
+	NumBytes      int64    `json:"num_bytes,omitempty"`
-	Operation     string  `json:"operation"`
+	Operation     string   `json:"operation"`
-	Proto         string  `json:"proto,omitempty"`
+	Proto         string   `json:"proto,omitempty"`
-	T             float64 `json:"t"`
+	T             float64  `json:"t"`
-	TransactionID int64   `json:"transaction_id,omitempty"`
+	Tags          []string `json:"tags,omitempty"`
+	TransactionID int64    `json:"transaction_id,omitempty"`
 }
 
 // NewNetworkEventsList returns a list of DNS queries.
@@ -547,6 +550,7 @@ type TLSHandshake struct {
 	PeerCertificates   []MaybeBinaryValue `json:"peer_certificates"`
 	ServerName         string             `json:"server_name"`
 	T                  float64            `json:"t"`
+	Tags               []string           `json:"tags"`
 	TLSVersion         string             `json:"tls_version"`
 	TransactionID      int64              `json:"transaction_id,omitempty"`
 }

internal/engine/netx/archival/archival_internal_test.go

@@ -0,0 +1,41 @@
+package archival
+
+import "testing"
+
+func TestDNSQueryIPOfType(t *testing.T) {
+	type expectation struct {
+		qtype  dnsQueryType
+		ip     string
+		output bool
+	}
+	var expectations = []expectation{{
+		qtype:  "A",
+		ip:     "8.8.8.8",
+		output: true,
+	}, {
+		qtype:  "A",
+		ip:     "2a00:1450:4002:801::2004",
+		output: false,
+	}, {
+		qtype:  "AAAA",
+		ip:     "8.8.8.8",
+		output: false,
+	}, {
+		qtype:  "AAAA",
+		ip:     "2a00:1450:4002:801::2004",
+		output: true,
+	}, {
+		qtype:  "ANTANI",
+		ip:     "2a00:1450:4002:801::2004",
+		output: false,
+	}, {
+		qtype:  "ANTANI",
+		ip:     "8.8.8.8",
+		output: false,
+	}}
+	for _, exp := range expectations {
+		if exp.qtype.ipoftype(exp.ip) != exp.output {
+			t.Fatalf("failure for %+v", exp)
+		}
+	}
+}

internal/engine/netx/archival/archival_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
-	"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
 )
 
 func TestNewTCPConnectList(t *testing.T) {
@@ -285,15 +284,10 @@ func TestNewRequestList(t *testing.T) {
 }
 
 func TestNewDNSQueriesList(t *testing.T) {
-	err := (&resourcesmanager.CopyWorker{DestDir: "../../testdata"}).Ensure()
-	if err != nil {
-		t.Fatal(err)
-	}
 	begin := time.Now()
 	type args struct {
 		begin  time.Time
 		events []trace.Event
-		dbpath string
 	}
 	tests := []struct {
 		name string
@@ -334,9 +328,13 @@ func TestNewDNSQueriesList(t *testing.T) {
 		},
 		want: []archival.DNSQueryEntry{{
 			Answers: []archival.DNSAnswerEntry{{
+				ASN:        15169,
+				ASOrgName:  "Google LLC",
 				AnswerType: "A",
 				IPv4:       "8.8.8.8",
 			}, {
+				ASN:        15169,
+				ASOrgName:  "Google LLC",
 				AnswerType: "A",
 				IPv4:       "8.8.4.4",
 			}},
@@ -357,27 +355,6 @@ func TestNewDNSQueriesList(t *testing.T) {
 				Time:      begin.Add(200 * time.Millisecond),
 			}},
 		},
-		want: []archival.DNSQueryEntry{{
-			Answers: []archival.DNSAnswerEntry{{
-				AnswerType: "AAAA",
-				IPv6:       "2001:4860:4860::8888",
-			}},
-			Hostname:  "dns.google.com",
-			QueryType: "AAAA",
-			T:         0.2,
-		}},
-	}, {
-		name: "run with ASN DB",
-		args: args{
-			begin: begin,
-			events: []trace.Event{{
-				Addresses: []string{"2001:4860:4860::8888"},
-				Hostname:  "dns.google.com",
-				Name:      "resolve_done",
-				Time:      begin.Add(200 * time.Millisecond),
-			}},
-			dbpath: "../../testdata/asn.mmdb",
-		},
 		want: []archival.DNSQueryEntry{{
 			Answers: []archival.DNSAnswerEntry{{
 				ASN:        15169,
@@ -399,7 +376,6 @@ func TestNewDNSQueriesList(t *testing.T) {
 				Name:     "resolve_done",
 				Time:     begin.Add(200 * time.Millisecond),
 			}},
-			dbpath: "../../testdata/asn.mmdb",
 		},
 		want: []archival.DNSQueryEntry{{
 			Answers: nil,
@@ -419,9 +395,9 @@ func TestNewDNSQueriesList(t *testing.T) {
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := archival.NewDNSQueriesList(
+			got := archival.NewDNSQueriesList(tt.args.begin, tt.args.events)
-				tt.args.begin, tt.args.events, tt.args.dbpath); !reflect.DeepEqual(got, tt.want) {
+			if diff := cmp.Diff(tt.want, got); diff != "" {
-				t.Error(cmp.Diff(got, tt.want))
+				t.Fatal(diff)
 			}
 		})
 	}
@@ -1009,13 +985,6 @@ func TestNewFailure(t *testing.T) {
 	}
 }
 
-func TestDNSQueryTypeInvalidIPOfType(t *testing.T) {
-	qtype := archival.DNSQueryType("ANTANI")
-	if qtype.IPOfType("8.8.8.8") != false {
-		t.Fatal("unexpected return value")
-	}
-}
-
 func TestNewFailedOperation(t *testing.T) {
 	type args struct {
 		err error

internal/engine/netx/archival/archival_test_internal.go

@@ -1,8 +0,0 @@
-package archival
-
-// DNSQueryType allows to access dnsQueryType from unit tests
-type DNSQueryType = dnsQueryType
-
-func (qtype dnsQueryType) IPOfType(addr string) bool {
-	return qtype.ipoftype(addr)
-}

internal/engine/netx/errorx/errorx_test.go

@@ -175,6 +175,9 @@ func TestToFailureString(t *testing.T) {
 		defer cancel() // fail immediately
 		udpAddr := &net.UDPAddr{IP: net.ParseIP("216.58.212.164"), Port: 80, Zone: ""}
 		udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+		if err != nil {
+			t.Fatal(err)
+		}
 		sess, err := quic.DialEarlyContext(ctx, udpConn, udpAddr, "google.com:80", &tls.Config{}, &quic.Config{})
 		if err == nil {
 			t.Fatal("expected an error here")

internal/engine/netx/gocertifi/certifi.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2021-03-03 11:48:43.129132377 +0100 CET m=+2.301468593
+// 2021-03-31 16:50:03.221493757 +0200 CEST m=+1.318280953
 // https://curl.haxx.se/ca/cacert.pem
 
 package gocertifi
@@ -3241,7 +3241,7 @@ kpzNNIaRkPpkUZ3+/uul9XXeifdy
 `
 
 // CACerts builds an X.509 certificate pool containing the
-// certificate bundle from https://curl.haxx.se/ca/cacert.pem fetch on 2021-03-03 11:48:43.129132377 +0100 CET m=+2.301468593.
+// certificate bundle from https://curl.haxx.se/ca/cacert.pem fetch on 2021-03-31 16:50:03.221493757 +0200 CEST m=+1.318280953.
 // Returns nil on error along with an appropriate error code.
 func CACerts() (*x509.CertPool, error) {
 	pool := x509.NewCertPool()

internal/engine/netx/quicdialer/connectionstate.go

@@ -1,5 +1,3 @@
-// +build !go1.15
-
 package quicdialer
 
 import (
@@ -10,5 +8,5 @@ import (
 
 // ConnectionState returns the ConnectionState of a QUIC Session.
 func ConnectionState(sess quic.EarlySession) tls.ConnectionState {
-	return tls.ConnectionState{}
+	return sess.ConnectionState().TLS.ConnectionState
 }

internal/engine/netx/quicdialer/connectionstate_go1.15.go

@@ -1,14 +0,0 @@
-// +build go1.15
-
-package quicdialer
-
-import (
-	"crypto/tls"
-
-	"github.com/lucas-clemente/quic-go"
-)
-
-// ConnectionState returns the ConnectionState of a QUIC Session.
-func ConnectionState(sess quic.EarlySession) tls.ConnectionState {
-	return sess.ConnectionState().ConnectionState
-}

internal/engine/ooapi/apis.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2021-02-26 15:45:50.431349269 +0100 CET m=+0.000196051
+// 2021-03-31 16:50:03.708459546 +0200 CEST m=+0.000228783
 
 package ooapi
 
@@ -12,8 +12,8 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel"
 )
 
-// CheckReportIDAPI implements the CheckReportID API.
+// simpleCheckReportIDAPI implements the CheckReportID API.
-type CheckReportIDAPI struct {
+type simpleCheckReportIDAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -21,28 +21,28 @@ type CheckReportIDAPI struct {
 	UserAgent    string       // optional
 }
 
-func (api *CheckReportIDAPI) baseURL() string {
+func (api *simpleCheckReportIDAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *CheckReportIDAPI) requestMaker() RequestMaker {
+func (api *simpleCheckReportIDAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *CheckReportIDAPI) jsonCodec() JSONCodec {
+func (api *simpleCheckReportIDAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *CheckReportIDAPI) httpClient() HTTPClient {
+func (api *simpleCheckReportIDAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -50,7 +50,7 @@ func (api *CheckReportIDAPI) httpClient() HTTPClient {
 }
 
 // Call calls the CheckReportID API.
-func (api *CheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error) {
+func (api *simpleCheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -62,8 +62,8 @@ func (api *CheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckReport
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// CheckInAPI implements the CheckIn API.
+// simpleCheckInAPI implements the CheckIn API.
-type CheckInAPI struct {
+type simpleCheckInAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -71,28 +71,28 @@ type CheckInAPI struct {
 	UserAgent    string       // optional
 }
 
-func (api *CheckInAPI) baseURL() string {
+func (api *simpleCheckInAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *CheckInAPI) requestMaker() RequestMaker {
+func (api *simpleCheckInAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *CheckInAPI) jsonCodec() JSONCodec {
+func (api *simpleCheckInAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *CheckInAPI) httpClient() HTTPClient {
+func (api *simpleCheckInAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -100,7 +100,7 @@ func (api *CheckInAPI) httpClient() HTTPClient {
 }
 
 // Call calls the CheckIn API.
-func (api *CheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error) {
+func (api *simpleCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -112,8 +112,8 @@ func (api *CheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequest) (
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// LoginAPI implements the Login API.
+// simpleLoginAPI implements the Login API.
-type LoginAPI struct {
+type simpleLoginAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -121,28 +121,28 @@ type LoginAPI struct {
 	UserAgent    string       // optional
 }
 
-func (api *LoginAPI) baseURL() string {
+func (api *simpleLoginAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *LoginAPI) requestMaker() RequestMaker {
+func (api *simpleLoginAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *LoginAPI) jsonCodec() JSONCodec {
+func (api *simpleLoginAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *LoginAPI) httpClient() HTTPClient {
+func (api *simpleLoginAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -150,7 +150,7 @@ func (api *LoginAPI) httpClient() HTTPClient {
 }
 
 // Call calls the Login API.
-func (api *LoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error) {
+func (api *simpleLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -162,8 +162,8 @@ func (api *LoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) (*api
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// MeasurementMetaAPI implements the MeasurementMeta API.
+// simpleMeasurementMetaAPI implements the MeasurementMeta API.
-type MeasurementMetaAPI struct {
+type simpleMeasurementMetaAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -171,28 +171,28 @@ type MeasurementMetaAPI struct {
 	UserAgent    string       // optional
 }
 
-func (api *MeasurementMetaAPI) baseURL() string {
+func (api *simpleMeasurementMetaAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *MeasurementMetaAPI) requestMaker() RequestMaker {
+func (api *simpleMeasurementMetaAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *MeasurementMetaAPI) jsonCodec() JSONCodec {
+func (api *simpleMeasurementMetaAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *MeasurementMetaAPI) httpClient() HTTPClient {
+func (api *simpleMeasurementMetaAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -200,7 +200,7 @@ func (api *MeasurementMetaAPI) httpClient() HTTPClient {
 }
 
 // Call calls the MeasurementMeta API.
-func (api *MeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
+func (api *simpleMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -212,8 +212,8 @@ func (api *MeasurementMetaAPI) Call(ctx context.Context, req *apimodel.Measureme
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// RegisterAPI implements the Register API.
+// simpleRegisterAPI implements the Register API.
-type RegisterAPI struct {
+type simpleRegisterAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -221,28 +221,28 @@ type RegisterAPI struct {
 	UserAgent    string       // optional
 }
 
-func (api *RegisterAPI) baseURL() string {
+func (api *simpleRegisterAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *RegisterAPI) requestMaker() RequestMaker {
+func (api *simpleRegisterAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *RegisterAPI) jsonCodec() JSONCodec {
+func (api *simpleRegisterAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *RegisterAPI) httpClient() HTTPClient {
+func (api *simpleRegisterAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -250,7 +250,7 @@ func (api *RegisterAPI) httpClient() HTTPClient {
 }
 
 // Call calls the Register API.
-func (api *RegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error) {
+func (api *simpleRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -262,8 +262,8 @@ func (api *RegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRequest)
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// TestHelpersAPI implements the TestHelpers API.
+// simpleTestHelpersAPI implements the TestHelpers API.
-type TestHelpersAPI struct {
+type simpleTestHelpersAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -271,28 +271,28 @@ type TestHelpersAPI struct {
 	UserAgent    string       // optional
 }
 
-func (api *TestHelpersAPI) baseURL() string {
+func (api *simpleTestHelpersAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *TestHelpersAPI) requestMaker() RequestMaker {
+func (api *simpleTestHelpersAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *TestHelpersAPI) jsonCodec() JSONCodec {
+func (api *simpleTestHelpersAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *TestHelpersAPI) httpClient() HTTPClient {
+func (api *simpleTestHelpersAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -300,7 +300,7 @@ func (api *TestHelpersAPI) httpClient() HTTPClient {
 }
 
 // Call calls the TestHelpers API.
-func (api *TestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error) {
+func (api *simpleTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -312,8 +312,8 @@ func (api *TestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelpersRe
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// PsiphonConfigAPI implements the PsiphonConfig API.
+// simplePsiphonConfigAPI implements the PsiphonConfig API.
-type PsiphonConfigAPI struct {
+type simplePsiphonConfigAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -324,8 +324,8 @@ type PsiphonConfigAPI struct {
 
 // WithToken returns a copy of the API where the
 // value of the Token field is replaced with token.
-func (api *PsiphonConfigAPI) WithToken(token string) PsiphonConfigCaller {
+func (api *simplePsiphonConfigAPI) WithToken(token string) callerForPsiphonConfigAPI {
-	out := &PsiphonConfigAPI{}
+	out := &simplePsiphonConfigAPI{}
 	out.BaseURL = api.BaseURL
 	out.HTTPClient = api.HTTPClient
 	out.JSONCodec = api.JSONCodec
@@ -335,28 +335,28 @@ func (api *PsiphonConfigAPI) WithToken(token string) PsiphonConfigCaller {
 	return out
 }
 
-func (api *PsiphonConfigAPI) baseURL() string {
+func (api *simplePsiphonConfigAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *PsiphonConfigAPI) requestMaker() RequestMaker {
+func (api *simplePsiphonConfigAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *PsiphonConfigAPI) jsonCodec() JSONCodec {
+func (api *simplePsiphonConfigAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *PsiphonConfigAPI) httpClient() HTTPClient {
+func (api *simplePsiphonConfigAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -364,7 +364,7 @@ func (api *PsiphonConfigAPI) httpClient() HTTPClient {
 }
 
 // Call calls the PsiphonConfig API.
-func (api *PsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) {
+func (api *simplePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -380,8 +380,8 @@ func (api *PsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConf
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// TorTargetsAPI implements the TorTargets API.
+// simpleTorTargetsAPI implements the TorTargets API.
-type TorTargetsAPI struct {
+type simpleTorTargetsAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -392,8 +392,8 @@ type TorTargetsAPI struct {
 
 // WithToken returns a copy of the API where the
 // value of the Token field is replaced with token.
-func (api *TorTargetsAPI) WithToken(token string) TorTargetsCaller {
+func (api *simpleTorTargetsAPI) WithToken(token string) callerForTorTargetsAPI {
-	out := &TorTargetsAPI{}
+	out := &simpleTorTargetsAPI{}
 	out.BaseURL = api.BaseURL
 	out.HTTPClient = api.HTTPClient
 	out.JSONCodec = api.JSONCodec
@@ -403,28 +403,28 @@ func (api *TorTargetsAPI) WithToken(token string) TorTargetsCaller {
 	return out
 }
 
-func (api *TorTargetsAPI) baseURL() string {
+func (api *simpleTorTargetsAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *TorTargetsAPI) requestMaker() RequestMaker {
+func (api *simpleTorTargetsAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *TorTargetsAPI) jsonCodec() JSONCodec {
+func (api *simpleTorTargetsAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *TorTargetsAPI) httpClient() HTTPClient {
+func (api *simpleTorTargetsAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -432,7 +432,7 @@ func (api *TorTargetsAPI) httpClient() HTTPClient {
 }
 
 // Call calls the TorTargets API.
-func (api *TorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) {
+func (api *simpleTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -448,8 +448,8 @@ func (api *TorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequ
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// URLsAPI implements the URLs API.
+// simpleURLsAPI implements the URLs API.
-type URLsAPI struct {
+type simpleURLsAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -457,28 +457,28 @@ type URLsAPI struct {
 	UserAgent    string       // optional
 }
 
-func (api *URLsAPI) baseURL() string {
+func (api *simpleURLsAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *URLsAPI) requestMaker() RequestMaker {
+func (api *simpleURLsAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *URLsAPI) jsonCodec() JSONCodec {
+func (api *simpleURLsAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *URLsAPI) httpClient() HTTPClient {
+func (api *simpleURLsAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -486,7 +486,7 @@ func (api *URLsAPI) httpClient() HTTPClient {
 }
 
 // Call calls the URLs API.
-func (api *URLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error) {
+func (api *simpleURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -498,8 +498,8 @@ func (api *URLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*apimo
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// OpenReportAPI implements the OpenReport API.
+// simpleOpenReportAPI implements the OpenReport API.
-type OpenReportAPI struct {
+type simpleOpenReportAPI struct {
 	BaseURL      string       // optional
 	HTTPClient   HTTPClient   // optional
 	JSONCodec    JSONCodec    // optional
@@ -507,28 +507,28 @@ type OpenReportAPI struct {
 	UserAgent    string       // optional
 }
 
-func (api *OpenReportAPI) baseURL() string {
+func (api *simpleOpenReportAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *OpenReportAPI) requestMaker() RequestMaker {
+func (api *simpleOpenReportAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *OpenReportAPI) jsonCodec() JSONCodec {
+func (api *simpleOpenReportAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *OpenReportAPI) httpClient() HTTPClient {
+func (api *simpleOpenReportAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -536,7 +536,7 @@ func (api *OpenReportAPI) httpClient() HTTPClient {
 }
 
 // Call calls the OpenReport API.
-func (api *OpenReportAPI) Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error) {
+func (api *simpleOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err
@@ -548,45 +548,45 @@ func (api *OpenReportAPI) Call(ctx context.Context, req *apimodel.OpenReportRequ
 	return api.newResponse(api.httpClient().Do(httpReq))
 }
 
-// SubmitMeasurementAPI implements the SubmitMeasurement API.
+// simpleSubmitMeasurementAPI implements the SubmitMeasurement API.
-type SubmitMeasurementAPI struct {
+type simpleSubmitMeasurementAPI struct {
 	BaseURL          string           // optional
 	HTTPClient       HTTPClient       // optional
 	JSONCodec        JSONCodec        // optional
 	RequestMaker     RequestMaker     // optional
-	TemplateExecutor TemplateExecutor // optional
+	TemplateExecutor templateExecutor // optional
 	UserAgent        string           // optional
 }
 
-func (api *SubmitMeasurementAPI) baseURL() string {
+func (api *simpleSubmitMeasurementAPI) baseURL() string {
 	if api.BaseURL != "" {
 		return api.BaseURL
 	}
 	return "https://ps1.ooni.io"
 }
 
-func (api *SubmitMeasurementAPI) requestMaker() RequestMaker {
+func (api *simpleSubmitMeasurementAPI) requestMaker() RequestMaker {
 	if api.RequestMaker != nil {
 		return api.RequestMaker
 	}
 	return &defaultRequestMaker{}
 }
 
-func (api *SubmitMeasurementAPI) jsonCodec() JSONCodec {
+func (api *simpleSubmitMeasurementAPI) jsonCodec() JSONCodec {
 	if api.JSONCodec != nil {
 		return api.JSONCodec
 	}
 	return &defaultJSONCodec{}
 }
 
-func (api *SubmitMeasurementAPI) templateExecutor() TemplateExecutor {
+func (api *simpleSubmitMeasurementAPI) templateExecutor() templateExecutor {
 	if api.TemplateExecutor != nil {
 		return api.TemplateExecutor
 	}
 	return &defaultTemplateExecutor{}
 }
 
-func (api *SubmitMeasurementAPI) httpClient() HTTPClient {
+func (api *simpleSubmitMeasurementAPI) httpClient() HTTPClient {
 	if api.HTTPClient != nil {
 		return api.HTTPClient
 	}
@@ -594,7 +594,7 @@ func (api *SubmitMeasurementAPI) httpClient() HTTPClient {
 }
 
 // Call calls the SubmitMeasurement API.
-func (api *SubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error) {
+func (api *simpleSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error) {
 	httpReq, err := api.newRequest(ctx, req)
 	if err != nil {
 		return nil, err

internal/engine/ooapi/apis_test.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2021-02-26 15:45:50.81792142 +0100 CET m=+0.000095792
+// 2021-03-31 16:50:04.057051721 +0200 CEST m=+0.000093878
 
 package ooapi
 
@@ -22,7 +22,7 @@ import (
 )
 
 func TestCheckReportIDInvalidURL(t *testing.T) {
-	api := &CheckReportIDAPI{
+	api := &simpleCheckReportIDAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -41,7 +41,7 @@ func TestCheckReportIDInvalidURL(t *testing.T) {
 func TestCheckReportIDWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &CheckReportIDAPI{
+	api := &simpleCheckReportIDAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -59,7 +59,7 @@ func TestCheckReportIDWithHTTPErr(t *testing.T) {
 
 func TestCheckReportIDWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &CheckReportIDAPI{
+	api := &simpleCheckReportIDAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -77,7 +77,7 @@ func TestCheckReportIDWithNewRequestErr(t *testing.T) {
 
 func TestCheckReportIDWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &CheckReportIDAPI{
+	api := &simpleCheckReportIDAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -95,7 +95,7 @@ func TestCheckReportIDWith401(t *testing.T) {
 
 func TestCheckReportIDWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &CheckReportIDAPI{
+	api := &simpleCheckReportIDAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -117,7 +117,7 @@ func TestCheckReportIDWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &CheckReportIDAPI{
+	api := &simpleCheckReportIDAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -139,7 +139,7 @@ func TestCheckReportIDWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &CheckReportIDAPI{
+	api := &simpleCheckReportIDAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -209,7 +209,7 @@ func TestCheckReportIDRoundTrip(t *testing.T) {
 	req := &apimodel.CheckReportIDRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &CheckReportIDAPI{BaseURL: srvr.URL}
+	api := &simpleCheckReportIDAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -252,7 +252,7 @@ func TestCheckReportIDMandatoryFields(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{
 		StatusCode: 500,
 	}}
-	api := &CheckReportIDAPI{
+	api := &simpleCheckReportIDAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -267,7 +267,7 @@ func TestCheckReportIDMandatoryFields(t *testing.T) {
 }
 
 func TestCheckInInvalidURL(t *testing.T) {
-	api := &CheckInAPI{
+	api := &simpleCheckInAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -286,7 +286,7 @@ func TestCheckInInvalidURL(t *testing.T) {
 func TestCheckInWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &CheckInAPI{
+	api := &simpleCheckInAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -304,7 +304,7 @@ func TestCheckInWithHTTPErr(t *testing.T) {
 
 func TestCheckInMarshalErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &CheckInAPI{
+	api := &simpleCheckInAPI{
 		JSONCodec: &FakeCodec{EncodeErr: errMocked},
 	}
 	ctx := context.Background()
@@ -322,7 +322,7 @@ func TestCheckInMarshalErr(t *testing.T) {
 
 func TestCheckInWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &CheckInAPI{
+	api := &simpleCheckInAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -340,7 +340,7 @@ func TestCheckInWithNewRequestErr(t *testing.T) {
 
 func TestCheckInWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &CheckInAPI{
+	api := &simpleCheckInAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -358,7 +358,7 @@ func TestCheckInWith401(t *testing.T) {
 
 func TestCheckInWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &CheckInAPI{
+	api := &simpleCheckInAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -380,7 +380,7 @@ func TestCheckInWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &CheckInAPI{
+	api := &simpleCheckInAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -402,7 +402,7 @@ func TestCheckInWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &CheckInAPI{
+	api := &simpleCheckInAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -472,7 +472,7 @@ func TestCheckInRoundTrip(t *testing.T) {
 	req := &apimodel.CheckInRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &CheckInAPI{BaseURL: srvr.URL}
+	api := &simpleCheckInAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -512,7 +512,7 @@ func TestCheckInRoundTrip(t *testing.T) {
 }
 
 func TestLoginInvalidURL(t *testing.T) {
-	api := &LoginAPI{
+	api := &simpleLoginAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -531,7 +531,7 @@ func TestLoginInvalidURL(t *testing.T) {
 func TestLoginWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &LoginAPI{
+	api := &simpleLoginAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -549,7 +549,7 @@ func TestLoginWithHTTPErr(t *testing.T) {
 
 func TestLoginMarshalErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &LoginAPI{
+	api := &simpleLoginAPI{
 		JSONCodec: &FakeCodec{EncodeErr: errMocked},
 	}
 	ctx := context.Background()
@@ -567,7 +567,7 @@ func TestLoginMarshalErr(t *testing.T) {
 
 func TestLoginWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &LoginAPI{
+	api := &simpleLoginAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -585,7 +585,7 @@ func TestLoginWithNewRequestErr(t *testing.T) {
 
 func TestLoginWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &LoginAPI{
+	api := &simpleLoginAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -603,7 +603,7 @@ func TestLoginWith401(t *testing.T) {
 
 func TestLoginWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &LoginAPI{
+	api := &simpleLoginAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -625,7 +625,7 @@ func TestLoginWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &LoginAPI{
+	api := &simpleLoginAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -647,7 +647,7 @@ func TestLoginWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &LoginAPI{
+	api := &simpleLoginAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -717,7 +717,7 @@ func TestLoginRoundTrip(t *testing.T) {
 	req := &apimodel.LoginRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &LoginAPI{BaseURL: srvr.URL}
+	api := &simpleLoginAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -757,7 +757,7 @@ func TestLoginRoundTrip(t *testing.T) {
 }
 
 func TestMeasurementMetaInvalidURL(t *testing.T) {
-	api := &MeasurementMetaAPI{
+	api := &simpleMeasurementMetaAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -776,7 +776,7 @@ func TestMeasurementMetaInvalidURL(t *testing.T) {
 func TestMeasurementMetaWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &MeasurementMetaAPI{
+	api := &simpleMeasurementMetaAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -794,7 +794,7 @@ func TestMeasurementMetaWithHTTPErr(t *testing.T) {
 
 func TestMeasurementMetaWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &MeasurementMetaAPI{
+	api := &simpleMeasurementMetaAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -812,7 +812,7 @@ func TestMeasurementMetaWithNewRequestErr(t *testing.T) {
 
 func TestMeasurementMetaWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &MeasurementMetaAPI{
+	api := &simpleMeasurementMetaAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -830,7 +830,7 @@ func TestMeasurementMetaWith401(t *testing.T) {
 
 func TestMeasurementMetaWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &MeasurementMetaAPI{
+	api := &simpleMeasurementMetaAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -852,7 +852,7 @@ func TestMeasurementMetaWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &MeasurementMetaAPI{
+	api := &simpleMeasurementMetaAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -874,7 +874,7 @@ func TestMeasurementMetaWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &MeasurementMetaAPI{
+	api := &simpleMeasurementMetaAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -944,7 +944,7 @@ func TestMeasurementMetaRoundTrip(t *testing.T) {
 	req := &apimodel.MeasurementMetaRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &MeasurementMetaAPI{BaseURL: srvr.URL}
+	api := &simpleMeasurementMetaAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -987,7 +987,7 @@ func TestMeasurementMetaMandatoryFields(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{
 		StatusCode: 500,
 	}}
-	api := &MeasurementMetaAPI{
+	api := &simpleMeasurementMetaAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1002,7 +1002,7 @@ func TestMeasurementMetaMandatoryFields(t *testing.T) {
 }
 
 func TestRegisterInvalidURL(t *testing.T) {
-	api := &RegisterAPI{
+	api := &simpleRegisterAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -1021,7 +1021,7 @@ func TestRegisterInvalidURL(t *testing.T) {
 func TestRegisterWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &RegisterAPI{
+	api := &simpleRegisterAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1039,7 +1039,7 @@ func TestRegisterWithHTTPErr(t *testing.T) {
 
 func TestRegisterMarshalErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &RegisterAPI{
+	api := &simpleRegisterAPI{
 		JSONCodec: &FakeCodec{EncodeErr: errMocked},
 	}
 	ctx := context.Background()
@@ -1057,7 +1057,7 @@ func TestRegisterMarshalErr(t *testing.T) {
 
 func TestRegisterWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &RegisterAPI{
+	api := &simpleRegisterAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -1075,7 +1075,7 @@ func TestRegisterWithNewRequestErr(t *testing.T) {
 
 func TestRegisterWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &RegisterAPI{
+	api := &simpleRegisterAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1093,7 +1093,7 @@ func TestRegisterWith401(t *testing.T) {
 
 func TestRegisterWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &RegisterAPI{
+	api := &simpleRegisterAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1115,7 +1115,7 @@ func TestRegisterWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &RegisterAPI{
+	api := &simpleRegisterAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1137,7 +1137,7 @@ func TestRegisterWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &RegisterAPI{
+	api := &simpleRegisterAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -1207,7 +1207,7 @@ func TestRegisterRoundTrip(t *testing.T) {
 	req := &apimodel.RegisterRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &RegisterAPI{BaseURL: srvr.URL}
+	api := &simpleRegisterAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -1247,7 +1247,7 @@ func TestRegisterRoundTrip(t *testing.T) {
 }
 
 func TestTestHelpersInvalidURL(t *testing.T) {
-	api := &TestHelpersAPI{
+	api := &simpleTestHelpersAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -1266,7 +1266,7 @@ func TestTestHelpersInvalidURL(t *testing.T) {
 func TestTestHelpersWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &TestHelpersAPI{
+	api := &simpleTestHelpersAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1284,7 +1284,7 @@ func TestTestHelpersWithHTTPErr(t *testing.T) {
 
 func TestTestHelpersWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &TestHelpersAPI{
+	api := &simpleTestHelpersAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -1302,7 +1302,7 @@ func TestTestHelpersWithNewRequestErr(t *testing.T) {
 
 func TestTestHelpersWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &TestHelpersAPI{
+	api := &simpleTestHelpersAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1320,7 +1320,7 @@ func TestTestHelpersWith401(t *testing.T) {
 
 func TestTestHelpersWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &TestHelpersAPI{
+	api := &simpleTestHelpersAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1342,7 +1342,7 @@ func TestTestHelpersWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &TestHelpersAPI{
+	api := &simpleTestHelpersAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1364,7 +1364,7 @@ func TestTestHelpersWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &TestHelpersAPI{
+	api := &simpleTestHelpersAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -1434,7 +1434,7 @@ func TestTestHelpersRoundTrip(t *testing.T) {
 	req := &apimodel.TestHelpersRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &TestHelpersAPI{BaseURL: srvr.URL}
+	api := &simpleTestHelpersAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -1478,7 +1478,7 @@ func TestTestHelpersResponseLiteralNull(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`null`)},
 	}}
-	api := &TestHelpersAPI{
+	api := &simpleTestHelpersAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -1495,7 +1495,7 @@ func TestTestHelpersResponseLiteralNull(t *testing.T) {
 }
 
 func TestPsiphonConfigInvalidURL(t *testing.T) {
-	api := &PsiphonConfigAPI{
+	api := &simplePsiphonConfigAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -1512,7 +1512,7 @@ func TestPsiphonConfigInvalidURL(t *testing.T) {
 }
 
 func TestPsiphonConfigWithMissingToken(t *testing.T) {
-	api := &PsiphonConfigAPI{} // no token
+	api := &simplePsiphonConfigAPI{} // no token
 	ctx := context.Background()
 	req := &apimodel.PsiphonConfigRequest{}
 	ff := &fakeFill{}
@@ -1529,7 +1529,7 @@ func TestPsiphonConfigWithMissingToken(t *testing.T) {
 func TestPsiphonConfigWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &PsiphonConfigAPI{
+	api := &simplePsiphonConfigAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1548,7 +1548,7 @@ func TestPsiphonConfigWithHTTPErr(t *testing.T) {
 
 func TestPsiphonConfigWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &PsiphonConfigAPI{
+	api := &simplePsiphonConfigAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 		Token:        "fakeToken",
 	}
@@ -1567,7 +1567,7 @@ func TestPsiphonConfigWithNewRequestErr(t *testing.T) {
 
 func TestPsiphonConfigWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &PsiphonConfigAPI{
+	api := &simplePsiphonConfigAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1586,7 +1586,7 @@ func TestPsiphonConfigWith401(t *testing.T) {
 
 func TestPsiphonConfigWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &PsiphonConfigAPI{
+	api := &simplePsiphonConfigAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1609,7 +1609,7 @@ func TestPsiphonConfigWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &PsiphonConfigAPI{
+	api := &simplePsiphonConfigAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1632,7 +1632,7 @@ func TestPsiphonConfigWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &PsiphonConfigAPI{
+	api := &simplePsiphonConfigAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 		Token:      "fakeToken",
@@ -1703,7 +1703,7 @@ func TestPsiphonConfigRoundTrip(t *testing.T) {
 	req := &apimodel.PsiphonConfigRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &PsiphonConfigAPI{BaseURL: srvr.URL}
+	api := &simplePsiphonConfigAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	ff.fill(&api.Token)
 	// issue request
@@ -1748,7 +1748,7 @@ func TestPsiphonConfigResponseLiteralNull(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`null`)},
 	}}
-	api := &PsiphonConfigAPI{
+	api := &simplePsiphonConfigAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1766,7 +1766,7 @@ func TestPsiphonConfigResponseLiteralNull(t *testing.T) {
 }
 
 func TestTorTargetsInvalidURL(t *testing.T) {
-	api := &TorTargetsAPI{
+	api := &simpleTorTargetsAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -1783,7 +1783,7 @@ func TestTorTargetsInvalidURL(t *testing.T) {
 }
 
 func TestTorTargetsWithMissingToken(t *testing.T) {
-	api := &TorTargetsAPI{} // no token
+	api := &simpleTorTargetsAPI{} // no token
 	ctx := context.Background()
 	req := &apimodel.TorTargetsRequest{}
 	ff := &fakeFill{}
@@ -1800,7 +1800,7 @@ func TestTorTargetsWithMissingToken(t *testing.T) {
 func TestTorTargetsWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &TorTargetsAPI{
+	api := &simpleTorTargetsAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1819,7 +1819,7 @@ func TestTorTargetsWithHTTPErr(t *testing.T) {
 
 func TestTorTargetsWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &TorTargetsAPI{
+	api := &simpleTorTargetsAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 		Token:        "fakeToken",
 	}
@@ -1838,7 +1838,7 @@ func TestTorTargetsWithNewRequestErr(t *testing.T) {
 
 func TestTorTargetsWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &TorTargetsAPI{
+	api := &simpleTorTargetsAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1857,7 +1857,7 @@ func TestTorTargetsWith401(t *testing.T) {
 
 func TestTorTargetsWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &TorTargetsAPI{
+	api := &simpleTorTargetsAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1880,7 +1880,7 @@ func TestTorTargetsWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &TorTargetsAPI{
+	api := &simpleTorTargetsAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -1903,7 +1903,7 @@ func TestTorTargetsWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &TorTargetsAPI{
+	api := &simpleTorTargetsAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 		Token:      "fakeToken",
@@ -1974,7 +1974,7 @@ func TestTorTargetsRoundTrip(t *testing.T) {
 	req := &apimodel.TorTargetsRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &TorTargetsAPI{BaseURL: srvr.URL}
+	api := &simpleTorTargetsAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	ff.fill(&api.Token)
 	// issue request
@@ -2019,7 +2019,7 @@ func TestTorTargetsResponseLiteralNull(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`null`)},
 	}}
-	api := &TorTargetsAPI{
+	api := &simpleTorTargetsAPI{
 		HTTPClient: clnt,
 		Token:      "fakeToken",
 	}
@@ -2037,7 +2037,7 @@ func TestTorTargetsResponseLiteralNull(t *testing.T) {
 }
 
 func TestURLsInvalidURL(t *testing.T) {
-	api := &URLsAPI{
+	api := &simpleURLsAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -2056,7 +2056,7 @@ func TestURLsInvalidURL(t *testing.T) {
 func TestURLsWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &URLsAPI{
+	api := &simpleURLsAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2074,7 +2074,7 @@ func TestURLsWithHTTPErr(t *testing.T) {
 
 func TestURLsWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &URLsAPI{
+	api := &simpleURLsAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -2092,7 +2092,7 @@ func TestURLsWithNewRequestErr(t *testing.T) {
 
 func TestURLsWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &URLsAPI{
+	api := &simpleURLsAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2110,7 +2110,7 @@ func TestURLsWith401(t *testing.T) {
 
 func TestURLsWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &URLsAPI{
+	api := &simpleURLsAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2132,7 +2132,7 @@ func TestURLsWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &URLsAPI{
+	api := &simpleURLsAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2154,7 +2154,7 @@ func TestURLsWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &URLsAPI{
+	api := &simpleURLsAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -2224,7 +2224,7 @@ func TestURLsRoundTrip(t *testing.T) {
 	req := &apimodel.URLsRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &URLsAPI{BaseURL: srvr.URL}
+	api := &simpleURLsAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -2264,7 +2264,7 @@ func TestURLsRoundTrip(t *testing.T) {
 }
 
 func TestOpenReportInvalidURL(t *testing.T) {
-	api := &OpenReportAPI{
+	api := &simpleOpenReportAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -2283,7 +2283,7 @@ func TestOpenReportInvalidURL(t *testing.T) {
 func TestOpenReportWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &OpenReportAPI{
+	api := &simpleOpenReportAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2301,7 +2301,7 @@ func TestOpenReportWithHTTPErr(t *testing.T) {
 
 func TestOpenReportMarshalErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &OpenReportAPI{
+	api := &simpleOpenReportAPI{
 		JSONCodec: &FakeCodec{EncodeErr: errMocked},
 	}
 	ctx := context.Background()
@@ -2319,7 +2319,7 @@ func TestOpenReportMarshalErr(t *testing.T) {
 
 func TestOpenReportWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &OpenReportAPI{
+	api := &simpleOpenReportAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -2337,7 +2337,7 @@ func TestOpenReportWithNewRequestErr(t *testing.T) {
 
 func TestOpenReportWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &OpenReportAPI{
+	api := &simpleOpenReportAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2355,7 +2355,7 @@ func TestOpenReportWith401(t *testing.T) {
 
 func TestOpenReportWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &OpenReportAPI{
+	api := &simpleOpenReportAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2377,7 +2377,7 @@ func TestOpenReportWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &OpenReportAPI{
+	api := &simpleOpenReportAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2399,7 +2399,7 @@ func TestOpenReportWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &OpenReportAPI{
+	api := &simpleOpenReportAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -2469,7 +2469,7 @@ func TestOpenReportRoundTrip(t *testing.T) {
 	req := &apimodel.OpenReportRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &OpenReportAPI{BaseURL: srvr.URL}
+	api := &simpleOpenReportAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -2509,7 +2509,7 @@ func TestOpenReportRoundTrip(t *testing.T) {
 }
 
 func TestSubmitMeasurementInvalidURL(t *testing.T) {
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		BaseURL: "\t", // invalid
 	}
 	ctx := context.Background()
@@ -2528,7 +2528,7 @@ func TestSubmitMeasurementInvalidURL(t *testing.T) {
 func TestSubmitMeasurementWithHTTPErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	clnt := &FakeHTTPClient{Err: errMocked}
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2546,7 +2546,7 @@ func TestSubmitMeasurementWithHTTPErr(t *testing.T) {
 
 func TestSubmitMeasurementMarshalErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		JSONCodec: &FakeCodec{EncodeErr: errMocked},
 	}
 	ctx := context.Background()
@@ -2564,7 +2564,7 @@ func TestSubmitMeasurementMarshalErr(t *testing.T) {
 
 func TestSubmitMeasurementWithNewRequestErr(t *testing.T) {
 	errMocked := errors.New("mocked error")
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		RequestMaker: &FakeRequestMaker{Err: errMocked},
 	}
 	ctx := context.Background()
@@ -2582,7 +2582,7 @@ func TestSubmitMeasurementWithNewRequestErr(t *testing.T) {
 
 func TestSubmitMeasurementWith401(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2600,7 +2600,7 @@ func TestSubmitMeasurementWith401(t *testing.T) {
 
 func TestSubmitMeasurementWith400(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2622,7 +2622,7 @@ func TestSubmitMeasurementWithResponseBodyReadErr(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Err: errMocked},
 	}}
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		HTTPClient: clnt,
 	}
 	ctx := context.Background()
@@ -2644,7 +2644,7 @@ func TestSubmitMeasurementWithUnmarshalFailure(t *testing.T) {
 		StatusCode: 200,
 		Body:       &FakeBody{Data: []byte(`{}`)},
 	}}
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		HTTPClient: clnt,
 		JSONCodec:  &FakeCodec{DecodeErr: errMocked},
 	}
@@ -2714,7 +2714,7 @@ func TestSubmitMeasurementRoundTrip(t *testing.T) {
 	req := &apimodel.SubmitMeasurementRequest{}
 	ff := &fakeFill{}
 	ff.fill(&req)
-	api := &SubmitMeasurementAPI{BaseURL: srvr.URL}
+	api := &simpleSubmitMeasurementAPI{BaseURL: srvr.URL}
 	ff.fill(&api.UserAgent)
 	// issue request
 	ctx := context.Background()
@@ -2758,7 +2758,7 @@ func TestSubmitMeasurementTemplateErr(t *testing.T) {
 	clnt := &FakeHTTPClient{Resp: &http.Response{
 		StatusCode: 500,
 	}}
-	api := &SubmitMeasurementAPI{
+	api := &simpleSubmitMeasurementAPI{
 		HTTPClient:       clnt,
 		TemplateExecutor: &FakeTemplateExecutor{Err: errMocked},
 	}

internal/engine/ooapi/caching.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2021-02-26 15:45:51.194159684 +0100 CET m=+0.000175181
+// 2021-03-31 16:50:04.317557242 +0200 CEST m=+0.000161626
 
 package ooapi
 
@@ -12,20 +12,20 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel"
 )
 
-// MeasurementMetaCache implements caching for MeasurementMetaAPI.
+// withCacheMeasurementMetaAPI implements caching for simpleMeasurementMetaAPI.
-type MeasurementMetaCache struct {
+type withCacheMeasurementMetaAPI struct {
-	API      MeasurementMetaCaller // mandatory
+	API      callerForMeasurementMetaAPI // mandatory
-	GobCodec GobCodec              // optional
+	GobCodec GobCodec                    // optional
-	KVStore  KVStore               // mandatory
+	KVStore  KVStore                     // mandatory
 }
 
-type cacheEntryForMeasurementMeta struct {
+type cacheEntryForMeasurementMetaAPI struct {
 	Req  *apimodel.MeasurementMetaRequest
 	Resp *apimodel.MeasurementMetaResponse
 }
 
 // Call calls the API and implements caching.
-func (c *MeasurementMetaCache) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
+func (c *withCacheMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
 	if resp, _ := c.readcache(req); resp != nil {
 		return resp, nil
 	}
@@ -39,26 +39,26 @@ func (c *MeasurementMetaCache) Call(ctx context.Context, req *apimodel.Measureme
 	return resp, nil
 }
 
-func (c *MeasurementMetaCache) gobCodec() GobCodec {
+func (c *withCacheMeasurementMetaAPI) gobCodec() GobCodec {
 	if c.GobCodec != nil {
 		return c.GobCodec
 	}
 	return &defaultGobCodec{}
 }
 
-func (c *MeasurementMetaCache) getcache() ([]cacheEntryForMeasurementMeta, error) {
+func (c *withCacheMeasurementMetaAPI) getcache() ([]cacheEntryForMeasurementMetaAPI, error) {
 	data, err := c.KVStore.Get("MeasurementMeta.cache")
 	if err != nil {
 		return nil, err
 	}
-	var out []cacheEntryForMeasurementMeta
+	var out []cacheEntryForMeasurementMetaAPI
 	if err := c.gobCodec().Decode(data, &out); err != nil {
 		return nil, err
 	}
 	return out, nil
 }
 
-func (c *MeasurementMetaCache) setcache(in []cacheEntryForMeasurementMeta) error {
+func (c *withCacheMeasurementMetaAPI) setcache(in []cacheEntryForMeasurementMetaAPI) error {
 	data, err := c.gobCodec().Encode(in)
 	if err != nil {
 		return err
@@ -66,7 +66,7 @@ func (c *MeasurementMetaCache) setcache(in []cacheEntryForMeasurementMeta) error
 	return c.KVStore.Set("MeasurementMeta.cache", data)
 }
 
-func (c *MeasurementMetaCache) readcache(req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
+func (c *withCacheMeasurementMetaAPI) readcache(req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
 	cache, err := c.getcache()
 	if err != nil {
 		return nil, err
@@ -79,9 +79,9 @@ func (c *MeasurementMetaCache) readcache(req *apimodel.MeasurementMetaRequest) (
 	return nil, errCacheNotFound
 }
 
-func (c *MeasurementMetaCache) writecache(req *apimodel.MeasurementMetaRequest, resp *apimodel.MeasurementMetaResponse) error {
+func (c *withCacheMeasurementMetaAPI) writecache(req *apimodel.MeasurementMetaRequest, resp *apimodel.MeasurementMetaResponse) error {
 	cache, _ := c.getcache()
-	out := []cacheEntryForMeasurementMeta{{Req: req, Resp: resp}}
+	out := []cacheEntryForMeasurementMetaAPI{{Req: req, Resp: resp}}
 	const toomany = 64
 	for idx, cur := range cache {
 		if reflect.DeepEqual(req, cur.Req) {
@@ -95,4 +95,4 @@ func (c *MeasurementMetaCache) writecache(req *apimodel.MeasurementMetaRequest,
 	return c.setcache(out)
 }
 
-var _ MeasurementMetaCaller = &MeasurementMetaCache{}
+var _ callerForMeasurementMetaAPI = &withCacheMeasurementMetaAPI{}

internal/engine/ooapi/caching_test.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2021-02-26 15:45:51.49660021 +0100 CET m=+0.000217672
+// 2021-03-31 16:50:04.580341787 +0200 CEST m=+0.000148802
 
 package ooapi
 
@@ -14,15 +14,15 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel"
 )
 
-func TestCacheMeasurementMetaAPISuccess(t *testing.T) {
+func TestCachesimpleMeasurementMetaAPISuccess(t *testing.T) {
 	ff := &fakeFill{}
 	var expect *apimodel.MeasurementMetaResponse
 	ff.fill(&expect)
-	cache := &MeasurementMetaCache{
+	cache := &withCacheMeasurementMetaAPI{
 		API: &FakeMeasurementMetaAPI{
 			Response: expect,
 		},
-		KVStore: &memkvstore{},
+		KVStore: &MemKVStore{},
 	}
 	var req *apimodel.MeasurementMetaRequest
 	ff.fill(&req)
@@ -39,12 +39,12 @@ func TestCacheMeasurementMetaAPISuccess(t *testing.T) {
 	}
 }
 
-func TestCacheMeasurementMetaAPIWriteCacheError(t *testing.T) {
+func TestCachesimpleMeasurementMetaAPIWriteCacheError(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	ff := &fakeFill{}
 	var expect *apimodel.MeasurementMetaResponse
 	ff.fill(&expect)
-	cache := &MeasurementMetaCache{
+	cache := &withCacheMeasurementMetaAPI{
 		API: &FakeMeasurementMetaAPI{
 			Response: expect,
 		},
@@ -62,14 +62,14 @@ func TestCacheMeasurementMetaAPIWriteCacheError(t *testing.T) {
 	}
 }
 
-func TestCacheMeasurementMetaAPIFailureWithNoCache(t *testing.T) {
+func TestCachesimpleMeasurementMetaAPIFailureWithNoCache(t *testing.T) {
 	errMocked := errors.New("mocked error")
 	ff := &fakeFill{}
-	cache := &MeasurementMetaCache{
+	cache := &withCacheMeasurementMetaAPI{
 		API: &FakeMeasurementMetaAPI{
 			Err: errMocked,
 		},
-		KVStore: &memkvstore{},
+		KVStore: &MemKVStore{},
 	}
 	var req *apimodel.MeasurementMetaRequest
 	ff.fill(&req)
@@ -83,16 +83,16 @@ func TestCacheMeasurementMetaAPIFailureWithNoCache(t *testing.T) {
 	}
 }
 
-func TestCacheMeasurementMetaAPIFailureWithPreviousCache(t *testing.T) {
+func TestCachesimpleMeasurementMetaAPIFailureWithPreviousCache(t *testing.T) {
 	ff := &fakeFill{}
 	var expect *apimodel.MeasurementMetaResponse
 	ff.fill(&expect)
 	fakeapi := &FakeMeasurementMetaAPI{
 		Response: expect,
 	}
-	cache := &MeasurementMetaCache{
+	cache := &withCacheMeasurementMetaAPI{
 		API:     fakeapi,
-		KVStore: &memkvstore{},
+		KVStore: &MemKVStore{},
 	}
 	var req *apimodel.MeasurementMetaRequest
 	ff.fill(&req)
@@ -127,12 +127,12 @@ func TestCacheMeasurementMetaAPIFailureWithPreviousCache(t *testing.T) {
 	}
 }
 
-func TestCacheMeasurementMetaAPISetcacheWithEncodeError(t *testing.T) {
+func TestCachesimpleMeasurementMetaAPISetcacheWithEncodeError(t *testing.T) {
 	ff := &fakeFill{}
 	errMocked := errors.New("mocked error")
-	var in []cacheEntryForMeasurementMeta
+	var in []cacheEntryForMeasurementMetaAPI
 	ff.fill(&in)
-	cache := &MeasurementMetaCache{
+	cache := &withCacheMeasurementMetaAPI{
 		GobCodec: &FakeCodec{EncodeErr: errMocked},
 	}
 	err := cache.setcache(in)
@@ -141,12 +141,12 @@ func TestCacheMeasurementMetaAPISetcacheWithEncodeError(t *testing.T) {
 	}
 }
 
-func TestCacheMeasurementMetaAPIReadCacheNotFound(t *testing.T) {
+func TestCachesimpleMeasurementMetaAPIReadCacheNotFound(t *testing.T) {
 	ff := &fakeFill{}
-	var incache []cacheEntryForMeasurementMeta
+	var incache []cacheEntryForMeasurementMetaAPI
 	ff.fill(&incache)
-	cache := &MeasurementMetaCache{
+	cache := &withCacheMeasurementMetaAPI{
-		KVStore: &memkvstore{},
+		KVStore: &MemKVStore{},
 	}
 	err := cache.setcache(incache)
 	if err != nil {
@@ -163,7 +163,7 @@ func TestCacheMeasurementMetaAPIReadCacheNotFound(t *testing.T) {
 	}
 }
 
-func TestCacheMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) {
+func TestCachesimpleMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) {
 	ff := &fakeFill{}
 	var req *apimodel.MeasurementMetaRequest
 	ff.fill(&req)
@@ -171,8 +171,8 @@ func TestCacheMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) {
 	ff.fill(&resp1)
 	var resp2 *apimodel.MeasurementMetaResponse
 	ff.fill(&resp2)
-	cache := &MeasurementMetaCache{
+	cache := &withCacheMeasurementMetaAPI{
-		KVStore: &memkvstore{},
+		KVStore: &MemKVStore{},
 	}
 	err := cache.writecache(req, resp1)
 	if err != nil {
@@ -194,10 +194,10 @@ func TestCacheMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) {
 	}
 }
 
-func TestCacheMeasurementMetaAPICacheSizeLimited(t *testing.T) {
+func TestCachesimpleMeasurementMetaAPICacheSizeLimited(t *testing.T) {
 	ff := &fakeFill{}
-	cache := &MeasurementMetaCache{
+	cache := &withCacheMeasurementMetaAPI{
-		KVStore: &memkvstore{},
+		KVStore: &MemKVStore{},
 	}
 	var prev int
 	for {

internal/engine/ooapi/callers.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2021-02-26 15:45:51.773813223 +0100 CET m=+0.000114768
+// 2021-03-31 16:50:04.79291318 +0200 CEST m=+0.000077728
 
 package ooapi
 
@@ -11,68 +11,68 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel"
 )
 
-// CheckReportIDCaller represents any type exposing a method
+// callerForCheckReportIDAPI represents any type exposing a method
-// like CheckReportIDAPI.Call.
+// like simpleCheckReportIDAPI.Call.
-type CheckReportIDCaller interface {
+type callerForCheckReportIDAPI interface {
 	Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error)
 }
 
-// CheckInCaller represents any type exposing a method
+// callerForCheckInAPI represents any type exposing a method
-// like CheckInAPI.Call.
+// like simpleCheckInAPI.Call.
-type CheckInCaller interface {
+type callerForCheckInAPI interface {
 	Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error)
 }
 
-// LoginCaller represents any type exposing a method
+// callerForLoginAPI represents any type exposing a method
-// like LoginAPI.Call.
+// like simpleLoginAPI.Call.
-type LoginCaller interface {
+type callerForLoginAPI interface {
 	Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error)
 }
 
-// MeasurementMetaCaller represents any type exposing a method
+// callerForMeasurementMetaAPI represents any type exposing a method
-// like MeasurementMetaAPI.Call.
+// like simpleMeasurementMetaAPI.Call.
-type MeasurementMetaCaller interface {
+type callerForMeasurementMetaAPI interface {
 	Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error)
 }
 
-// RegisterCaller represents any type exposing a method
+// callerForRegisterAPI represents any type exposing a method
-// like RegisterAPI.Call.
+// like simpleRegisterAPI.Call.
-type RegisterCaller interface {
+type callerForRegisterAPI interface {
 	Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error)
 }
 
-// TestHelpersCaller represents any type exposing a method
+// callerForTestHelpersAPI represents any type exposing a method
-// like TestHelpersAPI.Call.
+// like simpleTestHelpersAPI.Call.
-type TestHelpersCaller interface {
+type callerForTestHelpersAPI interface {
 	Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error)
 }
 
-// PsiphonConfigCaller represents any type exposing a method
+// callerForPsiphonConfigAPI represents any type exposing a method
-// like PsiphonConfigAPI.Call.
+// like simplePsiphonConfigAPI.Call.
-type PsiphonConfigCaller interface {
+type callerForPsiphonConfigAPI interface {
 	Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error)
 }
 
-// TorTargetsCaller represents any type exposing a method
+// callerForTorTargetsAPI represents any type exposing a method
-// like TorTargetsAPI.Call.
+// like simpleTorTargetsAPI.Call.
-type TorTargetsCaller interface {
+type callerForTorTargetsAPI interface {
 	Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error)
 }
 
-// URLsCaller represents any type exposing a method
+// callerForURLsAPI represents any type exposing a method
-// like URLsAPI.Call.
+// like simpleURLsAPI.Call.
-type URLsCaller interface {
+type callerForURLsAPI interface {
 	Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error)
 }
 
-// OpenReportCaller represents any type exposing a method
+// callerForOpenReportAPI represents any type exposing a method
-// like OpenReportAPI.Call.
+// like simpleOpenReportAPI.Call.
-type OpenReportCaller interface {
+type callerForOpenReportAPI interface {
 	Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error)
 }
 
-// SubmitMeasurementCaller represents any type exposing a method
+// callerForSubmitMeasurementAPI represents any type exposing a method
-// like SubmitMeasurementAPI.Call.
+// like simpleSubmitMeasurementAPI.Call.
-type SubmitMeasurementCaller interface {
+type callerForSubmitMeasurementAPI interface {
 	Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error)
 }

internal/engine/ooapi/client.go

@@ -0,0 +1,13 @@
+package ooapi
+
+// Client is a client for speaking with the OONI API. Make sure you
+// fill in the mandatory fields.
+type Client struct {
+	BaseURL      string       // optional
+	GobCodec     GobCodec     // optional
+	HTTPClient   HTTPClient   // optional
+	JSONCodec    JSONCodec    // optional
+	KVStore      KVStore      // mandatory
+	RequestMaker RequestMaker // optional
+	UserAgent    string       // optional
+}

internal/engine/ooapi/clientcall.go

@@ -0,0 +1,214 @@
+// Code generated by go generate; DO NOT EDIT.
+// 2021-03-31 16:50:05.02947443 +0200 CEST m=+0.000143471
+
+package ooapi
+
+//go:generate go run ./internal/generator -file clientcall.go
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel"
+)
+
+func (c *Client) newCheckReportIDCaller() callerForCheckReportIDAPI {
+	return &simpleCheckReportIDAPI{
+		BaseURL:      c.BaseURL,
+		HTTPClient:   c.HTTPClient,
+		JSONCodec:    c.JSONCodec,
+		RequestMaker: c.RequestMaker,
+		UserAgent:    c.UserAgent,
+	}
+}
+
+// CheckReportID calls the CheckReportID API.
+func (c *Client) CheckReportID(
+	ctx context.Context, req *apimodel.CheckReportIDRequest,
+) (*apimodel.CheckReportIDResponse, error) {
+	api := c.newCheckReportIDCaller()
+	return api.Call(ctx, req)
+}
+
+func (c *Client) newCheckInCaller() callerForCheckInAPI {
+	return &simpleCheckInAPI{
+		BaseURL:      c.BaseURL,
+		HTTPClient:   c.HTTPClient,
+		JSONCodec:    c.JSONCodec,
+		RequestMaker: c.RequestMaker,
+		UserAgent:    c.UserAgent,
+	}
+}
+
+// CheckIn calls the CheckIn API.
+func (c *Client) CheckIn(
+	ctx context.Context, req *apimodel.CheckInRequest,
+) (*apimodel.CheckInResponse, error) {
+	api := c.newCheckInCaller()
+	return api.Call(ctx, req)
+}
+
+func (c *Client) newMeasurementMetaCaller() callerForMeasurementMetaAPI {
+	return &withCacheMeasurementMetaAPI{
+		API: &simpleMeasurementMetaAPI{
+			BaseURL:      c.BaseURL,
+			HTTPClient:   c.HTTPClient,
+			JSONCodec:    c.JSONCodec,
+			RequestMaker: c.RequestMaker,
+			UserAgent:    c.UserAgent,
+		},
+		GobCodec: c.GobCodec,
+		KVStore:  c.KVStore,
+	}
+}
+
+// MeasurementMeta calls the MeasurementMeta API.
+func (c *Client) MeasurementMeta(
+	ctx context.Context, req *apimodel.MeasurementMetaRequest,
+) (*apimodel.MeasurementMetaResponse, error) {
+	api := c.newMeasurementMetaCaller()
+	return api.Call(ctx, req)
+}
+
+func (c *Client) newTestHelpersCaller() callerForTestHelpersAPI {
+	return &simpleTestHelpersAPI{
+		BaseURL:      c.BaseURL,
+		HTTPClient:   c.HTTPClient,
+		JSONCodec:    c.JSONCodec,
+		RequestMaker: c.RequestMaker,
+		UserAgent:    c.UserAgent,
+	}
+}
+
+// TestHelpers calls the TestHelpers API.
+func (c *Client) TestHelpers(
+	ctx context.Context, req *apimodel.TestHelpersRequest,
+) (apimodel.TestHelpersResponse, error) {
+	api := c.newTestHelpersCaller()
+	return api.Call(ctx, req)
+}
+
+func (c *Client) newPsiphonConfigCaller() callerForPsiphonConfigAPI {
+	return &withLoginPsiphonConfigAPI{
+		API: &simplePsiphonConfigAPI{
+			BaseURL:      c.BaseURL,
+			HTTPClient:   c.HTTPClient,
+			JSONCodec:    c.JSONCodec,
+			RequestMaker: c.RequestMaker,
+			UserAgent:    c.UserAgent,
+		},
+		JSONCodec: c.JSONCodec,
+		KVStore:   c.KVStore,
+		RegisterAPI: &simpleRegisterAPI{
+			BaseURL:      c.BaseURL,
+			HTTPClient:   c.HTTPClient,
+			JSONCodec:    c.JSONCodec,
+			RequestMaker: c.RequestMaker,
+			UserAgent:    c.UserAgent,
+		},
+		LoginAPI: &simpleLoginAPI{
+			BaseURL:      c.BaseURL,
+			HTTPClient:   c.HTTPClient,
+			JSONCodec:    c.JSONCodec,
+			RequestMaker: c.RequestMaker,
+			UserAgent:    c.UserAgent,
+		},
+	}
+}
+
+// PsiphonConfig calls the PsiphonConfig API.
+func (c *Client) PsiphonConfig(
+	ctx context.Context, req *apimodel.PsiphonConfigRequest,
+) (apimodel.PsiphonConfigResponse, error) {
+	api := c.newPsiphonConfigCaller()
+	return api.Call(ctx, req)
+}
+
+func (c *Client) newTorTargetsCaller() callerForTorTargetsAPI {
+	return &withLoginTorTargetsAPI{
+		API: &simpleTorTargetsAPI{
+			BaseURL:      c.BaseURL,
+			HTTPClient:   c.HTTPClient,
+			JSONCodec:    c.JSONCodec,
+			RequestMaker: c.RequestMaker,
+			UserAgent:    c.UserAgent,
+		},
+		JSONCodec: c.JSONCodec,
+		KVStore:   c.KVStore,
+		RegisterAPI: &simpleRegisterAPI{
+			BaseURL:      c.BaseURL,
+			HTTPClient:   c.HTTPClient,
+			JSONCodec:    c.JSONCodec,
+			RequestMaker: c.RequestMaker,
+			UserAgent:    c.UserAgent,
+		},
+		LoginAPI: &simpleLoginAPI{
+			BaseURL:      c.BaseURL,
+			HTTPClient:   c.HTTPClient,
+			JSONCodec:    c.JSONCodec,
+			RequestMaker: c.RequestMaker,
+			UserAgent:    c.UserAgent,
+		},
+	}
+}
+
+// TorTargets calls the TorTargets API.
+func (c *Client) TorTargets(
+	ctx context.Context, req *apimodel.TorTargetsRequest,
+) (apimodel.TorTargetsResponse, error) {
+	api := c.newTorTargetsCaller()
+	return api.Call(ctx, req)
+}
+
+func (c *Client) newURLsCaller() callerForURLsAPI {
+	return &simpleURLsAPI{
+		BaseURL:      c.BaseURL,
+		HTTPClient:   c.HTTPClient,
+		JSONCodec:    c.JSONCodec,
+		RequestMaker: c.RequestMaker,
+		UserAgent:    c.UserAgent,
+	}
+}
+
+// URLs calls the URLs API.
+func (c *Client) URLs(
+	ctx context.Context, req *apimodel.URLsRequest,
+) (*apimodel.URLsResponse, error) {
+	api := c.newURLsCaller()
+	return api.Call(ctx, req)
+}
+
+func (c *Client) newOpenReportCaller() callerForOpenReportAPI {
+	return &simpleOpenReportAPI{
+		BaseURL:      c.BaseURL,
+		HTTPClient:   c.HTTPClient,
+		JSONCodec:    c.JSONCodec,
+		RequestMaker: c.RequestMaker,
+		UserAgent:    c.UserAgent,
+	}
+}
+
+// OpenReport calls the OpenReport API.
+func (c *Client) OpenReport(
+	ctx context.Context, req *apimodel.OpenReportRequest,
+) (*apimodel.OpenReportResponse, error) {
+	api := c.newOpenReportCaller()
+	return api.Call(ctx, req)
+}
+
+func (c *Client) newSubmitMeasurementCaller() callerForSubmitMeasurementAPI {
+	return &simpleSubmitMeasurementAPI{
+		BaseURL:      c.BaseURL,
+		HTTPClient:   c.HTTPClient,
+		JSONCodec:    c.JSONCodec,
+		RequestMaker: c.RequestMaker,
+		UserAgent:    c.UserAgent,
+	}
+}
+
+// SubmitMeasurement calls the SubmitMeasurement API.
+func (c *Client) SubmitMeasurement(
+	ctx context.Context, req *apimodel.SubmitMeasurementRequest,
+) (*apimodel.SubmitMeasurementResponse, error) {
+	api := c.newSubmitMeasurementCaller()
+	return api.Call(ctx, req)
+}

internal/engine/ooapi/clientcall_test.go

@@ -0,0 +1,898 @@
+// Code generated by go generate; DO NOT EDIT.
+// 2021-03-31 16:50:05.248051533 +0200 CEST m=+0.000086484
+
+package ooapi
+
+//go:generate go run ./internal/generator -file clientcall_test.go
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sync"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel"
+)
+
+type handleClientCallCheckReportID struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        *apimodel.CheckReportIDResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallCheckReportID) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out *apimodel.CheckReportIDResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestCheckReportIDClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallCheckReportID{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.CheckReportIDRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.CheckReportID(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "GET" {
+		t.Fatal("invalid method")
+	}
+	// check the query
+	api := &simpleCheckReportIDAPI{BaseURL: srvr.URL}
+	httpReq, err := api.newRequest(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
+		t.Fatal(diff)
+	}
+	if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+type handleClientCallCheckIn struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        *apimodel.CheckInResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out *apimodel.CheckInResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestCheckInClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallCheckIn{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.CheckInRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.CheckIn(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "POST" {
+		t.Fatal("invalid method")
+	}
+	// check the body
+	if handler.contentType != "application/json" {
+		t.Fatal("invalid content-type header")
+	}
+	got := &apimodel.CheckInRequest{}
+	if err := json.Unmarshal(handler.body, &got); err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(req, got); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+type handleClientCallMeasurementMeta struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        *apimodel.MeasurementMetaResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out *apimodel.MeasurementMetaResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestMeasurementMetaClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallMeasurementMeta{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.MeasurementMetaRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.MeasurementMeta(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "GET" {
+		t.Fatal("invalid method")
+	}
+	// check the query
+	api := &simpleMeasurementMetaAPI{BaseURL: srvr.URL}
+	httpReq, err := api.newRequest(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
+		t.Fatal(diff)
+	}
+	if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+type handleClientCallTestHelpers struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        apimodel.TestHelpersResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out apimodel.TestHelpersResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestTestHelpersClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallTestHelpers{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.TestHelpersRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.TestHelpers(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "GET" {
+		t.Fatal("invalid method")
+	}
+	// check the query
+	api := &simpleTestHelpersAPI{BaseURL: srvr.URL}
+	httpReq, err := api.newRequest(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
+		t.Fatal(diff)
+	}
+	if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+type handleClientCallPsiphonConfig struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        apimodel.PsiphonConfigResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallPsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	if r.URL.Path == "/api/v1/register" {
+		var out apimodel.RegisterResponse
+		ff.fill(&out)
+		data, err := json.Marshal(out)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		w.Write(data)
+		return
+	}
+	if r.URL.Path == "/api/v1/login" {
+		var out apimodel.LoginResponse
+		ff.fill(&out)
+		data, err := json.Marshal(out)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		w.Write(data)
+		return
+	}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out apimodel.PsiphonConfigResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestPsiphonConfigClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallPsiphonConfig{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.PsiphonConfigRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.PsiphonConfig(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "GET" {
+		t.Fatal("invalid method")
+	}
+	// check the query
+	api := &simplePsiphonConfigAPI{BaseURL: srvr.URL}
+	httpReq, err := api.newRequest(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
+		t.Fatal(diff)
+	}
+	if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+type handleClientCallTorTargets struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        apimodel.TorTargetsResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	if r.URL.Path == "/api/v1/register" {
+		var out apimodel.RegisterResponse
+		ff.fill(&out)
+		data, err := json.Marshal(out)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		w.Write(data)
+		return
+	}
+	if r.URL.Path == "/api/v1/login" {
+		var out apimodel.LoginResponse
+		ff.fill(&out)
+		data, err := json.Marshal(out)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		w.Write(data)
+		return
+	}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out apimodel.TorTargetsResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestTorTargetsClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallTorTargets{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.TorTargetsRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.TorTargets(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "GET" {
+		t.Fatal("invalid method")
+	}
+	// check the query
+	api := &simpleTorTargetsAPI{BaseURL: srvr.URL}
+	httpReq, err := api.newRequest(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
+		t.Fatal(diff)
+	}
+	if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+type handleClientCallURLs struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        *apimodel.URLsResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallURLs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out *apimodel.URLsResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestURLsClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallURLs{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.URLsRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.URLs(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "GET" {
+		t.Fatal("invalid method")
+	}
+	// check the query
+	api := &simpleURLsAPI{BaseURL: srvr.URL}
+	httpReq, err := api.newRequest(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
+		t.Fatal(diff)
+	}
+	if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+type handleClientCallOpenReport struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        *apimodel.OpenReportResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out *apimodel.OpenReportResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestOpenReportClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallOpenReport{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.OpenReportRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.OpenReport(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "POST" {
+		t.Fatal("invalid method")
+	}
+	// check the body
+	if handler.contentType != "application/json" {
+		t.Fatal("invalid content-type header")
+	}
+	got := &apimodel.OpenReportRequest{}
+	if err := json.Unmarshal(handler.body, &got); err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(req, got); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+type handleClientCallSubmitMeasurement struct {
+	accept      string
+	body        []byte
+	contentType string
+	count       int32
+	method      string
+	mu          sync.Mutex
+	resp        *apimodel.SubmitMeasurementResponse
+	url         *url.URL
+	userAgent   string
+}
+
+func (h *handleClientCallSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ff := fakeFill{}
+	defer h.mu.Unlock()
+	h.mu.Lock()
+	if h.count > 0 {
+		w.WriteHeader(400)
+		return
+	}
+	h.count++
+	if r.Body != nil {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			w.WriteHeader(400)
+			return
+		}
+		h.body = data
+	}
+	h.method = r.Method
+	h.url = r.URL
+	h.accept = r.Header.Get("Accept")
+	h.contentType = r.Header.Get("Content-Type")
+	h.userAgent = r.Header.Get("User-Agent")
+	var out *apimodel.SubmitMeasurementResponse
+	ff.fill(&out)
+	h.resp = out
+	data, err := json.Marshal(out)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	w.Write(data)
+}
+
+func TestSubmitMeasurementClientCallRoundTrip(t *testing.T) {
+	// setup
+	handler := &handleClientCallSubmitMeasurement{}
+	srvr := httptest.NewServer(handler)
+	defer srvr.Close()
+	req := &apimodel.SubmitMeasurementRequest{}
+	ff := &fakeFill{}
+	ff.fill(&req)
+	clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}
+	ff.fill(&clnt.UserAgent)
+	// issue request
+	ctx := context.Background()
+	resp, err := clnt.SubmitMeasurement(ctx, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("expected non-nil response here")
+	}
+	// compare our response and server's one
+	if diff := cmp.Diff(handler.resp, resp); diff != "" {
+		t.Fatal(diff)
+	}
+	// check whether headers are OK
+	if handler.accept != "application/json" {
+		t.Fatal("invalid accept header")
+	}
+	if handler.userAgent != clnt.UserAgent {
+		t.Fatal("invalid user-agent header")
+	}
+	// check whether the method is OK
+	if handler.method != "POST" {
+		t.Fatal("invalid method")
+	}
+	// check the body
+	if handler.contentType != "application/json" {
+		t.Fatal("invalid content-type header")
+	}
+	got := &apimodel.SubmitMeasurementRequest{}
+	if err := json.Unmarshal(handler.body, &got); err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(req, got); diff != "" {
+		t.Fatal(diff)
+	}
+}

internal/engine/ooapi/cloners.go

@@ -1,18 +1,18 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2021-02-26 15:45:52.108352268 +0100 CET m=+0.000275862
+// 2021-03-31 16:50:05.487854795 +0200 CEST m=+0.000140777
 
 package ooapi
 
 //go:generate go run ./internal/generator -file cloners.go
 
-// PsiphonConfigCaller represents any type exposing a method
+// clonerForPsiphonConfigAPI represents any type exposing a method
-// like PsiphonConfigAPI.WithToken.
+// like simplePsiphonConfigAPI.WithToken.
-type PsiphonConfigCloner interface {
+type clonerForPsiphonConfigAPI interface {
-	WithToken(token string) PsiphonConfigCaller
+	WithToken(token string) callerForPsiphonConfigAPI
 }
 
-// TorTargetsCaller represents any type exposing a method
+// clonerForTorTargetsAPI represents any type exposing a method
-// like TorTargetsAPI.WithToken.
+// like simpleTorTargetsAPI.WithToken.
-type TorTargetsCloner interface {
+type clonerForTorTargetsAPI interface {
-	WithToken(token string) TorTargetsCaller
+	WithToken(token string) callerForTorTargetsAPI
 }

internal/engine/ooapi/dependencies.go

@@ -21,8 +21,8 @@ type RequestMaker interface {
 	NewRequest(ctx context.Context, method, URL string, body io.Reader) (*http.Request, error)
 }
 
-// TemplateExecutor parses and executes a text template.
+// templateExecutor parses and executes a text template.
-type TemplateExecutor interface {
+type templateExecutor interface {
 	// Execute takes in input a template string and some piece of data. It
 	// returns either a string where template parameters have been replaced,
 	// on success, or an error, on failure.

internal/engine/ooapi/doc.go

@@ -1,108 +1,19 @@
-// Package ooapi contains clients for the OONI API. We
+// Package ooapi contains a client for the OONI API. We
-// automatically generate the code in this package from
+// automatically generate the code in this package from the
-// the apimodel and internal/generator packages. For
+// apimodel and internal/generator packages.
-// each OONI API, we define up to three data structures:
 //
-// 1. a data structure representing the API;
+// Usage
 //
-// 2. a caching data structure, if the API
+// You need to create a Client. Make sure you set all
-// supports caching;
+// the mandatory fields. You will then have a function
+// for every supported OONI API. This function will
+// take in input a context and a request. You need to
+// fill the request, of course. The return value is
+// either a response or an error.
 //
-// 3. an auto-login data structure, if the API
+// If an API requires login, we will automatically
-// requires login.
+// perform the login. If an API uses caching, we will
-//
+// automatically use the cache.
-// The rest of this documentation page describes these
-// three data structures and the design and architecture
-// of this package. Refer to subpackages for more
-// information on how to specify an API.
-//
-// API data structure
-//
-// For each API, this package defines a data structure
-// representing the API. For example, for the TorTargets API,
-// we define the TorTargetsAPI data structure.
-//
-// The API data structure defines a method named Call that
-// allows calling the specified API. Call takes as arguments
-// a context and the request for the API and returns the
-// API response or an error.
-//
-// Request and response messages live inside the apimodel
-// subpackage. We name them after the API. Thus, for
-// the TorTargets API, the request is TorTargetsRequest,
-// and the response is TorTargetsResponse.
-//
-// API data structures are cheap to create and do not
-// mutate. They should be used in place and then forgotten
-// off once the API call is complete.
-//
-// Unless explicitly indicated, the zero value of every
-// API data structure is a valid API data structure.
-//
-// In terms of dependencies, APIs certainly need an http.Client
-// to communicate with the OONI backend. To represent such a
-// client, we use the HTTPClient interface. If you do not tell
-// an API which http.Client to use, we will default to the
-// standard library's http.DefaultClient.
-//
-// An API also depends on a JSONCodec. That is, on a data
-// structures that encodes data to/from JSON. If you do not
-// specify explicitly a JSONCodec, we will use the Go
-// standard library's JSON implementation.
-//
-// When an API requires authentication, you need to tell
-// it which authentication token to use. This gives you
-// control over obtaining the token and is the low-level
-// way of interacting with authenticated APIs. We recommend
-// using the auto-login wrappers instead (see below).
-//
-// Authenticated APIs also define the WithToken method. This
-// method takes as argument a token and returns a copy of the
-// original API using the given token. We use this method
-// to implement auto-login wrappers.
-//
-// For each API, we also define two interfaces:
-//
-// 1. the Caller interface represents the possibility of
-// calling a specific API with the correct arguments;
-//
-// 2. the Cloner interface represents the possibility of
-// calling WithToken on the given API.
-//
-// They abstract the interaction between the API type and
-// its caching and auto-login wrappers.
-//
-// Caching
-//
-// If an API supports caching, we define a type whose name
-// ends in Cache. The TorTargets API cache, for example,
-// is TorTargetsCache. These caching types wrap the API type
-// and provide the caching functionality.
-//
-// Because the cache needs to read from and write to the
-// disk, a caching type needs a KVStore. A KVStore is
-// an interface that allow you to bind a specific key to
-// a given blob of bytes and to retrieve such bytes later.
-//
-// Caches use the gob data format from the Go standard
-// library (`encoding/gob`). We abstract this dependency
-// using the GobCodec interface. By default, when you
-// do not specify a GobCodec we use the implementation
-// of gob from the Go standard library.
-//
-// See the example describing caching for more information
-// on how to use caching.
-//
-// Auto-login
-//
-// If an API supports auto-login, we define a type whose
-// name ends with WithLogin. The TorTargets auto-login struct,
-// for example, is called TorTargetsAPIWithLogin.
-//
-// Auto-login wrappers need to store persistent data. We
-// use a KVStore for that (see above). We encode login data
-// using JSON. To this end, we use a JSONCodec (also
-// described above).
 //
 // See the example describing auto-login for more information
 // on how to use auto-login.
@@ -142,22 +53,4 @@
 // The ./internal/generator contains code to generate most
 // code in this package. In particular, the spec.go file is
 // the specification of the APIs.
-//
-// Notable generated files
-//
-// - apis.go: contains APIs (e.g., TorTargetsAPI);
-//
-// - caching.go: contains caching wrappers for every API
-// that declares that it needs a cache (e.g., TorTargetsCache);
-//
-// - callers.go: contains Callers;
-//
-// - cloners.go: contains the Cloners;
-//
-// - login.go: contains auto-login wrappers (e.g.,
-// TorTargetsAPIWithLogin);
-//
-// - requests.go: contains code to generate http.Requests.
-//
-// - responses.go: code to parse http.Responses.
 package ooapi

internal/engine/ooapi/errors.go

@@ -2,13 +2,13 @@ package ooapi
 
 import "errors"
 
-// Errors defined by this package. In addition to these errors, this
+// Errors defined by this package.
-// package may of course return any other stdlib specific error.
 var (
-	ErrEmptyField      = errors.New("apiclient: empty field")
+	ErrAPICallFailed   = errors.New("ooapi: API call failed")
-	ErrHTTPFailure     = errors.New("apiclient: http request failed")
+	ErrEmptyField      = errors.New("ooapi: empty field")
-	ErrJSONLiteralNull = errors.New("apiclient: server returned us a literal null")
+	ErrHTTPFailure     = errors.New("ooapi: http request failed")
-	ErrMissingToken    = errors.New("apiclient: missing auth token")
+	ErrJSONLiteralNull = errors.New("ooapi: server returned us a literal null")
-	ErrUnauthorized    = errors.New("apiclient: not authorized")
+	ErrMissingToken    = errors.New("ooapi: missing auth token")
-	errCacheNotFound   = errors.New("apiclient: not found in cache")
+	ErrUnauthorized    = errors.New("ooapi: not authorized")
+	errCacheNotFound   = errors.New("ooapi: not found in cache")
 )

internal/engine/ooapi/fakeapi_test.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2021-02-26 15:45:52.357709034 +0100 CET m=+0.000208565
+// 2021-03-31 16:50:05.702711633 +0200 CEST m=+0.000209246
 
 package ooapi
 
@@ -24,7 +24,7 @@ func (fapi *FakeCheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckR
 }
 
 var (
-	_ CheckReportIDCaller = &FakeCheckReportIDAPI{}
+	_ callerForCheckReportIDAPI = &FakeCheckReportIDAPI{}
 )
 
 type FakeCheckInAPI struct {
@@ -39,7 +39,7 @@ func (fapi *FakeCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInReque
 }
 
 var (
-	_ CheckInCaller = &FakeCheckInAPI{}
+	_ callerForCheckInAPI = &FakeCheckInAPI{}
 )
 
 type FakeLoginAPI struct {
@@ -54,7 +54,7 @@ func (fapi *FakeLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest)
 }
 
 var (
-	_ LoginCaller = &FakeLoginAPI{}
+	_ callerForLoginAPI = &FakeLoginAPI{}
 )
 
 type FakeMeasurementMetaAPI struct {
@@ -69,7 +69,7 @@ func (fapi *FakeMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.Meas
 }
 
 var (
-	_ MeasurementMetaCaller = &FakeMeasurementMetaAPI{}
+	_ callerForMeasurementMetaAPI = &FakeMeasurementMetaAPI{}
 )
 
 type FakeRegisterAPI struct {
@@ -84,7 +84,7 @@ func (fapi *FakeRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterReq
 }
 
 var (
-	_ RegisterCaller = &FakeRegisterAPI{}
+	_ callerForRegisterAPI = &FakeRegisterAPI{}
 )
 
 type FakeTestHelpersAPI struct {
@@ -99,11 +99,11 @@ func (fapi *FakeTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelp
 }
 
 var (
-	_ TestHelpersCaller = &FakeTestHelpersAPI{}
+	_ callerForTestHelpersAPI = &FakeTestHelpersAPI{}
 )
 
 type FakePsiphonConfigAPI struct {
-	WithResult PsiphonConfigCaller
+	WithResult callerForPsiphonConfigAPI
 	Err        error
 	Response   apimodel.PsiphonConfigResponse
 	CountCall  int32
@@ -114,17 +114,17 @@ func (fapi *FakePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.Psipho
 	return fapi.Response, fapi.Err
 }
 
-func (fapi *FakePsiphonConfigAPI) WithToken(token string) PsiphonConfigCaller {
+func (fapi *FakePsiphonConfigAPI) WithToken(token string) callerForPsiphonConfigAPI {
 	return fapi.WithResult
 }
 
 var (
-	_ PsiphonConfigCaller = &FakePsiphonConfigAPI{}
+	_ callerForPsiphonConfigAPI = &FakePsiphonConfigAPI{}
-	_ PsiphonConfigCloner = &FakePsiphonConfigAPI{}
+	_ clonerForPsiphonConfigAPI = &FakePsiphonConfigAPI{}
 )
 
 type FakeTorTargetsAPI struct {
-	WithResult TorTargetsCaller
+	WithResult callerForTorTargetsAPI
 	Err        error
 	Response   apimodel.TorTargetsResponse
 	CountCall  int32
@@ -135,13 +135,13 @@ func (fapi *FakeTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTarget
 	return fapi.Response, fapi.Err
 }
 
-func (fapi *FakeTorTargetsAPI) WithToken(token string) TorTargetsCaller {
+func (fapi *FakeTorTargetsAPI) WithToken(token string) callerForTorTargetsAPI {
 	return fapi.WithResult
 }
 
 var (
-	_ TorTargetsCaller = &FakeTorTargetsAPI{}
+	_ callerForTorTargetsAPI = &FakeTorTargetsAPI{}
-	_ TorTargetsCloner = &FakeTorTargetsAPI{}
+	_ clonerForTorTargetsAPI = &FakeTorTargetsAPI{}
 )
 
 type FakeURLsAPI struct {
@@ -156,7 +156,7 @@ func (fapi *FakeURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*
 }
 
 var (
-	_ URLsCaller = &FakeURLsAPI{}
+	_ callerForURLsAPI = &FakeURLsAPI{}
 )
 
 type FakeOpenReportAPI struct {
@@ -171,7 +171,7 @@ func (fapi *FakeOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenRepor
 }
 
 var (
-	_ OpenReportCaller = &FakeOpenReportAPI{}
+	_ callerForOpenReportAPI = &FakeOpenReportAPI{}
 )
 
 type FakeSubmitMeasurementAPI struct {
@@ -186,5 +186,5 @@ func (fapi *FakeSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.Su
 }
 
 var (
-	_ SubmitMeasurementCaller = &FakeSubmitMeasurementAPI{}
+	_ callerForSubmitMeasurementAPI = &FakeSubmitMeasurementAPI{}
 )

internal/engine/ooapi/httpclient_test.go

@@ -0,0 +1,21 @@
+package ooapi
+
+import (
+	"net/http"
+	"testing"
+)
+
+type VerboseHTTPClient struct {
+	T *testing.T
+}
+
+func (c *VerboseHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	c.T.Logf("> %s %s", req.Method, req.URL.String())
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		c.T.Logf("< %s", err.Error())
+		return nil, err
+	}
+	c.T.Logf("< %d", resp.StatusCode)
+	return resp, nil
+}