fix(ooniprobe): dnscheck,stunreachability run w/ default input (#633)

This diff is part of https://github.com/ooni/probe/issues/1814 and
teaches `ooniprobe` to run dnscheck and stunreachability by using the
default static input feature of the `InputLoader`.

I've manually tested that we can still run `websites` like
we did before (including category filtering).

I've also manually tested that now we can run `experimental` and
get parseable results for dnscheck and stunreachability.

With this diff in, we have fixed the original problem highlighted in
the https://github.com/ooni/probe/issues/1814 issue.

Yet, because of the way in which I solved the problem, there is
more work to do. My changes have broken stunreachability for
mobile and now it's time I apply fixes to make it work again.

This diff was extracted from https://github.com/ooni/probe-cli/pull/539,
which at this point only basically contains the remaining fixes to
ensure we can run stunreachability on mobile.

Co-authored-by: Arturo Filastò <arturo@filasto.net>

Co-authored-by: Arturo Filastò <arturo@filasto.net>

cmd/ooniprobe/internal/nettests/dnscheck.go

@@ -1,60 +1,42 @@
 package nettests
 
 import (
-	"encoding/json"
+	"context"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	engine "github.com/ooni/probe-cli/v3/internal/engine"
-	"github.com/ooni/probe-cli/v3/internal/engine/experiment/run"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
-	"github.com/ooni/probe-cli/v3/internal/runtimex"
 )
 
 // DNSCheck nettest implementation.
 type DNSCheck struct{}
 
-var dnsCheckDefaultInput []string
+func (n DNSCheck) lookupURLs(ctl *Controller) ([]string, error) {
-
+	inputloader := &engine.InputLoader{
-func dnsCheckMustMakeInput(input *run.StructuredInput) string {
+		CheckInConfig: &model.CheckInConfig{
-	data, err := json.Marshal(input)
+			// not needed because we have default static input in the engine
-	runtimex.PanicOnError(err, "json.Marshal failed")
+		},
-	return string(data)
+		ExperimentName: "dnscheck",
-}
+		InputPolicy:    engine.InputOrStaticDefault,
-
+		Session:        ctl.Session,
-func init() {
+		SourceFiles:    ctl.InputFiles,
-	// The following code just adds a minimal set of URLs to
+		StaticInputs:   ctl.Inputs,
-	// test using DNSCheck, so we start exposing it.
+	}
-	//
+	testlist, err := inputloader.Load(context.Background())
-	// TODO(bassosimone):
+	if err != nil {
-	//
+		return nil, err
-	// 1. we should be getting input from the backend instead of
+	}
-	// having an hardcoded list of inputs here.
+	return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
-	//
-	// 2. we should modify dnscheck to accept http3://... as a
-	// shortcut for https://... with h3. If we don't do that, we
-	// are stuck with the h3 results hiding h2 results in OONI
-	// Explorer because they use the same URL.
-	//
-	// 3. it seems we have the problem that dnscheck results
-	// appear as the `run` nettest in `ooniprobe list <ID>` because
-	// dnscheck is run using the `run` functionality.
-	dnsCheckDefaultInput = append(dnsCheckDefaultInput, dnsCheckMustMakeInput(
-		&run.StructuredInput{
-			DNSCheck: dnscheck.Config{},
-			Name:     "dnscheck",
-			Input:    "https://dns.google/dns-query",
-		}))
-	dnsCheckDefaultInput = append(dnsCheckDefaultInput, dnsCheckMustMakeInput(
-		&run.StructuredInput{
-			DNSCheck: dnscheck.Config{},
-			Name:     "dnscheck",
-			Input:    "https://cloudflare-dns.com/dns-query",
-		}))
 }
 
 // Run starts the nettest.
 func (n DNSCheck) Run(ctl *Controller) error {
-	builder, err := ctl.Session.NewExperimentBuilder("run")
+	builder, err := ctl.Session.NewExperimentBuilder("dnscheck")
 	if err != nil {
 		return err
 	}
-	return ctl.Run(builder, dnsCheckDefaultInput)
+	urls, err := n.lookupURLs(ctl)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, urls)
 }

cmd/ooniprobe/internal/nettests/nettests.go

@@ -13,6 +13,7 @@ import (
 	engine "github.com/ooni/probe-cli/v3/internal/engine"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/pkg/errors"
+	"upper.io/db.v3/lib/sqlbuilder"
 )
 
 // Nettest interface. Every Nettest should implement this.
@@ -64,12 +65,49 @@ type Controller struct {
 	curInputIdx int
 }
 
-// SetInputIdxMap is used to set the mapping of index into input. This mapping
+// BuildAndSetInputIdxMap takes in input a list of URLs in the format
-// is used to reference, for example, a particular URL based on the index inside
+// returned by the check-in API (i.e., model.URLInfo) and performs
-// of the input list and the index of it in the database.
+// the following actions:
-func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error {
+//
-	c.inputIdxMap = inputIdxMap
+// 1. inserts each URL into the database;
-	return nil
+//
+// 2. builds a list of bare URLs to be tested;
+//
+// 3. registers a mapping between each URL and an index
+// and stores it into the controller.
+//
+// Arguments:
+//
+// - db is the database in which to register the URL;
+//
+// - testlist is the result from the check-in API (or possibly
+// a manually constructed list when applicable, e.g., for dnscheck
+// until we have an API for serving its input).
+//
+// Results:
+//
+// - on success, a list of strings containing URLs to test;
+//
+// - on failure, an error.
+func (c *Controller) BuildAndSetInputIdxMap(
+	db sqlbuilder.Database, testlist []model.URLInfo) ([]string, error) {
+	var urls []string
+	urlIDMap := make(map[int64]int64)
+	for idx, url := range testlist {
+		log.Debugf("Going over URL %d", idx)
+		urlID, err := database.CreateOrUpdateURL(
+			db, url.URL, url.CategoryCode, url.CountryCode,
+		)
+		if err != nil {
+			log.Error("failed to add to the URL table")
+			return nil, err
+		}
+		log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
+		urlIDMap[int64(idx)] = urlID
+		urls = append(urls, url.URL)
+	}
+	c.inputIdxMap = urlIDMap
+	return urls, nil
 }
 
 // SetNettestIndex is used to set the current nettest index and total nettest

cmd/ooniprobe/internal/nettests/stunreachability.go

@@ -1,13 +1,42 @@
 package nettests
 
+import (
+	"context"
+
+	engine "github.com/ooni/probe-cli/v3/internal/engine"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
 // STUNReachability nettest implementation.
 type STUNReachability struct{}
 
+func (n STUNReachability) lookupURLs(ctl *Controller) ([]string, error) {
+	inputloader := &engine.InputLoader{
+		CheckInConfig: &model.CheckInConfig{
+			// not needed because we have default static input in the engine
+		},
+		ExperimentName: "stunreachability",
+		InputPolicy:    engine.InputOrStaticDefault,
+		Session:        ctl.Session,
+		SourceFiles:    ctl.InputFiles,
+		StaticInputs:   ctl.Inputs,
+	}
+	testlist, err := inputloader.Load(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
+}
+
 // Run starts the nettest.
 func (n STUNReachability) Run(ctl *Controller) error {
 	builder, err := ctl.Session.NewExperimentBuilder("stunreachability")
 	if err != nil {
 		return err
 	}
-	return ctl.Run(builder, []string{""})
+	urls, err := n.lookupURLs(ctl)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, urls)
 }

cmd/ooniprobe/internal/nettests/web_connectivity.go

@@ -4,12 +4,11 @@ import (
 	"context"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	engine "github.com/ooni/probe-cli/v3/internal/engine"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 
-func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64, error) {
+func (n WebConnectivity) lookupURLs(ctl *Controller, categories []string) ([]string, error) {
 	inputloader := &engine.InputLoader{
 		CheckInConfig: &model.CheckInConfig{
 			// Setting Charging and OnWiFi to true causes the CheckIn
@@ -22,31 +21,17 @@ func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64
 				CategoryCodes: categories,
 			},
 		},
-		InputPolicy:  engine.InputOrQueryBackend,
+		ExperimentName: "web_connectivity",
-		Session:      ctl.Session,
+		InputPolicy:    engine.InputOrQueryBackend,
-		SourceFiles:  ctl.InputFiles,
+		Session:        ctl.Session,
-		StaticInputs: ctl.Inputs,
+		SourceFiles:    ctl.InputFiles,
+		StaticInputs:   ctl.Inputs,
 	}
 	testlist, err := inputloader.Load(context.Background())
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	var urls []string
+	return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
-	urlIDMap := make(map[int64]int64)
-	for idx, url := range testlist {
-		log.Debugf("Going over URL %d", idx)
-		urlID, err := database.CreateOrUpdateURL(
-			ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode,
-		)
-		if err != nil {
-			log.Error("failed to add to the URL table")
-			return nil, nil, err
-		}
-		log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
-		urlIDMap[int64(idx)] = urlID
-		urls = append(urls, url.URL)
-	}
-	return urls, urlIDMap, nil
 }
 
 // WebConnectivity test implementation
@@ -55,14 +40,11 @@ type WebConnectivity struct{}
 // Run starts the test
 func (n WebConnectivity) Run(ctl *Controller) error {
 	log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
-	urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
+	urls, err := n.lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
 	if err != nil {
 		return err
 	}
-	ctl.SetInputIdxMap(urlIDMap)
+	builder, err := ctl.Session.NewExperimentBuilder("web_connectivity")
-	builder, err := ctl.Session.NewExperimentBuilder(
-		"web_connectivity",
-	)
 	if err != nil {
 		return err
 	}