diff --git a/cmd/ooniprobe/internal/nettests/dnscheck.go b/cmd/ooniprobe/internal/nettests/dnscheck.go index 5fba9a9..0054c39 100644 --- a/cmd/ooniprobe/internal/nettests/dnscheck.go +++ b/cmd/ooniprobe/internal/nettests/dnscheck.go @@ -1,60 +1,42 @@ package nettests import ( - "encoding/json" + "context" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/run" - "github.com/ooni/probe-cli/v3/internal/runtimex" + engine "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/model" ) // DNSCheck nettest implementation. type DNSCheck struct{} -var dnsCheckDefaultInput []string - -func dnsCheckMustMakeInput(input *run.StructuredInput) string { - data, err := json.Marshal(input) - runtimex.PanicOnError(err, "json.Marshal failed") - return string(data) -} - -func init() { - // The following code just adds a minimal set of URLs to - // test using DNSCheck, so we start exposing it. - // - // TODO(bassosimone): - // - // 1. we should be getting input from the backend instead of - // having an hardcoded list of inputs here. - // - // 2. we should modify dnscheck to accept http3://... as a - // shortcut for https://... with h3. If we don't do that, we - // are stuck with the h3 results hiding h2 results in OONI - // Explorer because they use the same URL. - // - // 3. it seems we have the problem that dnscheck results - // appear as the `run` nettest in `ooniprobe list ` because - // dnscheck is run using the `run` functionality. - dnsCheckDefaultInput = append(dnsCheckDefaultInput, dnsCheckMustMakeInput( - &run.StructuredInput{ - DNSCheck: dnscheck.Config{}, - Name: "dnscheck", - Input: "https://dns.google/dns-query", - })) - dnsCheckDefaultInput = append(dnsCheckDefaultInput, dnsCheckMustMakeInput( - &run.StructuredInput{ - DNSCheck: dnscheck.Config{}, - Name: "dnscheck", - Input: "https://cloudflare-dns.com/dns-query", - })) +func (n DNSCheck) lookupURLs(ctl *Controller) ([]string, error) { + inputloader := &engine.InputLoader{ + CheckInConfig: &model.CheckInConfig{ + // not needed because we have default static input in the engine + }, + ExperimentName: "dnscheck", + InputPolicy: engine.InputOrStaticDefault, + Session: ctl.Session, + SourceFiles: ctl.InputFiles, + StaticInputs: ctl.Inputs, + } + testlist, err := inputloader.Load(context.Background()) + if err != nil { + return nil, err + } + return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist) } // Run starts the nettest. func (n DNSCheck) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("run") + builder, err := ctl.Session.NewExperimentBuilder("dnscheck") if err != nil { return err } - return ctl.Run(builder, dnsCheckDefaultInput) + urls, err := n.lookupURLs(ctl) + if err != nil { + return err + } + return ctl.Run(builder, urls) } diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index 4a27893..1176526 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -13,6 +13,7 @@ import ( engine "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/pkg/errors" + "upper.io/db.v3/lib/sqlbuilder" ) // Nettest interface. Every Nettest should implement this. @@ -64,12 +65,49 @@ type Controller struct { curInputIdx int } -// SetInputIdxMap is used to set the mapping of index into input. This mapping -// is used to reference, for example, a particular URL based on the index inside -// of the input list and the index of it in the database. -func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error { - c.inputIdxMap = inputIdxMap - return nil +// BuildAndSetInputIdxMap takes in input a list of URLs in the format +// returned by the check-in API (i.e., model.URLInfo) and performs +// the following actions: +// +// 1. inserts each URL into the database; +// +// 2. builds a list of bare URLs to be tested; +// +// 3. registers a mapping between each URL and an index +// and stores it into the controller. +// +// Arguments: +// +// - db is the database in which to register the URL; +// +// - testlist is the result from the check-in API (or possibly +// a manually constructed list when applicable, e.g., for dnscheck +// until we have an API for serving its input). +// +// Results: +// +// - on success, a list of strings containing URLs to test; +// +// - on failure, an error. +func (c *Controller) BuildAndSetInputIdxMap( + db sqlbuilder.Database, testlist []model.URLInfo) ([]string, error) { + var urls []string + urlIDMap := make(map[int64]int64) + for idx, url := range testlist { + log.Debugf("Going over URL %d", idx) + urlID, err := database.CreateOrUpdateURL( + db, url.URL, url.CategoryCode, url.CountryCode, + ) + if err != nil { + log.Error("failed to add to the URL table") + return nil, err + } + log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID) + urlIDMap[int64(idx)] = urlID + urls = append(urls, url.URL) + } + c.inputIdxMap = urlIDMap + return urls, nil } // SetNettestIndex is used to set the current nettest index and total nettest diff --git a/cmd/ooniprobe/internal/nettests/stunreachability.go b/cmd/ooniprobe/internal/nettests/stunreachability.go index 14aa46c..61adcc4 100644 --- a/cmd/ooniprobe/internal/nettests/stunreachability.go +++ b/cmd/ooniprobe/internal/nettests/stunreachability.go @@ -1,13 +1,42 @@ package nettests +import ( + "context" + + engine "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/model" +) + // STUNReachability nettest implementation. type STUNReachability struct{} +func (n STUNReachability) lookupURLs(ctl *Controller) ([]string, error) { + inputloader := &engine.InputLoader{ + CheckInConfig: &model.CheckInConfig{ + // not needed because we have default static input in the engine + }, + ExperimentName: "stunreachability", + InputPolicy: engine.InputOrStaticDefault, + Session: ctl.Session, + SourceFiles: ctl.InputFiles, + StaticInputs: ctl.Inputs, + } + testlist, err := inputloader.Load(context.Background()) + if err != nil { + return nil, err + } + return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist) +} + // Run starts the nettest. func (n STUNReachability) Run(ctl *Controller) error { builder, err := ctl.Session.NewExperimentBuilder("stunreachability") if err != nil { return err } - return ctl.Run(builder, []string{""}) + urls, err := n.lookupURLs(ctl) + if err != nil { + return err + } + return ctl.Run(builder, urls) } diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index 4ea021f..ad113fc 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -4,12 +4,11 @@ import ( "context" "github.com/apex/log" - "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database" engine "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/engine/model" ) -func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64, error) { +func (n WebConnectivity) lookupURLs(ctl *Controller, categories []string) ([]string, error) { inputloader := &engine.InputLoader{ CheckInConfig: &model.CheckInConfig{ // Setting Charging and OnWiFi to true causes the CheckIn @@ -22,31 +21,17 @@ func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64 CategoryCodes: categories, }, }, - InputPolicy: engine.InputOrQueryBackend, - Session: ctl.Session, - SourceFiles: ctl.InputFiles, - StaticInputs: ctl.Inputs, + ExperimentName: "web_connectivity", + InputPolicy: engine.InputOrQueryBackend, + Session: ctl.Session, + SourceFiles: ctl.InputFiles, + StaticInputs: ctl.Inputs, } testlist, err := inputloader.Load(context.Background()) if err != nil { - return nil, nil, err + return nil, err } - var urls []string - urlIDMap := make(map[int64]int64) - for idx, url := range testlist { - log.Debugf("Going over URL %d", idx) - urlID, err := database.CreateOrUpdateURL( - ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode, - ) - if err != nil { - log.Error("failed to add to the URL table") - return nil, nil, err - } - log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID) - urlIDMap[int64(idx)] = urlID - urls = append(urls, url.URL) - } - return urls, urlIDMap, nil + return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist) } // WebConnectivity test implementation @@ -55,14 +40,11 @@ type WebConnectivity struct{} // Run starts the test func (n WebConnectivity) Run(ctl *Controller) error { log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) - urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) + urls, err := n.lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) if err != nil { return err } - ctl.SetInputIdxMap(urlIDMap) - builder, err := ctl.Session.NewExperimentBuilder( - "web_connectivity", - ) + builder, err := ctl.Session.NewExperimentBuilder("web_connectivity") if err != nil { return err }