c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ooniprobe): dnscheck,stunreachability run w/ default input (#633)

Browse Source 
This diff is part of https://github.com/ooni/probe/issues/1814 and
teaches `ooniprobe` to run dnscheck and stunreachability by using the
default static input feature of the `InputLoader`.

I've manually tested that we can still run `websites` like
we did before (including category filtering).

I've also manually tested that now we can run `experimental` and
get parseable results for dnscheck and stunreachability.

With this diff in, we have fixed the original problem highlighted in
the https://github.com/ooni/probe/issues/1814 issue.

Yet, because of the way in which I solved the problem, there is
more work to do. My changes have broken stunreachability for
mobile and now it's time I apply fixes to make it work again.

This diff was extracted from https://github.com/ooni/probe-cli/pull/539,
which at this point only basically contains the remaining fixes to
ensure we can run stunreachability on mobile.

Co-authored-by: Arturo Filastò <arturo@filasto.net>

Co-authored-by: Arturo Filastò <arturo@filasto.net>
This commit is contained in:
Simone Basso 2021-12-03 16:10:55 +01:00 committed by GitHub
parent 2044b78a5a
commit 1896d2172a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 109 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
cmd/ooniprobe/internal/nettests/dnscheck.go
View File

@ -1,60 +1,42 @@
package nettests
import (
"encoding/json"
"context"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/run"
"github.com/ooni/probe-cli/v3/internal/runtimex"
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// DNSCheck nettest implementation.
type DNSCheck struct{}
var dnsCheckDefaultInput []string
func dnsCheckMustMakeInput(input *run.StructuredInput) string {
data, err := json.Marshal(input)
runtimex.PanicOnError(err, "json.Marshal failed")
return string(data)
}
func init() {
// The following code just adds a minimal set of URLs to
// test using DNSCheck, so we start exposing it.
//
// TODO(bassosimone):
//
// 1. we should be getting input from the backend instead of
// having an hardcoded list of inputs here.
//
// 2. we should modify dnscheck to accept http3://... as a
// shortcut for https://... with h3. If we don't do that, we
// are stuck with the h3 results hiding h2 results in OONI
// Explorer because they use the same URL.
//
// 3. it seems we have the problem that dnscheck results
// appear as the `run` nettest in `ooniprobe list <ID>` because
// dnscheck is run using the `run` functionality.
dnsCheckDefaultInput = append(dnsCheckDefaultInput, dnsCheckMustMakeInput(
&run.StructuredInput{
DNSCheck: dnscheck.Config{},
Name: "dnscheck",
Input: "https://dns.google/dns-query",
}))
dnsCheckDefaultInput = append(dnsCheckDefaultInput, dnsCheckMustMakeInput(
&run.StructuredInput{
DNSCheck: dnscheck.Config{},
Name: "dnscheck",
Input: "https://cloudflare-dns.com/dns-query",
}))
func (n DNSCheck) lookupURLs(ctl *Controller) ([]string, error) {
inputloader := &engine.InputLoader{
CheckInConfig: &model.CheckInConfig{
// not needed because we have default static input in the engine
},
ExperimentName: "dnscheck",
InputPolicy: engine.InputOrStaticDefault,
Session: ctl.Session,
SourceFiles: ctl.InputFiles,
StaticInputs: ctl.Inputs,
}
testlist, err := inputloader.Load(context.Background())
if err != nil {
return nil, err
}
return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
}
// Run starts the nettest.
func (n DNSCheck) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder("run")
builder, err := ctl.Session.NewExperimentBuilder("dnscheck")
if err != nil {
return err
}
return ctl.Run(builder, dnsCheckDefaultInput)
urls, err := n.lookupURLs(ctl)
if err != nil {
return err
}
return ctl.Run(builder, urls)
}

50
cmd/ooniprobe/internal/nettests/nettests.go
View File

@ -13,6 +13,7 @@ import (
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/pkg/errors"
"upper.io/db.v3/lib/sqlbuilder"
)
// Nettest interface. Every Nettest should implement this.
@ -64,12 +65,49 @@ type Controller struct {
curInputIdx int
}
// SetInputIdxMap is used to set the mapping of index into input. This mapping
// is used to reference, for example, a particular URL based on the index inside
// of the input list and the index of it in the database.
func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error {
c.inputIdxMap = inputIdxMap
return nil
// BuildAndSetInputIdxMap takes in input a list of URLs in the format
// returned by the check-in API (i.e., model.URLInfo) and performs
// the following actions:
//
// 1. inserts each URL into the database;
//
// 2. builds a list of bare URLs to be tested;
//
// 3. registers a mapping between each URL and an index
// and stores it into the controller.
//
// Arguments:
//
// - db is the database in which to register the URL;
//
// - testlist is the result from the check-in API (or possibly
// a manually constructed list when applicable, e.g., for dnscheck
// until we have an API for serving its input).
//
// Results:
//
// - on success, a list of strings containing URLs to test;
//
// - on failure, an error.
func (c *Controller) BuildAndSetInputIdxMap(
db sqlbuilder.Database, testlist []model.URLInfo) ([]string, error) {
var urls []string
urlIDMap := make(map[int64]int64)
for idx, url := range testlist {
log.Debugf("Going over URL %d", idx)
urlID, err := database.CreateOrUpdateURL(
db, url.URL, url.CategoryCode, url.CountryCode,
)
if err != nil {
log.Error("failed to add to the URL table")
return nil, err
}
log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
urlIDMap[int64(idx)] = urlID
urls = append(urls, url.URL)
}
c.inputIdxMap = urlIDMap
return urls, nil
}
// SetNettestIndex is used to set the current nettest index and total nettest

31
cmd/ooniprobe/internal/nettests/stunreachability.go
View File

@ -1,13 +1,42 @@
package nettests
import (
"context"
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// STUNReachability nettest implementation.
type STUNReachability struct{}
func (n STUNReachability) lookupURLs(ctl *Controller) ([]string, error) {
inputloader := &engine.InputLoader{
CheckInConfig: &model.CheckInConfig{
// not needed because we have default static input in the engine
},
ExperimentName: "stunreachability",
InputPolicy: engine.InputOrStaticDefault,
Session: ctl.Session,
SourceFiles: ctl.InputFiles,
StaticInputs: ctl.Inputs,
}
testlist, err := inputloader.Load(context.Background())
if err != nil {
return nil, err
}
return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
}
// Run starts the nettest.
func (n STUNReachability) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder("stunreachability")
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
urls, err := n.lookupURLs(ctl)
if err != nil {
return err
}
return ctl.Run(builder, urls)
}

38
cmd/ooniprobe/internal/nettests/web_connectivity.go
View File

@ -4,12 +4,11 @@ import (
"context"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64, error) {
func (n WebConnectivity) lookupURLs(ctl *Controller, categories []string) ([]string, error) {
inputloader := &engine.InputLoader{
CheckInConfig: &model.CheckInConfig{
// Setting Charging and OnWiFi to true causes the CheckIn
@ -22,31 +21,17 @@ func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64
CategoryCodes: categories,
},
},
InputPolicy: engine.InputOrQueryBackend,
Session: ctl.Session,
SourceFiles: ctl.InputFiles,
StaticInputs: ctl.Inputs,
ExperimentName: "web_connectivity",
InputPolicy: engine.InputOrQueryBackend,
Session: ctl.Session,
SourceFiles: ctl.InputFiles,
StaticInputs: ctl.Inputs,
}
testlist, err := inputloader.Load(context.Background())
if err != nil {
return nil, nil, err
return nil, err
}
var urls []string
urlIDMap := make(map[int64]int64)
for idx, url := range testlist {
log.Debugf("Going over URL %d", idx)
urlID, err := database.CreateOrUpdateURL(
ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode,
)
if err != nil {
log.Error("failed to add to the URL table")
return nil, nil, err
}
log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
urlIDMap[int64(idx)] = urlID
urls = append(urls, url.URL)
}
return urls, urlIDMap, nil
return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
}
// WebConnectivity test implementation
@ -55,14 +40,11 @@ type WebConnectivity struct{}
// Run starts the test
func (n WebConnectivity) Run(ctl *Controller) error {
log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
urls, err := n.lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
if err != nil {
return err
}
ctl.SetInputIdxMap(urlIDMap)
builder, err := ctl.Session.NewExperimentBuilder(
"web_connectivity",
)
builder, err := ctl.Session.NewExperimentBuilder("web_connectivity")
if err != nil {
return err
}