refactor: nettests is now an internal package (#165)

I'm moving everything into internal packages since this ain't a library.

While there, remove typo that was causing a build breakage.

internal/cli/run/run.go

@@ -10,7 +10,7 @@ import (
 	"github.com/ooni/probe-cli/internal/cli/onboard"
 	"github.com/ooni/probe-cli/internal/cli/root"
 	"github.com/ooni/probe-cli/internal/database"
-	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-cli/internal/nettests"
 )
 
 func runNettestGroup(tg string, ctx *ooni.Context, network *database.Network) error {

internal/log/handlers/cli/results.go

@@ -84,7 +84,6 @@ func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string)
 
 func logResultItem(w io.Writer, f log.Fields) error {
 	colWidth := 24
-Try running
 	rID := f.Get("id").(int64)
 	name := f.Get("name").(string)
 	isDone := f.Get("is_done").(bool)

internal/nettests/dash.go

@@ -0,0 +1,63 @@
+package nettests
+
+import (
+	"github.com/pkg/errors"
+)
+
+// Dash test implementation
+type Dash struct {
+}
+
+// Run starts the test
+func (d Dash) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder("dash")
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// DashTestKeys for the test
+// TODO: process 'receiver_data' to provide an array of performance for a chart.
+type DashTestKeys struct {
+	Latency   float64 `json:"connect_latency"`
+	Bitrate   float64 `json:"median_bitrate"`
+	Delay     float64 `json:"min_playout_delay"`
+	IsAnomaly bool    `json:"-"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (d Dash) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	var err error
+
+	testKeys := DashTestKeys{IsAnomaly: false}
+
+	simple, ok := tk["simple"].(map[string]interface{})
+	if !ok {
+		return testKeys, errors.New("simple key is not of the expected type")
+	}
+
+	latency, ok := simple["connect_latency"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "connect_latency is invalid")
+	}
+	testKeys.Latency = latency
+
+	bitrate, ok := simple["median_bitrate"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "median_bitrate is invalid")
+	}
+	testKeys.Bitrate = bitrate
+
+	delay, ok := simple["min_playout_delay"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "min_playout_delay is invalid")
+	}
+	testKeys.Delay = delay
+	return testKeys, err
+}
+
+// LogSummary writes the summary to the standard output
+func (d Dash) LogSummary(s string) error {
+	return nil
+}

internal/nettests/facebook_messenger.go

@@ -0,0 +1,53 @@
+package nettests
+
+// FacebookMessenger test implementation
+type FacebookMessenger struct {
+}
+
+// Run starts the test
+func (h FacebookMessenger) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"facebook_messenger",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// FacebookMessengerTestKeys for the test
+type FacebookMessengerTestKeys struct {
+	DNSBlocking bool `json:"facebook_dns_blocking"`
+	TCPBlocking bool `json:"facebook_tcp_blocking"`
+	IsAnomaly   bool `json:"-"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (h FacebookMessenger) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	var (
+		dnsBlocking bool
+		tcpBlocking bool
+	)
+	if tk["facebook_dns_blocking"] == nil {
+		dnsBlocking = false
+	} else {
+		dnsBlocking = tk["facebook_dns_blocking"].(bool)
+	}
+
+	if tk["facebook_tcp_blocking"] == nil {
+		tcpBlocking = false
+	} else {
+		tcpBlocking = tk["facebook_tcp_blocking"].(bool)
+	}
+
+	return FacebookMessengerTestKeys{
+		DNSBlocking: dnsBlocking,
+		TCPBlocking: tcpBlocking,
+		IsAnomaly:   dnsBlocking || tcpBlocking,
+	}, nil
+}
+
+// LogSummary writes the summary to the standard output
+func (h FacebookMessenger) LogSummary(s string) error {
+	return nil
+}

internal/nettests/groups.go

@@ -0,0 +1,46 @@
+package nettests
+
+// NettestGroup base structure
+type NettestGroup struct {
+	Label    string
+	Nettests []Nettest
+}
+
+// NettestGroups that can be run by the user
+var NettestGroups = map[string]NettestGroup{
+	"websites": {
+		Label: "Websites",
+		Nettests: []Nettest{
+			WebConnectivity{},
+		},
+	},
+	"performance": {
+		Label: "Performance",
+		Nettests: []Nettest{
+			Dash{},
+			NDT{},
+		},
+	},
+	"middlebox": {
+		Label: "Middleboxes",
+		Nettests: []Nettest{
+			HTTPInvalidRequestLine{},
+			HTTPHeaderFieldManipulation{},
+		},
+	},
+	"im": {
+		Label: "Instant Messaging",
+		Nettests: []Nettest{
+			FacebookMessenger{},
+			Telegram{},
+			WhatsApp{},
+		},
+	},
+	"circumvention": {
+		Label: "Circumvention Tools",
+		Nettests: []Nettest{
+			Psiphon{},
+			Tor{},
+		},
+	},
+}

internal/nettests/http_header_field_manipulation.go

@@ -0,0 +1,48 @@
+package nettests
+
+import (
+	"errors"
+)
+
+// HTTPHeaderFieldManipulation test implementation
+type HTTPHeaderFieldManipulation struct {
+}
+
+// Run starts the test
+func (h HTTPHeaderFieldManipulation) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"http_header_field_manipulation",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// HTTPHeaderFieldManipulationTestKeys for the test
+type HTTPHeaderFieldManipulationTestKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetTestKeys returns a projection of the tests keys needed for the views
+func (h HTTPHeaderFieldManipulation) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	testKeys := HTTPHeaderFieldManipulationTestKeys{IsAnomaly: false}
+	tampering, ok := tk["tampering"].(map[string]interface{})
+	if !ok {
+		return testKeys, errors.New("tampering testkey is invalid")
+	}
+	for _, v := range tampering {
+		t, ok := v.(bool)
+		// Ignore non booleans in the tampering map
+		if ok && t == true {
+			testKeys.IsAnomaly = true
+		}
+	}
+
+	return testKeys, nil
+}
+
+// LogSummary writes the summary to the standard output
+func (h HTTPHeaderFieldManipulation) LogSummary(s string) error {
+	return nil
+}

internal/nettests/http_invalid_request_line.go

@@ -0,0 +1,43 @@
+package nettests
+
+import (
+	"errors"
+)
+
+// HTTPInvalidRequestLine test implementation
+type HTTPInvalidRequestLine struct {
+}
+
+// Run starts the test
+func (h HTTPInvalidRequestLine) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"http_invalid_request_line",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// HTTPInvalidRequestLineTestKeys for the test
+type HTTPInvalidRequestLineTestKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (h HTTPInvalidRequestLine) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	testKeys := HTTPInvalidRequestLineTestKeys{IsAnomaly: false}
+
+	tampering, ok := tk["tampering"].(bool)
+	if !ok {
+		return testKeys, errors.New("tampering is not bool")
+	}
+	testKeys.IsAnomaly = tampering
+
+	return testKeys, nil
+}
+
+// LogSummary writes the summary to the standard output
+func (h HTTPInvalidRequestLine) LogSummary(s string) error {
+	return nil
+}

internal/nettests/ndt.go

@@ -0,0 +1,83 @@
+package nettests
+
+import (
+	"github.com/pkg/errors"
+)
+
+// NDT test implementation. We use v7 of NDT since 2020-03-12.
+type NDT struct {
+}
+
+// Run starts the test
+func (n NDT) Run(ctl *Controller) error {
+	// Since 2020-03-18 probe-engine exports v7 as "ndt".
+	builder, err := ctl.Session.NewExperimentBuilder("ndt")
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// NDTTestKeys for the test
+type NDTTestKeys struct {
+	Upload         float64 `json:"upload"`
+	Download       float64 `json:"download"`
+	Ping           float64 `json:"ping"`
+	MaxRTT         float64 `json:"max_rtt"`
+	AvgRTT         float64 `json:"avg_rtt"`
+	MinRTT         float64 `json:"min_rtt"`
+	MSS            float64 `json:"mss"`
+	RetransmitRate float64 `json:"retransmit_rate"`
+	IsAnomaly      bool    `json:"-"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (n NDT) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	var err error
+	testKeys := NDTTestKeys{IsAnomaly: false}
+
+	summary, ok := tk["summary"].(map[string]interface{})
+	if !ok {
+		return testKeys, errors.New("summary key is invalid")
+	}
+
+	// XXX there is likely a better pattern for this
+	testKeys.Upload, ok = summary["upload"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "upload key invalid")
+	}
+	testKeys.Download, ok = summary["download"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "download key invalid")
+	}
+	testKeys.Ping, ok = summary["ping"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "ping key invalid")
+	}
+	testKeys.MaxRTT, ok = summary["max_rtt"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "max_rtt key invalid")
+	}
+	testKeys.AvgRTT, ok = summary["avg_rtt"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "avg_rtt key invalid")
+	}
+	testKeys.MinRTT, ok = summary["min_rtt"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "min_rtt key invalid")
+	}
+	testKeys.MSS, ok = summary["mss"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "mss key invalid")
+	}
+	testKeys.RetransmitRate, ok = summary["retransmit_rate"].(float64)
+	if !ok {
+		err = errors.Wrap(err, "retransmit_rate key invalid")
+	}
+	return testKeys, err
+}
+
+// LogSummary writes the summary to the standard output
+func (n NDT) LogSummary(s string) error {
+	return nil
+}

internal/nettests/nettests.go

@@ -0,0 +1,220 @@
+package nettests
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/fatih/color"
+	ooni "github.com/ooni/probe-cli"
+	"github.com/ooni/probe-cli/internal/database"
+	"github.com/ooni/probe-cli/internal/output"
+	engine "github.com/ooni/probe-engine"
+	"github.com/ooni/probe-engine/model"
+	"github.com/pkg/errors"
+)
+
+// Nettest interface. Every Nettest should implement this.
+type Nettest interface {
+	Run(*Controller) error
+	GetTestKeys(map[string]interface{}) (interface{}, error)
+	LogSummary(string) error
+}
+
+// NewController creates a nettest controller
+func NewController(
+	nt Nettest, ctx *ooni.Context, res *database.Result, sess *engine.Session) *Controller {
+	return &Controller{
+		Ctx:     ctx,
+		nt:      nt,
+		res:     res,
+		Session: sess,
+	}
+}
+
+// Controller is passed to the run method of every Nettest
+// each nettest instance has one controller
+type Controller struct {
+	Ctx         *ooni.Context
+	Session     *engine.Session
+	res         *database.Result
+	nt          Nettest
+	ntCount     int
+	ntIndex     int
+	ntStartTime time.Time // used to calculate the eta
+	msmts       map[int64]*database.Measurement
+	inputIdxMap map[int64]int64 // Used to map mk idx to database id
+
+	// numInputs is the total number of inputs
+	numInputs int
+
+	// curInputIdx is the current input index
+	curInputIdx int
+}
+
+// SetInputIdxMap is used to set the mapping of index into input. This mapping
+// is used to reference, for example, a particular URL based on the index inside
+// of the input list and the index of it in the database.
+func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error {
+	c.inputIdxMap = inputIdxMap
+	return nil
+}
+
+// SetNettestIndex is used to set the current nettest index and total nettest
+// count to compute a different progress percentage.
+func (c *Controller) SetNettestIndex(i, n int) {
+	c.ntCount = n
+	c.ntIndex = i
+}
+
+// Run runs the selected nettest using the related experiment
+// with the specified inputs.
+//
+// This function will continue to run in most cases but will
+// immediately halt if something's wrong with the file system.
+func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) error {
+	// This will configure the controller as handler for the callbacks
+	// called by ooni/probe-engine/experiment.Experiment.
+	builder.SetCallbacks(model.ExperimentCallbacks(c))
+	c.numInputs = len(inputs)
+	exp := builder.NewExperiment()
+	defer func() {
+		c.res.DataUsageDown += exp.KibiBytesReceived()
+		c.res.DataUsageUp += exp.KibiBytesSent()
+	}()
+
+	c.msmts = make(map[int64]*database.Measurement)
+
+	// These values are shared by every measurement
+	var reportID sql.NullString
+	resultID := c.res.ID
+
+	log.Debug(color.RedString("status.queued"))
+	log.Debug(color.RedString("status.started"))
+
+	if c.Ctx.Config.Sharing.UploadResults {
+		if err := exp.OpenReport(); err != nil {
+			log.Debugf(
+				"%s: %s", color.RedString("failure.report_create"), err.Error(),
+			)
+		} else {
+			defer exp.CloseReport()
+			log.Debugf(color.RedString("status.report_create"))
+			reportID = sql.NullString{String: exp.ReportID(), Valid: true}
+		}
+	}
+
+	c.ntStartTime = time.Now()
+	for idx, input := range inputs {
+		if c.Ctx.IsTerminated() == true {
+			log.Debug("isTerminated == true, breaking the input loop")
+			break
+		}
+		c.curInputIdx = idx // allow for precise progress
+		idx64 := int64(idx)
+		log.Debug(color.RedString("status.measurement_start"))
+		var urlID sql.NullInt64
+		if c.inputIdxMap != nil {
+			urlID = sql.NullInt64{Int64: c.inputIdxMap[idx64], Valid: true}
+		}
+
+		msmt, err := database.CreateMeasurement(
+			c.Ctx.DB, reportID, exp.Name(), c.res.MeasurementDir, idx, resultID, urlID,
+		)
+		if err != nil {
+			return errors.Wrap(err, "failed to create measurement")
+		}
+		c.msmts[idx64] = msmt
+
+		if input != "" {
+			c.OnProgress(0, fmt.Sprintf("processing input: %s", input))
+		}
+		measurement, err := exp.Measure(input)
+		if err != nil {
+			log.WithError(err).Debug(color.RedString("failure.measurement"))
+			if err := c.msmts[idx64].Failed(c.Ctx.DB, err.Error()); err != nil {
+				return errors.Wrap(err, "failed to mark measurement as failed")
+			}
+			// Even with a failed measurement, we want to continue. We want to
+			// record and submit the information we have. Saving the information
+			// is useful for local inspection. Submitting it is useful to us to
+			// undertsand what went wrong (censorship? bug? anomaly?).
+		}
+
+		if c.Ctx.Config.Sharing.UploadResults {
+			// Implementation note: SubmitMeasurement will fail here if we did fail
+			// to open the report but we still want to continue. There will be a
+			// bit of a spew in the logs, perhaps, but stopping seems less efficient.
+			if err := exp.SubmitAndUpdateMeasurement(measurement); err != nil {
+				log.Debug(color.RedString("failure.measurement_submission"))
+				if err := c.msmts[idx64].UploadFailed(c.Ctx.DB, err.Error()); err != nil {
+					return errors.Wrap(err, "failed to mark upload as failed")
+				}
+			} else if err := c.msmts[idx64].UploadSucceeded(c.Ctx.DB); err != nil {
+				return errors.Wrap(err, "failed to mark upload as succeeded")
+			}
+		}
+
+		if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil {
+			return errors.Wrap(err, "failed to save measurement on disk")
+		}
+		if err := c.msmts[idx64].Done(c.Ctx.DB); err != nil {
+			return errors.Wrap(err, "failed to mark measurement as done")
+		}
+
+		// We're not sure whether it's enough to log the error or we should
+		// instead also mark the measurement as failed. Strictly speaking this
+		// is an inconsistency between the code that generate the measurement
+		// and the code that process the measurement. We do have some data
+		// but we're not gonna have a summary. To be reconsidered.
+		genericTk, err := measurement.MakeGenericTestKeys()
+		if err != nil {
+			log.WithError(err).Error("failed to cast the test keys")
+			continue
+		}
+		tk, err := c.nt.GetTestKeys(genericTk)
+		if err != nil {
+			log.WithError(err).Error("failed to obtain testKeys")
+			continue
+		}
+		log.Debugf("Fetching: %d %v", idx, c.msmts[idx64])
+		if err := database.AddTestKeys(c.Ctx.DB, c.msmts[idx64], tk); err != nil {
+			return errors.Wrap(err, "failed to add test keys to summary")
+		}
+	}
+
+	log.Debugf("status.end")
+	return nil
+}
+
+// OnProgress should be called when a new progress event is available.
+func (c *Controller) OnProgress(perc float64, msg string) {
+	log.Debugf("OnProgress: %f - %s", perc, msg)
+	var eta float64
+	eta = -1.0
+	if c.numInputs > 1 {
+		// make the percentage relative to the current input over all inputs
+		floor := (float64(c.curInputIdx) / float64(c.numInputs))
+		step := 1.0 / float64(c.numInputs)
+		perc = floor + perc*step
+		if c.curInputIdx > 0 {
+			eta = (time.Now().Sub(c.ntStartTime).Seconds() / float64(c.curInputIdx)) * float64(c.numInputs-c.curInputIdx)
+		}
+	}
+	if c.ntCount > 0 {
+		// make the percentage relative to the current nettest over all nettests
+		perc = float64(c.ntIndex)/float64(c.ntCount) + perc/float64(c.ntCount)
+	}
+	key := fmt.Sprintf("%T", c.nt)
+	output.Progress(key, perc, eta, msg)
+}
+
+// OnDataUsage should be called when we have a data usage update.
+func (c *Controller) OnDataUsage(dloadKiB, uploadKiB float64) {
+	// Unused as 2020-04-05: we're now using directly the accessors
+	// provided by the experiment. This callback is going to be removed
+	// from probe-engine in May or June.
+	//
+	// TODO(bassosimone): create an issue for this?
+}

internal/nettests/nettests_test.go

@@ -0,0 +1,52 @@
+package nettests
+
+import (
+	"io/ioutil"
+	"path"
+	"testing"
+
+	ooni "github.com/ooni/probe-cli"
+	"github.com/ooni/probe-cli/internal/database"
+	"github.com/ooni/probe-cli/utils/shutil"
+)
+
+func newTestingContext(t *testing.T) *ooni.Context {
+	homePath, err := ioutil.TempDir("", "ooniprobetests")
+	if err != nil {
+		t.Fatal(err)
+	}
+	configPath := path.Join(homePath, "config.json")
+	testingConfig := path.Join("..", "..", "testdata", "testing-config.json")
+	shutil.Copy(testingConfig, configPath, false)
+	ctx := ooni.NewContext(configPath, homePath)
+	swName := "ooniprobe-cli-tests"
+	swVersion := "3.0.0-alpha"
+	err = ctx.Init(swName, swVersion)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ctx
+}
+
+func TestCreateContext(t *testing.T) {
+	newTestingContext(t)
+}
+
+func TestRun(t *testing.T) {
+	ctx := newTestingContext(t)
+	sess, err := ctx.NewSession()
+	if err != nil {
+		t.Fatal(err)
+	}
+	network, err := database.CreateNetwork(ctx.DB, sess)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res, err := database.CreateResult(ctx.DB, ctx.Home, "middlebox", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	nt := HTTPInvalidRequestLine{}
+	ctl := NewController(nt, ctx, res, sess)
+	nt.Run(ctl)
+}

internal/nettests/psiphon.go

@@ -0,0 +1,51 @@
+package nettests
+
+import "github.com/pkg/errors"
+
+// Psiphon test implementation
+type Psiphon struct {
+}
+
+// Run starts the test
+func (h Psiphon) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"psiphon",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// PsiphonTestKeys contains the test keys
+type PsiphonTestKeys struct {
+	IsAnomaly     bool    `json:"-"`
+	BootstrapTime float64 `json:"bootstrap_time"`
+	Failure       string  `json:"failure"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (h Psiphon) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	var (
+		ok       bool
+		testKeys PsiphonTestKeys
+	)
+	if tk["failure"] != nil {
+		testKeys.IsAnomaly = true
+		failure, ok := tk["failure"].(*string)
+		if !ok {
+			return testKeys, errors.New("failure key invalid")
+		}
+		testKeys.Failure = *failure
+	}
+	testKeys.BootstrapTime, ok = tk["bootstrap_time"].(float64)
+	if !ok {
+		return testKeys, errors.New("bootstrap_time key invalid")
+	}
+	return testKeys, nil
+}
+
+// LogSummary writes the summary to the standard output
+func (h Psiphon) LogSummary(s string) error {
+	return nil
+}

internal/nettests/telegram.go

@@ -0,0 +1,61 @@
+package nettests
+
+// Telegram test implementation
+type Telegram struct {
+}
+
+// Run starts the test
+func (h Telegram) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"telegram",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// TelegramTestKeys for the test
+type TelegramTestKeys struct {
+	HTTPBlocking bool `json:"telegram_http_blocking"`
+	TCPBlocking  bool `json:"telegram_tcp_blocking"`
+	WebBlocking  bool `json:"telegram_web_blocking"`
+	IsAnomaly    bool `json:"-"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (h Telegram) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	var (
+		tcpBlocking  bool
+		httpBlocking bool
+		webBlocking  bool
+	)
+
+	if tk["telegram_tcp_blocking"] == nil {
+		tcpBlocking = false
+	} else {
+		tcpBlocking = tk["telegram_tcp_blocking"].(bool)
+	}
+	if tk["telegram_http_blocking"] == nil {
+		httpBlocking = false
+	} else {
+		httpBlocking = tk["telegram_http_blocking"].(bool)
+	}
+	if tk["telegram_web_status"] == nil {
+		webBlocking = false
+	} else {
+		webBlocking = tk["telegram_web_status"].(string) == "blocked"
+	}
+
+	return TelegramTestKeys{
+		TCPBlocking:  tcpBlocking,
+		HTTPBlocking: httpBlocking,
+		WebBlocking:  webBlocking,
+		IsAnomaly:    webBlocking || httpBlocking || tcpBlocking,
+	}, nil
+}
+
+// LogSummary writes the summary to the standard output
+func (h Telegram) LogSummary(s string) error {
+	return nil
+}

internal/nettests/tor.go

@@ -0,0 +1,72 @@
+package nettests
+
+// Tor test implementation
+type Tor struct {
+}
+
+// Run starts the test
+func (h Tor) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"tor",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// TorTestKeys contains the test keys
+type TorTestKeys struct {
+	DirPortTotal            int64 `json:"dir_port_total"`
+	DirPortAccessible       int64 `json:"dir_port_accessible"`
+	IsAnomaly               bool  `json:"-"`
+	OBFS4Total              int64 `json:"obfs4_total"`
+	OBFS4Accessible         int64 `json:"obfs4_accessible"`
+	ORPortDirauthTotal      int64 `json:"or_port_dirauth_total"`
+	ORPortDirauthAccessible int64 `json:"or_port_dirauth_accessible"`
+	ORPortTotal             int64 `json:"or_port_total"`
+	ORPortAccessible        int64 `json:"or_port_accessible"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (h Tor) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	testKeys := TorTestKeys{IsAnomaly: false}
+	// Implementation note: when Go marshals into an interface, it marshals to
+	// float64 rather than int64, so we need to do some more work here.
+	//
+	// See <https://golang.org/pkg/encoding/json/#Unmarshal>.
+	if tk["dir_port_total"] != nil {
+		testKeys.DirPortTotal = int64(tk["dir_port_total"].(float64))
+	}
+	if tk["dir_port_accessible"] != nil {
+		testKeys.DirPortAccessible = int64(tk["dir_port_accessible"].(float64))
+	}
+	if tk["obfs4_total"] != nil {
+		testKeys.OBFS4Total = int64(tk["obfs4_total"].(float64))
+	}
+	if tk["obfs4_accessible"] != nil {
+		testKeys.OBFS4Accessible = int64(tk["obfs4_accessible"].(float64))
+	}
+	if tk["or_port_dirauth_total"] != nil {
+		testKeys.ORPortDirauthTotal = int64(tk["or_port_dirauth_total"].(float64))
+	}
+	if tk["or_port_dirauth_accessible"] != nil {
+		testKeys.ORPortDirauthAccessible = int64(tk["or_port_dirauth_accessible"].(float64))
+	}
+	if tk["or_port_total"] != nil {
+		testKeys.ORPortTotal = int64(tk["or_port_total"].(float64))
+	}
+	if tk["or_port_accessible"] != nil {
+		testKeys.ORPortAccessible = int64(tk["or_port_accessible"].(float64))
+	}
+	testKeys.IsAnomaly = ((testKeys.DirPortAccessible <= 0 && testKeys.DirPortTotal > 0) ||
+		(testKeys.OBFS4Accessible <= 0 && testKeys.OBFS4Total > 0) ||
+		(testKeys.ORPortDirauthAccessible <= 0 && testKeys.ORPortDirauthTotal > 0) ||
+		(testKeys.ORPortAccessible <= 0 && testKeys.ORPortTotal > 0))
+	return testKeys, nil
+}
+
+// LogSummary writes the summary to the standard output
+func (h Tor) LogSummary(s string) error {
+	return nil
+}

internal/nettests/web_connectivity.go

@@ -0,0 +1,101 @@
+package nettests
+
+import (
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/internal/database"
+	engine "github.com/ooni/probe-engine"
+)
+
+func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
+	var urls []string
+	urlIDMap := make(map[int64]int64)
+	testlist, err := ctl.Session.QueryTestListsURLs(&engine.TestListsURLsConfig{
+		Limit:      limit,
+		Categories: categories,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	for idx, url := range testlist.Result {
+		log.Debugf("Going over URL %d", idx)
+		urlID, err := database.CreateOrUpdateURL(
+			ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode,
+		)
+		if err != nil {
+			log.Error("failed to add to the URL table")
+			return nil, nil, err
+		}
+		log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
+		urlIDMap[int64(idx)] = urlID
+		urls = append(urls, url.URL)
+	}
+	return urls, urlIDMap, nil
+}
+
+// WebConnectivity test implementation
+type WebConnectivity struct {
+}
+
+// Run starts the test
+func (n WebConnectivity) Run(ctl *Controller) error {
+	log.Debugf("Enabled category codes are the following %v", ctl.Ctx.Config.Nettests.WebsitesEnabledCategoryCodes)
+	urls, urlIDMap, err := lookupURLs(ctl, ctl.Ctx.Config.Nettests.WebsitesURLLimit, ctl.Ctx.Config.Nettests.WebsitesEnabledCategoryCodes)
+	if err != nil {
+		return err
+	}
+	ctl.SetInputIdxMap(urlIDMap)
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"web_connectivity",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, urls)
+}
+
+// WebConnectivityTestKeys for the test
+type WebConnectivityTestKeys struct {
+	Accessible bool   `json:"accessible"`
+	Blocking   string `json:"blocking"`
+	IsAnomaly  bool   `json:"-"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (n WebConnectivity) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	var (
+		blocked    bool
+		blocking   string
+		accessible bool
+	)
+
+	// We need to do these complicated type assertions, because some of the fields
+	// are "nullable" and/or can be of different types
+	switch v := tk["blocking"].(type) {
+	case bool:
+		blocked = false
+		blocking = "none"
+	case string:
+		blocked = true
+		blocking = v
+	default:
+		blocked = false
+		blocking = "none"
+	}
+
+	if tk["accessible"] == nil {
+		accessible = false
+	} else {
+		accessible = tk["accessible"].(bool)
+	}
+
+	return WebConnectivityTestKeys{
+		Accessible: accessible,
+		Blocking:   blocking,
+		IsAnomaly:  blocked,
+	}, nil
+}
+
+// LogSummary writes the summary to the standard output
+func (n WebConnectivity) LogSummary(s string) error {
+	return nil
+}

internal/nettests/whatsapp.go

@@ -0,0 +1,59 @@
+package nettests
+
+// WhatsApp test implementation
+type WhatsApp struct {
+}
+
+// Run starts the test
+func (h WhatsApp) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"whatsapp",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}
+
+// WhatsAppTestKeys for the test
+type WhatsAppTestKeys struct {
+	RegistrationServerBlocking bool `json:"registration_server_blocking"`
+	WebBlocking                bool `json:"whatsapp_web_blocking"`
+	EndpointsBlocking          bool `json:"whatsapp_endpoints_blocking"`
+	IsAnomaly                  bool `json:"-"`
+}
+
+// GetTestKeys generates a summary for a test run
+func (h WhatsApp) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+	var (
+		webBlocking          bool
+		registrationBlocking bool
+		endpointsBlocking    bool
+	)
+
+	var computeBlocking = func(key string) bool {
+		const blk = "blocked"
+		if tk[key] == nil {
+			return false
+		}
+		if tk[key].(string) == blk {
+			return true
+		}
+		return false
+	}
+	registrationBlocking = computeBlocking("registration_server_status")
+	webBlocking = computeBlocking("whatsapp_web_status")
+	endpointsBlocking = computeBlocking("whatsapp_endpoints_status")
+
+	return WhatsAppTestKeys{
+		RegistrationServerBlocking: registrationBlocking,
+		WebBlocking:                webBlocking,
+		EndpointsBlocking:          endpointsBlocking,
+		IsAnomaly:                  registrationBlocking || webBlocking || endpointsBlocking,
+	}, nil
+}
+
+// LogSummary writes the summary to the standard output
+func (h WhatsApp) LogSummary(s string) error {
+	return nil
+}