Use more general ooni/probe-engine API (#67)

No functional change, just importing less stuff and meddling
much less with the internals of ooni/probe-engine.

nettests/im/facebook_messenger.go

@@ -2,7 +2,6 @@ package im
 
 import (
 	"github.com/ooni/probe-cli/nettests"
-	"github.com/ooni/probe-engine/experiment/fbmessenger"
 )
 
 // FacebookMessenger test implementation
@@ -11,10 +10,16 @@ type FacebookMessenger struct {
 
 // Run starts the test
 func (h FacebookMessenger) Run(ctl *nettests.Controller) error {
-	experiment := fbmessenger.NewExperiment(ctl.Ctx.Session, fbmessenger.Config{
-		LogLevel: "INFO",
-	})
-	return ctl.Run(experiment, []string{""})
+	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+		"facebook_messenger",
+	)
+	if err != nil {
+		return err
+	}
+	if err := builder.SetOptionString("LogLevel", "INFO"); err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
 }
 
 // FacebookMessengerTestKeys for the test

nettests/im/telegram.go

@@ -2,7 +2,6 @@ package im
 
 import (
 	"github.com/ooni/probe-cli/nettests"
-	"github.com/ooni/probe-engine/experiment/telegram"
 )
 
 // Telegram test implementation
@@ -11,10 +10,16 @@ type Telegram struct {
 
 // Run starts the test
 func (h Telegram) Run(ctl *nettests.Controller) error {
-	experiment := telegram.NewExperiment(ctl.Ctx.Session, telegram.Config{
-		LogLevel: "INFO",
-	})
-	return ctl.Run(experiment, []string{""})
+	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+		"telegram",
+	)
+	if err != nil {
+		return err
+	}
+	if err := builder.SetOptionString("LogLevel", "INFO"); err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
 }
 
 // TelegramTestKeys for the test

nettests/im/whatsapp.go

@@ -2,7 +2,6 @@ package im
 
 import (
 	"github.com/ooni/probe-cli/nettests"
-	"github.com/ooni/probe-engine/experiment/whatsapp"
 )
 
 // WhatsApp test implementation
@@ -11,10 +10,16 @@ type WhatsApp struct {
 
 // Run starts the test
 func (h WhatsApp) Run(ctl *nettests.Controller) error {
-	experiment := whatsapp.NewExperiment(ctl.Ctx.Session, whatsapp.Config{
-		LogLevel: "INFO",
-	})
-	return ctl.Run(experiment, []string{""})
+	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+		"whatsapp",
+	)
+	if err != nil {
+		return err
+	}
+	if err := builder.SetOptionString("LogLevel", "INFO"); err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
 }
 
 // WhatsAppTestKeys for the test

nettests/middlebox/http_header_field_manipulation.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 
 	"github.com/ooni/probe-cli/nettests"
-	"github.com/ooni/probe-engine/experiment/hhfm"
 )
 
 // HTTPHeaderFieldManipulation test implementation
@@ -13,10 +12,16 @@ type HTTPHeaderFieldManipulation struct {
 
 // Run starts the test
 func (h HTTPHeaderFieldManipulation) Run(ctl *nettests.Controller) error {
-	experiment := hhfm.NewExperiment(ctl.Ctx.Session, hhfm.Config{
-		LogLevel: "INFO",
-	})
-	return ctl.Run(experiment, []string{""})
+	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+		"http_header_field_manipulation",
+	)
+	if err != nil {
+		return err
+	}
+	if err := builder.SetOptionString("LogLevel", "INFO"); err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
 }
 
 // HTTPHeaderFieldManipulationTestKeys for the test

nettests/middlebox/http_invalid_request_line.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 
 	"github.com/ooni/probe-cli/nettests"
-	"github.com/ooni/probe-engine/experiment/hirl"
 )
 
 // HTTPInvalidRequestLine test implementation
@@ -13,10 +12,16 @@ type HTTPInvalidRequestLine struct {
 
 // Run starts the test
 func (h HTTPInvalidRequestLine) Run(ctl *nettests.Controller) error {
-	experiment := hirl.NewExperiment(ctl.Ctx.Session, hirl.Config{
-		LogLevel: "INFO",
-	})
-	return ctl.Run(experiment, []string{""})
+	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+		"http_invalid_request_line",
+	)
+	if err != nil {
+		return err
+	}
+	if err := builder.SetOptionString("LogLevel", "INFO"); err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
 }
 
 // HTTPInvalidRequestLineTestKeys for the test

nettests/nettests.go

@@ -1,7 +1,6 @@
 package nettests
 
 import (
-	"context"
 	"database/sql"
 	"fmt"
 	"path/filepath"
@@ -11,12 +10,9 @@ import (
 	"github.com/fatih/color"
 	ooni "github.com/ooni/probe-cli"
 	"github.com/ooni/probe-cli/internal/database"
-	"github.com/ooni/probe-cli/internal/enginex"
 	"github.com/ooni/probe-cli/internal/output"
 	"github.com/ooni/probe-cli/utils"
-	"github.com/ooni/probe-engine/experiment"
-	"github.com/ooni/probe-engine/experiment/handler"
-	"github.com/ooni/probe-engine/model"
+	engine "github.com/ooni/probe-engine"
 	"github.com/pkg/errors"
 )
 
@@ -79,13 +75,13 @@ func (c *Controller) SetNettestIndex(i, n int) {
 //
 // This function will continue to run in most cases but will
 // immediately halt if something's wrong with the file system.
-func (c *Controller) Run(exp *experiment.Experiment, inputs []string) error {
-	ctx := context.Background()
+func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) error {
 
 	// This will configure the controller as handler for the callbacks
 	// called by ooni/probe-engine/experiment.Experiment.
-	exp.Callbacks = handler.Callbacks(c)
+	builder.SetCallbacks(engine.Callbacks(c))
 	c.numInputs = len(inputs)
+	exp := builder.Build()
 
 	c.msmts = make(map[int64]*database.Measurement)
 
@@ -98,12 +94,12 @@ func (c *Controller) Run(exp *experiment.Experiment, inputs []string) error {
 	log.Debugf("OutputPath: %s", c.msmtPath)
 
 	if c.Ctx.Config.Sharing.UploadResults {
-		if err := exp.OpenReport(ctx); err != nil {
+		if err := exp.OpenReport(); err != nil {
 			log.Debugf(
 				"%s: %s", color.RedString("failure.report_create"), err.Error(),
 			)
 		} else {
-			defer exp.CloseReport(ctx)
+			defer exp.CloseReport()
 			log.Debugf(color.RedString("status.report_create"))
 			reportID = sql.NullString{String: exp.ReportID(), Valid: true}
 		}
@@ -118,14 +114,14 @@ func (c *Controller) Run(exp *experiment.Experiment, inputs []string) error {
 			urlID = sql.NullInt64{Int64: c.inputIdxMap[idx64], Valid: true}
 		}
 		msmt, err := database.CreateMeasurement(
-			c.Ctx.DB, reportID, exp.TestName, resultID, c.msmtPath, urlID,
+			c.Ctx.DB, reportID, exp.Name(), resultID, c.msmtPath, urlID,
 		)
 		if err != nil {
 			return errors.Wrap(err, "failed to create measurement")
 		}
 		c.msmts[idx64] = msmt
 
-		measurement, err := exp.Measure(ctx, input)
+		measurement, err := exp.Measure(input)
 		if err != nil {
 			log.WithError(err).Debug(color.RedString("failure.measurement"))
 			if err := c.msmts[idx64].Failed(c.Ctx.DB, err.Error()); err != nil {
@@ -134,22 +130,11 @@ func (c *Controller) Run(exp *experiment.Experiment, inputs []string) error {
 			continue
 		}
 
-		// Make sure we share what the user wants us to share.
-		if c.Ctx.Config.Sharing.IncludeIP == false {
-			measurement.ProbeIP = model.DefaultProbeIP
-		}
-		if c.Ctx.Config.Sharing.IncludeASN == false {
-			measurement.ProbeASN = fmt.Sprintf("AS%d", model.DefaultProbeASN)
-		}
-		if c.Ctx.Config.Sharing.IncludeCountry == false {
-			measurement.ProbeCC = model.DefaultProbeCC
-		}
-
 		if c.Ctx.Config.Sharing.UploadResults {
 			// Implementation note: SubmitMeasurement will fail here if we did fail
 			// to open the report but we still want to continue. There will be a
 			// bit of a spew in the logs, perhaps, but stopping seems less efficient.
-			if err := exp.SubmitMeasurement(ctx, &measurement); err != nil {
+			if err := exp.SubmitAndUpdateMeasurement(measurement); err != nil {
 				log.Debug(color.RedString("failure.measurement_submission"))
 				if err := c.msmts[idx64].UploadFailed(c.Ctx.DB, err.Error()); err != nil {
 					return errors.Wrap(err, "failed to mark upload as failed")
@@ -171,7 +156,7 @@ func (c *Controller) Run(exp *experiment.Experiment, inputs []string) error {
 		// is an inconsistency between the code that generate the measurement
 		// and the code that process the measurement. We do have some data
 		// but we're not gonna have a summary. To be reconsidered.
-		genericTk, err := enginex.MakeGenericTestKeys(measurement)
+		genericTk, err := measurement.MakeGenericTestKeys()
 		if err != nil {
 			log.WithError(err).Error("failed to cast the test keys")
 			continue
@@ -204,7 +189,7 @@ func (c *Controller) OnProgress(perc float64, msg string) {
 		// make the percentage relative to the current input over all inputs
 		floor := (float64(c.curInputIdx) / float64(c.numInputs))
 		step := 1.0 / float64(c.numInputs)
-		perc = floor + perc * step
+		perc = floor + perc*step
 	}
 	if c.ntCount > 0 {
 		// make the percentage relative to the current nettest over all nettests

nettests/performance/dash.go

@@ -4,7 +4,6 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/ooni/probe-cli/nettests"
-	"github.com/ooni/probe-engine/experiment/dash"
 )
 
 // Dash test implementation
@@ -13,10 +12,11 @@ type Dash struct {
 
 // Run starts the test
 func (d Dash) Run(ctl *nettests.Controller) error {
-	experiment := dash.NewExperiment(
-		ctl.Ctx.Session, dash.Config{},
-	)
-	return ctl.Run(experiment, []string{""})
+	builder, err := ctl.Ctx.Session.NewExperimentBuilder("dash")
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
 }
 
 // DashTestKeys for the test

nettests/performance/ndt.go

@@ -2,7 +2,6 @@ package performance
 
 import (
 	"github.com/ooni/probe-cli/nettests"
-	"github.com/ooni/probe-engine/experiment/ndt"
 	"github.com/pkg/errors"
 )
 
@@ -12,10 +11,11 @@ type NDT struct {
 
 // Run starts the test
 func (n NDT) Run(ctl *nettests.Controller) error {
-	experiment := ndt.NewExperiment(
-		ctl.Ctx.Session, ndt.Config{},
-	)
-	return ctl.Run(experiment, []string{""})
+	builder, err := ctl.Ctx.Session.NewExperimentBuilder("ndt")
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
 }
 
 // NDTTestKeys for the test

nettests/websites/web_connectivity.go

@@ -1,28 +1,25 @@
 package websites
 
 import (
-	"context"
-
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/internal/database"
 	"github.com/ooni/probe-cli/nettests"
-	"github.com/ooni/probe-engine/experiment/web_connectivity"
-	"github.com/ooni/probe-engine/orchestra/testlists"
 )
 
 func lookupURLs(ctl *nettests.Controller, limit int) ([]string, map[int64]int64, error) {
 	var urls []string
 	urlIDMap := make(map[int64]int64)
-	testlist, err := testlists.NewClient(ctl.Ctx.Session).Do(
-		context.Background(), ctl.Ctx.Session.ProbeCC(), limit,
-	)
+	config := ctl.Ctx.Session.NewTestListsConfig()
+	config.Limit = limit
+	client := ctl.Ctx.Session.NewTestListsClient()
+	testlist, err := client.Fetch(config)
 	if err != nil {
 		return nil, nil, err
 	}
 	for idx, url := range testlist {
 		log.Debugf("Going over URL %d", idx)
 		urlID, err := database.CreateOrUpdateURL(
-			ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode,
+			ctl.Ctx.DB, url.URL(), url.CategoryCode(), url.CountryCode(),
 		)
 		if err != nil {
 			log.Error("failed to add to the URL table")
@@ -30,7 +27,7 @@ func lookupURLs(ctl *nettests.Controller, limit int) ([]string, map[int64]int64,
 		}
 		log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
 		urlIDMap[int64(idx)] = urlID
-		urls = append(urls, url.URL)
+		urls = append(urls, url.URL())
 	}
 	return urls, urlIDMap, nil
 }
@@ -46,11 +43,16 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error {
 		return err
 	}
 	ctl.SetInputIdxMap(urlIDMap)
-	experiment := web_connectivity.NewExperiment(
-		ctl.Ctx.Session,
-		web_connectivity.Config{LogLevel: "INFO"},
+	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+		"web_connectivity",
 	)
-	return ctl.Run(experiment, urls)
+	if err != nil {
+		return err
+	}
+	if err := builder.SetOptionString("LogLevel", "INFO"); err != nil {
+		return err
+	}
+	return ctl.Run(builder, urls)
 }
 
 // WebConnectivityTestKeys for the test