Basic SMTP+StartTLS / SMTPS test

See https://github.com/ooni/probe/issues/2352

Co-authored-by: decfox <decfox@github.com>

CLI/go-build-generic

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+	echo "" 1>&2
+	echo "Compiler for a Go PACKAGE producing GOOS/GOARCH binaries in the" 1>&2
+	echo "top-level directory (should not be used for releasing)." 1>&2
+	echo "" 1>&2
+	echo "usage: $0 PACKAGE..." 1>&2
+	echo "" 1>&2
+	echo "Features:" 1>&2
+	echo "" 1>&2
+	echo "* automatically sets -tags=ooni_psiphon_config when possible;" 1>&2
+	echo "" 1>&2
+	echo "* if GOLANG_EXTRA_FLAGS is set, pass it to the Go compiler." 1>&2
+	echo "" 1>&2
+	echo "Example:" 1>&2
+	echo "" 1>&2
+	echo "  ./CLI/go-build-generic ./internal/cmd/miniooni" 1>&2
+	echo "" 1>&2
+	exit 1
+fi
+
+if [[ -f ./internal/engine/psiphon-config.json.age &&
+	-f ./internal/engine/psiphon-config.key ]]; then
+	OONI_PSIPHON_TAGS=ooni_psiphon_config
+else
+	OONI_PSIPHON_TAGS=""
+fi
+
+set -x
+for pkg in "$@"; do
+	go build -tags=$OONI_PSIPHON_TAGS -ldflags="-s -w" ${GOLANG_EXTRA_FLAGS:-} $pkg
+done

Makefile

@@ -129,6 +129,20 @@ CLI/linux-static-armv7: search/for/docker maybe/copypsiphon
 CLI/linux-static-arm64: search/for/docker maybe/copypsiphon
 	./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) arm64 ./cmd/ooniprobe ./internal/cmd/miniooni
 
+#help:
+#help: The `make CLI/miniooni` command creates a build of miniooni, for the current
+#help: system, putting the binary in the top-level directory.
+.PHONY: CLI/miniooni
+CLI/miniooni: maybe/copypsiphon search/for/go
+	./CLI/go-build-generic ./internal/cmd/miniooni
+
+#help:
+#help: The `make CLI/ooniprobe` command creates a build of ooniprobe, for the current
+#help: system, putting the binary in the top-level directory.
+.PHONY: CLI/ooniprobe
+CLI/ooniprobe: maybe/copypsiphon search/for/go
+	./CLI/go-build-generic ./cmd/ooniprobe
+
 #help:
 #help: The `make CLI/windows` command builds the ooniprobe and miniooni
 #help: command line clients for windows/386 and windows/amd64.

PULL_REQUEST_TEMPLATE.md

@@ -3,6 +3,7 @@
 - [ ] I have read the [contribution guidelines](https://github.com/ooni/probe-cli/blob/master/CONTRIBUTING.md)
 - [ ] reference issue for this pull request: <!-- add URL here -->
 - [ ] if you changed anything related how experiments work and you need to reflect these changes in the ooni/spec repository, please link to the related ooni/spec pull request: <!-- add URL here -->
+- [ ] if you change code inside an experiment, make sure you bump its version number
 
 <!-- Reminder: Location of the issue tracker: https://github.com/ooni/probe -->
 

cmd/ooniprobe/internal/cli/list/list.go

@@ -6,7 +6,6 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
 )
 
@@ -20,7 +19,7 @@ func init() {
 			return err
 		}
 		if *resultID > 0 {
-			measurements, err := database.ListMeasurements(probeCLI.DB(), *resultID)
+			measurements, err := probeCLI.DB().ListMeasurements(*resultID)
 			if err != nil {
 				log.WithError(err).Error("failed to list measurements")
 				return err
@@ -63,7 +62,7 @@ func init() {
 			}
 			output.MeasurementSummary(msmtSummary)
 		} else {
-			doneResults, incompleteResults, err := database.ListResults(probeCLI.DB())
+			doneResults, incompleteResults, err := probeCLI.DB().ListResults()
 			if err != nil {
 				log.WithError(err).Error("failed to list results")
 				return err

cmd/ooniprobe/internal/cli/reset/reset.go

@@ -25,7 +25,7 @@ func init() {
 			log.WithError(err).Error("failed to close the DB")
 			return err
 		}
-		if *force == true {
+		if *force {
 			os.RemoveAll(ctx.Home())
 			log.Infof("Deleted %s", ctx.Home())
 		} else {

cmd/ooniprobe/internal/cli/rm/rm.go

@@ -8,11 +8,11 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/internal/database"
 	"github.com/upper/db/v4"
 )
 
-func deleteAll(sess db.Session, skipInteractive bool) error {
+func deleteAll(d *database.Database, skipInteractive bool) error {
 	if skipInteractive == false {
 		answer := ""
 		confirm := &survey.Select{
@@ -25,21 +25,21 @@ func deleteAll(sess db.Session, skipInteractive bool) error {
 			return errors.New("canceled by user")
 		}
 	}
-	doneResults, incompleteResults, err := database.ListResults(sess)
+	doneResults, incompleteResults, err := d.ListResults()
 	if err != nil {
 		log.WithError(err).Error("failed to list results")
 		return err
 	}
 	cnt := 0
 	for _, result := range incompleteResults {
-		err = database.DeleteResult(sess, result.Result.ID)
+		err = d.DeleteResult(result.Result.ID)
 		if err == db.ErrNoMoreRows {
 			log.WithError(err).Errorf("failed to delete result #%d", result.Result.ID)
 		}
 		cnt++
 	}
 	for _, result := range doneResults {
-		err = database.DeleteResult(sess, result.Result.ID)
+		err = d.DeleteResult(result.Result.ID)
 		if err == db.ErrNoMoreRows {
 			log.WithError(err).Errorf("failed to delete result #%d", result.Result.ID)
 		}
@@ -68,7 +68,7 @@ func init() {
 		}
 
 		if *yes == true {
-			err = database.DeleteResult(ctx.DB(), *resultID)
+			err = ctx.DB().DeleteResult(*resultID)
 			if err == db.ErrNoMoreRows {
 				return errors.New("result not found")
 			}
@@ -84,7 +84,7 @@ func init() {
 		if answer == "false" {
 			return errors.New("canceled by user")
 		}
-		err = database.DeleteResult(ctx.DB(), *resultID)
+		err = ctx.DB().DeleteResult(*resultID)
 		if err == db.ErrNoMoreRows {
 			return errors.New("result not found")
 		}

cmd/ooniprobe/internal/cli/root/root.go

@@ -44,6 +44,9 @@ func init() {
 	softwareVersion := Cmd.Flag(
 		"software-version", "Override the application version",
 	).Default(version.Version).String()
+	proxy := Cmd.Flag(
+		"proxy", "specify a proxy address for speaking to the OONI Probe backend (use: --proxy=psiphon:/// for psiphon)",
+	).String()
 
 	Cmd.PreAction(func(ctx *kingpin.ParseContext) error {
 		// TODO(bassosimone): we need to properly deprecate --batch
@@ -78,7 +81,7 @@ func init() {
 			}
 
 			probe := ooni.NewProbe(*configPath, homePath)
-			err = probe.Init(*softwareName, *softwareVersion)
+			err = probe.Init(*softwareName, *softwareVersion, *proxy)
 			if err != nil {
 				return nil, err
 			}

cmd/ooniprobe/internal/cli/show/show.go

@@ -4,7 +4,6 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
 )
 
@@ -17,7 +16,7 @@ func init() {
 			log.WithError(err).Error("failed to initialize root context")
 			return err
 		}
-		msmt, err := database.GetMeasurementJSON(ctx.DB(), *msmtID)
+		msmt, err := ctx.DB().GetMeasurementJSON(*msmtID)
 		if err != nil {
 			log.Errorf("error: %v", err)
 			return err

cmd/ooniprobe/internal/log/handlers/cli/results.go

@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+	"github.com/ooni/probe-cli/v3/internal/database"
 )
 
 func formatSpeed(speed float64) string {

cmd/ooniprobe/internal/nettests/dnscheck.go

@@ -25,7 +25,7 @@ func (n DNSCheck) lookupURLs(ctl *Controller) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
+	return ctl.BuildAndSetInputIdxMap(testlist)
 }
 
 // Run starts the nettest.

cmd/ooniprobe/internal/nettests/nettests.go

@@ -8,13 +8,12 @@ import (
 
 	"github.com/apex/log"
 	"github.com/fatih/color"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
+	"github.com/ooni/probe-cli/v3/internal/database"
 	engine "github.com/ooni/probe-cli/v3/internal/engine"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/pkg/errors"
-	"github.com/upper/db/v4"
 )
 
 // Nettest interface. Every Nettest should implement this.
@@ -90,14 +89,13 @@ type Controller struct {
 // - on success, a list of strings containing URLs to test;
 //
 // - on failure, an error.
-func (c *Controller) BuildAndSetInputIdxMap(
+func (c *Controller) BuildAndSetInputIdxMap(testlist []model.OOAPIURLInfo) ([]string, error) {
-	sess db.Session, testlist []model.OOAPIURLInfo) ([]string, error) {
 	var urls []string
 	urlIDMap := make(map[int64]int64)
 	for idx, url := range testlist {
 		log.Debugf("Going over URL %d", idx)
-		urlID, err := database.CreateOrUpdateURL(
+		urlID, err := c.Probe.DB().CreateOrUpdateURL(
-			sess, url.URL, url.CategoryCode, url.CountryCode,
+			url.URL, url.CategoryCode, url.CountryCode,
 		)
 		if err != nil {
 			log.Error("failed to add to the URL table")
@@ -124,6 +122,7 @@ func (c *Controller) SetNettestIndex(i, n int) {
 // This function will continue to run in most cases but will
 // immediately halt if something's wrong with the file system.
 func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error {
+	db := c.Probe.DB()
 	// This will configure the controller as handler for the callbacks
 	// called by ooni/probe-engine/experiment.Experiment.
 	builder.SetCallbacks(model.ExperimentCallbacks(c))
@@ -168,6 +167,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
 		log.Debug("disabling maxRuntime with user-provided input")
 		maxRuntime = 0
 	}
+	sess := db.Session()
 	start := time.Now()
 	c.ntStartTime = start
 	for idx, input := range inputs {
@@ -187,8 +187,8 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
 			urlID = sql.NullInt64{Int64: c.inputIdxMap[idx64], Valid: true}
 		}
 
-		msmt, err := database.CreateMeasurement(
+		msmt, err := db.CreateMeasurement(
-			c.Probe.DB(), reportID, exp.Name(), c.res.MeasurementDir, idx, resultID, urlID,
+			reportID, exp.Name(), c.res.MeasurementDir, idx, resultID, urlID,
 		)
 		if err != nil {
 			return errors.Wrap(err, "failed to create measurement")
@@ -201,7 +201,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
 		measurement, err := exp.MeasureWithContext(context.Background(), input)
 		if err != nil {
 			log.WithError(err).Debug(color.RedString("failure.measurement"))
-			if err := c.msmts[idx64].Failed(c.Probe.DB(), err.Error()); err != nil {
+			if err := c.msmts[idx64].Failed(sess, err.Error()); err != nil {
 				return errors.Wrap(err, "failed to mark measurement as failed")
 			}
 			// Since https://github.com/ooni/probe-cli/pull/527, the Measure
@@ -221,10 +221,10 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
 			// bit of a spew in the logs, perhaps, but stopping seems less efficient.
 			if err := exp.SubmitAndUpdateMeasurementContext(context.Background(), measurement); err != nil {
 				log.Debug(color.RedString("failure.measurement_submission"))
-				if err := c.msmts[idx64].UploadFailed(c.Probe.DB(), err.Error()); err != nil {
+				if err := c.msmts[idx64].UploadFailed(sess, err.Error()); err != nil {
 					return errors.Wrap(err, "failed to mark upload as failed")
 				}
-			} else if err := c.msmts[idx64].UploadSucceeded(c.Probe.DB()); err != nil {
+			} else if err := c.msmts[idx64].UploadSucceeded(sess); err != nil {
 				return errors.Wrap(err, "failed to mark upload as succeeded")
 			} else {
 				// Everything went OK, don't save to disk
@@ -238,7 +238,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
 			}
 		}
 
-		if err := c.msmts[idx64].Done(c.Probe.DB()); err != nil {
+		if err := c.msmts[idx64].Done(sess); err != nil {
 			return errors.Wrap(err, "failed to mark measurement as done")
 		}
 
@@ -253,11 +253,11 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
 			continue
 		}
 		log.Debugf("Fetching: %d %v", idx, c.msmts[idx64])
-		if err := database.AddTestKeys(c.Probe.DB(), c.msmts[idx64], tk); err != nil {
+		if err := db.AddTestKeys(c.msmts[idx64], tk); err != nil {
 			return errors.Wrap(err, "failed to add test keys to summary")
 		}
 	}
-	database.UpdateUploadedStatus(c.Probe.DB(), c.res)
+	db.UpdateUploadedStatus(c.res)
 	log.Debugf("status.end")
 	return nil
 }

cmd/ooniprobe/internal/nettests/nettests_test.go

@@ -7,7 +7,6 @@ import (
 	"path"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
 	"github.com/ooni/probe-cli/v3/internal/model"
 )
@@ -33,7 +32,7 @@ func newOONIProbe(t *testing.T) *ooni.Probe {
 	probe := ooni.NewProbe(configPath, homePath)
 	swName := "ooniprobe-cli-tests"
 	swVersion := "3.0.0-alpha"
-	err = probe.Init(swName, swVersion)
+	err = probe.Init(swName, swVersion, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -53,11 +52,12 @@ func TestRun(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	network, err := database.CreateNetwork(probe.DB(), sess)
+	db := probe.DB()
+	network, err := db.CreateNetwork(sess)
 	if err != nil {
 		t.Fatal(err)
 	}
-	res, err := database.CreateResult(probe.DB(), probe.Home(), "middlebox", network.ID)
+	res, err := db.CreateResult(probe.Home(), "middlebox", network.ID)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/ooniprobe/internal/nettests/run.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/pkg/errors"
@@ -72,7 +71,8 @@ func RunGroup(config RunGroupConfig) error {
 		log.WithError(err).Error("Failed to lookup the location of the probe")
 		return err
 	}
-	network, err := database.CreateNetwork(config.Probe.DB(), sess)
+	db := config.Probe.DB()
+	network, err := db.CreateNetwork(sess)
 	if err != nil {
 		log.WithError(err).Error("Failed to create the network row")
 		return err
@@ -89,8 +89,8 @@ func RunGroup(config RunGroupConfig) error {
 	}
 	log.Debugf("Running test group %s", group.Label)
 
-	result, err := database.CreateResult(
+	result, err := db.CreateResult(
-		config.Probe.DB(), config.Probe.Home(), config.GroupName, network.ID)
+		config.Probe.Home(), config.GroupName, network.ID)
 	if err != nil {
 		log.Errorf("DB result error: %s", err)
 		return err
@@ -131,8 +131,8 @@ func RunGroup(config RunGroupConfig) error {
 	if err != nil {
 		os.Remove(result.MeasurementDir)
 	}
-
+	dbSess := db.Session()
-	if err = result.Finished(config.Probe.DB()); err != nil {
+	if err = result.Finished(dbSess); err != nil {
 		return err
 	}
 	return nil

cmd/ooniprobe/internal/nettests/stunreachability.go

@@ -25,7 +25,7 @@ func (n STUNReachability) lookupURLs(ctl *Controller) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
+	return ctl.BuildAndSetInputIdxMap(testlist)
 }
 
 // Run starts the nettest.

cmd/ooniprobe/internal/nettests/web_connectivity.go

@@ -31,7 +31,7 @@ func (n WebConnectivity) lookupURLs(ctl *Controller, categories []string) ([]str
 	if err != nil {
 		return nil, err
 	}
-	return ctl.BuildAndSetInputIdxMap(ctl.Probe.DB(), testlist)
+	return ctl.BuildAndSetInputIdxMap(testlist)
 }
 
 // WebConnectivity test implementation

cmd/ooniprobe/internal/ooni/ooni.go

@@ -4,31 +4,35 @@ import (
 	"context"
 	_ "embed" // because we embed a file
 	"io/ioutil"
+	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
 
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
 	"github.com/ooni/probe-cli/v3/internal/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/database"
 	"github.com/ooni/probe-cli/v3/internal/engine"
 	"github.com/ooni/probe-cli/v3/internal/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/legacy/assetsdir"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/pkg/errors"
-	"github.com/upper/db/v4"
 )
 
 // DefaultSoftwareName is the default software name.
 const DefaultSoftwareName = "ooniprobe-cli"
 
+// logger is the logger used by the engine.
+var logger = log.WithFields(log.Fields{
+	"type": "engine",
+})
+
 // ProbeCLI is the OONI Probe CLI context.
 type ProbeCLI interface {
 	Config() *config.Config
-	DB() db.Session
+	DB() *database.Database
 	IsBatch() bool
 	Home() string
 	TempDir() string
@@ -48,7 +52,7 @@ type ProbeEngine interface {
 // Probe contains the ooniprobe CLI context.
 type Probe struct {
 	config  *config.Config
-	db      db.Session
+	db      *database.Database
 	isBatch bool
 
 	home      string
@@ -62,6 +66,7 @@ type Probe struct {
 
 	softwareName    string
 	softwareVersion string
+	proxyURL        *url.URL
 }
 
 // SetIsBatch sets the value of isBatch.
@@ -80,7 +85,7 @@ func (p *Probe) Config() *config.Config {
 }
 
 // DB returns the database we're using
-func (p *Probe) DB() db.Session {
+func (p *Probe) DB() *database.Database {
 	return p.db
 }
 
@@ -124,7 +129,7 @@ func (p *Probe) ListenForSignals() {
 // MaybeListenForStdinClosed will treat any error on stdin just
 // like SIGTERM if and only if
 //
-//     os.Getenv("OONI_STDIN_EOF_IMPLIES_SIGTERM") == "true"
+//	os.Getenv("OONI_STDIN_EOF_IMPLIES_SIGTERM") == "true"
 //
 // When this feature is enabled, a collateral effect is that we swallow
 // whatever is passed to us on the standard input.
@@ -151,7 +156,7 @@ func (p *Probe) MaybeListenForStdinClosed() {
 }
 
 // Init the OONI manager
-func (p *Probe) Init(softwareName, softwareVersion string) error {
+func (p *Probe) Init(softwareName, softwareVersion, proxy string) error {
 	var err error
 
 	if err = MaybeInitializeHome(p.home); err != nil {
@@ -174,7 +179,7 @@ func (p *Probe) Init(softwareName, softwareVersion string) error {
 
 	p.dbPath = utils.DBDir(p.home, "main")
 	log.Debugf("Connecting to database sqlite3://%s", p.dbPath)
-	db, err := database.Connect(p.dbPath)
+	db, err := database.Open(p.dbPath)
 	if err != nil {
 		return err
 	}
@@ -197,6 +202,12 @@ func (p *Probe) Init(softwareName, softwareVersion string) error {
 
 	p.softwareName = softwareName
 	p.softwareVersion = softwareVersion
+	if proxy != "" {
+		p.proxyURL, err = url.Parse(proxy)
+		if err != nil {
+			return errors.Wrap(err, "invalid proxy URL")
+		}
+	}
 	return nil
 }
 
@@ -223,11 +234,12 @@ func (p *Probe) NewSession(ctx context.Context, runType model.RunType) (*engine.
 	}
 	return engine.NewSession(ctx, engine.SessionConfig{
 		KVStore:         kvstore,
-		Logger:          enginex.Logger,
+		Logger:          logger,
 		SoftwareName:    softwareName,
 		SoftwareVersion: p.softwareVersion,
 		TempDir:         p.tempDir,
 		TunnelDir:       p.tunnelDir,
+		ProxyURL:        p.proxyURL,
 	})
 }
 

cmd/ooniprobe/internal/ooni/ooni_test.go

@@ -17,7 +17,7 @@ func TestInit(t *testing.T) {
 	probe := NewProbe("", ooniHome)
 	swName := "ooniprobe-cli-tests"
 	swVersion := "3.0.0-alpha"
-	if err := probe.Init(swName, swVersion); err != nil {
+	if err := probe.Init(swName, swVersion, ""); err != nil {
 		t.Error(err)
 		t.Fatal("failed to init the context")
 	}

cmd/ooniprobe/internal/oonitest/oonitest.go

@@ -8,8 +8,8 @@ import (
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/ooni/probe-cli/v3/internal/database"
 	"github.com/ooni/probe-cli/v3/internal/model"
-	"github.com/upper/db/v4"
 )
 
 // FakeOutput allows to fake the output package.
@@ -28,7 +28,7 @@ func (fo *FakeOutput) SectionTitle(s string) {
 // FakeProbeCLI fakes ooni.ProbeCLI
 type FakeProbeCLI struct {
 	FakeConfig         *config.Config
-	FakeDB             db.Session
+	FakeDB             *database.Database
 	FakeIsBatch        bool
 	FakeHome           string
 	FakeTempDir        string
@@ -42,7 +42,7 @@ func (cli *FakeProbeCLI) Config() *config.Config {
 }
 
 // DB implements ProbeCLI.DB
-func (cli *FakeProbeCLI) DB() db.Session {
+func (cli *FakeProbeCLI) DB() *database.Database {
 	return cli.FakeDB
 }
 

cmd/ooniprobe/internal/output/output.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/apex/log"
 	"github.com/mitchellh/go-wordwrap"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/internal/database"
 )
 
 // MeasurementJSON prints the JSON of a measurement

cmd/ooniprobe/internal/utils/paths.go

@@ -1,11 +1,9 @@
 package utils
 
 import (
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
-	"time"
 
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils/homedir"
 )
@@ -53,26 +51,6 @@ func FileExists(path string) bool {
 	return err == nil && stat.Mode().IsRegular()
 }
 
-// ResultTimestamp is a windows friendly timestamp
-const ResultTimestamp = "2006-01-02T150405.999999999Z0700"
-
-// MakeResultsDir creates and returns a directory for the result
-func MakeResultsDir(home string, name string, ts time.Time) (string, error) {
-	p := filepath.Join(home, "msmts",
-		fmt.Sprintf("%s-%s", name, ts.Format(ResultTimestamp)))
-
-	// If the path already exists, this is a problem. It should not clash, because
-	// we are using nanosecond precision for the starttime.
-	if _, e := os.Stat(p); e == nil {
-		return "", errors.New("results path already exists")
-	}
-	err := os.MkdirAll(p, 0700)
-	if err != nil {
-		return "", err
-	}
-	return p, nil
-}
-
 // GetOONIHome returns the path to the OONI Home
 func GetOONIHome() (string, error) {
 	if ooniHome := os.Getenv("OONI_HOME"); ooniHome != "" {
@@ -96,8 +74,5 @@ func DidLegacyInformedConsent() bool {
 	}
 
 	path := filepath.Join(filepath.Join(home, ".ooni"), "initialized")
-	if FileExists(path) {
+	return FileExists(path)
-		return true
-	}
-	return false
 }

internal/database/actions.go

@@ -12,16 +12,36 @@ import (
 	"time"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex"
+	"github.com/ooni/probe-cli/v3/internal/engine"
-	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
 	"github.com/pkg/errors"
 	"github.com/upper/db/v4"
 )
 
+// Open returns a new database instance
+func Open(dbpath string) (*Database, error) {
+	db, err := Connect(dbpath)
+	if err != nil {
+		return nil, err
+	}
+	return &Database{
+		sess: db,
+	}, nil
+}
+
+// Database is a database instance to store measurements
+type Database struct {
+	sess db.Session
+}
+
+// Session returns the database session
+func (d *Database) Session() db.Session {
+	return d.sess
+}
+
 // ListMeasurements given a result ID
-func ListMeasurements(sess db.Session, resultID int64) ([]MeasurementURLNetwork, error) {
+func (d *Database) ListMeasurements(resultID int64) ([]MeasurementURLNetwork, error) {
 	measurements := []MeasurementURLNetwork{}
-	req := sess.SQL().Select(
+	req := d.sess.SQL().Select(
 		db.Raw("networks.*"),
 		db.Raw("urls.*"),
 		db.Raw("measurements.*"),
@@ -40,12 +60,12 @@ func ListMeasurements(sess db.Session, resultID int64) ([]MeasurementURLNetwork,
 }
 
 // GetMeasurementJSON returns a map[string]interface{} given a database and a measurementID
-func GetMeasurementJSON(sess db.Session, measurementID int64) (map[string]interface{}, error) {
+func (d *Database) GetMeasurementJSON(measurementID int64) (map[string]interface{}, error) {
 	var (
 		measurement MeasurementURLNetwork
 		msmtJSON    map[string]interface{}
 	)
-	req := sess.SQL().Select(
+	req := d.sess.SQL().Select(
 		db.Raw("urls.*"),
 		db.Raw("measurements.*"),
 	).From("measurements").
@@ -65,7 +85,7 @@ func GetMeasurementJSON(sess db.Session, measurementID int64) (map[string]interf
 		}
 		query := url.Values{}
 		query.Add("report_id", reportID)
-		if measurement.URL.URL.Valid == true {
+		if measurement.URL.URL.Valid {
 			query.Add("input", measurement.URL.URL.String)
 		}
 		measurementURL.RawQuery = query.Encode()
@@ -84,7 +104,7 @@ func GetMeasurementJSON(sess db.Session, measurementID int64) (map[string]interf
 	}
 	// MeasurementFilePath might be NULL because the measurement from a
 	// 3.0.0-beta install
-	if measurement.Measurement.MeasurementFilePath.Valid == false {
+	if !measurement.Measurement.MeasurementFilePath.Valid {
 		log.Error("invalid measurement_file_path")
 		log.Error("backup your OONI_HOME and run `ooniprobe reset`")
 		return nil, errors.New("cannot access measurement file")
@@ -103,10 +123,10 @@ func GetMeasurementJSON(sess db.Session, measurementID int64) (map[string]interf
 }
 
 // ListResults return the list of results
-func ListResults(sess db.Session) ([]ResultNetwork, []ResultNetwork, error) {
+func (d *Database) ListResults() ([]ResultNetwork, []ResultNetwork, error) {
 	doneResults := []ResultNetwork{}
 	incompleteResults := []ResultNetwork{}
-	req := sess.SQL().Select(
+	req := d.sess.SQL().Select(
 		db.Raw("networks.network_name"),
 		db.Raw("networks.network_type"),
 		db.Raw("networks.ip"),
@@ -166,9 +186,9 @@ func ListResults(sess db.Session) ([]ResultNetwork, []ResultNetwork, error) {
 
 // DeleteResult will delete a particular result and the relative measurement on
 // disk.
-func DeleteResult(sess db.Session, resultID int64) error {
+func (d *Database) DeleteResult(resultID int64) error {
 	var result Result
-	res := sess.Collection("results").Find("result_id", resultID)
+	res := d.sess.Collection("results").Find("result_id", resultID)
 	if err := res.One(&result); err != nil {
 		if err == db.ErrNoMoreRows {
 			return err
@@ -186,8 +206,8 @@ func DeleteResult(sess db.Session, resultID int64) error {
 }
 
 // UpdateUploadedStatus will check if all the measurements inside of a given result set have been uploaded and if so will set the is_uploaded flag to true
-func UpdateUploadedStatus(sess db.Session, result *Result) error {
+func (d *Database) UpdateUploadedStatus(result *Result) error {
-	err := sess.Tx(func(tx db.Session) error {
+	err := d.sess.Tx(func(tx db.Session) error {
 		uploadedTotal := UploadedTotalCount{}
 		req := tx.SQL().Select(
 			db.Raw("SUM(measurements.measurement_is_uploaded)"),
@@ -223,7 +243,7 @@ func UpdateUploadedStatus(sess db.Session, result *Result) error {
 
 // CreateMeasurement writes the measurement to the database a returns a pointer
 // to the Measurement
-func CreateMeasurement(sess db.Session, reportID sql.NullString, testName string, measurementDir string, idx int, resultID int64, urlID sql.NullInt64) (*Measurement, error) {
+func (d *Database) CreateMeasurement(reportID sql.NullString, testName string, measurementDir string, idx int, resultID int64, urlID sql.NullInt64) (*Measurement, error) {
 	// TODO we should look into generating this file path in a more robust way.
 	// If there are two identical test_names in the same test group there is
 	// going to be a clash of test_name
@@ -241,7 +261,7 @@ func CreateMeasurement(sess db.Session, reportID sql.NullString, testName string
 		TestKeys:  "",
 	}
 
-	newID, err := sess.Collection("measurements").Insert(msmt)
+	newID, err := d.sess.Collection("measurements").Insert(msmt)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating measurement")
 	}
@@ -251,10 +271,10 @@ func CreateMeasurement(sess db.Session, reportID sql.NullString, testName string
 
 // CreateResult writes the Result to the database a returns a pointer
 // to the Result
-func CreateResult(sess db.Session, homePath string, testGroupName string, networkID int64) (*Result, error) {
+func (d *Database) CreateResult(homePath string, testGroupName string, networkID int64) (*Result, error) {
 	startTime := time.Now().UTC()
 
-	p, err := utils.MakeResultsDir(homePath, testGroupName, startTime)
+	p, err := makeResultsDir(homePath, testGroupName, startTime)
 	if err != nil {
 		return nil, err
 	}
@@ -267,7 +287,7 @@ func CreateResult(sess db.Session, homePath string, testGroupName string, networ
 	result.MeasurementDir = p
 	log.Debugf("Creating result %v", result)
 
-	newID, err := sess.Collection("results").Insert(result)
+	newID, err := d.sess.Collection("results").Insert(result)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating result")
 	}
@@ -276,7 +296,7 @@ func CreateResult(sess db.Session, homePath string, testGroupName string, networ
 }
 
 // CreateNetwork will create a new network in the network table
-func CreateNetwork(sess db.Session, loc enginex.LocationProvider) (*Network, error) {
+func (d *Database) CreateNetwork(loc engine.LocationProvider) (*Network, error) {
 	network := Network{
 		ASN:         loc.ProbeASN(),
 		CountryCode: loc.ProbeCC(),
@@ -285,7 +305,7 @@ func CreateNetwork(sess db.Session, loc enginex.LocationProvider) (*Network, err
 		NetworkType: "wifi",
 		IP:          loc.ProbeIP(),
 	}
-	newID, err := sess.Collection("networks").Insert(network)
+	newID, err := d.sess.Collection("networks").Insert(network)
 	if err != nil {
 		return nil, err
 	}
@@ -297,10 +317,10 @@ func CreateNetwork(sess db.Session, loc enginex.LocationProvider) (*Network, err
 // CreateOrUpdateURL will create a new URL entry to the urls table if it doesn't
 // exists, otherwise it will update the category code of the one already in
 // there.
-func CreateOrUpdateURL(sess db.Session, urlStr string, categoryCode string, countryCode string) (int64, error) {
+func (d *Database) CreateOrUpdateURL(urlStr string, categoryCode string, countryCode string) (int64, error) {
 	var url URL
 
-	err := sess.Tx(func(tx db.Session) error {
+	err := d.sess.Tx(func(tx db.Session) error {
 		res := tx.Collection("urls").Find(
 			db.Cond{"url": urlStr, "url_country_code": countryCode},
 		)
@@ -337,7 +357,7 @@ func CreateOrUpdateURL(sess db.Session, urlStr string, categoryCode string, coun
 }
 
 // AddTestKeys writes the summary to the measurement
-func AddTestKeys(sess db.Session, msmt *Measurement, tk interface{}) error {
+func (d *Database) AddTestKeys(msmt *Measurement, tk interface{}) error {
 	var (
 		isAnomaly      bool
 		isAnomalyValid bool
@@ -351,17 +371,22 @@ func AddTestKeys(sess db.Session, msmt *Measurement, tk interface{}) error {
 	// the IsAnomaly field of bool type.
 	// Maybe generics are not so bad after-all, heh golang?
 	isAnomalyValue := reflect.ValueOf(tk).FieldByName("IsAnomaly")
-	if isAnomalyValue.IsValid() == true && isAnomalyValue.Kind() == reflect.Bool {
+	if isAnomalyValue.IsValid() && isAnomalyValue.Kind() == reflect.Bool {
 		isAnomaly = isAnomalyValue.Bool()
 		isAnomalyValid = true
 	}
 	msmt.TestKeys = string(tkBytes)
 	msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: isAnomalyValid}
 
-	err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
+	err = d.sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
 	if err != nil {
 		log.WithError(err).Error("failed to update measurement")
 		return errors.Wrap(err, "updating measurement")
 	}
 	return nil
 }
+
+// Close closes the database session
+func (d *Database) Close() error {
+	return d.sess.Close()
+}

internal/database/actions_test.go

@@ -43,6 +43,31 @@ func (lp *locationInfo) ResolverIP() string {
 	return lp.resolverIP
 }
 
+func TestNewDatabase(t *testing.T) {
+	t.Run("with empty path", func(t *testing.T) {
+		dbpath := ""
+		database, err := Open(dbpath)
+		if database != nil {
+			t.Fatal("unexpected database instance")
+		}
+		if err.Error() != "Expecting file:// connection scheme." {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("with valid path", func(t *testing.T) {
+		tmpfile, err := ioutil.TempFile("", "dbtest")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.Remove(tmpfile.Name())
+		_, err = Open(tmpfile.Name())
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
 func TestMeasurementWorkflow(t *testing.T) {
 	tmpfile, err := ioutil.TempFile("", "dbtest")
 	if err != nil {
@@ -56,7 +81,7 @@ func TestMeasurementWorkflow(t *testing.T) {
 	}
 	defer os.RemoveAll(tmpdir)
 
-	sess, err := Connect(tmpfile.Name())
+	database, err := Open(tmpfile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -66,12 +91,13 @@ func TestMeasurementWorkflow(t *testing.T) {
 		countryCode: "IT",
 		networkName: "Unknown",
 	}
-	network, err := CreateNetwork(sess, &location)
+	sess := database.Session()
+	network, err := database.CreateNetwork(&location)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	result, err := database.CreateResult(tmpdir, "websites", network.ID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -82,7 +108,7 @@ func TestMeasurementWorkflow(t *testing.T) {
 	msmtFilePath := tmpdir
 	urlID := sql.NullInt64{Int64: 0, Valid: false}
 
-	m1, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	m1, err := database.CreateMeasurement(reportID, testName, msmtFilePath, 0, resultID, urlID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -93,7 +119,7 @@ func TestMeasurementWorkflow(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	m2, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	m2, err := database.CreateMeasurement(reportID, testName, msmtFilePath, 0, resultID, urlID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -107,7 +133,7 @@ func TestMeasurementWorkflow(t *testing.T) {
 	if m2.ResultID != m1.ResultID {
 		t.Error("result_id mismatch")
 	}
-	err = UpdateUploadedStatus(sess, result)
+	err = database.UpdateUploadedStatus(result)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -122,7 +148,7 @@ func TestMeasurementWorkflow(t *testing.T) {
 		t.Error("result should be marked as not uploaded")
 	}
 
-	done, incomplete, err := ListResults(sess)
+	done, incomplete, err := database.ListResults()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -141,7 +167,7 @@ func TestMeasurementWorkflow(t *testing.T) {
 		t.Error("there should be a total of 1 anomalies in the result")
 	}
 
-	msmts, err := ListMeasurements(sess, resultID)
+	msmts, err := database.ListMeasurements(resultID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -163,7 +189,7 @@ func TestDeleteResult(t *testing.T) {
 	}
 	defer os.RemoveAll(tmpdir)
 
-	sess, err := Connect(tmpfile.Name())
+	database, err := Open(tmpfile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -173,12 +199,13 @@ func TestDeleteResult(t *testing.T) {
 		countryCode: "IT",
 		networkName: "Unknown",
 	}
-	network, err := CreateNetwork(sess, &location)
+	sess := database.Session()
+	network, err := database.CreateNetwork(&location)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	result, err := database.CreateResult(tmpdir, "websites", network.ID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -189,7 +216,7 @@ func TestDeleteResult(t *testing.T) {
 	msmtFilePath := tmpdir
 	urlID := sql.NullInt64{Int64: 0, Valid: false}
 
-	m1, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	m1, err := database.CreateMeasurement(reportID, testName, msmtFilePath, 0, resultID, urlID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -204,7 +231,7 @@ func TestDeleteResult(t *testing.T) {
 		t.Error("result_id mismatch")
 	}
 
-	err = DeleteResult(sess, resultID)
+	err = database.DeleteResult(resultID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -223,7 +250,7 @@ func TestDeleteResult(t *testing.T) {
 		t.Fatal("measurements should be zero")
 	}
 
-	err = DeleteResult(sess, 20)
+	err = database.DeleteResult(20)
 	if err != db.ErrNoMoreRows {
 		t.Fatal(err)
 	}
@@ -236,7 +263,7 @@ func TestNetworkCreate(t *testing.T) {
 	}
 	defer os.Remove(tmpfile.Name())
 
-	sess, err := Connect(tmpfile.Name())
+	database, err := Open(tmpfile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -253,12 +280,12 @@ func TestNetworkCreate(t *testing.T) {
 		networkName: "Fufnet",
 	}
 
-	_, err = CreateNetwork(sess, &l1)
+	_, err = database.CreateNetwork(&l1)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	_, err = CreateNetwork(sess, &l2)
+	_, err = database.CreateNetwork(&l2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -272,32 +299,32 @@ func TestURLCreation(t *testing.T) {
 	}
 	defer os.Remove(tmpfile.Name())
 
-	sess, err := Connect(tmpfile.Name())
+	database, err := Open(tmpfile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	newID1, err := CreateOrUpdateURL(sess, "https://google.com", "GMB", "XX")
+	newID1, err := database.CreateOrUpdateURL("https://google.com", "GMB", "XX")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	newID2, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX")
+	newID2, err := database.CreateOrUpdateURL("https://google.com", "SRCH", "XX")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	newID3, err := CreateOrUpdateURL(sess, "https://facebook.com", "GRP", "XX")
+	newID3, err := database.CreateOrUpdateURL("https://facebook.com", "GRP", "XX")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	newID4, err := CreateOrUpdateURL(sess, "https://facebook.com", "GMP", "XX")
+	newID4, err := database.CreateOrUpdateURL("https://facebook.com", "GMP", "XX")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	newID5, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX")
+	newID5, err := database.CreateOrUpdateURL("https://google.com", "SRCH", "XX")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -351,22 +378,22 @@ func TestGetMeasurementJSON(t *testing.T) {
 	}
 	defer os.RemoveAll(tmpdir)
 
-	sess, err := Connect(tmpfile.Name())
+	database, err := Open(tmpfile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
-
+	sess := database.Session()
 	location := locationInfo{
 		asn:         0,
 		countryCode: "IT",
 		networkName: "Unknown",
 	}
-	network, err := CreateNetwork(sess, &location)
+	network, err := database.CreateNetwork(&location)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	result, err := database.CreateResult(tmpdir, "websites", network.ID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -377,7 +404,7 @@ func TestGetMeasurementJSON(t *testing.T) {
 	msmtFilePath := tmpdir
 	urlID := sql.NullInt64{Int64: 0, Valid: false}
 
-	msmt, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	msmt, err := database.CreateMeasurement(reportID, testName, msmtFilePath, 0, resultID, urlID)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -387,7 +414,7 @@ func TestGetMeasurementJSON(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	tk, err := GetMeasurementJSON(sess, msmt.ID)
+	tk, err := database.GetMeasurementJSON(msmt.ID)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/database/models.go

@@ -99,7 +99,7 @@ type PerformanceTestKeys struct {
 
 // Finished marks the result as done and sets the runtime
 func (r *Result) Finished(sess db.Session) error {
-	if r.IsDone == true || r.Runtime != 0 {
+	if r.IsDone || r.Runtime != 0 {
 		return errors.New("Result is already finished")
 	}
 	r.Runtime = time.Now().UTC().Sub(r.StartTime).Seconds()

internal/database/utils.go

@@ -0,0 +1,29 @@
+package database
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// resultTimestamp is a windows friendly timestamp
+const resultTimestamp = "2006-01-02T150405.999999999Z0700"
+
+// makeResultsDir creates and returns a directory for the result
+func makeResultsDir(home string, name string, ts time.Time) (string, error) {
+	p := filepath.Join(home, "msmts",
+		fmt.Sprintf("%s-%s", name, ts.Format(resultTimestamp)))
+
+	// If the path already exists, this is a problem. It should not clash, because
+	// we are using nanosecond precision for the starttime.
+	if _, e := os.Stat(p); e == nil {
+		return "", errors.New("results path already exists")
+	}
+	err := os.MkdirAll(p, 0700)
+	if err != nil {
+		return "", err
+	}
+	return p, nil
+}

internal/engine/experiment/smtp/smtp.go

@@ -0,0 +1,314 @@
+package smtp
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"github.com/pkg/errors"
+	"net"
+	"net/smtp"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/measurexlite"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/tracex"
+)
+
+var (
+	// errNoInputProvided indicates you didn't provide any input
+	errNoInputProvided = errors.New("not input provided")
+
+	// errInputIsNotAnURL indicates that input is not an URL
+	errInputIsNotAnURL = errors.New("input is not an URL")
+
+	// errInvalidScheme indicates that the scheme is invalid
+	errInvalidScheme = errors.New("scheme must be smtp(s)")
+
+	// errInvalidPort indicates that the port provided could not be parsed as an int
+	errInvalidPort = errors.New("Port number is not a valid integer")
+)
+
+const (
+	testName    = "smtp"
+	testVersion = "0.0.1"
+)
+
+// Config contains the experiment config.
+type Config struct {
+	host       string
+	port       string
+	forced_tls bool
+	noop_count uint8
+}
+
+func config(input model.MeasurementTarget) (*Config, error) {
+	if input == "" {
+		// TODO: static input data (eg. gmail/riseup..)
+		return nil, errNoInputProvided
+	}
+
+	parsed, err := url.Parse(string(input))
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error())
+	}
+	if parsed.Scheme != "smtp" && parsed.Scheme != "smtps" {
+		return nil, errInvalidScheme
+	}
+
+	port := ""
+
+	if parsed.Port() == "" {
+		// Default ports for StartTLS and forced TLS respectively
+		if parsed.Scheme == "smtp" {
+			port = "587"
+		} else {
+			port = "465"
+		}
+	} else {
+		// Check that requested port is a valid integer
+		_, err := strconv.Atoi(parsed.Port())
+		if err != nil {
+			return nil, errInvalidPort
+		} else {
+			port = parsed.Port()
+		}
+	}
+
+	valid_config := Config{
+		host:       parsed.Hostname(),
+		forced_tls: parsed.Scheme == "smtps",
+		port:       port,
+		noop_count: 10,
+	}
+
+	return &valid_config, nil
+}
+
+// TestKeys contains the experiment results
+type TestKeys struct {
+	Queries       []*model.ArchivalDNSLookupResult          `json:"queries"`
+	TCPConnect    []*model.ArchivalTCPConnectResult         `json:"tcp_connect"`
+	TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"`
+	SMTPErrors    []*string                                 `json:"smtp"`
+	NoOpCounter   uint8                                     `json:"successful_noops"`
+	// Used for global failure (DNS resolution)
+	Failure string `json:"failure"`
+}
+
+type Measurer struct {
+	// Config contains the experiment settings. If empty we
+	// will be using default settings.
+	Config Config
+
+	// Getter is an optional getter to be used for testing.
+	Getter urlgetter.MultiGetter
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+type SMTPRunner struct {
+	trace     *measurexlite.Trace
+	logger    model.Logger
+	ctx       context.Context
+	tk        *TestKeys
+	tlsconfig *tls.Config
+}
+
+func (r SMTPRunner) resolve(host string) ([]string, bool) {
+	r.logger.Infof("Resolving DNS for %s", host)
+	resolver := r.trace.NewStdlibResolver(r.logger)
+	addrs, err := resolver.LookupHost(r.ctx, host)
+	r.tk.Queries = append(r.tk.Queries, r.trace.DNSLookupsFromRoundTrip()...)
+	if err != nil {
+		r.tk.Failure = *tracex.NewFailure(err)
+		return []string{}, false
+	}
+	r.logger.Infof("Finished DNS for %s: %v", host, addrs)
+
+	return addrs, true
+}
+
+func (r SMTPRunner) conn(addr string, port string) (net.Conn, bool) {
+	dialer := r.trace.NewDialerWithoutResolver(r.logger)
+	conn, err := dialer.DialContext(r.ctx, "tcp", net.JoinHostPort(addr, port))
+	r.tk.TCPConnect = append(r.tk.TCPConnect, r.trace.TCPConnects()...)
+	if err != nil {
+		r.tk.Failure = *tracex.NewFailure(err)
+		return nil, false
+	}
+	return conn, true
+}
+
+func (r SMTPRunner) handshake(conn net.Conn, host string, port string) (net.Conn, bool) {
+	r.logger.Infof("Starting TLS handshake with %s:%s", host, port)
+	thx := r.trace.NewTLSHandshakerStdlib(r.logger)
+	tconn, _, err := thx.Handshake(r.ctx, conn, r.tlsconfig)
+	r.tk.TLSHandshakes = append(r.tk.TLSHandshakes, r.trace.FirstTLSHandshakeOrNil())
+	if err != nil {
+		return nil, false
+	}
+	r.logger.Infof("Handshake succeeded")
+	return tconn, true
+}
+
+func (r SMTPRunner) starttls(conn net.Conn, host string, port string, message string) (net.Conn, bool) {
+	r.logger.Infof("Asking for StartTLS upgrade")
+	conn.Write([]byte(message))
+	tconn, success := r.handshake(conn, host, port)
+	return tconn, success
+}
+
+func (r SMTPRunner) smtp(conn net.Conn, ehlo string, noop uint8) bool {
+	client, err := smtp.NewClient(conn, ehlo)
+	if err != nil {
+		r.tk.SMTPErrors = append(r.tk.SMTPErrors, []*string{tracex.NewFailure(err)}...)
+		return false
+	}
+	err = client.Hello(ehlo)
+	if err != nil {
+		r.tk.SMTPErrors = append(r.tk.SMTPErrors, []*string{tracex.NewFailure(err)}...)
+		return false
+	}
+
+	if noop > 0 {
+		r.logger.Infof("Trying to generate more no-op traffic")
+		// TODO: noop counter per IP address
+		r.tk.NoOpCounter = 0
+		for r.tk.NoOpCounter < noop {
+			r.tk.NoOpCounter += 1
+			r.logger.Infof("NoOp Iteration %d", r.tk.NoOpCounter)
+			err = client.Noop()
+			if err != nil {
+				r.tk.SMTPErrors = append(r.tk.SMTPErrors, []*string{tracex.NewFailure(err)}...)
+				break
+			}
+		}
+
+		if r.tk.NoOpCounter == noop {
+			r.logger.Infof("Successfully generated no-op traffic")
+			return true
+		} else {
+			r.logger.Infof("Failed no-op traffic at iteration %d", r.tk.NoOpCounter)
+			return false
+		}
+	}
+
+	return true
+}
+
+// Run implements ExperimentMeasurer.Run
+func (m Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+
+	log := sess.Logger()
+	trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved)
+
+	config, err := config(measurement.Input)
+	if err != nil {
+		// Invalid input data, we don't even generate report
+		return err
+	}
+
+	tk := new(TestKeys)
+	measurement.TestKeys = tk
+
+	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+
+	tlsconfig := tls.Config{
+		InsecureSkipVerify: false,
+		ServerName:         config.host,
+	}
+
+	runner := SMTPRunner{
+		trace:     trace,
+		logger:    log,
+		ctx:       ctx,
+		tk:        tk,
+		tlsconfig: &tlsconfig,
+	}
+
+	// First resolve DNS
+	addrs, success := runner.resolve(config.host)
+	if !success {
+		return nil
+	}
+
+	for _, addr := range addrs {
+		conn, success := runner.conn(addr, config.port)
+		if !success {
+			return nil
+		}
+		defer conn.Close()
+
+		if config.forced_tls {
+			// Direct TLS connection
+			tconn, success := runner.handshake(conn, config.host, config.port)
+			if !success {
+				continue
+			}
+			defer tconn.Close()
+
+			// Try EHLO + NoOps
+			if !runner.smtp(tconn, "localhost", 10) {
+				continue
+			}
+		} else {
+			// StartTLS... first try plaintext EHLO
+			if !runner.smtp(conn, "localhost", 0) {
+				continue
+			}
+
+			// Upgrade via StartTLS and try EHLO + NoOps
+			tconn, success := runner.starttls(conn, config.host, config.port, "STARTTLS\n")
+			if !success {
+				continue
+			}
+			defer tconn.Close()
+
+			if !runner.smtp(tconn, "localhost", 10) {
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with ooniprobe
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	//DNSBlocking bool `json:"facebook_dns_blocking"`
+	//TCPBlocking bool `json:"facebook_tcp_blocking"`
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	_, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	return sk, nil
+}

internal/engine/location.go

@@ -1,14 +1,4 @@
-// Package enginex contains ooni/probe-engine extensions.
+package engine
-package enginex
-
-import (
-	"github.com/apex/log"
-)
-
-// Logger is the logger used by the engine.
-var Logger = log.WithFields(log.Fields{
-	"type": "engine",
-})
 
 // LocationProvider is an interface that returns the current location. The
 // github.com/ooni/probe-cli/v3/internal/engine/session.Session implements it.

internal/registry/smtp.go

@@ -0,0 +1,22 @@
+package registry
+
+//
+// Registers the `dnsping' experiment.
+//
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/smtp"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func init() {
+	AllExperiments["smtp"] = &Factory{
+		build: func(config interface{}) model.ExperimentMeasurer {
+			return smtp.NewExperimentMeasurer(
+				*config.(*smtp.Config),
+			)
+		},
+		config:      &smtp.Config{},
+		inputPolicy: model.InputOrStaticDefault,
+	}
+}