Basic SMTP/IMAP probe: look for bad TLS and StartTLS downgrade attacks

Closes https://github.com/ooni/probe/issues/2358.

internal/engine/experiment.go

@@ -92,7 +92,12 @@ func (eaw *experimentAsyncWrapper) RunAsync(
 	out := make(chan *model.ExperimentAsyncTestKeys)
 	measurement := eaw.experiment.newMeasurement(input)
 	start := time.Now()
-	err := eaw.experiment.measurer.Run(ctx, eaw.session, measurement, eaw.callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   eaw.callbacks,
+		Measurement: measurement,
+		Session:     eaw.session,
+	}
+	err := eaw.experiment.measurer.Run(ctx, args)
 	stop := time.Now()
 	if err != nil {
 		return nil, err

internal/engine/experiment/dash/dash.go

@@ -249,10 +249,10 @@ func (m Measurer) ExperimentVersion() string {
 }
 
 // Run implements model.ExperimentMeasurer.Run.
-func (m Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	tk := new(TestKeys)
 	measurement.TestKeys = tk
 	saver := &tracex.Saver{}

internal/engine/experiment/dash/dash_test.go

@@ -270,15 +270,15 @@ func TestMeasureWithCancelledContext(t *testing.T) {
 	cancel() // cause failure
 	measurement := new(model.Measurement)
 	m := &Measurer{}
-	err := m.Run(
-		ctx,
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableHTTPClient: http.DefaultClient,
 			MockableLogger:     log.Log,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := m.Run(ctx, args)
 	// See corresponding comment in Measurer.Run implementation to
 	// understand why here it's correct to return nil.
 	if !errors.Is(err, nil) {

internal/engine/experiment/dnscheck/dnscheck.go

@@ -120,10 +120,11 @@ var (
 )
 
 // Run implements model.ExperimentSession.Run
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
+
 	// 1. fill the measurement with test keys
 	tk := new(TestKeys)
 	tk.Lookups = make(map[string]urlgetter.TestKeys)

internal/engine/experiment/dnscheck/dnscheck_test.go

@@ -56,12 +56,12 @@ func TestExperimentNameAndVersion(t *testing.T) {
 
 func TestDNSCheckFailsWithoutInput(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{Domain: "example.com"})
-	err := measurer.Run(
-		context.Background(),
-		newsession(),
-		new(model.Measurement),
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: new(model.Measurement),
+		Session:     newsession(),
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, ErrInputRequired) {
 		t.Fatal("expected no input error")
 	}
@@ -69,12 +69,12 @@ func TestDNSCheckFailsWithoutInput(t *testing.T) {
 
 func TestDNSCheckFailsWithInvalidURL(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
-	err := measurer.Run(
-		context.Background(),
-		newsession(),
-		&model.Measurement{Input: "Not a valid URL \x7f"},
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: &model.Measurement{Input: "Not a valid URL \x7f"},
+		Session:     newsession(),
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, ErrInvalidURL) {
 		t.Fatal("expected invalid input error")
 	}
@@ -82,12 +82,12 @@ func TestDNSCheckFailsWithInvalidURL(t *testing.T) {
 
 func TestDNSCheckFailsWithUnsupportedProtocol(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
-	err := measurer.Run(
-		context.Background(),
-		newsession(),
-		&model.Measurement{Input: "file://1.1.1.1"},
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: &model.Measurement{Input: "file://1.1.1.1"},
+		Session:     newsession(),
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, ErrUnsupportedURLScheme) {
 		t.Fatal("expected unsupported scheme error")
 	}
@@ -100,12 +100,12 @@ func TestWithCancelledContext(t *testing.T) {
 		DefaultAddrs: "1.1.1.1 1.0.0.1",
 	})
 	measurement := &model.Measurement{Input: "dot://one.one.one.one"}
-	err := measurer.Run(
-		ctx,
-		newsession(),
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     newsession(),
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -147,12 +147,12 @@ func TestDNSCheckValid(t *testing.T) {
 		DefaultAddrs: "1.1.1.1 1.0.0.1",
 	})
 	measurement := model.Measurement{Input: "dot://one.one.one.one:853"}
-	err := measurer.Run(
-		context.Background(),
-		newsession(),
-		&measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: &measurement,
+		Session:     newsession(),
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err.Error())
 	}
@@ -195,12 +195,12 @@ func TestDNSCheckWait(t *testing.T) {
 	measurer := &Measurer{Endpoints: endpoints}
 	run := func(input string) {
 		measurement := model.Measurement{Input: model.MeasurementTarget(input)}
-		err := measurer.Run(
-			context.Background(),
-			newsession(),
-			&measurement,
-			model.NewPrinterCallbacks(log.Log),
-		)
+		args := &model.ExperimentArgs{
+			Callbacks:   model.NewPrinterCallbacks(log.Log),
+			Measurement: &measurement,
+			Session:     newsession(),
+		}
+		err := measurer.Run(context.Background(), args)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err.Error())
 		}

internal/engine/experiment/dnsping/dnsping.go

@@ -85,12 +85,10 @@ var (
 )
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	if measurement.Input == "" {
 		return errNoInputProvided
 	}

internal/engine/experiment/dnsping/dnsping_test.go

@@ -61,7 +61,12 @@ func TestMeasurer_run(t *testing.T) {
 			MockableLogger: model.DiscardLogger,
 		}
 		callbacks := model.NewPrinterCallbacks(model.DiscardLogger)
-		err := m.Run(ctx, sess, meas, callbacks)
+		args := &model.ExperimentArgs{
+			Callbacks:   callbacks,
+			Measurement: meas,
+			Session:     sess,
+		}
+		err := m.Run(ctx, args)
 		return meas, m, err
 	}
 

internal/engine/experiment/example/example.go

@@ -57,10 +57,10 @@ func (m Measurer) ExperimentVersion() string {
 var ErrFailure = errors.New("mocked error")
 
 // Run implements model.ExperimentMeasurer.Run.
-func (m Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	var err error
 	if m.config.ReturnError {
 		err = ErrFailure

internal/engine/experiment/example/example_test.go

@@ -26,7 +26,12 @@ func TestSuccess(t *testing.T) {
 	sess := &mockable.Session{MockableLogger: log.Log}
 	callbacks := model.NewPrinterCallbacks(sess.Logger())
 	measurement := new(model.Measurement)
-	err := m.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := m.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -47,7 +52,12 @@ func TestFailure(t *testing.T) {
 	ctx := context.Background()
 	sess := &mockable.Session{MockableLogger: log.Log}
 	callbacks := model.NewPrinterCallbacks(sess.Logger())
-	err := m.Run(ctx, sess, new(model.Measurement), callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: new(model.Measurement),
+		Session:     sess,
+	}
+	err := m.Run(ctx, args)
 	if !errors.Is(err, example.ErrFailure) {
 		t.Fatal("expected an error here")
 	}

internal/engine/experiment/fbmessenger/fbmessenger.go

@@ -157,10 +157,10 @@ func (m Measurer) ExperimentVersion() string {
 }
 
 // Run implements ExperimentMeasurer.Run
-func (m Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
 	urlgetter.RegisterExtensions(measurement)

internal/engine/experiment/fbmessenger/fbmessenger_test.go

@@ -35,7 +35,12 @@ func TestSuccess(t *testing.T) {
 	sess := newsession(t)
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -97,7 +102,12 @@ func TestWithCancelledContext(t *testing.T) {
 	sess := &mockable.Session{MockableLogger: log.Log}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/experiment/hhfm/hhfm.go

@@ -90,10 +90,10 @@ var (
 )
 
 // Run implements ExperimentMeasurer.Run.
-func (m Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
 	urlgetter.RegisterExtensions(measurement)

internal/engine/experiment/hhfm/hhfm_test.go

@@ -45,7 +45,12 @@ func TestSuccess(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +158,12 @@ func TestCancelledContext(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -259,7 +269,12 @@ func TestNoHelpers(t *testing.T) {
 	sess := &mockable.Session{}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, hhfm.ErrNoAvailableTestHelpers) {
 		t.Fatal("not the error we expected")
 	}
@@ -309,7 +324,12 @@ func TestNoActualHelpersInList(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, hhfm.ErrNoAvailableTestHelpers) {
 		t.Fatal("not the error we expected")
 	}
@@ -362,7 +382,12 @@ func TestWrongTestHelperType(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, hhfm.ErrInvalidHelperType) {
 		t.Fatal("not the error we expected")
 	}
@@ -415,7 +440,12 @@ func TestNewRequestFailure(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
 		t.Fatal("not the error we expected")
 	}
@@ -472,7 +502,12 @@ func TestInvalidJSONBody(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/experiment/hirl/hirl.go

@@ -78,10 +78,10 @@ var (
 )
 
 // Run implements ExperimentMeasurer.Run.
-func (m Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	tk := new(TestKeys)
 	measurement.TestKeys = tk
 	if len(m.Methods) < 1 {

internal/engine/experiment/hirl/hirl_test.go

@@ -42,7 +42,12 @@ func TestSuccess(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -91,7 +96,12 @@ func TestCancelledContext(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -190,7 +200,12 @@ func TestWithFakeMethods(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -251,7 +266,12 @@ func TestWithNoMethods(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, hirl.ErrNoMeasurementMethod) {
 		t.Fatal("not the error we expected")
 	}
@@ -279,7 +299,12 @@ func TestNoHelpers(t *testing.T) {
 	sess := &mockable.Session{}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, hirl.ErrNoAvailableTestHelpers) {
 		t.Fatal("not the error we expected")
 	}
@@ -311,7 +336,12 @@ func TestNoActualHelperInList(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, hirl.ErrNoAvailableTestHelpers) {
 		t.Fatal("not the error we expected")
 	}
@@ -346,7 +376,12 @@ func TestWrongTestHelperType(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, hirl.ErrInvalidHelperType) {
 		t.Fatal("not the error we expected")
 	}

internal/engine/experiment/httphostheader/httphostheader.go

@@ -46,12 +46,10 @@ func (m *Measurer) ExperimentVersion() string {
 }
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	if measurement.Input == "" {
 		return errors.New("experiment requires input")
 	}

internal/engine/experiment/httphostheader/httphostheader_test.go

@@ -30,12 +30,12 @@ func TestMeasurerMeasureNoMeasurementInput(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{
 		TestHelperURL: "http://www.google.com",
 	})
-	err := measurer.Run(
-		context.Background(),
-		newsession(),
-		new(model.Measurement),
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: &model.Measurement{},
+		Session:     newsession(),
+	}
+	err := measurer.Run(context.Background(), args)
 	if err == nil || err.Error() != "experiment requires input" {
 		t.Fatal("not the error we expected")
 	}
@@ -44,12 +44,12 @@ func TestMeasurerMeasureNoMeasurementInput(t *testing.T) {
 func TestMeasurerMeasureNoTestHelper(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
 	measurement := &model.Measurement{Input: "x.org"}
-	err := measurer.Run(
-		context.Background(),
-		newsession(),
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     newsession(),
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -75,12 +75,12 @@ func TestRunnerHTTPSetHostHeader(t *testing.T) {
 	measurement := &model.Measurement{
 		Input: "x.org",
 	}
-	err := measurer.Run(
-		context.Background(),
-		newsession(),
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     newsession(),
+	}
+	err := measurer.Run(context.Background(), args)
 	if host != "x.org" {
 		t.Fatal("not the host we expected")
 	}

internal/engine/experiment/imap/imap.go

@@ -0,0 +1,325 @@
+package imap
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"github.com/pkg/errors"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/measurexlite"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/tcprunner"
+	"github.com/ooni/probe-cli/v3/internal/tracex"
+)
+
+var (
+	// errNoInputProvided indicates you didn't provide any input
+	errNoInputProvided = errors.New("not input provided")
+
+	// errInputIsNotAnURL indicates that input is not an URL
+	errInputIsNotAnURL = errors.New("input is not an URL")
+
+	// errInvalidScheme indicates that the scheme is invalid
+	errInvalidScheme = errors.New("scheme must be imap(s)")
+)
+
+const (
+	testName    = "imap"
+	testVersion = "0.0.1"
+)
+
+// Config contains the experiment config.
+type Config struct{}
+
+type runtimeConfig struct {
+	host      string
+	port      string
+	forcedTLS bool
+	noopCount uint8
+}
+
+func config(input model.MeasurementTarget) (*runtimeConfig, error) {
+	if input == "" {
+		// TODO: static input data (eg. gmail/riseup..)
+		return nil, errNoInputProvided
+	}
+
+	parsed, err := url.Parse(string(input))
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error())
+	}
+	if parsed.Scheme != "imap" && parsed.Scheme != "imaps" {
+		return nil, errInvalidScheme
+	}
+
+	port := ""
+
+	if parsed.Port() == "" {
+		// Default ports for StartTLS and forced TLS respectively
+		if parsed.Scheme == "imap" {
+			port = "143"
+		} else {
+			port = "993"
+		}
+	} else {
+		// Valid port is checked by URL parsing
+		port = parsed.Port()
+	}
+
+	validConfig := runtimeConfig{
+		host:      parsed.Hostname(),
+		forcedTLS: parsed.Scheme == "imaps",
+		port:      port,
+		noopCount: 10,
+	}
+
+	return &validConfig, nil
+}
+
+// TestKeys contains the experiment results for an entire domain host
+type TestKeys struct {
+	Host    string                           `json:"hostname"`
+	Queries []*model.ArchivalDNSLookupResult `json:"queries"`
+	// Individual IP/port results
+	Runs []*IndividualTestKeys `json:"runs"`
+	// Used for global failure (DNS resolution)
+	Failure string `json:"failure"`
+}
+
+func newTestKeys(host string) *TestKeys {
+	tk := new(TestKeys)
+	tk.Host = host
+	return tk
+}
+
+// Hostname TCPRunnerModel
+func (tk *TestKeys) Hostname(host string) {
+	tk.Host = host
+}
+
+// DNSResults TCPRunnerModel
+func (tk *TestKeys) DNSResults(res []*model.ArchivalDNSLookupResult) {
+	// TODO: not sure if we are passed the overall trace results and should overwrite key, or just append
+	tk.Queries = append(tk.Queries, res...)
+}
+
+// Failed TCPRunnerModel
+func (tk *TestKeys) Failed(msg string) {
+	tk.Failure = msg
+}
+
+// NewRun TCPRunnerModel
+func (tk *TestKeys) NewRun(addr string, port string) tcprunner.TCPSessionModel {
+	itk := newIndividualTestKeys(addr, port)
+	tk.Runs = append(tk.Runs, itk)
+	return itk
+}
+
+// IndividualTestKeys contains the experiment results for a single IP/port combo
+type IndividualTestKeys struct {
+	TCPConnect   []*model.ArchivalTCPConnectResult       `json:"tcp_connect"`
+	TLSHandshake *model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"`
+	Failure      string                                  `json:"failure"`
+	FailureStep  string                                  `json:"failed_step"`
+	IP           string                                  `json:"ip"`
+	Port         string                                  `json:"port"`
+	noopCounter  uint8
+}
+
+func newIndividualTestKeys(addr string, port string) *IndividualTestKeys {
+	itk := new(IndividualTestKeys)
+	itk.IP = addr
+	itk.Port = port
+	return itk
+}
+
+// IPPort TCPSessionModel
+func (itk *IndividualTestKeys) IPPort(ip string, port string) {
+	itk.IP = ip
+	itk.Port = port
+}
+
+// ConnectResults TCPSessionModel
+func (itk *IndividualTestKeys) ConnectResults(res []*model.ArchivalTCPConnectResult) {
+	itk.TCPConnect = append(itk.TCPConnect, res...)
+}
+
+// HandshakeResult TCPSessionModel
+func (itk *IndividualTestKeys) HandshakeResult(res *model.ArchivalTLSOrQUICHandshakeResult) {
+	itk.TLSHandshake = res
+}
+
+// FailedStep TCPSessionModel
+func (itk *IndividualTestKeys) FailedStep(failure string, step string) {
+	itk.Failure = failure
+	itk.FailureStep = step
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	// Config contains the experiment settings. If empty we
+	// will be using default settings.
+	Config Config
+
+	// Getter is an optional getter to be used for testing.
+	Getter urlgetter.MultiGetter
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	sess := args.Session
+	measurement := args.Measurement
+	log := sess.Logger()
+	trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved)
+
+	config, err := config(measurement.Input)
+	if err != nil {
+		// Invalid input data, we don't even generate report
+		return err
+	}
+
+	tk := new(TestKeys)
+	measurement.TestKeys = tk
+
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	tlsconfig := tls.Config{
+		InsecureSkipVerify: false,
+		ServerName:         config.host,
+	}
+
+	runner := &tcprunner.TCPRunner{
+		Tk:        tk,
+		Trace:     trace,
+		Logger:    log,
+		Ctx:       ctx,
+		Tlsconfig: &tlsconfig,
+	}
+
+	// First resolve DNS
+	addrs, success := runner.Resolve(config.host)
+	if !success {
+		return nil
+	}
+
+	for _, addr := range addrs {
+		tcpSession, success := runner.Conn(addr, config.port)
+		if !success {
+			continue
+		}
+		defer tcpSession.Close()
+
+		if config.forcedTLS {
+			log.Infof("Running direct TLS mode to %s:%s", addr, config.port)
+
+			if !tcpSession.Handshake() {
+				continue
+			}
+
+			// Try NoOps
+			if !testIMAP(tcpSession, config.noopCount) {
+				continue
+			}
+		} else {
+			log.Infof("Running StartTLS mode to %s:%s", addr, config.port)
+
+			// Upgrade via StartTLS and try NoOps
+			if !tcpSession.StartTLS("A1 STARTTLS\n", "TLS") {
+				continue
+			}
+
+			if !testIMAP(tcpSession, config.noopCount) {
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
+func testIMAP(s *tcprunner.TCPSession, noop uint8) bool {
+	// Auto-choose plaintext/TCP session
+	// TODO: move to Debugf
+	s.Runner.Logger.Infof("Retrieving existing connection")
+	conn := s.CurrentConn()
+	s.Runner.Logger.Infof("Starting IMAP query")
+
+	command, err := bufio.NewReader(conn).ReadString('\n')
+	if err != nil {
+		s.FailedStep(*tracex.NewFailure(err), "imap_wait_capability")
+		return false
+	}
+
+	if !strings.Contains(command, "CAPABILITY") {
+		s.FailedStep(fmt.Sprintf("Received unexpected IMAP response: %s", command), "imap_wrong_capability")
+		return false
+	}
+
+	s.Runner.Logger.Infof("Finished starting IMAP")
+
+	if noop > 0 {
+		// Downcast TCPSession's itk into typed IndividualTestKeys to access noopCounter field
+		concreteITK := s.Itk.(*IndividualTestKeys)
+		s.Runner.Logger.Infof("Trying to generate more no-op traffic")
+		concreteITK.noopCounter = 0
+		for concreteITK.noopCounter < noop {
+			concreteITK.noopCounter++
+			s.Runner.Logger.Infof("NoOp Iteration %d", concreteITK.noopCounter)
+			_, err = conn.Write([]byte("A1 NOOP\n"))
+			if err != nil {
+				s.FailedStep(*tracex.NewFailure(err), fmt.Sprintf("imap_noop_%d", concreteITK.noopCounter))
+				break
+			}
+		}
+
+		if concreteITK.noopCounter == noop {
+			s.Runner.Logger.Infof("Successfully generated no-op traffic")
+			return true
+		}
+		s.Runner.Logger.Warnf("Failed no-op traffic at iteration %d", concreteITK.noopCounter)
+		return false
+	}
+
+	return true
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with ooniprobe
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	//DNSBlocking bool `json:"facebook_dns_blocking"`
+	//TCPBlocking bool `json:"facebook_tcp_blocking"`
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	_, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	return sk, nil
+}

internal/engine/experiment/imap/imap_test.go

@@ -0,0 +1,192 @@
+package imap
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	//"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/mockable"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func plaintextListener() net.Listener {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
+			panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err))
+		}
+	}
+	return l
+}
+
+func tlsListener(l net.Listener) net.Listener {
+	return tls.NewListener(l, &tls.Config{})
+}
+
+func listenerAddr(l net.Listener) string {
+	return l.Addr().String()
+}
+
+func ValidIMAPServer(conn net.Conn) {
+	starttls := false
+	for {
+		command, err := bufio.NewReader(conn).ReadString('\n')
+		if err != nil {
+			return
+		}
+
+		if strings.Contains(command, "NOOP") {
+			conn.Write([]byte("A1 OK NOOP completed.\n"))
+		} else if command == "STARTTLS" {
+			starttls = true
+			conn.Write([]byte("A1 OK Begin TLS negotiation now.\n"))
+			// TODO: conn.Close does not actually close connection? or does client not detect it?
+			//conn.Close()
+			return
+		} else if starttls {
+			conn.Write([]byte("GARBAGE TO BREAK STARTTLS"))
+		}
+		conn.Write([]byte("\n"))
+	}
+}
+
+func TCPServer(l net.Listener) {
+	for {
+		conn, err := l.Accept()
+		if err != nil {
+			continue
+		}
+		defer conn.Close()
+		conn.Write([]byte("* OK [CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+ STARTTLS LOGINDISABLED] howdy, ready.\n"))
+		ValidIMAPServer(conn)
+	}
+}
+
+func TestMeasurer_run(t *testing.T) {
+	// runHelper is an helper function to run this set of tests.
+	runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) {
+		m := NewExperimentMeasurer(Config{})
+		if m.ExperimentName() != "imap" {
+			t.Fatal("invalid experiment name")
+		}
+		if m.ExperimentVersion() != "0.0.1" {
+			t.Fatal("invalid experiment version")
+		}
+		ctx := context.Background()
+		meas := &model.Measurement{
+			Input: model.MeasurementTarget(input),
+		}
+		sess := &mockable.Session{
+			MockableLogger: model.DiscardLogger,
+		}
+
+		args := &model.ExperimentArgs{
+			Callbacks:   model.NewPrinterCallbacks(model.DiscardLogger),
+			Measurement: meas,
+			Session:     sess,
+		}
+
+		err := m.Run(ctx, args)
+		return meas, m, err
+	}
+
+	t.Run("with empty input", func(t *testing.T) {
+		_, _, err := runHelper("")
+		if !errors.Is(err, errNoInputProvided) {
+			t.Fatal("unexpected error", err)
+		}
+	})
+
+	t.Run("with invalid URL", func(t *testing.T) {
+		_, _, err := runHelper("\t")
+		if !errors.Is(err, errInputIsNotAnURL) {
+			t.Fatal("unexpected error", err)
+		}
+	})
+
+	t.Run("with invalid scheme", func(t *testing.T) {
+		_, _, err := runHelper("https://8.8.8.8:443/")
+		if !errors.Is(err, errInvalidScheme) {
+			t.Fatal("unexpected error", err)
+		}
+	})
+
+	t.Run("with broken TLS", func(t *testing.T) {
+		p := plaintextListener()
+		defer p.Close()
+
+		l := tlsListener(p)
+		defer l.Close()
+		addr := listenerAddr(l)
+		go TCPServer(l)
+
+		meas, m, err := runHelper("imaps://" + addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		tk := meas.TestKeys.(*TestKeys)
+
+		for _, run := range tk.Runs {
+			if *run.TLSHandshake.Failure != "unknown_failure: remote error: tls: unrecognized name" {
+				t.Fatal("expected unrecognized_name in TLS handshake")
+			}
+
+			if run.noopCounter != 0 {
+				t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter)
+			}
+		}
+
+		ask, err := m.GetSummaryKeys(meas)
+		if err != nil {
+			t.Fatal("cannot obtain summary")
+		}
+		summary := ask.(SummaryKeys)
+		if summary.IsAnomaly {
+			t.Fatal("expected no anomaly")
+		}
+	})
+
+	t.Run("with broken starttls", func(t *testing.T) {
+		l := plaintextListener()
+		defer l.Close()
+		addr := listenerAddr(l)
+
+		go TCPServer(l)
+
+		meas, m, err := runHelper("imap://" + addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		tk := meas.TestKeys.(*TestKeys)
+		//bs, _ := json.Marshal(tk)
+		//fmt.Println(string(bs))
+
+		for _, run := range tk.Runs {
+			if *run.TLSHandshake.Failure != "unknown_failure: tls: first record does not look like a TLS handshake" {
+				t.Fatalf("s%ss", *run.TLSHandshake.Failure)
+				t.Fatal("expected broken handshake")
+			}
+
+			if run.noopCounter != 0 {
+				t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter)
+			}
+		}
+
+		ask, err := m.GetSummaryKeys(meas)
+		if err != nil {
+			t.Fatal("cannot obtain summary")
+		}
+		summary := ask.(SummaryKeys)
+		if summary.IsAnomaly {
+			t.Fatal("expected no anomaly")
+		}
+	})
+}

internal/engine/experiment/ndt7/ndt7.go

@@ -210,10 +210,10 @@ func (m *Measurer) doUpload(
 }
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	tk := new(TestKeys)
 	tk.Protocol = 7
 	measurement.TestKeys = tk

internal/engine/experiment/ndt7/ndt7_test.go

@@ -84,7 +84,12 @@ func TestRunWithCancelledContext(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel() // immediately cancel
 	meas := &model.Measurement{}
-	err := m.Run(ctx, sess, meas, model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: meas,
+		Session:     sess,
+	}
+	err := m.Run(ctx, args)
 	// Here we get nil because we still want to submit this measurement
 	if !errors.Is(err, nil) {
 		t.Fatal("not the error we expected")
@@ -104,15 +109,15 @@ func TestGood(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	measurer := NewExperimentMeasurer(Config{})
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableHTTPClient: http.DefaultClient,
 			MockableLogger:     log.Log,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -133,15 +138,15 @@ func TestFailDownload(t *testing.T) {
 		cancel()
 	}
 	meas := &model.Measurement{}
-	err := measurer.Run(
-		ctx,
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: meas,
+		Session: &mockable.Session{
 			MockableHTTPClient: http.DefaultClient,
 			MockableLogger:     log.Log,
 		},
-		meas,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(ctx, args)
 	// We expect a nil failure here because we want to submit anyway
 	// a measurement that failed to connect to m-lab.
 	if err != nil {
@@ -164,15 +169,15 @@ func TestFailUpload(t *testing.T) {
 		cancel()
 	}
 	meas := &model.Measurement{}
-	err := measurer.Run(
-		ctx,
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: meas,
+		Session: &mockable.Session{
 			MockableHTTPClient: http.DefaultClient,
 			MockableLogger:     log.Log,
 		},
-		meas,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(ctx, args)
 	// Here we expect a nil error because we want to submit this measurement
 	if err != nil {
 		t.Fatal(err)
@@ -197,15 +202,15 @@ func TestDownloadJSONUnmarshalFail(t *testing.T) {
 		seenError = true
 		return expected
 	}
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: &model.Measurement{},
+		Session: &mockable.Session{
 			MockableHTTPClient: http.DefaultClient,
 			MockableLogger:     log.Log,
 		},
-		new(model.Measurement),
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/experiment/portfiltering/measurer.go

@@ -38,12 +38,10 @@ var (
 )
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	// TODO(DecFox): Replace the localhost deployment with an OONI testhelper
 	// Ensure that we only do this once we have a deployed testhelper
 	testhelper := "http://127.0.0.1"

internal/engine/experiment/portfiltering/measurer_test.go

@@ -29,7 +29,12 @@ func TestMeasurer_run(t *testing.T) {
 	}
 	callbacks := model.NewPrinterCallbacks(model.DiscardLogger)
 	ctx := context.Background()
-	err := m.Run(ctx, sess, meas, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: meas,
+		Session:     sess,
+	}
+	err := m.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/experiment/psiphon/psiphon.go

@@ -66,10 +66,10 @@ func (m *Measurer) printprogress(
 }
 
 // Run runs the measurement
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	const maxruntime = 300
 	ctx, cancel := context.WithTimeout(ctx, maxruntime*time.Second)
 	var (

internal/engine/experiment/psiphon/psiphon_test.go

@@ -33,8 +33,12 @@ func TestRunWithCancelledContext(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel() // fail immediately
 	measurement := new(model.Measurement)
-	err := measurer.Run(ctx, newfakesession(), measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     newfakesession(),
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, nil) { // nil because we want to submit the measurement
 		t.Fatal("expected another error here")
 	}
@@ -64,8 +68,12 @@ func TestRunWithCustomInputAndCancelledContext(t *testing.T) {
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel() // fail immediately
-	err := measurer.Run(ctx, newfakesession(), measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     newfakesession(),
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, nil) { // nil because we want to submit the measurement
 		t.Fatal("expected another error here")
 	}
@@ -84,7 +92,12 @@ func TestRunWillPrintSomethingWithCancelledContext(t *testing.T) {
 		cancel() // fail after we've given the printer a chance to run
 	}
 	observer := observerCallbacks{progress: &atomicx.Int64{}}
-	err := measurer.Run(ctx, newfakesession(), measurement, observer)
+	args := &model.ExperimentArgs{
+		Callbacks:   observer,
+		Measurement: measurement,
+		Session:     newfakesession(),
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, nil) { // nil because we want to submit the measurement
 		t.Fatal("expected another error here")
 	}

internal/engine/experiment/quicping/quicping.go

@@ -221,12 +221,11 @@ func (m *Measurer) receiver(
 }
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
+
 	host := string(measurement.Input)
 	// allow URL input
 	if u, err := url.ParseRequestURI(host); err == nil {

internal/engine/experiment/quicping/quicping_test.go

@@ -33,8 +33,12 @@ func TestInvalidHost(t *testing.T) {
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("a.a.a.a")
 	sess := &mockable.Session{MockableLogger: log.Log}
-	err := measurer.Run(context.Background(), sess, measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err == nil {
 		t.Fatal("expected an error here")
 	}
@@ -53,8 +57,12 @@ func TestURLInput(t *testing.T) {
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("https://google.com/")
 	sess := &mockable.Session{MockableLogger: log.Log}
-	err := measurer.Run(context.Background(), sess, measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal("unexpected error")
 	}
@@ -73,8 +81,12 @@ func TestSuccess(t *testing.T) {
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("google.com")
 	sess := &mockable.Session{MockableLogger: log.Log}
-	err := measurer.Run(context.Background(), sess, measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal("did not expect an error here")
 	}
@@ -117,8 +129,12 @@ func TestWithCancelledContext(t *testing.T) {
 	sess := &mockable.Session{MockableLogger: log.Log}
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
-	err := measurer.Run(ctx, sess, measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal("did not expect an error here")
 	}
@@ -138,8 +154,12 @@ func TestListenFails(t *testing.T) {
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("google.com")
 	sess := &mockable.Session{MockableLogger: log.Log}
-	err := measurer.Run(context.Background(), sess, measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err == nil {
 		t.Fatal("expected an error here")
 	}
@@ -182,8 +202,12 @@ func TestWriteFails(t *testing.T) {
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("google.com")
 	sess := &mockable.Session{MockableLogger: log.Log}
-	err := measurer.Run(context.Background(), sess, measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal("unexpected error")
 	}
@@ -239,8 +263,12 @@ func TestReadFails(t *testing.T) {
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("google.com")
 	sess := &mockable.Session{MockableLogger: log.Log}
-	err := measurer.Run(context.Background(), sess, measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal("unexpected error")
 	}
@@ -271,8 +299,12 @@ func TestNoResponse(t *testing.T) {
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("ooni.org")
 	sess := &mockable.Session{MockableLogger: log.Log}
-	err := measurer.Run(context.Background(), sess, measurement,
-		model.NewPrinterCallbacks(log.Log))
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal("did not expect an error here")
 	}

internal/engine/experiment/riseupvpn/riseupvpn.go

@@ -175,8 +175,11 @@ func (m Measurer) ExperimentVersion() string {
 }
 
 // Run implements ExperimentMeasurer.Run.
-func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
+
 	ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
 	defer cancel()
 	testkeys := NewTestKeys()

internal/engine/experiment/riseupvpn/riseupvpn_test.go

@@ -100,7 +100,7 @@ const (
 					"cert": "XXXXXXXXXXXXXXXXXXXXXXXXX",
 					"iatMode": "0"
 				  }
-				},              
+				},
 				{
 				  "type":"openvpn",
 				  "protocols":[
@@ -328,7 +328,12 @@ func TestInvalidCaCert(t *testing.T) {
 	sess := &mockable.Session{MockableLogger: log.Log}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -599,7 +604,12 @@ func TestMissingTransport(t *testing.T) {
 	sess := &mockable.Session{MockableLogger: log.Log}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err = measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err = measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -790,14 +800,14 @@ func runDefaultMockTest(t *testing.T, multiGetter urlgetter.MultiGetter) *model.
 	}
 
 	measurement := new(model.Measurement)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: log.Log,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 
 	if err != nil {
 		t.Fatal(err)

internal/engine/experiment/run/dnscheck.go

@@ -21,5 +21,10 @@ func (m *dnsCheckMain) do(ctx context.Context, input StructuredInput,
 	measurement.TestName = exp.ExperimentName()
 	measurement.TestVersion = exp.ExperimentVersion()
 	measurement.Input = model.MeasurementTarget(input.Input)
-	return exp.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	return exp.Run(ctx, args)
 }

internal/engine/experiment/run/run.go

@@ -46,10 +46,10 @@ type StructuredInput struct {
 }
 
 // Run implements ExperimentMeasurer.ExperimentVersion.
-func (Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	var input StructuredInput
 	if err := json.Unmarshal([]byte(measurement.Input), &input); err != nil {
 		return err

internal/engine/experiment/run/run_test.go

@@ -31,7 +31,12 @@ func TestRunDNSCheckWithCancelledContext(t *testing.T) {
 	cancel() // fail immediately
 	sess := &mockable.Session{MockableLogger: log.Log}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	// TODO(bassosimone): here we could improve the tests by checking
 	// whether the result makes sense for a cancelled context.
 	if err != nil {
@@ -62,7 +67,12 @@ func TestRunURLGetterWithCancelledContext(t *testing.T) {
 	cancel() // fail immediately
 	sess := &mockable.Session{MockableLogger: log.Log}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil { // here we expected nil b/c we want to submit the measurement
 		t.Fatal(err)
 	}
@@ -86,7 +96,12 @@ func TestRunWithInvalidJSON(t *testing.T) {
 	ctx := context.Background()
 	sess := &mockable.Session{MockableLogger: log.Log}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err == nil || err.Error() != "invalid character '}' looking for beginning of value" {
 		t.Fatalf("not the error we expected: %+v", err)
 	}
@@ -100,7 +115,12 @@ func TestRunWithUnknownExperiment(t *testing.T) {
 	ctx := context.Background()
 	sess := &mockable.Session{MockableLogger: log.Log}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err == nil || err.Error() != "no such experiment: antani" {
 		t.Fatalf("not the error we expected: %+v", err)
 	}

internal/engine/experiment/run/urlgetter.go

@@ -18,5 +18,10 @@ func (m *urlGetterMain) do(ctx context.Context, input StructuredInput,
 	measurement.TestName = exp.ExperimentName()
 	measurement.TestVersion = exp.ExperimentVersion()
 	measurement.Input = model.MeasurementTarget(input.Input)
-	return exp.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	return exp.Run(ctx, args)
 }

internal/engine/experiment/signal/signal.go

@@ -141,8 +141,10 @@ func (m Measurer) ExperimentVersion() string {
 }
 
 // Run implements ExperimentMeasurer.Run
-func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
 	urlgetter.RegisterExtensions(measurement)

internal/engine/experiment/signal/signal_test.go

@@ -25,14 +25,14 @@ func TestNewExperimentMeasurer(t *testing.T) {
 func TestGood(t *testing.T) {
 	measurer := signal.NewExperimentMeasurer(signal.Config{})
 	measurement := new(model.Measurement)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: log.Log,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -103,14 +103,14 @@ func TestBadSignalCA(t *testing.T) {
 		SignalCA: "INVALIDCA",
 	})
 	measurement := new(model.Measurement)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: log.Log,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if err.Error() != "AppendCertsFromPEM failed" {
 		t.Fatal("not the error we expected")
 	}

internal/engine/experiment/simplequicping/simplequicping.go

@@ -112,12 +112,11 @@ var (
 )
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
+
 	if measurement.Input == "" {
 		return errNoInputProvided
 	}

internal/engine/experiment/simplequicping/simplequicping_test.go

@@ -65,7 +65,12 @@ func TestMeasurer_run(t *testing.T) {
 			MockableLogger: model.DiscardLogger,
 		}
 		callbacks := model.NewPrinterCallbacks(model.DiscardLogger)
-		err := m.Run(ctx, sess, meas, callbacks)
+		args := &model.ExperimentArgs{
+			Callbacks:   callbacks,
+			Measurement: meas,
+			Session:     sess,
+		}
+		err := m.Run(ctx, args)
 		return meas, m, err
 	}
 

internal/engine/experiment/smtp/smtp.go

@@ -0,0 +1,333 @@
+package smtp
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"github.com/pkg/errors"
+	"net/smtp"
+	"net/url"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/measurexlite"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/tcprunner"
+	"github.com/ooni/probe-cli/v3/internal/tracex"
+)
+
+var (
+	// errNoInputProvided indicates you didn't provide any input
+	errNoInputProvided = errors.New("not input provided")
+
+	// errInputIsNotAnURL indicates that input is not an URL
+	errInputIsNotAnURL = errors.New("input is not an URL")
+
+	// errInvalidScheme indicates that the scheme is invalid
+	errInvalidScheme = errors.New("scheme must be smtp(s)")
+)
+
+const (
+	testName    = "smtp"
+	testVersion = "0.0.1"
+)
+
+// Config contains the experiment config.
+type Config struct{}
+
+type runtimeConfig struct {
+	host      string
+	port      string
+	forcedTLS bool
+	noopCount uint8
+}
+
+func config(input model.MeasurementTarget) (*runtimeConfig, error) {
+	if input == "" {
+		// TODO: static input data (eg. gmail/riseup..)
+		return nil, errNoInputProvided
+	}
+
+	parsed, err := url.Parse(string(input))
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error())
+	}
+	if parsed.Scheme != "smtp" && parsed.Scheme != "smtps" {
+		return nil, errInvalidScheme
+	}
+
+	port := ""
+
+	if parsed.Port() == "" {
+		// Default ports for StartTLS and forced TLS respectively
+		if parsed.Scheme == "smtp" {
+			port = "587"
+		} else {
+			port = "465"
+		}
+	} else {
+		// Valid port is checked by URL parsing
+		port = parsed.Port()
+	}
+
+	validConfig := runtimeConfig{
+		host:      parsed.Hostname(),
+		forcedTLS: parsed.Scheme == "smtps",
+		port:      port,
+		noopCount: 10,
+	}
+
+	return &validConfig, nil
+}
+
+// TestKeys contains the experiment results for an entire domain host
+type TestKeys struct {
+	Host    string                           `json:"hostname"`
+	Queries []*model.ArchivalDNSLookupResult `json:"queries"`
+	// Individual IP/port results
+	Runs []*IndividualTestKeys `json:"runs"`
+	// Used for global failure (DNS resolution)
+	Failure string `json:"failure"`
+}
+
+func newTestKeys(host string) *TestKeys {
+	tk := new(TestKeys)
+	tk.Host = host
+	return tk
+}
+
+// Hostname TCPRunnerModel
+func (tk *TestKeys) Hostname(host string) {
+	tk.Host = host
+}
+
+// DNSResults TCPRunnerModel
+func (tk *TestKeys) DNSResults(res []*model.ArchivalDNSLookupResult) {
+	// TODO: not sure if we are passed the overall trace results and should overwrite key, or just append
+	tk.Queries = append(tk.Queries, res...)
+}
+
+// Failed TCPRunnerModel
+func (tk *TestKeys) Failed(msg string) {
+	tk.Failure = msg
+}
+
+// NewRun TCPRunnerModel
+func (tk *TestKeys) NewRun(addr string, port string) tcprunner.TCPSessionModel {
+	itk := newIndividualTestKeys(addr, port)
+	tk.Runs = append(tk.Runs, itk)
+	return itk
+}
+
+// IndividualTestKeys contains the experiment results for a single IP/port combo
+type IndividualTestKeys struct {
+	TCPConnect   []*model.ArchivalTCPConnectResult       `json:"tcp_connect"`
+	TLSHandshake *model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"`
+	Failure      string                                  `json:"failure"`
+	FailureStep  string                                  `json:"failed_step"`
+	IP           string                                  `json:"ip"`
+	Port         string                                  `json:"port"`
+	noopCounter  uint8
+}
+
+func newIndividualTestKeys(addr string, port string) *IndividualTestKeys {
+	itk := new(IndividualTestKeys)
+	itk.IP = addr
+	itk.Port = port
+	return itk
+}
+
+// IPPort TCPSessionModel
+func (itk *IndividualTestKeys) IPPort(ip string, port string) {
+	itk.IP = ip
+	itk.Port = port
+}
+
+// ConnectResults TCPSessionModel
+func (itk *IndividualTestKeys) ConnectResults(res []*model.ArchivalTCPConnectResult) {
+	itk.TCPConnect = append(itk.TCPConnect, res...)
+}
+
+// HandshakeResult TCPSessionModel
+func (itk *IndividualTestKeys) HandshakeResult(res *model.ArchivalTLSOrQUICHandshakeResult) {
+	itk.TLSHandshake = res
+}
+
+// FailedStep TCPSessionModel
+func (itk *IndividualTestKeys) FailedStep(failure string, step string) {
+	itk.Failure = failure
+	itk.FailureStep = step
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	// Config contains the experiment settings. If empty we
+	// will be using default settings.
+	Config Config
+
+	// Getter is an optional getter to be used for testing.
+	Getter urlgetter.MultiGetter
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	sess := args.Session
+	measurement := args.Measurement
+	log := sess.Logger()
+	trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved)
+
+	config, err := config(measurement.Input)
+	if err != nil {
+		// Invalid input data, we don't even generate report
+		return err
+	}
+
+	tk := new(TestKeys)
+	measurement.TestKeys = tk
+
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	tlsconfig := tls.Config{
+		InsecureSkipVerify: false,
+		ServerName:         config.host,
+	}
+
+	runner := &tcprunner.TCPRunner{
+		Tk:        tk,
+		Trace:     trace,
+		Logger:    log,
+		Ctx:       ctx,
+		Tlsconfig: &tlsconfig,
+	}
+
+	// First resolve DNS
+	addrs, success := runner.Resolve(config.host)
+	if !success {
+		return nil
+	}
+
+	for _, addr := range addrs {
+		tcpSession, success := runner.Conn(addr, config.port)
+		if !success {
+			continue
+		}
+		defer tcpSession.Close()
+
+		if config.forcedTLS {
+			log.Infof("Running direct TLS mode to %s:%s", addr, config.port)
+
+			if !tcpSession.Handshake() {
+				continue
+			}
+
+			// Try EHLO + NoOps
+			if !testSMTP(tcpSession, "localhost", config.noopCount) {
+				continue
+			}
+		} else {
+			log.Infof("Running StartTLS mode to %s:%s", addr, config.port)
+
+			if !testSMTP(tcpSession, "localhost", 0) {
+				continue
+			}
+
+			// Upgrade via StartTLS and try EHLO + NoOps
+			if !tcpSession.StartTLS("STARTTLS\n", "TLS") {
+				continue
+			}
+
+			if !testSMTP(tcpSession, "localhost", config.noopCount) {
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
+func testSMTP(s *tcprunner.TCPSession, ehlo string, noop uint8) bool {
+	// Auto-choose plaintext/TCP session
+	// TODO: move to Debugf
+	s.Runner.Logger.Infof("Retrieving existing connection")
+	conn := s.CurrentConn()
+	s.Runner.Logger.Infof("Initializing SMTP client")
+	client, err := smtp.NewClient(conn, ehlo)
+	if err != nil {
+		s.FailedStep(*tracex.NewFailure(err), "smtp_init")
+		return false
+	}
+
+	s.Runner.Logger.Infof("Starting SMTP EHLO")
+	err = client.Hello(ehlo)
+	if err != nil {
+		if s.TLS {
+			s.FailedStep(*tracex.NewFailure(err), "smtp_tls_ehlo")
+		} else {
+			s.FailedStep(*tracex.NewFailure(err), "smtp_plaintext_ehlo")
+		}
+		return false
+	}
+
+	s.Runner.Logger.Infof("Finished SMTP EHLO")
+
+	if noop > 0 {
+		// Downcast TCPSession's itk into typed IndividualTestKeys to access noopCounter field
+		concreteITK := s.Itk.(*IndividualTestKeys)
+		s.Runner.Logger.Infof("Trying to generate more no-op traffic")
+		concreteITK.noopCounter = 0
+		for concreteITK.noopCounter < noop {
+			concreteITK.noopCounter++
+			s.Runner.Logger.Infof("NoOp Iteration %d", concreteITK.noopCounter)
+			err = client.Noop()
+			if err != nil {
+				s.FailedStep(*tracex.NewFailure(err), fmt.Sprintf("smtp_noop_%d", concreteITK.noopCounter))
+				break
+			}
+		}
+
+		if concreteITK.noopCounter == noop {
+			s.Runner.Logger.Infof("Successfully generated no-op traffic")
+			return true
+		}
+		s.Runner.Logger.Warnf("Failed no-op traffic at iteration %d", concreteITK.noopCounter)
+		return false
+	}
+
+	return true
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with ooniprobe
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	//DNSBlocking bool `json:"facebook_dns_blocking"`
+	//TCPBlocking bool `json:"facebook_tcp_blocking"`
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	_, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	return sk, nil
+}

internal/engine/experiment/smtp/smtp_test.go

@@ -0,0 +1,186 @@
+package smtp
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/mockable"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func plaintextListener() net.Listener {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
+			panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err))
+		}
+	}
+	return l
+}
+
+func tlsListener(l net.Listener) net.Listener {
+	return tls.NewListener(l, &tls.Config{})
+}
+
+func listenerAddr(l net.Listener) string {
+	return l.Addr().String()
+}
+
+func ValidSMTPServer(conn net.Conn) {
+	for {
+		command, err := bufio.NewReader(conn).ReadString('\n')
+		if err != nil {
+			return
+		}
+
+		if command == "" {
+		} else if command == "NOOP" {
+			conn.Write([]byte("250 2.0.0 Ok\n"))
+		} else if command == "STARTTLS" {
+			conn.Write([]byte("220 2.0.0 Ready to start TLS\n"))
+			// TODO: conn.Close does not actually close connection? or does client not detect it?
+			conn.Close()
+			return
+		} else if strings.HasPrefix(command, "EHLO") {
+			conn.Write([]byte("250 mock.example.com\n"))
+		}
+		conn.Write([]byte("\n"))
+	}
+}
+
+func TCPServer(l net.Listener) {
+	for {
+		conn, err := l.Accept()
+		if err != nil {
+			continue
+		}
+		defer conn.Close()
+		conn.Write([]byte("220 mock.example.com ESMTP (spam is not appreciated)\n"))
+		ValidSMTPServer(conn)
+	}
+}
+
+func TestMeasurer_run(t *testing.T) {
+	// runHelper is an helper function to run this set of tests.
+	runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) {
+		m := NewExperimentMeasurer(Config{})
+		if m.ExperimentName() != "smtp" {
+			t.Fatal("invalid experiment name")
+		}
+		if m.ExperimentVersion() != "0.0.1" {
+			t.Fatal("invalid experiment version")
+		}
+		ctx := context.Background()
+		meas := &model.Measurement{
+			Input: model.MeasurementTarget(input),
+		}
+		sess := &mockable.Session{
+			MockableLogger: model.DiscardLogger,
+		}
+
+		args := &model.ExperimentArgs{
+			Callbacks:   model.NewPrinterCallbacks(model.DiscardLogger),
+			Measurement: meas,
+			Session:     sess,
+		}
+		err := m.Run(ctx, args)
+		return meas, m, err
+	}
+
+	t.Run("with empty input", func(t *testing.T) {
+		_, _, err := runHelper("")
+		if !errors.Is(err, errNoInputProvided) {
+			t.Fatal("unexpected error", err)
+		}
+	})
+
+	t.Run("with invalid URL", func(t *testing.T) {
+		_, _, err := runHelper("\t")
+		if !errors.Is(err, errInputIsNotAnURL) {
+			t.Fatal("unexpected error", err)
+		}
+	})
+
+	t.Run("with invalid scheme", func(t *testing.T) {
+		_, _, err := runHelper("https://8.8.8.8:443/")
+		if !errors.Is(err, errInvalidScheme) {
+			t.Fatal("unexpected error", err)
+		}
+	})
+
+	t.Run("with broken TLS", func(t *testing.T) {
+		p := plaintextListener()
+		defer p.Close()
+
+		l := tlsListener(p)
+		defer l.Close()
+		addr := listenerAddr(l)
+		go TCPServer(l)
+
+		meas, m, err := runHelper("smtps://" + addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		tk := meas.TestKeys.(*TestKeys)
+
+		for _, run := range tk.Runs {
+			if *run.TLSHandshake.Failure != "unknown_failure: remote error: tls: unrecognized name" {
+				t.Fatal("expected unrecognized_name in TLS handshake")
+			}
+
+			if run.noopCounter != 0 {
+				t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter)
+			}
+		}
+
+		ask, err := m.GetSummaryKeys(meas)
+		if err != nil {
+			t.Fatal("cannot obtain summary")
+		}
+		summary := ask.(SummaryKeys)
+		if summary.IsAnomaly {
+			t.Fatal("expected no anomaly")
+		}
+	})
+
+	t.Run("with broken starttls", func(t *testing.T) {
+		l := plaintextListener()
+		defer l.Close()
+		addr := listenerAddr(l)
+
+		go TCPServer(l)
+
+		meas, m, err := runHelper("smtp://" + addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		tk := meas.TestKeys.(*TestKeys)
+
+		for _, run := range tk.Runs {
+			if *run.TLSHandshake.Failure != "generic_timeout_error" {
+				t.Fatal("expected timeout in TLS handshake")
+			}
+
+			if run.noopCounter != 0 {
+				t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter)
+			}
+		}
+
+		ask, err := m.GetSummaryKeys(meas)
+		if err != nil {
+			t.Fatal("cannot obtain summary")
+		}
+		summary := ask.(SummaryKeys)
+		if summary.IsAnomaly {
+			t.Fatal("expected no anomaly")
+		}
+	})
+}

internal/engine/experiment/sniblocking/sniblocking.go

@@ -233,12 +233,10 @@ func maybeURLToSNI(input model.MeasurementTarget) (model.MeasurementTarget, erro
 }
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	m.mu.Lock()
 	if m.cache == nil {
 		m.cache = make(map[string]Subresult)

internal/engine/experiment/sniblocking/sniblocking_test.go

@@ -116,12 +116,12 @@ func TestMeasurerMeasureNoMeasurementInput(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{
 		ControlSNI: "example.com",
 	})
-	err := measurer.Run(
-		context.Background(),
-		newsession(),
-		new(model.Measurement),
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: &model.Measurement{},
+		Session:     newsession(),
+	}
+	err := measurer.Run(context.Background(), args)
 	if err.Error() != "Experiment requires measurement.Input" {
 		t.Fatal("not the error we expected")
 	}
@@ -136,12 +136,12 @@ func TestMeasurerMeasureWithInvalidInput(t *testing.T) {
 	measurement := &model.Measurement{
 		Input: "\t",
 	}
-	err := measurer.Run(
-		ctx,
-		newsession(),
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     newsession(),
+	}
+	err := measurer.Run(ctx, args)
 	if err == nil {
 		t.Fatal("expected an error here")
 	}
@@ -156,12 +156,12 @@ func TestMeasurerMeasureWithCancelledContext(t *testing.T) {
 	measurement := &model.Measurement{
 		Input: "kernel.org",
 	}
-	err := measurer.Run(
-		ctx,
-		newsession(),
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     newsession(),
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/experiment/stunreachability/stunreachability.go

@@ -73,10 +73,10 @@ var errStunMissingPortInURL = errors.New("stun: missing port in URL")
 var errUnsupportedURLScheme = errors.New("stun: unsupported URL scheme")
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	tk := new(TestKeys)
 	measurement.TestKeys = tk
 	registerExtensions(measurement)

internal/engine/experiment/stunreachability/stunreachability_test.go

@@ -32,12 +32,12 @@ func TestMeasurerExperimentNameVersion(t *testing.T) {
 func TestRunWithoutInput(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
 	measurement := new(model.Measurement)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, errStunMissingInput) {
 		t.Fatal("not the error we expected", err)
 	}
@@ -47,12 +47,12 @@ func TestRunWithInvalidURL(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("\t") // <- invalid URL
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := measurer.Run(context.Background(), args)
 	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
 		t.Fatal("not the error we expected", err)
 	}
@@ -62,12 +62,12 @@ func TestRunWithNoPort(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("stun://stun.ekiga.net")
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, errStunMissingPortInURL) {
 		t.Fatal("not the error we expected", err)
 	}
@@ -77,12 +77,12 @@ func TestRunWithUnsupportedURLScheme(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("https://stun.ekiga.net:3478")
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, errUnsupportedURLScheme) {
 		t.Fatal("not the error we expected", err)
 	}
@@ -92,14 +92,14 @@ func TestRunWithInput(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget(defaultInput)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: model.DiscardLogger,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -124,14 +124,14 @@ func TestCancelledContext(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget(defaultInput)
-	err := measurer.Run(
-		ctx,
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: model.DiscardLogger,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, nil) { // nil because we want to submit
 		t.Fatal("not the error we expected", err)
 	}
@@ -166,14 +166,14 @@ func TestNewClientFailure(t *testing.T) {
 	measurer := NewExperimentMeasurer(*config)
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget(defaultInput)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: model.DiscardLogger,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, nil) { // nil because we want to submit
 		t.Fatal("not the error we expected")
 	}
@@ -202,14 +202,14 @@ func TestStartFailure(t *testing.T) {
 	measurer := NewExperimentMeasurer(*config)
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget(defaultInput)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: model.DiscardLogger,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, nil) { // nil because we want to submit
 		t.Fatal("not the error we expected")
 	}
@@ -242,14 +242,14 @@ func TestReadFailure(t *testing.T) {
 	measurer := NewExperimentMeasurer(*config)
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget(defaultInput)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: model.DiscardLogger,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, nil) { // nil because we want to submit
 		t.Fatal("not the error we expected")
 	}

internal/engine/experiment/tcpping/tcpping.go

@@ -82,12 +82,10 @@ var (
 )
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	if measurement.Input == "" {
 		return errNoInputProvided
 	}

internal/engine/experiment/tcpping/tcpping_test.go

@@ -51,7 +51,12 @@ func TestMeasurer_run(t *testing.T) {
 			MockableLogger: model.DiscardLogger,
 		}
 		callbacks := model.NewPrinterCallbacks(model.DiscardLogger)
-		err := m.Run(ctx, sess, meas, callbacks)
+		args := &model.ExperimentArgs{
+			Callbacks:   callbacks,
+			Measurement: meas,
+			Session:     sess,
+		}
+		err := m.Run(ctx, args)
 		return meas, m, err
 	}
 

internal/engine/experiment/telegram/telegram.go

@@ -101,8 +101,11 @@ func (m Measurer) ExperimentVersion() string {
 }
 
 // Run implements ExperimentMeasurer.Run
-func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
+
 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
 	urlgetter.RegisterExtensions(measurement)

internal/engine/experiment/telegram/telegram_test.go

@@ -28,14 +28,14 @@ func TestNewExperimentMeasurer(t *testing.T) {
 func TestGood(t *testing.T) {
 	measurer := telegram.NewExperimentMeasurer(telegram.Config{})
 	measurement := new(model.Measurement)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: log.Log,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -297,7 +297,12 @@ func TestWeConfigureWebChecksToFailOnHTTPError(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := measurer.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	if called.Load() < 1 {

internal/engine/experiment/tlsmiddlebox/measurer.go

@@ -52,12 +52,10 @@ var (
 )
 
 // // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	if measurement.Input == "" {
 		return errNoInputProvided
 	}

internal/engine/experiment/tlsmiddlebox/measurer_test.go

@@ -38,7 +38,12 @@ func TestMeasurer_input_failure(t *testing.T) {
 			},
 		}
 		callbacks := model.NewPrinterCallbacks(model.DiscardLogger)
-		err := m.Run(ctx, sess, meas, callbacks)
+		args := &model.ExperimentArgs{
+			Callbacks:   callbacks,
+			Measurement: meas,
+			Session:     sess,
+		}
+		err := m.Run(ctx, args)
 		return meas, m, err
 	}
 

internal/engine/experiment/tlsping/tlsping.go

@@ -112,12 +112,10 @@ var (
 )
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	if measurement.Input == "" {
 		return errNoInputProvided
 	}

internal/engine/experiment/tlsping/tlsping_test.go

@@ -58,7 +58,12 @@ func TestMeasurer_run(t *testing.T) {
 			MockableLogger: model.DiscardLogger,
 		}
 		callbacks := model.NewPrinterCallbacks(model.DiscardLogger)
-		err := m.Run(ctx, sess, meas, callbacks)
+		args := &model.ExperimentArgs{
+			Callbacks:   callbacks,
+			Measurement: meas,
+			Session:     sess,
+		}
+		err := m.Run(ctx, args)
 		return meas, m, err
 	}
 

internal/engine/experiment/tlstool/tlstool.go

@@ -78,12 +78,11 @@ var allMethods = []method{{
 }}
 
 // Run implements ExperimentMeasurer.Run.
-func (m Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
+
 	// TODO(bassosimone): wondering whether this experiment should
 	// actually be merged with sniblocking instead?
 	tk := new(TestKeys)

internal/engine/experiment/tlstool/tlstool_test.go

@@ -27,12 +27,12 @@ func TestRunWithExplicitSNI(t *testing.T) {
 	})
 	measurement := new(model.Measurement)
 	measurement.Input = "8.8.8.8:853"
-	err := measurer.Run(
-		ctx,
-		&mockable.Session{},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -43,12 +43,12 @@ func TestRunWithImplicitSNI(t *testing.T) {
 	measurer := tlstool.NewExperimentMeasurer(tlstool.Config{})
 	measurement := new(model.Measurement)
 	measurement.Input = "dns.google:853"
-	err := measurer.Run(
-		ctx,
-		&mockable.Session{},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -60,12 +60,12 @@ func TestRunWithCancelledContext(t *testing.T) {
 	measurer := tlstool.NewExperimentMeasurer(tlstool.Config{})
 	measurement := new(model.Measurement)
 	measurement.Input = "dns.google:853"
-	err := measurer.Run(
-		ctx,
-		&mockable.Session{},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/experiment/tor/tor.go

@@ -166,12 +166,10 @@ func (m *Measurer) ExperimentVersion() string {
 }
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	targets, err := m.gimmeTargets(ctx, sess)
 	if err != nil {
 		return err // fail the measurement if we cannot get any target

internal/engine/experiment/tor/tor_test.go

@@ -36,14 +36,14 @@ func TestMeasurerMeasureFetchTorTargetsError(t *testing.T) {
 	measurer.fetchTorTargets = func(ctx context.Context, sess model.ExperimentSession, cc string) (map[string]model.OOAPITorTarget, error) {
 		return nil, expected
 	}
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: &model.Measurement{},
+		Session: &mockable.Session{
 			MockableLogger: log.Log,
 		},
-		new(model.Measurement),
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if !errors.Is(err, expected) {
 		t.Fatal("not the error we expected")
 	}
@@ -55,14 +55,14 @@ func TestMeasurerMeasureFetchTorTargetsEmptyList(t *testing.T) {
 		return nil, nil
 	}
 	measurement := new(model.Measurement)
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session: &mockable.Session{
 			MockableLogger: log.Log,
 		},
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -79,14 +79,14 @@ func TestMeasurerMeasureGoodWithMockedOrchestra(t *testing.T) {
 	measurer.fetchTorTargets = func(ctx context.Context, sess model.ExperimentSession, cc string) (map[string]model.OOAPITorTarget, error) {
 		return nil, nil
 	}
-	err := measurer.Run(
-		context.Background(),
-		&mockable.Session{
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: &model.Measurement{},
+		Session: &mockable.Session{
 			MockableLogger: log.Log,
 		},
-		new(model.Measurement),
-		model.NewPrinterCallbacks(log.Log),
-	)
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -99,12 +99,12 @@ func TestMeasurerMeasureGood(t *testing.T) {
 	measurer := NewMeasurer(Config{})
 	sess := newsession()
 	measurement := new(model.Measurement)
-	err := measurer.Run(
-		context.Background(),
-		sess,
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -142,12 +142,12 @@ func TestMeasurerMeasureSanitiseOutput(t *testing.T) {
 		key: staticPrivateTestingTarget,
 	}
 	measurement := new(model.Measurement)
-	err := measurer.Run(
-		context.Background(),
-		sess,
-		measurement,
-		model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(context.Background(), args)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/experiment/torsf/integration_test.go

@@ -34,7 +34,12 @@ func TestRunWithExistingTor(t *testing.T) {
 		MockableLogger:  log.Log,
 		MockableTempDir: tempdir,
 	}
-	if err = m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err = m.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 }

internal/engine/experiment/torsf/torsf.go

@@ -124,10 +124,10 @@ const maxRuntime = 600 * time.Second
 // set the relevant OONI error inside of the measurement and
 // return nil. This is important because the caller may not submit
 // the measurement if this method returns an error.
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	ptl, sfdialer, err := m.setup(ctx, sess.Logger())
 	if err != nil {
 		// we cannot setup the experiment

internal/engine/experiment/torsf/torsf_test.go

@@ -47,7 +47,12 @@ func TestFailureWithInvalidRendezvousMethod(t *testing.T) {
 	callbacks := &model.PrinterCallbacks{
 		Logger: model.DiscardLogger,
 	}
-	err := m.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := m.Run(ctx, args)
 	if !errors.Is(err, ptx.ErrSnowflakeNoSuchRendezvousMethod) {
 		t.Fatal("unexpected error", err)
 	}
@@ -70,7 +75,12 @@ func TestFailureToStartPTXListener(t *testing.T) {
 	callbacks := &model.PrinterCallbacks{
 		Logger: model.DiscardLogger,
 	}
-	if err := m.Run(ctx, sess, measurement, callbacks); !errors.Is(err, expected) {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := m.Run(ctx, args); !errors.Is(err, expected) {
 		t.Fatal("not the error we expected", err)
 	}
 	if tk := measurement.TestKeys; tk != nil {
@@ -108,7 +118,12 @@ func TestSuccessWithMockedTunnelStart(t *testing.T) {
 	callbacks := &model.PrinterCallbacks{
 		Logger: model.DiscardLogger,
 	}
-	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := m.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	if called.Load() != 1 {
@@ -168,7 +183,12 @@ func TestWithCancelledContext(t *testing.T) {
 	callbacks := &model.PrinterCallbacks{
 		Logger: model.DiscardLogger,
 	}
-	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := m.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	tk := measurement.TestKeys.(*TestKeys)
@@ -231,7 +251,12 @@ func TestFailureToStartTunnel(t *testing.T) {
 	callbacks := &model.PrinterCallbacks{
 		Logger: model.DiscardLogger,
 	}
-	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := m.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	tk := measurement.TestKeys.(*TestKeys)

internal/engine/experiment/urlgetter/urlgetter.go

@@ -97,10 +97,10 @@ func (m Measurer) ExperimentVersion() string {
 }
 
 // Run implements model.ExperimentSession.Run
-func (m Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	// When using the urlgetter experiment directly, there is a nonconfigurable
 	// default timeout that applies. When urlgetter is used as a library, it's
 	// instead the responsibility of the user of urlgetter to set timeouts. Note

internal/engine/experiment/urlgetter/urlgetter_test.go

@@ -23,10 +23,12 @@ func TestMeasurer(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	measurement.Input = "https://www.google.com"
-	err := m.Run(
-		ctx, &mockable.Session{},
-		measurement, model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := m.Run(ctx, args)
 	if !errors.Is(err, nil) { // nil because we want to submit the measurement
 		t.Fatal("not the error we expected")
 	}
@@ -60,10 +62,12 @@ func TestMeasurerDNSCache(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	measurement.Input = "https://www.google.com"
-	err := m.Run(
-		ctx, &mockable.Session{},
-		measurement, model.NewPrinterCallbacks(log.Log),
-	)
+	args := &model.ExperimentArgs{
+		Callbacks:   model.NewPrinterCallbacks(log.Log),
+		Measurement: measurement,
+		Session:     &mockable.Session{},
+	}
+	err := m.Run(ctx, args)
 	if !errors.Is(err, nil) { // nil because we want to submit the measurement
 		t.Fatal("not the error we expected")
 	}

internal/engine/experiment/vanillator/integration_test.go

@@ -34,7 +34,12 @@ func TestRunWithExistingTor(t *testing.T) {
 		MockableLogger:  log.Log,
 		MockableTempDir: tempdir,
 	}
-	if err = m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err = m.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 }

internal/engine/experiment/vanillator/vanillator.go

@@ -106,10 +106,10 @@ const maxRuntime = 200 * time.Second
 // set the relevant OONI error inside of the measurement and
 // return nil. This is important because the caller may not submit
 // the measurement if this method returns an error.
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	m.registerExtensions(measurement)
 	start := time.Now()
 	ctx, cancel := context.WithTimeout(ctx, maxRuntime)

internal/engine/experiment/vanillator/vanillator_test.go

@@ -59,7 +59,12 @@ func TestSuccessWithMockedTunnelStart(t *testing.T) {
 	callbacks := &model.PrinterCallbacks{
 		Logger: model.DiscardLogger,
 	}
-	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := m.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	if called.Load() != 1 {
@@ -113,7 +118,12 @@ func TestWithCancelledContext(t *testing.T) {
 	callbacks := &model.PrinterCallbacks{
 		Logger: model.DiscardLogger,
 	}
-	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := m.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	tk := measurement.TestKeys.(*TestKeys)
@@ -170,7 +180,12 @@ func TestFailureToStartTunnel(t *testing.T) {
 	callbacks := &model.PrinterCallbacks{
 		Logger: model.DiscardLogger,
 	}
-	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := m.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	tk := measurement.TestKeys.(*TestKeys)

internal/engine/experiment/webconnectivity/control.go

@@ -4,9 +4,10 @@ import (
 	"context"
 
 	"github.com/ooni/probe-cli/v3/internal/geoipx"
-	"github.com/ooni/probe-cli/v3/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/httpapi"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
 )
 
 // Redirect to types defined inside the model package
@@ -21,22 +22,23 @@ type (
 // Control performs the control request and returns the response.
 func Control(
 	ctx context.Context, sess model.ExperimentSession,
-	thAddr string, creq ControlRequest) (out ControlResponse, err error) {
-	clnt := &httpx.APIClientTemplate{
-		BaseURL:    thAddr,
-		HTTPClient: sess.DefaultHTTPClient(),
-		Logger:     sess.Logger(),
-		UserAgent:  sess.UserAgent(),
-	}
+	testhelpers []model.OOAPIService, creq ControlRequest) (ControlResponse, *model.OOAPIService, error) {
+	seqCaller := httpapi.NewSequenceCaller(
+		httpapi.MustNewPOSTJSONWithJSONResponseDescriptor(sess.Logger(), "/", creq).WithBodyLogging(true),
+		httpapi.NewEndpointList(sess.DefaultHTTPClient(), sess.UserAgent(), testhelpers...)...,
+	)
 	sess.Logger().Infof("control for %s...", creq.HTTPRequest)
-	// make sure error is wrapped
-	err = clnt.WithBodyLogging().Build().PostJSON(ctx, "/", creq, &out)
-	if err != nil {
-		err = netxlite.NewTopLevelGenericErrWrapper(err)
-	}
+	var out ControlResponse
+	idx, err := seqCaller.CallWithJSONResponse(ctx, &out)
 	sess.Logger().Infof("control for %s... %+v", creq.HTTPRequest, model.ErrorToStringOrOK(err))
+	if err != nil {
+		// make sure error is wrapped
+		err = netxlite.NewTopLevelGenericErrWrapper(err)
+		return ControlResponse{}, nil, err
+	}
 	fillASNs(&out.DNS)
-	return
+	runtimex.Assert(idx >= 0 && idx < len(testhelpers), "idx out of bounds")
+	return out, &testhelpers[idx], nil
 }
 
 // fillASNs fills the ASNs array of ControlDNSResult. For each Addr inside

internal/engine/experiment/webconnectivity/webconnectivity.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	testName    = "web_connectivity"
-	testVersion = "0.4.1"
+	testVersion = "0.4.2"
 )
 
 // Config contains the experiment config.
@@ -121,12 +121,11 @@ const (
 )
 
 // Run implements ExperimentMeasurer.Run.
-func (m Measurer) Run(
-	ctx context.Context,
-	sess model.ExperimentSession,
-	measurement *model.Measurement,
-	callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
+
 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
 	tk := new(TestKeys)
@@ -145,19 +144,9 @@ func (m Measurer) Run(
 	}
 	// 1. find test helper
 	testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
-	var testhelper *model.OOAPIService
-	for _, th := range testhelpers {
-		if th.Type == "https" {
-			testhelper = &th
-			break
-		}
-	}
-	if testhelper == nil {
+	if len(testhelpers) < 1 {
 		return ErrNoAvailableTestHelpers
 	}
-	measurement.TestHelpers = map[string]interface{}{
-		"backend": testhelper,
-	}
 	// 2. perform the DNS lookup step
 	dnsBegin := time.Now()
 	dnsResult := DNSLookup(ctx, DNSLookupConfig{
@@ -167,10 +156,11 @@ func (m Measurer) Run(
 	tk.Queries = append(tk.Queries, dnsResult.TestKeys.Queries...)
 	tk.DNSExperimentFailure = dnsResult.Failure
 	epnts := NewEndpoints(URL, dnsResult.Addresses())
-	sess.Logger().Infof("using control: %s", testhelper.Address)
+	sess.Logger().Infof("using control: %+v", testhelpers)
 	// 3. perform the control measurement
 	thBegin := time.Now()
-	tk.Control, err = Control(ctx, sess, testhelper.Address, ControlRequest{
+	var usedTH *model.OOAPIService
+	tk.Control, usedTH, err = Control(ctx, sess, testhelpers, ControlRequest{
 		HTTPRequest: URL.String(),
 		HTTPRequestHeaders: map[string][]string{
 			"Accept":          {model.HTTPHeaderAccept},
@@ -179,6 +169,11 @@ func (m Measurer) Run(
 		},
 		TCPConnect: epnts.Endpoints(),
 	})
+	if usedTH != nil {
+		measurement.TestHelpers = map[string]interface{}{
+			"backend": usedTH,
+		}
+	}
 	tk.THRuntime = time.Since(thBegin)
 	tk.ControlFailure = tracex.NewFailure(err)
 	// 4. analyze DNS results

internal/engine/experiment/webconnectivity/webconnectivity_test.go

@@ -21,7 +21,7 @@ func TestNewExperimentMeasurer(t *testing.T) {
 	if measurer.ExperimentName() != "web_connectivity" {
 		t.Fatal("unexpected name")
 	}
-	if measurer.ExperimentVersion() != "0.4.1" {
+	if measurer.ExperimentVersion() != "0.4.2" {
 		t.Fatal("unexpected version")
 	}
 }
@@ -37,7 +37,12 @@ func TestSuccess(t *testing.T) {
 	sess := newsession(t, true)
 	measurement := &model.Measurement{Input: "http://www.example.com"}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,7 +70,12 @@ func TestMeasureWithCancelledContext(t *testing.T) {
 	sess := newsession(t, true)
 	measurement := &model.Measurement{Input: "http://www.example.com"}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := measurer.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	tk := measurement.TestKeys.(*webconnectivity.TestKeys)
@@ -99,7 +109,12 @@ func TestMeasureWithNoInput(t *testing.T) {
 	sess := newsession(t, true)
 	measurement := &model.Measurement{Input: ""}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, webconnectivity.ErrNoInput) {
 		t.Fatal(err)
 	}
@@ -127,7 +142,12 @@ func TestMeasureWithInputNotBeingAnURL(t *testing.T) {
 	sess := newsession(t, true)
 	measurement := &model.Measurement{Input: "\t\t\t\t\t\t"}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, webconnectivity.ErrInputIsNotAnURL) {
 		t.Fatal(err)
 	}
@@ -155,7 +175,12 @@ func TestMeasureWithUnsupportedInput(t *testing.T) {
 	sess := newsession(t, true)
 	measurement := &model.Measurement{Input: "dnslookup://example.com"}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, webconnectivity.ErrUnsupportedInput) {
 		t.Fatal(err)
 	}
@@ -183,7 +208,12 @@ func TestMeasureWithNoAvailableTestHelpers(t *testing.T) {
 	sess := newsession(t, false)
 	measurement := &model.Measurement{Input: "https://www.example.com"}
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if !errors.Is(err, webconnectivity.ErrNoAvailableTestHelpers) {
 		t.Fatal(err)
 	}

internal/engine/experiment/whatsapp/whatsapp.go

@@ -154,10 +154,11 @@ func (m Measurer) ExperimentVersion() string {
 }
 
 // Run implements ExperimentMeasurer.Run
-func (m Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
+
 	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()
 	urlgetter.RegisterExtensions(measurement)

internal/engine/experiment/whatsapp/whatsapp_test.go

@@ -35,7 +35,12 @@ func TestSuccess(t *testing.T) {
 	sess := &mockable.Session{MockableLogger: log.Log}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -70,7 +75,12 @@ func TestFailureAllEndpoints(t *testing.T) {
 	sess := &mockable.Session{MockableLogger: log.Log}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	err := measurer.Run(ctx, sess, measurement, callbacks)
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	err := measurer.Run(ctx, args)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -598,7 +608,12 @@ func TestWeConfigureWebChecksCorrectly(t *testing.T) {
 	}
 	measurement := new(model.Measurement)
 	callbacks := model.NewPrinterCallbacks(log.Log)
-	if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err := measurer.Run(ctx, args); err != nil {
 		t.Fatal(err)
 	}
 	if called.Load() != 263 {

internal/engine/experiment_integration_test.go

@@ -475,10 +475,7 @@ func (am *antaniMeasurer) ExperimentVersion() string {
 	return "0.1.1"
 }
 
-func (am *antaniMeasurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (am *antaniMeasurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
 	return nil
 }
 

internal/experiment/webconnectivity/cleartextflow.go

@@ -285,7 +285,7 @@ func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Res
 			WaitGroup:    t.WaitGroup,
 			Referer:      resp.Request.URL.String(),
 			Session:      nil, // no need to issue another control request
-			THAddr:       "",  // ditto
+			TestHelpers:  nil, // ditto
 			UDPAddress:   t.UDPAddress,
 		}
 		resolvers.Start(ctx)

internal/experiment/webconnectivity/control.go

@@ -8,10 +8,11 @@ import (
 	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
-	"github.com/ooni/probe-cli/v3/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/httpapi"
 	"github.com/ooni/probe-cli/v3/internal/measurexlite"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
 )
 
 // EndpointMeasurementsStarter is used by Control to start extra
@@ -51,8 +52,8 @@ type Control struct {
 	// Session is the MANDATORY session to use.
 	Session model.ExperimentSession
 
-	// THAddr is the MANDATORY TH's URL.
-	THAddr string
+	// TestHelpers is the MANDATORY list of test helpers.
+	TestHelpers []model.OOAPIService
 
 	// URL is the MANDATORY URL we are measuring.
 	URL *url.URL
@@ -102,26 +103,20 @@ func (c *Control) Run(parentCtx context.Context) {
 	// create logger for this operation
 	ol := measurexlite.NewOperationLogger(
 		c.Logger,
-		"control for %s using %s",
+		"control for %s using %+v",
 		creq.HTTPRequest,
-		c.THAddr,
+		c.TestHelpers,
 	)
 
-	// create an API client
-	clnt := (&httpx.APIClientTemplate{
-		Accept:        "",
-		Authorization: "",
-		BaseURL:       c.THAddr,
-		HTTPClient:    c.Session.DefaultHTTPClient(),
-		Host:          "", // use the one inside the URL
-		LogBody:       true,
-		Logger:        c.Logger,
-		UserAgent:     c.Session.UserAgent(),
-	}).Build()
+	// create an httpapi sequence caller
+	seqCaller := httpapi.NewSequenceCaller(
+		httpapi.MustNewPOSTJSONWithJSONResponseDescriptor(c.Logger, "/", creq).WithBodyLogging(true),
+		httpapi.NewEndpointList(c.Session.DefaultHTTPClient(), c.Session.UserAgent(), c.TestHelpers...)...,
+	)
 
 	// issue the control request and wait for the response
 	var cresp webconnectivity.ControlResponse
-	err := clnt.PostJSON(opCtx, "/", creq, &cresp)
+	idx, err := seqCaller.CallWithJSONResponse(opCtx, &cresp)
 	if err != nil {
 		// make sure error is wrapped
 		err = netxlite.NewTopLevelGenericErrWrapper(err)
@@ -134,6 +129,10 @@ func (c *Control) Run(parentCtx context.Context) {
 	c.TestKeys.SetControl(&cresp)
 	ol.Stop(nil)
 
+	// record the specific TH that worked
+	runtimex.Assert(idx >= 0 && idx < len(c.TestHelpers), "idx out of bounds")
+	c.TestKeys.setTestHelper(&c.TestHelpers[idx])
+
 	// if the TH returned us addresses we did not previously were
 	// aware of, make sure we also measure them
 	c.maybeStartExtraMeasurements(parentCtx, cresp.DNS.Addrs)

internal/experiment/webconnectivity/dnsresolvers.go

@@ -67,8 +67,9 @@ type DNSResolvers struct {
 	// always follow the redirect chain caused by the provided URL.
 	Session model.ExperimentSession
 
-	// THAddr is the OPTIONAL test helper address.
-	THAddr string
+	// TestHelpers is the OPTIONAL list of test helpers. If the list is
+	// empty, we are not going to try to contact any test helper.
+	TestHelpers []model.OOAPIService
 
 	// UDPAddress is the OPTIONAL address of the UDP resolver to use. If this
 	// field is not set we use a default one (e.g., `8.8.8.8:53`).
@@ -498,15 +499,15 @@ func (t *DNSResolvers) startSecureFlows(
 	}
 }
 
-// maybeStartControlFlow starts the control flow iff .Session and .THAddr are set.
+// maybeStartControlFlow starts the control flow iff .Session and .TestHelpers are set.
 func (t *DNSResolvers) maybeStartControlFlow(
 	ctx context.Context,
 	ps *prioritySelector,
 	addresses []DNSEntry,
 ) {
-	// note: for subsequent requests we don't set .Session and .THAddr hence
+	// note: for subsequent requests we don't set .Session and .TestHelpers hence
 	// we are not going to query the test helper more than once
-	if t.Session != nil && t.THAddr != "" {
+	if t.Session != nil && len(t.TestHelpers) > 0 {
 		var addrs []string
 		for _, addr := range addresses {
 			addrs = append(addrs, addr.Addr)
@@ -518,7 +519,7 @@ func (t *DNSResolvers) maybeStartControlFlow(
 			PrioSelector:             ps,
 			TestKeys:                 t.TestKeys,
 			Session:                  t.Session,
-			THAddr:                   t.THAddr,
+			TestHelpers:              t.TestHelpers,
 			URL:                      t.URL,
 			WaitGroup:                t.WaitGroup,
 		}

internal/experiment/webconnectivity/measurer.go

@@ -36,15 +36,17 @@ func (m *Measurer) ExperimentName() string {
 
 // ExperimentVersion implements model.ExperimentMeasurer.
 func (m *Measurer) ExperimentVersion() string {
-	return "0.5.18"
+	return "0.5.19"
 }
 
 // Run implements model.ExperimentMeasurer.
-func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
 	// Reminder: when this function returns an error, the measurement result
 	// WILL NOT be submitted to the OONI backend. You SHOULD only return an error
 	// for fundamental errors (e.g., the input is invalid or missing).
+	_ = args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 
 	// make sure we have a cancellable context such that we can stop any
 	// goroutine running in the background (e.g., priority.go's ones)
@@ -89,17 +91,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession,
 
 	// obtain the test helper's address
 	testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
-	var thAddr string
-	for _, th := range testhelpers {
-		if th.Type == "https" {
-			thAddr = th.Address
-			measurement.TestHelpers = map[string]any{
-				"backend": &th,
-			}
-			break
-		}
-	}
-	if thAddr == "" {
+	if len(testhelpers) < 1 {
 		sess.Logger().Warnf("continuing without a valid TH address")
 		tk.SetControlFailure(webconnectivity.ErrNoAvailableTestHelpers)
 	}
@@ -120,7 +112,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession,
 		CookieJar:    jar,
 		Referer:      "",
 		Session:      sess,
-		THAddr:       thAddr,
+		TestHelpers:  testhelpers,
 		UDPAddress:   "",
 	}
 	resos.Start(ctx)
@@ -137,6 +129,16 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession,
 	// perform any deferred computation on the test keys
 	tk.Finalize(sess.Logger())
 
+	// set the test helper we used
+	// TODO(bassosimone): it may be more informative to know about all the
+	// test helpers we _tried_ to use, however the data format does not have
+	// support for that as far as I can tell...
+	if th := tk.getTestHelper(); th != nil {
+		measurement.TestHelpers = map[string]interface{}{
+			"backend": th,
+		}
+	}
+
 	// return whether there was a fundamental failure, which would prevent
 	// the measurement from being submitted to the OONI collector.
 	return tk.fundamentalFailure

internal/experiment/webconnectivity/secureflow.go

@@ -337,7 +337,7 @@ func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Respon
 			WaitGroup:    t.WaitGroup,
 			Referer:      resp.Request.URL.String(),
 			Session:      nil, // no need to issue another control request
-			THAddr:       "",  // ditto
+			TestHelpers:  nil, // ditto
 			UDPAddress:   t.UDPAddress,
 		}
 		resolvers.Start(ctx)

internal/experiment/webconnectivity/testkeys.go

@@ -134,6 +134,10 @@ type TestKeys struct {
 
 	// mu provides mutual exclusion for accessing the test keys.
 	mu *sync.Mutex
+
+	// testHelper is used to communicate the TH that worked to the main
+	// goroutine such that we can fill measurement.TestHelpers.
+	testHelper *model.OOAPIService
 }
 
 // ConnPriorityLogEntry is an entry in the TestKeys.ConnPriorityLog slice.
@@ -302,6 +306,21 @@ func (tk *TestKeys) AppendConnPriorityLogEntry(entry *ConnPriorityLogEntry) {
 	tk.mu.Unlock()
 }
 
+// setTestHelper sets .testHelper in a thread safe way
+func (tk *TestKeys) setTestHelper(th *model.OOAPIService) {
+	tk.mu.Lock()
+	tk.testHelper = th
+	tk.mu.Unlock()
+}
+
+// getTestHelper gets .testHelper in a thread safe way
+func (tk *TestKeys) getTestHelper() (th *model.OOAPIService) {
+	tk.mu.Lock()
+	th = tk.testHelper
+	tk.mu.Unlock()
+	return
+}
+
 // NewTestKeys creates a new instance of TestKeys.
 func NewTestKeys() *TestKeys {
 	return &TestKeys{
@@ -348,6 +367,7 @@ func NewTestKeys() *TestKeys {
 		ControlRequest:        nil,
 		fundamentalFailure:    nil,
 		mu:                    &sync.Mutex{},
+		testHelper:            nil,
 	}
 }
 

internal/httpapi/call.go

@@ -0,0 +1,181 @@
+package httpapi
+
+//
+// Calling HTTP APIs.
+//
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// joinURLPath appends |resourcePath| to |urlPath|.
+func joinURLPath(urlPath, resourcePath string) string {
+	if resourcePath == "" {
+		if urlPath == "" {
+			return "/"
+		}
+		return urlPath
+	}
+	if !strings.HasSuffix(urlPath, "/") {
+		urlPath += "/"
+	}
+	resourcePath = strings.TrimPrefix(resourcePath, "/")
+	return urlPath + resourcePath
+}
+
+// newRequest creates a new http.Request from the given |ctx|, |endpoint|, and |desc|.
+func newRequest(ctx context.Context, endpoint *Endpoint, desc *Descriptor) (*http.Request, error) {
+	URL, err := url.Parse(endpoint.BaseURL)
+	if err != nil {
+		return nil, err
+	}
+	// BaseURL and resource URL are joined if they have a path
+	URL.Path = joinURLPath(URL.Path, desc.URLPath)
+	if len(desc.URLQuery) > 0 {
+		URL.RawQuery = desc.URLQuery.Encode()
+	} else {
+		URL.RawQuery = "" // as documented we only honour desc.URLQuery
+	}
+	var reqBody io.Reader
+	if len(desc.RequestBody) > 0 {
+		reqBody = bytes.NewReader(desc.RequestBody)
+		desc.Logger.Debugf("httpapi: request body length: %d", len(desc.RequestBody))
+		if desc.LogBody {
+			desc.Logger.Debugf("httpapi: request body: %s", string(desc.RequestBody))
+		}
+	}
+	request, err := http.NewRequestWithContext(ctx, desc.Method, URL.String(), reqBody)
+	if err != nil {
+		return nil, err
+	}
+	request.Host = endpoint.Host // allow cloudfronting
+	if desc.Authorization != "" {
+		request.Header.Set("Authorization", desc.Authorization)
+	}
+	if desc.ContentType != "" {
+		request.Header.Set("Content-Type", desc.ContentType)
+	}
+	if desc.Accept != "" {
+		request.Header.Set("Accept", desc.Accept)
+	}
+	if endpoint.UserAgent != "" {
+		request.Header.Set("User-Agent", endpoint.UserAgent)
+	}
+	return request, nil
+}
+
+// ErrHTTPRequestFailed indicates that the server returned >= 400.
+type ErrHTTPRequestFailed struct {
+	// StatusCode is the status code that failed.
+	StatusCode int
+}
+
+// Error implements error.
+func (err *ErrHTTPRequestFailed) Error() string {
+	return fmt.Sprintf("httpapi: http request failed: %d", err.StatusCode)
+}
+
+// errMaybeCensorship indicates that there was an error at the networking layer
+// including, e.g., DNS, TCP connect, TLS. When we see this kind of error, we
+// will consider retrying with another endpoint under the assumption that it
+// may be that the current endpoint is censored.
+type errMaybeCensorship struct {
+	// Err is the underlying error
+	Err error
+}
+
+// Error implements error
+func (err *errMaybeCensorship) Error() string {
+	return err.Err.Error()
+}
+
+// Unwrap allows to get the underlying error
+func (err *errMaybeCensorship) Unwrap() error {
+	return err.Err
+}
+
+// docall calls the API represented by the given request |req| on the given |endpoint|
+// and returns the response and its body or an error.
+func docall(endpoint *Endpoint, desc *Descriptor, request *http.Request) (*http.Response, []byte, error) {
+	// Implementation note: remember to mark errors for which you want
+	// to retry with another endpoint using errMaybeCensorship.
+	response, err := endpoint.HTTPClient.Do(request)
+	if err != nil {
+		return nil, nil, &errMaybeCensorship{err}
+	}
+	defer response.Body.Close()
+	// Implementation note: always read and log the response body since
+	// it's quite useful to see the response JSON on API error.
+	r := io.LimitReader(response.Body, DefaultMaxBodySize)
+	data, err := netxlite.ReadAllContext(request.Context(), r)
+	if err != nil {
+		return response, nil, &errMaybeCensorship{err}
+	}
+	desc.Logger.Debugf("httpapi: response body length: %d bytes", len(data))
+	if desc.LogBody {
+		desc.Logger.Debugf("httpapi: response body: %s", string(data))
+	}
+	if response.StatusCode >= 400 {
+		return response, nil, &ErrHTTPRequestFailed{response.StatusCode}
+	}
+	return response, data, nil
+}
+
+// call is like Call but also returns the response.
+func call(ctx context.Context, desc *Descriptor, endpoint *Endpoint) (*http.Response, []byte, error) {
+	timeout := desc.Timeout
+	if timeout <= 0 {
+		timeout = DefaultCallTimeout // as documented
+	}
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	request, err := newRequest(ctx, endpoint, desc)
+	if err != nil {
+		return nil, nil, err
+	}
+	return docall(endpoint, desc, request)
+}
+
+// Call invokes the API described by |desc| on the given HTTP |endpoint| and
+// returns the response body (as a slice of bytes) or an error.
+//
+// Note: this function returns ErrHTTPRequestFailed if the HTTP status code is
+// greater or equal than 400. You could use errors.As to obtain a copy of the
+// error that was returned and see for yourself the actual status code.
+func Call(ctx context.Context, desc *Descriptor, endpoint *Endpoint) ([]byte, error) {
+	_, rawResponseBody, err := call(ctx, desc, endpoint)
+	return rawResponseBody, err
+}
+
+// goodContentTypeForJSON tracks known-good content-types for JSON. If the content-type
+// is not in this map, |CallWithJSONResponse| emits a warning message.
+var goodContentTypeForJSON = map[string]bool{
+	applicationJSON: true,
+}
+
+// CallWithJSONResponse is like Call but also assumes that the response is a
+// JSON body and attempts to parse it into the |response| field.
+//
+// Note: this function returns ErrHTTPRequestFailed if the HTTP status code is
+// greater or equal than 400. You could use errors.As to obtain a copy of the
+// error that was returned and see for yourself the actual status code.
+func CallWithJSONResponse(ctx context.Context, desc *Descriptor, endpoint *Endpoint, response any) error {
+	httpResp, rawRespBody, err := call(ctx, desc, endpoint)
+	if err != nil {
+		return err
+	}
+	if ctype := httpResp.Header.Get("Content-Type"); !goodContentTypeForJSON[ctype] {
+		desc.Logger.Warnf("httpapi: unexpected content-type: %s", ctype)
+		// fallthrough
+	}
+	return json.Unmarshal(rawRespBody, response)
+}

internal/httpapi/descriptor.go

@@ -0,0 +1,155 @@
+package httpapi
+
+//
+// HTTP API descriptor (e.g., GET /api/v1/test-list/urls)
+//
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+)
+
+// Descriptor contains the parameters for calling a given HTTP
+// API (e.g., GET /api/v1/test-list/urls).
+//
+// The zero value of this struct is invalid. Please, fill all the
+// fields marked as MANDATORY for correct initialization.
+type Descriptor struct {
+	// Accept contains the OPTIONAL accept header.
+	Accept string
+
+	// Authorization is the OPTIONAL authorization.
+	Authorization string
+
+	// ContentType is the OPTIONAL content-type header.
+	ContentType string
+
+	// LogBody OPTIONALLY enables logging bodies.
+	LogBody bool
+
+	// Logger is the MANDATORY logger to use.
+	//
+	// For example, model.DiscardLogger.
+	Logger model.Logger
+
+	// MaxBodySize is the OPTIONAL maximum response body size. If
+	// not set, we use the |DefaultMaxBodySize| constant.
+	MaxBodySize int64
+
+	// Method is the MANDATORY request method.
+	Method string
+
+	// RequestBody is the OPTIONAL request body.
+	RequestBody []byte
+
+	// Timeout is the OPTIONAL timeout for this call. If no timeout
+	// is specified we will use the |DefaultCallTimeout| const.
+	Timeout time.Duration
+
+	// URLPath is the MANDATORY URL path.
+	URLPath string
+
+	// URLQuery is the OPTIONAL query.
+	URLQuery url.Values
+}
+
+// WithBodyLogging returns a SHALLOW COPY of |Descriptor| with LogBody set to |value|. You SHOULD
+// only use this method when initializing the descriptor you want to use.
+func (desc *Descriptor) WithBodyLogging(value bool) *Descriptor {
+	out := &Descriptor{}
+	*out = *desc
+	out.LogBody = value
+	return out
+}
+
+// DefaultMaxBodySize is the default value for the maximum
+// body size you can fetch using the httpapi package.
+const DefaultMaxBodySize = 1 << 22
+
+// DefaultCallTimeout is the default timeout for an httpapi call.
+const DefaultCallTimeout = 60 * time.Second
+
+// NewGETJSONDescriptor is a convenience factory for creating a new descriptor
+// that uses the GET method and expects a JSON response.
+func NewGETJSONDescriptor(logger model.Logger, urlPath string) *Descriptor {
+	return NewGETJSONWithQueryDescriptor(logger, urlPath, url.Values{})
+}
+
+// applicationJSON is the content-type for JSON
+const applicationJSON = "application/json"
+
+// NewGETJSONWithQueryDescriptor is like NewGETJSONDescriptor but it also
+// allows you to provide |query| arguments. Leaving |query| nil or empty
+// is equivalent to calling NewGETJSONDescriptor directly.
+func NewGETJSONWithQueryDescriptor(logger model.Logger, urlPath string, query url.Values) *Descriptor {
+	return &Descriptor{
+		Accept:        applicationJSON,
+		Authorization: "",
+		ContentType:   "",
+		LogBody:       false,
+		Logger:        logger,
+		MaxBodySize:   DefaultMaxBodySize,
+		Method:        http.MethodGet,
+		RequestBody:   nil,
+		Timeout:       DefaultCallTimeout,
+		URLPath:       urlPath,
+		URLQuery:      query,
+	}
+}
+
+// NewPOSTJSONWithJSONResponseDescriptor creates a descriptor that POSTs a JSON document
+// and expects to receive back a JSON document from the API.
+//
+// This function ONLY fails if we cannot serialize the |request| to JSON. So, if you know
+// that |request| is JSON-serializable, you can safely call MustNewPostJSONWithJSONResponseDescriptor instead.
+func NewPOSTJSONWithJSONResponseDescriptor(logger model.Logger, urlPath string, request any) (*Descriptor, error) {
+	rawRequest, err := json.Marshal(request)
+	if err != nil {
+		return nil, err
+	}
+	desc := &Descriptor{
+		Accept:        applicationJSON,
+		Authorization: "",
+		ContentType:   applicationJSON,
+		LogBody:       false,
+		Logger:        logger,
+		MaxBodySize:   DefaultMaxBodySize,
+		Method:        http.MethodPost,
+		RequestBody:   rawRequest,
+		Timeout:       DefaultCallTimeout,
+		URLPath:       urlPath,
+		URLQuery:      nil,
+	}
+	return desc, nil
+}
+
+// MustNewPOSTJSONWithJSONResponseDescriptor is like NewPOSTJSONWithJSONResponseDescriptor except that
+// it panics in case it's not possible to JSON serialize the |request|.
+func MustNewPOSTJSONWithJSONResponseDescriptor(logger model.Logger, urlPath string, request any) *Descriptor {
+	desc, err := NewPOSTJSONWithJSONResponseDescriptor(logger, urlPath, request)
+	runtimex.PanicOnError(err, "NewPOSTJSONWithJSONResponseDescriptor failed")
+	return desc
+}
+
+// NewGETResourceDescriptor creates a generic descriptor for GETting a
+// resource of unspecified type using the given |urlPath|.
+func NewGETResourceDescriptor(logger model.Logger, urlPath string) *Descriptor {
+	return &Descriptor{
+		Accept:        "",
+		Authorization: "",
+		ContentType:   "",
+		LogBody:       false,
+		Logger:        logger,
+		MaxBodySize:   DefaultMaxBodySize,
+		Method:        http.MethodGet,
+		RequestBody:   nil,
+		Timeout:       DefaultCallTimeout,
+		URLPath:       urlPath,
+		URLQuery:      url.Values{},
+	}
+}

internal/httpapi/descriptor_test.go

@@ -0,0 +1,248 @@
+package httpapi
+
+import (
+	"log"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func TestDescriptor_WithBodyLogging(t *testing.T) {
+	type fields struct {
+		Accept        string
+		Authorization string
+		ContentType   string
+		LogBody       bool
+		Logger        model.Logger
+		MaxBodySize   int64
+		Method        string
+		RequestBody   []byte
+		Timeout       time.Duration
+		URLPath       string
+		URLQuery      url.Values
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   *Descriptor
+	}{{
+		name:   "with empty fields",
+		fields: fields{}, // LogBody defaults to false
+		want: &Descriptor{
+			LogBody: true,
+		},
+	}, {
+		name: "with nonempty fields",
+		fields: fields{
+			Accept:        "xx",
+			Authorization: "y",
+			ContentType:   "zzz",
+			LogBody:       false, // obviously must be false
+			Logger:        model.DiscardLogger,
+			MaxBodySize:   123,
+			Method:        "POST",
+			RequestBody:   []byte("123"),
+			Timeout:       15555,
+			URLPath:       "/",
+			URLQuery: map[string][]string{
+				"a": {"b"},
+			},
+		},
+		want: &Descriptor{
+			Accept:        "xx",
+			Authorization: "y",
+			ContentType:   "zzz",
+			LogBody:       true,
+			Logger:        model.DiscardLogger,
+			MaxBodySize:   123,
+			Method:        "POST",
+			RequestBody:   []byte("123"),
+			Timeout:       15555,
+			URLPath:       "/",
+			URLQuery: map[string][]string{
+				"a": {"b"},
+			},
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			desc := &Descriptor{
+				Accept:        tt.fields.Accept,
+				Authorization: tt.fields.Authorization,
+				ContentType:   tt.fields.ContentType,
+				LogBody:       tt.fields.LogBody,
+				Logger:        tt.fields.Logger,
+				MaxBodySize:   tt.fields.MaxBodySize,
+				Method:        tt.fields.Method,
+				RequestBody:   tt.fields.RequestBody,
+				Timeout:       tt.fields.Timeout,
+				URLPath:       tt.fields.URLPath,
+				URLQuery:      tt.fields.URLQuery,
+			}
+			got := desc.WithBodyLogging(true)
+			if diff := cmp.Diff(tt.want, got); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestNewGetJSONDescriptor(t *testing.T) {
+	expected := &Descriptor{
+		Accept:        "application/json",
+		Authorization: "",
+		ContentType:   "",
+		LogBody:       false,
+		Logger:        model.DiscardLogger,
+		MaxBodySize:   DefaultMaxBodySize,
+		Method:        http.MethodGet,
+		RequestBody:   nil,
+		Timeout:       DefaultCallTimeout,
+		URLPath:       "/robots.txt",
+		URLQuery:      url.Values{},
+	}
+	got := NewGETJSONDescriptor(model.DiscardLogger, "/robots.txt")
+	if diff := cmp.Diff(expected, got); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestNewGetJSONWithQueryDescriptor(t *testing.T) {
+	query := url.Values{
+		"a": {"b"},
+		"c": {"d"},
+	}
+	expected := &Descriptor{
+		Accept:        "application/json",
+		Authorization: "",
+		ContentType:   "",
+		LogBody:       false,
+		Logger:        model.DiscardLogger,
+		MaxBodySize:   DefaultMaxBodySize,
+		Method:        http.MethodGet,
+		RequestBody:   nil,
+		Timeout:       DefaultCallTimeout,
+		URLPath:       "/robots.txt",
+		URLQuery:      query,
+	}
+	got := NewGETJSONWithQueryDescriptor(model.DiscardLogger, "/robots.txt", query)
+	if diff := cmp.Diff(expected, got); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestNewPOSTJSONWithJSONResponseDescriptor(t *testing.T) {
+	type request struct {
+		Name string
+		Age  int64
+	}
+
+	t.Run("with failure", func(t *testing.T) {
+		request := make(chan int64)
+		got, err := NewPOSTJSONWithJSONResponseDescriptor(model.DiscardLogger, "/robots.txt", request)
+		if err == nil || err.Error() != "json: unsupported type: chan int64" {
+			log.Fatal("unexpected err", err)
+		}
+		if got != nil {
+			log.Fatal("expected to get a nil Descriptor")
+		}
+	})
+
+	t.Run("with success", func(t *testing.T) {
+		request := request{
+			Name: "sbs",
+			Age:  99,
+		}
+		expected := &Descriptor{
+			Accept:        "application/json",
+			Authorization: "",
+			ContentType:   "application/json",
+			LogBody:       false,
+			Logger:        model.DiscardLogger,
+			MaxBodySize:   DefaultMaxBodySize,
+			Method:        http.MethodPost,
+			RequestBody:   []byte(`{"Name":"sbs","Age":99}`),
+			Timeout:       DefaultCallTimeout,
+			URLPath:       "/robots.txt",
+			URLQuery:      nil,
+		}
+		got, err := NewPOSTJSONWithJSONResponseDescriptor(model.DiscardLogger, "/robots.txt", request)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if diff := cmp.Diff(expected, got); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+}
+
+func TestMustNewPOSTJSONWithJSONResponseDescriptor(t *testing.T) {
+	type request struct {
+		Name string
+		Age  int64
+	}
+
+	t.Run("with failure", func(t *testing.T) {
+		var panicked bool
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					panicked = true
+				}
+			}()
+			request := make(chan int64)
+			_ = MustNewPOSTJSONWithJSONResponseDescriptor(model.DiscardLogger, "/robots.txt", request)
+		}()
+		if !panicked {
+			t.Fatal("did not panic")
+		}
+	})
+
+	t.Run("with success", func(t *testing.T) {
+		request := request{
+			Name: "sbs",
+			Age:  99,
+		}
+		expected := &Descriptor{
+			Accept:        "application/json",
+			Authorization: "",
+			ContentType:   "application/json",
+			LogBody:       false,
+			Logger:        model.DiscardLogger,
+			MaxBodySize:   DefaultMaxBodySize,
+			Method:        http.MethodPost,
+			RequestBody:   []byte(`{"Name":"sbs","Age":99}`),
+			Timeout:       DefaultCallTimeout,
+			URLPath:       "/robots.txt",
+			URLQuery:      nil,
+		}
+		got := MustNewPOSTJSONWithJSONResponseDescriptor(model.DiscardLogger, "/robots.txt", request)
+		if diff := cmp.Diff(expected, got); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+}
+
+func TestNewGetResourceDescriptor(t *testing.T) {
+	expected := &Descriptor{
+		Accept:        "",
+		Authorization: "",
+		ContentType:   "",
+		LogBody:       false,
+		Logger:        model.DiscardLogger,
+		MaxBodySize:   DefaultMaxBodySize,
+		Method:        http.MethodGet,
+		RequestBody:   nil,
+		Timeout:       DefaultCallTimeout,
+		URLPath:       "/robots.txt",
+		URLQuery:      url.Values{},
+	}
+	got := NewGETResourceDescriptor(model.DiscardLogger, "/robots.txt")
+	if diff := cmp.Diff(expected, got); diff != "" {
+		t.Fatal(diff)
+	}
+}

internal/httpapi/doc.go

@@ -0,0 +1,15 @@
+// Package httpapi contains code for calling HTTP APIs.
+//
+// We model HTTP APIs as follows:
+//
+// 1. |Endpoint| is an API endpoint (e.g., https://api.ooni.io);
+//
+// 2. |Descriptor| describes the specific API you want to use (e.g.,
+// GET /api/v1/test-list/urls with JSON response body).
+//
+// Generally, you use |Call| to call the API identified by a |Descriptor|
+// on the specified |Endpoint|. However, there are cases where you
+// need more complex calling patterns. For example, with |SequenceCaller|
+// you can invoke the same API |Descriptor| with multiple equivalent
+// API |Endpoint|s until one of them succeeds or all fail.
+package httpapi

internal/httpapi/endpoint.go

@@ -0,0 +1,76 @@
+package httpapi
+
+//
+// HTTP API Endpoint (e.g., https://api.ooni.io)
+//
+
+import "github.com/ooni/probe-cli/v3/internal/model"
+
+// Endpoint models an HTTP endpoint on which you can call
+// several HTTP APIs (e.g., https://api.ooni.io) using a
+// given HTTP client potentially using a circumvention tunnel
+// mechanism such as psiphon or torsf.
+//
+// The zero value of this struct is invalid. Please, fill all the
+// fields marked as MANDATORY for correct initialization.
+type Endpoint struct {
+	// BaseURL is the MANDATORY endpoint base URL. We will honour the
+	// path of this URL and prepend it to the actual path specified inside
+	// a |Descriptor.URLPath|. However, we will always discard any query
+	// that may have been set inside the BaseURL. The only query string
+	// will be composed from the |Descriptor.URLQuery| values.
+	//
+	// For example, https://api.ooni.io.
+	BaseURL string
+
+	// HTTPClient is the MANDATORY HTTP client to use.
+	//
+	// For example, http.DefaultClient. You can introduce circumvention
+	// here by using an HTTPClient bound to a specific tunnel.
+	HTTPClient model.HTTPClient
+
+	// Host is the OPTIONAL host header to use.
+	//
+	// If this field is empty we use the BaseURL's hostname. A specific
+	// host header may be needed when using cloudfronting.
+	Host string
+
+	// User-Agent is the OPTIONAL user-agent to use. If empty,
+	// we'll use the stdlib's default user-agent string.
+	UserAgent string
+}
+
+// NewEndpointList constructs a list of API endpoints from |services|
+// returned by the OONI backend (or known in advance).
+//
+// Arguments:
+//
+// - httpClient is the HTTP client to use for accessing the endpoints;
+//
+// - userAgent is the user agent you would like to use;
+//
+// - service is the list of services gathered from the backend.
+func NewEndpointList(httpClient model.HTTPClient,
+	userAgent string, services ...model.OOAPIService) (out []*Endpoint) {
+	for _, svc := range services {
+		switch svc.Type {
+		case "https":
+			out = append(out, &Endpoint{
+				BaseURL:    svc.Address,
+				HTTPClient: httpClient,
+				Host:       "",
+				UserAgent:  userAgent,
+			})
+		case "cloudfront":
+			out = append(out, &Endpoint{
+				BaseURL:    svc.Address,
+				HTTPClient: httpClient,
+				Host:       svc.Front,
+				UserAgent:  userAgent,
+			})
+		default:
+			// nothing!
+		}
+	}
+	return
+}

internal/httpapi/endpoint_test.go

@@ -0,0 +1,69 @@
+package httpapi
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/model/mocks"
+)
+
+func TestNewEndpointList(t *testing.T) {
+	type args struct {
+		httpClient model.HTTPClient
+		userAgent  string
+		services   []model.OOAPIService
+	}
+	defaultHTTPClient := &mocks.HTTPClient{}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut []*Endpoint
+	}{{
+		name: "with no services",
+		args: args{
+			httpClient: defaultHTTPClient,
+			userAgent:  model.HTTPHeaderUserAgent,
+			services:   nil,
+		},
+		wantOut: nil,
+	}, {
+		name: "common cases",
+		args: args{
+			httpClient: defaultHTTPClient,
+			userAgent:  model.HTTPHeaderUserAgent,
+			services: []model.OOAPIService{{
+				Address: "https://www.example.com/",
+				Type:    "https",
+				Front:   "",
+			}, {
+				Address: "https://www.example.org/",
+				Type:    "cloudfront",
+				Front:   "example.org.it",
+			}, {
+				Address: "https://nonexistent.onion/",
+				Type:    "onion",
+				Front:   "",
+			}},
+		},
+		wantOut: []*Endpoint{{
+			BaseURL:    "https://www.example.com/",
+			HTTPClient: defaultHTTPClient,
+			Host:       "",
+			UserAgent:  model.HTTPHeaderUserAgent,
+		}, {
+			BaseURL:    "https://www.example.org/",
+			HTTPClient: defaultHTTPClient,
+			Host:       "example.org.it",
+			UserAgent:  model.HTTPHeaderUserAgent,
+		}},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := NewEndpointList(tt.args.httpClient, tt.args.userAgent, tt.args.services...)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}

internal/httpapi/sequence.go

@@ -0,0 +1,92 @@
+package httpapi
+
+//
+// Sequentially call available API endpoints until one succeed
+// or all of them fail. A future implementation of this code may
+// (probably should?) take into account knowledge of what is
+// working and what is not working to optimize the order with
+// which to try different alternatives.
+//
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ooni/probe-cli/v3/internal/multierror"
+)
+
+// SequenceCaller calls the API specified by |Descriptor| once for each of
+// the available |Endpoints| until one of them succeeds.
+//
+// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when
+// the error originates in the HTTP round trip or while reading the body.
+type SequenceCaller struct {
+	// Descriptor is the API |Descriptor|.
+	Descriptor *Descriptor
+
+	// Endpoints is the list of |Endpoint| to use.
+	Endpoints []*Endpoint
+}
+
+// NewSequenceCaller is a factory for creating a |SequenceCaller|.
+func NewSequenceCaller(desc *Descriptor, endpoints ...*Endpoint) *SequenceCaller {
+	return &SequenceCaller{
+		Descriptor: desc,
+		Endpoints:  endpoints,
+	}
+}
+
+// ErrAllEndpointsFailed indicates that all endpoints failed.
+var ErrAllEndpointsFailed = errors.New("httpapi: all endpoints failed")
+
+// shouldRetry returns true when we should try with another endpoint given the
+// value of |err| which could (obviously) be nil in case of success.
+func (sc *SequenceCaller) shouldRetry(err error) bool {
+	var kind *errMaybeCensorship
+	belongs := errors.As(err, &kind)
+	return belongs
+}
+
+// Call calls |Call| for each |Endpoint| and |Descriptor| until one endpoint succeeds. The
+// return value is the response body and the selected endpoint index or the error.
+//
+// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when
+// the error originates in the HTTP round trip or while reading the body.
+func (sc *SequenceCaller) Call(ctx context.Context) ([]byte, int, error) {
+	var selected int
+	merr := multierror.New(ErrAllEndpointsFailed)
+	for _, epnt := range sc.Endpoints {
+		respBody, err := Call(ctx, sc.Descriptor, epnt)
+		if sc.shouldRetry(err) {
+			merr.Add(err)
+			selected++
+			continue
+		}
+		// Note: some errors will lead us to return
+		// early as documented for this method
+		return respBody, selected, err
+	}
+	return nil, -1, merr
+}
+
+// CallWithJSONResponse is like |SequenceCaller.Call| except that it invokes the
+// underlying |CallWithJSONResponse| rather than invoking |Call|.
+//
+// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when
+// the error originates in the HTTP round trip or while reading the body.
+func (sc *SequenceCaller) CallWithJSONResponse(ctx context.Context, response any) (int, error) {
+	var selected int
+	merr := multierror.New(ErrAllEndpointsFailed)
+	for _, epnt := range sc.Endpoints {
+		err := CallWithJSONResponse(ctx, sc.Descriptor, epnt, response)
+		if sc.shouldRetry(err) {
+			merr.Add(err)
+			selected++
+			continue
+		}
+		// Note: some errors will lead us to return
+		// early as documented for this method
+		return selected, err
+	}
+	return -1, merr
+}

internal/httpapi/sequence_test.go

@@ -0,0 +1,358 @@
+package httpapi
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/model/mocks"
+)
+
+func TestSequenceCaller(t *testing.T) {
+	t.Run("Call", func(t *testing.T) {
+		t.Run("first success", func(t *testing.T) {
+			sc := NewSequenceCaller(
+				&Descriptor{
+					Logger:  model.DiscardLogger,
+					Method:  http.MethodGet,
+					URLPath: "/",
+				},
+				&Endpoint{
+					BaseURL: "https://a.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							resp := &http.Response{
+								StatusCode: 200,
+								Body:       io.NopCloser(strings.NewReader("deadbeef")),
+							}
+							return resp, nil
+						},
+					},
+				},
+				&Endpoint{
+					BaseURL: "https://b.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF
+						},
+					},
+				},
+			)
+			data, idx, err := sc.Call(context.Background())
+			if err != nil {
+				t.Fatal(err)
+			}
+			if idx != 0 {
+				t.Fatal("invalid idx")
+			}
+			if diff := cmp.Diff([]byte("deadbeef"), data); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+
+		t.Run("first HTTP failure and we immediately stop", func(t *testing.T) {
+			sc := NewSequenceCaller(
+				&Descriptor{
+					Logger:  model.DiscardLogger,
+					Method:  http.MethodGet,
+					URLPath: "/",
+				},
+				&Endpoint{
+					BaseURL: "https://a.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							resp := &http.Response{
+								StatusCode: 403, // should cause us to return early
+								Body:       io.NopCloser(strings.NewReader("deadbeef")),
+							}
+							return resp, nil
+						},
+					},
+				},
+				&Endpoint{
+					BaseURL: "https://b.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF
+						},
+					},
+				},
+			)
+			data, idx, err := sc.Call(context.Background())
+			var failure *ErrHTTPRequestFailed
+			if !errors.As(err, &failure) || failure.StatusCode != 403 {
+				t.Fatal("unexpected err", err)
+			}
+			if idx != 0 {
+				t.Fatal("invalid idx")
+			}
+			if len(data) > 0 {
+				t.Fatal("expected to see no response body")
+			}
+		})
+
+		t.Run("first network failure, second success", func(t *testing.T) {
+			sc := NewSequenceCaller(
+				&Descriptor{
+					Logger:  model.DiscardLogger,
+					Method:  http.MethodGet,
+					URLPath: "/",
+				},
+				&Endpoint{
+					BaseURL: "https://a.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF // should cause us to cycle to the second entry
+						},
+					},
+				},
+				&Endpoint{
+					BaseURL: "https://b.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							resp := &http.Response{
+								StatusCode: 200,
+								Body:       io.NopCloser(strings.NewReader("abad1dea")),
+							}
+							return resp, nil
+						},
+					},
+				},
+			)
+			data, idx, err := sc.Call(context.Background())
+			if err != nil {
+				t.Fatal(err)
+			}
+			if idx != 1 {
+				t.Fatal("invalid idx")
+			}
+			if diff := cmp.Diff([]byte("abad1dea"), data); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+
+		t.Run("all network failure", func(t *testing.T) {
+			sc := NewSequenceCaller(
+				&Descriptor{
+					Logger:  model.DiscardLogger,
+					Method:  http.MethodGet,
+					URLPath: "/",
+				},
+				&Endpoint{
+					BaseURL: "https://a.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF // should cause us to cycle to the next entry
+						},
+					},
+				},
+				&Endpoint{
+					BaseURL: "https://b.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF // should cause us to cycle to the next entry
+						},
+					},
+				},
+			)
+			data, idx, err := sc.Call(context.Background())
+			if !errors.Is(err, ErrAllEndpointsFailed) {
+				t.Fatal("unexpected err", err)
+			}
+			if idx != -1 {
+				t.Fatal("invalid idx")
+			}
+			if len(data) > 0 {
+				t.Fatal("expected zero-length data")
+			}
+		})
+	})
+
+	t.Run("CallWithJSONResponse", func(t *testing.T) {
+		type response struct {
+			Name string
+			Age  int64
+		}
+
+		t.Run("first success", func(t *testing.T) {
+			sc := NewSequenceCaller(
+				&Descriptor{
+					Logger:  model.DiscardLogger,
+					Method:  http.MethodGet,
+					URLPath: "/",
+				},
+				&Endpoint{
+					BaseURL: "https://a.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							resp := &http.Response{
+								StatusCode: 200,
+								Body:       io.NopCloser(strings.NewReader(`{"Name":"sbs","Age":99}`)),
+							}
+							return resp, nil
+						},
+					},
+				},
+				&Endpoint{
+					BaseURL: "https://b.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							resp := &http.Response{
+								StatusCode: 200,
+								Body:       io.NopCloser(strings.NewReader(`{}`)), // different
+							}
+							return resp, nil
+						},
+					},
+				},
+			)
+			expect := response{
+				Name: "sbs",
+				Age:  99,
+			}
+			var got response
+			idx, err := sc.CallWithJSONResponse(context.Background(), &got)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if idx != 0 {
+				t.Fatal("invalid idx")
+			}
+			if diff := cmp.Diff(expect, got); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+
+		t.Run("first HTTP failure and we immediately stop", func(t *testing.T) {
+			sc := NewSequenceCaller(
+				&Descriptor{
+					Logger:  model.DiscardLogger,
+					Method:  http.MethodGet,
+					URLPath: "/",
+				},
+				&Endpoint{
+					BaseURL: "https://a.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							resp := &http.Response{
+								StatusCode: 403, // should be enough to cause us fail immediately
+								Body:       io.NopCloser(strings.NewReader(`{"Age": 155, "Name": "sbs"}`)),
+							}
+							return resp, nil
+						},
+					},
+				},
+				&Endpoint{
+					BaseURL: "https://b.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF
+						},
+					},
+				},
+			)
+			// even though there is a JSON body we don't care about reading it
+			// and so we expect to see in output the zero-value struct
+			expect := response{
+				Name: "",
+				Age:  0,
+			}
+			var got response
+			idx, err := sc.CallWithJSONResponse(context.Background(), &got)
+			var failure *ErrHTTPRequestFailed
+			if !errors.As(err, &failure) || failure.StatusCode != 403 {
+				t.Fatal("unexpected err", err)
+			}
+			if idx != 0 {
+				t.Fatal("invalid idx")
+			}
+			if diff := cmp.Diff(expect, got); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+
+		t.Run("first network failure, second success", func(t *testing.T) {
+			sc := NewSequenceCaller(
+				&Descriptor{
+					Logger:  model.DiscardLogger,
+					Method:  http.MethodGet,
+					URLPath: "/",
+				},
+				&Endpoint{
+					BaseURL: "https://a.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF // should cause us to try the next entry
+						},
+					},
+				},
+				&Endpoint{
+					BaseURL: "https://b.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							resp := &http.Response{
+								StatusCode: 200,
+								Body:       io.NopCloser(strings.NewReader(`{"Age":155}`)),
+							}
+							return resp, nil
+						},
+					},
+				},
+			)
+			expect := response{
+				Name: "",
+				Age:  155,
+			}
+			var got response
+			idx, err := sc.CallWithJSONResponse(context.Background(), &got)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if idx != 1 {
+				t.Fatal("invalid idx")
+			}
+			if diff := cmp.Diff(expect, got); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+
+		t.Run("all network failure", func(t *testing.T) {
+			sc := NewSequenceCaller(
+				&Descriptor{
+					Logger:  model.DiscardLogger,
+					Method:  http.MethodGet,
+					URLPath: "/",
+				},
+				&Endpoint{
+					BaseURL: "https://a.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF // should cause us to try the next entry
+						},
+					},
+				},
+				&Endpoint{
+					BaseURL: "https://b.example.com/",
+					HTTPClient: &mocks.HTTPClient{
+						MockDo: func(req *http.Request) (*http.Response, error) {
+							return nil, io.EOF // should cause us to try the next entry
+						},
+					},
+				},
+			)
+			var got response
+			idx, err := sc.CallWithJSONResponse(context.Background(), &got)
+			if !errors.Is(err, ErrAllEndpointsFailed) {
+				t.Fatal("unexpected err", err)
+			}
+			if idx != -1 {
+				t.Fatal("invalid idx")
+			}
+		})
+	})
+}

internal/httpx/httpx.go

@@ -1,4 +1,8 @@
 // Package httpx contains http extensions.
+//
+// Deprecated: new code should use httpapi instead. While this package and httpapi
+// are basically using the same implementation, the API exposed by httpapi allows
+// us to try the same request with multiple HTTP endpoints.
 package httpx
 
 import (

internal/model/experiment.go

@@ -117,6 +117,19 @@ func (d PrinterCallbacks) OnProgress(percentage float64, message string) {
 	d.Logger.Infof("[%5.1f%%] %s", percentage*100, message)
 }
 
+// ExperimentArgs contains the arguments passed to an experiment.
+type ExperimentArgs struct {
+	// Callbacks contains MANDATORY experiment callbacks.
+	Callbacks ExperimentCallbacks
+
+	// Measurement is the MANDATORY measurement in which the experiment
+	// must write the results of the measurement.
+	Measurement *Measurement
+
+	// Session is the MANDATORY session the experiment can use.
+	Session ExperimentSession
+}
+
 // ExperimentMeasurer is the interface that allows to run a
 // measurement for a specific experiment.
 type ExperimentMeasurer interface {
@@ -133,10 +146,7 @@ type ExperimentMeasurer interface {
 	// set the relevant OONI error inside of the measurement and
 	// return nil. This is important because the caller WILL NOT submit
 	// the measurement if this method returns an error.
-	Run(
-		ctx context.Context, sess ExperimentSession,
-		measurement *Measurement, callbacks ExperimentCallbacks,
-	) error
+	Run(ctx context.Context, args *ExperimentArgs) error
 
 	// GetSummaryKeys returns summary keys expected by ooni/probe-cli.
 	GetSummaryKeys(*Measurement) (interface{}, error)

internal/registry/imap.go

@@ -0,0 +1,22 @@
+package registry
+
+//
+// Registers the 'imap' experiment.
+//
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/imap"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func init() {
+	AllExperiments["imap"] = &Factory{
+		build: func(config interface{}) model.ExperimentMeasurer {
+			return imap.NewExperimentMeasurer(
+				*config.(*imap.Config),
+			)
+		},
+		config:      &imap.Config{},
+		inputPolicy: model.InputOrStaticDefault,
+	}
+}

internal/registry/smtp.go

@@ -0,0 +1,22 @@
+package registry
+
+//
+// Registers the 'smtp' experiment.
+//
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/smtp"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func init() {
+	AllExperiments["smtp"] = &Factory{
+		build: func(config interface{}) model.ExperimentMeasurer {
+			return smtp.NewExperimentMeasurer(
+				*config.(*smtp.Config),
+			)
+		},
+		config:      &smtp.Config{},
+		inputPolicy: model.InputOrStaticDefault,
+	}
+}

internal/tcprunner/tcprunner.go

@@ -0,0 +1,192 @@
+package tcprunner
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"net"
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/measurexlite"
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/tracex"
+)
+
+// Model describes a type that does a DNS lookup(s), then attempts several TCP sessions
+type Model interface {
+	// Stores the provided hostname
+	Hostname(string)
+	// Store DNS query result
+	DNSResults([]*model.ArchivalDNSLookupResult)
+	// Indicates one or more steps failed (can be overwritten)
+	Failed(string)
+	// Stores a new individual test key (for a TCP session) and returns a pointer to it
+	NewRun(string, string) TCPSessionModel
+}
+
+// TCPSessionModel describes a type that does a single TCP connection and TLS handshake with a given IP/Port combo
+type TCPSessionModel interface {
+	// Store IP/port address used for this session
+	IPPort(string, string)
+	// Store TCP connect result
+	ConnectResults([]*model.ArchivalTCPConnectResult)
+	// Store TLS handshake result
+	HandshakeResult(*model.ArchivalTLSOrQUICHandshakeResult)
+	// Indicates a failure string, as well as an identifier for the failed step
+	FailedStep(string, string)
+}
+
+// TCPRunner manages sequential TCP sessions to the same hostname (over different IPs)
+type TCPRunner struct {
+	Tk        Model
+	Trace     *measurexlite.Trace
+	Logger    model.Logger
+	Ctx       context.Context
+	Tlsconfig *tls.Config
+}
+
+// TCPSession Manages a single TCP session and TLS handshake to a given ip:port
+type TCPSession struct {
+	Itk     TCPSessionModel
+	Runner  *TCPRunner
+	Addr    string
+	Port    string
+	TLS     bool
+	RawConn *net.Conn
+	TLSConn *net.Conn
+}
+
+// FailedStep saves a failure (with an associated failed step identifier) into IndividualTestKeys
+func (s *TCPSession) FailedStep(failure string, step string) {
+	// Save FailedStep inside ITK
+	s.Itk.FailedStep(failure, step)
+	// Copy FailedStep to global TK
+	s.Runner.Tk.Failed(failure)
+	// Print the warning message
+	s.Runner.Logger.Warn(failure)
+}
+
+// Close closes the open TCP connections
+func (s *TCPSession) Close() {
+	if s.TLS {
+		var conn = *s.TLSConn
+		conn.Close()
+	} else {
+		// TODO: should raw connection be closed anyway?
+		var conn = *s.RawConn
+		conn.Close()
+	}
+}
+
+// CurrentConn returns the currently active connection (TLS or plaintext)
+func (s *TCPSession) CurrentConn() net.Conn {
+	if s.TLS {
+		// TODO: move to Debugf
+		s.Runner.Logger.Infof("Reusing TLS connection")
+		return *s.TLSConn
+	}
+	s.Runner.Logger.Infof("Reusing plaintext connection")
+	return *s.RawConn
+}
+
+// Conn initializes a new Run and IndividualTestKeys
+func (r *TCPRunner) Conn(addr string, port string) (*TCPSession, bool) {
+	// Get new individual test keys
+	itk := r.Tk.NewRun(addr, port)
+
+	s := new(TCPSession)
+	s.Runner = r
+	s.Itk = itk
+	s.Addr = addr
+	s.Port = port
+	s.TLS = false
+
+	if !s.Conn(addr, port) {
+		return nil, false
+	}
+	return s, true
+}
+
+// Conn starts a new TCP/IP connection to addr/port
+func (s *TCPSession) Conn(addr string, port string) bool {
+	dialer := s.Runner.Trace.NewDialerWithoutResolver(s.Runner.Logger)
+	s.Runner.Logger.Infof("Dialing to %s:%s", addr, port)
+	conn, err := dialer.DialContext(s.Runner.Ctx, "tcp", net.JoinHostPort(addr, port))
+	s.Itk.ConnectResults(s.Runner.Trace.TCPConnects())
+	if err != nil {
+		s.FailedStep(*tracex.NewFailure(err), "tcp_connect")
+		return false
+	}
+	s.RawConn = &conn
+
+	return true
+}
+
+// Resolve resolves a hostname to a list of addresses
+func (r *TCPRunner) Resolve(host string) ([]string, bool) {
+	r.Logger.Infof("Resolving DNS for %s", host)
+	resolver := r.Trace.NewStdlibResolver(r.Logger)
+	addrs, err := resolver.LookupHost(r.Ctx, host)
+	r.Tk.DNSResults(r.Trace.DNSLookupsFromRoundTrip())
+	if err != nil {
+		r.Tk.Failed(*tracex.NewFailure(err))
+		return []string{}, false
+	}
+	r.Logger.Infof("Finished DNS for %s: %v", host, addrs)
+
+	return addrs, true
+}
+
+// Handshake performs a TLS handshake over the currently active connection
+func (s *TCPSession) Handshake() bool {
+	if s.TLS {
+		// TLS already initialized...
+		return true
+	}
+	s.Runner.Logger.Infof("Starting TLS handshake with %s:%s", s.Addr, s.Port)
+	thx := s.Runner.Trace.NewTLSHandshakerStdlib(s.Runner.Logger)
+	tconn, _, err := thx.Handshake(s.Runner.Ctx, *s.RawConn, s.Runner.Tlsconfig)
+	s.Itk.HandshakeResult(s.Runner.Trace.FirstTLSHandshakeOrNil())
+	if err != nil {
+		s.FailedStep(*tracex.NewFailure(err), "tls_handshake")
+		return false
+	}
+
+	s.TLS = true
+	s.TLSConn = &tconn
+	s.Runner.Logger.Infof("Handshake succeeded")
+	return true
+}
+
+// StartTLS performs a StartTLS exchange by sending a message over the plaintext connection, waiting for a specific
+// response, then performing a TLS handshake
+func (s *TCPSession) StartTLS(message string, waitForResponse string) bool {
+	if s.TLS {
+		s.Runner.Logger.Warn("Requested TCPSession to do StartTLS when TLS is already enabled")
+		return true
+	}
+
+	if message != "" {
+		s.Runner.Logger.Infof("Asking for StartTLS upgrade")
+		s.CurrentConn().Write([]byte(message))
+	}
+
+	if waitForResponse != "" {
+		s.Runner.Logger.Infof("Waiting for server response containing: %s", waitForResponse)
+		conn := s.CurrentConn()
+		for {
+			line, err := bufio.NewReader(conn).ReadString('\n')
+			if err != nil {
+				s.FailedStep(*tracex.NewFailure(err), "starttls_wait_ok")
+				return false
+			}
+			s.Runner.Logger.Debugf("Received: %s", line)
+			if strings.Contains(line, waitForResponse) {
+				s.Runner.Logger.Infof("Server is ready for StartTLS")
+				break
+			}
+		}
+	}
+
+	return s.Handshake()
+}

internal/tutorial/experiment/torsf/chapter01/README.md

@@ -211,7 +211,12 @@ need any fancy context and we pass a `context.Background` to `Run`.
 
 ```Go
 	ctx := context.Background()
-	if err = m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err = m.Run(ctx, args); err != nil {
 		log.WithError(err).Fatal("torsf experiment failed")
 	}
 ```

internal/tutorial/experiment/torsf/chapter01/main.go

@@ -212,7 +212,12 @@ func main() {
 	//
 	// ```Go
 	ctx := context.Background()
-	if err = m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err = m.Run(ctx, args); err != nil {
 		log.WithError(err).Fatal("torsf experiment failed")
 	}
 	// ```

internal/tutorial/experiment/torsf/chapter02/README.md

@@ -117,10 +117,10 @@ chapters, finally, we will modify this function until it is a
 minimal implementation of the `torsf` experiment.
 
 ```Go
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	_ = args.Measurement
+	sess := args.Session
 ```
 As you can see, this is just a stub implementation that sleeps
 for one second and prints a logging message.

internal/tutorial/experiment/torsf/chapter02/main.go

@@ -54,7 +54,12 @@ func main() {
 		MockableLogger:  log.Log,
 		MockableTempDir: tempdir,
 	}
-	if err = m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err = m.Run(ctx, args); err != nil {
 		log.WithError(err).Fatal("torsf experiment failed")
 	}
 	data, err := json.Marshal(measurement)

internal/tutorial/experiment/torsf/chapter02/torsf.go

@@ -93,10 +93,10 @@ func (m *Measurer) ExperimentVersion() string {
 // minimal implementation of the `torsf` experiment.
 //
 // ```Go
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	_ = args.Callbacks
+	_ = args.Measurement
+	sess := args.Session
 	// ```
 	// As you can see, this is just a stub implementation that sleeps
 	// for one second and prints a logging message.

internal/tutorial/experiment/torsf/chapter03/README.md

@@ -32,10 +32,10 @@ print periodic updates via the `callbacks`. We will defer the
 real work to a private function called `run`.
 
 ```Go
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 ```
 
 Let's create an instance of `TestKeys` and let's modify

internal/tutorial/experiment/torsf/chapter03/main.go

@@ -28,7 +28,12 @@ func main() {
 		MockableLogger:  log.Log,
 		MockableTempDir: tempdir,
 	}
-	if err = m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err = m.Run(ctx, args); err != nil {
 		log.WithError(err).Fatal("torsf experiment failed")
 	}
 	data, err := json.Marshal(measurement)

internal/tutorial/experiment/torsf/chapter03/torsf.go

@@ -65,10 +65,10 @@ type TestKeys struct {
 // real work to a private function called `run`.
 //
 // ```Go
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	// ```
 	//
 	// Let's create an instance of `TestKeys` and let's modify

internal/tutorial/experiment/torsf/chapter04/main.go

@@ -28,7 +28,12 @@ func main() {
 		MockableLogger:  log.Log,
 		MockableTempDir: tempdir,
 	}
-	if err = m.Run(ctx, sess, measurement, callbacks); err != nil {
+	args := &model.ExperimentArgs{
+		Callbacks:   callbacks,
+		Measurement: measurement,
+		Session:     sess,
+	}
+	if err = m.Run(ctx, args); err != nil {
 		log.WithError(err).Fatal("torsf experiment failed")
 	}
 	data, err := json.Marshal(measurement)

internal/tutorial/experiment/torsf/chapter04/torsf.go

@@ -99,10 +99,10 @@ type TestKeys struct {
 }
 
 // Run implements ExperimentMeasurer.Run.
-func (m *Measurer) Run(
-	ctx context.Context, sess model.ExperimentSession,
-	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
-) error {
+func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
+	callbacks := args.Callbacks
+	measurement := args.Measurement
+	sess := args.Session
 	testkeys := &TestKeys{}
 	measurement.TestKeys = testkeys
 	start := time.Now()