chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/experiment/README.md

@@ -0,0 +1,24 @@
+# Directory github.com/ooni/probe-engine/experiment
+
+This directory contains the implementation of all the supported
+experiments, one for each directory. The [OONI spec repository
+contains a description of all the specified experiments](
+https://github.com/ooni/spec/tree/master/nettests).
+
+Note that in the OONI spec repository experiments are called
+nettests. Originally, they were also called nettests here but
+that created confusion with nettests in [ooni/probe-cli](
+https://github.com/ooni/probe-cli). Therefore, we now use the
+term experiment to indicate the implementation and the term
+nettest to indicate the user facing view of such implementation.
+
+Note that some experiments implemented here are not part of
+the OONI specification. For example, the [urlgetter](urlgetter)
+experiment is not in the OONI spec repository. The reason why
+this happens is that `urlgetter` is an experiment "library" that
+other experiments use to implement their functionality.
+
+Likewise, the [example](example) experiment is a minimal
+experiment that does nothing and you could use to bootstrap
+the implementation of a new experiment. Of course, this
+experiment is not part of the OONI specification.

internal/engine/experiment/dash/collect.go

@@ -0,0 +1,57 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type collectDeps interface {
+	HTTPClient() *http.Client
+	JSONMarshal(v interface{}) ([]byte, error)
+	Logger() model.Logger
+	NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
+	ReadAll(r io.Reader) ([]byte, error)
+	Scheme() string
+	UserAgent() string
+}
+
+func collect(ctx context.Context, fqdn, authorization string,
+	results []clientResults, deps collectDeps) error {
+	data, err := deps.JSONMarshal(results)
+	if err != nil {
+		return err
+	}
+	deps.Logger().Debugf("dash: body: %s", string(data))
+	var URL url.URL
+	URL.Scheme = deps.Scheme()
+	URL.Host = fqdn
+	URL.Path = collectPath
+	req, err := deps.NewHTTPRequest("POST", URL.String(), bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", deps.UserAgent())
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", authorization)
+	resp, err := deps.HTTPClient().Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != 200 {
+		return errHTTPRequestFailed
+	}
+	defer resp.Body.Close()
+	data, err = deps.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	deps.Logger().Debugf("dash: body: %s", string(data))
+	var serverResults []serverResults
+	return json.Unmarshal(data, &serverResults)
+}

internal/engine/experiment/dash/collect_test.go

@@ -0,0 +1,113 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestCollectJSONMarshalError(t *testing.T) {
+	expected := errors.New("mocked error")
+	deps := FakeDeps{jsonMarshalErr: expected}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectNewHTTPRequestFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	deps := FakeDeps{newHTTPRequestErr: expected}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectHTTPClientDoFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{err: expected}
+	deps := FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+		Header: http.Header{},
+		URL:    &url.URL{},
+	}}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectInternalError(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{StatusCode: 500}}
+	deps := FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+		Header: http.Header{},
+		URL:    &url.URL{},
+	}}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, errHTTPRequestFailed) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectReadAllFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllErr: expected,
+	}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectInvalidJSON(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte("["),
+	}
+	err := collect(context.Background(), "", "", nil, deps)
+	if err == nil || !strings.HasSuffix(err.Error(), "unexpected end of JSON input") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectSuccess(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte("[]"),
+	}
+	err := collect(context.Background(), "", "", nil, deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

internal/engine/experiment/dash/dash.go

@@ -0,0 +1,307 @@
+// Package dash implements the DASH network experiment.
+//
+// Spec: https://github.com/ooni/spec/blob/master/nettests/ts-021-dash.md
+package dash
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"runtime"
+	"time"
+
+	"github.com/montanaflynn/stats"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+const (
+	defaultTimeout = 120 * time.Second
+	magicVersion   = "0.008000000"
+	testName       = "dash"
+	testVersion    = "0.12.0"
+	totalStep      = 15.0
+)
+
+var (
+	errServerBusy        = errors.New("dash: server busy; try again later")
+	errHTTPRequestFailed = errors.New("dash: request failed")
+)
+
+// Config contains the experiment config.
+type Config struct{}
+
+// Simple contains the experiment total summary
+type Simple struct {
+	ConnectLatency  float64 `json:"connect_latency"`
+	MedianBitrate   int64   `json:"median_bitrate"`
+	MinPlayoutDelay float64 `json:"min_playout_delay"`
+}
+
+// ServerInfo contains information on the selected server
+//
+// This is currently an extension to the DASH specification
+// until the data format of the new mlab locate is clear.
+type ServerInfo struct {
+	Hostname string `json:"hostname"`
+	Site     string `json:"site,omitempty"`
+}
+
+// TestKeys contains the test keys
+type TestKeys struct {
+	Server       ServerInfo      `json:"server"`
+	Simple       Simple          `json:"simple"`
+	Failure      *string         `json:"failure"`
+	ReceiverData []clientResults `json:"receiver_data"`
+}
+
+type runner struct {
+	callbacks  model.ExperimentCallbacks
+	httpClient *http.Client
+	saver      *trace.Saver
+	sess       model.ExperimentSession
+	tk         *TestKeys
+}
+
+func (r runner) HTTPClient() *http.Client {
+	return r.httpClient
+}
+
+func (r runner) JSONMarshal(v interface{}) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+func (r runner) Logger() model.Logger {
+	return r.sess.Logger()
+}
+
+func (r runner) NewHTTPRequest(meth, url string, body io.Reader) (*http.Request, error) {
+	return http.NewRequest(meth, url, body)
+}
+
+func (r runner) ReadAll(reader io.Reader) ([]byte, error) {
+	return ioutil.ReadAll(reader)
+}
+
+func (r runner) Scheme() string {
+	return "https"
+}
+
+func (r runner) UserAgent() string {
+	return r.sess.UserAgent()
+}
+
+func (r runner) loop(ctx context.Context, numIterations int64) error {
+	locateResult, err := locate(ctx, r)
+	if err != nil {
+		return err
+	}
+	r.tk.Server = ServerInfo{
+		Hostname: locateResult.FQDN,
+		Site:     locateResult.Site,
+	}
+	fqdn := locateResult.FQDN
+	r.callbacks.OnProgress(0.0, fmt.Sprintf("streaming: server: %s", fqdn))
+	negotiateResp, err := negotiate(ctx, fqdn, r)
+	if err != nil {
+		return err
+	}
+	if err := r.measure(ctx, fqdn, negotiateResp, numIterations); err != nil {
+		return err
+	}
+	// TODO(bassosimone): it seems we're not saving the server data?
+	err = collect(ctx, fqdn, negotiateResp.Authorization, r.tk.ReceiverData, r)
+	if err != nil {
+		return err
+	}
+	return r.tk.analyze()
+}
+
+func (r runner) measure(
+	ctx context.Context, fqdn string, negotiateResp negotiateResponse,
+	numIterations int64) error {
+	// Note: according to a comment in MK sources 3000 kbit/s was the
+	// minimum speed recommended by Netflix for SD quality in 2017.
+	//
+	// See: <https://help.netflix.com/en/node/306>.
+	const initialBitrate = 3000
+	current := clientResults{
+		ElapsedTarget: 2,
+		Platform:      runtime.GOOS,
+		Rate:          initialBitrate,
+		RealAddress:   negotiateResp.RealAddress,
+		Version:       magicVersion,
+	}
+	var (
+		begin       = time.Now()
+		connectTime float64
+		total       int64
+	)
+	for current.Iteration < numIterations {
+		result, err := download(ctx, downloadConfig{
+			authorization: negotiateResp.Authorization,
+			begin:         begin,
+			currentRate:   current.Rate,
+			deps:          r,
+			elapsedTarget: current.ElapsedTarget,
+			fqdn:          fqdn,
+		})
+		if err != nil {
+			// Implementation note: ndt7 controls the connection much
+			// more than us and it can tell whether an error occurs when
+			// connecting or later. We cannot say that very precisely
+			// because, in principle, we may reconnect. So we always
+			// return error here. This comment is being introduced so
+			// that we don't do https://github.com/ooni/probe-cli/v3/internal/engine/pull/526
+			// again, because that isn't accurate.
+			return err
+		}
+		current.Elapsed = result.elapsed
+		current.Received = result.received
+		current.RequestTicks = result.requestTicks
+		current.Timestamp = result.timestamp
+		current.ServerURL = result.serverURL
+		// Read the events so far and possibly update our measurement
+		// of the latest connect time. We should have one sample in most
+		// cases, because the connection should be persistent.
+		for _, ev := range r.saver.Read() {
+			if ev.Name == errorx.ConnectOperation {
+				connectTime = ev.Duration.Seconds()
+			}
+		}
+		current.ConnectTime = connectTime
+		r.tk.ReceiverData = append(r.tk.ReceiverData, current)
+		total += current.Received
+		avgspeed := 8 * float64(total) / time.Now().Sub(begin).Seconds()
+		percentage := float64(current.Iteration) / float64(numIterations)
+		message := fmt.Sprintf("streaming: speed: %s", humanizex.SI(avgspeed, "bit/s"))
+		r.callbacks.OnProgress(percentage, message)
+		current.Iteration++
+		speed := float64(current.Received) / float64(current.Elapsed)
+		speed *= 8.0    // to bits per second
+		speed /= 1000.0 // to kbit/s
+		current.Rate = int64(speed)
+	}
+	return nil
+}
+
+func (tk *TestKeys) analyze() error {
+	var (
+		rates          []float64
+		frameReadyTime float64
+		playTime       float64
+	)
+	for _, results := range tk.ReceiverData {
+		rates = append(rates, float64(results.Rate))
+		// Same in all samples if we're using a single connection
+		tk.Simple.ConnectLatency = results.ConnectTime
+		// Rationale: first segment plays when it arrives. Subsequent segments
+		// would play in ElapsedTarget seconds. However, will play when they
+		// arrive. Stall is the time we need to wait for a frame to arrive with
+		// the video stopped and the spinning icon.
+		frameReadyTime += results.Elapsed
+		if playTime == 0.0 {
+			playTime += frameReadyTime
+		} else {
+			playTime += float64(results.ElapsedTarget)
+		}
+		stall := frameReadyTime - playTime
+		if stall > tk.Simple.MinPlayoutDelay {
+			tk.Simple.MinPlayoutDelay = stall
+		}
+	}
+	median, err := stats.Median(rates)
+	tk.Simple.MedianBitrate = int64(median)
+	return err
+}
+
+func (r runner) do(ctx context.Context) error {
+	defer r.callbacks.OnProgress(1, "streaming: done")
+	const numIterations = 15
+	err := r.loop(ctx, numIterations)
+	if err != nil {
+		s := err.Error()
+		r.tk.Failure = &s
+		// fallthrough
+	}
+	return err
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config Config
+}
+
+// ExperimentName implements model.ExperimentMeasurer.ExperimentName.
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements model.ExperimentMeasurer.ExperimentVersion.
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements model.ExperimentMeasurer.Run.
+func (m Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	tk := new(TestKeys)
+	measurement.TestKeys = tk
+	saver := &trace.Saver{}
+	httpClient := &http.Client{
+		Transport: netx.NewHTTPTransport(netx.Config{
+			ContextByteCounting: true,
+			DialSaver:           saver,
+			Logger:              sess.Logger(),
+		}),
+	}
+	defer httpClient.CloseIdleConnections()
+	r := runner{
+		callbacks:  callbacks,
+		httpClient: httpClient,
+		saver:      saver,
+		sess:       sess,
+		tk:         tk,
+	}
+	ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
+	defer cancel()
+	return r.do(ctx)
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	Latency   float64 `json:"connect_latency"`
+	Bitrate   float64 `json:"median_bitrate"`
+	Delay     float64 `json:"min_playout_delay"`
+	IsAnomaly bool    `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	sk.Latency = tk.Simple.ConnectLatency
+	sk.Bitrate = float64(tk.Simple.MedianBitrate)
+	sk.Delay = tk.Simple.MinPlayoutDelay
+	return sk, nil
+}

internal/engine/experiment/dash/dash_test.go

@@ -0,0 +1,328 @@
+package dash
+
+import (
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/montanaflynn/stats"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+func TestRunnerLoopLocateFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: FakeHTTPTransport{
+				err: expected,
+			},
+		},
+		saver: new(trace.Saver),
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerLoopNegotiateFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: &FakeHTTPTransportStack{
+				all: []FakeHTTPTransport{
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"fqdn": "ams01.measurementlab.net"}`)),
+							StatusCode: 200,
+						},
+					},
+					{err: expected},
+				},
+			},
+		},
+		saver: new(trace.Saver),
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerLoopMeasureFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: &FakeHTTPTransportStack{
+				all: []FakeHTTPTransport{
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"fqdn": "ams01.measurementlab.net"}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"authorization": "xx", "unchoked": 1}`)),
+							StatusCode: 200,
+						},
+					},
+					{err: expected},
+				},
+			},
+		},
+		saver: new(trace.Saver),
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerLoopCollectFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	saver := new(trace.Saver)
+	saver.Write(trace.Event{Name: errorx.ConnectOperation, Duration: 150 * time.Millisecond})
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: &FakeHTTPTransportStack{
+				all: []FakeHTTPTransport{
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"fqdn": "ams01.measurementlab.net"}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"authorization": "xx", "unchoked": 1}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body:       ioutil.NopCloser(strings.NewReader(`1234567`)),
+							StatusCode: 200,
+						},
+					},
+					{err: expected},
+				},
+			},
+		},
+		saver: saver,
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerLoopSuccess(t *testing.T) {
+	saver := new(trace.Saver)
+	saver.Write(trace.Event{Name: errorx.ConnectOperation, Duration: 150 * time.Millisecond})
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: &FakeHTTPTransportStack{
+				all: []FakeHTTPTransport{
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"fqdn": "ams01.measurementlab.net"}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"authorization": "xx", "unchoked": 1}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body:       ioutil.NopCloser(strings.NewReader(`1234567`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body:       ioutil.NopCloser(strings.NewReader(`[]`)),
+							StatusCode: 200,
+						},
+					},
+				},
+			},
+		},
+		saver: saver,
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestTestKeysAnalyzeWithNoData(t *testing.T) {
+	tk := &TestKeys{}
+	err := tk.analyze()
+	if !errors.Is(err, stats.EmptyInputErr) {
+		t.Fatal("expected an error here")
+	}
+}
+
+func TestTestKeysAnalyzeMedian(t *testing.T) {
+	tk := &TestKeys{
+		ReceiverData: []clientResults{
+			{
+				Rate: 1,
+			},
+			{
+				Rate: 2,
+			},
+			{
+				Rate: 3,
+			},
+		},
+	}
+	err := tk.analyze()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Simple.MedianBitrate != 2 {
+		t.Fatal("unexpected median value")
+	}
+}
+
+func TestTestKeysAnalyzeMinPlayoutDelay(t *testing.T) {
+	tk := &TestKeys{
+		ReceiverData: []clientResults{
+			{
+				ElapsedTarget: 2,
+				Elapsed:       1.4,
+			},
+			{
+				ElapsedTarget: 2,
+				Elapsed:       3.0,
+			},
+			{
+				ElapsedTarget: 2,
+				Elapsed:       1.8,
+			},
+		},
+	}
+	err := tk.analyze()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Simple.MinPlayoutDelay < 0.99 || tk.Simple.MinPlayoutDelay > 1.01 {
+		t.Fatal("unexpected min-playout-delay value")
+	}
+}
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	if measurer.ExperimentName() != "dash" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.12.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestMeasureWithCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // cause failure
+	measurement := new(model.Measurement)
+	m := &Measurer{}
+	err := m.Run(
+		ctx,
+		&mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("unexpected error value")
+	}
+	sk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysGood(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &TestKeys{Simple: Simple{
+		ConnectLatency:  1234,
+		MedianBitrate:   123,
+		MinPlayoutDelay: 12,
+	}}}
+	m := &Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(SummaryKeys)
+	if sk.Latency != 1234 {
+		t.Fatal("invalid latency")
+	}
+	if sk.Bitrate != 123 {
+		t.Fatal("invalid bitrate")
+	}
+	if sk.Delay != 12 {
+		t.Fatal("invalid delay")
+	}
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/dash/download.go

@@ -0,0 +1,74 @@
+package dash
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+type downloadDeps interface {
+	HTTPClient() *http.Client
+	NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
+	ReadAll(r io.Reader) ([]byte, error)
+	Scheme() string
+	UserAgent() string
+}
+
+type downloadConfig struct {
+	authorization string
+	begin         time.Time
+	currentRate   int64
+	deps          downloadDeps
+	elapsedTarget int64
+	fqdn          string
+}
+
+type downloadResult struct {
+	elapsed      float64
+	received     int64
+	requestTicks float64
+	serverURL    string
+	timestamp    int64
+}
+
+func download(ctx context.Context, config downloadConfig) (downloadResult, error) {
+	nbytes := (config.currentRate * 1000 * config.elapsedTarget) >> 3
+	var URL url.URL
+	URL.Scheme = config.deps.Scheme()
+	URL.Host = config.fqdn
+	URL.Path = fmt.Sprintf("%s%d", downloadPath, nbytes)
+	req, err := config.deps.NewHTTPRequest("GET", URL.String(), nil)
+	var result downloadResult
+	if err != nil {
+		return result, err
+	}
+	result.serverURL = URL.String()
+	req.Header.Set("User-Agent", config.deps.UserAgent())
+	req.Header.Set("Authorization", config.authorization)
+	savedTicks := time.Now()
+	resp, err := config.deps.HTTPClient().Do(req.WithContext(ctx))
+	if err != nil {
+		return result, err
+	}
+	if resp.StatusCode != 200 {
+		return result, errHTTPRequestFailed
+	}
+	defer resp.Body.Close()
+	data, err := config.deps.ReadAll(resp.Body)
+	if err != nil {
+		return result, err
+	}
+	// Implementation note: MK contains a comment that says that Neubot uses
+	// the elapsed time since when we start receiving the response but it
+	// turns out that Neubot and MK do the same. So, we do what they do. At
+	// the same time, we are currently not able to include the overhead that
+	// is caused by HTTP headers etc. So, we're a bit less precise.
+	result.elapsed = time.Now().Sub(savedTicks).Seconds()
+	result.received = int64(len(data))
+	result.requestTicks = savedTicks.Sub(config.begin).Seconds()
+	result.timestamp = time.Now().Unix()
+	return result, nil
+}

internal/engine/experiment/dash/download_test.go

@@ -0,0 +1,104 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+func TestDownloadNewHTTPRequestFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	_, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{newHTTPRequestErr: expected},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadHTTPClientDoFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{err: expected}
+	_, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		}},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadInternalError(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{StatusCode: 500}}
+	_, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		}},
+	})
+	if !errors.Is(err, errHTTPRequestFailed) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadReadAllFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	_, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{
+			httpTransport: txp,
+			newHTTPRequestResult: &http.Request{
+				Header: http.Header{},
+				URL:    &url.URL{},
+			},
+			readAllErr: expected,
+		},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadSuccess(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	result, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{
+			httpTransport: txp,
+			newHTTPRequestResult: &http.Request{
+				Header: http.Header{},
+				URL:    &url.URL{},
+			},
+			readAllResult: []byte("[]"),
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.elapsed <= 0 {
+		t.Fatal("invalid elapsed")
+	}
+	if result.received <= 0 {
+		t.Fatal("invalid received")
+	}
+	if result.requestTicks <= 0 {
+		t.Fatal("invalid requestTicks")
+	}
+	if result.serverURL == "" {
+		t.Fatal("invalid serverURL")
+	}
+	if result.timestamp <= 0 {
+		t.Fatal("invalid timestamp")
+	}
+}

internal/engine/experiment/dash/fake_test.go

@@ -0,0 +1,69 @@
+package dash
+
+import (
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type FakeDeps struct {
+	httpTransport        http.RoundTripper
+	jsonMarshalErr       error
+	jsonMarshalResult    []byte
+	newHTTPRequestErr    error
+	newHTTPRequestResult *http.Request
+	readAllErr           error
+	readAllResult        []byte
+}
+
+func (d FakeDeps) HTTPClient() *http.Client {
+	return &http.Client{Transport: d.httpTransport}
+}
+
+func (d FakeDeps) JSONMarshal(v interface{}) ([]byte, error) {
+	return d.jsonMarshalResult, d.jsonMarshalErr
+}
+
+func (d FakeDeps) Logger() model.Logger {
+	return log.Log
+}
+
+func (d FakeDeps) NewHTTPRequest(
+	method string, url string, body io.Reader) (*http.Request, error) {
+	return d.newHTTPRequestResult, d.newHTTPRequestErr
+}
+
+func (d FakeDeps) ReadAll(r io.Reader) ([]byte, error) {
+	return d.readAllResult, d.readAllErr
+}
+
+func (d FakeDeps) Scheme() string {
+	return "https"
+}
+
+func (d FakeDeps) UserAgent() string {
+	return "miniooni/0.1.0-dev"
+}
+
+type FakeHTTPTransport struct {
+	err  error
+	resp *http.Response
+}
+
+func (txp FakeHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	time.Sleep(10 * time.Microsecond)
+	return txp.resp, txp.err
+}
+
+type FakeHTTPTransportStack struct {
+	all []FakeHTTPTransport
+}
+
+func (txp *FakeHTTPTransportStack) RoundTrip(req *http.Request) (*http.Response, error) {
+	frame := txp.all[0]
+	txp.all = txp.all[1:]
+	return frame.RoundTrip(req)
+}

internal/engine/experiment/dash/locate.go

@@ -0,0 +1,20 @@
+package dash
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mlablocate"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type locateDeps interface {
+	HTTPClient() *http.Client
+	Logger() model.Logger
+	UserAgent() string
+}
+
+func locate(ctx context.Context, deps locateDeps) (mlablocate.Result, error) {
+	return mlablocate.NewClient(
+		deps.HTTPClient(), deps.Logger(), deps.UserAgent()).Query(ctx, "neubot")
+}

internal/engine/experiment/dash/model.go

@@ -0,0 +1,47 @@
+package dash
+
+// clientResults contains the results measured by the client. This data
+// structure is sent to the server in the collection phase.
+//
+// All the fields listed here are part of the original specification
+// of DASH, except ServerURL, added in MK v0.10.6.
+type clientResults struct {
+	ConnectTime     float64 `json:"connect_time"`
+	DeltaSysTime    float64 `json:"delta_sys_time"`
+	DeltaUserTime   float64 `json:"delta_user_time"`
+	Elapsed         float64 `json:"elapsed"`
+	ElapsedTarget   int64   `json:"elapsed_target"`
+	InternalAddress string  `json:"internal_address"`
+	Iteration       int64   `json:"iteration"`
+	Platform        string  `json:"platform"`
+	Rate            int64   `json:"rate"`
+	RealAddress     string  `json:"real_address"`
+	Received        int64   `json:"received"`
+	RemoteAddress   string  `json:"remote_address"`
+	RequestTicks    float64 `json:"request_ticks"`
+	ServerURL       string  `json:"server_url"`
+	Timestamp       int64   `json:"timestamp"`
+	UUID            string  `json:"uuid"`
+	Version         string  `json:"version"`
+}
+
+// serverResults contains the server results. This data structure is sent
+// to the client during the collection phase of DASH.
+type serverResults struct {
+	Iteration int64   `json:"iteration"`
+	Ticks     float64 `json:"ticks"`
+	Timestamp int64   `json:"timestamp"`
+}
+
+// negotiateRequest contains the request of negotiation
+type negotiateRequest struct {
+	DASHRates []int64 `json:"dash_rates"`
+}
+
+// negotiateResponse contains the response of negotiation
+type negotiateResponse struct {
+	Authorization string `json:"authorization"`
+	QueuePos      int64  `json:"queue_pos"`
+	RealAddress   string `json:"real_address"`
+	Unchoked      int    `json:"unchoked"`
+}

internal/engine/experiment/dash/negotiate.go

@@ -0,0 +1,68 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type negotiateDeps interface {
+	HTTPClient() *http.Client
+	JSONMarshal(v interface{}) ([]byte, error)
+	Logger() model.Logger
+	NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
+	ReadAll(r io.Reader) ([]byte, error)
+	Scheme() string
+	UserAgent() string
+}
+
+func negotiate(
+	ctx context.Context, fqdn string, deps negotiateDeps) (negotiateResponse, error) {
+	var negotiateResp negotiateResponse
+	data, err := deps.JSONMarshal(negotiateRequest{DASHRates: defaultRates})
+	if err != nil {
+		return negotiateResp, err
+	}
+	deps.Logger().Debugf("dash: body: %s", string(data))
+	var URL url.URL
+	URL.Scheme = deps.Scheme()
+	URL.Host = fqdn
+	URL.Path = negotiatePath
+	req, err := deps.NewHTTPRequest("POST", URL.String(), bytes.NewReader(data))
+	if err != nil {
+		return negotiateResp, err
+	}
+	req.Header.Set("User-Agent", deps.UserAgent())
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "")
+	resp, err := deps.HTTPClient().Do(req.WithContext(ctx))
+	if err != nil {
+		return negotiateResp, err
+	}
+	if resp.StatusCode != 200 {
+		return negotiateResp, errHTTPRequestFailed
+	}
+	defer resp.Body.Close()
+	data, err = deps.ReadAll(resp.Body)
+	if err != nil {
+		return negotiateResp, err
+	}
+	deps.Logger().Debugf("dash: body: %s", string(data))
+	err = json.Unmarshal(data, &negotiateResp)
+	if err != nil {
+		return negotiateResp, err
+	}
+	// Implementation oddity: Neubot is using an integer rather than a
+	// boolean for the unchoked, with obvious semantics. I wonder why
+	// I choose an integer over a boolean, given that Python does have
+	// support for booleans. I don't remember 🤷.
+	if negotiateResp.Authorization == "" || negotiateResp.Unchoked == 0 {
+		return negotiateResp, errServerBusy
+	}
+	return negotiateResp, nil
+}

internal/engine/experiment/dash/negotiate_test.go

@@ -0,0 +1,178 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestNegotiateJSONMarshalError(t *testing.T) {
+	expected := errors.New("mocked error")
+	deps := FakeDeps{jsonMarshalErr: expected}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateNewHTTPRequestFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	deps := FakeDeps{newHTTPRequestErr: expected}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateHTTPClientDoFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{err: expected}
+	deps := FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+		Header: http.Header{},
+		URL:    &url.URL{},
+	}}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateInternalError(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{StatusCode: 500}}
+	deps := FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+		Header: http.Header{},
+		URL:    &url.URL{},
+	}}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, errHTTPRequestFailed) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateReadAllFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllErr: expected,
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateInvalidJSON(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte("["),
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if err == nil || !strings.HasSuffix(err.Error(), "unexpected end of JSON input") {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateServerBusyFirstCase(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte(`{"authorization": ""}`),
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, errServerBusy) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateServerBusyThirdCase(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte(`{}`),
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, errServerBusy) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateSuccess(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte(`{"authorization": "xx", "unchoked": 1}`),
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Authorization != "xx" || result.Unchoked != 1 {
+		t.Fatal("invalid result")
+	}
+}

internal/engine/experiment/dash/spec.go

@@ -0,0 +1,26 @@
+package dash
+
+const (
+	// currentServerSchemaVersion is the version of the server schema that
+	// will be adopted by this implementation. Version 3 is the one that is
+	// Neubot uses. We needed to bump the version because Web100 is not on
+	// M-Lab anymore and hence we need to make a breaking change.
+	currentServerSchemaVersion = 4
+
+	// negotiatePath is the URL path used to negotiate
+	negotiatePath = "/negotiate/dash"
+
+	// downloadPath is the URL path used to request DASH segments. You can
+	// append to this path an integer indicating how many bytes you would like
+	// the server to send you as part of the next chunk.
+	downloadPath = "/dash/download/"
+
+	// collectPath is the URL path used to collect
+	collectPath = "/collect/dash"
+)
+
+// defaultRates contains the default DASH rates in kbit/s.
+var defaultRates = []int64{
+	100, 150, 200, 250, 300, 400, 500, 700, 900, 1200, 1500, 2000,
+	2500, 3000, 4000, 5000, 6000, 7000, 10000, 20000,
+}

internal/engine/experiment/dnscheck/dnscheck.go

@@ -0,0 +1,323 @@
+// Package dnscheck contains the DNS check experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-028-dnscheck.md.
+package dnscheck
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+const (
+	testName      = "dnscheck"
+	testVersion   = "0.9.0"
+	defaultDomain = "example.org"
+)
+
+// Endpoints keeps track of repeatedly measured endpoints.
+type Endpoints struct {
+	WaitTime  time.Duration
+	count     uint32
+	nextVisit map[string]time.Time
+	mu        sync.Mutex
+}
+
+func (e *Endpoints) maybeSleep(resolverURL string, logger model.Logger) {
+	if e == nil {
+		return
+	}
+	defer e.mu.Unlock()
+	e.mu.Lock()
+	nextTime, found := e.nextVisit[resolverURL]
+	now := time.Now()
+	if !found || now.After(nextTime) {
+		return
+	}
+	sleepTime := nextTime.Sub(now)
+	atomic.AddUint32(&e.count, 1)
+	logger.Infof("waiting %v before testing %s again", sleepTime, resolverURL)
+	time.Sleep(sleepTime)
+}
+
+func (e *Endpoints) maybeRegister(resolverURL string) {
+	if e != nil && !strings.HasPrefix(resolverURL, "udp://") {
+		defer e.mu.Unlock()
+		e.mu.Lock()
+		if e.nextVisit == nil {
+			e.nextVisit = make(map[string]time.Time)
+		}
+		waitTime := 180 * time.Second
+		if e.WaitTime > 0 {
+			waitTime = e.WaitTime
+		}
+		e.nextVisit[resolverURL] = time.Now().Add(waitTime)
+	}
+}
+
+// Config contains the experiment's configuration.
+type Config struct {
+	DefaultAddrs  string `json:"default_addrs" ooni:"default addresses for domain"`
+	Domain        string `json:"domain" ooni:"domain to resolve using the specified resolver"`
+	HTTP3Enabled  bool   `json:"http3_enabled" ooni:"use http3 instead of http/1.1 or http2"`
+	HTTPHost      string `json:"http_host" ooni:"force using specific HTTP Host header"`
+	TLSServerName string `json:"tls_server_name" ooni:"force TLS to using a specific SNI in Client Hello"`
+	TLSVersion    string `json:"tls_version" ooni:"Force specific TLS version (e.g. 'TLSv1.3')"`
+}
+
+// TestKeys contains the results of the dnscheck experiment.
+type TestKeys struct {
+	DefaultAddrs     string                        `json:"x_default_addrs"`
+	Domain           string                        `json:"domain"`
+	HTTP3Enabled     bool                          `json:"x_http3_enabled,omitempty"`
+	HTTPHost         string                        `json:"x_http_host,omitempty"`
+	TLSServerName    string                        `json:"x_tls_server_name,omitempty"`
+	TLSVersion       string                        `json:"x_tls_version,omitempty"`
+	Bootstrap        *urlgetter.TestKeys           `json:"bootstrap"`
+	BootstrapFailure *string                       `json:"bootstrap_failure"`
+	Lookups          map[string]urlgetter.TestKeys `json:"lookups"`
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	Config
+	Endpoints *Endpoints
+}
+
+// ExperimentName implements model.ExperimentSession.ExperimentName
+func (m *Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements model.ExperimentSession.ExperimentVersion
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// The following errors may be returned by this experiment. Of course these
+// errors are in addition to any other errors returned by the low level packages
+// that are used by this experiment to implement its functionality.
+var (
+	ErrInputRequired        = errors.New("this experiment needs input")
+	ErrInvalidURL           = errors.New("the input URL is invalid")
+	ErrUnsupportedURLScheme = errors.New("unsupported URL scheme")
+)
+
+// Run implements model.ExperimentSession.Run
+func (m *Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	// 1. fill the measurement with test keys
+	tk := new(TestKeys)
+	tk.Lookups = make(map[string]urlgetter.TestKeys)
+	measurement.TestKeys = tk
+	urlgetter.RegisterExtensions(measurement)
+
+	// 2. select the domain to resolve or use default and, while there, also
+	// ensure that we register all the other options we're using.
+	domain := m.Config.Domain
+	if domain == "" {
+		domain = defaultDomain
+	}
+	tk.DefaultAddrs = m.Config.DefaultAddrs
+	tk.Domain = domain
+	tk.HTTP3Enabled = m.Config.HTTP3Enabled
+	tk.HTTPHost = m.Config.HTTPHost
+	tk.TLSServerName = m.Config.TLSServerName
+	tk.TLSVersion = m.Config.TLSVersion
+
+	// 3. parse the input URL describing the resolver to use
+	input := string(measurement.Input)
+	if input == "" {
+		return ErrInputRequired
+	}
+	URL, err := url.Parse(input)
+	if err != nil {
+		return fmt.Errorf("%w: %s", ErrInvalidURL, err.Error())
+	}
+	switch URL.Scheme {
+	case "https", "dot", "udp", "tcp":
+		// all good
+	default:
+		return ErrUnsupportedURLScheme
+	}
+
+	// 4. possibly expand a domain to a list of IP addresses.
+	//
+	// Implementation note: because the resolver we constructed also deals
+	// with IP addresses successfully, we just get back the IPs when we are
+	// passing as input an IP address rather than a domain name.
+	begin := measurement.MeasurementStartTimeSaved
+	evsaver := new(trace.Saver)
+	resolver := netx.NewResolver(netx.Config{
+		BogonIsError: true,
+		Logger:       sess.Logger(),
+		ResolveSaver: evsaver,
+	})
+	addrs, err := m.lookupHost(ctx, URL.Hostname(), resolver)
+	queries := archival.NewDNSQueriesList(begin, evsaver.Read(), sess.ASNDatabasePath())
+	tk.BootstrapFailure = archival.NewFailure(err)
+	if len(queries) > 0 {
+		// We get no queries in case we are resolving an IP address, since
+		// the address resolver doesn't generate events
+		tk.Bootstrap = &urlgetter.TestKeys{Queries: queries}
+	}
+
+	// 5. merge default addresses for the domain with the ones that
+	// we did discover here and measure them all.
+	allAddrs := make(map[string]bool)
+	for _, addr := range addrs {
+		allAddrs[addr] = true
+	}
+	for _, addr := range strings.Split(m.Config.DefaultAddrs, " ") {
+		if addr != "" {
+			allAddrs[addr] = true
+		}
+	}
+
+	// 6. determine all the domain lookups we need to perform
+	const maxParallelism = 10
+	parallelism := maxParallelism
+	if parallelism > len(allAddrs) {
+		parallelism = len(allAddrs)
+	}
+	var inputs []urlgetter.MultiInput
+	multi := urlgetter.Multi{Begin: begin, Parallelism: parallelism, Session: sess}
+	for addr := range allAddrs {
+		inputs = append(inputs, urlgetter.MultiInput{
+			Config: urlgetter.Config{
+				DNSHTTPHost:      m.httpHost(URL.Host),
+				DNSTLSServerName: m.tlsServerName(URL.Hostname()),
+				DNSTLSVersion:    m.Config.TLSVersion,
+				HTTP3Enabled:     m.Config.HTTP3Enabled,
+				RejectDNSBogons:  true, // bogons are errors in this context
+				ResolverURL:      makeResolverURL(URL, addr),
+				Timeout:          45 * time.Second,
+			},
+			Target: fmt.Sprintf("dnslookup://%s", domain), // urlgetter wants a URL
+		})
+	}
+
+	// 7. make sure we don't test the same endpoint too frequently
+	// because this may cause residual censorship.
+	for _, input := range inputs {
+		resolverURL := input.Config.ResolverURL
+		m.Endpoints.maybeSleep(resolverURL, sess.Logger())
+	}
+
+	// 8. perform all the required resolutions
+	for output := range Collect(ctx, multi, inputs, callbacks) {
+		resolverURL := output.Input.Config.ResolverURL
+		tk.Lookups[resolverURL] = output.TestKeys
+		m.Endpoints.maybeRegister(resolverURL)
+	}
+	return nil
+}
+
+func (m *Measurer) lookupHost(ctx context.Context, hostname string, r netx.Resolver) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	defer cancel()
+	return r.LookupHost(ctx, hostname)
+}
+
+// httpHost returns the configured HTTP host, if set, otherwise
+// it will return the host provide as argument.
+func (m *Measurer) httpHost(httpHost string) string {
+	if m.Config.HTTPHost != "" {
+		return m.Config.HTTPHost
+	}
+	return httpHost
+}
+
+// tlsServerName is like httpHost for the TLS server name.
+func (m *Measurer) tlsServerName(tlsServerName string) string {
+	if m.Config.TLSServerName != "" {
+		return m.Config.TLSServerName
+	}
+	return tlsServerName
+}
+
+// Collect prints on the output channel the result of running dnscheck
+// on every provided input. It closes the output channel when done.
+func Collect(ctx context.Context, multi urlgetter.Multi, inputs []urlgetter.MultiInput,
+	callbacks model.ExperimentCallbacks) <-chan urlgetter.MultiOutput {
+	outputch := make(chan urlgetter.MultiOutput)
+	expect := len(inputs)
+	inputch := multi.Run(ctx, inputs)
+	go func() {
+		var count int
+		defer close(outputch)
+		for count < expect {
+			entry := <-inputch
+			count++
+			percentage := float64(count) / float64(expect)
+			callbacks.OnProgress(percentage, fmt.Sprintf(
+				"dnscheck: measure %s: %+v", entry.Input.Config.ResolverURL, entry.Err,
+			))
+			outputch <- entry
+		}
+	}()
+	return outputch
+}
+
+// makeResolverURL rewrites the input URL to replace the domain in
+// the input URL with the given addr. When the input URL already contains
+// an addr, this operation will return the same URL.
+func makeResolverURL(URL *url.URL, addr string) string {
+	// 1. determine the hostname in the resulting URL
+	hostname := URL.Hostname()
+	if net.ParseIP(hostname) == nil {
+		hostname = addr
+	}
+	// 2. adjust hostname if we also have a port
+	if hasPort := URL.Port() != ""; hasPort {
+		_, port, err := net.SplitHostPort(URL.Host)
+		// We say this cannot fail because we already parsed the URL to validate
+		// its scheme and hence the URL hostname should be well formed.
+		runtimex.PanicOnError(err, "net.SplitHostPort should not fail here")
+		hostname = net.JoinHostPort(hostname, port)
+	} else if idx := strings.Index(addr, ":"); idx >= 0 {
+		// Make sure an IPv6 address hostname without a port is properly
+		// quoted to avoid breaking the URL parser down the line.
+		hostname = "[" + addr + "]"
+	}
+	// 3. reassemble the URL
+	return (&url.URL{
+		Scheme:   URL.Scheme,
+		Host:     hostname,
+		Path:     URL.Path,
+		RawQuery: URL.RawQuery,
+	}).String()
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return &Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/dnscheck/dnscheck_test.go

@@ -0,0 +1,227 @@
+package dnscheck
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestHTTPHostWithOverride(t *testing.T) {
+	m := Measurer{Config: Config{HTTPHost: "antani"}}
+	result := m.httpHost("mascetti")
+	if result != "antani" {
+		t.Fatal("not the result we expected")
+	}
+}
+
+func TestHTTPHostWithoutOverride(t *testing.T) {
+	m := Measurer{Config: Config{}}
+	result := m.httpHost("mascetti")
+	if result != "mascetti" {
+		t.Fatal("not the result we expected")
+	}
+}
+
+func TestTLSServerNameWithOverride(t *testing.T) {
+	m := Measurer{Config: Config{TLSServerName: "antani"}}
+	result := m.tlsServerName("mascetti")
+	if result != "antani" {
+		t.Fatal("not the result we expected")
+	}
+}
+
+func TestTLSServerNameWithoutOverride(t *testing.T) {
+	m := Measurer{Config: Config{}}
+	result := m.tlsServerName("mascetti")
+	if result != "mascetti" {
+		t.Fatal("not the result we expected")
+	}
+}
+
+func TestExperimentNameAndVersion(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{Domain: "example.com"})
+	if measurer.ExperimentName() != "dnscheck" {
+		t.Error("unexpected experiment name")
+	}
+	if measurer.ExperimentVersion() != "0.9.0" {
+		t.Error("unexpected experiment version")
+	}
+}
+
+func TestDNSCheckFailsWithoutInput(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{Domain: "example.com"})
+	err := measurer.Run(
+		context.Background(),
+		newsession(),
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, ErrInputRequired) {
+		t.Fatal("expected no input error")
+	}
+}
+
+func TestDNSCheckFailsWithInvalidURL(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	err := measurer.Run(
+		context.Background(),
+		newsession(),
+		&model.Measurement{Input: "Not a valid URL \x7f"},
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, ErrInvalidURL) {
+		t.Fatal("expected invalid input error")
+	}
+}
+
+func TestDNSCheckFailsWithUnsupportedProtocol(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	err := measurer.Run(
+		context.Background(),
+		newsession(),
+		&model.Measurement{Input: "file://1.1.1.1"},
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, ErrUnsupportedURLScheme) {
+		t.Fatal("expected unsupported scheme error")
+	}
+}
+
+func TestWithCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel the context
+	measurer := NewExperimentMeasurer(Config{
+		DefaultAddrs: "1.1.1.1 1.0.0.1",
+	})
+	measurement := &model.Measurement{Input: "dot://one.one.one.one"}
+	err := measurer.Run(
+		ctx,
+		newsession(),
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestMakeResolverURL(t *testing.T) {
+	// test address substitution
+	addr := "255.255.255.0"
+	resolver := makeResolverURL(&url.URL{Host: "example.com"}, addr)
+	resolverURL, err := url.Parse(resolver)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resolverURL.Host != addr {
+		t.Fatal("expected address to be set as host")
+	}
+
+	// test IPv6 URLs are quoted
+	addr = "2001:db8:85a3:8d3:1319:8a2e:370"
+	resolver = makeResolverURL(&url.URL{Host: "example.com"}, addr)
+	resolverURL, err = url.Parse(resolver)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resolverURL.Host != "["+addr+"]" {
+		t.Fatal("expected URL host to be quoted")
+	}
+}
+
+func TestDNSCheckValid(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{
+		DefaultAddrs: "1.1.1.1 1.0.0.1",
+	})
+	measurement := model.Measurement{Input: "dot://one.one.one.one:853"}
+	err := measurer.Run(
+		context.Background(),
+		newsession(),
+		&measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err.Error())
+	}
+	tk := measurement.TestKeys.(*TestKeys)
+	if tk.Domain != defaultDomain {
+		t.Fatal("unexpected default value for domain")
+	}
+	if tk.Bootstrap == nil {
+		t.Fatal("unexpected value for bootstrap")
+	}
+	if tk.BootstrapFailure != nil {
+		t.Fatal("unexpected value for bootstrap_failure")
+	}
+	if len(tk.Lookups) <= 0 {
+		t.Fatal("unexpected value for lookups")
+	}
+}
+
+func newsession() model.ExperimentSession {
+	return &mockable.Session{MockableLogger: log.Log}
+}
+
+func TestSummaryKeysGeneric(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &TestKeys{}}
+	m := &Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(SummaryKeys)
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}
+
+func TestDNSCheckWait(t *testing.T) {
+	endpoints := &Endpoints{
+		WaitTime: 1 * time.Second,
+	}
+	measurer := &Measurer{Endpoints: endpoints}
+	run := func(input string) {
+		measurement := model.Measurement{Input: model.MeasurementTarget(input)}
+		err := measurer.Run(
+			context.Background(),
+			newsession(),
+			&measurement,
+			model.NewPrinterCallbacks(log.Log),
+		)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err.Error())
+		}
+		tk := measurement.TestKeys.(*TestKeys)
+		if tk.Domain != defaultDomain {
+			t.Fatal("unexpected default value for domain")
+		}
+		if tk.Bootstrap == nil {
+			t.Fatalf("unexpected value for bootstrap: %+v", tk.Bootstrap)
+		}
+		if tk.BootstrapFailure != nil {
+			t.Fatal("unexpected value for bootstrap_failure")
+		}
+		if len(tk.Lookups) <= 0 {
+			t.Fatal("unexpected value for lookups")
+		}
+	}
+	run("dot://one.one.one.one")
+	run("dot://1dot1dot1dot1.cloudflare-dns.com")
+	if atomic.LoadUint32(&endpoints.count) < 1 {
+		t.Fatal("did not sleep")
+	}
+}

internal/engine/experiment/example/example.go

@@ -0,0 +1,95 @@
+// Package example contains a simple example experiment.
+//
+// You could use this code to boostrap the implementation of
+// a new experiment that you are working on.
+package example
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+const testVersion = "0.1.0"
+
+// Config contains the experiment config.
+//
+// This contains all the settings that user can set to modify the behaviour
+// of this experiment. By tagging these variables with `ooni:"..."`, we allow
+// miniooni's -O flag to find them and set them.
+type Config struct {
+	Message     string `ooni:"Message to emit at test completion"`
+	ReturnError bool   `ooni:"Toogle to return a mocked error"`
+	SleepTime   int64  `ooni:"Amount of time to sleep for"`
+}
+
+// TestKeys contains the experiment's result.
+//
+// This is what will end up into the Measurement.TestKeys field
+// when you run this experiment.
+//
+// In other words, the variables in this struct will be
+// the specific results of this experiment.
+type TestKeys struct {
+	Success bool `json:"success"`
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config   Config
+	testName string
+}
+
+// ExperimentName implements model.ExperimentMeasurer.ExperimentName.
+func (m Measurer) ExperimentName() string {
+	return m.testName
+}
+
+// ExperimentVersion implements model.ExperimentMeasurer.ExperimentVersion.
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// ErrFailure is the error returned when you set the
+// config.ReturnError field to true.
+var ErrFailure = errors.New("mocked error")
+
+// Run implements model.ExperimentMeasurer.Run.
+func (m Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	var err error
+	if m.config.ReturnError {
+		err = ErrFailure
+	}
+	testkeys := &TestKeys{Success: err == nil}
+	measurement.TestKeys = testkeys
+	sess.Logger().Warnf("%s", "Follow the white rabbit.")
+	ctx, cancel := context.WithTimeout(ctx, time.Duration(m.config.SleepTime))
+	defer cancel()
+	<-ctx.Done()
+	sess.Logger().Infof("%s", "Knock, knock, Neo.")
+	callbacks.OnProgress(1.0, m.config.Message)
+	return err
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config, testName string) model.ExperimentMeasurer {
+	return Measurer{config: config, testName: testName}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/example/example_test.go

@@ -0,0 +1,67 @@
+package example_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/example"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestSuccess(t *testing.T) {
+	m := example.NewExperimentMeasurer(example.Config{
+		SleepTime: int64(2 * time.Millisecond),
+	}, "example")
+	if m.ExperimentName() != "example" {
+		t.Fatal("invalid ExperimentName")
+	}
+	if m.ExperimentVersion() != "0.1.0" {
+		t.Fatal("invalid ExperimentVersion")
+	}
+	ctx := context.Background()
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(sess.Logger())
+	measurement := new(model.Measurement)
+	err := m.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(example.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestFailure(t *testing.T) {
+	m := example.NewExperimentMeasurer(example.Config{
+		SleepTime:   int64(2 * time.Millisecond),
+		ReturnError: true,
+	}, "example")
+	ctx := context.Background()
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(sess.Logger())
+	err := m.Run(ctx, sess, new(model.Measurement), callbacks)
+	if !errors.Is(err, example.ErrFailure) {
+		t.Fatal("expected an error here")
+	}
+}
+
+func TestSummaryKeysGeneric(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &example.TestKeys{}}
+	m := &example.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(example.SummaryKeys)
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/fbmessenger/fbmessenger.go

@@ -0,0 +1,227 @@
+// Package fbmessenger contains the Facebook Messenger network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-019-facebook-messenger.md
+package fbmessenger
+
+import (
+	"context"
+	"errors"
+	"math/rand"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+const (
+	// FacebookASN is Facebook's ASN
+	FacebookASN = 32934
+
+	// ServiceSTUN is the STUN service
+	ServiceSTUN = "dnslookup://stun.fbsbx.com"
+
+	// ServiceBAPI is the b-api service
+	ServiceBAPI = "tcpconnect://b-api.facebook.com:443"
+
+	// ServiceBGraph is the b-graph service
+	ServiceBGraph = "tcpconnect://b-graph.facebook.com:443"
+
+	// ServiceEdge is the edge service
+	ServiceEdge = "tcpconnect://edge-mqtt.facebook.com:443"
+
+	// ServiceExternalCDN is the external CDN service
+	ServiceExternalCDN = "tcpconnect://external.xx.fbcdn.net:443"
+
+	// ServiceScontentCDN is the scontent CDN service
+	ServiceScontentCDN = "tcpconnect://scontent.xx.fbcdn.net:443"
+
+	// ServiceStar is the star service
+	ServiceStar = "tcpconnect://star.c10r.facebook.com:443"
+
+	testName    = "facebook_messenger"
+	testVersion = "0.2.0"
+)
+
+// Config contains the experiment config.
+type Config struct{}
+
+// TestKeys contains the experiment results
+type TestKeys struct {
+	urlgetter.TestKeys
+	FacebookBAPIDNSConsistent        *bool `json:"facebook_b_api_dns_consistent"`
+	FacebookBAPIReachable            *bool `json:"facebook_b_api_reachable"`
+	FacebookBGraphDNSConsistent      *bool `json:"facebook_b_graph_dns_consistent"`
+	FacebookBGraphReachable          *bool `json:"facebook_b_graph_reachable"`
+	FacebookEdgeDNSConsistent        *bool `json:"facebook_edge_dns_consistent"`
+	FacebookEdgeReachable            *bool `json:"facebook_edge_reachable"`
+	FacebookExternalCDNDNSConsistent *bool `json:"facebook_external_cdn_dns_consistent"`
+	FacebookExternalCDNReachable     *bool `json:"facebook_external_cdn_reachable"`
+	FacebookScontentCDNDNSConsistent *bool `json:"facebook_scontent_cdn_dns_consistent"`
+	FacebookScontentCDNReachable     *bool `json:"facebook_scontent_cdn_reachable"`
+	FacebookStarDNSConsistent        *bool `json:"facebook_star_dns_consistent"`
+	FacebookStarReachable            *bool `json:"facebook_star_reachable"`
+	FacebookSTUNDNSConsistent        *bool `json:"facebook_stun_dns_consistent"`
+	FacebookSTUNReachable            *bool `json:"facebook_stun_reachable"`
+	FacebookDNSBlocking              *bool `json:"facebook_dns_blocking"`
+	FacebookTCPBlocking              *bool `json:"facebook_tcp_blocking"`
+}
+
+// Update updates the TestKeys using the given MultiOutput result.
+func (tk *TestKeys) Update(v urlgetter.MultiOutput) {
+	// Update the easy to update entries first
+	tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
+	tk.Queries = append(tk.Queries, v.TestKeys.Queries...)
+	tk.Requests = append(tk.Requests, v.TestKeys.Requests...)
+	tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
+	tk.TLSHandshakes = append(tk.TLSHandshakes, v.TestKeys.TLSHandshakes...)
+	// Set the status of endpoints
+	switch v.Input.Target {
+	case ServiceSTUN:
+		var ignored *bool
+		tk.ComputeEndpointStatus(v, &tk.FacebookSTUNDNSConsistent, &ignored)
+	case ServiceBAPI:
+		tk.ComputeEndpointStatus(
+			v, &tk.FacebookBAPIDNSConsistent, &tk.FacebookBAPIReachable)
+	case ServiceBGraph:
+		tk.ComputeEndpointStatus(
+			v, &tk.FacebookBGraphDNSConsistent, &tk.FacebookBGraphReachable)
+	case ServiceEdge:
+		tk.ComputeEndpointStatus(
+			v, &tk.FacebookEdgeDNSConsistent, &tk.FacebookEdgeReachable)
+	case ServiceExternalCDN:
+		tk.ComputeEndpointStatus(
+			v, &tk.FacebookExternalCDNDNSConsistent, &tk.FacebookExternalCDNReachable)
+	case ServiceScontentCDN:
+		tk.ComputeEndpointStatus(
+			v, &tk.FacebookScontentCDNDNSConsistent, &tk.FacebookScontentCDNReachable)
+	case ServiceStar:
+		tk.ComputeEndpointStatus(
+			v, &tk.FacebookStarDNSConsistent, &tk.FacebookStarReachable)
+	}
+}
+
+var (
+	trueValue  = true
+	falseValue = false
+)
+
+// ComputeEndpointStatus computes the DNS and TCP status of a specific endpoint.
+func (tk *TestKeys) ComputeEndpointStatus(v urlgetter.MultiOutput, dns, tcp **bool) {
+	// start where all is unknown
+	*dns, *tcp = nil, nil
+	// process DNS first
+	if v.TestKeys.FailedOperation != nil && *v.TestKeys.FailedOperation == errorx.ResolveOperation {
+		tk.FacebookDNSBlocking = &trueValue
+		*dns = &falseValue
+		return // we know that the DNS has failed
+	}
+	for _, query := range v.TestKeys.Queries {
+		for _, ans := range query.Answers {
+			if ans.ASN != FacebookASN {
+				tk.FacebookDNSBlocking = &trueValue
+				*dns = &falseValue
+				return // because DNS is lying
+			}
+		}
+	}
+	*dns = &trueValue
+	// now process connect
+	if v.TestKeys.FailedOperation != nil && *v.TestKeys.FailedOperation == errorx.ConnectOperation {
+		tk.FacebookTCPBlocking = &trueValue
+		*tcp = &falseValue
+		return // because connect failed
+	}
+	// all good
+	*tcp = &trueValue
+}
+
+// Measurer performs the measurement
+type Measurer struct {
+	// Config contains the experiment settings. If empty we
+	// will be using default settings.
+	Config Config
+
+	// Getter is an optional getter to be used for testing.
+	Getter urlgetter.MultiGetter
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run
+func (m Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+	urlgetter.RegisterExtensions(measurement)
+	// generate targets
+	services := []string{
+		ServiceSTUN, ServiceBAPI, ServiceBGraph, ServiceEdge, ServiceExternalCDN,
+		ServiceScontentCDN, ServiceStar,
+	}
+	var inputs []urlgetter.MultiInput
+	for _, service := range services {
+		inputs = append(inputs, urlgetter.MultiInput{Target: service})
+	}
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+	rnd.Shuffle(len(inputs), func(i, j int) {
+		inputs[i], inputs[j] = inputs[j], inputs[i]
+	})
+	// measure in parallel
+	multi := urlgetter.Multi{Begin: time.Now(), Getter: m.Getter, Session: sess}
+	testkeys := new(TestKeys)
+	testkeys.Agent = "redirect"
+	measurement.TestKeys = testkeys
+	for entry := range multi.Collect(ctx, inputs, "facebook_messenger", callbacks) {
+		testkeys.Update(entry)
+	}
+	// if we haven't yet determined the status of DNS blocking and TCP blocking
+	// then no blocking has been detected and we can set them
+	if testkeys.FacebookDNSBlocking == nil {
+		testkeys.FacebookDNSBlocking = &falseValue
+	}
+	if testkeys.FacebookTCPBlocking == nil {
+		testkeys.FacebookTCPBlocking = &falseValue
+	}
+	return nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	DNSBlocking bool `json:"facebook_dns_blocking"`
+	TCPBlocking bool `json:"facebook_tcp_blocking"`
+	IsAnomaly   bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	dnsBlocking := tk.FacebookDNSBlocking != nil && *tk.FacebookDNSBlocking
+	tcpBlocking := tk.FacebookTCPBlocking != nil && *tk.FacebookTCPBlocking
+	sk.DNSBlocking = dnsBlocking
+	sk.TCPBlocking = tcpBlocking
+	sk.IsAnomaly = dnsBlocking || tcpBlocking
+	return sk, nil
+}

internal/engine/experiment/fbmessenger/fbmessenger_test.go

@@ -0,0 +1,363 @@
+package fbmessenger_test
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/apex/log"
+	engine "github.com/ooni/probe-cli/v3/internal/engine"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := fbmessenger.NewExperimentMeasurer(fbmessenger.Config{})
+	if measurer.ExperimentName() != "facebook_messenger" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.2.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := fbmessenger.NewExperimentMeasurer(fbmessenger.Config{})
+	ctx := context.Background()
+	// we need a real session because we need the ASN database
+	sess := newsession(t)
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*fbmessenger.TestKeys)
+	if *tk.FacebookBAPIDNSConsistent != true {
+		t.Fatal("invalid FacebookBAPIDNSConsistent")
+	}
+	if *tk.FacebookBAPIReachable != true {
+		t.Fatal("invalid FacebookBAPIReachable")
+	}
+	if *tk.FacebookBGraphDNSConsistent != true {
+		t.Fatal("invalid FacebookBGraphDNSConsistent")
+	}
+	if *tk.FacebookBGraphReachable != true {
+		t.Fatal("invalid FacebookBGraphReachable")
+	}
+	if *tk.FacebookEdgeDNSConsistent != true {
+		t.Fatal("invalid FacebookEdgeDNSConsistent")
+	}
+	if *tk.FacebookEdgeReachable != true {
+		t.Fatal("invalid FacebookEdgeReachable")
+	}
+	if *tk.FacebookExternalCDNDNSConsistent != true {
+		t.Fatal("invalid FacebookExternalCDNDNSConsistent")
+	}
+	if *tk.FacebookExternalCDNReachable != true {
+		t.Fatal("invalid FacebookExternalCDNReachable")
+	}
+	if *tk.FacebookScontentCDNDNSConsistent != true {
+		t.Fatal("invalid FacebookScontentCDNDNSConsistent")
+	}
+	if *tk.FacebookScontentCDNReachable != true {
+		t.Fatal("invalid FacebookScontentCDNReachable")
+	}
+	if *tk.FacebookStarDNSConsistent != true {
+		t.Fatal("invalid FacebookStarDNSConsistent")
+	}
+	if *tk.FacebookStarReachable != true {
+		t.Fatal("invalid FacebookStarReachable")
+	}
+	if *tk.FacebookSTUNDNSConsistent != true {
+		t.Fatal("invalid FacebookSTUNDNSConsistent")
+	}
+	if tk.FacebookSTUNReachable != nil {
+		t.Fatal("invalid FacebookSTUNReachable")
+	}
+	if *tk.FacebookDNSBlocking != false {
+		t.Fatal("invalid FacebookDNSBlocking")
+	}
+	if *tk.FacebookTCPBlocking != false {
+		t.Fatal("invalid FacebookTCPBlocking")
+	}
+}
+
+func TestWithCancelledContext(t *testing.T) {
+	measurer := fbmessenger.NewExperimentMeasurer(fbmessenger.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // so we fail immediately
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*fbmessenger.TestKeys)
+	if *tk.FacebookBAPIDNSConsistent != false {
+		t.Fatal("invalid FacebookBAPIDNSConsistent")
+	}
+	if tk.FacebookBAPIReachable != nil {
+		t.Fatal("invalid FacebookBAPIReachable")
+	}
+	if *tk.FacebookBGraphDNSConsistent != false {
+		t.Fatal("invalid FacebookBGraphDNSConsistent")
+	}
+	if tk.FacebookBGraphReachable != nil {
+		t.Fatal("invalid FacebookBGraphReachable")
+	}
+	if *tk.FacebookEdgeDNSConsistent != false {
+		t.Fatal("invalid FacebookEdgeDNSConsistent")
+	}
+	if tk.FacebookEdgeReachable != nil {
+		t.Fatal("invalid FacebookEdgeReachable")
+	}
+	if *tk.FacebookExternalCDNDNSConsistent != false {
+		t.Fatal("invalid FacebookExternalCDNDNSConsistent")
+	}
+	if tk.FacebookExternalCDNReachable != nil {
+		t.Fatal("invalid FacebookExternalCDNReachable")
+	}
+	if *tk.FacebookScontentCDNDNSConsistent != false {
+		t.Fatal("invalid FacebookScontentCDNDNSConsistent")
+	}
+	if tk.FacebookScontentCDNReachable != nil {
+		t.Fatal("invalid FacebookScontentCDNReachable")
+	}
+	if *tk.FacebookStarDNSConsistent != false {
+		t.Fatal("invalid FacebookStarDNSConsistent")
+	}
+	if tk.FacebookStarReachable != nil {
+		t.Fatal("invalid FacebookStarReachable")
+	}
+	if *tk.FacebookSTUNDNSConsistent != false {
+		t.Fatal("invalid FacebookSTUNDNSConsistent")
+	}
+	if tk.FacebookSTUNReachable != nil {
+		t.Fatal("invalid FacebookSTUNReachable")
+	}
+	if *tk.FacebookDNSBlocking != true {
+		t.Fatal("invalid FacebookDNSBlocking")
+	}
+	// no TCP blocking because we didn't ever reach TCP connect
+	if *tk.FacebookTCPBlocking != false {
+		t.Fatal("invalid FacebookTCPBlocking")
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(fbmessenger.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestComputeEndpointStatsTCPBlocking(t *testing.T) {
+	failure := io.EOF.Error()
+	operation := errorx.ConnectOperation
+	tk := fbmessenger.TestKeys{}
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{Target: fbmessenger.ServiceEdge},
+		TestKeys: urlgetter.TestKeys{
+			Failure:         &failure,
+			FailedOperation: &operation,
+			Queries: []archival.DNSQueryEntry{{
+				Answers: []archival.DNSAnswerEntry{{
+					ASN: fbmessenger.FacebookASN,
+				}},
+			}},
+		},
+	})
+	if *tk.FacebookEdgeDNSConsistent != true {
+		t.Fatal("invalid FacebookEdgeDNSConsistent")
+	}
+	if *tk.FacebookEdgeReachable != false {
+		t.Fatal("invalid FacebookEdgeReachable")
+	}
+	if tk.FacebookDNSBlocking != nil { // meaning: not determined yet
+		t.Fatal("invalid FacebookDNSBlocking")
+	}
+	if *tk.FacebookTCPBlocking != true {
+		t.Fatal("invalid FacebookTCPBlocking")
+	}
+}
+
+func TestComputeEndpointStatsDNSIsLying(t *testing.T) {
+	failure := io.EOF.Error()
+	operation := errorx.ConnectOperation
+	tk := fbmessenger.TestKeys{}
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{Target: fbmessenger.ServiceEdge},
+		TestKeys: urlgetter.TestKeys{
+			Failure:         &failure,
+			FailedOperation: &operation,
+			Queries: []archival.DNSQueryEntry{{
+				Answers: []archival.DNSAnswerEntry{{
+					ASN: 0,
+				}},
+			}},
+		},
+	})
+	if *tk.FacebookEdgeDNSConsistent != false {
+		t.Fatal("invalid FacebookEdgeDNSConsistent")
+	}
+	if tk.FacebookEdgeReachable != nil {
+		t.Fatal("invalid FacebookEdgeReachable")
+	}
+	if *tk.FacebookDNSBlocking != true {
+		t.Fatal("invalid FacebookDNSBlocking")
+	}
+	if tk.FacebookTCPBlocking != nil { // meaning: not determined yet
+		t.Fatal("invalid FacebookTCPBlocking")
+	}
+}
+
+func newsession(t *testing.T) model.ExperimentSession {
+	sess, err := engine.NewSession(engine.SessionConfig{
+		AssetsDir: "../../testdata",
+		AvailableProbeServices: []model.Service{{
+			Address: "https://ams-pg-test.ooni.org",
+			Type:    "https",
+		}},
+		Logger:          log.Log,
+		SoftwareName:    "ooniprobe-engine",
+		SoftwareVersion: "0.0.1",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := sess.MaybeLookupLocation(); err != nil {
+		t.Fatal(err)
+	}
+	return sess
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &fbmessenger.Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysWithNils(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{}}
+	m := &fbmessenger.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(fbmessenger.SummaryKeys)
+	if sk.DNSBlocking {
+		t.Fatal("invalid dnsBlocking")
+	}
+	if sk.TCPBlocking {
+		t.Fatal("invalid tcpBlocking")
+	}
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}
+
+func TestSummaryKeysWithFalseFalse(t *testing.T) {
+	falsy := false
+	measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{
+		FacebookTCPBlocking: &falsy,
+		FacebookDNSBlocking: &falsy,
+	}}
+	m := &fbmessenger.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(fbmessenger.SummaryKeys)
+	if sk.DNSBlocking {
+		t.Fatal("invalid dnsBlocking")
+	}
+	if sk.TCPBlocking {
+		t.Fatal("invalid tcpBlocking")
+	}
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}
+
+func TestSummaryKeysWithFalseTrue(t *testing.T) {
+	falsy := false
+	truy := true
+	measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{
+		FacebookTCPBlocking: &falsy,
+		FacebookDNSBlocking: &truy,
+	}}
+	m := &fbmessenger.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(fbmessenger.SummaryKeys)
+	if sk.DNSBlocking == false {
+		t.Fatal("invalid dnsBlocking")
+	}
+	if sk.TCPBlocking {
+		t.Fatal("invalid tcpBlocking")
+	}
+	if sk.IsAnomaly == false {
+		t.Fatal("invalid isAnomaly")
+	}
+}
+
+func TestSummaryKeysWithTrueFalse(t *testing.T) {
+	falsy := false
+	truy := true
+	measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{
+		FacebookTCPBlocking: &truy,
+		FacebookDNSBlocking: &falsy,
+	}}
+	m := &fbmessenger.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(fbmessenger.SummaryKeys)
+	if sk.DNSBlocking {
+		t.Fatal("invalid dnsBlocking")
+	}
+	if sk.TCPBlocking == false {
+		t.Fatal("invalid tcpBlocking")
+	}
+	if sk.IsAnomaly == false {
+		t.Fatal("invalid isAnomaly")
+	}
+}
+
+func TestSummaryKeysWithTrueTrue(t *testing.T) {
+	truy := true
+	measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{
+		FacebookTCPBlocking: &truy,
+		FacebookDNSBlocking: &truy,
+	}}
+	m := &fbmessenger.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(fbmessenger.SummaryKeys)
+	if sk.DNSBlocking == false {
+		t.Fatal("invalid dnsBlocking")
+	}
+	if sk.TCPBlocking == false {
+		t.Fatal("invalid tcpBlocking")
+	}
+	if sk.IsAnomaly == false {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/hhfm/fake_test.go

@@ -0,0 +1,56 @@
+package hhfm_test
+
+import (
+	"context"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"time"
+)
+
+type FakeDialer struct {
+	Conn net.Conn
+	Err  error
+}
+
+func (d FakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	time.Sleep(10 * time.Microsecond)
+	return d.Conn, d.Err
+}
+
+type FakeTransport struct {
+	Err  error
+	Func func(*http.Request) (*http.Response, error)
+	Resp *http.Response
+}
+
+func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	time.Sleep(10 * time.Microsecond)
+	if txp.Func != nil {
+		return txp.Func(req)
+	}
+	if req.Body != nil {
+		ioutil.ReadAll(req.Body)
+		req.Body.Close()
+	}
+	if txp.Err != nil {
+		return nil, txp.Err
+	}
+	txp.Resp.Request = req // non thread safe but it doesn't matter
+	return txp.Resp, nil
+}
+
+func (txp FakeTransport) CloseIdleConnections() {}
+
+type FakeBody struct {
+	Err error
+}
+
+func (fb FakeBody) Read(p []byte) (int, error) {
+	time.Sleep(10 * time.Microsecond)
+	return 0, fb.Err
+}
+
+func (fb FakeBody) Close() error {
+	return nil
+}

internal/engine/experiment/hhfm/hhfm.go

@@ -0,0 +1,370 @@
+// Package hhfm contains the HTTP Header Field Manipulation network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-006-header-field-manipulation.md
+package hhfm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"sort"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
+)
+
+const (
+	testName    = "http_header_field_manipulation"
+	testVersion = "0.2.0"
+)
+
+// Config contains the experiment config.
+type Config struct{}
+
+// TestKeys contains the experiment test keys.
+//
+// Here we are emitting for the same set of test keys that are
+// produced by the MK implementation.
+type TestKeys struct {
+	Agent      string                  `json:"agent"`
+	Failure    *string                 `json:"failure"`
+	Requests   []archival.RequestEntry `json:"requests"`
+	SOCKSProxy *string                 `json:"socksproxy"`
+	Tampering  Tampering               `json:"tampering"`
+}
+
+// Tampering describes the detected forms of tampering.
+//
+// The meaning of these fields is described in the specification.
+type Tampering struct {
+	HeaderFieldName           bool     `json:"header_field_name"`
+	HeaderFieldNumber         bool     `json:"header_field_number"`
+	HeaderFieldValue          bool     `json:"header_field_value"`
+	HeaderNameCapitalization  bool     `json:"header_name_capitalization"`
+	HeaderNameDiff            []string `json:"header_name_diff"`
+	RequestLineCapitalization bool     `json:"request_line_capitalization"`
+	Total                     bool     `json:"total"`
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// Transport is the definition of http.RoundTripper used by this package.
+type Transport interface {
+	RoundTrip(req *http.Request) (*http.Response, error)
+	CloseIdleConnections()
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	Config    Config
+	Transport Transport // for testing
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+var (
+	// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
+	ErrNoAvailableTestHelpers = errors.New("no available helpers")
+
+	// ErrInvalidHelperType is emitted when the helper type is invalid.
+	ErrInvalidHelperType = errors.New("invalid helper type")
+)
+
+// Run implements ExperimentMeasurer.Run.
+func (m Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	urlgetter.RegisterExtensions(measurement)
+	tk := new(TestKeys)
+	tk.Agent = "agent"
+	tk.Tampering.HeaderNameDiff = []string{}
+	measurement.TestKeys = tk
+	// parse helper
+	const helperName = "http-return-json-headers"
+	helpers, ok := sess.GetTestHelpersByName(helperName)
+	if !ok || len(helpers) < 1 {
+		return ErrNoAvailableTestHelpers
+	}
+	helper := helpers[0]
+	if helper.Type != "legacy" {
+		return ErrInvalidHelperType
+	}
+	measurement.TestHelpers = map[string]interface{}{
+		"backend": helper.Address,
+	}
+	// prepare request
+	req, err := http.NewRequest("GeT", helper.Address, nil)
+	if err != nil {
+		return err
+	}
+	headers := map[string]string{
+		randx.ChangeCapitalization("Accept"):          httpheader.Accept(),
+		randx.ChangeCapitalization("Accept-Charset"):  "ISO-8859-1,utf-8;q=0.7,*;q=0.3",
+		randx.ChangeCapitalization("Accept-Encoding"): "gzip,deflate,sdch",
+		randx.ChangeCapitalization("Accept-Language"): httpheader.AcceptLanguage(),
+		randx.ChangeCapitalization("Host"):            randx.Letters(15) + ".com",
+		randx.ChangeCapitalization("User-Agent"):      httpheader.UserAgent(),
+	}
+	for key, value := range headers {
+		// Implementation note: Golang will normalize the header names. We will use
+		// a custom dialer to restore the random capitalisation.
+		req.Header.Set(key, value)
+	}
+	req.Host = req.Header.Get("Host")
+	// fill tk.Requests[0]
+	tk.Requests = NewRequestEntryList(req, headers)
+	// prepare transport
+	txp := m.Transport
+	if txp == nil {
+		ht := http.DefaultTransport.(*http.Transport).Clone() // basically: use defaults
+		ht.DisableCompression = true                          // disable sending Accept: gzip
+		ht.ForceAttemptHTTP2 = false
+		ht.DialContext = Dialer{Headers: headers}.DialContext
+		txp = ht
+	}
+	defer txp.CloseIdleConnections()
+	// round trip and read body
+	// TODO(bassosimone): this implementation will lead to false positives if the
+	// network is really bad. Yet, this seems what MK does, so I'd rather start
+	// from that and then see to improve the robustness in the future.
+	resp, data, err := Transact(txp, req.WithContext(ctx), callbacks)
+	if err != nil {
+		tk.Failure = archival.NewFailure(err)
+		tk.Requests[0].Failure = tk.Failure
+		tk.Tampering.Total = true
+		return nil // measurement did not fail, we measured tampering
+	}
+	// fill tk.Requests[0].Response
+	tk.Requests[0].Response = NewHTTPResponse(resp, data)
+	// parse response body
+	var jsonHeaders JSONHeaders
+	if err := json.Unmarshal(data, &jsonHeaders); err != nil {
+		failure := errorx.FailureJSONParseError
+		tk.Failure = &failure
+		tk.Tampering.Total = true
+		return nil // measurement did not fail, we measured tampering
+	}
+	// fill tampering
+	tk.FillTampering(req, jsonHeaders, headers)
+	return nil
+}
+
+// Transact performs the HTTP transaction which consists of performing
+// the HTTP round trip and then reading the body.
+func Transact(txp Transport, req *http.Request,
+	callbacks model.ExperimentCallbacks) (*http.Response, []byte, error) {
+	// make sure that we return a wrapped error here
+	resp, data, err := transact(txp, req, callbacks)
+	err = errorx.SafeErrWrapperBuilder{
+		Error: err, Operation: errorx.TopLevelOperation}.MaybeBuild()
+	return resp, data, err
+}
+
+func transact(txp Transport, req *http.Request,
+	callbacks model.ExperimentCallbacks) (*http.Response, []byte, error) {
+	callbacks.OnProgress(0.25, "sending request...")
+	resp, err := txp.RoundTrip(req)
+	callbacks.OnProgress(0.50, fmt.Sprintf("got reseponse headers... %+v", err))
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, nil, urlgetter.ErrHTTPRequestFailed
+	}
+	callbacks.OnProgress(0.75, "reading response body...")
+	data, err := ioutil.ReadAll(resp.Body)
+	callbacks.OnProgress(1.00, fmt.Sprintf("got reseponse body... %+v", err))
+	if err != nil {
+		return nil, nil, err
+	}
+	return resp, data, nil
+}
+
+// FillTampering fills the tampering structure in the TestKeys
+// based on the value of other fields of the TestKeys, the original
+// HTTP request, the response from the test helper, and the
+// headers with modified capitalisation.
+func (tk *TestKeys) FillTampering(
+	req *http.Request, jsonHeaders JSONHeaders, headers map[string]string) {
+	tk.Tampering.RequestLineCapitalization = (fmt.Sprintf(
+		"%s / HTTP/1.1", req.Method) != jsonHeaders.RequestLine)
+	tk.Tampering.HeaderFieldNumber = len(headers) != len(jsonHeaders.HeadersDict)
+	expectedHeaderKeys := make(map[string]string)
+	for key := range headers {
+		expectedHeaderKeys[http.CanonicalHeaderKey(key)] = key
+	}
+	receivedHeaderKeys := make(map[string]string)
+	for key := range jsonHeaders.HeadersDict {
+		receivedHeaderKeys[http.CanonicalHeaderKey(key)] = key
+	}
+	commonHeaderKeys := make(map[string]int)
+	for key := range expectedHeaderKeys {
+		commonHeaderKeys[key]++
+	}
+	for key := range receivedHeaderKeys {
+		commonHeaderKeys[key]++
+	}
+	for key, count := range commonHeaderKeys {
+		if count != 2 {
+			continue // not in common
+		}
+		expectedKey, receivedKey := expectedHeaderKeys[key], receivedHeaderKeys[key]
+		if expectedKey != receivedKey {
+			tk.Tampering.HeaderNameCapitalization = true
+			tk.Tampering.HeaderNameDiff = append(tk.Tampering.HeaderNameDiff, expectedKey)
+			tk.Tampering.HeaderNameDiff = append(tk.Tampering.HeaderNameDiff, receivedKey)
+		}
+		expectedValue := headers[expectedKey]
+		receivedValue := jsonHeaders.HeadersDict[receivedKey]
+		if len(receivedValue) != 1 || expectedValue != receivedValue[0] {
+			tk.Tampering.HeaderFieldValue = true
+		}
+	}
+}
+
+// NewRequestEntryList creates a new []archival.RequestEntry given a
+// specific *http.Request and headers with random case.
+func NewRequestEntryList(req *http.Request, headers map[string]string) (out []archival.RequestEntry) {
+	out = []archival.RequestEntry{{
+		Request: archival.HTTPRequest{
+			Headers:     make(map[string]archival.MaybeBinaryValue),
+			HeadersList: []archival.HTTPHeader{},
+			Method:      req.Method,
+			URL:         req.URL.String(),
+		},
+	}}
+	for key, value := range headers {
+		// Using the random capitalization headers here
+		mbv := archival.MaybeBinaryValue{Value: value}
+		out[0].Request.Headers[key] = mbv
+		out[0].Request.HeadersList = append(out[0].Request.HeadersList,
+			archival.HTTPHeader{Key: key, Value: mbv})
+	}
+	sort.Slice(out[0].Request.HeadersList, func(i, j int) bool {
+		return out[0].Request.HeadersList[i].Key < out[0].Request.HeadersList[j].Key
+	})
+	return
+}
+
+// NewHTTPResponse creates a new archival.HTTPResponse given a
+// specific *http.Response instance and its body.
+func NewHTTPResponse(resp *http.Response, data []byte) (out archival.HTTPResponse) {
+	out = archival.HTTPResponse{
+		Body:        archival.HTTPBody{Value: string(data)},
+		Code:        int64(resp.StatusCode),
+		Headers:     make(map[string]archival.MaybeBinaryValue),
+		HeadersList: []archival.HTTPHeader{},
+	}
+	for key := range resp.Header {
+		mbv := archival.MaybeBinaryValue{Value: resp.Header.Get(key)}
+		out.Headers[key] = mbv
+		out.HeadersList = append(out.HeadersList, archival.HTTPHeader{Key: key, Value: mbv})
+	}
+	sort.Slice(out.HeadersList, func(i, j int) bool {
+		return out.HeadersList[i].Key < out.HeadersList[j].Key
+	})
+	return
+}
+
+// JSONHeaders contains the response from the backend server.
+//
+// Here we're defining only the fields we care about.
+type JSONHeaders struct {
+	HeadersDict map[string][]string `json:"headers_dict"`
+	RequestLine string              `json:"request_line"`
+}
+
+// Dialer is a dialer that performs headers transformations.
+//
+// Because Golang will canonicalize header names, we need to reintroduce
+// the random capitalization when emitting the request.
+//
+// This implementation rests on the assumption that we shall use the
+// same connection just once, which is guarantee by the implementation
+// of HHFM above. If using this code elsewhere, make sure that you
+// guarantee that the connection is used for a single request and that
+// such a request does not contain any body.
+type Dialer struct {
+	Dialer  netx.Dialer // used for testing
+	Headers map[string]string
+}
+
+// DialContext dials a specific connection and arranges such that
+// headers in the outgoing request are transformed.
+func (d Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	dialer := d.Dialer
+	if dialer == nil {
+		dialer = selfcensor.DefaultDialer
+	}
+	conn, err := dialer.DialContext(ctx, network, address)
+	if err != nil {
+		return nil, err
+	}
+	return Conn{Conn: conn, Headers: d.Headers}, nil
+}
+
+// Conn is a connection where headers in the outgoing request
+// are transformed according to a transform table.
+type Conn struct {
+	net.Conn
+	Headers map[string]string
+}
+
+// Write implements Conn.Write.
+func (c Conn) Write(b []byte) (int, error) {
+	for key := range c.Headers {
+		b = bytes.Replace(b, []byte(http.CanonicalHeaderKey(key)+":"), []byte(key+":"), 1)
+	}
+	return c.Conn.Write(b)
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	sk.IsAnomaly = (tk.Tampering.HeaderFieldName ||
+		tk.Tampering.HeaderFieldNumber ||
+		tk.Tampering.HeaderFieldValue ||
+		tk.Tampering.HeaderNameCapitalization ||
+		tk.Tampering.RequestLineCapitalization ||
+		tk.Tampering.Total)
+	return sk, nil
+}

internal/engine/experiment/hhfm/hhfm_test.go

@@ -0,0 +1,906 @@
+package hhfm_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+	engine "github.com/ooni/probe-cli/v3/internal/engine"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := hhfm.NewExperimentMeasurer(hhfm.Config{})
+	if measurer.ExperimentName() != "http_header_field_manipulation" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.2.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestSuccess(t *testing.T) {
+	measurer := hhfm.NewExperimentMeasurer(hhfm.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+		MockableTestHelpers: map[string][]model.Service{
+			"http-return-json-headers": {{
+				Address: "http://37.218.241.94:80",
+				Type:    "legacy",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*hhfm.TestKeys)
+	if tk.Agent != "agent" {
+		t.Fatal("invalid Agent")
+	}
+	if tk.Failure != nil {
+		t.Fatal("invalid Failure")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("invalid Requests")
+	}
+	request := tk.Requests[0]
+	if request.Failure != nil {
+		t.Fatal("invalid Requests[0].Failure")
+	}
+	if request.Request.Body.Value != "" {
+		t.Fatal("invalid Requests[0].Request.Body.Value")
+	}
+	if request.Request.BodyIsTruncated != false {
+		t.Fatal("invalid Requests[0].Request.BodyIsTruncated")
+	}
+	if len(request.Request.HeadersList) != 6 {
+		t.Fatal("invalid Requests[0].Request.HeadersList length")
+	}
+	if len(request.Request.Headers) != 6 {
+		t.Fatal("invalid Requests[0].Request.Headers length")
+	}
+	if strings.ToUpper(request.Request.Method) != "GET" {
+		t.Fatal("invalid Requests[0].Request.Method")
+	}
+	if request.Request.Tor.ExitIP != nil {
+		t.Fatal("invalid Requests[0].Request.Tor.ExitIP")
+	}
+	if request.Request.Tor.ExitName != nil {
+		t.Fatal("invalid Requests[0].Request.Tor.ExitName")
+	}
+	if request.Request.Tor.IsTor != false {
+		t.Fatal("invalid Requests[0].Request.Tor.IsTor")
+	}
+	ths, ok := sess.GetTestHelpersByName("http-return-json-headers")
+	if !ok || len(ths) < 1 || ths[0].Type != "legacy" {
+		t.Fatal("cannot get the test helper")
+	}
+	if request.Request.URL != ths[0].Address {
+		t.Fatal("invalid Requests[0].Request.URL")
+	}
+	if len(request.Response.Body.Value) < 1 {
+		t.Fatal("invalid Requests[0].Response.Body.Value length")
+	}
+	if request.Response.BodyIsTruncated != false {
+		t.Fatal("invalid Requests[0].Response.BodyIsTruncated")
+	}
+	if request.Response.Code != 200 {
+		t.Fatal("invalid Requests[0].Code")
+	}
+	if len(request.Response.HeadersList) != 0 {
+		t.Fatal("invalid Requests[0].HeadersList length")
+	}
+	if len(request.Response.Headers) != 0 {
+		t.Fatal("invalid Requests[0].Headers length")
+	}
+	if request.T != 0 {
+		t.Fatal("invalid Requests[0].T")
+	}
+	if request.TransactionID != 0 {
+		t.Fatal("invalid Requests[0].TransactionID")
+	}
+	if tk.SOCKSProxy != nil {
+		t.Fatal("invalid SOCKSProxy")
+	}
+	if tk.Tampering.HeaderFieldName != false {
+		t.Fatal("invalid Tampering.HeaderFieldName")
+	}
+	if tk.Tampering.HeaderFieldNumber != false {
+		t.Fatal("invalid Tampering.HeaderFieldNumber")
+	}
+	if tk.Tampering.HeaderFieldValue != false {
+		t.Fatal("invalid Tampering.HeaderFieldValue")
+	}
+	if tk.Tampering.HeaderNameCapitalization != false {
+		t.Fatal("invalid Tampering.HeaderNameCapitalization")
+	}
+	if len(tk.Tampering.HeaderNameDiff) != 0 {
+		t.Fatal("invalid Tampering.HeaderNameDiff")
+	}
+	if tk.Tampering.RequestLineCapitalization != false {
+		t.Fatal("invalid Tampering.RequestLineCapitalization")
+	}
+	if tk.Tampering.Total != false {
+		t.Fatal("invalid Tampering.Total")
+	}
+}
+
+func TestCancelledContext(t *testing.T) {
+	measurer := hhfm.NewExperimentMeasurer(hhfm.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+		MockableTestHelpers: map[string][]model.Service{
+			"http-return-json-headers": {{
+				Address: "http://37.218.241.94:80",
+				Type:    "legacy",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*hhfm.TestKeys)
+	if tk.Agent != "agent" {
+		t.Fatal("invalid Agent")
+	}
+	if *tk.Failure != errorx.FailureInterrupted {
+		t.Fatal("invalid Failure")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("invalid Requests")
+	}
+	request := tk.Requests[0]
+	if *request.Failure != errorx.FailureInterrupted {
+		t.Fatal("invalid Requests[0].Failure")
+	}
+	if request.Request.Body.Value != "" {
+		t.Fatal("invalid Requests[0].Request.Body.Value")
+	}
+	if request.Request.BodyIsTruncated != false {
+		t.Fatal("invalid Requests[0].Request.BodyIsTruncated")
+	}
+	if len(request.Request.HeadersList) != 6 {
+		t.Fatal("invalid Requests[0].Request.HeadersList length")
+	}
+	if len(request.Request.Headers) != 6 {
+		t.Fatal("invalid Requests[0].Request.Headers length")
+	}
+	if strings.ToUpper(request.Request.Method) != "GET" {
+		t.Fatal("invalid Requests[0].Request.Method")
+	}
+	if request.Request.Tor.ExitIP != nil {
+		t.Fatal("invalid Requests[0].Request.Tor.ExitIP")
+	}
+	if request.Request.Tor.ExitName != nil {
+		t.Fatal("invalid Requests[0].Request.Tor.ExitName")
+	}
+	if request.Request.Tor.IsTor != false {
+		t.Fatal("invalid Requests[0].Request.Tor.IsTor")
+	}
+	ths, ok := sess.GetTestHelpersByName("http-return-json-headers")
+	if !ok || len(ths) < 1 || ths[0].Type != "legacy" {
+		t.Fatal("cannot get the test helper")
+	}
+	if request.Request.URL != ths[0].Address {
+		t.Fatal("invalid Requests[0].Request.URL")
+	}
+	if len(request.Response.Body.Value) != 0 {
+		t.Fatal("invalid Requests[0].Response.Body.Value length")
+	}
+	if request.Response.BodyIsTruncated != false {
+		t.Fatal("invalid Requests[0].Response.BodyIsTruncated")
+	}
+	if request.Response.Code != 0 {
+		t.Fatal("invalid Requests[0].Code")
+	}
+	if len(request.Response.HeadersList) != 0 {
+		t.Fatal("invalid Requests[0].HeadersList length")
+	}
+	if len(request.Response.Headers) != 0 {
+		t.Fatal("invalid Requests[0].Headers length")
+	}
+	if request.T != 0 {
+		t.Fatal("invalid Requests[0].T")
+	}
+	if request.TransactionID != 0 {
+		t.Fatal("invalid Requests[0].TransactionID")
+	}
+	if tk.SOCKSProxy != nil {
+		t.Fatal("invalid SOCKSProxy")
+	}
+	if tk.Tampering.HeaderFieldName != false {
+		t.Fatal("invalid Tampering.HeaderFieldName")
+	}
+	if tk.Tampering.HeaderFieldNumber != false {
+		t.Fatal("invalid Tampering.HeaderFieldNumber")
+	}
+	if tk.Tampering.HeaderFieldValue != false {
+		t.Fatal("invalid Tampering.HeaderFieldValue")
+	}
+	if tk.Tampering.HeaderNameCapitalization != false {
+		t.Fatal("invalid Tampering.HeaderNameCapitalization")
+	}
+	if len(tk.Tampering.HeaderNameDiff) != 0 {
+		t.Fatal("invalid Tampering.HeaderNameDiff")
+	}
+	if tk.Tampering.RequestLineCapitalization != false {
+		t.Fatal("invalid Tampering.RequestLineCapitalization")
+	}
+	if tk.Tampering.Total != true {
+		t.Fatal("invalid Tampering.Total")
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(hhfm.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestNoHelpers(t *testing.T) {
+	measurer := hhfm.NewExperimentMeasurer(hhfm.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if !errors.Is(err, hhfm.ErrNoAvailableTestHelpers) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*hhfm.TestKeys)
+	if tk.Agent != "agent" {
+		t.Fatal("invalid Agent")
+	}
+	if tk.Failure != nil {
+		t.Fatal("invalid Failure")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("invalid Requests")
+	}
+	if tk.SOCKSProxy != nil {
+		t.Fatal("invalid SOCKSProxy")
+	}
+	if tk.Tampering.HeaderFieldName != false {
+		t.Fatal("invalid Tampering.HeaderFieldName")
+	}
+	if tk.Tampering.HeaderFieldNumber != false {
+		t.Fatal("invalid Tampering.HeaderFieldNumber")
+	}
+	if tk.Tampering.HeaderFieldValue != false {
+		t.Fatal("invalid Tampering.HeaderFieldValue")
+	}
+	if tk.Tampering.HeaderNameCapitalization != false {
+		t.Fatal("invalid Tampering.HeaderNameCapitalization")
+	}
+	if len(tk.Tampering.HeaderNameDiff) != 0 {
+		t.Fatal("invalid Tampering.HeaderNameDiff")
+	}
+	if tk.Tampering.RequestLineCapitalization != false {
+		t.Fatal("invalid Tampering.RequestLineCapitalization")
+	}
+	if tk.Tampering.Total != false {
+		t.Fatal("invalid Tampering.Total")
+	}
+}
+
+func TestNoActualHelpersInList(t *testing.T) {
+	measurer := hhfm.NewExperimentMeasurer(hhfm.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableTestHelpers: map[string][]model.Service{
+			"http-return-json-headers": nil,
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if !errors.Is(err, hhfm.ErrNoAvailableTestHelpers) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*hhfm.TestKeys)
+	if tk.Agent != "agent" {
+		t.Fatal("invalid Agent")
+	}
+	if tk.Failure != nil {
+		t.Fatal("invalid Failure")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("invalid Requests")
+	}
+	if tk.SOCKSProxy != nil {
+		t.Fatal("invalid SOCKSProxy")
+	}
+	if tk.Tampering.HeaderFieldName != false {
+		t.Fatal("invalid Tampering.HeaderFieldName")
+	}
+	if tk.Tampering.HeaderFieldNumber != false {
+		t.Fatal("invalid Tampering.HeaderFieldNumber")
+	}
+	if tk.Tampering.HeaderFieldValue != false {
+		t.Fatal("invalid Tampering.HeaderFieldValue")
+	}
+	if tk.Tampering.HeaderNameCapitalization != false {
+		t.Fatal("invalid Tampering.HeaderNameCapitalization")
+	}
+	if len(tk.Tampering.HeaderNameDiff) != 0 {
+		t.Fatal("invalid Tampering.HeaderNameDiff")
+	}
+	if tk.Tampering.RequestLineCapitalization != false {
+		t.Fatal("invalid Tampering.RequestLineCapitalization")
+	}
+	if tk.Tampering.Total != false {
+		t.Fatal("invalid Tampering.Total")
+	}
+}
+
+func TestWrongTestHelperType(t *testing.T) {
+	measurer := hhfm.NewExperimentMeasurer(hhfm.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableTestHelpers: map[string][]model.Service{
+			"http-return-json-headers": {{
+				Address: "http://127.0.0.1",
+				Type:    "antani",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if !errors.Is(err, hhfm.ErrInvalidHelperType) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*hhfm.TestKeys)
+	if tk.Agent != "agent" {
+		t.Fatal("invalid Agent")
+	}
+	if tk.Failure != nil {
+		t.Fatal("invalid Failure")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("invalid Requests")
+	}
+	if tk.SOCKSProxy != nil {
+		t.Fatal("invalid SOCKSProxy")
+	}
+	if tk.Tampering.HeaderFieldName != false {
+		t.Fatal("invalid Tampering.HeaderFieldName")
+	}
+	if tk.Tampering.HeaderFieldNumber != false {
+		t.Fatal("invalid Tampering.HeaderFieldNumber")
+	}
+	if tk.Tampering.HeaderFieldValue != false {
+		t.Fatal("invalid Tampering.HeaderFieldValue")
+	}
+	if tk.Tampering.HeaderNameCapitalization != false {
+		t.Fatal("invalid Tampering.HeaderNameCapitalization")
+	}
+	if len(tk.Tampering.HeaderNameDiff) != 0 {
+		t.Fatal("invalid Tampering.HeaderNameDiff")
+	}
+	if tk.Tampering.RequestLineCapitalization != false {
+		t.Fatal("invalid Tampering.RequestLineCapitalization")
+	}
+	if tk.Tampering.Total != false {
+		t.Fatal("invalid Tampering.Total")
+	}
+}
+
+func TestNewRequestFailure(t *testing.T) {
+	measurer := hhfm.NewExperimentMeasurer(hhfm.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableTestHelpers: map[string][]model.Service{
+			"http-return-json-headers": {{
+				Address: "http://127.0.0.1\t\t\t", // invalid
+				Type:    "legacy",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*hhfm.TestKeys)
+	if tk.Agent != "agent" {
+		t.Fatal("invalid Agent")
+	}
+	if tk.Failure != nil {
+		t.Fatal("invalid Failure")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("invalid Requests")
+	}
+	if tk.SOCKSProxy != nil {
+		t.Fatal("invalid SOCKSProxy")
+	}
+	if tk.Tampering.HeaderFieldName != false {
+		t.Fatal("invalid Tampering.HeaderFieldName")
+	}
+	if tk.Tampering.HeaderFieldNumber != false {
+		t.Fatal("invalid Tampering.HeaderFieldNumber")
+	}
+	if tk.Tampering.HeaderFieldValue != false {
+		t.Fatal("invalid Tampering.HeaderFieldValue")
+	}
+	if tk.Tampering.HeaderNameCapitalization != false {
+		t.Fatal("invalid Tampering.HeaderNameCapitalization")
+	}
+	if len(tk.Tampering.HeaderNameDiff) != 0 {
+		t.Fatal("invalid Tampering.HeaderNameDiff")
+	}
+	if tk.Tampering.RequestLineCapitalization != false {
+		t.Fatal("invalid Tampering.RequestLineCapitalization")
+	}
+	if tk.Tampering.Total != false {
+		t.Fatal("invalid Tampering.Total")
+	}
+}
+
+func TestInvalidJSONBody(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "Hello, client") // not valid JSON
+	}))
+	defer server.Close()
+	measurer := hhfm.NewExperimentMeasurer(hhfm.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableTestHelpers: map[string][]model.Service{
+			"http-return-json-headers": {{
+				Address: server.URL,
+				Type:    "legacy",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*hhfm.TestKeys)
+	if tk.Agent != "agent" {
+		t.Fatal("invalid Agent")
+	}
+	if *tk.Failure != errorx.FailureJSONParseError {
+		t.Fatal("invalid Failure")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("invalid Requests")
+	}
+	// we already check the content of Requests in other tests
+	if tk.SOCKSProxy != nil {
+		t.Fatal("invalid SOCKSProxy")
+	}
+	if tk.Tampering.HeaderFieldName != false {
+		t.Fatal("invalid Tampering.HeaderFieldName")
+	}
+	if tk.Tampering.HeaderFieldNumber != false {
+		t.Fatal("invalid Tampering.HeaderFieldNumber")
+	}
+	if tk.Tampering.HeaderFieldValue != false {
+		t.Fatal("invalid Tampering.HeaderFieldValue")
+	}
+	if tk.Tampering.HeaderNameCapitalization != false {
+		t.Fatal("invalid Tampering.HeaderNameCapitalization")
+	}
+	if len(tk.Tampering.HeaderNameDiff) != 0 {
+		t.Fatal("invalid Tampering.HeaderNameDiff")
+	}
+	if tk.Tampering.RequestLineCapitalization != false {
+		t.Fatal("invalid Tampering.RequestLineCapitalization")
+	}
+	if tk.Tampering.Total != true {
+		t.Fatal("invalid Tampering.Total")
+	}
+}
+
+func TestTransactStatusCodeFailure(t *testing.T) {
+	txp := FakeTransport{Resp: &http.Response{
+		Body:       ioutil.NopCloser(strings.NewReader("")),
+		StatusCode: 500,
+	}}
+	resp, body, err := hhfm.Transact(txp, &http.Request{},
+		model.NewPrinterCallbacks(log.Log))
+	if !errors.Is(err, urlgetter.ErrHTTPRequestFailed) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("resp is not nil")
+	}
+	if body != nil {
+		t.Fatal("body is not nil")
+	}
+}
+
+func TestTransactCannotReadBody(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeTransport{Resp: &http.Response{
+		Body:       &FakeBody{Err: expected},
+		StatusCode: 200,
+	}}
+	resp, body, err := hhfm.Transact(txp, &http.Request{},
+		model.NewPrinterCallbacks(log.Log))
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("resp is not nil")
+	}
+	if body != nil {
+		t.Fatal("body is not nil")
+	}
+}
+
+func newsession(t *testing.T) model.ExperimentSession {
+	sess, err := engine.NewSession(engine.SessionConfig{
+		AssetsDir: "../../testdata",
+		AvailableProbeServices: []model.Service{{
+			Address: "https://ams-pg-test.ooni.org",
+			Type:    "https",
+		}},
+		Logger:          log.Log,
+		SoftwareName:    "ooniprobe-engine",
+		SoftwareVersion: "0.0.1",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := sess.MaybeLookupBackends(); err != nil {
+		t.Fatal(err)
+	}
+	return sess
+}
+
+func TestTestKeys_FillTampering(t *testing.T) {
+	type fields struct {
+		Agent      string
+		Failure    *string
+		Requests   []archival.RequestEntry
+		SOCKSProxy *string
+		Tampering  hhfm.Tampering
+	}
+	type args struct {
+		req         *http.Request
+		jsonHeaders hhfm.JSONHeaders
+		headers     map[string]string
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+	}{{
+		name: "Request line capitalisation",
+		fields: fields{
+			Tampering: hhfm.Tampering{
+				RequestLineCapitalization: true,
+			},
+		},
+		args: args{
+			req: &http.Request{
+				Method: "GeT",
+			},
+			jsonHeaders: hhfm.JSONHeaders{
+				RequestLine: "GET / HTTP/1.1",
+			},
+		},
+	}, {
+		name: "Header field number",
+		fields: fields{
+			Tampering: hhfm.Tampering{
+				HeaderFieldNumber: true,
+			},
+		},
+		args: args{
+			req: &http.Request{
+				Method: "GeT",
+			},
+			jsonHeaders: hhfm.JSONHeaders{
+				RequestLine: "GeT / HTTP/1.1",
+			},
+			headers: map[string]string{
+				"UsEr-AgENt": "miniooni/0.1.0-dev",
+			},
+		},
+	}, {
+		name: "Header name diff",
+		fields: fields{
+			Tampering: hhfm.Tampering{
+				HeaderNameCapitalization: true,
+				HeaderNameDiff:           []string{"UsEr-AgENt", "User-Agent"},
+			},
+		},
+		args: args{
+			req: &http.Request{
+				Method: "GeT",
+			},
+			jsonHeaders: hhfm.JSONHeaders{
+				RequestLine: "GeT / HTTP/1.1",
+				HeadersDict: map[string][]string{
+					"User-Agent": {"miniooni/0.1.0-dev"},
+				},
+			},
+			headers: map[string]string{
+				"UsEr-AgENt": "miniooni/0.1.0-dev",
+			},
+		},
+	}, {
+		name: "Header value diff",
+		fields: fields{
+			Tampering: hhfm.Tampering{
+				HeaderFieldValue: true,
+			},
+		},
+		args: args{
+			req: &http.Request{
+				Method: "GeT",
+			},
+			jsonHeaders: hhfm.JSONHeaders{
+				RequestLine: "GeT / HTTP/1.1",
+				HeadersDict: map[string][]string{
+					"UsEr-AgENt": {"MINIOONI/0.1.0-dev"},
+				},
+			},
+			headers: map[string]string{
+				"UsEr-AgENt": "miniooni/0.1.0-dev",
+			},
+		},
+	}, {
+		name: "Number of headers per key diffs",
+		fields: fields{
+			Tampering: hhfm.Tampering{
+				HeaderFieldValue: true,
+			},
+		},
+		args: args{
+			req: &http.Request{
+				Method: "GeT",
+			},
+			jsonHeaders: hhfm.JSONHeaders{
+				RequestLine: "GeT / HTTP/1.1",
+				HeadersDict: map[string][]string{
+					"UsEr-AgENt": {"miniooni/0.1.0-dev", "ooniprobe-engine/0.1.0-dev"},
+				},
+			},
+			headers: map[string]string{
+				"UsEr-AgENt": "miniooni/0.1.0-dev",
+			},
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tk := &hhfm.TestKeys{
+				Agent:      tt.fields.Agent,
+				Failure:    tt.fields.Failure,
+				Requests:   tt.fields.Requests,
+				SOCKSProxy: tt.fields.SOCKSProxy,
+			}
+			tk.FillTampering(tt.args.req, tt.args.jsonHeaders, tt.args.headers)
+			if diff := cmp.Diff(tt.fields.Tampering, tk.Tampering); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestNewRequestEntryList(t *testing.T) {
+	type args struct {
+		req     *http.Request
+		headers map[string]string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut []archival.RequestEntry
+	}{{
+		name: "common case",
+		args: args{
+			req: &http.Request{
+				Method: "GeT",
+				URL: &url.URL{
+					Scheme: "http",
+					Host:   "10.0.0.1",
+					Path:   "/",
+				},
+			},
+			headers: map[string]string{
+				"ContENt-tYPE": "text/plain",
+				"User-aGENT":   "foo/1.0",
+			},
+		},
+		wantOut: []archival.RequestEntry{{
+			Request: archival.HTTPRequest{
+				HeadersList: []archival.HTTPHeader{{
+					Key:   "ContENt-tYPE",
+					Value: archival.MaybeBinaryValue{Value: "text/plain"},
+				}, {
+					Key:   "User-aGENT",
+					Value: archival.MaybeBinaryValue{Value: "foo/1.0"},
+				}},
+				Headers: map[string]archival.MaybeBinaryValue{
+					"ContENt-tYPE": {Value: "text/plain"},
+					"User-aGENT":   {Value: "foo/1.0"},
+				},
+				Method: "GeT",
+				URL:    "http://10.0.0.1/",
+			},
+		}},
+	}, {
+		name: "without headers",
+		args: args{
+			req: &http.Request{
+				Method: "GeT",
+				URL: &url.URL{
+					Scheme: "http",
+					Host:   "10.0.0.1",
+					Path:   "/",
+				},
+			},
+		},
+		wantOut: []archival.RequestEntry{{
+			Request: archival.HTTPRequest{
+				Method:      "GeT",
+				Headers:     make(map[string]archival.MaybeBinaryValue),
+				HeadersList: []archival.HTTPHeader{},
+				URL:         "http://10.0.0.1/",
+			},
+		}},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := hhfm.NewRequestEntryList(tt.args.req, tt.args.headers)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestNewHTTPResponse(t *testing.T) {
+	type args struct {
+		resp *http.Response
+		data []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut archival.HTTPResponse
+	}{{
+		name: "common case",
+		args: args{
+			resp: &http.Response{
+				StatusCode: 200,
+				Header: http.Header{
+					"Content-Type": []string{"text/plain"},
+					"User-Agent":   []string{"foo/1.0"},
+				},
+			},
+			data: []byte("deadbeef"),
+		},
+		wantOut: archival.HTTPResponse{
+			Body: archival.MaybeBinaryValue{Value: "deadbeef"},
+			Code: 200,
+			HeadersList: []archival.HTTPHeader{{
+				Key:   "Content-Type",
+				Value: archival.MaybeBinaryValue{Value: "text/plain"},
+			}, {
+				Key:   "User-Agent",
+				Value: archival.MaybeBinaryValue{Value: "foo/1.0"},
+			}},
+			Headers: map[string]archival.MaybeBinaryValue{
+				"Content-Type": {Value: "text/plain"},
+				"User-Agent":   {Value: "foo/1.0"},
+			},
+		},
+	}, {
+		name: "with no HTTP header and body",
+		args: args{
+			resp: &http.Response{StatusCode: 200},
+		},
+		wantOut: archival.HTTPResponse{
+			Body:        archival.MaybeBinaryValue{Value: ""},
+			Code:        200,
+			HeadersList: []archival.HTTPHeader{},
+			Headers:     map[string]archival.MaybeBinaryValue{},
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := hhfm.NewHTTPResponse(tt.args.resp, tt.args.data)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestDialerDialContext(t *testing.T) {
+	expected := errors.New("mocked error")
+	d := hhfm.Dialer{Dialer: FakeDialer{Err: expected}}
+	conn, err := d.DialContext(context.Background(), "tcp", "127.0.0.1:80")
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if conn != nil {
+		t.Fatal("conn is not nil")
+	}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &hhfm.Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysWorksAsIntended(t *testing.T) {
+	tests := []struct {
+		tampering hhfm.Tampering
+		isAnomaly bool
+	}{{
+		tampering: hhfm.Tampering{},
+		isAnomaly: false,
+	}, {
+		tampering: hhfm.Tampering{HeaderFieldName: true},
+		isAnomaly: true,
+	}, {
+		tampering: hhfm.Tampering{HeaderFieldNumber: true},
+		isAnomaly: true,
+	}, {
+		tampering: hhfm.Tampering{HeaderFieldValue: true},
+		isAnomaly: true,
+	}, {
+		tampering: hhfm.Tampering{HeaderNameCapitalization: true},
+		isAnomaly: true,
+	}, {
+		tampering: hhfm.Tampering{RequestLineCapitalization: true},
+		isAnomaly: true,
+	}, {
+		tampering: hhfm.Tampering{Total: true},
+		isAnomaly: true,
+	}}
+	for idx, tt := range tests {
+		t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
+			m := &hhfm.Measurer{}
+			measurement := &model.Measurement{TestKeys: &hhfm.TestKeys{
+				Tampering: tt.tampering,
+			}}
+			got, err := m.GetSummaryKeys(measurement)
+			if err != nil {
+				t.Fatal(err)
+				return
+			}
+			sk := got.(hhfm.SummaryKeys)
+			if sk.IsAnomaly != tt.isAnomaly {
+				t.Fatal("unexpected isAnomaly value")
+			}
+		})
+	}
+}

internal/engine/experiment/hirl/fake_test.go

@@ -0,0 +1,71 @@
+package hirl_test
+
+import (
+	"context"
+	"io"
+	"net"
+	"time"
+)
+
+type FakeDialer struct {
+	Conn net.Conn
+	Err  error
+}
+
+func (d FakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	time.Sleep(10 * time.Microsecond)
+	return d.Conn, d.Err
+}
+
+type FakeConn struct {
+	ReadError             error
+	ReadData              []byte
+	SetDeadlineError      error
+	SetReadDeadlineError  error
+	SetWriteDeadlineError error
+	WriteError            error
+}
+
+func (c *FakeConn) Read(b []byte) (int, error) {
+	if len(c.ReadData) > 0 {
+		n := copy(b, c.ReadData)
+		c.ReadData = c.ReadData[n:]
+		return n, nil
+	}
+	if c.ReadError != nil {
+		return 0, c.ReadError
+	}
+	return 0, io.EOF
+}
+
+func (c *FakeConn) Write(b []byte) (n int, err error) {
+	if c.WriteError != nil {
+		return 0, c.WriteError
+	}
+	n = len(b)
+	return
+}
+
+func (*FakeConn) Close() (err error) {
+	return
+}
+
+func (*FakeConn) LocalAddr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+func (*FakeConn) RemoteAddr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+func (c *FakeConn) SetDeadline(t time.Time) (err error) {
+	return c.SetDeadlineError
+}
+
+func (c *FakeConn) SetReadDeadline(t time.Time) (err error) {
+	return c.SetReadDeadlineError
+}
+
+func (c *FakeConn) SetWriteDeadline(t time.Time) (err error) {
+	return c.SetWriteDeadlineError
+}

internal/engine/experiment/hirl/hirl.go

@@ -0,0 +1,323 @@
+// Package hirl contains the HTTP Invalid Request Line network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-007-http-invalid-request-line.md
+package hirl
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+const (
+	testName    = "http_invalid_request_line"
+	testVersion = "0.2.0"
+	timeout     = 5 * time.Second
+)
+
+// Config contains the experiment config.
+type Config struct{}
+
+// TestKeys contains the experiment test keys.
+type TestKeys struct {
+	FailureList   []*string                   `json:"failure_list"`
+	Received      []archival.MaybeBinaryValue `json:"received"`
+	Sent          []string                    `json:"sent"`
+	TamperingList []bool                      `json:"tampering_list"`
+	Tampering     bool                        `json:"tampering"`
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{
+		Config: config,
+		Methods: []Method{
+			randomInvalidMethod{},
+			randomInvalidFieldCount{},
+			randomBigRequestMethod{},
+			randomInvalidVersionNumber{},
+			squidCacheManager{},
+		},
+	}
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	Config  Config
+	Methods []Method
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+var (
+	// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
+	ErrNoAvailableTestHelpers = errors.New("no available helpers")
+
+	// ErrInvalidHelperType is emitted when the helper type is invalid.
+	ErrInvalidHelperType = errors.New("invalid helper type")
+
+	// ErrNoMeasurementMethod is emitted when Measurer.Methods is empty.
+	ErrNoMeasurementMethod = errors.New("no configured measurement method")
+)
+
+// Run implements ExperimentMeasurer.Run.
+func (m Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	tk := new(TestKeys)
+	measurement.TestKeys = tk
+	if len(m.Methods) < 1 {
+		return ErrNoMeasurementMethod
+	}
+	const helperName = "tcp-echo"
+	helpers, ok := sess.GetTestHelpersByName(helperName)
+	if !ok || len(helpers) < 1 {
+		return ErrNoAvailableTestHelpers
+	}
+	helper := helpers[0]
+	if helper.Type != "legacy" {
+		return ErrInvalidHelperType
+	}
+	measurement.TestHelpers = map[string]interface{}{
+		"backend": helper.Address,
+	}
+	out := make(chan MethodResult)
+	for _, method := range m.Methods {
+		callbacks.OnProgress(0.0, fmt.Sprintf("%s...", method.Name()))
+		go method.Run(ctx, MethodConfig{
+			Address: helper.Address,
+			Logger:  sess.Logger(),
+			Out:     out,
+		})
+	}
+	var (
+		completed int
+		progress  float64
+		result    MethodResult
+	)
+	for {
+		select {
+		case result = <-out:
+		case <-time.After(500 * time.Millisecond):
+			if completed <= 0 {
+				progress += 0.05
+				callbacks.OnProgress(progress, "waiting for results...")
+			}
+			continue
+		}
+		failure := archival.NewFailure(result.Err)
+		tk.FailureList = append(tk.FailureList, failure)
+		tk.Received = append(tk.Received, result.Received)
+		tk.Sent = append(tk.Sent, result.Sent)
+		tk.TamperingList = append(tk.TamperingList, result.Tampering)
+		tk.Tampering = (tk.Tampering || result.Tampering)
+		completed++
+		percentage := (float64(completed)/float64(len(m.Methods)))*0.5 + 0.5
+		callbacks.OnProgress(percentage, fmt.Sprintf("%s... %+v", result.Name, result.Err))
+		if completed >= len(m.Methods) {
+			break
+		}
+	}
+	return nil
+}
+
+// MethodConfig contains the settings for a specific measuring method.
+type MethodConfig struct {
+	Address string
+	Logger  model.Logger
+	Out     chan<- MethodResult
+}
+
+// MethodResult is the result of one of the methods implemented by this experiment.
+type MethodResult struct {
+	Err       error
+	Name      string
+	Received  archival.MaybeBinaryValue
+	Sent      string
+	Tampering bool
+}
+
+// Method is one of the methods implemented by this experiment.
+type Method interface {
+	Name() string
+	Run(ctx context.Context, config MethodConfig)
+}
+
+type randomInvalidMethod struct{}
+
+func (randomInvalidMethod) Name() string {
+	return "random_invalid_method"
+}
+
+func (meth randomInvalidMethod) Run(ctx context.Context, config MethodConfig) {
+	RunMethod(ctx, RunMethodConfig{
+		MethodConfig: config,
+		Name:         meth.Name(),
+		RequestLine:  randx.LettersUppercase(4) + " / HTTP/1.1\n\r",
+	})
+}
+
+type randomInvalidFieldCount struct{}
+
+func (randomInvalidFieldCount) Name() string {
+	return "random_invalid_field_count"
+}
+
+func (meth randomInvalidFieldCount) Run(ctx context.Context, config MethodConfig) {
+	RunMethod(ctx, RunMethodConfig{
+		MethodConfig: config,
+		Name:         meth.Name(),
+		RequestLine: strings.Join([]string{
+			randx.LettersUppercase(5),
+			" ",
+			randx.LettersUppercase(5),
+			" ",
+			randx.LettersUppercase(5),
+			" ",
+			randx.LettersUppercase(5),
+			"\r\n",
+		}, ""),
+	})
+}
+
+type randomBigRequestMethod struct{}
+
+func (randomBigRequestMethod) Name() string {
+	return "random_big_request_method"
+}
+
+func (meth randomBigRequestMethod) Run(ctx context.Context, config MethodConfig) {
+	RunMethod(ctx, RunMethodConfig{
+		MethodConfig: config,
+		Name:         meth.Name(),
+		RequestLine: strings.Join([]string{
+			randx.LettersUppercase(1024),
+			" / HTTP/1.1\r\n",
+		}, ""),
+	})
+}
+
+type randomInvalidVersionNumber struct{}
+
+func (randomInvalidVersionNumber) Name() string {
+	return "random_invalid_version_number"
+}
+
+func (meth randomInvalidVersionNumber) Run(ctx context.Context, config MethodConfig) {
+	RunMethod(ctx, RunMethodConfig{
+		MethodConfig: config,
+		Name:         meth.Name(),
+		RequestLine: strings.Join([]string{
+			"GET / HTTP/",
+			randx.LettersUppercase(3),
+			"\r\n",
+		}, ""),
+	})
+}
+
+type squidCacheManager struct{}
+
+func (squidCacheManager) Name() string {
+	return "squid_cache_manager"
+}
+
+func (meth squidCacheManager) Run(ctx context.Context, config MethodConfig) {
+	RunMethod(ctx, RunMethodConfig{
+		MethodConfig: config,
+		Name:         meth.Name(),
+		RequestLine:  "GET cache_object://localhost/ HTTP/1.0\n\r",
+	})
+}
+
+// RunMethodConfig contains the config for RunMethod
+type RunMethodConfig struct {
+	MethodConfig
+	Name        string
+	NewDialer   func(config netx.Config) netx.Dialer
+	RequestLine string
+}
+
+// RunMethod runs the specific method using the given config and context
+func RunMethod(ctx context.Context, config RunMethodConfig) {
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	result := MethodResult{Name: config.Name}
+	defer func() {
+		config.Out <- result
+	}()
+	if config.NewDialer == nil {
+		config.NewDialer = netx.NewDialer
+	}
+	dialer := config.NewDialer(netx.Config{
+		ContextByteCounting: true,
+		Logger:              config.Logger,
+	})
+	conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(config.Address, "80"))
+	if err != nil {
+		result.Err = err
+		return
+	}
+	deadline := time.Now().Add(timeout)
+	if err := conn.SetDeadline(deadline); err != nil {
+		result.Err = err
+		return
+	}
+	if _, err := conn.Write([]byte(config.RequestLine)); err != nil {
+		result.Err = err
+		return
+	}
+	result.Sent = config.RequestLine
+	data := make([]byte, 4096)
+	defer func() {
+		result.Tampering = (result.Sent != result.Received.Value)
+	}()
+	for {
+		count, err := conn.Read(data)
+		if err != nil {
+			// We expect this method to terminate w/ timeout
+			if err.Error() == errorx.FailureGenericTimeoutError {
+				err = nil
+			}
+			result.Err = err
+			return
+		}
+		result.Received.Value += string(data[:count])
+	}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	sk.IsAnomaly = tk.Tampering
+	return sk, nil
+}

internal/engine/experiment/hirl/hirl_test.go

@@ -0,0 +1,598 @@
+package hirl_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := hirl.NewExperimentMeasurer(hirl.Config{})
+	if measurer.ExperimentName() != "http_invalid_request_line" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.2.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestSuccess(t *testing.T) {
+	measurer := hirl.NewExperimentMeasurer(hirl.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "37.218.241.93",
+				Type:    "legacy",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*hirl.TestKeys)
+	if len(tk.FailureList) != len(tk.Received) {
+		t.Fatal("FailureList and Received have different lengths")
+	}
+	if len(tk.Received) != len(tk.Sent) {
+		t.Fatal("Received and Sent have different lengths")
+	}
+	if len(tk.Sent) != len(tk.TamperingList) {
+		t.Fatal("Sent and TamperingList have different lengths")
+	}
+	for _, failure := range tk.FailureList {
+		if failure != nil {
+			t.Fatal(*failure)
+		}
+	}
+	for idx, received := range tk.Received {
+		if received.Value != tk.Sent[idx] {
+			t.Fatal("mismatch between received and sent")
+		}
+	}
+	for _, entry := range tk.TamperingList {
+		if entry != false {
+			t.Fatal("found entry with tampering")
+		}
+	}
+	if tk.Tampering != false {
+		t.Fatal("overall there is tampering?!")
+	}
+}
+
+func TestCancelledContext(t *testing.T) {
+	measurer := hirl.NewExperimentMeasurer(hirl.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "37.218.241.93",
+				Type:    "legacy",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*hirl.TestKeys)
+	if len(tk.FailureList) != 5 {
+		t.Fatal("unexpected FailureList length")
+	}
+	for _, failure := range tk.FailureList {
+		if *failure != errorx.FailureInterrupted {
+			t.Fatal("unexpected failure")
+		}
+	}
+	if len(tk.Received) != 5 {
+		t.Fatal("unexpected Received length")
+	}
+	for _, entry := range tk.Received {
+		if entry.Value != "" {
+			t.Fatal("unexpected received entry")
+		}
+	}
+	if len(tk.Sent) != 5 {
+		t.Fatal("unexpected Sent length")
+	}
+	for _, entry := range tk.Sent {
+		if entry != "" {
+			t.Fatal("unexpected sent entry")
+		}
+	}
+	if len(tk.TamperingList) != 5 {
+		t.Fatal("unexpected TamperingList length")
+	}
+	for _, entry := range tk.TamperingList {
+		if entry != false {
+			t.Fatal("unexpected tampering entry")
+		}
+	}
+	if tk.Tampering != false {
+		t.Fatal("overall there is tampering?!")
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(hirl.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+type FakeMethodSuccessful struct{}
+
+func (FakeMethodSuccessful) Name() string {
+	return "success"
+}
+
+func (meth FakeMethodSuccessful) Run(ctx context.Context, config hirl.MethodConfig) {
+	config.Out <- hirl.MethodResult{
+		Name:      meth.Name(),
+		Received:  archival.MaybeBinaryValue{Value: "antani"},
+		Sent:      "antani",
+		Tampering: false,
+	}
+}
+
+type FakeMethodFailure struct{}
+
+func (FakeMethodFailure) Name() string {
+	return "failure"
+}
+
+func (meth FakeMethodFailure) Run(ctx context.Context, config hirl.MethodConfig) {
+	config.Out <- hirl.MethodResult{
+		Name:      meth.Name(),
+		Received:  archival.MaybeBinaryValue{Value: "antani"},
+		Sent:      "melandri",
+		Tampering: true,
+	}
+}
+
+func TestWithFakeMethods(t *testing.T) {
+	measurer := hirl.Measurer{
+		Config: hirl.Config{},
+		Methods: []hirl.Method{
+			FakeMethodSuccessful{},
+			FakeMethodFailure{},
+			FakeMethodSuccessful{},
+		},
+	}
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "127.0.0.1",
+				Type:    "legacy",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*hirl.TestKeys)
+	if len(tk.FailureList) != len(tk.Received) {
+		t.Fatal("FailureList and Received have different lengths")
+	}
+	if len(tk.Received) != len(tk.Sent) {
+		t.Fatal("Received and Sent have different lengths")
+	}
+	if len(tk.Sent) != len(tk.TamperingList) {
+		t.Fatal("Sent and TamperingList have different lengths")
+	}
+	for _, failure := range tk.FailureList {
+		if failure != nil {
+			t.Fatal(*failure)
+		}
+	}
+	for _, received := range tk.Received {
+		if received.Value != "antani" {
+			t.Fatal("unexpected received value")
+		}
+	}
+	for _, sent := range tk.Sent {
+		if sent != "antani" && sent != "melandri" {
+			t.Fatal("unexpected sent value")
+		}
+	}
+	var falses, trues int
+	for _, entry := range tk.TamperingList {
+		if entry {
+			trues++
+		} else {
+			falses++
+		}
+	}
+	if falses != 2 && trues != 1 {
+		t.Fatal("not the right values in tampering list")
+	}
+	if tk.Tampering != true {
+		t.Fatal("overall there is no tampering?!")
+	}
+}
+
+func TestWithNoMethods(t *testing.T) {
+	measurer := hirl.Measurer{
+		Config:  hirl.Config{},
+		Methods: []hirl.Method{},
+	}
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "127.0.0.1",
+				Type:    "legacy",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if !errors.Is(err, hirl.ErrNoMeasurementMethod) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*hirl.TestKeys)
+	if len(tk.FailureList) != 0 {
+		t.Fatal("unexpected FailureList length")
+	}
+	if len(tk.Received) != 0 {
+		t.Fatal("unexpected Received length")
+	}
+	if len(tk.Sent) != 0 {
+		t.Fatal("unexpected Sent length")
+	}
+	if len(tk.TamperingList) != 0 {
+		t.Fatal("unexpected TamperingList length")
+	}
+	if tk.Tampering != false {
+		t.Fatal("overall there is tampering?!")
+	}
+}
+
+func TestNoHelpers(t *testing.T) {
+	measurer := hirl.NewExperimentMeasurer(hirl.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if !errors.Is(err, hirl.ErrNoAvailableTestHelpers) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*hirl.TestKeys)
+	if len(tk.FailureList) != 0 {
+		t.Fatal("expected an empty FailureList")
+	}
+	if len(tk.FailureList) != len(tk.Received) {
+		t.Fatal("FailureList and Received have different lengths")
+	}
+	if len(tk.Received) != len(tk.Sent) {
+		t.Fatal("Received and Sent have different lengths")
+	}
+	if len(tk.Sent) != len(tk.TamperingList) {
+		t.Fatal("Sent and TamperingList have different lengths")
+	}
+	if tk.Tampering != false {
+		t.Fatal("overall there is tampering?!")
+	}
+}
+
+func TestNoActualHelperInList(t *testing.T) {
+	measurer := hirl.NewExperimentMeasurer(hirl.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": nil,
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if !errors.Is(err, hirl.ErrNoAvailableTestHelpers) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*hirl.TestKeys)
+	if len(tk.FailureList) != 0 {
+		t.Fatal("expected an empty FailureList")
+	}
+	if len(tk.FailureList) != len(tk.Received) {
+		t.Fatal("FailureList and Received have different lengths")
+	}
+	if len(tk.Received) != len(tk.Sent) {
+		t.Fatal("Received and Sent have different lengths")
+	}
+	if len(tk.Sent) != len(tk.TamperingList) {
+		t.Fatal("Sent and TamperingList have different lengths")
+	}
+	if tk.Tampering != false {
+		t.Fatal("overall there is tampering?!")
+	}
+}
+
+func TestWrongTestHelperType(t *testing.T) {
+	measurer := hirl.NewExperimentMeasurer(hirl.Config{})
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "127.0.0.1",
+				Type:    "antani",
+			}},
+		},
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if !errors.Is(err, hirl.ErrInvalidHelperType) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*hirl.TestKeys)
+	if len(tk.FailureList) != 0 {
+		t.Fatal("expected an empty FailureList")
+	}
+	if len(tk.FailureList) != len(tk.Received) {
+		t.Fatal("FailureList and Received have different lengths")
+	}
+	if len(tk.Received) != len(tk.Sent) {
+		t.Fatal("Received and Sent have different lengths")
+	}
+	if len(tk.Sent) != len(tk.TamperingList) {
+		t.Fatal("Sent and TamperingList have different lengths")
+	}
+	if tk.Tampering != false {
+		t.Fatal("overall there is tampering?!")
+	}
+}
+
+func TestRunMethodDialFailure(t *testing.T) {
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "37.218.241.93",
+				Type:    "legacy",
+			}},
+		},
+	}
+	helpers, ok := sess.GetTestHelpersByName("tcp-echo")
+	if len(helpers) < 1 || !ok {
+		t.Fatal("cannot get helper")
+	}
+	expected := errors.New("mocked error")
+	out := make(chan hirl.MethodResult)
+	config := hirl.RunMethodConfig{
+		MethodConfig: hirl.MethodConfig{
+			Address: helpers[0].Address,
+			Logger:  log.Log,
+			Out:     out,
+		},
+		Name: "random_invalid_version_number",
+		NewDialer: func(config netx.Config) netx.Dialer {
+			return FakeDialer{Err: expected}
+		},
+		RequestLine: "GET / HTTP/ABC",
+	}
+	go hirl.RunMethod(context.Background(), config)
+	result := <-out
+	if !errors.Is(result.Err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Name != "random_invalid_version_number" {
+		t.Fatal("unexpected Name")
+	}
+	if result.Received.Value != "" {
+		t.Fatal("unexpected Received.Value")
+	}
+	if result.Sent != "" {
+		t.Fatal("unexpected Sent")
+	}
+	if result.Tampering != false {
+		t.Fatal("unexpected Tampering")
+	}
+}
+
+func TestRunMethodSetDeadlineFailure(t *testing.T) {
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "37.218.241.93",
+				Type:    "legacy",
+			}},
+		},
+	}
+	helpers, ok := sess.GetTestHelpersByName("tcp-echo")
+	if len(helpers) < 1 || !ok {
+		t.Fatal("cannot get helper")
+	}
+	expected := errors.New("mocked error")
+	out := make(chan hirl.MethodResult)
+	config := hirl.RunMethodConfig{
+		MethodConfig: hirl.MethodConfig{
+			Address: helpers[0].Address,
+			Logger:  log.Log,
+			Out:     out,
+		},
+		Name: "random_invalid_version_number",
+		NewDialer: func(config netx.Config) netx.Dialer {
+			return FakeDialer{Conn: &FakeConn{
+				SetDeadlineError: expected,
+			}}
+		},
+		RequestLine: "GET / HTTP/ABC",
+	}
+	go hirl.RunMethod(context.Background(), config)
+	result := <-out
+	if !errors.Is(result.Err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Name != "random_invalid_version_number" {
+		t.Fatal("unexpected Name")
+	}
+	if result.Received.Value != "" {
+		t.Fatal("unexpected Received.Value")
+	}
+	if result.Sent != "" {
+		t.Fatal("unexpected Sent")
+	}
+	if result.Tampering != false {
+		t.Fatal("unexpected Tampering")
+	}
+}
+
+func TestRunMethodWriteFailure(t *testing.T) {
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "37.218.241.93",
+				Type:    "legacy",
+			}},
+		},
+	}
+	helpers, ok := sess.GetTestHelpersByName("tcp-echo")
+	if len(helpers) < 1 || !ok {
+		t.Fatal("cannot get helper")
+	}
+	expected := errors.New("mocked error")
+	out := make(chan hirl.MethodResult)
+	config := hirl.RunMethodConfig{
+		MethodConfig: hirl.MethodConfig{
+			Address: helpers[0].Address,
+			Logger:  log.Log,
+			Out:     out,
+		},
+		Name: "random_invalid_version_number",
+		NewDialer: func(config netx.Config) netx.Dialer {
+			return FakeDialer{Conn: &FakeConn{
+				WriteError: expected,
+			}}
+		},
+		RequestLine: "GET / HTTP/ABC",
+	}
+	go hirl.RunMethod(context.Background(), config)
+	result := <-out
+	if !errors.Is(result.Err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Name != "random_invalid_version_number" {
+		t.Fatal("unexpected Name")
+	}
+	if result.Received.Value != "" {
+		t.Fatal("unexpected Received.Value")
+	}
+	if result.Sent != "" {
+		t.Fatal("unexpected Sent")
+	}
+	if result.Tampering != false {
+		t.Fatal("unexpected Tampering")
+	}
+}
+
+func TestRunMethodReadEOFWithWrongData(t *testing.T) {
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+		MockableTestHelpers: map[string][]model.Service{
+			"tcp-echo": {{
+				Address: "37.218.241.93",
+				Type:    "legacy",
+			}},
+		},
+	}
+	helpers, ok := sess.GetTestHelpersByName("tcp-echo")
+	if len(helpers) < 1 || !ok {
+		t.Fatal("cannot get helper")
+	}
+	out := make(chan hirl.MethodResult)
+	config := hirl.RunMethodConfig{
+		MethodConfig: hirl.MethodConfig{
+			Address: helpers[0].Address,
+			Logger:  log.Log,
+			Out:     out,
+		},
+		Name: "random_invalid_version_number",
+		NewDialer: func(config netx.Config) netx.Dialer {
+			return FakeDialer{Conn: &FakeConn{
+				ReadData: []byte("0xdeadbeef"),
+			}}
+		},
+		RequestLine: "GET / HTTP/ABC",
+	}
+	go hirl.RunMethod(context.Background(), config)
+	result := <-out
+	if !errors.Is(result.Err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Name != "random_invalid_version_number" {
+		t.Fatal("unexpected Name")
+	}
+	if result.Received.Value != "0xdeadbeef" {
+		t.Fatal("unexpected Received.Value")
+	}
+	if result.Sent != "GET / HTTP/ABC" {
+		t.Fatal("unexpected Sent")
+	}
+	if result.Tampering != true {
+		t.Fatal("unexpected Tampering")
+	}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &hirl.Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysFalse(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &hirl.TestKeys{
+		Tampering: false,
+	}}
+	m := &hirl.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(hirl.SummaryKeys)
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}
+
+func TestSummaryKeysTrue(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &hirl.TestKeys{
+		Tampering: true,
+	}}
+	m := &hirl.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(hirl.SummaryKeys)
+	if sk.IsAnomaly == false {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/httphostheader/httphostheader.go

@@ -0,0 +1,94 @@
+// Package httphostheader contains the HTTP host header network experiment.
+//
+// This experiment has not been specified yet. It is nonetheless available for testing
+// and as a building block that other experiments could reuse.
+package httphostheader
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+const (
+	testName    = "http_host_header"
+	testVersion = "0.3.0"
+)
+
+// Config contains the experiment config.
+type Config struct {
+	// TestHelperURL is the address of the test helper.
+	TestHelperURL string
+}
+
+// TestKeys contains httphost test keys.
+type TestKeys struct {
+	urlgetter.TestKeys
+	THAddress string `json:"th_address"`
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config Config
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m *Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run.
+func (m *Measurer) Run(
+	ctx context.Context,
+	sess model.ExperimentSession,
+	measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks,
+) error {
+	if measurement.Input == "" {
+		return errors.New("experiment requires input")
+	}
+	if m.config.TestHelperURL == "" {
+		m.config.TestHelperURL = "http://www.example.org"
+	}
+	urlgetter.RegisterExtensions(measurement)
+	g := urlgetter.Getter{
+		Begin: measurement.MeasurementStartTimeSaved,
+		Config: urlgetter.Config{
+			HTTPHost: string(measurement.Input),
+		},
+		Session: sess,
+		Target:  fmt.Sprintf(m.config.TestHelperURL),
+	}
+	tk, _ := g.Get(ctx)
+	measurement.TestKeys = &TestKeys{
+		TestKeys:  tk,
+		THAddress: m.config.TestHelperURL,
+	}
+	return nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return &Measurer{config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/httphostheader/httphostheader_test.go

@@ -0,0 +1,107 @@
+package httphostheader
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+const (
+	softwareName    = "ooniprobe-example"
+	softwareVersion = "0.0.1"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	if measurer.ExperimentName() != "http_host_header" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.3.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestMeasurerMeasureNoMeasurementInput(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{
+		TestHelperURL: "http://www.google.com",
+	})
+	err := measurer.Run(
+		context.Background(),
+		newsession(),
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err == nil || err.Error() != "experiment requires input" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestMeasurerMeasureNoTestHelper(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	measurement := &model.Measurement{Input: "x.org"}
+	err := measurer.Run(
+		context.Background(),
+		newsession(),
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestRunnerHTTPSetHostHeader(t *testing.T) {
+	var host string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		host = r.Host
+		w.WriteHeader(200)
+	}))
+	defer server.Close()
+	measurer := NewExperimentMeasurer(Config{
+		TestHelperURL: server.URL,
+	})
+	measurement := &model.Measurement{
+		Input: "x.org",
+	}
+	err := measurer.Run(
+		context.Background(),
+		newsession(),
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if host != "x.org" {
+		t.Fatal("not the host we expected")
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func newsession() model.ExperimentSession {
+	return &mockable.Session{MockableLogger: log.Log}
+}
+
+func TestSummaryKeysGeneric(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &TestKeys{}}
+	m := &Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(SummaryKeys)
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/ndt7/callback.go

@@ -0,0 +1,8 @@
+package ndt7
+
+import "time"
+
+type (
+	callbackJSON        func(data []byte) error
+	callbackPerformance func(elapsed time.Duration, count int64)
+)

internal/engine/experiment/ndt7/callback_test.go

@@ -0,0 +1,10 @@
+package ndt7
+
+import "time"
+
+func defaultCallbackJSON(data []byte) error {
+	return nil
+}
+
+func defaultCallbackPerformance(elapsed time.Duration, count int64) {
+}

internal/engine/experiment/ndt7/dial.go

@@ -0,0 +1,91 @@
+package ndt7
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+	"net/url"
+
+	"github.com/gorilla/websocket"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
+)
+
+type dialManager struct {
+	ndt7URL         string
+	logger          model.Logger
+	proxyURL        *url.URL
+	readBufferSize  int
+	tlsConfig       *tls.Config
+	userAgent       string
+	writeBufferSize int
+}
+
+func newDialManager(ndt7URL string, logger model.Logger, userAgent string) dialManager {
+	return dialManager{
+		ndt7URL:         ndt7URL,
+		logger:          logger,
+		readBufferSize:  paramMaxBufferSize,
+		userAgent:       userAgent,
+		writeBufferSize: paramMaxBufferSize,
+	}
+}
+
+func (mgr dialManager) dialWithTestName(ctx context.Context, testName string) (*websocket.Conn, error) {
+	var reso resolver.Resolver = resolver.SystemResolver{}
+	reso = resolver.LoggingResolver{Resolver: reso, Logger: mgr.logger}
+	var dlr dialer.Dialer = selfcensor.SystemDialer{}
+	dlr = dialer.TimeoutDialer{Dialer: dlr}
+	dlr = dialer.ErrorWrapperDialer{Dialer: dlr}
+	dlr = dialer.LoggingDialer{Dialer: dlr, Logger: mgr.logger}
+	dlr = dialer.DNSDialer{Dialer: dlr, Resolver: reso}
+	dlr = dialer.ProxyDialer{Dialer: dlr, ProxyURL: mgr.proxyURL}
+	dlr = dialer.ByteCounterDialer{Dialer: dlr}
+	dlr = dialer.ShapingDialer{Dialer: dlr}
+	dialer := websocket.Dialer{
+		NetDialContext:  dlr.DialContext,
+		ReadBufferSize:  mgr.readBufferSize,
+		TLSClientConfig: mgr.tlsConfig,
+		WriteBufferSize: mgr.writeBufferSize,
+	}
+	headers := http.Header{}
+	headers.Add("Sec-WebSocket-Protocol", "net.measurementlab.ndt.v7")
+	headers.Add("User-Agent", mgr.userAgent)
+	mgr.logrequest(mgr.ndt7URL, headers)
+	conn, _, err := dialer.DialContext(ctx, mgr.ndt7URL, headers)
+	mgr.logresponse(err)
+	return conn, err
+}
+
+func (mgr dialManager) logrequest(url string, headers http.Header) {
+	mgr.logger.Debugf("> GET %s", url)
+	for key, values := range headers {
+		for _, v := range values {
+			mgr.logger.Debugf("> %s: %s", key, v)
+		}
+	}
+	mgr.logger.Debug("> Connection: upgrade")
+	mgr.logger.Debug("> Upgrade: websocket")
+	mgr.logger.Debug(">")
+}
+
+func (mgr dialManager) logresponse(err error) {
+	if err != nil {
+		mgr.logger.Debugf("< %+v", err)
+		return
+	}
+	mgr.logger.Debug("< 101")
+	mgr.logger.Debug("< Connection: upgrade")
+	mgr.logger.Debug("< Upgrade: websocket")
+	mgr.logger.Debug("<")
+}
+
+func (mgr dialManager) dialDownload(ctx context.Context) (*websocket.Conn, error) {
+	return mgr.dialWithTestName(ctx, "download")
+}
+
+func (mgr dialManager) dialUpload(ctx context.Context) (*websocket.Conn, error) {
+	return mgr.dialWithTestName(ctx, "upload")
+}

internal/engine/experiment/ndt7/dial_test.go

@@ -0,0 +1,70 @@
+package ndt7
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/gorilla/websocket"
+)
+
+func TestDialDownloadWithCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately halt
+	mgr := newDialManager("wss://hostname.fake", log.Log, "miniooni/0.1.0-dev")
+	conn, err := mgr.dialDownload(ctx)
+	if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
+		t.Fatal("not the error we expected")
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn here")
+	}
+}
+
+func TestDialUploadWithCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately halt
+	mgr := newDialManager("wss://hostname.fake", log.Log, "miniooni/0.1.0-dev")
+	conn, err := mgr.dialUpload(ctx)
+	if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
+		t.Fatal("not the error we expected")
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn here")
+	}
+}
+
+func TestDialIncludesUserAgent(t *testing.T) {
+	do := func(testName string) {
+		var userAgent string
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			userAgent = r.UserAgent()
+			w.WriteHeader(500)
+		})
+		server := httptest.NewServer(handler)
+		defer server.Close()
+		url, err := url.Parse(server.URL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		url.Scheme = "ws"
+		mgr := newDialManager(url.String(), log.Log, "miniooni/0.1.0-dev")
+		conn, err := mgr.dialWithTestName(context.Background(), testName)
+		if !errors.Is(err, websocket.ErrBadHandshake) {
+			t.Fatal("not the error we expected")
+		}
+		if conn != nil {
+			t.Fatal("expected nil conn here")
+		}
+		if userAgent != "miniooni/0.1.0-dev" {
+			t.Fatal("User-Agent not sent")
+		}
+	}
+	do("download")
+	do("upload")
+}

internal/engine/experiment/ndt7/download.go

@@ -0,0 +1,73 @@
+package ndt7
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+type downloadManager struct {
+	conn            mockableConn
+	maxMessageSize  int64
+	maxRuntime      time.Duration
+	measureInterval time.Duration
+	onJSON          callbackJSON
+	onPerformance   callbackPerformance
+}
+
+func newDownloadManager(
+	conn mockableConn, onPerformance callbackPerformance,
+	onJSON callbackJSON,
+) downloadManager {
+	return downloadManager{
+		conn:            conn,
+		maxMessageSize:  paramMaxMessageSize,
+		maxRuntime:      paramMaxRuntime,
+		measureInterval: paramMeasureInterval,
+		onJSON:          onJSON,
+		onPerformance:   onPerformance,
+	}
+}
+
+func (mgr downloadManager) run(ctx context.Context) error {
+	var total int64
+	start := time.Now()
+	if err := mgr.conn.SetReadDeadline(start.Add(mgr.maxRuntime)); err != nil {
+		return err
+	}
+	mgr.conn.SetReadLimit(mgr.maxMessageSize)
+	ticker := time.NewTicker(mgr.measureInterval)
+	defer ticker.Stop()
+	for ctx.Err() == nil {
+		kind, reader, err := mgr.conn.NextReader()
+		if err != nil {
+			return err
+		}
+		if kind == websocket.TextMessage {
+			data, err := ioutil.ReadAll(reader)
+			if err != nil {
+				return err
+			}
+			total += int64(len(data))
+			if err := mgr.onJSON(data); err != nil {
+				return err
+			}
+			continue
+		}
+		n, err := io.Copy(ioutil.Discard, reader)
+		if err != nil {
+			return err
+		}
+		total += int64(n)
+		select {
+		case now := <-ticker.C:
+			mgr.onPerformance(now.Sub(start), total)
+		default:
+			// NOTHING
+		}
+	}
+	return nil
+}

internal/engine/experiment/ndt7/download_test.go

@@ -0,0 +1,145 @@
+package ndt7
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+func TestDownloadSetReadDeadlineFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	mgr := newDownloadManager(
+		&mockableConnMock{
+			ReadDeadlineErr: expected,
+		},
+		defaultCallbackPerformance,
+		defaultCallbackJSON,
+	)
+	err := mgr.run(context.Background())
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadNextReaderFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	mgr := newDownloadManager(
+		&mockableConnMock{
+			NextReaderErr: expected,
+		},
+		defaultCallbackPerformance,
+		defaultCallbackJSON,
+	)
+	err := mgr.run(context.Background())
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadTextMessageReadAllFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	mgr := newDownloadManager(
+		&mockableConnMock{
+			NextReaderMsgType: websocket.TextMessage,
+			NextReaderReader: func() io.Reader {
+				return &alwaysFailingReader{
+					Err: expected,
+				}
+			},
+		},
+		defaultCallbackPerformance,
+		defaultCallbackJSON,
+	)
+	err := mgr.run(context.Background())
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+type alwaysFailingReader struct {
+	Err error
+}
+
+func (r *alwaysFailingReader) Read(p []byte) (int, error) {
+	return 0, r.Err
+}
+
+func TestDownloadBinaryMessageReadAllFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	mgr := newDownloadManager(
+		&mockableConnMock{
+			NextReaderMsgType: websocket.BinaryMessage,
+			NextReaderReader: func() io.Reader {
+				return &alwaysFailingReader{
+					Err: expected,
+				}
+			},
+		},
+		defaultCallbackPerformance,
+		defaultCallbackJSON,
+	)
+	err := mgr.run(context.Background())
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadOnJSONCallbackError(t *testing.T) {
+	mgr := newDownloadManager(
+		&mockableConnMock{
+			NextReaderMsgType: websocket.TextMessage,
+			NextReaderReader: func() io.Reader {
+				return &invalidJSONReader{}
+			},
+		},
+		defaultCallbackPerformance,
+		func(data []byte) error {
+			var v interface{}
+			return json.Unmarshal(data, &v)
+		},
+	)
+	err := mgr.run(context.Background())
+	if err == nil || !strings.HasSuffix(err.Error(), "unexpected end of JSON input") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+type invalidJSONReader struct{}
+
+func (r *invalidJSONReader) Read(p []byte) (int, error) {
+	return copy(p, []byte(`{`)), io.EOF
+}
+
+func TestDownloadOnJSONLoop(t *testing.T) {
+	mgr := newDownloadManager(
+		&mockableConnMock{
+			NextReaderMsgType: websocket.TextMessage,
+			NextReaderReader: func() io.Reader {
+				return &goodJSONReader{}
+			},
+		},
+		defaultCallbackPerformance,
+		func(data []byte) error {
+			var v interface{}
+			return json.Unmarshal(data, &v)
+		},
+	)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+	err := mgr.run(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+type goodJSONReader struct{}
+
+func (r *goodJSONReader) Read(p []byte) (int, error) {
+	return copy(p, []byte(`{}`)), io.EOF
+}

internal/engine/experiment/ndt7/mockable.go

@@ -0,0 +1,16 @@
+package ndt7
+
+import (
+	"io"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+type mockableConn interface {
+	NextReader() (int, io.Reader, error)
+	SetReadDeadline(time.Time) error
+	SetReadLimit(int64)
+	SetWriteDeadline(time.Time) error
+	WritePreparedMessage(*websocket.PreparedMessage) error
+}

internal/engine/experiment/ndt7/mockable_test.go

@@ -0,0 +1,39 @@
+package ndt7
+
+import (
+	"io"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+type mockableConnMock struct {
+	NextReaderMsgType       int
+	NextReaderErr           error
+	NextReaderReader        func() io.Reader
+	ReadDeadlineErr         error
+	WriteDeadlineErr        error
+	WritePreparedMessageErr error
+}
+
+func (c *mockableConnMock) NextReader() (int, io.Reader, error) {
+	var reader io.Reader
+	if c.NextReaderReader != nil {
+		reader = c.NextReaderReader()
+	}
+	return c.NextReaderMsgType, reader, c.NextReaderErr
+}
+
+func (c *mockableConnMock) SetReadDeadline(time.Time) error {
+	return c.ReadDeadlineErr
+}
+
+func (c *mockableConnMock) SetReadLimit(int64) {}
+
+func (c *mockableConnMock) SetWriteDeadline(time.Time) error {
+	return c.WriteDeadlineErr
+}
+
+func (c *mockableConnMock) WritePreparedMessage(*websocket.PreparedMessage) error {
+	return c.WritePreparedMessageErr
+}

internal/engine/experiment/ndt7/ndt7.go

@@ -0,0 +1,301 @@
+// Package ndt7 contains the ndt7 network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-022-ndt.md
+package ndt7
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mlablocatev2"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+const (
+	testName    = "ndt"
+	testVersion = "0.8.0"
+)
+
+// Config contains the experiment settings
+type Config struct {
+	noDownload bool
+	noUpload   bool
+}
+
+// Summary is the measurement summary
+type Summary struct {
+	AvgRTT         float64 `json:"avg_rtt"`         // Average RTT [ms]
+	Download       float64 `json:"download"`        // download speed [kbit/s]
+	MSS            int64   `json:"mss"`             // MSS
+	MaxRTT         float64 `json:"max_rtt"`         // Max AvgRTT sample seen [ms]
+	MinRTT         float64 `json:"min_rtt"`         // Min RTT according to kernel [ms]
+	Ping           float64 `json:"ping"`            // Equivalent to MinRTT [ms]
+	RetransmitRate float64 `json:"retransmit_rate"` // bytes_retrans/bytes_sent [0..1]
+	Upload         float64 `json:"upload"`          // upload speed [kbit/s]
+}
+
+// ServerInfo contains information on the selected server
+//
+// Site is currently an extension to the NDT specification
+// until the data format of the new mlab locate is clear.
+type ServerInfo struct {
+	Hostname string `json:"hostname"`
+	Site     string `json:"site,omitempty"`
+}
+
+// TestKeys contains the test keys
+type TestKeys struct {
+	// Download contains download results
+	Download []Measurement `json:"download"`
+
+	// Failure is the failure string
+	Failure *string `json:"failure"`
+
+	// Protocol contains the version of the ndt protocol
+	Protocol int64 `json:"protocol"`
+
+	// Server contains information on the selected server
+	Server ServerInfo `json:"server"`
+
+	// Summary contains the measurement summary
+	Summary Summary `json:"summary"`
+
+	// Upload contains upload results
+	Upload []Measurement `json:"upload"`
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config          Config
+	jsonUnmarshal   func(data []byte, v interface{}) error
+	preDownloadHook func()
+	preUploadHook   func()
+}
+
+func (m *Measurer) discover(
+	ctx context.Context, sess model.ExperimentSession) (mlablocatev2.NDT7Result, error) {
+	httpClient := &http.Client{
+		Transport: netx.NewHTTPTransport(netx.Config{
+			Logger: sess.Logger(),
+		}),
+	}
+	defer httpClient.CloseIdleConnections()
+	client := mlablocatev2.NewClient(httpClient, sess.Logger(), sess.UserAgent())
+	out, err := client.QueryNDT7(ctx)
+	if err != nil {
+		return mlablocatev2.NDT7Result{}, err
+	}
+	return out[0], nil // same as with locate services v1
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m *Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+func (m *Measurer) doDownload(
+	ctx context.Context, sess model.ExperimentSession,
+	callbacks model.ExperimentCallbacks, tk *TestKeys,
+	URL string,
+) error {
+	if m.config.noDownload == true {
+		return nil // useful to make tests faster
+	}
+	conn, err := newDialManager(URL,
+		sess.Logger(), sess.UserAgent()).dialDownload(ctx)
+	if err != nil {
+		return err
+	}
+	defer callbacks.OnProgress(0.5, " download: done")
+	defer conn.Close()
+	mgr := newDownloadManager(
+		conn,
+		func(timediff time.Duration, count int64) {
+			elapsed := timediff.Seconds()
+			// The percentage of completion of download goes from 0 to
+			// 50% of the whole experiment, hence the `/2.0`.
+			percentage := elapsed / paramMaxRuntimeUpperBound / 2.0
+			speed := float64(count) * 8.0 / elapsed
+			message := fmt.Sprintf(" download: speed %s", humanizex.SI(
+				float64(speed), "bit/s"))
+			tk.Summary.Download = speed / 1e03 /* bit/s => kbit/s */
+			callbacks.OnProgress(percentage, message)
+			tk.Download = append(tk.Download, Measurement{
+				AppInfo: &AppInfo{
+					ElapsedTime: int64(timediff / time.Microsecond),
+					NumBytes:    count,
+				},
+				Origin: "client",
+				Test:   "download",
+			})
+		},
+		func(data []byte) error {
+			sess.Logger().Debugf("%s", string(data))
+			var measurement Measurement
+			if err := m.jsonUnmarshal(data, &measurement); err != nil {
+				return err
+			}
+			if measurement.TCPInfo != nil {
+				rtt := float64(measurement.TCPInfo.RTT) / 1e03 /* us => ms */
+				tk.Summary.AvgRTT = rtt
+				tk.Summary.MSS = int64(measurement.TCPInfo.AdvMSS)
+				if tk.Summary.MaxRTT < rtt {
+					tk.Summary.MaxRTT = rtt
+				}
+				tk.Summary.MinRTT = float64(measurement.TCPInfo.MinRTT) / 1e03 /* us => ms */
+				tk.Summary.Ping = tk.Summary.MinRTT
+				if measurement.TCPInfo.BytesSent > 0 {
+					tk.Summary.RetransmitRate = (float64(measurement.TCPInfo.BytesRetrans) /
+						float64(measurement.TCPInfo.BytesSent))
+				}
+				measurement.BBRInfo = nil        // don't encourage people to use it
+				measurement.ConnectionInfo = nil // do we need to save it?
+				measurement.Origin = "server"
+				measurement.Test = "download"
+				tk.Download = append(tk.Download, measurement)
+			}
+			return nil
+		},
+	)
+	if err := mgr.run(ctx); err != nil && err.Error() != "generic_timeout_error" {
+		sess.Logger().Warnf("download: %s", err)
+	}
+	return nil // failure is only when we cannot connect
+}
+
+func (m *Measurer) doUpload(
+	ctx context.Context, sess model.ExperimentSession,
+	callbacks model.ExperimentCallbacks, tk *TestKeys,
+	URL string,
+) error {
+	if m.config.noUpload == true {
+		return nil // useful to make tests faster
+	}
+	conn, err := newDialManager(URL,
+		sess.Logger(), sess.UserAgent()).dialUpload(ctx)
+	if err != nil {
+		return err
+	}
+	defer callbacks.OnProgress(1, "   upload: done")
+	defer conn.Close()
+	mgr := newUploadManager(
+		conn,
+		func(timediff time.Duration, count int64) {
+			elapsed := timediff.Seconds()
+			// The percentage of completion of upload goes from 50% to 100% of
+			// the whole experiment, hence `0.5 +` and `/2.0`.
+			percentage := 0.5 + elapsed/paramMaxRuntimeUpperBound/2.0
+			speed := float64(count) * 8.0 / elapsed
+			message := fmt.Sprintf("   upload: speed %s", humanizex.SI(
+				float64(speed), "bit/s"))
+			tk.Summary.Upload = speed / 1e03 /* bit/s => kbit/s */
+			callbacks.OnProgress(percentage, message)
+			tk.Upload = append(tk.Upload, Measurement{
+				AppInfo: &AppInfo{
+					ElapsedTime: int64(timediff / time.Microsecond),
+					NumBytes:    count,
+				},
+				Origin: "client",
+				Test:   "upload",
+			})
+		},
+	)
+	if err := mgr.run(ctx); err != nil && err.Error() != "generic_timeout_error" {
+		sess.Logger().Warnf("upload: %s", err)
+	}
+	return nil // failure is only when we cannot connect
+}
+
+// Run implements ExperimentMeasurer.Run.
+func (m *Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	tk := new(TestKeys)
+	tk.Protocol = 7
+	measurement.TestKeys = tk
+	locateResult, err := m.discover(ctx, sess)
+	if err != nil {
+		tk.Failure = failureFromError(err)
+		return err
+	}
+	tk.Server = ServerInfo{
+		Hostname: locateResult.Hostname,
+		Site:     locateResult.Site,
+	}
+	callbacks.OnProgress(0, fmt.Sprintf(" download: url: %s", locateResult.WSSDownloadURL))
+	if m.preDownloadHook != nil {
+		m.preDownloadHook()
+	}
+	if err := m.doDownload(ctx, sess, callbacks, tk, locateResult.WSSDownloadURL); err != nil {
+		tk.Failure = failureFromError(err)
+		return err
+	}
+	callbacks.OnProgress(0.5, fmt.Sprintf("   upload: url: %s", locateResult.WSSUploadURL))
+	if m.preUploadHook != nil {
+		m.preUploadHook()
+	}
+	if err := m.doUpload(ctx, sess, callbacks, tk, locateResult.WSSUploadURL); err != nil {
+		tk.Failure = failureFromError(err)
+		return err
+	}
+	return nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return &Measurer{config: config, jsonUnmarshal: json.Unmarshal}
+}
+
+func failureFromError(err error) (failure *string) {
+	if err != nil {
+		s := err.Error()
+		failure = &s
+	}
+	return
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	Upload         float64 `json:"upload"`
+	Download       float64 `json:"download"`
+	Ping           float64 `json:"ping"`
+	MaxRTT         float64 `json:"max_rtt"`
+	AvgRTT         float64 `json:"avg_rtt"`
+	MinRTT         float64 `json:"min_rtt"`
+	MSS            float64 `json:"mss"`
+	RetransmitRate float64 `json:"retransmit_rate"`
+	IsAnomaly      bool    `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	sk.Upload = tk.Summary.Upload
+	sk.Download = tk.Summary.Download
+	sk.Ping = tk.Summary.Ping
+	sk.MaxRTT = tk.Summary.MaxRTT
+	sk.AvgRTT = tk.Summary.AvgRTT
+	sk.MinRTT = tk.Summary.MinRTT
+	sk.MSS = float64(tk.Summary.MSS)
+	sk.RetransmitRate = tk.Summary.RetransmitRate
+	return sk, nil
+}

internal/engine/experiment/ndt7/ndt7_test.go

@@ -0,0 +1,250 @@
+package ndt7
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	if measurer.ExperimentName() != "ndt" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.8.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestDiscoverCancelledContext(t *testing.T) {
+	m := new(Measurer)
+	sess := &mockable.Session{
+		MockableHTTPClient: http.DefaultClient,
+		MockableLogger:     log.Log,
+		MockableUserAgent:  "miniooni/0.1.0-dev",
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel
+	locateResult, err := m.discover(ctx, sess)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if locateResult.Hostname != "" {
+		t.Fatal("not the Hostname we expected")
+	}
+}
+
+type verifyRequestTransport struct {
+	ExpectedError error
+}
+
+func (txp *verifyRequestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.URL.RawQuery != "ip=1.2.3.4" {
+		return nil, errors.New("invalid req.URL.RawQuery")
+	}
+	return nil, txp.ExpectedError
+}
+
+func TestDoDownloadWithCancelledContext(t *testing.T) {
+	m := new(Measurer)
+	sess := &mockable.Session{
+		MockableHTTPClient: http.DefaultClient,
+		MockableLogger:     log.Log,
+		MockableUserAgent:  "miniooni/0.1.0-dev",
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel
+	err := m.doDownload(
+		ctx, sess, model.NewPrinterCallbacks(log.Log), new(TestKeys),
+		"ws://host.name")
+	if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDoUploadWithCancelledContext(t *testing.T) {
+	m := new(Measurer)
+	sess := &mockable.Session{
+		MockableHTTPClient: http.DefaultClient,
+		MockableLogger:     log.Log,
+		MockableUserAgent:  "miniooni/0.1.0-dev",
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel
+	err := m.doUpload(
+		ctx, sess, model.NewPrinterCallbacks(log.Log), new(TestKeys),
+		"ws://host.name")
+	if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunWithCancelledContext(t *testing.T) {
+	m := new(Measurer)
+	sess := &mockable.Session{
+		MockableHTTPClient: http.DefaultClient,
+		MockableLogger:     log.Log,
+		MockableUserAgent:  "miniooni/0.1.0-dev",
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel
+	err := m.Run(ctx, sess, new(model.Measurement), model.NewPrinterCallbacks(log.Log))
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestGood(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurement := new(model.Measurement)
+	measurer := NewExperimentMeasurer(Config{})
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestFailDownload(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	measurer := NewExperimentMeasurer(Config{}).(*Measurer)
+	measurer.preDownloadHook = func() {
+		cancel()
+	}
+	err := measurer.Run(
+		ctx,
+		&mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
+		t.Fatal(err)
+	}
+}
+
+func TestFailUpload(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	measurer := NewExperimentMeasurer(Config{noDownload: true}).(*Measurer)
+	measurer.preUploadHook = func() {
+		cancel()
+	}
+	err := measurer.Run(
+		ctx,
+		&mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
+		t.Fatal(err)
+	}
+}
+
+func TestDownloadJSONUnmarshalFail(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{noUpload: true}).(*Measurer)
+	var seenError bool
+	expected := errors.New("expected error")
+	measurer.jsonUnmarshal = func(data []byte, v interface{}) error {
+		seenError = true
+		return expected
+	}
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !seenError {
+		t.Fatal("did not see expected error")
+	}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysGood(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &TestKeys{Summary: Summary{
+		RetransmitRate: 1,
+		MSS:            2,
+		MinRTT:         3,
+		AvgRTT:         4,
+		MaxRTT:         5,
+		Ping:           6,
+		Download:       7,
+		Upload:         8,
+	}}}
+	m := &Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(SummaryKeys)
+	if sk.RetransmitRate != 1 {
+		t.Fatal("invalid retransmitRate")
+	}
+	if sk.MSS != 2 {
+		t.Fatal("invalid mss")
+	}
+	if sk.MinRTT != 3 {
+		t.Fatal("invalid minRTT")
+	}
+	if sk.AvgRTT != 4 {
+		t.Fatal("invalid minRTT")
+	}
+	if sk.MaxRTT != 5 {
+		t.Fatal("invalid minRTT")
+	}
+	if sk.Ping != 6 {
+		t.Fatal("invalid minRTT")
+	}
+	if sk.Download != 7 {
+		t.Fatal("invalid minRTT")
+	}
+	if sk.Upload != 8 {
+		t.Fatal("invalid minRTT")
+	}
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/ndt7/param.go

@@ -0,0 +1,14 @@
+package ndt7
+
+import "time"
+
+const (
+	paramFractionForScaling   = 16
+	paramMinMessageSize       = 1 << 10
+	paramMaxBufferSize        = 1 << 20
+	paramMaxScaledMessageSize = 1 << 20
+	paramMaxMessageSize       = 1 << 24
+	paramMaxRuntimeUpperBound = 15.0 // seconds
+	paramMaxRuntime           = 10 * time.Second
+	paramMeasureInterval      = 250 * time.Millisecond
+)

internal/engine/experiment/ndt7/spec.go

@@ -0,0 +1,177 @@
+package ndt7
+
+// This file vendors data structures from the following repositories:
+//
+// - github.com/m-lab/ndt7-client-go
+// - github.com/m-lab/ndt-server
+// - github.com/m-lab/tcp-info
+//
+// It is available under the Apache License v2.0.
+//
+// Because m-lab uses mainly Linux as a development platform, they may
+// unwillingly break our Windows builds. Also, they use lots of depdencies
+// that we don't actually need. Hence, vendoring FTW.
+//
+// The data structures are supposed to stay constant in time or to not
+// change dramatically, hence this vendoring shouldn't be too bad.
+
+type (
+	// OriginKind indicates the origin of a measurement.
+	OriginKind string
+
+	// TestKind indicates the direction of a measurement.
+	TestKind string
+)
+
+const (
+	// OriginClient indicates that the measurement origin is the client.
+	OriginClient = OriginKind("client")
+
+	// OriginServer indicates that the measurement origin is the server.
+	OriginServer = OriginKind("server")
+
+	// TestDownload indicates that this is a download.
+	TestDownload = TestKind("download")
+
+	// TestUpload indicates that this is an upload.
+	TestUpload = TestKind("upload")
+)
+
+// LinuxTCPInfo is the linux defined structure returned in RouteAttr DIAG_INFO messages.
+// It corresponds to the struct tcp_info in include/uapi/linux/tcp.h
+type LinuxTCPInfo struct {
+	State       uint8 `csv:"TCP.State"`
+	CAState     uint8 `csv:"TCP.CAState"`
+	Retransmits uint8 `csv:"TCP.Retransmits"`
+	Probes      uint8 `csv:"TCP.Probes"`
+	Backoff     uint8 `csv:"TCP.Backoff"`
+	Options     uint8 `csv:"TCP.Options"`
+	WScale      uint8 `csv:"TCP.WScale"`     //snd_wscale : 4, tcpi_rcv_wscale : 4;
+	AppLimited  uint8 `csv:"TCP.AppLimited"` //delivery_rate_app_limited:1;
+
+	RTO    uint32 `csv:"TCP.RTO"` // offset 8
+	ATO    uint32 `csv:"TCP.ATO"`
+	SndMSS uint32 `csv:"TCP.SndMSS"`
+	RcvMSS uint32 `csv:"TCP.RcvMSS"`
+
+	Unacked uint32 `csv:"TCP.Unacked"` // offset 24
+	Sacked  uint32 `csv:"TCP.Sacked"`
+	Lost    uint32 `csv:"TCP.Lost"`
+	Retrans uint32 `csv:"TCP.Retrans"`
+	Fackets uint32 `csv:"TCP.Fackets"`
+
+	/* Times. */
+	// These seem to be elapsed time, so they increase on almost every sample.
+	// We can probably use them to get more info about intervals between samples.
+	LastDataSent uint32 `csv:"TCP.LastDataSent"` // offset 44
+	LastAckSent  uint32 `csv:"TCP.LastAckSent"`  /* Not remembered, sorry. */ // offset 48
+	LastDataRecv uint32 `csv:"TCP.LastDataRecv"` // offset 52
+	LastAckRecv  uint32 `csv:"TCP.LastDataRecv"` // offset 56
+
+	/* Metrics. */
+	PMTU        uint32 `csv:"TCP.PMTU"`
+	RcvSsThresh uint32 `csv:"TCP.RcvSsThresh"`
+	RTT         uint32 `csv:"TCP.RTT"`
+	RTTVar      uint32 `csv:"TCP.RTTVar"`
+	SndSsThresh uint32 `csv:"TCP.SndSsThresh"`
+	SndCwnd     uint32 `csv:"TCP.SndCwnd"`
+	AdvMSS      uint32 `csv:"TCP.AdvMSS"`
+	Reordering  uint32 `csv:"TCP.Reordering"`
+
+	RcvRTT   uint32 `csv:"TCP.RcvRTT"`
+	RcvSpace uint32 `csv:"TCP.RcvSpace"`
+
+	TotalRetrans uint32 `csv:"TCP.TotalRetrans"`
+
+	PacingRate    int64 `csv:"TCP.PacingRate"`    // This is often -1, so better for it to be signed
+	MaxPacingRate int64 `csv:"TCP.MaxPacingRate"` // This is often -1, so better to be signed.
+
+	// NOTE: In linux, these are uint64, but we make them int64 here for compatibility with BigQuery
+	BytesAcked    int64 `csv:"TCP.BytesAcked"`    /* RFC4898 tcpEStatsAppHCThruOctetsAcked */
+	BytesReceived int64 `csv:"TCP.BytesReceived"` /* RFC4898 tcpEStatsAppHCThruOctetsReceived */
+	SegsOut       int32 `csv:"TCP.SegsOut"`       /* RFC4898 tcpEStatsPerfSegsOut */
+	SegsIn        int32 `csv:"TCP.SegsIn"`        /* RFC4898 tcpEStatsPerfSegsIn */
+
+	NotsentBytes uint32 `csv:"TCP.NotsentBytes"`
+	MinRTT       uint32 `csv:"TCP.MinRTT"`
+	DataSegsIn   uint32 `csv:"TCP.DataSegsIn"`  /* RFC4898 tcpEStatsDataSegsIn */
+	DataSegsOut  uint32 `csv:"TCP.DataSegsOut"` /* RFC4898 tcpEStatsDataSegsOut */
+
+	// NOTE: In linux, this is uint64, but we make it int64 here for compatibility with BigQuery
+	DeliveryRate int64 `csv:"TCP.DeliveryRate"`
+
+	BusyTime      int64 `csv:"TCP.BusyTime"`      /* Time (usec) busy sending data */
+	RWndLimited   int64 `csv:"TCP.RWndLimited"`   /* Time (usec) limited by receive window */
+	SndBufLimited int64 `csv:"TCP.SndBufLimited"` /* Time (usec) limited by send buffer */
+
+	Delivered   uint32 `csv:"TCP.Delivered"`
+	DeliveredCE uint32 `csv:"TCP.DeliveredCE"`
+
+	// NOTE: In linux, these are uint64, but we make them int64 here for compatibility with BigQuery
+	BytesSent    int64 `csv:"TCP.BytesSent"`    /* RFC4898 tcpEStatsPerfHCDataOctetsOut */
+	BytesRetrans int64 `csv:"TCP.BytesRetrans"` /* RFC4898 tcpEStatsPerfOctetsRetrans */
+
+	DSackDups uint32 `csv:"TCP.DSackDups"` /* RFC4898 tcpEStatsStackDSACKDups */
+	ReordSeen uint32 `csv:"TCP.ReordSeen"` /* reordering events seen */
+}
+
+// AppInfo contains an application level measurement. This structure is
+// described in the ndt7 specification.
+type AppInfo struct {
+	NumBytes    int64
+	ElapsedTime int64
+}
+
+// ConnectionInfo contains connection info. This structure is described
+// in the ndt7 specification.
+type ConnectionInfo struct {
+	Client string
+	Server string
+	UUID   string `json:",omitempty"`
+}
+
+// InetDiagBBRInfo implements the struct associated with INET_DIAG_BBRINFO attribute, corresponding with
+// linux struct tcp_bbr_info in uapi/linux/inet_diag.h.
+type InetDiagBBRInfo struct {
+	BW         int64  `csv:"BBR.BW"`         // Max-filtered BW (app throughput) estimate in bytes/second
+	MinRTT     uint32 `csv:"BBR.MinRTT"`     // Min-filtered RTT in uSec
+	PacingGain uint32 `csv:"BBR.PacingGain"` // Pacing gain shifted left 8 bits
+	CwndGain   uint32 `csv:"BBR.CwndGain"`   // Cwnd gain shifted left 8 bits
+}
+
+// The BBRInfo struct contains information measured using BBR. This structure is
+// an extension to the ndt7 specification. Variables here have the same
+// measurement unit that is used by the Linux kernel.
+type BBRInfo struct {
+	InetDiagBBRInfo
+	ElapsedTime int64
+}
+
+// The TCPInfo struct contains information measured using TCP_INFO. This
+// structure is described in the ndt7 specification.
+type TCPInfo struct {
+	LinuxTCPInfo
+	ElapsedTime int64
+}
+
+// The Measurement struct contains measurement results. This message is
+// an extension of the one inside of v0.9.0 of the ndt7 spec.
+type Measurement struct {
+	// AppInfo contains application level measurements.
+	AppInfo *AppInfo `json:",omitempty"`
+
+	// BBRInfo is the data measured using TCP BBR instrumentation.
+	BBRInfo *BBRInfo `json:",omitempty"`
+
+	// ConnectionInfo contains info on the connection.
+	ConnectionInfo *ConnectionInfo `json:",omitempty"`
+
+	// Origin indicates who performed this measurement.
+	Origin OriginKind `json:",omitempty"`
+
+	// Test contains the test name.
+	Test TestKind `json:",omitempty"`
+
+	// TCPInfo contains metrics measured using TCP_INFO instrumentation.
+	TCPInfo *TCPInfo `json:",omitempty"`
+}

internal/engine/experiment/ndt7/upload.go

@@ -0,0 +1,75 @@
+package ndt7
+
+import (
+	"context"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+func newMessage(n int) (*websocket.PreparedMessage, error) {
+	return websocket.NewPreparedMessage(websocket.BinaryMessage, make([]byte, n))
+}
+
+type uploadManager struct {
+	conn                 mockableConn
+	fractionForScaling   int64
+	maxRuntime           time.Duration
+	maxMessageSize       int
+	maxScaledMessageSize int
+	measureInterval      time.Duration
+	minMessageSize       int
+	newMessage           func(int) (*websocket.PreparedMessage, error)
+	onPerformance        callbackPerformance
+}
+
+func newUploadManager(
+	conn mockableConn, onPerformance callbackPerformance,
+) uploadManager {
+	return uploadManager{
+		conn:                 conn,
+		fractionForScaling:   paramFractionForScaling,
+		maxRuntime:           paramMaxRuntime,
+		maxMessageSize:       paramMaxMessageSize,
+		maxScaledMessageSize: paramMaxScaledMessageSize,
+		measureInterval:      paramMeasureInterval,
+		minMessageSize:       paramMinMessageSize,
+		newMessage:           newMessage,
+		onPerformance:        onPerformance,
+	}
+}
+
+func (mgr uploadManager) run(ctx context.Context) error {
+	var total int64
+	start := time.Now()
+	if err := mgr.conn.SetWriteDeadline(time.Now().Add(mgr.maxRuntime)); err != nil {
+		return err
+	}
+	size := mgr.minMessageSize
+	message, err := mgr.newMessage(size)
+	if err != nil {
+		return err
+	}
+	ticker := time.NewTicker(mgr.measureInterval)
+	defer ticker.Stop()
+	for ctx.Err() == nil {
+		if err := mgr.conn.WritePreparedMessage(message); err != nil {
+			return err
+		}
+		total += int64(size)
+		select {
+		case now := <-ticker.C:
+			mgr.onPerformance(now.Sub(start), total)
+		default:
+			// NOTHING
+		}
+		if size >= mgr.maxScaledMessageSize || int64(size) >= (total/mgr.fractionForScaling) {
+			continue
+		}
+		size <<= 1
+		if message, err = mgr.newMessage(size); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/engine/experiment/ndt7/upload_test.go

@@ -0,0 +1,89 @@
+package ndt7
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+func TestUploadSetWriteDeadlineFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	mgr := newUploadManager(
+		&mockableConnMock{
+			WriteDeadlineErr: expected,
+		},
+		defaultCallbackPerformance,
+	)
+	err := mgr.run(context.Background())
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestUploadNewMessageFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	mgr := newUploadManager(
+		&mockableConnMock{},
+		defaultCallbackPerformance,
+	)
+	mgr.newMessage = func(int) (*websocket.PreparedMessage, error) {
+		return nil, expected
+	}
+	err := mgr.run(context.Background())
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestUploadWritePreparedMessageFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	mgr := newUploadManager(
+		&mockableConnMock{
+			WritePreparedMessageErr: expected,
+		},
+		defaultCallbackPerformance,
+	)
+	err := mgr.run(context.Background())
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestUploadWritePreparedMessageSubsequentFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	mgr := newUploadManager(
+		&mockableConnMock{},
+		defaultCallbackPerformance,
+	)
+	var already bool
+	mgr.newMessage = func(int) (*websocket.PreparedMessage, error) {
+		if !already {
+			already = true
+			return new(websocket.PreparedMessage), nil
+		}
+		return nil, expected
+	}
+	err := mgr.run(context.Background())
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestUploadLoop(t *testing.T) {
+	mgr := newUploadManager(
+		&mockableConnMock{},
+		defaultCallbackPerformance,
+	)
+	mgr.newMessage = func(int) (*websocket.PreparedMessage, error) {
+		return new(websocket.PreparedMessage), nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+	err := mgr.run(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

internal/engine/experiment/psiphon/psiphon.go

@@ -0,0 +1,133 @@
+// Package psiphon implements the psiphon network experiment. This
+// implements, in particular, v0.2.0 of the spec.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-015-psiphon.md
+package psiphon
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+const (
+	testName    = "psiphon"
+	testVersion = "0.5.0"
+)
+
+// Config contains the experiment's configuration.
+type Config struct{}
+
+// TestKeys contains the experiment's result.
+type TestKeys struct {
+	urlgetter.TestKeys
+	MaxRuntime float64 `json:"max_runtime"`
+}
+
+// Measurer is the psiphon measurer.
+type Measurer struct {
+	BeforeGetHook func(g urlgetter.Getter)
+	Config        Config
+}
+
+// ExperimentName returns the experiment name
+func (m *Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion returns the experiment version
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+func (m *Measurer) printprogress(
+	ctx context.Context, wg *sync.WaitGroup,
+	maxruntime int, callbacks model.ExperimentCallbacks,
+) {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	step := 1 / float64(maxruntime)
+	var progress float64
+	defer callbacks.OnProgress(1.0, "psiphon experiment complete")
+	defer wg.Done()
+	for {
+		select {
+		case <-ticker.C:
+			progress += step
+			callbacks.OnProgress(progress, "psiphon experiment running")
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// Run runs the measurement
+func (m *Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	const maxruntime = 60
+	ctx, cancel := context.WithTimeout(ctx, maxruntime*time.Second)
+	var (
+		wg     sync.WaitGroup
+		config urlgetter.Config
+	)
+	wg.Add(1)
+	go m.printprogress(ctx, &wg, maxruntime, callbacks)
+	config.Tunnel = "psiphon" // force to use psiphon tunnel
+	urlgetter.RegisterExtensions(measurement)
+	target := "https://www.google.com/humans.txt"
+	if measurement.Input != "" {
+		target = string(measurement.Input)
+	}
+	g := urlgetter.Getter{
+		Config:  config,
+		Session: sess,
+		Target:  target,
+	}
+	if m.BeforeGetHook != nil {
+		m.BeforeGetHook(g)
+	}
+	tk, err := g.Get(ctx)
+	cancel()
+	wg.Wait()
+	measurement.TestKeys = &TestKeys{
+		TestKeys:   tk,
+		MaxRuntime: maxruntime,
+	}
+	return err
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return &Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	BootstrapTime float64 `json:"bootstrap_time"`
+	Failure       string  `json:"failure"`
+	IsAnomaly     bool    `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	if tk.Failure != nil {
+		sk.Failure = *tk.Failure
+		sk.IsAnomaly = true
+	}
+	sk.BootstrapTime = tk.BootstrapTime
+	return sk, nil
+}

internal/engine/experiment/psiphon/psiphon_test.go

@@ -0,0 +1,163 @@
+package psiphon_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/psiphon"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// Implementation note: integration test performed by
+// the $topdir/experiment_test.go file
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := psiphon.NewExperimentMeasurer(psiphon.Config{})
+	if measurer.ExperimentName() != "psiphon" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.5.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestRunWithCancelledContext(t *testing.T) {
+	measurer := psiphon.NewExperimentMeasurer(psiphon.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	measurement := new(model.Measurement)
+	err := measurer.Run(ctx, newfakesession(), measurement,
+		model.NewPrinterCallbacks(log.Log))
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("expected another error here")
+	}
+	tk := measurement.TestKeys.(*psiphon.TestKeys)
+	if tk.MaxRuntime <= 0 {
+		t.Fatal("you did not set the max runtime")
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(psiphon.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestRunWithCustomInputAndCancelledContext(t *testing.T) {
+	expected := "http://x.org"
+	measurement := &model.Measurement{
+		Input: model.MeasurementTarget(expected),
+	}
+	measurer := psiphon.NewExperimentMeasurer(psiphon.Config{})
+	measurer.(*psiphon.Measurer).BeforeGetHook = func(g urlgetter.Getter) {
+		if g.Target != expected {
+			t.Fatal("target was not correctly set")
+		}
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	err := measurer.Run(ctx, newfakesession(), measurement,
+		model.NewPrinterCallbacks(log.Log))
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("expected another error here")
+	}
+	tk := measurement.TestKeys.(*psiphon.TestKeys)
+	if tk.MaxRuntime <= 0 {
+		t.Fatal("you did not set the max runtime")
+	}
+}
+
+func TestRunWillPrintSomethingWithCancelledContext(t *testing.T) {
+	measurement := new(model.Measurement)
+	measurer := psiphon.NewExperimentMeasurer(psiphon.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	measurer.(*psiphon.Measurer).BeforeGetHook = func(g urlgetter.Getter) {
+		time.Sleep(2 * time.Second)
+		cancel() // fail after we've given the printer a chance to run
+	}
+	observer := observerCallbacks{progress: atomicx.NewInt64()}
+	err := measurer.Run(ctx, newfakesession(), measurement, observer)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("expected another error here")
+	}
+	tk := measurement.TestKeys.(*psiphon.TestKeys)
+	if tk.MaxRuntime <= 0 {
+		t.Fatal("you did not set the max runtime")
+	}
+	if observer.progress.Load() < 2 {
+		t.Fatal("not enough progress emitted?!")
+	}
+}
+
+type observerCallbacks struct {
+	progress *atomicx.Int64
+}
+
+func (d observerCallbacks) OnProgress(percentage float64, message string) {
+	d.progress.Add(1)
+}
+
+func newfakesession() model.ExperimentSession {
+	return &mockable.Session{MockableLogger: log.Log}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &psiphon.Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysGood(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &psiphon.TestKeys{TestKeys: urlgetter.TestKeys{
+		BootstrapTime: 123,
+	}}}
+	m := &psiphon.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(psiphon.SummaryKeys)
+	if sk.BootstrapTime != 123 {
+		t.Fatal("invalid latency")
+	}
+	if sk.Failure != "" {
+		t.Fatal("invalid failure")
+	}
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}
+
+func TestSummaryKeysFailure(t *testing.T) {
+	expected := io.EOF.Error()
+	measurement := &model.Measurement{TestKeys: &psiphon.TestKeys{TestKeys: urlgetter.TestKeys{
+		BootstrapTime: 123,
+		Failure:       &expected,
+	}}}
+	m := &psiphon.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(psiphon.SummaryKeys)
+	if sk.BootstrapTime != 123 {
+		t.Fatal("invalid latency")
+	}
+	if sk.Failure != expected {
+		t.Fatal("invalid failure")
+	}
+	if sk.IsAnomaly == false {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/riseupvpn/riseupvpn.go

@@ -0,0 +1,309 @@
+// Package riseupvpn contains the RiseupVPN network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-026-riseupvpn.md
+package riseupvpn
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+)
+
+const (
+	testName      = "riseupvpn"
+	testVersion   = "0.1.0"
+	eipServiceURL = "https://api.black.riseup.net:443/3/config/eip-service.json"
+	providerURL   = "https://riseup.net/provider.json"
+	geoServiceURL = "https://api.black.riseup.net:9001/json"
+	tcpConnect    = "tcpconnect://"
+)
+
+// EipService main json object of eip-service.json
+type EipService struct {
+	Gateways []GatewayV3
+}
+
+// GatewayV3 json obj Version 3
+type GatewayV3 struct {
+	Capabilities struct {
+		Transport []TransportV3
+	}
+	Host      string
+	IPAddress string `json:"ip_address"`
+}
+
+// TransportV3 json obj Version 3
+type TransportV3 struct {
+	Type      string
+	Protocols []string
+	Ports     []string
+	Options   map[string]string
+}
+
+// GatewayConnection describes the connection to a riseupvpn gateway
+type GatewayConnection struct {
+	IP            string `json:"ip"`
+	Port          int    `json:"port"`
+	TransportType string `json:"transport_type"`
+}
+
+// Config contains the riseupvpn experiment config.
+type Config struct {
+	urlgetter.Config
+}
+
+// TestKeys contains riseupvpn test keys.
+type TestKeys struct {
+	urlgetter.TestKeys
+	APIFailure      *string             `json:"api_failure"`
+	APIStatus       string              `json:"api_status"`
+	CACertStatus    bool                `json:"ca_cert_status"`
+	FailingGateways []GatewayConnection `json:"failing_gateways"`
+}
+
+// NewTestKeys creates new riseupvpn TestKeys.
+func NewTestKeys() *TestKeys {
+	return &TestKeys{
+		APIFailure:      nil,
+		APIStatus:       "ok",
+		CACertStatus:    true,
+		FailingGateways: nil,
+	}
+}
+
+// UpdateProviderAPITestKeys updates the TestKeys using the given MultiOutput result.
+func (tk *TestKeys) UpdateProviderAPITestKeys(v urlgetter.MultiOutput) {
+	tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
+	tk.Queries = append(tk.Queries, v.TestKeys.Queries...)
+	tk.Requests = append(tk.Requests, v.TestKeys.Requests...)
+	tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
+	tk.TLSHandshakes = append(tk.TLSHandshakes, v.TestKeys.TLSHandshakes...)
+	if tk.APIStatus != "ok" {
+		return // we already flipped the state
+	}
+	if v.TestKeys.Failure != nil {
+		tk.APIStatus = "blocked"
+		tk.APIFailure = v.TestKeys.Failure
+		return
+	}
+}
+
+// AddGatewayConnectTestKeys updates the TestKeys using the given MultiOutput result of gateway connectivity testing.
+func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transportType string) {
+	tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
+	tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
+	for _, tcpConnect := range v.TestKeys.TCPConnect {
+		if !tcpConnect.Status.Success {
+			gatewayConnection := newGatewayConnection(tcpConnect, transportType)
+			tk.FailingGateways = append(tk.FailingGateways, *gatewayConnection)
+		}
+	}
+	return
+}
+
+func newGatewayConnection(tcpConnect archival.TCPConnectEntry, transportType string) *GatewayConnection {
+	return &GatewayConnection{
+		IP:            tcpConnect.IP,
+		Port:          tcpConnect.Port,
+		TransportType: transportType,
+	}
+}
+
+// AddCACertFetchTestKeys Adding generic urlgetter.Get() testKeys to riseupvpn specific test keys
+func (tk *TestKeys) AddCACertFetchTestKeys(testKeys urlgetter.TestKeys) {
+	tk.NetworkEvents = append(tk.NetworkEvents, testKeys.NetworkEvents...)
+	tk.Queries = append(tk.Queries, testKeys.Queries...)
+	tk.Requests = append(tk.Requests, testKeys.Requests...)
+	tk.TCPConnect = append(tk.TCPConnect, testKeys.TCPConnect...)
+	tk.TLSHandshakes = append(tk.TLSHandshakes, testKeys.TLSHandshakes...)
+	if testKeys.Failure != nil {
+		tk.APIStatus = "blocked"
+		tk.APIFailure = tk.Failure
+		tk.CACertStatus = false
+	}
+}
+
+// Measurer performs the measurement
+type Measurer struct {
+	// Config contains the experiment settings. If empty we
+	// will be using default settings.
+	Config Config
+
+	// Getter is an optional getter to be used for testing.
+	Getter urlgetter.MultiGetter
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run
+func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
+	ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
+	defer cancel()
+	testkeys := NewTestKeys()
+	measurement.TestKeys = testkeys
+	urlgetter.RegisterExtensions(measurement)
+
+	caTarget := "https://black.riseup.net/ca.crt"
+	caGetter := urlgetter.Getter{
+		Config:  m.Config.Config,
+		Session: sess,
+		Target:  caTarget,
+	}
+	log.Info("Getting CA certificate; please be patient...")
+	tk, err := caGetter.Get(ctx)
+	testkeys.AddCACertFetchTestKeys(tk)
+
+	if err != nil {
+		log.Error("Getting CA certificate failed. Aborting test.")
+		return nil
+	}
+
+	certPool := netx.NewDefaultCertPool()
+	if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok {
+		testkeys.CACertStatus = false
+		testkeys.APIStatus = "blocked"
+		errorValue := "invalid_ca"
+		testkeys.APIFailure = &errorValue
+		return nil
+	}
+
+	inputs := []urlgetter.MultiInput{
+
+		// Here we need to provide the method explicitly. See
+		// https://github.com/ooni/probe-cli/v3/internal/engine/issues/827.
+		{Target: providerURL, Config: urlgetter.Config{
+			CertPool:        certPool,
+			Method:          "GET",
+			FailOnHTTPError: true,
+		}},
+		{Target: eipServiceURL, Config: urlgetter.Config{
+			CertPool:        certPool,
+			Method:          "GET",
+			FailOnHTTPError: true,
+		}},
+		{Target: geoServiceURL, Config: urlgetter.Config{
+			CertPool:        certPool,
+			Method:          "GET",
+			FailOnHTTPError: true,
+		}},
+	}
+	multi := urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
+
+	for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) {
+		testkeys.UpdateProviderAPITestKeys(entry)
+	}
+
+	// test gateways now
+	gateways := parseGateways(testkeys)
+	openvpnEndpoints := generateMultiInputs(gateways, "openvpn")
+	obfs4Endpoints := generateMultiInputs(gateways, "obfs4")
+	overallCount := len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints)
+
+	// measure openvpn in parallel
+	multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
+	for entry := range multi.CollectOverall(ctx, openvpnEndpoints, len(inputs), overallCount, "riseupvpn", callbacks) {
+		testkeys.AddGatewayConnectTestKeys(entry, "openvpn")
+	}
+
+	// measure obfs4 in parallel
+	multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
+	for entry := range multi.CollectOverall(ctx, obfs4Endpoints, len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) {
+		testkeys.AddGatewayConnectTestKeys(entry, "obfs4")
+	}
+
+	return nil
+}
+
+func generateMultiInputs(gateways []GatewayV3, transportType string) []urlgetter.MultiInput {
+	var gatewayInputs []urlgetter.MultiInput
+	for _, gateway := range gateways {
+		for _, transport := range gateway.Capabilities.Transport {
+			if transport.Type != transportType {
+				continue
+			}
+			supportsTCP := false
+			for _, protocol := range transport.Protocols {
+				if protocol == "tcp" {
+					supportsTCP = true
+				}
+			}
+			if !supportsTCP {
+				continue
+			}
+			for _, port := range transport.Ports {
+				tcpConnection := tcpConnect + gateway.IPAddress + ":" + port
+				gatewayInputs = append(gatewayInputs, urlgetter.MultiInput{Target: tcpConnection})
+			}
+		}
+	}
+	return gatewayInputs
+}
+
+func parseGateways(testKeys *TestKeys) []GatewayV3 {
+	for _, requestEntry := range testKeys.Requests {
+		if requestEntry.Request.URL == eipServiceURL && requestEntry.Failure == nil {
+			eipService, err := DecodeEIP3(requestEntry.Response.Body.Value)
+			if err == nil {
+				return eipService.Gateways
+			}
+		}
+	}
+	return nil
+}
+
+// DecodeEIP3 decodes eip-service.json version 3
+func DecodeEIP3(body string) (*EipService, error) {
+	var eip EipService
+	err := json.Unmarshal([]byte(body), &eip)
+	if err != nil {
+		return nil, err
+	}
+	return &eip, nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	APIBlocked      bool `json:"api_blocked"`
+	ValidCACert     bool `json:"valid_ca_cert"`
+	FailingGateways int  `json:"failing_gateways"`
+	IsAnomaly       bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	sk.APIBlocked = tk.APIStatus != "ok"
+	sk.ValidCACert = tk.CACertStatus
+	sk.FailingGateways = len(tk.FailingGateways)
+	sk.IsAnomaly = (sk.APIBlocked == true || tk.CACertStatus == false ||
+		sk.FailingGateways != 0)
+	return sk, nil
+}

internal/engine/experiment/riseupvpn/riseupvpn_test.go

@@ -0,0 +1,497 @@
+package riseupvpn_test
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	if measurer.ExperimentName() != "riseupvpn" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.1.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestGood(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.Agent != "" {
+		t.Fatal("unexpected Agent: " + tk.Agent)
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("unexpected FailedOperation")
+	}
+	if tk.Failure != nil {
+		t.Fatal("unexpected Failure")
+	}
+	if len(tk.NetworkEvents) <= 0 {
+		t.Fatal("no NetworkEvents?!")
+	}
+	if len(tk.Queries) <= 0 {
+		t.Fatal("no Queries?!")
+	}
+	if len(tk.Requests) <= 0 {
+		t.Fatal("no Requests?!")
+	}
+	if len(tk.TCPConnect) <= 0 {
+		t.Fatal("no TCPConnect?!")
+	}
+	if len(tk.TLSHandshakes) <= 0 {
+		t.Fatal("no TLSHandshakes?!")
+	}
+	if tk.APIFailure != nil {
+		t.Fatal("unexpected ApiFailure")
+	}
+	if tk.APIStatus != "ok" {
+		t.Fatal("unexpected ApiStatus")
+	}
+	if tk.CACertStatus != true {
+		t.Fatal("unexpected CaCertStatus")
+	}
+	if tk.FailingGateways != nil {
+		t.Fatal("unexpected FailingGateways value")
+	}
+}
+
+// TestUpdateWithMixedResults tests if one operation failed
+// ApiStatus is considered as blocked
+func TestUpdateWithMixedResults(t *testing.T) {
+	tk := riseupvpn.NewTestKeys()
+	tk.UpdateProviderAPITestKeys(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "https://api.black.riseup.net:443/3/config/eip-service.json",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseStatus: 200,
+		},
+	})
+	tk.UpdateProviderAPITestKeys(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "https://riseup.net/provider.json",
+		},
+		TestKeys: urlgetter.TestKeys{
+			FailedOperation: (func() *string {
+				s := errorx.HTTPRoundTripOperation
+				return &s
+			})(),
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	tk.UpdateProviderAPITestKeys(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "https://api.black.riseup.net:9001/json",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseStatus: 200,
+		},
+	})
+	if tk.APIStatus != "blocked" {
+		t.Fatal("ApiStatus should be blocked")
+	}
+	if *tk.APIFailure != errorx.FailureEOFError {
+		t.Fatal("invalid ApiFailure")
+	}
+}
+
+func TestFailureCaCertFetch(t *testing.T) {
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	// we're cancelling immediately so that the CA Cert fetch fails
+	cancel()
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus != false {
+		t.Fatal("invalid CACertStatus ")
+	}
+	if tk.APIStatus != "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure != nil {
+		t.Fatal("ApiFailure should be null")
+	}
+	if len(tk.Requests) > 1 {
+		t.Fatal("Unexpected requests")
+	}
+}
+
+func TestFailureEipServiceBlocked(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	selfcensor.Enable(`{"PoisonSystemDNS":{"api.black.riseup.net":["NXDOMAIN"]}}`)
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus != true {
+		t.Fatal("invalid CACertStatus ")
+	}
+
+	for _, entry := range tk.Requests {
+		if entry.Request.URL == "https://api.black.riseup.net:443/3/config/eip-service.json" {
+			if entry.Failure == nil {
+				t.Fatal("Failure for " + entry.Request.URL + " should not be null")
+			}
+		}
+	}
+
+	if tk.APIStatus != "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure == nil {
+		t.Fatal("ApiFailure should not be null")
+	}
+}
+
+func TestFailureProviderUrlBlocked(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.70:443":"REJECT"}}`)
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+
+	for _, entry := range tk.Requests {
+		if entry.Request.URL == "https://riseup.net/provider.json" {
+			if entry.Failure == nil {
+				t.Fatal("Failure for " + entry.Request.URL + " should not be null")
+			}
+		}
+	}
+
+	if tk.CACertStatus != true {
+		t.Fatal("invalid CACertStatus ")
+	}
+	if tk.APIStatus != "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure == nil {
+		t.Fatal("ApiFailure should not be null")
+	}
+}
+
+func TestFailureGeoIpServiceBlocked(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.107:9001":"REJECT"}}`)
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus != true {
+		t.Fatal("invalid CACertStatus ")
+	}
+
+	for _, entry := range tk.Requests {
+		if entry.Request.URL == "https://api.black.riseup.net:9001/json" {
+			if entry.Failure == nil {
+				t.Fatal("Failure for " + entry.Request.URL + " should not be null")
+			}
+		}
+	}
+
+	if tk.APIStatus != "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure == nil {
+		t.Fatal("ApiFailure should not be null")
+	}
+}
+
+func TestFailureGateway(t *testing.T) {
+	var testCases = [...]string{"openvpn", "obfs4"}
+	eipService, err := fetchEipService()
+	if err != nil {
+		t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error())
+		t.SkipNow()
+	}
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("testing censored transport %s", tc), func(t *testing.T) {
+			censoredGateway, err := selfCensorRandomGateway(eipService, tc)
+			if err == nil {
+				censorString := `{"BlockedEndpoints":{"` + censoredGateway.IP + `:` + censoredGateway.Port + `":"REJECT"}}`
+				selfcensor.Enable(censorString)
+			} else {
+				t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error())
+				t.SkipNow()
+			}
+
+			// - run measurement
+			runGatewayTest(t, censoredGateway)
+		})
+	}
+}
+
+type SelfCensoredGateway struct {
+	IP   string
+	Port string
+}
+
+func fetchEipService() (*riseupvpn.EipService, error) {
+	// - fetch client cert and add to certpool
+	caFetchClient := &http.Client{
+		Timeout: time.Second * 30,
+	}
+
+	caCertResponse, err := caFetchClient.Get("https://black.riseup.net/ca.crt")
+	if err != nil {
+		return nil, err
+	}
+
+	var bodyString string
+
+	if caCertResponse.StatusCode != http.StatusOK {
+		return nil, errors.New("unexpected HTTP response code")
+	}
+	bodyBytes, err := ioutil.ReadAll(caCertResponse.Body)
+	defer caCertResponse.Body.Close()
+
+	if err != nil {
+		return nil, err
+	}
+	bodyString = string(bodyBytes)
+
+	certs := x509.NewCertPool()
+	certs.AppendCertsFromPEM([]byte(bodyString))
+
+	// - fetch and parse eip-service.json
+	client := &http.Client{
+		Timeout: time.Second * 30,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: certs,
+			},
+		},
+	}
+
+	eipResponse, err := client.Get("https://api.black.riseup.net/3/config/eip-service.json")
+	if err != nil {
+		return nil, err
+	}
+	if eipResponse.StatusCode != http.StatusOK {
+		return nil, errors.New("Unexpected HTTP response code")
+	}
+
+	bodyBytes, err = ioutil.ReadAll(eipResponse.Body)
+	defer eipResponse.Body.Close()
+	if err != nil {
+		return nil, err
+	}
+	bodyString = string(bodyBytes)
+
+	eipService, err := riseupvpn.DecodeEIP3(bodyString)
+	if err != nil {
+		return nil, err
+	}
+	return eipService, nil
+}
+
+func selfCensorRandomGateway(eipService *riseupvpn.EipService, transportType string) (*SelfCensoredGateway, error) {
+
+	// - self censor random gateway
+	gateways := eipService.Gateways
+	if gateways == nil || len(gateways) == 0 {
+		return nil, errors.New("No gateways found")
+	}
+
+	var selfcensoredGateways []SelfCensoredGateway
+	for _, gateway := range gateways {
+		for _, transport := range gateway.Capabilities.Transport {
+			if transport.Type == transportType {
+				selfcensoredGateways = append(selfcensoredGateways, SelfCensoredGateway{IP: gateway.IPAddress, Port: transport.Ports[0]})
+			}
+		}
+	}
+
+	if len(selfcensoredGateways) == 0 {
+		return nil, errors.New("transport " + transportType + " doesn't seem to be supported.")
+	}
+
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+	min := 0
+	max := len(selfcensoredGateways) - 1
+	randomIndex := rnd.Intn(max-min+1) + min
+	return &selfcensoredGateways[randomIndex], nil
+
+}
+
+func runGatewayTest(t *testing.T, censoredGateway *SelfCensoredGateway) {
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus != true {
+		t.Fatal("invalid CACertStatus ")
+	}
+
+	if tk.FailingGateways == nil || len(tk.FailingGateways) != 1 {
+		t.Fatal("unexpected amount of failing gateways")
+	}
+
+	entry := tk.FailingGateways[0]
+	if entry.IP != censoredGateway.IP || fmt.Sprint(entry.Port) != censoredGateway.Port {
+		t.Fatal("unexpected failed gateway configuration")
+	}
+
+	if tk.APIStatus == "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure != nil {
+		t.Fatal("ApiFailure should be null")
+	}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &riseupvpn.Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysWorksAsIntended(t *testing.T) {
+	tests := []struct {
+		tk riseupvpn.TestKeys
+		sk riseupvpn.SummaryKeys
+	}{{
+		tk: riseupvpn.TestKeys{
+			APIStatus:       "blocked",
+			CACertStatus:    true,
+			FailingGateways: nil,
+		},
+		sk: riseupvpn.SummaryKeys{
+			APIBlocked:  true,
+			ValidCACert: true,
+			IsAnomaly:   true,
+		},
+	}, {
+		tk: riseupvpn.TestKeys{
+			APIStatus:       "ok",
+			CACertStatus:    false,
+			FailingGateways: nil,
+		},
+		sk: riseupvpn.SummaryKeys{
+			ValidCACert: false,
+			IsAnomaly:   true,
+		},
+	}, {
+		tk: riseupvpn.TestKeys{
+			APIStatus:    "ok",
+			CACertStatus: true,
+			FailingGateways: []riseupvpn.GatewayConnection{{
+				IP:            "1.1.1.1",
+				Port:          443,
+				TransportType: "obfs4",
+			}},
+		},
+		sk: riseupvpn.SummaryKeys{
+			FailingGateways: 1,
+			IsAnomaly:       true,
+			ValidCACert:     true,
+		},
+	}}
+	for idx, tt := range tests {
+		t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
+			m := &riseupvpn.Measurer{}
+			measurement := &model.Measurement{TestKeys: &tt.tk}
+			got, err := m.GetSummaryKeys(measurement)
+			if err != nil {
+				t.Fatal(err)
+				return
+			}
+			sk := got.(riseupvpn.SummaryKeys)
+			if diff := cmp.Diff(tt.sk, sk); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}

internal/engine/experiment/run/dnscheck.go

@@ -0,0 +1,27 @@
+package run
+
+import (
+	"context"
+	"sync"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type dnsCheckMain struct {
+	Endpoints *dnscheck.Endpoints
+	mu        sync.Mutex
+}
+
+func (m *dnsCheckMain) do(ctx context.Context, input StructuredInput,
+	sess model.ExperimentSession, measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks) error {
+	exp := dnscheck.Measurer{
+		Config:    input.DNSCheck,
+		Endpoints: m.Endpoints,
+	}
+	measurement.TestName = exp.ExperimentName()
+	measurement.TestVersion = exp.ExperimentVersion()
+	measurement.Input = model.MeasurementTarget(input.Input)
+	return exp.Run(ctx, sess, measurement, callbacks)
+}

internal/engine/experiment/run/run.go

@@ -0,0 +1,79 @@
+// Package run contains code to run other experiments.
+//
+// This code is currently alpha.
+package run
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// Config contains settings.
+type Config struct{}
+
+// Measurer runs the measurement.
+type Measurer struct{}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName.
+func (Measurer) ExperimentName() string {
+	return "run"
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (Measurer) ExperimentVersion() string {
+	return "0.2.0"
+}
+
+// StructuredInput contains structured input for this experiment.
+type StructuredInput struct {
+	// Annotations contains extra annotations to add to the
+	// final measurement.
+	Annotations map[string]string `json:"annotations"`
+
+	// DNSCheck contains settings for the dnscheck experiment.
+	DNSCheck dnscheck.Config `json:"dnscheck"`
+
+	// URLGetter contains settings for the urlgetter experiment.
+	URLGetter urlgetter.Config `json:"urlgetter"`
+
+	// Name is the name of the experiment to run.
+	Name string `json:"name"`
+
+	// Input is the input for this experiment.
+	Input string `json:"input"`
+}
+
+// Run implements ExperimentMeasurer.ExperimentVersion.
+func (Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	var input StructuredInput
+	if err := json.Unmarshal([]byte(measurement.Input), &input); err != nil {
+		return err
+	}
+	exprun, found := table[input.Name]
+	if !found {
+		return fmt.Errorf("no such experiment: %s", input.Name)
+	}
+	measurement.AddAnnotations(input.Annotations)
+	return exprun.do(ctx, input, sess, measurement, callbacks)
+}
+
+// GetSummaryKeys implements ExperimentMeasurer.GetSummaryKeys
+func (Measurer) GetSummaryKeys(*model.Measurement) (interface{}, error) {
+	// TODO(bassosimone): we could extend this interface to call the
+	// specific GetSummaryKeys of the experiment we're running.
+	return dnscheck.SummaryKeys{IsAnomaly: false}, nil
+}
+
+// NewExperimentMeasurer creates a new model.ExperimentMeasurer
+// implementing the run experiment.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{}
+}

internal/engine/experiment/run/run_test.go

@@ -0,0 +1,107 @@
+package run_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/run"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestExperimentNameAndVersion(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	if measurer.ExperimentName() != "run" {
+		t.Error("unexpected experiment name")
+	}
+	if measurer.ExperimentVersion() != "0.2.0" {
+		t.Error("unexpected experiment version")
+	}
+}
+
+func TestRunDNSCheckWithCancelledContext(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	input := `{"name": "dnscheck", "input": "https://dns.google/dns-query"}`
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	// TODO(bassosimone): here we could improve the tests by checking
+	// whether the result makes sense for a cancelled context.
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := measurement.TestKeys.(*dnscheck.TestKeys); !ok {
+		t.Fatal("invalid type for test keys")
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rsk, ok := sk.(dnscheck.SummaryKeys)
+	if !ok {
+		t.Fatal("cannot convert summary keys to specific type")
+	}
+	if rsk.IsAnomaly != false {
+		t.Fatal("unexpected IsAnomaly value")
+	}
+}
+
+func TestRunURLGetterWithCancelledContext(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	input := `{"name": "urlgetter", "input": "https://google.com"}`
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err == nil {
+		t.Fatal(err)
+	}
+	if len(measurement.Extensions) != 6 {
+		t.Fatal("not the expected number of extensions")
+	}
+	tk, ok := measurement.TestKeys.(*urlgetter.TestKeys)
+	if !ok {
+		t.Fatal("invalid type for test keys")
+	}
+	if len(tk.DNSCache) != 0 {
+		t.Fatal("not the DNSCache value we expected")
+	}
+}
+
+func TestRunWithInvalidJSON(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	input := `{"name": }`
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	ctx := context.Background()
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err == nil || err.Error() != "invalid character '}' looking for beginning of value" {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+}
+
+func TestRunWithUnknownExperiment(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	input := `{"name": "antani"}`
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	ctx := context.Background()
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err == nil || err.Error() != "no such experiment: antani" {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+}

internal/engine/experiment/run/table.go

@@ -0,0 +1,26 @@
+package run
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type experimentMain interface {
+	do(ctx context.Context, input StructuredInput,
+		sess model.ExperimentSession, measurement *model.Measurement,
+		callbacks model.ExperimentCallbacks) error
+}
+
+var table = map[string]experimentMain{
+	// TODO(bassosimone): before extending run to support more than
+	// single experiment, we need to handle the case in which we are
+	// including different experiments into the same report ID.
+	// Probably, the right way to implement this functionality is to
+	// use proveservices.Submitter to submit reports.
+	"dnscheck": &dnsCheckMain{
+		Endpoints: &dnscheck.Endpoints{},
+	},
+	"urlgetter": &urlGetterMain{},
+}

internal/engine/experiment/run/urlgetter.go

@@ -0,0 +1,22 @@
+package run
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type urlGetterMain struct {}
+
+func (m *urlGetterMain) do(ctx context.Context, input StructuredInput,
+	sess model.ExperimentSession, measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks) error {
+	exp := urlgetter.Measurer{
+		Config: input.URLGetter,
+	}
+	measurement.TestName = exp.ExperimentName()
+	measurement.TestVersion = exp.ExperimentVersion()
+	measurement.Input = model.MeasurementTarget(input.Input)
+	return exp.Run(ctx, sess, measurement, callbacks)
+}

internal/engine/experiment/sniblocking/sniblocking.go

@@ -0,0 +1,306 @@
+// Package sniblocking contains the SNI blocking network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-024-sni-blocking.md.
+package sniblocking
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+const (
+	testName    = "sni_blocking"
+	testVersion = "0.3.0"
+)
+
+// Config contains the experiment config.
+type Config struct {
+	// ControlSNI is the SNI to be used for the control.
+	ControlSNI string
+
+	// TestHelperAddress is the address of the test helper.
+	TestHelperAddress string
+}
+
+// Subresult contains the keys of a single measurement
+// that targets either the target or the control.
+type Subresult struct {
+	urlgetter.TestKeys
+	Cached    bool   `json:"-"`
+	SNI       string `json:"sni"`
+	THAddress string `json:"th_address"`
+}
+
+// TestKeys contains sniblocking test keys.
+type TestKeys struct {
+	Control Subresult `json:"control"`
+	Result  string    `json:"result"`
+	Target  Subresult `json:"target"`
+}
+
+const (
+	classAnomalyTestHelperUnreachable   = "anomaly.test_helper_unreachable"
+	classAnomalyTimeout                 = "anomaly.timeout"
+	classAnomalyUnexpectedFailure       = "anomaly.unexpected_failure"
+	classInterferenceClosed             = "interference.closed"
+	classInterferenceInvalidCertificate = "interference.invalid_certificate"
+	classInterferenceReset              = "interference.reset"
+	classInterferenceUnknownAuthority   = "interference.unknown_authority"
+	classSuccessGotServerHello          = "success.got_server_hello"
+)
+
+func (tk *TestKeys) classify() string {
+	if tk.Target.Failure == nil {
+		return classSuccessGotServerHello
+	}
+	switch *tk.Target.Failure {
+	case errorx.FailureConnectionRefused:
+		return classAnomalyTestHelperUnreachable
+	case errorx.FailureConnectionReset:
+		return classInterferenceReset
+	case errorx.FailureDNSNXDOMAINError:
+		return classAnomalyTestHelperUnreachable
+	case errorx.FailureEOFError:
+		return classInterferenceClosed
+	case errorx.FailureGenericTimeoutError:
+		if tk.Control.Failure != nil {
+			return classAnomalyTestHelperUnreachable
+		}
+		return classAnomalyTimeout
+	case errorx.FailureSSLInvalidCertificate:
+		return classInterferenceInvalidCertificate
+	case errorx.FailureSSLInvalidHostname:
+		return classSuccessGotServerHello
+	case errorx.FailureSSLUnknownAuthority:
+		return classInterferenceUnknownAuthority
+	}
+	return classAnomalyUnexpectedFailure
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	cache  map[string]Subresult
+	config Config
+	mu     sync.Mutex
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m *Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+func (m *Measurer) measureone(
+	ctx context.Context,
+	sess model.ExperimentSession,
+	beginning time.Time,
+	sni string,
+	thaddr string,
+) Subresult {
+	// slightly delay the measurement
+	gen := rand.New(rand.NewSource(time.Now().UnixNano()))
+	sleeptime := time.Duration(gen.Intn(250)) * time.Millisecond
+	select {
+	case <-time.After(sleeptime):
+	case <-ctx.Done():
+		s := errorx.FailureInterrupted
+		failedop := errorx.TopLevelOperation
+		return Subresult{
+			TestKeys: urlgetter.TestKeys{
+				FailedOperation: &failedop,
+				Failure:         &s,
+			},
+			THAddress: thaddr,
+			SNI:       sni,
+		}
+	}
+	// perform the measurement
+	g := urlgetter.Getter{
+		Begin:   beginning,
+		Config:  urlgetter.Config{TLSServerName: sni},
+		Session: sess,
+		Target:  fmt.Sprintf("tlshandshake://%s", thaddr),
+	}
+	// Ignoring the error because g.Get() sets the tk.Failure field
+	// to be the OONI equivalent of the error that occurred.
+	tk, _ := g.Get(ctx)
+	// assemble and publish the results
+	smk := Subresult{
+		SNI:       sni,
+		THAddress: thaddr,
+		TestKeys:  tk,
+	}
+	return smk
+}
+
+func (m *Measurer) measureonewithcache(
+	ctx context.Context,
+	output chan<- Subresult,
+	sess model.ExperimentSession,
+	beginning time.Time,
+	sni string,
+	thaddr string,
+) {
+	cachekey := sni + thaddr
+	m.mu.Lock()
+	smk, okay := m.cache[cachekey]
+	m.mu.Unlock()
+	if okay {
+		output <- smk
+		return
+	}
+	smk = m.measureone(ctx, sess, beginning, sni, thaddr)
+	output <- smk
+	smk.Cached = true
+	m.mu.Lock()
+	m.cache[cachekey] = smk
+	m.mu.Unlock()
+}
+
+func (m *Measurer) startall(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, inputs []string,
+) <-chan Subresult {
+	outputs := make(chan Subresult, len(inputs))
+	for _, input := range inputs {
+		go m.measureonewithcache(
+			ctx, outputs, sess,
+			measurement.MeasurementStartTimeSaved,
+			input, m.config.TestHelperAddress,
+		)
+	}
+	return outputs
+}
+
+func processall(
+	outputs <-chan Subresult,
+	measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks,
+	inputs []string,
+	sess model.ExperimentSession,
+	controlSNI string,
+) *TestKeys {
+	var (
+		current  int
+		testkeys = new(TestKeys)
+	)
+	for smk := range outputs {
+		if smk.SNI == controlSNI {
+			testkeys.Control = smk
+		} else if smk.SNI == string(measurement.Input) {
+			testkeys.Target = smk
+		} else {
+			panic("unexpected smk.SNI")
+		}
+		current++
+		sess.Logger().Debugf(
+			"sni_blocking: %s: %s [cached: %+v]", smk.SNI,
+			asString(smk.Failure), smk.Cached)
+		if current >= len(inputs) {
+			break
+		}
+	}
+	testkeys.Result = testkeys.classify()
+	sess.Logger().Infof("sni_blocking: result: %s", testkeys.Result)
+	return testkeys
+}
+
+// maybeURLToSNI handles the case where the input is from the test-lists
+// and hence every input is a URL rather than a domain.
+func maybeURLToSNI(input model.MeasurementTarget) (model.MeasurementTarget, error) {
+	parsed, err := url.Parse(string(input))
+	if err != nil {
+		return "", err
+	}
+	if parsed.Path == string(input) {
+		return input, nil
+	}
+	return model.MeasurementTarget(parsed.Hostname()), nil
+}
+
+// Run implements ExperimentMeasurer.Run.
+func (m *Measurer) Run(
+	ctx context.Context,
+	sess model.ExperimentSession,
+	measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks,
+) error {
+	m.mu.Lock()
+	if m.cache == nil {
+		m.cache = make(map[string]Subresult)
+	}
+	m.mu.Unlock()
+	if m.config.ControlSNI == "" {
+		m.config.ControlSNI = "example.org"
+	}
+	if measurement.Input == "" {
+		return errors.New("Experiment requires measurement.Input")
+	}
+	if m.config.TestHelperAddress == "" {
+		m.config.TestHelperAddress = net.JoinHostPort(
+			m.config.ControlSNI, "443",
+		)
+	}
+	urlgetter.RegisterExtensions(measurement)
+	// TODO(bassosimone): if the user has configured DoT or DoH, here we
+	// probably want to perform the name resolution before the measurements
+	// or to make sure that the classify logic is robust to that.
+	//
+	// See https://github.com/ooni/probe-cli/v3/internal/engine/issues/392.
+	maybeParsed, err := maybeURLToSNI(measurement.Input)
+	if err != nil {
+		return err
+	}
+	measurement.Input = maybeParsed
+	inputs := []string{m.config.ControlSNI}
+	if string(measurement.Input) != m.config.ControlSNI {
+		inputs = append(inputs, string(measurement.Input))
+	}
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second*time.Duration(len(inputs)))
+	defer cancel()
+	outputs := m.startall(ctx, sess, measurement, inputs)
+	measurement.TestKeys = processall(
+		outputs, measurement, callbacks, inputs, sess, m.config.ControlSNI,
+	)
+	return nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return &Measurer{config: config}
+}
+
+func asString(failure *string) (result string) {
+	result = "success"
+	if failure != nil {
+		result = *failure
+	}
+	return
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/sniblocking/sniblocking_test.go

@@ -0,0 +1,434 @@
+package sniblocking
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+const (
+	softwareName    = "ooniprobe-example"
+	softwareVersion = "0.0.1"
+)
+
+func TestTestKeysClassify(t *testing.T) {
+	asStringPtr := func(s string) *string {
+		return &s
+	}
+	t.Run("with tk.Target.Failure == nil", func(t *testing.T) {
+		tk := new(TestKeys)
+		if tk.classify() != classSuccessGotServerHello {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == connection_refused", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureConnectionRefused)
+		if tk.classify() != classAnomalyTestHelperUnreachable {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == dns_nxdomain_error", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureDNSNXDOMAINError)
+		if tk.classify() != classAnomalyTestHelperUnreachable {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == connection_reset", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureConnectionReset)
+		if tk.classify() != classInterferenceReset {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == eof_error", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureEOFError)
+		if tk.classify() != classInterferenceClosed {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == ssl_invalid_hostname", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureSSLInvalidHostname)
+		if tk.classify() != classSuccessGotServerHello {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == ssl_unknown_authority", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureSSLUnknownAuthority)
+		if tk.classify() != classInterferenceUnknownAuthority {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == ssl_invalid_certificate", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureSSLInvalidCertificate)
+		if tk.classify() != classInterferenceInvalidCertificate {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == generic_timeout_error #1", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureGenericTimeoutError)
+		if tk.classify() != classAnomalyTimeout {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == generic_timeout_error #2", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(errorx.FailureGenericTimeoutError)
+		tk.Control.Failure = asStringPtr(errorx.FailureGenericTimeoutError)
+		if tk.classify() != classAnomalyTestHelperUnreachable {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == unknown_failure", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr("unknown_failure")
+		if tk.classify() != classAnomalyUnexpectedFailure {
+			t.Fatal("unexpected result")
+		}
+	})
+}
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	if measurer.ExperimentName() != "sni_blocking" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.3.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestMeasurerMeasureNoMeasurementInput(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{
+		ControlSNI: "example.com",
+	})
+	err := measurer.Run(
+		context.Background(),
+		newsession(),
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err.Error() != "Experiment requires measurement.Input" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestMeasurerMeasureWithInvalidInput(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel the context
+	measurer := NewExperimentMeasurer(Config{
+		ControlSNI: "example.com",
+	})
+	measurement := &model.Measurement{
+		Input: "\t",
+	}
+	err := measurer.Run(
+		ctx,
+		newsession(),
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+}
+
+func TestMeasurerMeasureWithCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel the context
+	measurer := NewExperimentMeasurer(Config{
+		ControlSNI: "example.com",
+	})
+	measurement := &model.Measurement{
+		Input: "kernel.org",
+	}
+	err := measurer.Run(
+		ctx,
+		newsession(),
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestMeasureoneCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel the context
+	result := new(Measurer).measureone(
+		ctx,
+		&mockable.Session{MockableLogger: log.Log},
+		time.Now(),
+		"kernel.org",
+		"example.com:443",
+	)
+	if result.Agent != "" {
+		t.Fatal("not the expected Agent")
+	}
+	if result.BootstrapTime != 0.0 {
+		t.Fatal("not the expected BootstrapTime")
+	}
+	if result.DNSCache != nil {
+		t.Fatal("not the expected DNSCache")
+	}
+	if result.FailedOperation == nil || *result.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the expected FailedOperation")
+	}
+	if result.Failure == nil || *result.Failure != errorx.FailureInterrupted {
+		t.Fatal("not the expected failure")
+	}
+	if result.NetworkEvents != nil {
+		t.Fatal("not the expected NetworkEvents")
+	}
+	if result.Queries != nil {
+		t.Fatal("not the expected Queries")
+	}
+	if result.Requests != nil {
+		t.Fatal("not the expected Requests")
+	}
+	if result.SOCKSProxy != "" {
+		t.Fatal("not the expected SOCKSProxy")
+	}
+	if result.TCPConnect != nil {
+		t.Fatal("not the expected TCPConnect")
+	}
+	if result.TLSHandshakes != nil {
+		t.Fatal("not the expected TLSHandshakes")
+	}
+	if result.Tunnel != "" {
+		t.Fatal("not the expected Tunnel")
+	}
+	if result.SNI != "kernel.org" {
+		t.Fatal("unexpected SNI")
+	}
+	if result.THAddress != "example.com:443" {
+		t.Fatal("unexpected THAddress")
+	}
+}
+
+func TestMeasureoneWithPreMeasurementFailure(t *testing.T) {
+	result := new(Measurer).measureone(
+		context.Background(),
+		&mockable.Session{MockableLogger: log.Log},
+		time.Now(),
+		"kernel.org",
+		"example.com:443\t\t\t", // cause URL parse error
+	)
+	if result.Agent != "redirect" {
+		t.Fatal("not the expected Agent")
+	}
+	if result.BootstrapTime != 0.0 {
+		t.Fatal("not the expected BootstrapTime")
+	}
+	if result.DNSCache != nil {
+		t.Fatal("not the expected DNSCache")
+	}
+	if result.FailedOperation == nil || *result.FailedOperation != "top_level" {
+		t.Fatal("not the expected FailedOperation")
+	}
+	if result.Failure == nil || !strings.Contains(*result.Failure, "invalid target URL") {
+		t.Fatal("not the expected failure")
+	}
+	if result.NetworkEvents != nil {
+		t.Fatal("not the expected NetworkEvents")
+	}
+	if result.Queries != nil {
+		t.Fatal("not the expected Queries")
+	}
+	if result.Requests != nil {
+		t.Fatal("not the expected Requests")
+	}
+	if result.SOCKSProxy != "" {
+		t.Fatal("not the expected SOCKSProxy")
+	}
+	if result.TCPConnect != nil {
+		t.Fatal("not the expected TCPConnect")
+	}
+	if result.TLSHandshakes != nil {
+		t.Fatal("not the expected TLSHandshakes")
+	}
+	if result.Tunnel != "" {
+		t.Fatal("not the expected Tunnel")
+	}
+	if result.SNI != "kernel.org" {
+		t.Fatal("unexpected SNI")
+	}
+	if result.THAddress != "example.com:443\t\t\t" {
+		t.Fatal("unexpected THAddress")
+	}
+}
+
+func TestMeasureoneSuccess(t *testing.T) {
+	result := new(Measurer).measureone(
+		context.Background(),
+		&mockable.Session{MockableLogger: log.Log},
+		time.Now(),
+		"kernel.org",
+		"example.com:443",
+	)
+	if result.Agent != "redirect" {
+		t.Fatal("not the expected Agent")
+	}
+	if result.BootstrapTime != 0.0 {
+		t.Fatal("not the expected BootstrapTime")
+	}
+	if result.DNSCache != nil {
+		t.Fatal("not the expected DNSCache")
+	}
+	if result.FailedOperation == nil || *result.FailedOperation != errorx.TLSHandshakeOperation {
+		t.Fatal("not the expected FailedOperation")
+	}
+	if result.Failure == nil || *result.Failure != errorx.FailureSSLInvalidHostname {
+		t.Fatal("unexpected failure")
+	}
+	if len(result.NetworkEvents) < 1 {
+		t.Fatal("not the expected NetworkEvents")
+	}
+	if len(result.Queries) < 1 {
+		t.Fatal("not the expected Queries")
+	}
+	if result.Requests != nil {
+		t.Fatal("not the expected Requests")
+	}
+	if result.SOCKSProxy != "" {
+		t.Fatal("not the expected SOCKSProxy")
+	}
+	if len(result.TCPConnect) < 1 {
+		t.Fatal("not the expected TCPConnect")
+	}
+	if len(result.TLSHandshakes) < 1 {
+		t.Fatal("not the expected TLSHandshakes")
+	}
+	if result.Tunnel != "" {
+		t.Fatal("not the expected Tunnel")
+	}
+	if result.SNI != "kernel.org" {
+		t.Fatal("unexpected SNI")
+	}
+	if result.THAddress != "example.com:443" {
+		t.Fatal("unexpected THAddress")
+	}
+}
+
+func TestMeasureonewithcacheWorks(t *testing.T) {
+	measurer := &Measurer{cache: make(map[string]Subresult)}
+	output := make(chan Subresult, 2)
+	for i := 0; i < 2; i++ {
+		measurer.measureonewithcache(
+			context.Background(),
+			output,
+			&mockable.Session{MockableLogger: log.Log},
+			time.Now(),
+			"kernel.org",
+			"example.com:443",
+		)
+	}
+	for _, expected := range []bool{false, true} {
+		result := <-output
+		if result.Cached != expected {
+			t.Fatal("unexpected cached")
+		}
+		if *result.Failure != errorx.FailureSSLInvalidHostname {
+			t.Fatal("unexpected failure")
+		}
+		if result.SNI != "kernel.org" {
+			t.Fatal("unexpected SNI")
+		}
+	}
+}
+
+func TestProcessallPanicsIfInvalidSNI(t *testing.T) {
+	defer func() {
+		panicdata := recover()
+		if panicdata == nil {
+			t.Fatal("expected to see panic here")
+		}
+		if panicdata.(string) != "unexpected smk.SNI" {
+			t.Fatal("not the panic we expected")
+		}
+	}()
+	outputs := make(chan Subresult, 1)
+	measurement := &model.Measurement{
+		Input: "kernel.org",
+	}
+	go func() {
+		outputs <- Subresult{
+			SNI: "antani.io",
+		}
+	}()
+	processall(
+		outputs,
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+		[]string{"kernel.org", "example.com"},
+		newsession(),
+		"example.com",
+	)
+}
+
+func TestMaybeURLToSNI(t *testing.T) {
+	t.Run("for invalid URL", func(t *testing.T) {
+		parsed, err := maybeURLToSNI("\t")
+		if err == nil {
+			t.Fatal("expected an error here")
+		}
+		if parsed != "" {
+			t.Fatal("expected empty parsed here")
+		}
+	})
+	t.Run("for domain name", func(t *testing.T) {
+		parsed, err := maybeURLToSNI("kernel.org")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if parsed != "kernel.org" {
+			t.Fatal("expected different domain here")
+		}
+	})
+	t.Run("for valid URL", func(t *testing.T) {
+		parsed, err := maybeURLToSNI("https://kernel.org/robots.txt")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if parsed != "kernel.org" {
+			t.Fatal("expected different domain here")
+		}
+	})
+}
+
+func newsession() model.ExperimentSession {
+	return &mockable.Session{MockableLogger: log.Log}
+}
+
+func TestSummaryKeysGeneric(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &TestKeys{}}
+	m := &Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(SummaryKeys)
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/stunreachability/fake_test.go

@@ -0,0 +1,60 @@
+package stunreachability
+
+import (
+	"io"
+	"net"
+	"time"
+)
+
+type FakeConn struct {
+	ReadError             error
+	ReadData              []byte
+	SetDeadlineError      error
+	SetReadDeadlineError  error
+	SetWriteDeadlineError error
+	WriteError            error
+}
+
+func (c *FakeConn) Read(b []byte) (int, error) {
+	if len(c.ReadData) > 0 {
+		n := copy(b, c.ReadData)
+		c.ReadData = c.ReadData[n:]
+		return n, nil
+	}
+	if c.ReadError != nil {
+		return 0, c.ReadError
+	}
+	return 0, io.EOF
+}
+
+func (c *FakeConn) Write(b []byte) (n int, err error) {
+	if c.WriteError != nil {
+		return 0, c.WriteError
+	}
+	n = len(b)
+	return
+}
+
+func (*FakeConn) Close() (err error) {
+	return
+}
+
+func (*FakeConn) LocalAddr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+func (*FakeConn) RemoteAddr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+func (c *FakeConn) SetDeadline(t time.Time) (err error) {
+	return c.SetDeadlineError
+}
+
+func (c *FakeConn) SetReadDeadline(t time.Time) (err error) {
+	return c.SetReadDeadlineError
+}
+
+func (c *FakeConn) SetWriteDeadline(t time.Time) (err error) {
+	return c.SetWriteDeadlineError
+}

internal/engine/experiment/stunreachability/stunreachability.go

@@ -0,0 +1,169 @@
+// Package stunreachability contains the STUN reachability experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-025-stun-reachability.md.
+package stunreachability
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+	"github.com/pion/stun"
+)
+
+const (
+	testName    = "stun_reachability"
+	testVersion = "0.1.0"
+)
+
+// Config contains the experiment config.
+type Config struct {
+	dialContext func(ctx context.Context, network, address string) (net.Conn, error)
+	newClient   func(conn stun.Connection, options ...stun.ClientOption) (*stun.Client, error)
+}
+
+// TestKeys contains the experiment's result.
+type TestKeys struct {
+	Endpoint      string                   `json:"endpoint"`
+	Failure       *string                  `json:"failure"`
+	NetworkEvents []archival.NetworkEvent  `json:"network_events"`
+	Queries       []archival.DNSQueryEntry `json:"queries"`
+}
+
+func registerExtensions(m *model.Measurement) {
+	archival.ExtDNS.AddTo(m)
+	archival.ExtNetevents.AddTo(m)
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config Config
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m *Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+func wrap(err error) error {
+	return errorx.SafeErrWrapperBuilder{
+		Error:     err,
+		Operation: "stun",
+	}.MaybeBuild()
+}
+
+// Run implements ExperimentMeasurer.Run.
+func (m *Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	tk := new(TestKeys)
+	measurement.TestKeys = tk
+	registerExtensions(measurement)
+	if err := wrap(tk.run(ctx, m.config, sess, measurement, callbacks)); err != nil {
+		s := err.Error()
+		tk.Failure = &s
+		return err
+	}
+	return nil
+}
+
+func (tk *TestKeys) run(
+	ctx context.Context, config Config, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	const defaultAddress = "stun.l.google.com:19302"
+	endpoint := string(measurement.Input)
+	if endpoint == "" {
+		endpoint = defaultAddress
+	}
+	callbacks.OnProgress(0, fmt.Sprintf("stunreachability: measuring: %s...", endpoint))
+	defer callbacks.OnProgress(
+		1, fmt.Sprintf("stunreachability: measuring: %s... done", endpoint))
+	tk.Endpoint = endpoint
+	saver := new(trace.Saver)
+	begin := time.Now()
+	err := tk.do(ctx, config, netx.NewDialer(netx.Config{
+		ContextByteCounting: true,
+		DialSaver:           saver,
+		Logger:              sess.Logger(),
+		ReadWriteSaver:      saver,
+		ResolveSaver:        saver,
+	}), endpoint)
+	events := saver.Read()
+	tk.NetworkEvents = append(
+		tk.NetworkEvents, archival.NewNetworkEventsList(begin, events)...,
+	)
+	tk.Queries = append(
+		tk.Queries, archival.NewDNSQueriesList(begin, events, sess.ASNDatabasePath())...,
+	)
+	return err
+}
+
+func (tk *TestKeys) do(
+	ctx context.Context, config Config, dialer dialer.Dialer, endpoint string) error {
+	dialContext := dialer.DialContext
+	if config.dialContext != nil {
+		dialContext = config.dialContext
+	}
+	conn, err := dialContext(ctx, "udp", endpoint)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	newClient := stun.NewClient
+	if config.newClient != nil {
+		newClient = config.newClient
+	}
+	client, err := newClient(conn, stun.WithNoConnClose)
+	if err != nil {
+		return err
+	}
+	message := stun.MustBuild(stun.TransactionID, stun.BindingRequest)
+	ch := make(chan error)
+	err = client.Start(message, func(ev stun.Event) {
+		// As mentioned below this code will run after Start has returned.
+		if ev.Error != nil {
+			ch <- ev.Error
+			return
+		}
+		var xorAddr stun.XORMappedAddress
+		ch <- xorAddr.GetFrom(ev.Message)
+	})
+	// Implementation note: if we successfully started, then the callback
+	// will be called when we receive a response or fail.
+	if err != nil {
+		return err
+	}
+	return <-ch
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return &Measurer{config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/stunreachability/stunreachability_internal_test.go

@@ -0,0 +1,18 @@
+package stunreachability
+
+import (
+	"context"
+	"net"
+
+	"github.com/pion/stun"
+)
+
+func (c *Config) SetNewClient(
+	f func(conn stun.Connection, options ...stun.ClientOption) (*stun.Client, error)) {
+	c.newClient = f
+}
+
+func (c *Config) SetDialContext(
+	f func(ctx context.Context, network, address string) (net.Conn, error)) {
+	c.dialContext = f
+}

internal/engine/experiment/stunreachability/stunreachability_test.go

@@ -0,0 +1,242 @@
+package stunreachability_test
+
+import (
+	"context"
+	"errors"
+	"net"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/stunreachability"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/pion/stun"
+)
+
+func TestMeasurerExperimentNameVersion(t *testing.T) {
+	measurer := stunreachability.NewExperimentMeasurer(stunreachability.Config{})
+	if measurer.ExperimentName() != "stun_reachability" {
+		t.Fatal("unexpected ExperimentName")
+	}
+	if measurer.ExperimentVersion() != "0.1.0" {
+		t.Fatal("unexpected ExperimentVersion")
+	}
+}
+
+func TestRun(t *testing.T) {
+	if os.Getenv("GITHUB_ACTIONS") == "true" {
+		// See https://github.com/ooni/probe-cli/v3/internal/engine/issues/874#issuecomment-679850652
+		t.Skip("skipping broken test on GitHub Actions")
+	}
+	measurer := stunreachability.NewExperimentMeasurer(stunreachability.Config{})
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*stunreachability.TestKeys)
+	if tk.Failure != nil {
+		t.Fatal("expected nil failure here")
+	}
+	if tk.Endpoint != "stun.l.google.com:19302" {
+		t.Fatal("unexpected endpoint")
+	}
+	if len(tk.NetworkEvents) <= 0 {
+		t.Fatal("no network events?!")
+	}
+	if len(tk.Queries) <= 0 {
+		t.Fatal("no DNS queries?!")
+	}
+}
+
+func TestRunCustomInput(t *testing.T) {
+	input := "stun.ekiga.net:3478"
+	measurer := stunreachability.NewExperimentMeasurer(stunreachability.Config{})
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*stunreachability.TestKeys)
+	if tk.Failure != nil {
+		t.Fatal("expected nil failure here")
+	}
+	if tk.Endpoint != input {
+		t.Fatal("unexpected endpoint")
+	}
+	if len(tk.NetworkEvents) <= 0 {
+		t.Fatal("no network events?!")
+	}
+	if len(tk.Queries) <= 0 {
+		t.Fatal("no DNS queries?!")
+	}
+}
+
+func TestCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately fail everything
+	measurer := stunreachability.NewExperimentMeasurer(stunreachability.Config{})
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		ctx,
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err.Error() != "interrupted" {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*stunreachability.TestKeys)
+	if *tk.Failure != "interrupted" {
+		t.Fatal("expected different failure here")
+	}
+	if tk.Endpoint != "stun.l.google.com:19302" {
+		t.Fatal("unexpected endpoint")
+	}
+	if len(tk.NetworkEvents) <= 0 {
+		t.Fatal("no network events?!")
+	}
+	if len(tk.Queries) <= 0 {
+		t.Fatal("no DNS queries?!")
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(stunreachability.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestNewClientFailure(t *testing.T) {
+	config := &stunreachability.Config{}
+	expected := errors.New("mocked error")
+	config.SetNewClient(
+		func(conn stun.Connection, options ...stun.ClientOption) (*stun.Client, error) {
+			return nil, expected
+		})
+	measurer := stunreachability.NewExperimentMeasurer(*config)
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*stunreachability.TestKeys)
+	if !strings.HasPrefix(*tk.Failure, "unknown_failure") {
+		t.Fatal("expected different failure here")
+	}
+	if tk.Endpoint != "stun.l.google.com:19302" {
+		t.Fatal("unexpected endpoint")
+	}
+	if len(tk.NetworkEvents) <= 0 {
+		t.Fatal("no network events?!")
+	}
+	if len(tk.Queries) <= 0 {
+		t.Fatal("no DNS queries?!")
+	}
+}
+
+func TestStartFailure(t *testing.T) {
+	config := &stunreachability.Config{}
+	expected := errors.New("mocked error")
+	config.SetDialContext(
+		func(ctx context.Context, network, address string) (net.Conn, error) {
+			conn := &stunreachability.FakeConn{WriteError: expected}
+			return conn, nil
+		})
+	measurer := stunreachability.NewExperimentMeasurer(*config)
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*stunreachability.TestKeys)
+	if !strings.HasPrefix(*tk.Failure, "unknown_failure") {
+		t.Fatal("expected different failure here")
+	}
+	if tk.Endpoint != "stun.l.google.com:19302" {
+		t.Fatal("unexpected endpoint")
+	}
+	// We're bypassing normal network with custom dial function
+	if len(tk.NetworkEvents) > 0 {
+		t.Fatal("network events?!")
+	}
+	if len(tk.Queries) > 0 {
+		t.Fatal("DNS queries?!")
+	}
+}
+
+func TestReadFailure(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	config := &stunreachability.Config{}
+	expected := errors.New("mocked error")
+	config.SetDialContext(
+		func(ctx context.Context, network, address string) (net.Conn, error) {
+			conn := &stunreachability.FakeConn{ReadError: expected}
+			return conn, nil
+		})
+	measurer := stunreachability.NewExperimentMeasurer(*config)
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, stun.ErrTransactionTimeOut) {
+		t.Fatal("not the error we expected")
+	}
+	tk := measurement.TestKeys.(*stunreachability.TestKeys)
+	if *tk.Failure != errorx.FailureGenericTimeoutError {
+		t.Fatal("expected different failure here")
+	}
+	if tk.Endpoint != "stun.l.google.com:19302" {
+		t.Fatal("unexpected endpoint")
+	}
+	// We're bypassing normal network with custom dial function
+	if len(tk.NetworkEvents) > 0 {
+		t.Fatal("network events?!")
+	}
+	if len(tk.Queries) > 0 {
+		t.Fatal("DNS queries?!")
+	}
+}
+
+func TestSummaryKeysGeneric(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &stunreachability.TestKeys{}}
+	m := &stunreachability.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(stunreachability.SummaryKeys)
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/telegram/telegram.go

@@ -0,0 +1,174 @@
+// Package telegram contains the Telegram network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-020-telegram.md.
+package telegram
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+const (
+	testName    = "telegram"
+	testVersion = "0.2.0"
+)
+
+// Config contains the telegram experiment config.
+type Config struct{}
+
+// TestKeys contains telegram test keys.
+type TestKeys struct {
+	urlgetter.TestKeys
+	TelegramHTTPBlocking bool    `json:"telegram_http_blocking"`
+	TelegramTCPBlocking  bool    `json:"telegram_tcp_blocking"`
+	TelegramWebFailure   *string `json:"telegram_web_failure"`
+	TelegramWebStatus    string  `json:"telegram_web_status"`
+}
+
+// NewTestKeys creates new telegram TestKeys.
+func NewTestKeys() *TestKeys {
+	return &TestKeys{
+		TelegramHTTPBlocking: true,
+		TelegramTCPBlocking:  true,
+		TelegramWebFailure:   nil,
+		TelegramWebStatus:    "ok",
+	}
+}
+
+// Update updates the TestKeys using the given MultiOutput result.
+func (tk *TestKeys) Update(v urlgetter.MultiOutput) {
+	// update the easy to update entries first
+	tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
+	tk.Queries = append(tk.Queries, v.TestKeys.Queries...)
+	tk.Requests = append(tk.Requests, v.TestKeys.Requests...)
+	tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
+	tk.TLSHandshakes = append(tk.TLSHandshakes, v.TestKeys.TLSHandshakes...)
+	// then process access points
+	if v.Input.Config.Method != "GET" {
+		if v.TestKeys.Failure == nil {
+			tk.TelegramHTTPBlocking = false
+			tk.TelegramTCPBlocking = false
+			return // found successful access point connection
+		}
+		if v.TestKeys.FailedOperation == nil || *v.TestKeys.FailedOperation != errorx.ConnectOperation {
+			tk.TelegramTCPBlocking = false
+		}
+		return
+	}
+	// now take care of web
+	if tk.TelegramWebStatus != "ok" {
+		return // we already flipped the state
+	}
+	if v.TestKeys.Failure != nil {
+		tk.TelegramWebStatus = "blocked"
+		tk.TelegramWebFailure = v.TestKeys.Failure
+		return
+	}
+	title := `<title>Telegram Web</title>`
+	if strings.Contains(v.TestKeys.HTTPResponseBody, title) == false {
+		failureString := "telegram_missing_title_error"
+		tk.TelegramWebFailure = &failureString
+		tk.TelegramWebStatus = "blocked"
+		return
+	}
+	return
+}
+
+// Measurer performs the measurement
+type Measurer struct {
+	// Config contains the experiment settings. If empty we
+	// will be using default settings.
+	Config Config
+
+	// Getter is an optional getter to be used for testing.
+	Getter urlgetter.MultiGetter
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run
+func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
+	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+	urlgetter.RegisterExtensions(measurement)
+	inputs := []urlgetter.MultiInput{
+		{Target: "http://149.154.175.50/", Config: urlgetter.Config{Method: "POST"}},
+		{Target: "http://149.154.167.51/", Config: urlgetter.Config{Method: "POST"}},
+		{Target: "http://149.154.175.100/", Config: urlgetter.Config{Method: "POST"}},
+		{Target: "http://149.154.167.91/", Config: urlgetter.Config{Method: "POST"}},
+		{Target: "http://149.154.171.5/", Config: urlgetter.Config{Method: "POST"}},
+
+		{Target: "http://149.154.175.50:443/", Config: urlgetter.Config{Method: "POST"}},
+		{Target: "http://149.154.167.51:443/", Config: urlgetter.Config{Method: "POST"}},
+		{Target: "http://149.154.175.100:443/", Config: urlgetter.Config{Method: "POST"}},
+		{Target: "http://149.154.167.91:443/", Config: urlgetter.Config{Method: "POST"}},
+		{Target: "http://149.154.171.5:443/", Config: urlgetter.Config{Method: "POST"}},
+
+		// Here we need to provide the method explicitly. See
+		// https://github.com/ooni/probe-cli/v3/internal/engine/issues/827.
+		{Target: "http://web.telegram.org/", Config: urlgetter.Config{
+			Method:          "GET",
+			FailOnHTTPError: true,
+		}},
+		{Target: "https://web.telegram.org/", Config: urlgetter.Config{
+			Method:          "GET",
+			FailOnHTTPError: true,
+		}},
+	}
+	multi := urlgetter.Multi{Begin: time.Now(), Getter: m.Getter, Session: sess}
+	testkeys := NewTestKeys()
+	testkeys.Agent = "redirect"
+	measurement.TestKeys = testkeys
+	for entry := range multi.Collect(ctx, inputs, "telegram", callbacks) {
+		testkeys.Update(entry)
+	}
+	return nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	HTTPBlocking bool `json:"telegram_http_blocking"`
+	TCPBlocking  bool `json:"telegram_tcp_blocking"`
+	WebBlocking  bool `json:"telegram_web_blocking"`
+	IsAnomaly    bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	tcpBlocking := tk.TelegramTCPBlocking
+	httpBlocking := tk.TelegramHTTPBlocking
+	webBlocking := tk.TelegramWebFailure != nil
+	sk.TCPBlocking = tcpBlocking
+	sk.HTTPBlocking = httpBlocking
+	sk.WebBlocking = webBlocking
+	sk.IsAnomaly = webBlocking || httpBlocking || tcpBlocking
+	return sk, nil
+}

internal/engine/experiment/telegram/telegram_test.go

@@ -0,0 +1,416 @@
+package telegram_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := telegram.NewExperimentMeasurer(telegram.Config{})
+	if measurer.ExperimentName() != "telegram" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.2.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestGood(t *testing.T) {
+	measurer := telegram.NewExperimentMeasurer(telegram.Config{})
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*telegram.TestKeys)
+	if tk.Agent != "redirect" {
+		t.Fatal("unexpected Agent")
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("unexpected FailedOperation")
+	}
+	if tk.Failure != nil {
+		t.Fatal("unexpected Failure")
+	}
+	if len(tk.NetworkEvents) <= 0 {
+		t.Fatal("no NetworkEvents?!")
+	}
+	if len(tk.Queries) <= 0 {
+		t.Fatal("no Queries?!")
+	}
+	if len(tk.Requests) <= 0 {
+		t.Fatal("no Requests?!")
+	}
+	if len(tk.TCPConnect) <= 0 {
+		t.Fatal("no TCPConnect?!")
+	}
+	if len(tk.TLSHandshakes) <= 0 {
+		t.Fatal("no TLSHandshakes?!")
+	}
+	if tk.TelegramHTTPBlocking != false {
+		t.Fatal("unexpected TelegramHTTPBlocking")
+	}
+	if tk.TelegramTCPBlocking != false {
+		t.Fatal("unexpected TelegramTCPBlocking")
+	}
+	if tk.TelegramWebFailure != nil {
+		t.Fatal("unexpected TelegramWebFailure")
+	}
+	if tk.TelegramWebStatus != "ok" {
+		t.Fatal("unexpected TelegramWebStatus")
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(telegram.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestUpdateWithNoAccessPointsBlocking(t *testing.T) {
+	tk := telegram.NewTestKeys()
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "POST"},
+			Target: "http://149.154.175.50/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "POST"},
+			Target: "http://149.154.175.50:443/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			Failure: nil, // this should be enough to declare success
+		},
+	})
+	if tk.TelegramHTTPBlocking == true {
+		t.Fatal("there should be no TelegramHTTPBlocking")
+	}
+	if tk.TelegramTCPBlocking == true {
+		t.Fatal("there should be no TelegramTCPBlocking")
+	}
+}
+
+func TestUpdateWithNilFailedOperation(t *testing.T) {
+	tk := telegram.NewTestKeys()
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "POST"},
+			Target: "http://149.154.175.50/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "POST"},
+			Target: "http://149.154.175.50:443/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	if tk.TelegramHTTPBlocking == false {
+		t.Fatal("there should be TelegramHTTPBlocking")
+	}
+	if tk.TelegramTCPBlocking == true {
+		t.Fatal("there should be no TelegramTCPBlocking")
+	}
+}
+
+func TestUpdateWithNonConnectFailedOperation(t *testing.T) {
+	tk := telegram.NewTestKeys()
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "POST"},
+			Target: "http://149.154.175.50/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			FailedOperation: (func() *string {
+				s := errorx.ConnectOperation
+				return &s
+			})(),
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "POST"},
+			Target: "http://149.154.175.50:443/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			FailedOperation: (func() *string {
+				s := errorx.HTTPRoundTripOperation
+				return &s
+			})(),
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	if tk.TelegramHTTPBlocking == false {
+		t.Fatal("there should be TelegramHTTPBlocking")
+	}
+	if tk.TelegramTCPBlocking == true {
+		t.Fatal("there should be no TelegramTCPBlocking")
+	}
+}
+
+func TestUpdateWithAllConnectsFailed(t *testing.T) {
+	tk := telegram.NewTestKeys()
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "POST"},
+			Target: "http://149.154.175.50/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			FailedOperation: (func() *string {
+				s := errorx.ConnectOperation
+				return &s
+			})(),
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "POST"},
+			Target: "http://149.154.175.50:443/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			FailedOperation: (func() *string {
+				s := errorx.ConnectOperation
+				return &s
+			})(),
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	if tk.TelegramHTTPBlocking == false {
+		t.Fatal("there should be TelegramHTTPBlocking")
+	}
+	if tk.TelegramTCPBlocking == false {
+		t.Fatal("there should be TelegramTCPBlocking")
+	}
+}
+
+func TestUpdateWebWithMixedResults(t *testing.T) {
+	tk := telegram.NewTestKeys()
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "http://web.telegram.org/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			FailedOperation: (func() *string {
+				s := errorx.HTTPRoundTripOperation
+				return &s
+			})(),
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "https://web.telegram.org/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseBody:   `<title>Telegram Web</title>`,
+			HTTPResponseStatus: 200,
+		},
+	})
+	if tk.TelegramWebStatus != "blocked" {
+		t.Fatal("TelegramWebStatus should be blocked")
+	}
+	if *tk.TelegramWebFailure != errorx.FailureEOFError {
+		t.Fatal("invalid TelegramWebFailure")
+	}
+}
+
+func TestWeConfigureWebChecksToFailOnHTTPError(t *testing.T) {
+	called := atomicx.NewInt64()
+	failOnErrorHTTPS := atomicx.NewInt64()
+	failOnErrorHTTP := atomicx.NewInt64()
+	measurer := telegram.Measurer{
+		Config: telegram.Config{},
+		Getter: func(ctx context.Context, g urlgetter.Getter) (urlgetter.TestKeys, error) {
+			called.Add(1)
+			switch g.Target {
+			case "https://web.telegram.org/":
+				if g.Config.FailOnHTTPError {
+					failOnErrorHTTPS.Add(1)
+				}
+			case "http://web.telegram.org/":
+				if g.Config.FailOnHTTPError {
+					failOnErrorHTTP.Add(1)
+				}
+			}
+			return urlgetter.DefaultMultiGetter(ctx, g)
+		},
+	}
+	ctx := context.Background()
+	sess := &mockable.Session{
+		MockableLogger: log.Log,
+	}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil {
+		t.Fatal(err)
+	}
+	if called.Load() < 1 {
+		t.Fatal("not called")
+	}
+	if failOnErrorHTTPS.Load() != 1 {
+		t.Fatal("not configured fail on error for HTTPS")
+	}
+	if failOnErrorHTTP.Load() != 1 {
+		t.Fatal("not configured fail on error for HTTP")
+	}
+}
+
+func TestUpdateWithMissingTitle(t *testing.T) {
+	tk := telegram.NewTestKeys()
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "http://web.telegram.org/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseStatus: 200,
+			HTTPResponseBody:   "<HTML><title>Telegram Web</title></HTML>",
+		},
+	})
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "http://web.telegram.org/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseStatus: 200,
+			HTTPResponseBody:   "<HTML><title>Antani Web</title></HTML>",
+		},
+	})
+	if tk.TelegramWebStatus != "blocked" {
+		t.Fatal("TelegramWebStatus should be blocked")
+	}
+	if *tk.TelegramWebFailure != "telegram_missing_title_error" {
+		t.Fatal("invalid TelegramWebFailure")
+	}
+}
+
+func TestUpdateWithAllGood(t *testing.T) {
+	tk := telegram.NewTestKeys()
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "http://web.telegram.org/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseStatus: 200,
+			HTTPResponseBody:   "<HTML><title>Telegram Web</title></HTML>",
+		},
+	})
+	tk.Update(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "http://web.telegram.org/",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseStatus: 200,
+			HTTPResponseBody:   "<HTML><title>Telegram Web</title></HTML>",
+		},
+	})
+	if tk.TelegramWebStatus != "ok" {
+		t.Fatal("TelegramWebStatus should be ok")
+	}
+	if tk.TelegramWebFailure != nil {
+		t.Fatal("invalid TelegramWebFailure")
+	}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &telegram.Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysWorksAsIntended(t *testing.T) {
+	failure := io.EOF.Error()
+	tests := []struct {
+		tk        telegram.TestKeys
+		isAnomaly bool
+	}{{
+		tk:        telegram.TestKeys{},
+		isAnomaly: false,
+	}, {
+		tk:        telegram.TestKeys{TelegramTCPBlocking: true},
+		isAnomaly: true,
+	}, {
+		tk:        telegram.TestKeys{TelegramHTTPBlocking: true},
+		isAnomaly: true,
+	}, {
+		tk:        telegram.TestKeys{TelegramWebFailure: &failure},
+		isAnomaly: true,
+	}}
+	for idx, tt := range tests {
+		t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
+			m := &telegram.Measurer{}
+			measurement := &model.Measurement{TestKeys: &tt.tk}
+			got, err := m.GetSummaryKeys(measurement)
+			if err != nil {
+				t.Fatal(err)
+				return
+			}
+			sk := got.(telegram.SummaryKeys)
+			if sk.IsAnomaly != tt.isAnomaly {
+				t.Fatal("unexpected isAnomaly value")
+			}
+		})
+	}
+}

internal/engine/experiment/tlstool/internal/dialer.go

@@ -0,0 +1,29 @@
+package internal
+
+import (
+	"context"
+	"net"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+// Dialer creates net.Conn instances where (1) we delay writes if
+// a delay is configured and (2) we split outgoing buffers if there
+// is a configured splitter function.
+type Dialer struct {
+	netx.Dialer
+	Delay    time.Duration
+	Splitter func([]byte) [][]byte
+}
+
+// DialContext implements netx.Dialer.DialContext.
+func (d Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	conn, err := d.Dialer.DialContext(ctx, network, address)
+	if err != nil {
+		return nil, err
+	}
+	conn = SleeperWriter{Conn: conn, Delay: d.Delay}
+	conn = SplitterWriter{Conn: conn, Splitter: d.Splitter}
+	return conn, nil
+}

internal/engine/experiment/tlstool/internal/dialer_test.go

@@ -0,0 +1,56 @@
+package internal_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+)
+
+func TestDialerFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	dialer := internal.Dialer{Dialer: internal.FakeDialer{
+		Err: expected,
+	}}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "8.8.8.8:853")
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn here")
+	}
+}
+
+func TestDialerSuccess(t *testing.T) {
+	splitter := func([]byte) [][]byte {
+		return nil // any value is fine we just a need a splitter != nil here
+	}
+	innerconn := &internal.FakeConn{}
+	dialer := internal.Dialer{
+		Delay:    12345,
+		Dialer:   internal.FakeDialer{Conn: innerconn},
+		Splitter: splitter,
+	}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "8.8.8.8:853")
+	if err != nil {
+		t.Fatal(err)
+	}
+	sconn, ok := conn.(internal.SplitterWriter)
+	if !ok {
+		t.Fatal("the outer connection is not a splitter")
+	}
+	if sconn.Splitter == nil {
+		t.Fatal("not the splitter we expected")
+	}
+	dconn, ok := sconn.Conn.(internal.SleeperWriter)
+	if !ok {
+		t.Fatal("the inner connection is not a sleeper")
+	}
+	if dconn.Delay != 12345 {
+		t.Fatal("invalid delay")
+	}
+	if dconn.Conn != innerconn {
+		t.Fatal("invalid inner connection")
+	}
+}

internal/engine/experiment/tlstool/internal/fake_test.go

@@ -0,0 +1,73 @@
+package internal
+
+import (
+	"context"
+	"io"
+	"net"
+	"time"
+)
+
+type FakeDialer struct {
+	Conn net.Conn
+	Err  error
+}
+
+func (d FakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	time.Sleep(10 * time.Microsecond)
+	return d.Conn, d.Err
+}
+
+type FakeConn struct {
+	ReadError             error
+	ReadData              []byte
+	SetDeadlineError      error
+	SetReadDeadlineError  error
+	SetWriteDeadlineError error
+	WriteData             [][]byte
+	WriteError            error
+}
+
+func (c *FakeConn) Read(b []byte) (int, error) {
+	if len(c.ReadData) > 0 {
+		n := copy(b, c.ReadData)
+		c.ReadData = c.ReadData[n:]
+		return n, nil
+	}
+	if c.ReadError != nil {
+		return 0, c.ReadError
+	}
+	return 0, io.EOF
+}
+
+func (c *FakeConn) Write(b []byte) (n int, err error) {
+	if c.WriteError != nil {
+		return 0, c.WriteError
+	}
+	c.WriteData = append(c.WriteData, b)
+	n = len(b)
+	return
+}
+
+func (*FakeConn) Close() (err error) {
+	return
+}
+
+func (*FakeConn) LocalAddr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+func (*FakeConn) RemoteAddr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+func (c *FakeConn) SetDeadline(t time.Time) (err error) {
+	return c.SetDeadlineError
+}
+
+func (c *FakeConn) SetReadDeadline(t time.Time) (err error) {
+	return c.SetReadDeadlineError
+}
+
+func (c *FakeConn) SetWriteDeadline(t time.Time) (err error) {
+	return c.SetWriteDeadlineError
+}

internal/engine/experiment/tlstool/internal/internal.go

@@ -0,0 +1,57 @@
+// Package internal contains the implementation of tlstool.
+package internal
+
+import (
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+// DialerConfig contains the config for creating a dialer
+type DialerConfig struct {
+	Dialer netx.Dialer
+	Delay  time.Duration
+	SNI    string
+}
+
+// NewSNISplitterDialer creates a new dialer that splits
+// outgoing messages such that the SNI should end up being
+// splitted into different TCP segments.
+func NewSNISplitterDialer(config DialerConfig) Dialer {
+	return Dialer{
+		Dialer: config.Dialer,
+		Delay:  config.Delay,
+		Splitter: func(b []byte) [][]byte {
+			return SNISplitter(b, []byte(config.SNI))
+		},
+	}
+}
+
+// NewThriceSplitterDialer creates a new dialer that splits
+// outgoing messages in three parts according to the circumvention
+// technique described by Kevin Boch in the Internet Measurement
+// Village 2020 <https://youtu.be/ksojSRFLbBM?t=1140>.
+func NewThriceSplitterDialer(config DialerConfig) Dialer {
+	return Dialer{
+		Dialer:   config.Dialer,
+		Delay:    config.Delay,
+		Splitter: Splitter84rest,
+	}
+}
+
+// NewRandomSplitterDialer creates a new dialer that splits
+// the SNI like the fixed splitting schema used by outline. See
+// github.com/Jigsaw-Code/outline-go-tun2socks.
+func NewRandomSplitterDialer(config DialerConfig) Dialer {
+	return Dialer{
+		Dialer:   config.Dialer,
+		Delay:    config.Delay,
+		Splitter: Splitter3264rand,
+	}
+}
+
+// NewVanillaDialer creates a new vanilla dialer that does
+// nothing and is used to establish a baseline.
+func NewVanillaDialer(config DialerConfig) Dialer {
+	return Dialer{Dialer: config.Dialer}
+}

internal/engine/experiment/tlstool/internal/internal_test.go

@@ -0,0 +1,40 @@
+package internal_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+var config = internal.DialerConfig{
+	Dialer: netx.NewDialer(netx.Config{}),
+	Delay:  10,
+	SNI:    "dns.google",
+}
+
+func dial(t *testing.T, d netx.Dialer) {
+	td := netx.NewTLSDialer(netx.Config{Dialer: d})
+	conn, err := td.DialTLSContext(context.Background(), "tcp", "dns.google:853")
+	if err != nil {
+		t.Fatal(err)
+	}
+	conn.Close()
+}
+
+func TestNewSNISplitterDialer(t *testing.T) {
+	dial(t, internal.NewSNISplitterDialer(config))
+}
+
+func TestNewThriceSplitterDialer(t *testing.T) {
+	dial(t, internal.NewThriceSplitterDialer(config))
+}
+
+func TestNewRandomSplitterDialer(t *testing.T) {
+	dial(t, internal.NewRandomSplitterDialer(config))
+}
+
+func TestNewVanillaDialer(t *testing.T) {
+	dial(t, internal.NewVanillaDialer(config))
+}

internal/engine/experiment/tlstool/internal/splitter.go

@@ -0,0 +1,67 @@
+package internal
+
+import (
+	"bytes"
+	"math/rand"
+	"time"
+)
+
+// SNISplitter splits input such that SNI is splitted across
+// a bunch of different output buffers.
+func SNISplitter(input []byte, sni []byte) (output [][]byte) {
+	idx := bytes.Index(input, sni)
+	if idx < 0 {
+		output = append(output, input)
+		return
+	}
+	output = append(output, input[:idx])
+	// TODO(bassosimone): splitting every three bytes causes
+	// a bunch of Unicode chatacters (e.g., in Chinese) to be
+	// sent as part of the same segment. Is that OK?
+	const segmentsize = 3
+	var buf []byte
+	for _, chr := range input[idx : idx+len(sni)] {
+		buf = append(buf, chr)
+		if len(buf) == segmentsize {
+			output = append(output, buf)
+			buf = nil
+		}
+	}
+	if len(buf) > 0 {
+		output = append(output, buf)
+		buf = nil
+	}
+	output = append(output, input[idx+len(sni):])
+	return
+}
+
+// Splitter84rest segments the specified buffer into three
+// sub-buffers containing respectively 8 bytes, 4 bytes, and
+// the rest of the buffer. This segment technique has been
+// described by Kevin Bock during the Internet Measurements
+// Village 2020: https://youtu.be/ksojSRFLbBM?t=1140.
+func Splitter84rest(input []byte) (output [][]byte) {
+	if len(input) <= 12 {
+		output = append(output, input)
+		return
+	}
+	output = append(output, input[:8])
+	output = append(output, input[8:12])
+	output = append(output, input[12:])
+	return
+}
+
+// Splitter3264rand splits the specified buffer at a random
+// offset between 32 and 64 bytes. This is the methodology used
+// by github.com/Jigsaw-Code/outline-go-tun2socks.
+func Splitter3264rand(input []byte) (output [][]byte) {
+	if len(input) <= 64 {
+		output = append(output, input)
+		return
+	}
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+	offset := rnd.Intn(32) + 32
+	output = append(output, input[:offset])
+	output = append(output, input[offset:])
+	return
+}

internal/engine/experiment/tlstool/internal/splitter_test.go

@@ -0,0 +1,143 @@
+package internal_test
+
+import (
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
+)
+
+func TestSplitter84restSmall(t *testing.T) {
+	input := []byte("1111222")
+	output := internal.Splitter84rest(input)
+	if len(output) != 1 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != "1111222" {
+		t.Fatal("invalid output[0]")
+	}
+}
+
+func TestSplitter84restGood(t *testing.T) {
+	input := []byte("1111222233334")
+	output := internal.Splitter84rest(input)
+	if len(output) != 3 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != "11112222" {
+		t.Fatal("invalid output[0]")
+	}
+	if string(output[1]) != "3333" {
+		t.Fatal("invalid output[1]")
+	}
+	if string(output[2]) != "4" {
+		t.Fatal("invalid output[2]")
+	}
+}
+
+func TestSplitter3264randSmall(t *testing.T) {
+	input := randx.Letters(64)
+	output := internal.Splitter3264rand([]byte(input))
+	if len(output) != 1 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != input {
+		t.Fatal("invalid output[0]")
+	}
+}
+
+func TestSplitter3264Works(t *testing.T) {
+	input := randx.Letters(65)
+	output := internal.Splitter3264rand([]byte(input))
+	for i := 0; i < 32; i++ {
+		if len(output) != 2 {
+			t.Fatal("invalid output length")
+		}
+		if len(output[0]) < 32 || len(output[0]) > 64 {
+			t.Fatal("invalid output[0] length")
+		}
+	}
+}
+
+func TestSNISplitterEasyCase(t *testing.T) {
+	input := []byte("11112222334555foo.barbar.deadbeef.com6777778888")
+	sni := []byte("barbar.deadbeef.com")
+	output := internal.SNISplitter(input, sni)
+	if len(output) != 9 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != "11112222334555foo." {
+		t.Fatal("invalid output[0]")
+	}
+	if string(output[1]) != "bar" {
+		t.Fatal("invalid output[1]")
+	}
+	if string(output[2]) != "bar" {
+		t.Fatal("invalid output[2]")
+	}
+	if string(output[3]) != ".de" {
+		t.Fatal("invalid output[3]")
+	}
+	if string(output[4]) != "adb" {
+		t.Fatal("invalid output[4]")
+	}
+	if string(output[5]) != "eef" {
+		t.Fatal("invalid output[5]")
+	}
+	if string(output[6]) != ".co" {
+		t.Fatal("invalid output[6]")
+	}
+	if string(output[7]) != "m" {
+		t.Fatal("invalid output[7]")
+	}
+	if string(output[8]) != "6777778888" {
+		t.Fatal("invalid output[8]")
+	}
+}
+
+func TestSNISplitterNoMatch(t *testing.T) {
+	input := []byte("11112222334555foo.barbar.deadbeef.com6777778888")
+	sni := []byte("www.google.com")
+	output := internal.SNISplitter(input, sni)
+	if len(output) != 1 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != string(input) {
+		t.Fatal("invalid output[0]")
+	}
+}
+
+func TestSNISplitterWithUnicode(t *testing.T) {
+	input := []byte("11112222334555你好世界.com6777778888")
+	sni := []byte("你好世界.com")
+	output := internal.SNISplitter(input, sni)
+	t.Log(string(output[2]))
+	t.Log(output)
+	if len(output) != 8 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != "11112222334555" {
+		t.Fatal("invalid output[0]")
+	}
+	if string(output[1]) != "你" {
+		t.Fatal("invalid output[1]")
+	}
+	if string(output[2]) != "好" {
+		t.Fatal("invalid output[2]")
+	}
+	if string(output[3]) != "世" {
+		t.Fatal("invalid output[3]")
+	}
+	if string(output[4]) != "界" {
+		t.Fatal("invalid output[4]")
+	}
+	if string(output[5]) != ".co" {
+		t.Fatal("invalid output[5]")
+	}
+	if string(output[6]) != "m" {
+		t.Fatal("invalid output[6]")
+	}
+	if string(output[7]) != "6777778888" {
+		t.Fatal("invalid output[7]")
+	}
+}

internal/engine/experiment/tlstool/internal/writer.go

@@ -0,0 +1,57 @@
+package internal
+
+import (
+	"net"
+	"time"
+)
+
+// SleeperWriter is a net.Conn that optionally sleeps for the
+// specified delay before posting each write.
+type SleeperWriter struct {
+	net.Conn
+	Delay time.Duration
+}
+
+func (c SleeperWriter) Write(b []byte) (int, error) {
+	<-time.After(c.Delay)
+	return c.Conn.Write(b)
+}
+
+// SplitterWriter is a writer that splits every outgoing buffer
+// according to the rules specified by the Splitter.
+//
+// Caveat
+//
+// The TLS ClientHello may be retransmitted if the server is
+// requesting us to restart the negotiation. Therefore, it is
+// not safe to just run the splitting once. Since this code
+// is meant to investigate TLS blocking, that's fine.
+type SplitterWriter struct {
+	net.Conn
+	Splitter func([]byte) [][]byte
+}
+
+// Write implements net.Conn.Write
+func (c SplitterWriter) Write(b []byte) (int, error) {
+	if c.Splitter != nil {
+		return Writev(c.Conn, c.Splitter(b))
+	}
+	return c.Conn.Write(b)
+}
+
+// Writev writes all the vectors inside datalist using the specified
+// conn. Returns either an error or the number of bytes sent. Note
+// that this function skips any empty entry in datalist.
+func Writev(conn net.Conn, datalist [][]byte) (int, error) {
+	var total int
+	for _, data := range datalist {
+		if len(data) > 0 {
+			count, err := conn.Write(data)
+			if err != nil {
+				return 0, err
+			}
+			total += count
+		}
+	}
+	return total, nil
+}

internal/engine/experiment/tlstool/internal/writer_test.go

@@ -0,0 +1,164 @@
+package internal_test
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+)
+
+func TestSleeperWriterWorksAsIntended(t *testing.T) {
+	origconn := &internal.FakeConn{}
+	const outdata = "deadbeefbadidea"
+	conn := internal.SleeperWriter{
+		Conn:  origconn,
+		Delay: 1 * time.Second,
+	}
+	before := time.Now()
+	count, err := conn.Write([]byte(outdata))
+	elapsed := time.Since(before)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != len(outdata) {
+		t.Fatal("unexpected count")
+	}
+	if len(origconn.WriteData) != 1 {
+		t.Fatal("wrong length of written data queue")
+	}
+	if string(origconn.WriteData[0]) != outdata {
+		t.Fatal("we did not write the right data")
+	}
+	if elapsed < 750*time.Millisecond {
+		t.Fatalf("unexpected elapsed time: %+v", elapsed)
+	}
+}
+
+func TestSplitterWriterNoSplitSuccess(t *testing.T) {
+	innerconn := &internal.FakeConn{}
+	conn := internal.SplitterWriter{Conn: innerconn}
+	const data = "deadbeef"
+	count, err := conn.Write([]byte(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != len(data) {
+		t.Fatal("invalid count")
+	}
+	if len(innerconn.WriteData) != 1 {
+		t.Fatal("invalid data queue")
+	}
+	if string(innerconn.WriteData[0]) != data {
+		t.Fatal("invalid written data")
+	}
+}
+
+func TestSplitterWriterNoSplitFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	innerconn := &internal.FakeConn{WriteError: expected}
+	conn := internal.SplitterWriter{Conn: innerconn}
+	const data = "deadbeef"
+	count, err := conn.Write([]byte(data))
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if count != 0 {
+		t.Fatal("invalid count")
+	}
+	if len(innerconn.WriteData) != 0 {
+		t.Fatal("invalid data queue")
+	}
+}
+
+func TestSplitterWriterSplitSuccess(t *testing.T) {
+	innerconn := &internal.FakeConn{}
+	conn := internal.SplitterWriter{
+		Conn: innerconn,
+		Splitter: func(b []byte) [][]byte {
+			return [][]byte{
+				b[:2], b[2:],
+			}
+		},
+	}
+	const data = "deadbeef"
+	count, err := conn.Write([]byte(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != len(data) {
+		t.Fatal("invalid count")
+	}
+	if len(innerconn.WriteData) != 2 {
+		t.Fatal("invalid data queue")
+	}
+	if string(innerconn.WriteData[0]) != "de" {
+		t.Fatal("invalid written data[0]")
+	}
+	if string(innerconn.WriteData[1]) != "adbeef" {
+		t.Fatal("invalid written data[1]")
+	}
+}
+
+func TestSplitterWriterSplitFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	innerconn := &internal.FakeConn{WriteError: expected}
+	conn := internal.SplitterWriter{
+		Conn: innerconn,
+		Splitter: func(b []byte) [][]byte {
+			return [][]byte{
+				b[:2], b[2:],
+			}
+		},
+	}
+	const data = "deadbeef"
+	count, err := conn.Write([]byte(data))
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if count != 0 {
+		t.Fatal("invalid count")
+	}
+	if len(innerconn.WriteData) != 0 {
+		t.Fatal("invalid data queue")
+	}
+}
+
+func TestWritevWorksWithAlsoEmptyData(t *testing.T) {
+	conn := &internal.FakeConn{}
+	datalist := [][]byte{
+		[]byte("deadbeef"),
+		[]byte(""),
+		[]byte("dead"),
+		nil,
+		[]byte("badidea"),
+		nil,
+	}
+	count, err := internal.Writev(conn, datalist)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != 19 {
+		t.Fatal("invalid number of bytes written")
+	}
+}
+
+func TestWritevFailsAsIntended(t *testing.T) {
+	expected := errors.New("mocked error")
+	conn := &internal.FakeConn{WriteError: expected}
+	datalist := [][]byte{
+		[]byte("deadbeef"),
+		[]byte(""),
+		[]byte("dead"),
+		nil,
+		[]byte("badidea"),
+		nil,
+	}
+	count, err := internal.Writev(conn, datalist)
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if count != 0 {
+		t.Fatal("invalid number of bytes written")
+	}
+}

internal/engine/experiment/tlstool/tlstool.go

@@ -0,0 +1,177 @@
+// Package tlstool contains a TLS tool that we are currently using
+// for running quick and dirty experiments. This tool will change
+// without notice and may be removed without notice.
+//
+// Caveats
+//
+// In particular, this experiment MAY panic when passed incorrect
+// input. This is acceptable because this is not production ready code.
+package tlstool
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+)
+
+const (
+	testName    = "tlstool"
+	testVersion = "0.1.0"
+)
+
+// Config contains the experiment configuration.
+type Config struct {
+	Delay int64  `ooni:"Milliseconds to wait between writes"`
+	SNI   string `ooni:"Force using the specified SNI"`
+}
+
+// TestKeys contains the experiment results.
+type TestKeys struct {
+	Experiment map[string]*ExperimentKeys `json:"experiment"`
+}
+
+// ExperimentKeys contains the specific experiment results.
+type ExperimentKeys struct {
+	Failure *string `json:"failure"`
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config Config
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+type method struct {
+	name      string
+	newDialer func(internal.DialerConfig) internal.Dialer
+}
+
+var allMethods = []method{{
+	name:      "vanilla",
+	newDialer: internal.NewVanillaDialer,
+}, {
+	name:      "snisplit",
+	newDialer: internal.NewSNISplitterDialer,
+}, {
+	name:      "random",
+	newDialer: internal.NewRandomSplitterDialer,
+}, {
+	name:      "thrice",
+	newDialer: internal.NewThriceSplitterDialer,
+}}
+
+// Run implements ExperimentMeasurer.Run.
+func (m Measurer) Run(
+	ctx context.Context,
+	sess model.ExperimentSession,
+	measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks,
+) error {
+	// TODO(bassosimone): wondering whether this experiment should
+	// actually be merged with sniblocking instead?
+	tk := new(TestKeys)
+	tk.Experiment = make(map[string]*ExperimentKeys)
+	measurement.TestKeys = tk
+	address := string(measurement.Input)
+	for idx, meth := range allMethods {
+		// TODO(bassosimone): here we actually want to use urlgetter
+		// if possible and collect standard test keys.
+		err := m.run(ctx, runConfig{
+			address:   address,
+			logger:    sess.Logger(),
+			newDialer: meth.newDialer,
+		})
+		percent := float64(idx) / float64(len(allMethods))
+		callbacks.OnProgress(percent, fmt.Sprintf("%s: %+v", meth.name, err))
+		tk.Experiment[meth.name] = &ExperimentKeys{
+			Failure: archival.NewFailure(err),
+		}
+	}
+	return nil
+}
+
+func (m Measurer) newDialer(logger model.Logger) netx.Dialer {
+	// TODO(bassosimone): this is a resolver that should hopefully work
+	// in many places. Maybe allow to configure it?
+	resolver, err := netx.NewDNSClientWithOverrides(netx.Config{Logger: logger},
+		"https://cloudflare.com/dns-query", "dns.cloudflare.com", "", "")
+	runtimex.PanicOnError(err, "cannot initialize resolver")
+	return netx.NewDialer(netx.Config{FullResolver: resolver, Logger: logger})
+}
+
+type runConfig struct {
+	address   string
+	logger    model.Logger
+	newDialer func(internal.DialerConfig) internal.Dialer
+}
+
+func (m Measurer) run(ctx context.Context, config runConfig) error {
+	dialer := config.newDialer(internal.DialerConfig{
+		Dialer: m.newDialer(config.logger),
+		Delay:  time.Duration(m.config.Delay) * time.Millisecond,
+		SNI:    m.pattern(config.address),
+	})
+	tdialer := netx.NewTLSDialer(netx.Config{
+		Dialer:    dialer,
+		Logger:    config.logger,
+		TLSConfig: m.tlsConfig(),
+	})
+	conn, err := tdialer.DialTLSContext(ctx, "tcp", config.address)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	return nil
+}
+
+func (m Measurer) tlsConfig() *tls.Config {
+	if m.config.SNI != "" {
+		return &tls.Config{ServerName: m.config.SNI}
+	}
+	return nil
+}
+
+func (m Measurer) pattern(address string) string {
+	if m.config.SNI != "" {
+		return m.config.SNI
+	}
+	addr, _, err := net.SplitHostPort(address)
+	// TODO(bassosimone): replace this panic with proper error checking.
+	runtimex.PanicOnError(err, "cannot split address")
+	return addr
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/tlstool/tlstool_test.go

@@ -0,0 +1,92 @@
+package tlstool_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestMeasurerExperimentNameVersion(t *testing.T) {
+	measurer := tlstool.NewExperimentMeasurer(tlstool.Config{})
+	if measurer.ExperimentName() != "tlstool" {
+		t.Fatal("unexpected ExperimentName")
+	}
+	if measurer.ExperimentVersion() != "0.1.0" {
+		t.Fatal("unexpected ExperimentVersion")
+	}
+}
+
+func TestRunWithExplicitSNI(t *testing.T) {
+	ctx := context.Background()
+	measurer := tlstool.NewExperimentMeasurer(tlstool.Config{
+		SNI: "dns.google",
+	})
+	measurement := new(model.Measurement)
+	measurement.Input = "8.8.8.8:853"
+	err := measurer.Run(
+		ctx,
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRunWithImplicitSNI(t *testing.T) {
+	ctx := context.Background()
+	measurer := tlstool.NewExperimentMeasurer(tlstool.Config{})
+	measurement := new(model.Measurement)
+	measurement.Input = "dns.google:853"
+	err := measurer.Run(
+		ctx,
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRunWithCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // cause failure
+	measurer := tlstool.NewExperimentMeasurer(tlstool.Config{})
+	measurement := new(model.Measurement)
+	measurement.Input = "dns.google:853"
+	err := measurer.Run(
+		ctx,
+		&mockable.Session{},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(tlstool.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestSummaryKeysGeneric(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &tlstool.TestKeys{}}
+	m := &tlstool.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(tlstool.SummaryKeys)
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/tor/tor.go

@@ -0,0 +1,476 @@
+// Package tor contains the tor experiment.
+//
+// Spec: https://github.com/ooni/spec/blob/master/nettests/ts-023-tor.md
+package tor
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/netxlogger"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/oonidatamodel"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/oonitemplates"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+const (
+	// parallelism is the number of parallel threads we use for this experiment
+	parallelism = 2
+
+	// testName is the name of this experiment
+	testName = "tor"
+
+	// testVersion is the version of this experiment
+	testVersion = "0.3.0"
+)
+
+// Config contains the experiment config.
+type Config struct{}
+
+// Summary contains a summary of what happened.
+type Summary struct {
+	Failure *string `json:"failure"`
+}
+
+// TargetResults contains the results of measuring a target.
+type TargetResults struct {
+	Agent          string                          `json:"agent"`
+	Failure        *string                         `json:"failure"`
+	NetworkEvents  oonidatamodel.NetworkEventsList `json:"network_events"`
+	Queries        oonidatamodel.DNSQueriesList    `json:"queries"`
+	Requests       oonidatamodel.RequestList       `json:"requests"`
+	Summary        map[string]Summary              `json:"summary"`
+	TargetAddress  string                          `json:"target_address"`
+	TargetName     string                          `json:"target_name,omitempty"`
+	TargetProtocol string                          `json:"target_protocol"`
+	TargetSource   string                          `json:"target_source,omitempty"`
+	TCPConnect     oonidatamodel.TCPConnectList    `json:"tcp_connect"`
+	TLSHandshakes  oonidatamodel.TLSHandshakesList `json:"tls_handshakes"`
+}
+
+func registerExtensions(m *model.Measurement) {
+	oonidatamodel.ExtHTTP.AddTo(m)
+	oonidatamodel.ExtNetevents.AddTo(m)
+	oonidatamodel.ExtDNS.AddTo(m)
+	oonidatamodel.ExtTCPConnect.AddTo(m)
+	oonidatamodel.ExtTLSHandshake.AddTo(m)
+}
+
+// fillSummary fills the Summary field used by the UI.
+func (tr *TargetResults) fillSummary() {
+	tr.Summary = make(map[string]Summary)
+	if len(tr.TCPConnect) < 1 {
+		return
+	}
+	tr.Summary[errorx.ConnectOperation] = Summary{
+		Failure: tr.TCPConnect[0].Status.Failure,
+	}
+	switch tr.TargetProtocol {
+	case "dir_port":
+		// The UI currently doesn't care about this protocol
+		// as long as drawing a table is concerned.
+	case "obfs4":
+		// We currently only perform an OBFS4 handshake, hence
+		// the final Failure is the handshake result
+		tr.Summary["handshake"] = Summary{
+			Failure: tr.Failure,
+		}
+	case "or_port_dirauth", "or_port":
+		if len(tr.TLSHandshakes) < 1 {
+			return
+		}
+		tr.Summary["handshake"] = Summary{
+			Failure: tr.TLSHandshakes[0].Failure,
+		}
+	}
+}
+
+// TestKeys contains tor test keys.
+type TestKeys struct {
+	DirPortTotal            int64                    `json:"dir_port_total"`
+	DirPortAccessible       int64                    `json:"dir_port_accessible"`
+	OBFS4Total              int64                    `json:"obfs4_total"`
+	OBFS4Accessible         int64                    `json:"obfs4_accessible"`
+	ORPortDirauthTotal      int64                    `json:"or_port_dirauth_total"`
+	ORPortDirauthAccessible int64                    `json:"or_port_dirauth_accessible"`
+	ORPortTotal             int64                    `json:"or_port_total"`
+	ORPortAccessible        int64                    `json:"or_port_accessible"`
+	Targets                 map[string]TargetResults `json:"targets"`
+}
+
+func (tk *TestKeys) fillToplevelKeys() {
+	for _, value := range tk.Targets {
+		switch value.TargetProtocol {
+		case "dir_port":
+			tk.DirPortTotal++
+			if value.Failure == nil {
+				tk.DirPortAccessible++
+			}
+		case "obfs4":
+			tk.OBFS4Total++
+			if value.Failure == nil {
+				tk.OBFS4Accessible++
+			}
+		case "or_port_dirauth":
+			tk.ORPortDirauthTotal++
+			if value.Failure == nil {
+				tk.ORPortDirauthAccessible++
+			}
+		case "or_port":
+			tk.ORPortTotal++
+			if value.Failure == nil {
+				tk.ORPortAccessible++
+			}
+		}
+	}
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config             Config
+	fetchTorTargets    func(ctx context.Context, clnt model.ExperimentOrchestraClient, cc string) (map[string]model.TorTarget, error)
+	newOrchestraClient func(ctx context.Context, sess model.ExperimentSession) (model.ExperimentOrchestraClient, error)
+}
+
+// NewMeasurer creates a new Measurer
+func NewMeasurer(config Config) *Measurer {
+	return &Measurer{
+		config: config,
+		fetchTorTargets: func(ctx context.Context, clnt model.ExperimentOrchestraClient, cc string) (map[string]model.TorTarget, error) {
+			return clnt.FetchTorTargets(ctx, cc)
+		},
+		newOrchestraClient: func(ctx context.Context, sess model.ExperimentSession) (model.ExperimentOrchestraClient, error) {
+			return sess.NewOrchestraClient(ctx)
+		},
+	}
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m *Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run.
+func (m *Measurer) Run(
+	ctx context.Context,
+	sess model.ExperimentSession,
+	measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks,
+) error {
+	targets, err := m.gimmeTargets(ctx, sess)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(
+		ctx, 15*time.Second*time.Duration(len(targets)),
+	)
+	defer cancel()
+	registerExtensions(measurement)
+	m.measureTargets(ctx, sess, measurement, callbacks, targets)
+	return nil
+}
+
+func (m *Measurer) gimmeTargets(
+	ctx context.Context, sess model.ExperimentSession,
+) (map[string]model.TorTarget, error) {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	clnt, err := m.newOrchestraClient(ctx, sess)
+	if err != nil {
+		return nil, err
+	}
+	return m.fetchTorTargets(ctx, clnt, sess.ProbeCC())
+}
+
+// keytarget contains a key and the related target
+type keytarget struct {
+	key    string
+	target model.TorTarget
+}
+
+// private returns whether a target is private. We consider private
+// every target coming from a non-empty data source.
+func (kt keytarget) private() bool {
+	return kt.target.Source != ""
+}
+
+// maybeTargetAddress returns the target address if the target is
+// not private, otherwise it returns `"[scrubbed]""`.
+func (kt keytarget) maybeTargetAddress() (address string) {
+	address = "[scrubbed]"
+	if !kt.private() {
+		address = kt.target.Address
+	}
+	return
+}
+
+func (m *Measurer) measureTargets(
+	ctx context.Context,
+	sess model.ExperimentSession,
+	measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks,
+	targets map[string]model.TorTarget,
+) {
+	// run measurements in parallel
+	var waitgroup sync.WaitGroup
+	rc := newResultsCollector(sess, measurement, callbacks)
+	waitgroup.Add(len(targets))
+	workch := make(chan keytarget)
+	for i := 0; i < parallelism; i++ {
+		go func(ch <-chan keytarget, total int) {
+			for kt := range ch {
+				rc.measureSingleTarget(ctx, kt, total)
+				waitgroup.Done()
+			}
+		}(workch, len(targets))
+	}
+	for key, target := range targets {
+		workch <- keytarget{key: key, target: target}
+	}
+	close(workch)
+	waitgroup.Wait()
+	// fill the measurement entry
+	testkeys := &TestKeys{Targets: rc.targetresults}
+	testkeys.fillToplevelKeys()
+	measurement.TestKeys = testkeys
+}
+
+type resultsCollector struct {
+	callbacks       model.ExperimentCallbacks
+	completed       *atomicx.Int64
+	flexibleConnect func(context.Context, keytarget) (oonitemplates.Results, error)
+	measurement     *model.Measurement
+	mu              sync.Mutex
+	sess            model.ExperimentSession
+	targetresults   map[string]TargetResults
+}
+
+func newResultsCollector(
+	sess model.ExperimentSession,
+	measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks,
+) *resultsCollector {
+	rc := &resultsCollector{
+		callbacks:     callbacks,
+		completed:     atomicx.NewInt64(),
+		measurement:   measurement,
+		sess:          sess,
+		targetresults: make(map[string]TargetResults),
+	}
+	rc.flexibleConnect = rc.defaultFlexibleConnect
+	return rc
+}
+
+func maybeSanitize(input TargetResults, kt keytarget) TargetResults {
+	if !kt.private() {
+		return input
+	}
+	data, err := json.Marshal(input)
+	runtimex.PanicOnError(err, "json.Marshal should not fail here")
+	// Implementation note: here we are using a strict scrubbing policy where
+	// we remove all IP _endpoints_, mainly for convenience, because we already
+	// have a well tested implementation that does that.
+	data = []byte(errorx.Scrub(string(data)))
+	var out TargetResults
+	err = json.Unmarshal(data, &out)
+	runtimex.PanicOnError(err, "json.Unmarshal should not fail here")
+	return out
+}
+
+func (rc *resultsCollector) measureSingleTarget(
+	ctx context.Context, kt keytarget, total int,
+) {
+	tk, err := rc.flexibleConnect(ctx, kt)
+	tr := TargetResults{
+		Agent:         "redirect",
+		Failure:       setFailure(err),
+		NetworkEvents: oonidatamodel.NewNetworkEventsList(tk),
+		Queries:       oonidatamodel.NewDNSQueriesList(tk),
+		Requests:      oonidatamodel.NewRequestList(tk),
+		TCPConnect:    oonidatamodel.NewTCPConnectList(tk),
+		TLSHandshakes: oonidatamodel.NewTLSHandshakesList(tk),
+	}
+	tr.fillSummary()
+	tr = maybeSanitize(tr, kt)
+	rc.mu.Lock()
+	tr.TargetAddress = kt.maybeTargetAddress()
+	tr.TargetName = kt.target.Name
+	tr.TargetProtocol = kt.target.Protocol
+	tr.TargetSource = kt.target.Source
+	rc.targetresults[kt.key] = tr
+	rc.mu.Unlock()
+	sofar := rc.completed.Add(1)
+	percentage := 0.0
+	if total > 0 {
+		percentage = float64(sofar) / float64(total)
+	}
+	rc.callbacks.OnProgress(percentage, fmt.Sprintf(
+		"tor: access %s/%s: %s", kt.maybeTargetAddress(), kt.target.Protocol,
+		errString(err),
+	))
+}
+
+// scrubbingLogger is a logger that scrubs endpoints from its output. We are using
+// it only here, currently, since we pay some performance penalty in that we evaluate
+// the string to be logged regardless of the logging level.
+//
+// TODO(bassosimone): find a more efficient way of scrubbing logs.
+type scrubbingLogger struct {
+	model.Logger
+}
+
+func (sl scrubbingLogger) Debug(message string) {
+	sl.Logger.Debug(errorx.Scrub(message))
+}
+
+func (sl scrubbingLogger) Debugf(format string, v ...interface{}) {
+	sl.Debug(fmt.Sprintf(format, v...))
+}
+
+func (sl scrubbingLogger) Info(message string) {
+	sl.Logger.Info(errorx.Scrub(message))
+}
+
+func (sl scrubbingLogger) Infof(format string, v ...interface{}) {
+	sl.Info(fmt.Sprintf(format, v...))
+}
+
+func (sl scrubbingLogger) Warn(message string) {
+	sl.Logger.Warn(errorx.Scrub(message))
+}
+
+func (sl scrubbingLogger) Warnf(format string, v ...interface{}) {
+	sl.Warn(fmt.Sprintf(format, v...))
+}
+
+func maybeScrubbingLogger(input model.Logger, kt keytarget) model.Logger {
+	if !kt.private() {
+		return input
+	}
+	return scrubbingLogger{Logger: input}
+}
+
+func (rc *resultsCollector) defaultFlexibleConnect(
+	ctx context.Context, kt keytarget,
+) (tk oonitemplates.Results, err error) {
+	logger := maybeScrubbingLogger(rc.sess.Logger(), kt)
+	switch kt.target.Protocol {
+	case "dir_port":
+		url := url.URL{
+			Host:   kt.target.Address,
+			Path:   "/tor/status-vote/current/consensus.z",
+			Scheme: "http",
+		}
+		const snapshotsize = 1 << 8 // no need to include all in report
+		r := oonitemplates.HTTPDo(ctx, oonitemplates.HTTPDoConfig{
+			Accept:                  httpheader.Accept(),
+			AcceptLanguage:          httpheader.AcceptLanguage(),
+			Beginning:               rc.measurement.MeasurementStartTimeSaved,
+			MaxEventsBodySnapSize:   snapshotsize,
+			MaxResponseBodySnapSize: snapshotsize,
+			Handler:                 netxlogger.NewHandler(logger),
+			Method:                  "GET",
+			URL:                     url.String(),
+			UserAgent:               httpheader.UserAgent(),
+		})
+		tk, err = r.TestKeys, r.Error
+	case "or_port", "or_port_dirauth":
+		r := oonitemplates.TLSConnect(ctx, oonitemplates.TLSConnectConfig{
+			Address:            kt.target.Address,
+			Beginning:          rc.measurement.MeasurementStartTimeSaved,
+			InsecureSkipVerify: true,
+			Handler:            netxlogger.NewHandler(logger),
+		})
+		tk, err = r.TestKeys, r.Error
+	case "obfs4":
+		r := oonitemplates.OBFS4Connect(ctx, oonitemplates.OBFS4ConnectConfig{
+			Address:      kt.target.Address,
+			Beginning:    rc.measurement.MeasurementStartTimeSaved,
+			Handler:      netxlogger.NewHandler(logger),
+			Params:       kt.target.Params,
+			StateBaseDir: rc.sess.TempDir(),
+		})
+		tk, err = r.TestKeys, r.Error
+	default:
+		r := oonitemplates.TCPConnect(ctx, oonitemplates.TCPConnectConfig{
+			Address:   kt.target.Address,
+			Beginning: rc.measurement.MeasurementStartTimeSaved,
+			Handler:   netxlogger.NewHandler(logger),
+		})
+		tk, err = r.TestKeys, r.Error
+	}
+	return
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return NewMeasurer(config)
+}
+
+func errString(err error) (s string) {
+	s = "success"
+	if err != nil {
+		s = err.Error()
+	}
+	return
+}
+
+func setFailure(err error) (s *string) {
+	if err != nil {
+		descr := err.Error()
+		s = &descr
+	}
+	return
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	DirPortTotal            int64 `json:"dir_port_total"`
+	DirPortAccessible       int64 `json:"dir_port_accessible"`
+	OBFS4Total              int64 `json:"obfs4_total"`
+	OBFS4Accessible         int64 `json:"obfs4_accessible"`
+	ORPortDirauthTotal      int64 `json:"or_port_dirauth_total"`
+	ORPortDirauthAccessible int64 `json:"or_port_dirauth_accessible"`
+	ORPortTotal             int64 `json:"or_port_total"`
+	ORPortAccessible        int64 `json:"or_port_accessible"`
+	IsAnomaly               bool  `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	sk.DirPortTotal = tk.DirPortTotal
+	sk.DirPortAccessible = tk.DirPortAccessible
+	sk.OBFS4Total = tk.OBFS4Total
+	sk.OBFS4Accessible = tk.OBFS4Accessible
+	sk.ORPortDirauthTotal = tk.ORPortDirauthTotal
+	sk.ORPortDirauthAccessible = tk.ORPortDirauthAccessible
+	sk.ORPortTotal = tk.ORPortTotal
+	sk.ORPortAccessible = tk.ORPortAccessible
+	sk.IsAnomaly = ((sk.DirPortAccessible <= 0 && sk.DirPortTotal > 0) ||
+		(sk.OBFS4Accessible <= 0 && sk.OBFS4Total > 0) ||
+		(sk.ORPortDirauthAccessible <= 0 && sk.ORPortDirauthTotal > 0) ||
+		(sk.ORPortAccessible <= 0 && sk.ORPortTotal > 0))
+	return sk, nil
+}

internal/engine/experiment/tor/tor_test.go

@@ -0,0 +1,931 @@
+package tor
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/oonidatamodel"
+	"github.com/ooni/probe-cli/v3/internal/engine/legacy/oonitemplates"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	if measurer.ExperimentName() != "tor" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.3.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestMeasurerMeasureNewOrchestraClientError(t *testing.T) {
+	measurer := NewMeasurer(Config{})
+	expected := errors.New("mocked error")
+	measurer.newOrchestraClient = func(ctx context.Context, sess model.ExperimentSession) (model.ExperimentOrchestraClient, error) {
+		return nil, expected
+	}
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestMeasurerMeasureFetchTorTargetsError(t *testing.T) {
+	measurer := NewMeasurer(Config{})
+	expected := errors.New("mocked error")
+	measurer.newOrchestraClient = func(ctx context.Context, sess model.ExperimentSession) (model.ExperimentOrchestraClient, error) {
+		return new(probeservices.Client), nil
+	}
+	measurer.fetchTorTargets = func(ctx context.Context, clnt model.ExperimentOrchestraClient, cc string) (map[string]model.TorTarget, error) {
+		return nil, expected
+	}
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestMeasurerMeasureFetchTorTargetsEmptyList(t *testing.T) {
+	measurer := NewMeasurer(Config{})
+	measurer.newOrchestraClient = func(ctx context.Context, sess model.ExperimentSession) (model.ExperimentOrchestraClient, error) {
+		return new(probeservices.Client), nil
+	}
+	measurer.fetchTorTargets = func(ctx context.Context, clnt model.ExperimentOrchestraClient, cc string) (map[string]model.TorTarget, error) {
+		return nil, nil
+	}
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*TestKeys)
+	if len(tk.Targets) != 0 {
+		t.Fatal("expected no targets here")
+	}
+}
+
+func TestMeasurerMeasureGoodWithMockedOrchestra(t *testing.T) {
+	// This test mocks orchestra to return a nil list of targets, so the code runs
+	// but we don't perform any actualy network actions.
+	measurer := NewMeasurer(Config{})
+	measurer.newOrchestraClient = func(ctx context.Context, sess model.ExperimentSession) (model.ExperimentOrchestraClient, error) {
+		return new(probeservices.Client), nil
+	}
+	measurer.fetchTorTargets = func(ctx context.Context, clnt model.ExperimentOrchestraClient, cc string) (map[string]model.TorTarget, error) {
+		return nil, nil
+	}
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMeasurerMeasureGood(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := NewMeasurer(Config{})
+	sess := newsession()
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		sess,
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+var staticPrivateTestingTargetEndpoint = "192.95.36.142:443"
+
+var staticPrivateTestingTarget = model.TorTarget{
+	Address: staticPrivateTestingTargetEndpoint,
+	Params: map[string][]string{
+		"cert": {
+			"qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ",
+		},
+		"iat-mode": {"1"},
+	},
+	Protocol: "obfs4",
+	Source:   "bridgedb",
+}
+
+func TestMeasurerMeasureSanitiseOutput(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := NewMeasurer(Config{})
+	sess := newsession()
+	key := "xyz-xyz-xyz-theCh2ju-ahG4chei-Ai2eka0a"
+	sess.MockableOrchestraClient = &mockable.ExperimentOrchestraClient{
+		MockableFetchTorTargetsResult: map[string]model.TorTarget{
+			key: staticPrivateTestingTarget,
+		},
+	}
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		sess,
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := json.Marshal(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*TestKeys)
+	entry := tk.Targets[key]
+	if entry.Failure != nil {
+		t.Fatal("measurement failed unexpectedly")
+	}
+	if !bytes.Contains(data, []byte(key)) {
+		t.Fatal("cannot find expected key")
+	}
+	if bytes.Contains(data, []byte(staticPrivateTestingTargetEndpoint)) {
+		t.Fatal("endpoint found in serialized measurement")
+	}
+	if !bytes.Contains(data, []byte("[scrubbed]")) {
+		t.Fatal("[scrubbed] not found in serialized measurement")
+	}
+}
+
+var staticTestingTargets = []model.TorTarget{
+	{
+		Address: "192.95.36.142:443",
+		Params: map[string][]string{
+			"cert": {
+				"qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ",
+			},
+			"iat-mode": {"1"},
+		},
+		Protocol: "obfs4",
+	},
+	{
+		Address:  "66.111.2.131:9030",
+		Protocol: "dir_port",
+	},
+	{
+		Address:  "66.111.2.131:9001",
+		Protocol: "or_port",
+	},
+	{
+		Address:  "1.1.1.1:80",
+		Protocol: "tcp",
+	},
+}
+
+func TestMeasurerMeasureTargetsNoInput(t *testing.T) {
+	var measurement model.Measurement
+	measurer := new(Measurer)
+	measurer.measureTargets(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		&measurement,
+		model.NewPrinterCallbacks(log.Log),
+		nil,
+	)
+	if len(measurement.TestKeys.(*TestKeys).Targets) != 0 {
+		t.Fatal("expected no measurements here")
+	}
+}
+
+func TestMeasurerMeasureTargetsCanceledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // so we don't actually do anything
+	var measurement model.Measurement
+	measurer := new(Measurer)
+	measurer.measureTargets(
+		ctx,
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		&measurement,
+		model.NewPrinterCallbacks(log.Log),
+		map[string]model.TorTarget{
+			"xx": staticTestingTargets[0],
+		},
+	)
+	targets := measurement.TestKeys.(*TestKeys).Targets
+	if len(targets) != 1 {
+		t.Fatal("expected single measurements here")
+	}
+	if _, found := targets["xx"]; !found {
+		t.Fatal("the target we expected is missing")
+	}
+	tgt := targets["xx"]
+	if *tgt.Failure != "interrupted" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func wrapTestingTarget(tt model.TorTarget) keytarget {
+	return keytarget{
+		key:    "xx", // using an super simple key; should work anyway
+		target: tt,
+	}
+}
+
+func TestResultsCollectorMeasureSingleTargetGood(t *testing.T) {
+	rc := newResultsCollector(
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	rc.flexibleConnect = func(context.Context, keytarget) (oonitemplates.Results, error) {
+		return oonitemplates.Results{}, nil
+	}
+	rc.measureSingleTarget(
+		context.Background(), wrapTestingTarget(staticTestingTargets[0]),
+		len(staticTestingTargets),
+	)
+	if len(rc.targetresults) != 1 {
+		t.Fatal("wrong number of entries")
+	}
+	// Implementation note: here we won't bother with checking that
+	// oonidatamodel works correctly because we already test that.
+	if rc.targetresults["xx"].Agent != "redirect" {
+		t.Fatal("agent is invalid")
+	}
+	if rc.targetresults["xx"].Failure != nil {
+		t.Fatal("failure is invalid")
+	}
+	if rc.targetresults["xx"].TargetAddress != staticTestingTargets[0].Address {
+		t.Fatal("target address is invalid")
+	}
+	if rc.targetresults["xx"].TargetProtocol != staticTestingTargets[0].Protocol {
+		t.Fatal("target protocol is invalid")
+	}
+}
+
+func TestResultsCollectorMeasureSingleTargetWithFailure(t *testing.T) {
+	rc := newResultsCollector(
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	rc.flexibleConnect = func(context.Context, keytarget) (oonitemplates.Results, error) {
+		return oonitemplates.Results{}, errors.New("mocked error")
+	}
+	rc.measureSingleTarget(
+		context.Background(), keytarget{
+			key:    "xx", // using an super simple key; should work anyway
+			target: staticTestingTargets[0],
+		},
+		len(staticTestingTargets),
+	)
+	if len(rc.targetresults) != 1 {
+		t.Fatal("wrong number of entries")
+	}
+	// Implementation note: here we won't bother with checking that
+	// oonidatamodel works correctly because we already test that.
+	if rc.targetresults["xx"].Agent != "redirect" {
+		t.Fatal("agent is invalid")
+	}
+	if *rc.targetresults["xx"].Failure != "mocked error" {
+		t.Fatal("failure is invalid")
+	}
+	if rc.targetresults["xx"].TargetAddress != staticTestingTargets[0].Address {
+		t.Fatal("target address is invalid")
+	}
+	if rc.targetresults["xx"].TargetProtocol != staticTestingTargets[0].Protocol {
+		t.Fatal("target protocol is invalid")
+	}
+}
+
+func TestDefautFlexibleConnectDirPort(t *testing.T) {
+	rc := newResultsCollector(
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	tk, err := rc.defaultFlexibleConnect(ctx, wrapTestingTarget(staticTestingTargets[1]))
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if !strings.HasSuffix(err.Error(), "interrupted") {
+		t.Fatal("not the error we expected")
+	}
+	if tk.HTTPRequests == nil {
+		t.Fatal("expected HTTP data here")
+	}
+}
+
+func TestDefautFlexibleConnectOrPort(t *testing.T) {
+	rc := newResultsCollector(
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	tk, err := rc.defaultFlexibleConnect(ctx, wrapTestingTarget(staticTestingTargets[2]))
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if err.Error() != "interrupted" {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Connects == nil {
+		t.Fatal("expected connects data here")
+	}
+	if tk.NetworkEvents == nil {
+		t.Fatal("expected network events data here")
+	}
+}
+
+func TestDefautFlexibleConnectOBFS4(t *testing.T) {
+	rc := newResultsCollector(
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	tk, err := rc.defaultFlexibleConnect(ctx, wrapTestingTarget(staticTestingTargets[0]))
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if err.Error() != "interrupted" {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Connects == nil {
+		t.Fatal("expected connects data here")
+	}
+	if tk.NetworkEvents == nil {
+		t.Fatal("expected network events data here")
+	}
+}
+
+func TestDefautFlexibleConnectDefault(t *testing.T) {
+	rc := newResultsCollector(
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		new(model.Measurement),
+		model.NewPrinterCallbacks(log.Log),
+	)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	tk, err := rc.defaultFlexibleConnect(ctx, wrapTestingTarget(staticTestingTargets[3]))
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if err.Error() != "interrupted" {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if tk.Connects == nil {
+		t.Fatalf("expected connects data here, found: %+v", tk.Connects)
+	}
+}
+
+func TestErrString(t *testing.T) {
+	if errString(nil) != "success" {
+		t.Fatal("not working with nil")
+	}
+	if errString(errors.New("antani")) != "antani" {
+		t.Fatal("not working with error")
+	}
+}
+
+func TestSummary(t *testing.T) {
+	t.Run("without any piece of data", func(t *testing.T) {
+		tr := new(TargetResults)
+		tr.fillSummary()
+		if len(tr.Summary) != 0 {
+			t.Fatal("summary must be empty")
+		}
+	})
+
+	t.Run("with a TCP connect and nothing else", func(t *testing.T) {
+		tr := new(TargetResults)
+		failure := "mocked_error"
+		tr.TCPConnect = append(tr.TCPConnect, oonidatamodel.TCPConnectEntry{
+			Status: oonidatamodel.TCPConnectStatus{
+				Success: true,
+				Failure: &failure,
+			},
+		})
+		tr.fillSummary()
+		if len(tr.Summary) != 1 {
+			t.Fatal("cannot find expected entry")
+		}
+		if *tr.Summary[errorx.ConnectOperation].Failure != failure {
+			t.Fatal("invalid failure")
+		}
+	})
+
+	t.Run("for OBFS4", func(t *testing.T) {
+		tr := new(TargetResults)
+		tr.TCPConnect = append(tr.TCPConnect, oonidatamodel.TCPConnectEntry{
+			Status: oonidatamodel.TCPConnectStatus{
+				Success: true,
+			},
+		})
+		failure := "mocked_error"
+		tr.TargetProtocol = "obfs4"
+		tr.Failure = &failure
+		tr.fillSummary()
+		if len(tr.Summary) != 2 {
+			t.Fatal("cannot find expected entry")
+		}
+		if tr.Summary[errorx.ConnectOperation].Failure != nil {
+			t.Fatal("invalid failure")
+		}
+		if *tr.Summary["handshake"].Failure != failure {
+			t.Fatal("invalid failure")
+		}
+	})
+
+	t.Run("for or_port/or_port_dirauth", func(t *testing.T) {
+		doit := func(targetProtocol string, handshake *oonidatamodel.TLSHandshake) {
+			tr := new(TargetResults)
+			tr.TCPConnect = append(tr.TCPConnect, oonidatamodel.TCPConnectEntry{
+				Status: oonidatamodel.TCPConnectStatus{
+					Success: true,
+				},
+			})
+			tr.TargetProtocol = targetProtocol
+			if handshake != nil {
+				tr.TLSHandshakes = append(tr.TLSHandshakes, *handshake)
+			}
+			tr.fillSummary()
+			if len(tr.Summary) < 1 {
+				t.Fatal("cannot find expected entry")
+			}
+			if tr.Summary[errorx.ConnectOperation].Failure != nil {
+				t.Fatal("invalid failure")
+			}
+			if handshake == nil {
+				if len(tr.Summary) != 1 {
+					t.Fatal("unexpected summary length")
+				}
+				return
+			}
+			if len(tr.Summary) != 2 {
+				t.Fatal("unexpected summary length")
+			}
+			if tr.Summary["handshake"].Failure != handshake.Failure {
+				t.Fatal("the failure value is unexpected")
+			}
+		}
+		doit("or_port_dirauth", nil)
+		doit("or_port", nil)
+		doit("or_port", &oonidatamodel.TLSHandshake{
+			Failure: (func() *string {
+				s := io.EOF.Error()
+				return &s
+			})(),
+		})
+	})
+}
+
+func TestFillToplevelKeys(t *testing.T) {
+	var tr TargetResults
+	tr.TargetProtocol = "or_port"
+	tk := new(TestKeys)
+	tk.Targets = make(map[string]TargetResults)
+	tk.Targets["xxx"] = tr
+	tk.fillToplevelKeys()
+	if tk.ORPortTotal != 1 {
+		t.Fatal("unexpected ORPortTotal value")
+	}
+}
+
+func newsession() *mockable.Session {
+	return &mockable.Session{
+		MockableLogger:     log.Log,
+		MockableHTTPClient: http.DefaultClient,
+	}
+}
+
+var referenceTargetResult = []byte(`{
+	"agent": "redirect",
+	"failure": null,
+	"network_events": [
+	  {
+		"address": "85.31.186.98:443",
+		"conn_id": 19,
+		"dial_id": 21,
+		"failure": null,
+		"operation": "connect",
+		"proto": "tcp",
+		"t": 8.639313
+	  },
+	  {
+		"conn_id": 19,
+		"failure": null,
+		"num_bytes": 1915,
+		"operation": "write",
+		"proto": "tcp",
+		"t": 8.639686
+	  },
+	  {
+		"conn_id": 19,
+		"failure": null,
+		"num_bytes": 1440,
+		"operation": "read",
+		"proto": "tcp",
+		"t": 8.691708
+	  },
+	  {
+		"conn_id": 19,
+		"failure": null,
+		"num_bytes": 1440,
+		"operation": "read",
+		"proto": "tcp",
+		"t": 8.691912
+	  },
+	  {
+		"conn_id": 19,
+		"failure": null,
+		"num_bytes": 1383,
+		"operation": "read",
+		"proto": "tcp",
+		"t": 8.69234
+	  }
+	],
+	"queries": null,
+	"requests": null,
+	"summary": {
+	  "connect": {
+		"failure": null
+	  }
+	},
+	"target_address": "85.31.186.98:443",
+	"target_protocol": "obfs4",
+	"tcp_connect": [
+	  {
+		"conn_id": 19,
+		"dial_id": 21,
+		"ip": "85.31.186.98",
+		"port": 443,
+		"status": {
+		  "failure": null,
+		  "success": true
+		},
+		"t": 8.639313
+	  }
+	],
+	"tls_handshakes": null
+  }`)
+
+var scrubbedTargetResult = []byte(`{
+	"agent": "redirect",
+	"failure": null,
+	"network_events": [
+	  {
+		"address": "[scrubbed]",
+		"conn_id": 19,
+		"dial_id": 21,
+		"failure": null,
+		"operation": "connect",
+		"proto": "tcp",
+		"t": 8.639313
+	  },
+	  {
+		"conn_id": 19,
+		"failure": null,
+		"num_bytes": 1915,
+		"operation": "write",
+		"proto": "tcp",
+		"t": 8.639686
+	  },
+	  {
+		"conn_id": 19,
+		"failure": null,
+		"num_bytes": 1440,
+		"operation": "read",
+		"proto": "tcp",
+		"t": 8.691708
+	  },
+	  {
+		"conn_id": 19,
+		"failure": null,
+		"num_bytes": 1440,
+		"operation": "read",
+		"proto": "tcp",
+		"t": 8.691912
+	  },
+	  {
+		"conn_id": 19,
+		"failure": null,
+		"num_bytes": 1383,
+		"operation": "read",
+		"proto": "tcp",
+		"t": 8.69234
+	  }
+	],
+	"queries": null,
+	"requests": null,
+	"summary": {
+	  "connect": {
+		"failure": null
+	  }
+	},
+	"target_address": "[scrubbed]",
+	"target_protocol": "obfs4",
+	"tcp_connect": [
+	  {
+		"conn_id": 19,
+		"dial_id": 21,
+		"ip": "[scrubbed]",
+		"port": 443,
+		"status": {
+		  "failure": null,
+		  "success": true
+		},
+		"t": 8.639313
+	  }
+	],
+	"tls_handshakes": null
+  }`)
+
+func TestMaybeSanitize(t *testing.T) {
+	var input TargetResults
+	if err := json.Unmarshal(referenceTargetResult, &input); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("nothing to do", func(t *testing.T) {
+		out := maybeSanitize(input, keytarget{target: model.TorTarget{Source: ""}})
+		diff := cmp.Diff(input, out)
+		if diff != "" {
+			t.Fatal(diff)
+		}
+	})
+	t.Run("scrubbing to do", func(t *testing.T) {
+		var expected TargetResults
+		if err := json.Unmarshal(scrubbedTargetResult, &expected); err != nil {
+			t.Fatal(err)
+		}
+		out := maybeSanitize(input, keytarget{target: model.TorTarget{
+			Address: "85.31.186.98:443",
+			Source:  "bridgedb",
+		}})
+		diff := cmp.Diff(expected, out)
+		if diff != "" {
+			t.Fatal(diff)
+		}
+	})
+}
+
+type savingLogger struct {
+	debug []string
+	info  []string
+	warn  []string
+}
+
+func (sl *savingLogger) Debug(message string) {
+	sl.debug = append(sl.debug, message)
+}
+
+func (sl *savingLogger) Debugf(format string, v ...interface{}) {
+	sl.Debug(fmt.Sprintf(format, v...))
+}
+
+func (sl *savingLogger) Info(message string) {
+	sl.info = append(sl.info, message)
+}
+
+func (sl *savingLogger) Infof(format string, v ...interface{}) {
+	sl.Info(fmt.Sprintf(format, v...))
+}
+
+func (sl *savingLogger) Warn(message string) {
+	sl.warn = append(sl.warn, message)
+}
+
+func (sl *savingLogger) Warnf(format string, v ...interface{}) {
+	sl.Warn(fmt.Sprintf(format, v...))
+}
+
+func TestScrubLogger(t *testing.T) {
+	input := "failure: 130.192.91.211:443: no route the host"
+	expect := "failure: [scrubbed]: no route the host"
+
+	t.Run("for debug", func(t *testing.T) {
+		logger := new(savingLogger)
+		scrubber := scrubbingLogger{Logger: logger}
+		scrubber.Debug(input)
+		if len(logger.debug) != 1 && len(logger.info) != 0 && len(logger.warn) != 0 {
+			t.Fatal("unexpected number of log lines written")
+		}
+		if logger.debug[0] != expect {
+			t.Fatal("unexpected output written")
+		}
+	})
+
+	t.Run("for debugf", func(t *testing.T) {
+		logger := new(savingLogger)
+		scrubber := scrubbingLogger{Logger: logger}
+		scrubber.Debugf("%s", input)
+		if len(logger.debug) != 1 && len(logger.info) != 0 && len(logger.warn) != 0 {
+			t.Fatal("unexpected number of log lines written")
+		}
+		if logger.debug[0] != expect {
+			t.Fatal("unexpected output written")
+		}
+	})
+
+	t.Run("for info", func(t *testing.T) {
+		logger := new(savingLogger)
+		scrubber := scrubbingLogger{Logger: logger}
+		scrubber.Info(input)
+		if len(logger.debug) != 0 && len(logger.info) != 1 && len(logger.warn) != 0 {
+			t.Fatal("unexpected number of log lines written")
+		}
+		if logger.info[0] != expect {
+			t.Fatal("unexpected output written")
+		}
+	})
+
+	t.Run("for infof", func(t *testing.T) {
+		logger := new(savingLogger)
+		scrubber := scrubbingLogger{Logger: logger}
+		scrubber.Infof("%s", input)
+		if len(logger.debug) != 0 && len(logger.info) != 1 && len(logger.warn) != 0 {
+			t.Fatal("unexpected number of log lines written")
+		}
+		if logger.info[0] != expect {
+			t.Fatal("unexpected output written")
+		}
+	})
+
+	t.Run("for warn", func(t *testing.T) {
+		logger := new(savingLogger)
+		scrubber := scrubbingLogger{Logger: logger}
+		scrubber.Warn(input)
+		if len(logger.debug) != 0 && len(logger.info) != 0 && len(logger.warn) != 1 {
+			t.Fatal("unexpected number of log lines written")
+		}
+		if logger.warn[0] != expect {
+			t.Fatal("unexpected output written")
+		}
+	})
+
+	t.Run("for warnf", func(t *testing.T) {
+		logger := new(savingLogger)
+		scrubber := scrubbingLogger{Logger: logger}
+		scrubber.Warnf("%s", input)
+		if len(logger.debug) != 0 && len(logger.info) != 0 && len(logger.warn) != 1 {
+			t.Fatal("unexpected number of log lines written")
+		}
+		if logger.warn[0] != expect {
+			t.Fatal("unexpected output written")
+		}
+	})
+}
+
+func TestMaybeScrubbingLogger(t *testing.T) {
+	var input model.Logger = new(savingLogger)
+
+	t.Run("for when we don't need to save", func(t *testing.T) {
+		kt := keytarget{target: model.TorTarget{
+			Source: "",
+		}}
+		out := maybeScrubbingLogger(input, kt)
+		if out != input {
+			t.Fatal("not the output we expected")
+		}
+		if _, ok := out.(*savingLogger); !ok {
+			t.Fatal("not the output type we expected")
+		}
+	})
+
+	t.Run("for when we need to save", func(t *testing.T) {
+		kt := keytarget{target: model.TorTarget{
+			Source: "bridgedb",
+		}}
+		out := maybeScrubbingLogger(input, kt)
+		if out == input {
+			t.Fatal("not the output value we expected")
+		}
+		if _, ok := out.(scrubbingLogger); !ok {
+			t.Fatal("not the output type we expected")
+		}
+	})
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysWorksAsIntended(t *testing.T) {
+	tests := []struct {
+		tk        TestKeys
+		isAnomaly bool
+	}{{
+		tk:        TestKeys{},
+		isAnomaly: false,
+	}, {
+		tk:        TestKeys{DirPortAccessible: 1, DirPortTotal: 3},
+		isAnomaly: false,
+	}, {
+		tk:        TestKeys{DirPortAccessible: 0, DirPortTotal: 3},
+		isAnomaly: true,
+	}, {
+		tk:        TestKeys{OBFS4Accessible: 1, OBFS4Total: 3},
+		isAnomaly: false,
+	}, {
+		tk:        TestKeys{OBFS4Accessible: 0, OBFS4Total: 3},
+		isAnomaly: true,
+	}, {
+		tk:        TestKeys{ORPortDirauthAccessible: 1, ORPortDirauthTotal: 3},
+		isAnomaly: false,
+	}, {
+		tk:        TestKeys{ORPortDirauthAccessible: 0, ORPortDirauthTotal: 3},
+		isAnomaly: true,
+	}, {
+		tk:        TestKeys{ORPortAccessible: 1, ORPortTotal: 3},
+		isAnomaly: false,
+	}, {
+		tk:        TestKeys{ORPortAccessible: 0, ORPortTotal: 3},
+		isAnomaly: true,
+	}}
+	for idx, tt := range tests {
+		t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
+			m := &Measurer{}
+			measurement := &model.Measurement{TestKeys: &tt.tk}
+			got, err := m.GetSummaryKeys(measurement)
+			if err != nil {
+				t.Fatal(err)
+				return
+			}
+			sk := got.(SummaryKeys)
+			if sk.IsAnomaly != tt.isAnomaly {
+				t.Fatal("unexpected isAnomaly value")
+			}
+		})
+	}
+}

internal/engine/experiment/urlgetter/.gitignore

@@ -0,0 +1 @@
+/urlgetter-tunnel

internal/engine/experiment/urlgetter/configurer.go

@@ -0,0 +1,102 @@
+package urlgetter
+
+import (
+	"crypto/tls"
+	"errors"
+	"net"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+// The Configurer job is to construct a Configuration that can
+// later be used by the measurer to perform measurements.
+type Configurer struct {
+	Config   Config
+	Logger   model.Logger
+	ProxyURL *url.URL
+	Saver    *trace.Saver
+}
+
+// The Configuration is the configuration for running a measurement.
+type Configuration struct {
+	HTTPConfig netx.Config
+	DNSClient  netx.DNSClient
+}
+
+// CloseIdleConnections will close idle connections, if needed.
+func (c Configuration) CloseIdleConnections() {
+	c.DNSClient.CloseIdleConnections()
+}
+
+// NewConfiguration builds a new measurement configuration.
+func (c Configurer) NewConfiguration() (Configuration, error) {
+	// set up defaults
+	configuration := Configuration{
+		HTTPConfig: netx.Config{
+			BogonIsError:        c.Config.RejectDNSBogons,
+			CacheResolutions:    true,
+			CertPool:            c.Config.CertPool,
+			ContextByteCounting: true,
+			DialSaver:           c.Saver,
+			HTTP3Enabled:        c.Config.HTTP3Enabled,
+			HTTPSaver:           c.Saver,
+			Logger:              c.Logger,
+			ReadWriteSaver:      c.Saver,
+			ResolveSaver:        c.Saver,
+			TLSSaver:            c.Saver,
+		},
+	}
+	// fill DNS cache
+	if c.Config.DNSCache != "" {
+		entry := strings.Split(c.Config.DNSCache, " ")
+		if len(entry) < 2 {
+			return configuration, errors.New("invalid DNSCache string")
+		}
+		domainregex := regexp.MustCompile(`^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$`)
+		if !domainregex.MatchString(entry[0]) {
+			return configuration, errors.New("invalid domain in DNSCache")
+		}
+		var addresses []string
+		for i := 1; i < len(entry); i++ {
+			if net.ParseIP(entry[i]) == nil {
+				return configuration, errors.New("invalid IP in DNSCache")
+			}
+			addresses = append(addresses, entry[i])
+		}
+		configuration.HTTPConfig.DNSCache = map[string][]string{
+			entry[0]: addresses,
+		}
+	}
+	dnsclient, err := netx.NewDNSClientWithOverrides(
+		configuration.HTTPConfig, c.Config.ResolverURL,
+		c.Config.DNSHTTPHost, c.Config.DNSTLSServerName,
+		c.Config.DNSTLSVersion,
+	)
+	if err != nil {
+		return configuration, err
+	}
+	configuration.DNSClient = dnsclient
+	configuration.HTTPConfig.BaseResolver = dnsclient.Resolver
+	// configure TLS
+	configuration.HTTPConfig.TLSConfig = &tls.Config{
+		NextProtos: []string{"h2", "http/1.1"},
+	}
+	if c.Config.TLSServerName != "" {
+		configuration.HTTPConfig.TLSConfig.ServerName = c.Config.TLSServerName
+	}
+	err = netx.ConfigureTLSVersion(
+		configuration.HTTPConfig.TLSConfig, c.Config.TLSVersion,
+	)
+	if err != nil {
+		return configuration, err
+	}
+	configuration.HTTPConfig.NoTLSVerify = c.Config.NoTLSVerify
+	// configure proxy
+	configuration.HTTPConfig.ProxyURL = c.ProxyURL
+	return configuration, nil
+}

internal/engine/experiment/urlgetter/configurer_test.go

@@ -0,0 +1,734 @@
+package urlgetter_test
+
+import (
+	"crypto/tls"
+	"errors"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+func TestConfigurerNewConfigurationVanilla(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer configuration.CloseIdleConnections()
+	if configuration.HTTPConfig.BogonIsError != false {
+		t.Fatal("not the BogonIsError we expected")
+	}
+	if configuration.HTTPConfig.CacheResolutions != true {
+		t.Fatal("not the CacheResolutions we expected")
+	}
+	if configuration.HTTPConfig.ContextByteCounting != true {
+		t.Fatal("not the ContextByteCounting we expected")
+	}
+	if configuration.HTTPConfig.DialSaver != saver {
+		t.Fatal("not the DialSaver we expected")
+	}
+	if configuration.HTTPConfig.HTTPSaver != saver {
+		t.Fatal("not the HTTPSaver we expected")
+	}
+	if configuration.HTTPConfig.Logger != log.Log {
+		t.Fatal("not the Logger we expected")
+	}
+	if configuration.HTTPConfig.ReadWriteSaver != saver {
+		t.Fatal("not the ReadWriteSaver we expected")
+	}
+	if configuration.HTTPConfig.ResolveSaver != saver {
+		t.Fatal("not the ResolveSaver we expected")
+	}
+	if configuration.HTTPConfig.TLSSaver != saver {
+		t.Fatal("not the TLSSaver we expected")
+	}
+	if configuration.HTTPConfig.BaseResolver == nil {
+		t.Fatal("not the BaseResolver we expected")
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.NoTLSVerify == true {
+		t.Fatal("not the NoTLSVerify we expected")
+	}
+	if configuration.HTTPConfig.ProxyURL != nil {
+		t.Fatal("not the ProxyURL we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationResolverDNSOverHTTPSPowerdns(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			ResolverURL: "doh://google",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer configuration.CloseIdleConnections()
+	if configuration.HTTPConfig.BogonIsError != false {
+		t.Fatal("not the BogonIsError we expected")
+	}
+	if configuration.HTTPConfig.CacheResolutions != true {
+		t.Fatal("not the CacheResolutions we expected")
+	}
+	if configuration.HTTPConfig.ContextByteCounting != true {
+		t.Fatal("not the ContextByteCounting we expected")
+	}
+	if configuration.HTTPConfig.DialSaver != saver {
+		t.Fatal("not the DialSaver we expected")
+	}
+	if configuration.HTTPConfig.HTTPSaver != saver {
+		t.Fatal("not the HTTPSaver we expected")
+	}
+	if configuration.HTTPConfig.Logger != log.Log {
+		t.Fatal("not the Logger we expected")
+	}
+	if configuration.HTTPConfig.ReadWriteSaver != saver {
+		t.Fatal("not the ReadWriteSaver we expected")
+	}
+	if configuration.HTTPConfig.ResolveSaver != saver {
+		t.Fatal("not the ResolveSaver we expected")
+	}
+	if configuration.HTTPConfig.TLSSaver != saver {
+		t.Fatal("not the TLSSaver we expected")
+	}
+	if configuration.HTTPConfig.BaseResolver == nil {
+		t.Fatal("not the BaseResolver we expected")
+	}
+	sr, ok := configuration.HTTPConfig.BaseResolver.(resolver.SerialResolver)
+	if !ok {
+		t.Fatal("not the resolver we expected")
+	}
+	stxp, ok := sr.Txp.(resolver.SaverDNSTransport)
+	if !ok {
+		t.Fatal("not the DNS transport we expected")
+	}
+	dohtxp, ok := stxp.RoundTripper.(resolver.DNSOverHTTPS)
+	if !ok {
+		t.Fatal("not the DNS transport we expected")
+	}
+	if dohtxp.URL != "https://dns.google/dns-query" {
+		t.Fatal("not the DoH URL we expected")
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.NoTLSVerify == true {
+		t.Fatal("not the NoTLSVerify we expected")
+	}
+	if configuration.HTTPConfig.ProxyURL != nil {
+		t.Fatal("not the ProxyURL we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationResolverDNSOverHTTPSGoogle(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			ResolverURL: "doh://google",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer configuration.CloseIdleConnections()
+	if configuration.HTTPConfig.BogonIsError != false {
+		t.Fatal("not the BogonIsError we expected")
+	}
+	if configuration.HTTPConfig.CacheResolutions != true {
+		t.Fatal("not the CacheResolutions we expected")
+	}
+	if configuration.HTTPConfig.ContextByteCounting != true {
+		t.Fatal("not the ContextByteCounting we expected")
+	}
+	if configuration.HTTPConfig.DialSaver != saver {
+		t.Fatal("not the DialSaver we expected")
+	}
+	if configuration.HTTPConfig.HTTPSaver != saver {
+		t.Fatal("not the HTTPSaver we expected")
+	}
+	if configuration.HTTPConfig.Logger != log.Log {
+		t.Fatal("not the Logger we expected")
+	}
+	if configuration.HTTPConfig.ReadWriteSaver != saver {
+		t.Fatal("not the ReadWriteSaver we expected")
+	}
+	if configuration.HTTPConfig.ResolveSaver != saver {
+		t.Fatal("not the ResolveSaver we expected")
+	}
+	if configuration.HTTPConfig.TLSSaver != saver {
+		t.Fatal("not the TLSSaver we expected")
+	}
+	if configuration.HTTPConfig.BaseResolver == nil {
+		t.Fatal("not the BaseResolver we expected")
+	}
+	sr, ok := configuration.HTTPConfig.BaseResolver.(resolver.SerialResolver)
+	if !ok {
+		t.Fatal("not the resolver we expected")
+	}
+	stxp, ok := sr.Txp.(resolver.SaverDNSTransport)
+	if !ok {
+		t.Fatal("not the DNS transport we expected")
+	}
+	dohtxp, ok := stxp.RoundTripper.(resolver.DNSOverHTTPS)
+	if !ok {
+		t.Fatal("not the DNS transport we expected")
+	}
+	if dohtxp.URL != "https://dns.google/dns-query" {
+		t.Fatal("not the DoH URL we expected")
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.NoTLSVerify == true {
+		t.Fatal("not the NoTLSVerify we expected")
+	}
+	if configuration.HTTPConfig.ProxyURL != nil {
+		t.Fatal("not the ProxyURL we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationResolverDNSOverHTTPSCloudflare(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			ResolverURL: "doh://cloudflare",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer configuration.CloseIdleConnections()
+	if configuration.HTTPConfig.BogonIsError != false {
+		t.Fatal("not the BogonIsError we expected")
+	}
+	if configuration.HTTPConfig.CacheResolutions != true {
+		t.Fatal("not the CacheResolutions we expected")
+	}
+	if configuration.HTTPConfig.ContextByteCounting != true {
+		t.Fatal("not the ContextByteCounting we expected")
+	}
+	if configuration.HTTPConfig.DialSaver != saver {
+		t.Fatal("not the DialSaver we expected")
+	}
+	if configuration.HTTPConfig.HTTPSaver != saver {
+		t.Fatal("not the HTTPSaver we expected")
+	}
+	if configuration.HTTPConfig.Logger != log.Log {
+		t.Fatal("not the Logger we expected")
+	}
+	if configuration.HTTPConfig.ReadWriteSaver != saver {
+		t.Fatal("not the ReadWriteSaver we expected")
+	}
+	if configuration.HTTPConfig.ResolveSaver != saver {
+		t.Fatal("not the ResolveSaver we expected")
+	}
+	if configuration.HTTPConfig.TLSSaver != saver {
+		t.Fatal("not the TLSSaver we expected")
+	}
+	if configuration.HTTPConfig.BaseResolver == nil {
+		t.Fatal("not the BaseResolver we expected")
+	}
+	sr, ok := configuration.HTTPConfig.BaseResolver.(resolver.SerialResolver)
+	if !ok {
+		t.Fatal("not the resolver we expected")
+	}
+	stxp, ok := sr.Txp.(resolver.SaverDNSTransport)
+	if !ok {
+		t.Fatal("not the DNS transport we expected")
+	}
+	dohtxp, ok := stxp.RoundTripper.(resolver.DNSOverHTTPS)
+	if !ok {
+		t.Fatal("not the DNS transport we expected")
+	}
+	if dohtxp.URL != "https://cloudflare-dns.com/dns-query" {
+		t.Fatal("not the DoH URL we expected")
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.NoTLSVerify == true {
+		t.Fatal("not the NoTLSVerify we expected")
+	}
+	if configuration.HTTPConfig.ProxyURL != nil {
+		t.Fatal("not the ProxyURL we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationResolverUDP(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			ResolverURL: "udp://8.8.8.8:53",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer configuration.CloseIdleConnections()
+	if configuration.HTTPConfig.BogonIsError != false {
+		t.Fatal("not the BogonIsError we expected")
+	}
+	if configuration.HTTPConfig.CacheResolutions != true {
+		t.Fatal("not the CacheResolutions we expected")
+	}
+	if configuration.HTTPConfig.ContextByteCounting != true {
+		t.Fatal("not the ContextByteCounting we expected")
+	}
+	if configuration.HTTPConfig.DialSaver != saver {
+		t.Fatal("not the DialSaver we expected")
+	}
+	if configuration.HTTPConfig.HTTPSaver != saver {
+		t.Fatal("not the HTTPSaver we expected")
+	}
+	if configuration.HTTPConfig.Logger != log.Log {
+		t.Fatal("not the Logger we expected")
+	}
+	if configuration.HTTPConfig.ReadWriteSaver != saver {
+		t.Fatal("not the ReadWriteSaver we expected")
+	}
+	if configuration.HTTPConfig.ResolveSaver != saver {
+		t.Fatal("not the ResolveSaver we expected")
+	}
+	if configuration.HTTPConfig.TLSSaver != saver {
+		t.Fatal("not the TLSSaver we expected")
+	}
+	if configuration.HTTPConfig.BaseResolver == nil {
+		t.Fatal("not the BaseResolver we expected")
+	}
+	sr, ok := configuration.HTTPConfig.BaseResolver.(resolver.SerialResolver)
+	if !ok {
+		t.Fatal("not the resolver we expected")
+	}
+	stxp, ok := sr.Txp.(resolver.SaverDNSTransport)
+	if !ok {
+		t.Fatal("not the DNS transport we expected")
+	}
+	udptxp, ok := stxp.RoundTripper.(resolver.DNSOverUDP)
+	if !ok {
+		t.Fatal("not the DNS transport we expected")
+	}
+	if udptxp.Address() != "8.8.8.8:53" {
+		t.Fatal("not the DoH URL we expected")
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("not the TLSConfig we expected")
+	}
+	if configuration.HTTPConfig.NoTLSVerify == true {
+		t.Fatal("not the NoTLSVerify we expected")
+	}
+	if configuration.HTTPConfig.ProxyURL != nil {
+		t.Fatal("not the ProxyURL we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationDNSCacheInvalidString(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			DNSCache: "a",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	_, err := configurer.NewConfiguration()
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid DNSCache string") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationDNSCacheNotDomain(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			DNSCache: "b b",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	_, err := configurer.NewConfiguration()
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid domain in DNSCache") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationDNSCacheNotIP(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			DNSCache: "x.org b",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	_, err := configurer.NewConfiguration()
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid IP in DNSCache") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationDNSCacheGood(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			DNSCache: "dns.google.com 8.8.8.8 8.8.4.4",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(configuration.HTTPConfig.DNSCache) != 1 {
+		t.Fatal("invalid number of entries in DNSCache")
+	}
+	if len(configuration.HTTPConfig.DNSCache["dns.google.com"]) != 2 {
+		t.Fatal("invalid number of IPs saved in DNSCache")
+	}
+	if configuration.HTTPConfig.DNSCache["dns.google.com"][0] != "8.8.8.8" {
+		t.Fatal("invalid IPs saved in DNSCache")
+	}
+	if configuration.HTTPConfig.DNSCache["dns.google.com"][1] != "8.8.4.4" {
+		t.Fatal("invalid IPs saved in DNSCache")
+	}
+}
+
+func TestConfigurerNewConfigurationResolverInvalidURL(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			ResolverURL: "\t",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	_, err := configurer.NewConfiguration()
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationResolverInvalidURLScheme(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			ResolverURL: "antani://8.8.8.8:53",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	_, err := configurer.NewConfiguration()
+	if err == nil || !strings.HasSuffix(err.Error(), "unsupported resolver scheme") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationTLSServerName(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			TLSServerName: "www.x.org",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if configuration.HTTPConfig.TLSConfig.ServerName != "www.x.org" {
+		t.Fatal("invalid ServerName")
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("invalid len(NextProtos)")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("invalid NextProtos[0]")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("invalid NextProtos[1]")
+	}
+}
+
+func TestConfigurerNewConfigurationNoTLSVerify(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			NoTLSVerify: true,
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if configuration.HTTPConfig.NoTLSVerify != true {
+		t.Fatal("not the NoTLSVerify we expected")
+	}
+}
+
+func TestConfigurerNewConfigurationTLSv1(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			TLSVersion: "TLSv1",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("invalid len(NextProtos)")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("invalid NextProtos[0]")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("invalid NextProtos[1]")
+	}
+	if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS10 {
+		t.Fatal("invalid MinVersion")
+	}
+	if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS10 {
+		t.Fatal("invalid MaxVersion")
+	}
+}
+
+func TestConfigurerNewConfigurationTLSv1dot0(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			TLSVersion: "TLSv1.0",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("invalid len(NextProtos)")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("invalid NextProtos[0]")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("invalid NextProtos[1]")
+	}
+	if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS10 {
+		t.Fatal("invalid MinVersion")
+	}
+	if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS10 {
+		t.Fatal("invalid MaxVersion")
+	}
+}
+
+func TestConfigurerNewConfigurationTLSv1dot1(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			TLSVersion: "TLSv1.1",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("invalid len(NextProtos)")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("invalid NextProtos[0]")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("invalid NextProtos[1]")
+	}
+	if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS11 {
+		t.Fatal("invalid MinVersion")
+	}
+	if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS11 {
+		t.Fatal("invalid MaxVersion")
+	}
+}
+
+func TestConfigurerNewConfigurationTLSv1dot2(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			TLSVersion: "TLSv1.2",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("invalid len(NextProtos)")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("invalid NextProtos[0]")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("invalid NextProtos[1]")
+	}
+	if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS12 {
+		t.Fatal("invalid MinVersion")
+	}
+	if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS12 {
+		t.Fatal("invalid MaxVersion")
+	}
+}
+
+func TestConfigurerNewConfigurationTLSv1dot3(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			TLSVersion: "TLSv1.3",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("invalid len(NextProtos)")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("invalid NextProtos[0]")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("invalid NextProtos[1]")
+	}
+	if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS13 {
+		t.Fatal("invalid MinVersion")
+	}
+	if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS13 {
+		t.Fatal("invalid MaxVersion")
+	}
+}
+
+func TestConfigurerNewConfigurationTLSvDefault(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
+		t.Fatal("invalid len(NextProtos)")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
+		t.Fatal("invalid NextProtos[0]")
+	}
+	if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
+		t.Fatal("invalid NextProtos[1]")
+	}
+	if configuration.HTTPConfig.TLSConfig.MinVersion != 0 {
+		t.Fatal("invalid MinVersion")
+	}
+	if configuration.HTTPConfig.TLSConfig.MaxVersion != 0 {
+		t.Fatal("invalid MaxVersion")
+	}
+}
+
+func TestConfigurerNewConfigurationTLSvInvalid(t *testing.T) {
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			TLSVersion: "SSLv3",
+		},
+		Logger: log.Log,
+		Saver:  saver,
+	}
+	_, err := configurer.NewConfiguration()
+	if !errors.Is(err, netx.ErrInvalidTLSVersion) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+}
+
+func TestConfigurerNewConfigurationProxyURL(t *testing.T) {
+	URL, _ := url.Parse("socks5://127.0.0.1:9050")
+	saver := new(trace.Saver)
+	configurer := urlgetter.Configurer{
+		Logger:   log.Log,
+		Saver:    saver,
+		ProxyURL: URL,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if configuration.HTTPConfig.ProxyURL != URL {
+		t.Fatal("invalid ProxyURL")
+	}
+}

internal/engine/experiment/urlgetter/getter.go

@@ -0,0 +1,132 @@
+package urlgetter
+
+import (
+	"context"
+	"net/url"
+	"path/filepath"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/tunnel"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+// The Getter gets the specified target in the context of the
+// given session and with the specified config.
+//
+// Other OONI experiment should use the Getter to factor code when
+// the Getter implements the operations they wanna perform.
+type Getter struct {
+	// Begin is the time when the experiment begun. If you do not
+	// set this field, every target is measured independently.
+	Begin time.Time
+
+	// Config contains settings for this run. If not set, then
+	// we will use the default config.
+	Config Config
+
+	// Session is the session for this run. This field must
+	// be set otherwise the code will panic.
+	Session model.ExperimentSession
+
+	// Target is the thing to measure in this run. This field must
+	// be set otherwise the code won't know what to do.
+	Target string
+}
+
+// Get performs the action described by g using the given context
+// and returning the test keys and eventually an error
+func (g Getter) Get(ctx context.Context) (TestKeys, error) {
+	if g.Config.Timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, g.Config.Timeout)
+		defer cancel()
+	}
+	if g.Begin.IsZero() {
+		g.Begin = time.Now()
+	}
+	saver := new(trace.Saver)
+	tk, err := g.get(ctx, saver)
+	// Make sure we have an operation in cases where we fail before
+	// hitting our httptransport that does error wrapping.
+	err = errorx.SafeErrWrapperBuilder{
+		Error:     err,
+		Operation: errorx.TopLevelOperation,
+	}.MaybeBuild()
+	tk.FailedOperation = archival.NewFailedOperation(err)
+	tk.Failure = archival.NewFailure(err)
+	events := saver.Read()
+	tk.Queries = append(
+		tk.Queries, archival.NewDNSQueriesList(
+			g.Begin, events, g.Session.ASNDatabasePath())...,
+	)
+	tk.NetworkEvents = append(
+		tk.NetworkEvents, archival.NewNetworkEventsList(g.Begin, events)...,
+	)
+	tk.Requests = append(
+		tk.Requests, archival.NewRequestList(g.Begin, events)...,
+	)
+	if len(tk.Requests) > 0 {
+		// OONI's convention is that the last request appears first
+		tk.HTTPResponseStatus = tk.Requests[0].Response.Code
+		tk.HTTPResponseBody = tk.Requests[0].Response.Body.Value
+		tk.HTTPResponseLocations = tk.Requests[0].Response.Locations
+	}
+	tk.TCPConnect = append(
+		tk.TCPConnect, archival.NewTCPConnectList(g.Begin, events)...,
+	)
+	tk.TLSHandshakes = append(
+		tk.TLSHandshakes, archival.NewTLSHandshakesList(g.Begin, events)...,
+	)
+	return tk, err
+}
+
+func (g Getter) get(ctx context.Context, saver *trace.Saver) (TestKeys, error) {
+	tk := TestKeys{
+		Agent:  "redirect",
+		Tunnel: g.Config.Tunnel,
+	}
+	if g.Config.DNSCache != "" {
+		tk.DNSCache = []string{g.Config.DNSCache}
+	}
+	if g.Config.NoFollowRedirects {
+		tk.Agent = "agent"
+	}
+	// start tunnel
+	var proxyURL *url.URL
+	if g.Config.Tunnel != "" {
+		tun, err := tunnel.Start(ctx, tunnel.Config{
+			Name:    g.Config.Tunnel,
+			Session: g.Session,
+			WorkDir: filepath.Join(g.Session.TempDir(), "urlgetter-tunnel"),
+		})
+		if err != nil {
+			return tk, err
+		}
+		tk.BootstrapTime = tun.BootstrapTime().Seconds()
+		proxyURL = tun.SOCKS5ProxyURL()
+		tk.SOCKSProxy = proxyURL.String()
+		defer tun.Stop()
+	}
+	// create configuration
+	configurer := Configurer{
+		Config:   g.Config,
+		Logger:   g.Session.Logger(),
+		ProxyURL: proxyURL,
+		Saver:    saver,
+	}
+	configuration, err := configurer.NewConfiguration()
+	if err != nil {
+		return tk, err
+	}
+	defer configuration.CloseIdleConnections()
+	// run the measurement
+	runner := Runner{
+		Config:     g.Config,
+		HTTPConfig: configuration.HTTPConfig,
+		Target:     g.Target,
+	}
+	return tk, runner.Run(ctx)
+}

internal/engine/experiment/urlgetter/getter_test.go

@@ -0,0 +1,777 @@
+package urlgetter_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestGetterWithVeryShortTimeout(t *testing.T) {
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			Timeout: 1,
+		},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(context.Background())
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || *tk.Failure != "generic_timeout_error" {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 3 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
+		t.Fatal("not the NetworkEvents[0].Operation we expected")
+	}
+	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
+		t.Fatal("not the NetworkEvents[1].Operation we expected")
+	}
+	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
+		t.Fatal("not the NetworkEvents[2].Operation we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextVanilla(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // faily immediately
+	g := urlgetter.Getter{
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 3 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
+		t.Fatal("not the NetworkEvents[0].Operation we expected")
+	}
+	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
+		t.Fatal("not the NetworkEvents[1].Operation we expected")
+	}
+	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
+		t.Fatal("not the NetworkEvents[2].Operation we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextAndMethod(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // faily immediately
+	g := urlgetter.Getter{
+		Config:  urlgetter.Config{Method: "POST"},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 3 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
+		t.Fatal("not the NetworkEvents[0].Operation we expected")
+	}
+	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
+		t.Fatal("not the NetworkEvents[1].Operation we expected")
+	}
+	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
+		t.Fatal("not the NetworkEvents[2].Operation we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "POST" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextNoFollowRedirects(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // faily immediately
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true,
+		},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 3 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
+		t.Fatal("not the NetworkEvents[0].Operation we expected")
+	}
+	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
+		t.Fatal("not the NetworkEvents[1].Operation we expected")
+	}
+	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
+		t.Fatal("not the NetworkEvents[2].Operation we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextCannotStartTunnel(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			Tunnel: "psiphon",
+		},
+		Session: &mockable.Session{MockableLogger: log.Log},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || *tk.Failure != "interrupted" {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 0 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "psiphon" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextUnknownResolverURL(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // faily immediately
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			ResolverURL: "antani://8.8.8.8:53",
+		},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if err == nil || err.Error() != "unknown_failure: unsupported resolver scheme" {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || *tk.Failure != "unknown_failure: unsupported resolver scheme" {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 0 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterIntegrationHTTPS(t *testing.T) {
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true, // reduce number of events
+		},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure != nil {
+		t.Fatal("not the Failure we expected")
+	}
+	var (
+		httpTransactionStart     bool
+		httpRequestMetadata      bool
+		resolveStart             bool
+		resolveDone              bool
+		connect                  bool
+		tlsHandshakeStart        bool
+		tlsHandshakeDone         bool
+		httpWroteHeaders         bool
+		httpWroteRequest         bool
+		httpFirstResponseByte    bool
+		httpResponseMetadata     bool
+		httpResponseBodySnapshot bool
+		httpTransactionDone      bool
+	)
+	for _, ev := range tk.NetworkEvents {
+		switch ev.Operation {
+		case "http_transaction_start":
+			httpTransactionStart = true
+		case "http_request_metadata":
+			httpRequestMetadata = true
+		case "resolve_start":
+			resolveStart = true
+		case "resolve_done":
+			resolveDone = true
+		case errorx.ConnectOperation:
+			connect = true
+		case "tls_handshake_start":
+			tlsHandshakeStart = true
+		case "tls_handshake_done":
+			tlsHandshakeDone = true
+		case "http_wrote_headers":
+			httpWroteHeaders = true
+		case "http_wrote_request":
+			httpWroteRequest = true
+		case "http_first_response_byte":
+			httpFirstResponseByte = true
+		case "http_response_metadata":
+			httpResponseMetadata = true
+		case "http_response_body_snapshot":
+			httpResponseBodySnapshot = true
+		case "http_transaction_done":
+			httpTransactionDone = true
+		}
+	}
+	ok := true
+	ok = ok && httpTransactionStart
+	ok = ok && httpRequestMetadata
+	ok = ok && resolveStart
+	ok = ok && resolveDone
+	ok = ok && connect
+	ok = ok && tlsHandshakeStart
+	ok = ok && tlsHandshakeDone
+	ok = ok && httpWroteHeaders
+	ok = ok && httpWroteRequest
+	ok = ok && httpFirstResponseByte
+	ok = ok && httpResponseMetadata
+	ok = ok && httpResponseBodySnapshot
+	ok = ok && httpTransactionDone
+	if !ok {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 2 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 1 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 1 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 200 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if len(tk.HTTPResponseBody) <= 0 {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterIntegrationRedirect(t *testing.T) {
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config:  urlgetter.Config{NoFollowRedirects: true},
+		Session: &mockable.Session{},
+		Target:  "http://web.whatsapp.com",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.HTTPResponseStatus != 302 {
+		t.Fatal("unexpected status code")
+	}
+	if len(tk.HTTPResponseLocations) != 1 {
+		t.Fatal("missing redirect URL")
+	}
+	if tk.HTTPResponseLocations[0] != "https://web.whatsapp.com/" {
+		t.Fatal("invalid redirect URL")
+	}
+}
+
+func TestGetterIntegrationTLSHandshake(t *testing.T) {
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true, // reduce number of events
+		},
+		Session: &mockable.Session{},
+		Target:  "tlshandshake://www.google.com:443",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure != nil {
+		t.Fatal("not the Failure we expected")
+	}
+	var (
+		httpTransactionStart     bool
+		httpRequestMetadata      bool
+		resolveStart             bool
+		resolveDone              bool
+		connect                  bool
+		tlsHandshakeStart        bool
+		tlsHandshakeDone         bool
+		httpWroteHeaders         bool
+		httpWroteRequest         bool
+		httpFirstResponseByte    bool
+		httpResponseMetadata     bool
+		httpResponseBodySnapshot bool
+		httpTransactionDone      bool
+	)
+	for _, ev := range tk.NetworkEvents {
+		switch ev.Operation {
+		case "http_transaction_start":
+			httpTransactionStart = true
+		case "http_request_metadata":
+			httpRequestMetadata = true
+		case "resolve_start":
+			resolveStart = true
+		case "resolve_done":
+			resolveDone = true
+		case errorx.ConnectOperation:
+			connect = true
+		case "tls_handshake_start":
+			tlsHandshakeStart = true
+		case "tls_handshake_done":
+			tlsHandshakeDone = true
+		case "http_wrote_headers":
+			httpWroteHeaders = true
+		case "http_wrote_request":
+			httpWroteRequest = true
+		case "http_first_response_byte":
+			httpFirstResponseByte = true
+		case "http_response_metadata":
+			httpResponseMetadata = true
+		case "http_response_body_snapshot":
+			httpResponseBodySnapshot = true
+		case "http_transaction_done":
+			httpTransactionDone = true
+		}
+	}
+	ok := true
+	ok = ok && !httpTransactionStart
+	ok = ok && !httpRequestMetadata
+	ok = ok && resolveStart
+	ok = ok && resolveDone
+	ok = ok && connect
+	ok = ok && tlsHandshakeStart
+	ok = ok && tlsHandshakeDone
+	ok = ok && !httpWroteHeaders
+	ok = ok && !httpWroteRequest
+	ok = ok && !httpFirstResponseByte
+	ok = ok && !httpResponseMetadata
+	ok = ok && !httpResponseBodySnapshot
+	ok = ok && !httpTransactionDone
+	if !ok {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 2 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 1 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 1 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterIntegrationHTTPSWithTunnel(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true, // reduce number of events
+			Tunnel:            "psiphon",
+		},
+		Session: &mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		Target: "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime <= 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure != nil {
+		t.Fatal("not the Failure we expected")
+	}
+	var (
+		httpTransactionStart     bool
+		httpRequestMetadata      bool
+		resolveStart             bool
+		resolveDone              bool
+		connect                  bool
+		tlsHandshakeStart        bool
+		tlsHandshakeDone         bool
+		httpWroteHeaders         bool
+		httpWroteRequest         bool
+		httpFirstResponseByte    bool
+		httpResponseMetadata     bool
+		httpResponseBodySnapshot bool
+		httpTransactionDone      bool
+	)
+	for _, ev := range tk.NetworkEvents {
+		switch ev.Operation {
+		case "http_transaction_start":
+			httpTransactionStart = true
+		case "http_request_metadata":
+			httpRequestMetadata = true
+		case "resolve_start":
+			resolveStart = true
+		case "resolve_done":
+			resolveDone = true
+		case errorx.ConnectOperation:
+			connect = true
+		case "tls_handshake_start":
+			tlsHandshakeStart = true
+		case "tls_handshake_done":
+			tlsHandshakeDone = true
+		case "http_wrote_headers":
+			httpWroteHeaders = true
+		case "http_wrote_request":
+			httpWroteRequest = true
+		case "http_first_response_byte":
+			httpFirstResponseByte = true
+		case "http_response_metadata":
+			httpResponseMetadata = true
+		case "http_response_body_snapshot":
+			httpResponseBodySnapshot = true
+		case "http_transaction_done":
+			httpTransactionDone = true
+		}
+	}
+	ok := true
+	ok = ok && httpTransactionStart
+	ok = ok && httpRequestMetadata
+	ok = ok && resolveStart == false
+	ok = ok && resolveDone == false
+	ok = ok && connect
+	ok = ok && tlsHandshakeStart
+	ok = ok && tlsHandshakeDone
+	ok = ok && httpWroteHeaders
+	ok = ok && httpWroteRequest
+	ok = ok && httpFirstResponseByte
+	ok = ok && httpResponseMetadata
+	ok = ok && httpResponseBodySnapshot
+	ok = ok && httpTransactionDone
+	if !ok {
+		t.Fatalf("not the NetworkEvents we expected: %+v", tk.NetworkEvents)
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 1 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy == "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 1 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "psiphon" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 200 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if len(tk.HTTPResponseBody) <= 0 {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}

internal/engine/experiment/urlgetter/multi.go

@@ -0,0 +1,143 @@
+package urlgetter
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// MultiInput is the input for Multi.Run().
+type MultiInput struct {
+	// Config contains the configuration for this target.
+	Config Config
+
+	// Target contains the target URL to measure.
+	Target string
+}
+
+// MultiOutput is the output returned by Multi.Run()
+type MultiOutput struct {
+	// Input is the input for which we measured.
+	Input MultiInput
+
+	// Err contains the measurement error.
+	Err error
+
+	// TestKeys contains the measured test keys.
+	TestKeys TestKeys
+}
+
+// MultiGetter allows to override the behaviour of Multi for testing purposes.
+type MultiGetter func(ctx context.Context, g Getter) (TestKeys, error)
+
+// DefaultMultiGetter is the default MultiGetter
+func DefaultMultiGetter(ctx context.Context, g Getter) (TestKeys, error) {
+	return g.Get(ctx)
+}
+
+// Multi allows to run several urlgetters in paraller.
+type Multi struct {
+	// Begin is the time when the experiment begun. If you do not
+	// set this field, every target is measured independently.
+	Begin time.Time
+
+	// Getter is the Getter func to be used. If this is nil we use
+	// the default getter, which is what you typically want.
+	Getter MultiGetter
+
+	// Parallelism is the optional parallelism to be used. If this is
+	// zero, or negative, we use a reasonable default.
+	Parallelism int
+
+	// Session is the session to be used. If this is nil, the Run
+	// method will panic with a nil pointer error.
+	Session model.ExperimentSession
+}
+
+// Run performs several urlgetters in parallel. This function returns a channel
+// where each result is posted. This function will always perform all the requested
+// measurements: if the ctx is canceled or its deadline expires, then you will see
+// a bunch of failed measurements. Since all measurements are always performed,
+// you know you're done when you've read len(inputs) results in output.
+func (m Multi) Run(ctx context.Context, inputs []MultiInput) <-chan MultiOutput {
+	parallelism := m.Parallelism
+	if parallelism <= 0 {
+		const defaultParallelism = 3
+		parallelism = defaultParallelism
+	}
+	inputch := make(chan MultiInput)
+	outputch := make(chan MultiOutput)
+	go m.source(inputs, inputch)
+	for i := 0; i < parallelism; i++ {
+		go m.do(ctx, inputch, outputch)
+	}
+	return outputch
+}
+
+// Collect prints on the output channel the result of running urlgetter
+// on every provided input. It closes the output channel when done.
+func (m Multi) Collect(ctx context.Context, inputs []MultiInput,
+	prefix string, callbacks model.ExperimentCallbacks) <-chan MultiOutput {
+	return m.CollectOverall(ctx, inputs, 0, len(inputs), prefix, callbacks)
+}
+
+// CollectOverall prints on the output channel the result of running urlgetter
+// on every provided input. You can use this method if you perform multiple collection
+// tasks within one experiment as it allows to calculate the overall progress correctly
+func (m Multi) CollectOverall(ctx context.Context, inputChunk []MultiInput, overallStartIndex int, overallCount int,
+	prefix string, callbacks model.ExperimentCallbacks) <-chan MultiOutput {
+	outputch := make(chan MultiOutput)
+	go m.collect(len(inputChunk), overallStartIndex, overallCount, prefix, callbacks, m.Run(ctx, inputChunk), outputch)
+	return outputch
+}
+
+// collect drains inputch, prints progress, and emits to outputch. When done, this
+// function will close outputch to notify the calller.
+func (m Multi) collect(expect int, overallStartIndex int, overallCount int, prefix string, callbacks model.ExperimentCallbacks,
+	inputch <-chan MultiOutput, outputch chan<- MultiOutput) {
+	count := overallStartIndex
+	var index int
+	defer close(outputch)
+	for index < expect {
+		entry := <-inputch
+		index++
+		count++
+		percentage := float64(count) / float64(overallCount)
+		callbacks.OnProgress(percentage, fmt.Sprintf(
+			"%s: measure %s: %+v", prefix, entry.Input.Target, entry.Err,
+		))
+		outputch <- entry
+	}
+}
+
+// source posts all the inputs in the inputch. When done, this
+// method will close the input channel to notify the reader.
+func (m Multi) source(inputs []MultiInput, inputch chan<- MultiInput) {
+	defer close(inputch)
+	for _, input := range inputs {
+		inputch <- input
+	}
+}
+
+// do performs urlgetter on all the inputs read from the in channel and
+// writes the results on the out channel. If the context is canceled, or
+// its deadline expires, this function will continue performing all the
+// required measurements, which will all fail.
+func (m Multi) do(ctx context.Context, in <-chan MultiInput, out chan<- MultiOutput) {
+	for input := range in {
+		g := Getter{
+			Begin:   m.Begin,
+			Config:  input.Config,
+			Session: m.Session,
+			Target:  input.Target,
+		}
+		fn := m.Getter
+		if fn == nil {
+			fn = DefaultMultiGetter
+		}
+		tk, err := fn(ctx, g)
+		out <- MultiOutput{Input: input, Err: err, TestKeys: tk}
+	}
+}

internal/engine/experiment/urlgetter/multi_test.go

@@ -0,0 +1,256 @@
+package urlgetter_test
+
+import (
+	"context"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestMultiIntegration(t *testing.T) {
+	multi := urlgetter.Multi{Session: &mockable.Session{}}
+	inputs := []urlgetter.MultiInput{{
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.google.com",
+	}, {
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.facebook.com",
+	}, {
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.kernel.org",
+	}, {
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.instagram.com",
+	}}
+	outputs := multi.Collect(context.Background(), inputs, "integration-test",
+		model.NewPrinterCallbacks(log.Log))
+	var count int
+	for result := range outputs {
+		count++
+		switch result.Input.Target {
+		case "https://www.google.com":
+		case "https://www.facebook.com":
+		case "https://www.kernel.org":
+		case "https://www.instagram.com":
+		default:
+			t.Fatal("unexpected Input.Target")
+		}
+		if result.Input.Config.Method != "HEAD" {
+			t.Fatal("unexpected Input.Config.Method")
+		}
+		if result.Err != nil {
+			t.Fatal(result.Err)
+		}
+		if result.TestKeys.Agent != "agent" {
+			t.Fatal("invalid TestKeys.Agent")
+		}
+		if len(result.TestKeys.Queries) != 2 {
+			t.Fatal("invalid number of Queries")
+		}
+		if len(result.TestKeys.Requests) != 1 {
+			t.Fatal("invalid number of Requests")
+		}
+		if len(result.TestKeys.TCPConnect) != 1 {
+			t.Fatal("invalid number of TCPConnects")
+		}
+		if len(result.TestKeys.TLSHandshakes) != 1 {
+			t.Fatal("invalid number of TLSHandshakes")
+		}
+	}
+	if count != 4 {
+		t.Fatal("invalid number of outputs")
+	}
+}
+
+func TestMultiIntegrationWithBaseTime(t *testing.T) {
+	// We set a beginning of time that's significantly in the past and then
+	// fail the test if we see any T smaller than 3600 seconds.
+	begin := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
+	multi := urlgetter.Multi{
+		Begin:   begin,
+		Session: &mockable.Session{},
+	}
+	inputs := []urlgetter.MultiInput{{
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.google.com",
+	}, {
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.instagram.com",
+	}}
+	outputs := multi.Collect(context.Background(), inputs, "integration-test",
+		model.NewPrinterCallbacks(log.Log))
+	var count int
+	for result := range outputs {
+		for _, entry := range result.TestKeys.NetworkEvents {
+			if entry.T < 3600 {
+				t.Fatal("base time not correctly set")
+			}
+			count++
+		}
+		for _, entry := range result.TestKeys.Queries {
+			if entry.T < 3600 {
+				t.Fatal("base time not correctly set")
+			}
+			count++
+		}
+		for _, entry := range result.TestKeys.TCPConnect {
+			if entry.T < 3600 {
+				t.Fatal("base time not correctly set")
+			}
+			count++
+		}
+		for _, entry := range result.TestKeys.TLSHandshakes {
+			if entry.T < 3600 {
+				t.Fatal("base time not correctly set")
+			}
+			count++
+		}
+	}
+	if count <= 0 {
+		t.Fatal("unexpected number of entries processed")
+	}
+}
+
+func TestMultiIntegrationWithoutBaseTime(t *testing.T) {
+	// We use the default beginning of time and then fail the test
+	// if we see any T smaller than 60 seconds.
+	multi := urlgetter.Multi{Session: &mockable.Session{}}
+	inputs := []urlgetter.MultiInput{{
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.google.com",
+	}, {
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.instagram.com",
+	}}
+	outputs := multi.Collect(context.Background(), inputs, "integration-test",
+		model.NewPrinterCallbacks(log.Log))
+	var count int
+	for result := range outputs {
+		for _, entry := range result.TestKeys.NetworkEvents {
+			if entry.T > 60 {
+				t.Fatal("base time not correctly set")
+			}
+			count++
+		}
+		for _, entry := range result.TestKeys.Queries {
+			if entry.T > 60 {
+				t.Fatal("base time not correctly set")
+			}
+			count++
+		}
+		for _, entry := range result.TestKeys.TCPConnect {
+			if entry.T > 60 {
+				t.Fatal("base time not correctly set")
+			}
+			count++
+		}
+		for _, entry := range result.TestKeys.TLSHandshakes {
+			if entry.T > 60 {
+				t.Fatal("base time not correctly set")
+			}
+			count++
+		}
+	}
+	if count <= 0 {
+		t.Fatal("unexpected number of entries processed")
+	}
+}
+
+func TestMultiContextCanceled(t *testing.T) {
+	multi := urlgetter.Multi{Session: &mockable.Session{}}
+	inputs := []urlgetter.MultiInput{{
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.google.com",
+	}, {
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.facebook.com",
+	}, {
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.kernel.org",
+	}, {
+		Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
+		Target: "https://www.instagram.com",
+	}}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	outputs := multi.Collect(ctx, inputs, "integration-test",
+		model.NewPrinterCallbacks(log.Log))
+	var count int
+	for result := range outputs {
+		count++
+		switch result.Input.Target {
+		case "https://www.google.com":
+		case "https://www.facebook.com":
+		case "https://www.kernel.org":
+		case "https://www.instagram.com":
+		default:
+			t.Fatal("unexpected Input.Target")
+		}
+		if result.Input.Config.Method != "HEAD" {
+			t.Fatal("unexpected Input.Config.Method")
+		}
+		if !errors.Is(result.Err, context.Canceled) {
+			t.Fatal("unexpected error")
+		}
+		if result.TestKeys.Agent != "agent" {
+			t.Fatal("invalid TestKeys.Agent")
+		}
+		if len(result.TestKeys.Queries) != 0 {
+			t.Fatal("invalid number of Queries")
+		}
+		if len(result.TestKeys.Requests) != 1 {
+			t.Fatal("invalid number of Requests")
+		}
+		if len(result.TestKeys.TCPConnect) != 0 {
+			t.Fatal("invalid number of TCPConnects")
+		}
+		if len(result.TestKeys.TLSHandshakes) != 0 {
+			t.Fatal("invalid number of TLSHandshakes")
+		}
+	}
+	if count != 4 {
+		t.Fatal("invalid number of outputs")
+	}
+}
+
+func TestMultiWithSpecificCertPool(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "Hello, client")
+	}))
+	defer server.Close()
+	cert := server.Certificate()
+	certpool := x509.NewCertPool()
+	certpool.AddCert(cert)
+	multi := urlgetter.Multi{Session: &mockable.Session{}}
+	inputs := []urlgetter.MultiInput{{
+		Config: urlgetter.Config{
+			CertPool:          certpool,
+			Method:            "GET",
+			NoFollowRedirects: true,
+		},
+		Target: server.URL,
+	}}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	outputs := multi.Collect(ctx, inputs, "integration-test",
+		model.NewPrinterCallbacks(log.Log))
+	var count int
+	for result := range outputs {
+		count++
+		if result.Err != nil {
+			t.Fatal(result.Err)
+		}
+	}
+	if count != 1 {
+		t.Fatal("unexpected count value")
+	}
+}

internal/engine/experiment/urlgetter/runner.go

@@ -0,0 +1,129 @@
+package urlgetter
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+const httpRequestFailed = "http_request_failed"
+
+// ErrHTTPRequestFailed indicates that the HTTP request failed.
+var ErrHTTPRequestFailed = &errorx.ErrWrapper{
+	Failure:    httpRequestFailed,
+	Operation:  errorx.TopLevelOperation,
+	WrappedErr: errors.New(httpRequestFailed),
+}
+
+// The Runner job is to run a single measurement
+type Runner struct {
+	Config     Config
+	HTTPConfig netx.Config
+	Target     string
+}
+
+// Run runs a measurement and returns the measurement result
+func (r Runner) Run(ctx context.Context) error {
+	targetURL, err := url.Parse(r.Target)
+	if err != nil {
+		return fmt.Errorf("urlgetter: invalid target URL: %w", err)
+	}
+	switch targetURL.Scheme {
+	case "http", "https":
+		return r.httpGet(ctx, r.Target)
+	case "dnslookup":
+		return r.dnsLookup(ctx, targetURL.Hostname())
+	case "tlshandshake":
+		return r.tlsHandshake(ctx, targetURL.Host)
+	case "tcpconnect":
+		return r.tcpConnect(ctx, targetURL.Host)
+	default:
+		return errors.New("unknown targetURL scheme")
+	}
+}
+
+// MaybeUserAgent returns ua if ua is not empty. Otherwise it
+// returns httpheader.RandomUserAgent().
+func MaybeUserAgent(ua string) string {
+	if ua == "" {
+		ua = httpheader.UserAgent()
+	}
+	return ua
+}
+
+func (r Runner) httpGet(ctx context.Context, url string) error {
+	// Implementation note: empty Method implies using the GET method
+	req, err := http.NewRequest(r.Config.Method, url, nil)
+	runtimex.PanicOnError(err, "http.NewRequest failed")
+	req = req.WithContext(ctx)
+	req.Header.Set("Accept", httpheader.Accept())
+	req.Header.Set("Accept-Language", httpheader.AcceptLanguage())
+	req.Header.Set("User-Agent", MaybeUserAgent(r.Config.UserAgent))
+	if r.Config.HTTPHost != "" {
+		req.Host = r.Config.HTTPHost
+	}
+	// Implementation note: the following cookiejar accepts all cookies
+	// from all domains. As such, would not be safe for usage where cookies
+	// matter, but it's totally fine for performing measurements.
+	jar, err := cookiejar.New(nil)
+	runtimex.PanicOnError(err, "cookiejar.New failed")
+	httpClient := &http.Client{
+		Jar:       jar,
+		Transport: netx.NewHTTPTransport(r.HTTPConfig),
+	}
+	if r.Config.NoFollowRedirects {
+		httpClient.CheckRedirect = func(*http.Request, []*http.Request) error {
+			return http.ErrUseLastResponse
+		}
+	}
+	defer httpClient.CloseIdleConnections()
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if _, err = io.Copy(ioutil.Discard, resp.Body); err != nil {
+		return err
+	}
+	// Implementation note: we shall check for this error once we have read the
+	// whole body. Even though we discard the body, we want to know whether we
+	// see any error when reading the body before inspecting the HTTP status code.
+	if resp.StatusCode >= 400 && r.Config.FailOnHTTPError {
+		return ErrHTTPRequestFailed
+	}
+	return nil
+}
+
+func (r Runner) dnsLookup(ctx context.Context, hostname string) error {
+	resolver := netx.NewResolver(r.HTTPConfig)
+	_, err := resolver.LookupHost(ctx, hostname)
+	return err
+}
+
+func (r Runner) tlsHandshake(ctx context.Context, address string) error {
+	tlsDialer := netx.NewTLSDialer(r.HTTPConfig)
+	conn, err := tlsDialer.DialTLSContext(ctx, "tcp", address)
+	if conn != nil {
+		conn.Close()
+	}
+	return err
+}
+
+func (r Runner) tcpConnect(ctx context.Context, address string) error {
+	dialer := netx.NewDialer(r.HTTPConfig)
+	conn, err := dialer.DialContext(ctx, "tcp", address)
+	if conn != nil {
+		conn.Close()
+	}
+	return err
+}

internal/engine/experiment/urlgetter/runner_test.go

@@ -0,0 +1,287 @@
+package urlgetter_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+)
+
+func TestRunnerWithInvalidURLScheme(t *testing.T) {
+	r := urlgetter.Runner{Target: "antani://www.google.com"}
+	err := r.Run(context.Background())
+	if err == nil || err.Error() != "unknown targetURL scheme" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerHTTPWithContextCanceled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	r := urlgetter.Runner{Target: "https://www.google.com"}
+	err := r.Run(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerDNSLookupWithContextCanceled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	r := urlgetter.Runner{Target: "dnslookup://www.google.com"}
+	err := r.Run(ctx)
+	if err == nil || err.Error() != "interrupted" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerTLSHandshakeWithContextCanceled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	r := urlgetter.Runner{Target: "tlshandshake://www.google.com:443"}
+	err := r.Run(ctx)
+	if err == nil || err.Error() != "interrupted" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerTCPConnectWithContextCanceled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	r := urlgetter.Runner{Target: "tcpconnect://www.google.com:443"}
+	err := r.Run(ctx)
+	if err == nil || err.Error() != "interrupted" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerWithInvalidURL(t *testing.T) {
+	r := urlgetter.Runner{Target: "\t"}
+	err := r.Run(context.Background())
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerWithEmptyHostname(t *testing.T) {
+	r := urlgetter.Runner{Target: "http:///foo.txt"}
+	err := r.Run(context.Background())
+	if err == nil || !strings.HasSuffix(err.Error(), "no Host in request URL") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerTLSHandshakeSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	r := urlgetter.Runner{Target: "tlshandshake://www.google.com:443"}
+	err := r.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRunnerTCPConnectSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	r := urlgetter.Runner{Target: "tcpconnect://www.google.com:443"}
+	err := r.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRunnerDNSLookupSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	r := urlgetter.Runner{Target: "dnslookup://www.google.com"}
+	err := r.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRunnerHTTPSSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	r := urlgetter.Runner{Target: "https://www.google.com"}
+	err := r.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRunnerHTTPSetHostHeader(t *testing.T) {
+	var host string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		host = r.Host
+		w.WriteHeader(200)
+	}))
+	defer server.Close()
+	r := urlgetter.Runner{
+		Config: urlgetter.Config{
+			HTTPHost: "x.org",
+		},
+		Target: server.URL,
+	}
+	err := r.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if host != "x.org" {
+		t.Fatal("not the host we expected")
+	}
+}
+
+func TestRunnerHTTPNoRedirect(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add("Location", "http:///") // cause failure if we redirect
+		w.WriteHeader(302)
+	}))
+	defer server.Close()
+	r := urlgetter.Runner{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true,
+		},
+		Target: server.URL,
+	}
+	err := r.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRunnerHTTPCannotReadBody(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hijacker, ok := w.(http.Hijacker)
+		if !ok {
+			panic("hijacking not supported by this server")
+		}
+		conn, _, _ := hijacker.Hijack()
+		conn.Write([]byte("HTTP/1.1 200 Ok\r\n"))
+		conn.Write([]byte("Content-Length: 1024\r\n"))
+		conn.Write([]byte("\r\n"))
+		conn.Write([]byte("123456789"))
+		conn.Close()
+	}))
+	defer server.Close()
+	r := urlgetter.Runner{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true,
+		},
+		Target: server.URL,
+	}
+	err := r.Run(context.Background())
+	if !errors.Is(err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerHTTPWeHandle400Correctly(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(400)
+	}))
+	defer server.Close()
+	r := urlgetter.Runner{
+		Config: urlgetter.Config{
+			FailOnHTTPError:   true,
+			NoFollowRedirects: true,
+		},
+		Target: server.URL,
+	}
+	err := r.Run(context.Background())
+	if !errors.Is(err, urlgetter.ErrHTTPRequestFailed) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerHTTPCannotReadBodyWinsOver400(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hijacker, ok := w.(http.Hijacker)
+		if !ok {
+			panic("hijacking not supported by this server")
+		}
+		conn, _, _ := hijacker.Hijack()
+		conn.Write([]byte("HTTP/1.1 400 Bad Request\r\n"))
+		conn.Write([]byte("Content-Length: 1024\r\n"))
+		conn.Write([]byte("\r\n"))
+		conn.Write([]byte("123456789"))
+		conn.Close()
+	}))
+	defer server.Close()
+	r := urlgetter.Runner{
+		Config: urlgetter.Config{
+			FailOnHTTPError:   true,
+			NoFollowRedirects: true,
+		},
+		Target: server.URL,
+	}
+	err := r.Run(context.Background())
+	if !errors.Is(err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerWeCanForceUserAgent(t *testing.T) {
+	expected := "antani/1.23.4-dev"
+	found := atomicx.NewInt64()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("User-Agent") == expected {
+			found.Add(1)
+		}
+		w.WriteHeader(200)
+	}))
+	defer server.Close()
+	r := urlgetter.Runner{
+		Config: urlgetter.Config{
+			FailOnHTTPError:   true,
+			NoFollowRedirects: true,
+			UserAgent:         expected,
+		},
+		Target: server.URL,
+	}
+	err := r.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if found.Load() != 1 {
+		t.Fatal("we didn't override the user agent")
+	}
+}
+
+func TestRunnerDefaultUserAgent(t *testing.T) {
+	expected := httpheader.UserAgent()
+	found := atomicx.NewInt64()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("User-Agent") == expected {
+			found.Add(1)
+		}
+		w.WriteHeader(200)
+	}))
+	defer server.Close()
+	r := urlgetter.Runner{
+		Config: urlgetter.Config{
+			FailOnHTTPError:   true,
+			NoFollowRedirects: true,
+		},
+		Target: server.URL,
+	}
+	err := r.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if found.Load() != 1 {
+		t.Fatal("we didn't override the user agent")
+	}
+}

internal/engine/experiment/urlgetter/urlgetter.go

@@ -0,0 +1,133 @@
+// Package urlgetter implements a nettest that fetches a URL.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-027-urlgetter.md.
+package urlgetter
+
+import (
+	"context"
+	"crypto/x509"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+)
+
+const (
+	testName    = "urlgetter"
+	testVersion = "0.1.0"
+)
+
+// Config contains the experiment's configuration.
+type Config struct {
+	// not settable from command line
+	CertPool *x509.CertPool
+	Timeout  time.Duration
+
+	// settable from command line
+	DNSCache          string `ooni:"Add 'DOMAIN IP...' to cache"`
+	DNSHTTPHost       string `ooni:"Force using specific HTTP Host header for DNS requests"`
+	DNSTLSServerName  string `ooni:"Force TLS to using a specific SNI for encrypted DNS requests"`
+	DNSTLSVersion     string `ooni:"Force specific TLS version used for DoT/DoH (e.g. 'TLSv1.3')"`
+	FailOnHTTPError   bool   `ooni:"Fail HTTP request if status code is 400 or above"`
+	HTTP3Enabled      bool   `ooni:"use http3 instead of http/1.1 or http2"`
+	HTTPHost          string `ooni:"Force using specific HTTP Host header"`
+	Method            string `ooni:"Force HTTP method different than GET"`
+	NoFollowRedirects bool   `ooni:"Disable following redirects"`
+	NoTLSVerify       bool   `ooni:"Disable TLS verification"`
+	RejectDNSBogons   bool   `ooni:"Fail DNS lookup if response contains bogons"`
+	ResolverURL       string `ooni:"URL describing the resolver to use"`
+	TLSServerName     string `ooni:"Force TLS to using a specific SNI in Client Hello"`
+	TLSVersion        string `ooni:"Force specific TLS version (e.g. 'TLSv1.3')"`
+	Tunnel            string `ooni:"Run experiment over a tunnel, e.g. psiphon"`
+	UserAgent         string `ooni:"Use the specified User-Agent"`
+}
+
+// TestKeys contains the experiment's result.
+type TestKeys struct {
+	// The following fields are part of the typical JSON emitted by OONI.
+	Agent           string                     `json:"agent"`
+	BootstrapTime   float64                    `json:"bootstrap_time,omitempty"`
+	DNSCache        []string                   `json:"dns_cache,omitempty"`
+	FailedOperation *string                    `json:"failed_operation"`
+	Failure         *string                    `json:"failure"`
+	NetworkEvents   []archival.NetworkEvent    `json:"network_events"`
+	Queries         []archival.DNSQueryEntry   `json:"queries"`
+	Requests        []archival.RequestEntry    `json:"requests"`
+	SOCKSProxy      string                     `json:"socksproxy,omitempty"`
+	TCPConnect      []archival.TCPConnectEntry `json:"tcp_connect"`
+	TLSHandshakes   []archival.TLSHandshake    `json:"tls_handshakes"`
+	Tunnel          string                     `json:"tunnel,omitempty"`
+
+	// The following fields are not serialised but are useful to simplify
+	// analysing the measurements in telegram, whatsapp, etc.
+	HTTPResponseStatus    int64    `json:"-"`
+	HTTPResponseBody      string   `json:"-"`
+	HTTPResponseLocations []string `json:"-"`
+}
+
+// RegisterExtensions registers the extensions used by the urlgetter
+// experiment into the provided measurement.
+func RegisterExtensions(m *model.Measurement) {
+	archival.ExtHTTP.AddTo(m)
+	archival.ExtDNS.AddTo(m)
+	archival.ExtNetevents.AddTo(m)
+	archival.ExtTCPConnect.AddTo(m)
+	archival.ExtTLSHandshake.AddTo(m)
+	archival.ExtTunnel.AddTo(m)
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	Config
+}
+
+// ExperimentName implements model.ExperimentSession.ExperimentName
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements model.ExperimentSession.ExperimentVersion
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements model.ExperimentSession.Run
+func (m Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	// When using the urlgetter experiment directly, there is a nonconfigurable
+	// default timeout that applies. When urlgetter is used as a library, it's
+	// instead the responsibility of the user of urlgetter to set timeouts. Note
+	// that this code is indeed only called when using urlgetter directly.
+	if m.Config.Timeout <= 0 {
+		m.Config.Timeout = 45 * time.Second
+	}
+	RegisterExtensions(measurement)
+	g := Getter{
+		Config:  m.Config,
+		Session: sess,
+		Target:  string(measurement.Input),
+	}
+	tk, err := g.Get(ctx)
+	measurement.TestKeys = &tk
+	return err
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/urlgetter/urlgetter_test.go

@@ -0,0 +1,90 @@
+package urlgetter_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestMeasurer(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	m := urlgetter.NewExperimentMeasurer(urlgetter.Config{})
+	if m.ExperimentName() != "urlgetter" {
+		t.Fatal("invalid experiment name")
+	}
+	if m.ExperimentVersion() != "0.1.0" {
+		t.Fatal("invalid experiment version")
+	}
+	measurement := new(model.Measurement)
+	measurement.Input = "https://www.google.com"
+	err := m.Run(
+		ctx, &mockable.Session{},
+		measurement, model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if len(measurement.Extensions) != 6 {
+		t.Fatal("not the expected number of extensions")
+	}
+	tk := measurement.TestKeys.(*urlgetter.TestKeys)
+	if len(tk.DNSCache) != 0 {
+		t.Fatal("not the DNSCache value we expected")
+	}
+	sk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(urlgetter.SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestMeasurerDNSCache(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	m := urlgetter.NewExperimentMeasurer(urlgetter.Config{
+		DNSCache: "dns.google 8.8.8.8 8.8.4.4",
+	})
+	if m.ExperimentName() != "urlgetter" {
+		t.Fatal("invalid experiment name")
+	}
+	if m.ExperimentVersion() != "0.1.0" {
+		t.Fatal("invalid experiment version")
+	}
+	measurement := new(model.Measurement)
+	measurement.Input = "https://www.google.com"
+	err := m.Run(
+		ctx, &mockable.Session{},
+		measurement, model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if len(measurement.Extensions) != 6 {
+		t.Fatal("not the expected number of extensions")
+	}
+	tk := measurement.TestKeys.(*urlgetter.TestKeys)
+	if len(tk.DNSCache) != 1 || tk.DNSCache[0] != "dns.google 8.8.8.8 8.8.4.4" {
+		t.Fatal("invalid tk.DNSCache")
+	}
+}
+
+func TestSummaryKeysGeneric(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &urlgetter.TestKeys{}}
+	m := &urlgetter.Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(urlgetter.SummaryKeys)
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/webconnectivity/connects.go

@@ -0,0 +1,58 @@
+package webconnectivity
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// ConnectsConfig contains the config for Connects
+type ConnectsConfig struct {
+	Session       model.ExperimentSession
+	TargetURL     *url.URL
+	URLGetterURLs []string
+}
+
+// TODO(bassosimone): we should normalize the timings
+
+// ConnectsResult contains the results of Connects
+type ConnectsResult struct {
+	AllKeys   []urlgetter.TestKeys
+	Successes int
+	Total     int
+}
+
+// Connects performs 0..N connects (either using TCP or TLS) to
+// check whether the resolved endpoints are reachable.
+func Connects(ctx context.Context, config ConnectsConfig) (out ConnectsResult) {
+	out.AllKeys = []urlgetter.TestKeys{}
+	multi := urlgetter.Multi{Session: config.Session}
+	inputs := []urlgetter.MultiInput{}
+	for _, url := range config.URLGetterURLs {
+		inputs = append(inputs, urlgetter.MultiInput{
+			Config: urlgetter.Config{
+				TLSServerName: config.TargetURL.Hostname(),
+			},
+			Target: url,
+		})
+	}
+	outputs := multi.Collect(ctx, inputs, "check", ConnectsNoCallbacks{})
+	for multiout := range outputs {
+		out.AllKeys = append(out.AllKeys, multiout.TestKeys)
+		for _, entry := range multiout.TestKeys.TCPConnect {
+			if entry.Status.Success {
+				out.Successes++
+			}
+			out.Total++
+		}
+	}
+	return
+}
+
+// ConnectsNoCallbacks suppresses the callbacks
+type ConnectsNoCallbacks struct{}
+
+// OnProgress implements ExperimentCallbacks.OnProgress
+func (ConnectsNoCallbacks) OnProgress(percentage float64, message string) {}

internal/engine/experiment/webconnectivity/connects_test.go

@@ -0,0 +1,55 @@
+package webconnectivity_test
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+)
+
+func TestConnectsSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	ctx := context.Background()
+	r := webconnectivity.Connects(ctx, webconnectivity.ConnectsConfig{
+		Session:   newsession(t, false),
+		TargetURL: &url.URL{Scheme: "https", Host: "cloudflare-dns.com", Path: "/"},
+		URLGetterURLs: []string{
+			"tlshandshake://104.16.249.249:443", "tlshandshake://104.16.248.249:443",
+			"tlshandshake://[2606:4700::6810:f9f9]:443",
+			"tlshandshake://[2606:4700::6810:f8f9]:443",
+		},
+	})
+	if len(r.AllKeys) != 4 {
+		t.Fatal("unexpected number of TestKeys lists")
+	}
+	if r.Successes < 1 {
+		t.Fatal("no successes?!")
+	}
+	if r.Total != 4 {
+		t.Fatal("unexpected number of attempts")
+	}
+}
+
+func TestConnectsNoInput(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	ctx := context.Background()
+	r := webconnectivity.Connects(ctx, webconnectivity.ConnectsConfig{
+		Session:       newsession(t, false),
+		TargetURL:     &url.URL{Scheme: "https", Host: "cloudflare-dns.com", Path: "/"},
+		URLGetterURLs: []string{},
+	})
+	if len(r.AllKeys) != 0 {
+		t.Fatal("unexpected number of TestKeys lists")
+	}
+	if r.Successes != 0 {
+		t.Fatal("successes?!")
+	}
+	if r.Total != 0 {
+		t.Fatal("unexpected number of attempts")
+	}
+}

internal/engine/experiment/webconnectivity/control.go

@@ -0,0 +1,84 @@
+package webconnectivity
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+// ControlRequest is the request that we send to the control
+type ControlRequest struct {
+	HTTPRequest        string              `json:"http_request"`
+	HTTPRequestHeaders map[string][]string `json:"http_request_headers"`
+	TCPConnect         []string            `json:"tcp_connect"`
+}
+
+// ControlTCPConnectResult is the result of the TCP connect
+// attempt performed by the control vantage point.
+type ControlTCPConnectResult struct {
+	Status  bool    `json:"status"`
+	Failure *string `json:"failure"`
+}
+
+// ControlHTTPRequestResult is the result of the HTTP request
+// performed by the control vantage point.
+type ControlHTTPRequestResult struct {
+	BodyLength int64             `json:"body_length"`
+	Failure    *string           `json:"failure"`
+	Title      string            `json:"title"`
+	Headers    map[string]string `json:"headers"`
+	StatusCode int64             `json:"status_code"`
+}
+
+// ControlDNSResult is the result of the DNS lookup
+// performed by the control vantage point.
+type ControlDNSResult struct {
+	Failure *string  `json:"failure"`
+	Addrs   []string `json:"addrs"`
+	ASNs    []int64  `json:"-"` // not visible from the JSON
+}
+
+// ControlResponse is the response from the control service.
+type ControlResponse struct {
+	TCPConnect  map[string]ControlTCPConnectResult `json:"tcp_connect"`
+	HTTPRequest ControlHTTPRequestResult           `json:"http_request"`
+	DNS         ControlDNSResult                   `json:"dns"`
+}
+
+// Control performs the control request and returns the response.
+func Control(
+	ctx context.Context, sess model.ExperimentSession,
+	thAddr string, creq ControlRequest) (out ControlResponse, err error) {
+	clnt := httpx.Client{
+		BaseURL:    thAddr,
+		HTTPClient: sess.DefaultHTTPClient(),
+		Logger:     sess.Logger(),
+	}
+	sess.Logger().Infof("control %s...", creq.HTTPRequest)
+	// make sure error is wrapped
+	err = errorx.SafeErrWrapperBuilder{
+		Error:     clnt.PostJSON(ctx, "/", creq, &out),
+		Operation: errorx.TopLevelOperation,
+	}.MaybeBuild()
+	sess.Logger().Infof("control %s... %+v", creq.HTTPRequest, err)
+	(&out.DNS).FillASNs(sess)
+	return
+}
+
+// FillASNs fills the ASNs array of ControlDNSResult. For each Addr inside
+// of the ControlDNSResult structure, we obtain the corresponding ASN.
+//
+// This is very useful to know what ASNs were the IP addresses returned by
+// the control according to the probe's ASN database.
+func (dns *ControlDNSResult) FillASNs(sess model.ExperimentSession) {
+	dns.ASNs = []int64{}
+	for _, ip := range dns.Addrs {
+		// TODO(bassosimone): this would be more efficient if we'd open just
+		// once the database and then reuse it for every address.
+		asn, _, _ := geolocate.LookupASN(sess.ASNDatabasePath(), ip)
+		dns.ASNs = append(dns.ASNs, int64(asn))
+	}
+}

internal/engine/experiment/webconnectivity/control_test.go

@@ -0,0 +1,36 @@
+package webconnectivity_test
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+)
+
+func TestFillASNsEmpty(t *testing.T) {
+	dns := new(webconnectivity.ControlDNSResult)
+	dns.FillASNs(new(mockable.Session))
+	if diff := cmp.Diff(dns.ASNs, []int64{}); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestFillASNsNoDatabase(t *testing.T) {
+	dns := new(webconnectivity.ControlDNSResult)
+	dns.Addrs = []string{"8.8.8.8", "1.1.1.1"}
+	dns.FillASNs(new(mockable.Session))
+	if diff := cmp.Diff(dns.ASNs, []int64{0, 0}); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestFillASNsSuccess(t *testing.T) {
+	sess := newsession(t, false)
+	dns := new(webconnectivity.ControlDNSResult)
+	dns.Addrs = []string{"8.8.8.8", "1.1.1.1"}
+	dns.FillASNs(sess)
+	if diff := cmp.Diff(dns.ASNs, []int64{15169, 13335}); diff != "" {
+		t.Fatal(diff)
+	}
+}

internal/engine/experiment/webconnectivity/dnsanalysis.go

@@ -0,0 +1,99 @@
+package webconnectivity
+
+import (
+	"net"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+// DNSAnalysisResult contains the results of analysing comparing
+// the measurement and the control DNS results.
+type DNSAnalysisResult struct {
+	DNSConsistency *string `json:"dns_consistency"`
+}
+
+// DNSNameError is the error returned by the control on NXDOMAIN
+const DNSNameError = "dns_name_error"
+
+var (
+	// DNSConsistent indicates that the measurement and the
+	// control have consistent DNS results.
+	DNSConsistent = "consistent"
+
+	// DNSInconsistent indicates that the measurement and the
+	// control have inconsistent DNS results.
+	DNSInconsistent = "inconsistent"
+)
+
+// DNSAnalysis compares the measurement and the control DNS results. This
+// implementation is a simplified version of the implementation of the same
+// check implemented in Measurement Kit v0.10.11.
+func DNSAnalysis(URL *url.URL, measurement DNSLookupResult,
+	control ControlResponse) (out DNSAnalysisResult) {
+	// 0. start assuming it's not consistent
+	out.DNSConsistency = &DNSInconsistent
+	// 1. flip to consistent if we're targeting an IP address because the
+	// control will actually return dns_name_error in this case.
+	if net.ParseIP(URL.Hostname()) != nil {
+		out.DNSConsistency = &DNSConsistent
+		return
+	}
+	// 2. flip to consistent if the failures are compatible
+	if measurement.Failure != nil && control.DNS.Failure != nil {
+		switch *control.DNS.Failure {
+		case DNSNameError: // the control returns this on NXDOMAIN error
+			switch *measurement.Failure {
+			case errorx.FailureDNSNXDOMAINError:
+				out.DNSConsistency = &DNSConsistent
+			}
+		}
+		return
+	}
+	// 3. flip to consistent if measurement and control returned IP addresses
+	// that belong to the same Autonomous System(s).
+	//
+	// This specific check is present in MK's implementation.
+	//
+	// Note that this covers also the cases where the measurement contains only
+	// bogons while the control does not contain bogons.
+	//
+	// Note that this also covers the cases where results are equal.
+	const (
+		inMeasurement = 1 << 0
+		inControl     = 1 << 1
+		inBoth        = inMeasurement | inControl
+	)
+	asnmap := make(map[int64]int)
+	for _, asn := range measurement.Addrs {
+		asnmap[asn] |= inMeasurement
+	}
+	for _, asn := range control.DNS.ASNs {
+		asnmap[asn] |= inControl
+	}
+	for key, value := range asnmap {
+		// zero means that ASN lookup failed
+		if key != 0 && (value&inBoth) == inBoth {
+			out.DNSConsistency = &DNSConsistent
+			return
+		}
+	}
+	// 4. when ASN lookup failed (unlikely), check whether
+	// there is overlap in the returned IP addresses
+	ipmap := make(map[string]int)
+	for ip := range measurement.Addrs {
+		ipmap[ip] |= inMeasurement
+	}
+	for _, ip := range control.DNS.Addrs {
+		ipmap[ip] |= inControl
+	}
+	for key, value := range ipmap {
+		// just in case an empty string slipped through
+		if key != "" && (value&inBoth) == inBoth {
+			out.DNSConsistency = &DNSConsistent
+			return
+		}
+	}
+	// 5. conclude that measurement and control are inconsistent
+	return
+}

internal/engine/experiment/webconnectivity/dnsanalysis_test.go

@@ -0,0 +1,193 @@
+package webconnectivity_test
+
+import (
+	"io"
+	"net/url"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestDNSAnalysis(t *testing.T) {
+	measurementFailure := errorx.FailureDNSNXDOMAINError
+	controlFailure := webconnectivity.DNSNameError
+	eofFailure := io.EOF.Error()
+	type args struct {
+		URL         *url.URL
+		measurement webconnectivity.DNSLookupResult
+		control     webconnectivity.ControlResponse
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut webconnectivity.DNSAnalysisResult
+	}{{
+		name: "when the URL contains an IP address",
+		args: args{
+			URL: &url.URL{
+				Host: "10.0.0.1",
+			},
+			control: webconnectivity.ControlResponse{
+				DNS: webconnectivity.ControlDNSResult{
+					Failure: &controlFailure,
+				},
+			},
+		},
+		wantOut: webconnectivity.DNSAnalysisResult{
+			DNSConsistency: &webconnectivity.DNSConsistent,
+		},
+	}, {
+		name: "when the failures are not compatible",
+		args: args{
+			URL: &url.URL{
+				Host: "www.kerneltrap.org",
+			},
+			measurement: webconnectivity.DNSLookupResult{
+				Failure: &eofFailure,
+			},
+			control: webconnectivity.ControlResponse{
+				DNS: webconnectivity.ControlDNSResult{
+					Failure: &controlFailure,
+				},
+			},
+		},
+		wantOut: webconnectivity.DNSAnalysisResult{
+			DNSConsistency: &webconnectivity.DNSInconsistent,
+		},
+	}, {
+		name: "when the failures are compatible",
+		args: args{
+			URL: &url.URL{
+				Host: "www.kerneltrap.org",
+			},
+			measurement: webconnectivity.DNSLookupResult{
+				Failure: &measurementFailure,
+			},
+			control: webconnectivity.ControlResponse{
+				DNS: webconnectivity.ControlDNSResult{
+					Failure: &controlFailure,
+				},
+			},
+		},
+		wantOut: webconnectivity.DNSAnalysisResult{
+			DNSConsistency: &webconnectivity.DNSConsistent,
+		},
+	}, {
+		name: "when the ASNs are equal",
+		args: args{
+			URL: &url.URL{
+				Host: "fancy.dns",
+			},
+			measurement: webconnectivity.DNSLookupResult{
+				Addrs: map[string]int64{
+					"1.1.1.1": 15169,
+					"8.8.8.8": 13335,
+				},
+			},
+			control: webconnectivity.ControlResponse{
+				DNS: webconnectivity.ControlDNSResult{
+					ASNs: []int64{13335, 15169},
+				},
+			},
+		},
+		wantOut: webconnectivity.DNSAnalysisResult{
+			DNSConsistency: &webconnectivity.DNSConsistent,
+		},
+	}, {
+		name: "when the ASNs overlap",
+		args: args{
+			URL: &url.URL{
+				Host: "fancy.dns",
+			},
+			measurement: webconnectivity.DNSLookupResult{
+				Addrs: map[string]int64{
+					"1.1.1.1": 15169,
+					"8.8.8.8": 13335,
+				},
+			},
+			control: webconnectivity.ControlResponse{
+				DNS: webconnectivity.ControlDNSResult{
+					ASNs: []int64{13335, 13335},
+				},
+			},
+		},
+		wantOut: webconnectivity.DNSAnalysisResult{
+			DNSConsistency: &webconnectivity.DNSConsistent,
+		},
+	}, {
+		name: "when the ASNs do not overlap",
+		args: args{
+			URL: &url.URL{
+				Host: "fancy.dns",
+			},
+			measurement: webconnectivity.DNSLookupResult{
+				Addrs: map[string]int64{
+					"1.1.1.1": 15169,
+					"8.8.8.8": 15169,
+				},
+			},
+			control: webconnectivity.ControlResponse{
+				DNS: webconnectivity.ControlDNSResult{
+					ASNs: []int64{13335, 13335},
+				},
+			},
+		},
+		wantOut: webconnectivity.DNSAnalysisResult{
+			DNSConsistency: &webconnectivity.DNSInconsistent,
+		},
+	}, {
+		name: "when ASNs lookup fails but IPs overlap",
+		args: args{
+			URL: &url.URL{
+				Host: "fancy.dns",
+			},
+			measurement: webconnectivity.DNSLookupResult{
+				Addrs: map[string]int64{
+					"2001:4860:4860::8844": 0,
+					"8.8.4.4":              0,
+				},
+			},
+			control: webconnectivity.ControlResponse{
+				DNS: webconnectivity.ControlDNSResult{
+					Addrs: []string{"8.8.8.8", "2001:4860:4860::8844"},
+					ASNs:  []int64{0, 0},
+				},
+			},
+		},
+		wantOut: webconnectivity.DNSAnalysisResult{
+			DNSConsistency: &webconnectivity.DNSConsistent,
+		},
+	}, {
+		name: "when ASNs lookup fails and IPs do not overlap",
+		args: args{
+			URL: &url.URL{
+				Host: "fancy.dns",
+			},
+			measurement: webconnectivity.DNSLookupResult{
+				Addrs: map[string]int64{
+					"2001:4860:4860::8888": 0,
+					"8.8.8.8":              0,
+				},
+			},
+			control: webconnectivity.ControlResponse{
+				DNS: webconnectivity.ControlDNSResult{
+					Addrs: []string{"8.8.4.4", "2001:4860:4860::8844"},
+					ASNs:  []int64{0, 0},
+				},
+			},
+		},
+		wantOut: webconnectivity.DNSAnalysisResult{
+			DNSConsistency: &webconnectivity.DNSInconsistent,
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := webconnectivity.DNSAnalysis(tt.args.URL, tt.args.measurement, tt.args.control)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}

internal/engine/experiment/webconnectivity/dnslookup.go

@@ -0,0 +1,59 @@
+package webconnectivity
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"sort"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// DNSLookupConfig contains settings for the DNS lookup.
+type DNSLookupConfig struct {
+	Session model.ExperimentSession
+	URL     *url.URL
+}
+
+// DNSLookupResult contains the result of the DNS lookup.
+type DNSLookupResult struct {
+	Addrs    map[string]int64
+	Failure  *string
+	TestKeys urlgetter.TestKeys
+}
+
+// DNSLookup performs the DNS lookup part of Web Connectivity.
+func DNSLookup(ctx context.Context, config DNSLookupConfig) (out DNSLookupResult) {
+	target := fmt.Sprintf("dnslookup://%s", config.URL.Hostname())
+	config.Session.Logger().Infof("%s...", target)
+	result, err := urlgetter.Getter{Session: config.Session, Target: target}.Get(ctx)
+	out.Addrs = make(map[string]int64)
+	for _, query := range result.Queries {
+		for _, answer := range query.Answers {
+			if answer.IPv4 != "" {
+				out.Addrs[answer.IPv4] = answer.ASN
+				continue
+			}
+			if answer.IPv6 != "" {
+				out.Addrs[answer.IPv6] = answer.ASN
+			}
+		}
+	}
+	config.Session.Logger().Infof("%s... %+v", target, err)
+	out.Failure = result.Failure
+	out.TestKeys = result
+	return
+}
+
+// Addresses returns the IP addresses in the DNSLookupResult
+func (r DNSLookupResult) Addresses() (out []string) {
+	out = []string{}
+	for addr := range r.Addrs {
+		out = append(out, addr)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i] < out[j]
+	})
+	return
+}

internal/engine/experiment/webconnectivity/dnslookup_test.go

@@ -0,0 +1,79 @@
+package webconnectivity_test
+
+import (
+	"context"
+	"net"
+	"net/url"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+)
+
+func TestDNSLookup(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	config := webconnectivity.DNSLookupConfig{
+		Session: newsession(t, true),
+		URL:     &url.URL{Host: "dns.google"},
+	}
+	out := webconnectivity.DNSLookup(context.Background(), config)
+	if out.Failure != nil {
+		t.Fatal(*out.Failure)
+	}
+	if len(out.Addrs) < 1 {
+		t.Fatal("no addresses?!")
+	}
+	for addr, asn := range out.Addrs {
+		if net.ParseIP(addr) == nil {
+			t.Fatal("invalid addr")
+		}
+		if asn != 15169 {
+			t.Fatal("invalid asn")
+		}
+	}
+	if len(out.TestKeys.NetworkEvents) < 1 {
+		t.Fatal("no network events?!")
+	}
+	if len(out.TestKeys.Queries) < 1 {
+		t.Fatal("no queries?!")
+	}
+}
+
+func TestDNSLookupResult_Addresses(t *testing.T) {
+	type fields struct {
+		Addrs    map[string]int64
+		Failure  *string
+		TestKeys urlgetter.TestKeys
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantOut []string
+	}{{
+		name:    "with no entries",
+		fields:  fields{},
+		wantOut: []string{},
+	}, {
+		name: "with some entries",
+		fields: fields{
+			Addrs: map[string]int64{"1.1.1.1": 1, "2001:4860:4860::8844": 2},
+		},
+		wantOut: []string{"1.1.1.1", "2001:4860:4860::8844"},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := webconnectivity.DNSLookupResult{
+				Addrs:    tt.fields.Addrs,
+				Failure:  tt.fields.Failure,
+				TestKeys: tt.fields.TestKeys,
+			}
+			gotOut := r.Addresses()
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}

internal/engine/experiment/webconnectivity/endpoints.go

@@ -0,0 +1,77 @@
+package webconnectivity
+
+import (
+	"net"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
+)
+
+// EndpointInfo describes a TCP/TLS endpoint.
+type EndpointInfo struct {
+	String       string // String representation
+	URLGetterURL string // URL for urlgetter
+}
+
+// EndpointsList is a list of EndpointInfo
+type EndpointsList []EndpointInfo
+
+// Endpoints returns a list of endpoints for TCP connect
+func (el EndpointsList) Endpoints() (out []string) {
+	out = []string{}
+	for _, ei := range el {
+		out = append(out, ei.String)
+	}
+	return
+}
+
+// URLs returns a list of URLs for TCP urlgetter
+func (el EndpointsList) URLs() (out []string) {
+	out = []string{}
+	for _, ei := range el {
+		out = append(out, ei.URLGetterURL)
+	}
+	return
+}
+
+// NewEndpoints creates a list of TCP/TLS endpoints to test from the
+// target URL and the list of resolved IP addresses.
+func NewEndpoints(URL *url.URL, addrs []string) (out EndpointsList) {
+	out = EndpointsList{}
+	port := NewEndpointPort(URL)
+	for _, addr := range addrs {
+		endpoint := net.JoinHostPort(addr, port.Port)
+		out = append(out, EndpointInfo{
+			String:       endpoint,
+			URLGetterURL: (&url.URL{Scheme: port.URLGetterScheme, Host: endpoint}).String(),
+		})
+	}
+	return
+}
+
+// EndpointPort is the port to be used by a TCP/TLS endpoint.
+type EndpointPort struct {
+	URLGetterScheme string
+	Port            string
+}
+
+// NewEndpointPort creates an EndpointPort from the given URL. This function
+// panic if the scheme is not `http` or `https` as well as if the host is not
+// valid. The latter should not happen if you used url.Parse.
+func NewEndpointPort(URL *url.URL) (out EndpointPort) {
+	if URL.Scheme != "http" && URL.Scheme != "https" {
+		panic("passed an unexpected scheme")
+	}
+	switch URL.Scheme {
+	case "http":
+		out.URLGetterScheme, out.Port = "tcpconnect", "80"
+	case "https":
+		out.URLGetterScheme, out.Port = "tlshandshake", "443"
+	}
+	if URL.Host != URL.Hostname() {
+		_, port, err := net.SplitHostPort(URL.Host)
+		runtimex.PanicOnError(err, "SplitHostPort should not fail here")
+		out.Port = port
+	}
+	return
+}

internal/engine/experiment/webconnectivity/endpoints_test.go

@@ -0,0 +1,224 @@
+package webconnectivity_test
+
+import (
+	"net/url"
+	"sync"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+)
+
+func TestNewEndpointPortPanicsWithInvalidScheme(t *testing.T) {
+	counter := atomicx.NewInt64()
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer func() {
+			if recover() != nil {
+				counter.Add(1)
+			}
+			wg.Done()
+		}()
+		webconnectivity.NewEndpointPort(&url.URL{Scheme: "antani"})
+	}()
+	wg.Wait()
+	if counter.Load() != 1 {
+		t.Fatal("did not panic")
+	}
+}
+
+func TestNewEndpointPortPanicsWithInvalidHost(t *testing.T) {
+	counter := atomicx.NewInt64()
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer func() {
+			if recover() != nil {
+				counter.Add(1)
+			}
+			wg.Done()
+		}()
+		webconnectivity.NewEndpointPort(&url.URL{Scheme: "http", Host: "[::1"})
+	}()
+	wg.Wait()
+	if counter.Load() != 1 {
+		t.Fatal("did not panic")
+	}
+}
+
+func TestNewEndpointPortCommonCase(t *testing.T) {
+	type args struct {
+		URL *url.URL
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut webconnectivity.EndpointPort
+	}{{
+		name: "with http and no default port",
+		args: args{URL: &url.URL{
+			Scheme: "http",
+			Host:   "www.example.com",
+			Path:   "/",
+		}},
+		wantOut: webconnectivity.EndpointPort{
+			URLGetterScheme: "tcpconnect",
+			Port:            "80",
+		},
+	}, {
+		name: "with https and no default port",
+		args: args{URL: &url.URL{
+			Scheme: "https",
+			Host:   "www.example.com",
+			Path:   "/",
+		}},
+		wantOut: webconnectivity.EndpointPort{
+			URLGetterScheme: "tlshandshake",
+			Port:            "443",
+		},
+	}, {
+		name: "with http and custom port",
+		args: args{URL: &url.URL{
+			Scheme: "http",
+			Host:   "www.example.com:11",
+			Path:   "/",
+		}},
+		wantOut: webconnectivity.EndpointPort{
+			URLGetterScheme: "tcpconnect",
+			Port:            "11",
+		},
+	}, {
+		name: "with https and custom port",
+		args: args{URL: &url.URL{
+			Scheme: "https",
+			Host:   "www.example.com:11",
+			Path:   "/",
+		}},
+		wantOut: webconnectivity.EndpointPort{
+			URLGetterScheme: "tlshandshake",
+			Port:            "11",
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := webconnectivity.NewEndpointPort(tt.args.URL)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestNewEndpoints(t *testing.T) {
+	type args struct {
+		URL   *url.URL
+		addrs []string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut webconnectivity.EndpointsList
+	}{{
+		name: "with all empty",
+		args: args{
+			URL: &url.URL{
+				Scheme: "http",
+			},
+		},
+		wantOut: webconnectivity.EndpointsList{},
+	}, {
+		name: "with some https endpoints",
+		args: args{
+			URL: &url.URL{
+				Scheme: "https",
+			},
+			addrs: []string{"1.1.1.1", "8.8.8.8"},
+		},
+		wantOut: webconnectivity.EndpointsList{{
+			URLGetterURL: "tlshandshake://1.1.1.1:443",
+			String:       "1.1.1.1:443",
+		}, {
+			URLGetterURL: "tlshandshake://8.8.8.8:443",
+			String:       "8.8.8.8:443",
+		}},
+	}, {
+		name: "with some http endpoints",
+		args: args{
+			URL: &url.URL{
+				Scheme: "http",
+			},
+			addrs: []string{"2001:4860:4860::8888", "2001:4860:4860::8844"},
+		},
+		wantOut: webconnectivity.EndpointsList{{
+			URLGetterURL: "tcpconnect://[2001:4860:4860::8888]:80",
+			String:       "[2001:4860:4860::8888]:80",
+		}, {
+			URLGetterURL: "tcpconnect://[2001:4860:4860::8844]:80",
+			String:       "[2001:4860:4860::8844]:80",
+		}},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := webconnectivity.NewEndpoints(tt.args.URL, tt.args.addrs)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestEndpointsList_Endpoints(t *testing.T) {
+	tests := []struct {
+		name    string
+		el      webconnectivity.EndpointsList
+		wantOut []string
+	}{{
+		name:    "when empty",
+		wantOut: []string{},
+	}, {
+		name: "common case",
+		el: webconnectivity.EndpointsList{{
+			String: "1.1.1.1:443",
+		}, {
+			String: "8.8.8.8:80",
+		}},
+		wantOut: []string{"1.1.1.1:443", "8.8.8.8:80"},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := tt.el.Endpoints()
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestEndpointsList_URLs(t *testing.T) {
+	tests := []struct {
+		name    string
+		el      webconnectivity.EndpointsList
+		wantOut []string
+	}{{
+		name:    "when empty",
+		wantOut: []string{},
+	}, {
+		name: "common case",
+		el: webconnectivity.EndpointsList{{
+			URLGetterURL: "tlshandshake://1.1.1.1:443",
+		}, {
+			URLGetterURL: "tcpconnect://8.8.8.8:80",
+		}},
+		wantOut: []string{"tlshandshake://1.1.1.1:443", "tcpconnect://8.8.8.8:80"},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := tt.el.URLs()
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}

internal/engine/experiment/webconnectivity/httpanalysis.go

@@ -0,0 +1,250 @@
+package webconnectivity
+
+import (
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// HTTPAnalysisResult contains the results of the analysis performed on the
+// client. We obtain it by comparing the measurement and the control.
+type HTTPAnalysisResult struct {
+	BodyLengthMatch *bool   `json:"body_length_match"`
+	BodyProportion  float64 `json:"body_proportion"`
+	StatusCodeMatch *bool   `json:"status_code_match"`
+	HeadersMatch    *bool   `json:"headers_match"`
+	TitleMatch      *bool   `json:"title_match"`
+}
+
+// Log logs the results of the analysis
+func (har HTTPAnalysisResult) Log(logger model.Logger) {
+	logger.Infof("BodyLengthMatch: %+v", internal.BoolPointerToString(har.BodyLengthMatch))
+	logger.Infof("BodyProportion: %+v", har.BodyProportion)
+	logger.Infof("StatusCodeMatch: %+v", internal.BoolPointerToString(har.StatusCodeMatch))
+	logger.Infof("HeadersMatch: %+v", internal.BoolPointerToString(har.HeadersMatch))
+	logger.Infof("TitleMatch: %+v", internal.BoolPointerToString(har.TitleMatch))
+}
+
+// HTTPAnalysis performs follow-up analysis on the webconnectivity measurement by
+// comparing the measurement test keys and the control.
+func HTTPAnalysis(tk urlgetter.TestKeys, ctrl ControlResponse) (out HTTPAnalysisResult) {
+	out.BodyLengthMatch, out.BodyProportion = HTTPBodyLengthChecks(tk, ctrl)
+	out.StatusCodeMatch = HTTPStatusCodeMatch(tk, ctrl)
+	out.HeadersMatch = HTTPHeadersMatch(tk, ctrl)
+	out.TitleMatch = HTTPTitleMatch(tk, ctrl)
+	return
+}
+
+// HTTPBodyLengthChecks returns whether the measured body is reasonably
+// long as much as the control body as well as the proportion between
+// the two bodies. This check may return nil, nil when such a
+// comparison would actually not be applicable.
+func HTTPBodyLengthChecks(
+	tk urlgetter.TestKeys, ctrl ControlResponse) (match *bool, proportion float64) {
+	control := ctrl.HTTPRequest.BodyLength
+	if control <= 0 {
+		return
+	}
+	if len(tk.Requests) <= 0 {
+		return
+	}
+	response := tk.Requests[0].Response
+	if response.BodyIsTruncated {
+		return
+	}
+	measurement := int64(len(response.Body.Value))
+	if measurement <= 0 {
+		return
+	}
+	const bodyProportionFactor = 0.7
+	if measurement >= control {
+		proportion = float64(control) / float64(measurement)
+	} else {
+		proportion = float64(measurement) / float64(control)
+	}
+	v := proportion > bodyProportionFactor
+	match = &v
+	return
+}
+
+// HTTPStatusCodeMatch returns whether the status code of the measurement
+// matches the status code of the control, or nil if such comparison
+// is actually not applicable.
+func HTTPStatusCodeMatch(tk urlgetter.TestKeys, ctrl ControlResponse) (out *bool) {
+	control := ctrl.HTTPRequest.StatusCode
+	if len(tk.Requests) < 1 {
+		return // no real status code
+	}
+	measurement := tk.Requests[0].Response.Code
+	if control == 0 {
+		return // no real status code
+	}
+	if measurement == 0 {
+		return // no real status code
+	}
+	value := control == measurement
+	if value == true {
+		// if the status codes are equal, they clearly match
+		out = &value
+		return
+	}
+	// This fix is part of Web Connectivity in MK and in Python since
+	// basically forever; my recollection is that we want to work around
+	// cases where the test helper is failing(?!). Unlike previous
+	// implementations, this implementation avoids a false positive
+	// when both measurement and control statuses are 500.
+	if control/100 == 5 {
+		return
+	}
+	out = &value
+	return
+}
+
+// HTTPHeadersMatch returns whether uncommon headers match between control and
+// measurement, or nil if check is not applicable.
+func HTTPHeadersMatch(tk urlgetter.TestKeys, ctrl ControlResponse) *bool {
+	if len(tk.Requests) <= 0 {
+		return nil
+	}
+	if tk.Requests[0].Response.Code == 0 {
+		return nil
+	}
+	if ctrl.HTTPRequest.StatusCode == 0 {
+		return nil
+	}
+	control := ctrl.HTTPRequest.Headers
+	// Implementation note: using map because we only care about the
+	// keys being different and we ignore the values.
+	measurement := tk.Requests[0].Response.Headers
+	const (
+		inMeasurement = 1 << 0
+		inControl     = 1 << 1
+		inBoth        = inMeasurement | inControl
+	)
+	commonHeaders := map[string]bool{
+		"date":                      true,
+		"content-type":              true,
+		"server":                    true,
+		"cache-control":             true,
+		"vary":                      true,
+		"set-cookie":                true,
+		"location":                  true,
+		"expires":                   true,
+		"x-powered-by":              true,
+		"content-encoding":          true,
+		"last-modified":             true,
+		"accept-ranges":             true,
+		"pragma":                    true,
+		"x-frame-options":           true,
+		"etag":                      true,
+		"x-content-type-options":    true,
+		"age":                       true,
+		"via":                       true,
+		"p3p":                       true,
+		"x-xss-protection":          true,
+		"content-language":          true,
+		"cf-ray":                    true,
+		"strict-transport-security": true,
+		"link":                      true,
+		"x-varnish":                 true,
+	}
+	matching := make(map[string]int)
+	ours := make(map[string]bool)
+	for key := range measurement {
+		key = strings.ToLower(key)
+		if _, ok := commonHeaders[key]; !ok {
+			matching[key] |= inMeasurement
+		}
+		ours[key] = true
+	}
+	theirs := make(map[string]bool)
+	for key := range control {
+		key = strings.ToLower(key)
+		if _, ok := commonHeaders[key]; !ok {
+			matching[key] |= inControl
+		}
+		theirs[key] = true
+	}
+	// if they are equal we're done
+	if good := reflect.DeepEqual(ours, theirs); good {
+		return &good
+	}
+	// compute the intersection of uncommon headers
+	var intersection int
+	for _, value := range matching {
+		if (value & inBoth) == inBoth {
+			intersection++
+		}
+	}
+	good := intersection > 0
+	return &good
+}
+
+// GetTitle returns the title or an empty string.
+func GetTitle(measurementBody string) string {
+	re := regexp.MustCompile(`(?i)<title>([^<]{1,128})</title>`) // like MK
+	v := re.FindStringSubmatch(measurementBody)
+	if len(v) < 2 {
+		return ""
+	}
+	return v[1]
+}
+
+// HTTPTitleMatch returns whether the measurement and the control titles
+// reasonably match, or nil if not applicable.
+func HTTPTitleMatch(tk urlgetter.TestKeys, ctrl ControlResponse) (out *bool) {
+	if len(tk.Requests) <= 0 {
+		return
+	}
+	response := tk.Requests[0].Response
+	if response.Code == 0 {
+		return
+	}
+	if response.BodyIsTruncated {
+		return
+	}
+	if ctrl.HTTPRequest.StatusCode == 0 {
+		return
+	}
+	control := ctrl.HTTPRequest.Title
+	measurementBody := response.Body.Value
+	measurement := GetTitle(measurementBody)
+	if measurement == "" {
+		return
+	}
+	const (
+		inMeasurement = 1 << 0
+		inControl     = 1 << 1
+		inBoth        = inMeasurement | inControl
+	)
+	words := make(map[string]int)
+	// We don't consider to match words that are shorter than 5
+	// characters (5 is the average word length for english)
+	//
+	// The original implementation considered the word order but
+	// considering different languages it seems we could have less
+	// false positives by ignoring the word order.
+	const minWordLength = 5
+	for _, word := range strings.Split(measurement, " ") {
+		if len(word) >= minWordLength {
+			words[strings.ToLower(word)] |= inMeasurement
+		}
+	}
+	for _, word := range strings.Split(control, " ") {
+		if len(word) >= minWordLength {
+			words[strings.ToLower(word)] |= inControl
+		}
+	}
+	good := true
+	for _, score := range words {
+		if (score & inBoth) != inBoth {
+			good = false
+			break
+		}
+	}
+	return &good
+}

internal/engine/experiment/webconnectivity/httpanalysis_test.go

@@ -0,0 +1,760 @@
+package webconnectivity_test
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+)
+
+func TestHTTPBodyLengthChecks(t *testing.T) {
+	var (
+		trueValue  = true
+		falseValue = false
+	)
+	type args struct {
+		tk   urlgetter.TestKeys
+		ctrl webconnectivity.ControlResponse
+	}
+	tests := []struct {
+		name        string
+		args        args
+		lengthMatch *bool
+		proportion  float64
+	}{{
+		name:        "nothing",
+		args:        args{},
+		lengthMatch: nil,
+	}, {
+		name: "control length is nonzero",
+		args: args{
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					BodyLength: 1024,
+				},
+			},
+		},
+		lengthMatch: nil,
+	}, {
+		name: "response body is truncated",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						BodyIsTruncated: true,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					BodyLength: 1024,
+				},
+			},
+		},
+		lengthMatch: nil,
+	}, {
+		name: "response body length is zero",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					BodyLength: 1024,
+				},
+			},
+		},
+		lengthMatch: nil,
+	}, {
+		name: "match with bigger control",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Body: archival.MaybeBinaryValue{
+							Value: randx.Letters(768),
+						},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					BodyLength: 1024,
+				},
+			},
+		},
+		lengthMatch: &trueValue,
+		proportion:  0.75,
+	}, {
+		name: "match with bigger measurement",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Body: archival.MaybeBinaryValue{
+							Value: randx.Letters(1024),
+						},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					BodyLength: 768,
+				},
+			},
+		},
+		lengthMatch: &trueValue,
+		proportion:  0.75,
+	}, {
+		name: "not match with bigger control",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Body: archival.MaybeBinaryValue{
+							Value: randx.Letters(8),
+						},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					BodyLength: 16,
+				},
+			},
+		},
+		lengthMatch: &falseValue,
+		proportion:  0.5,
+	}, {
+		name: "match with bigger measurement",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Body: archival.MaybeBinaryValue{
+							Value: randx.Letters(16),
+						},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					BodyLength: 8,
+				},
+			},
+		},
+		lengthMatch: &falseValue,
+		proportion:  0.5,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			match, proportion := webconnectivity.HTTPBodyLengthChecks(tt.args.tk, tt.args.ctrl)
+			if diff := cmp.Diff(tt.lengthMatch, match); diff != "" {
+				t.Fatal(diff)
+			}
+			if diff := cmp.Diff(tt.proportion, proportion); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestStatusCodeMatch(t *testing.T) {
+	var (
+		trueValue  = true
+		falseValue = false
+	)
+	type args struct {
+		tk   urlgetter.TestKeys
+		ctrl webconnectivity.ControlResponse
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut *bool
+	}{{
+		name: "with all zero",
+		args: args{},
+	}, {
+		name: "with a request but zero status codes",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{}},
+			},
+		},
+	}, {
+		name: "with equal status codes including 5xx",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 501,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 501,
+				},
+			},
+		},
+		wantOut: &trueValue,
+	}, {
+		name: "with different status codes and the control being 5xx",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 407,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 501,
+				},
+			},
+		},
+		wantOut: nil,
+	}, {
+		name: "with different status codes and the control being not 5xx",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 407,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 200,
+				},
+			},
+		},
+		wantOut: &falseValue,
+	}, {
+		name: "with only response status code and no control status code",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 200,
+					},
+				}},
+			},
+		},
+	}, {
+		name: "with only control status code and no response status code",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 0,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 200,
+				},
+			},
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := webconnectivity.HTTPStatusCodeMatch(tt.args.tk, tt.args.ctrl)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestHeadersMatch(t *testing.T) {
+	var (
+		trueValue  = true
+		falseValue = false
+	)
+	type args struct {
+		tk   urlgetter.TestKeys
+		ctrl webconnectivity.ControlResponse
+	}
+	tests := []struct {
+		name string
+		args args
+		want *bool
+	}{{
+		name: "with no requests",
+		args: args{
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						"Date":   "Mon Jul 13 21:05:43 CEST 2020",
+						"Antani": "Mascetti",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: nil,
+	}, {
+		name: "with basically nothing",
+		args: args{},
+		want: nil,
+	}, {
+		name: "with request and no response status code",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						"Date":   "Mon Jul 13 21:05:43 CEST 2020",
+						"Antani": "Mascetti",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: nil,
+	}, {
+		name: "with no control status code",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Headers: map[string]archival.MaybeBinaryValue{
+							"Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"},
+						},
+						Code: 200,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{},
+		},
+		want: nil,
+	}, {
+		name: "with no uncommon headers",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Headers: map[string]archival.MaybeBinaryValue{
+							"Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"},
+						},
+						Code: 200,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						"Date": "Mon Jul 13 21:05:43 CEST 2020",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: &trueValue,
+	}, {
+		name: "with equal uncommon headers",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Headers: map[string]archival.MaybeBinaryValue{
+							"Date":   {Value: "Mon Jul 13 21:10:08 CEST 2020"},
+							"Antani": {Value: "MASCETTI"},
+						},
+						Code: 200,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						"Date":   "Mon Jul 13 21:05:43 CEST 2020",
+						"Antani": "MELANDRI",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: &trueValue,
+	}, {
+		name: "with different uncommon headers",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Headers: map[string]archival.MaybeBinaryValue{
+							"Date":   {Value: "Mon Jul 13 21:10:08 CEST 2020"},
+							"Antani": {Value: "MASCETTI"},
+						},
+						Code: 200,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						"Date":     "Mon Jul 13 21:05:43 CEST 2020",
+						"Melandri": "MASCETTI",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: &falseValue,
+	}, {
+		name: "with small uncommon intersection (X-Cache)",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Headers: map[string]archival.MaybeBinaryValue{
+							"Accept-Ranges":  {Value: "bytes"},
+							"Age":            {Value: "404727"},
+							"Cache-Control":  {Value: "max-age=604800"},
+							"Content-Length": {Value: "1256"},
+							"Content-Type":   {Value: "text/html; charset=UTF-8"},
+							"Date":           {Value: "Tue, 14 Jul 2020 22:26:09 GMT"},
+							"Etag":           {Value: "\"3147526947\""},
+							"Expires":        {Value: "Tue, 21 Jul 2020 22:26:09 GMT"},
+							"Last-Modified":  {Value: "Thu, 17 Oct 2019 07:18:26 GMT"},
+							"Server":         {Value: "ECS (dcb/7F3C)"},
+							"Vary":           {Value: "Accept-Encoding"},
+							"X-Cache":        {Value: "HIT"},
+						},
+						Code: 200,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						// Note: the test helper was probably requesting the
+						// resource in a different way. There is content-length
+						// in this response, maybe it's using HTTP/1.0?
+						"Accept-Ranges": "bytes",
+						"Age":           "469182",
+						"Cache-Control": "max-age=604800",
+						"Content-Type":  "text/html; charset=UTF-8",
+						"Date":          "Tue, 14 Jul 2020 22:26:08 GMT",
+						"Etag":          "\"3147526947\"",
+						"Expires":       "Tue, 21 Jul 2020 22:26:08 GMT",
+						"Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT",
+						"Server":        "ECS (nyb/1D07)",
+						"Vary":          "Accept-Encoding",
+						"X-Cache":       "HIT",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: &trueValue,
+	}, {
+		name: "with no uncommon intersection",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Headers: map[string]archival.MaybeBinaryValue{
+							"Accept-Ranges":  {Value: "bytes"},
+							"Age":            {Value: "404727"},
+							"Cache-Control":  {Value: "max-age=604800"},
+							"Content-Length": {Value: "1256"},
+							"Content-Type":   {Value: "text/html; charset=UTF-8"},
+							"Date":           {Value: "Tue, 14 Jul 2020 22:26:09 GMT"},
+							"Etag":           {Value: "\"3147526947\""},
+							"Expires":        {Value: "Tue, 21 Jul 2020 22:26:09 GMT"},
+							"Last-Modified":  {Value: "Thu, 17 Oct 2019 07:18:26 GMT"},
+							"Server":         {Value: "ECS (dcb/7F3C)"},
+							"Vary":           {Value: "Accept-Encoding"},
+						},
+						Code: 200,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						// Note: the test helper was probably requesting the
+						// resource in a different way. There is content-length
+						// in this response, maybe it's using HTTP/1.0?
+						"Accept-Ranges": "bytes",
+						"Age":           "469182",
+						"Cache-Control": "max-age=604800",
+						"Content-Type":  "text/html; charset=UTF-8",
+						"Date":          "Tue, 14 Jul 2020 22:26:08 GMT",
+						"Etag":          "\"3147526947\"",
+						"Expires":       "Tue, 21 Jul 2020 22:26:08 GMT",
+						"Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT",
+						"Server":        "ECS (nyb/1D07)",
+						"Vary":          "Accept-Encoding",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: &falseValue,
+	}, {
+		name: "with exactly equal headers",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Headers: map[string]archival.MaybeBinaryValue{
+							"Accept-Ranges": {Value: "bytes"},
+							"Age":           {Value: "404727"},
+							"Cache-Control": {Value: "max-age=604800"},
+							"Content-Type":  {Value: "text/html; charset=UTF-8"},
+							"Date":          {Value: "Tue, 14 Jul 2020 22:26:09 GMT"},
+							"Etag":          {Value: "\"3147526947\""},
+							"Expires":       {Value: "Tue, 21 Jul 2020 22:26:09 GMT"},
+							"Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"},
+							"Server":        {Value: "ECS (dcb/7F3C)"},
+							"Vary":          {Value: "Accept-Encoding"},
+						},
+						Code: 200,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						"Accept-Ranges": "bytes",
+						"Age":           "469182",
+						"Cache-Control": "max-age=604800",
+						"Content-Type":  "text/html; charset=UTF-8",
+						"Date":          "Tue, 14 Jul 2020 22:26:08 GMT",
+						"Etag":          "\"3147526947\"",
+						"Expires":       "Tue, 21 Jul 2020 22:26:08 GMT",
+						"Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT",
+						"Server":        "ECS (nyb/1D07)",
+						"Vary":          "Accept-Encoding",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: &trueValue,
+	}, {
+		name: "with equal headers except for the case",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Headers: map[string]archival.MaybeBinaryValue{
+							"accept-ranges": {Value: "bytes"},
+							"AGE":           {Value: "404727"},
+							"cache-Control": {Value: "max-age=604800"},
+							"Content-TyPe":  {Value: "text/html; charset=UTF-8"},
+							"DatE":          {Value: "Tue, 14 Jul 2020 22:26:09 GMT"},
+							"etag":          {Value: "\"3147526947\""},
+							"expires":       {Value: "Tue, 21 Jul 2020 22:26:09 GMT"},
+							"Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"},
+							"SerVer":        {Value: "ECS (dcb/7F3C)"},
+							"Vary":          {Value: "Accept-Encoding"},
+						},
+						Code: 200,
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					Headers: map[string]string{
+						"Accept-Ranges": "bytes",
+						"Age":           "469182",
+						"Cache-Control": "max-age=604800",
+						"Content-Type":  "text/html; charset=UTF-8",
+						"Date":          "Tue, 14 Jul 2020 22:26:08 GMT",
+						"Etag":          "\"3147526947\"",
+						"Expires":       "Tue, 21 Jul 2020 22:26:08 GMT",
+						"Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT",
+						"Server":        "ECS (nyb/1D07)",
+						"Vary":          "Accept-Encoding",
+					},
+					StatusCode: 200,
+				},
+			},
+		},
+		want: &trueValue,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := webconnectivity.HTTPHeadersMatch(tt.args.tk, tt.args.ctrl)
+			if diff := cmp.Diff(tt.want, got); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestTitleMatch(t *testing.T) {
+	var (
+		trueValue  = true
+		falseValue = false
+	)
+	type args struct {
+		tk   urlgetter.TestKeys
+		ctrl webconnectivity.ControlResponse
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut *bool
+	}{{
+		name:    "with all empty",
+		args:    args{},
+		wantOut: nil,
+	}, {
+		name: "with a request and no response",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{}},
+			},
+		},
+		wantOut: nil,
+	}, {
+		name: "with a response with truncated body",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code:            200,
+						BodyIsTruncated: true,
+					},
+				}},
+			},
+		},
+		wantOut: nil,
+	}, {
+		name: "with a response with good body",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 200,
+						Body: archival.MaybeBinaryValue{Value: "<HTML/>"},
+					},
+				}},
+			},
+		},
+		wantOut: nil,
+	}, {
+		name: "with all good but no titles",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 200,
+						Body: archival.MaybeBinaryValue{Value: "<HTML/>"},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 200,
+					Title:      "",
+				},
+			},
+		},
+		wantOut: nil,
+	}, {
+		name: "reasonably common case where it succeeds",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 200,
+						Body: archival.MaybeBinaryValue{
+							Value: "<HTML><TITLE>La community di MSN</TITLE></HTML>"},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 200,
+					Title:      "MSN Community",
+				},
+			},
+		},
+		wantOut: &trueValue,
+	}, {
+		name: "reasonably common case where it fails",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 200,
+						Body: archival.MaybeBinaryValue{
+							Value: "<HTML><TITLE>La communità di MSN</TITLE></HTML>"},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 200,
+					Title:      "MSN Community",
+				},
+			},
+		},
+		wantOut: &falseValue,
+	}, {
+		name: "when the title is too long",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 200,
+						Body: archival.MaybeBinaryValue{
+							Value: "<HTML><TITLE>" + randx.Letters(1024) + "</TITLE></HTML>"},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 200,
+					Title:      "MSN Community",
+				},
+			},
+		},
+		wantOut: nil,
+	}, {
+		name: "reasonably common case where it succeeds with case variations",
+		args: args{
+			tk: urlgetter.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Response: archival.HTTPResponse{
+						Code: 200,
+						Body: archival.MaybeBinaryValue{
+							Value: "<HTML><TiTLe>La commUNity di MSN</tITLE></HTML>"},
+					},
+				}},
+			},
+			ctrl: webconnectivity.ControlResponse{
+				HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+					StatusCode: 200,
+					Title:      "MSN COmmunity",
+				},
+			},
+		},
+		wantOut: &trueValue,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := webconnectivity.HTTPTitleMatch(tt.args.tk, tt.args.ctrl)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}

internal/engine/experiment/webconnectivity/httpget.go

@@ -0,0 +1,50 @@
+package webconnectivity
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// HTTPGetConfig contains the config for HTTPGet
+type HTTPGetConfig struct {
+	Addresses []string
+	Session   model.ExperimentSession
+	TargetURL *url.URL
+}
+
+// TODO(bassosimone): we should normalize the timings
+
+// HTTPGetResult contains the results of HTTPGet
+type HTTPGetResult struct {
+	TestKeys urlgetter.TestKeys
+	Failure  *string
+}
+
+// HTTPGet performs the HTTP/HTTPS part of Web Connectivity.
+func HTTPGet(ctx context.Context, config HTTPGetConfig) (out HTTPGetResult) {
+	addresses := strings.Join(config.Addresses, " ")
+	if addresses == "" {
+		// TODO(bassosimone): what to do in this case? We clearly
+		// cannot fill the DNS cache...
+		return
+	}
+	target := config.TargetURL.String()
+	config.Session.Logger().Infof("GET %s...", target)
+	domain := config.TargetURL.Hostname()
+	result, err := urlgetter.Getter{
+		Config: urlgetter.Config{
+			DNSCache: fmt.Sprintf("%s %s", domain, addresses),
+		},
+		Session: config.Session,
+		Target:  target,
+	}.Get(ctx)
+	config.Session.Logger().Infof("GET %s... %+v", target, err)
+	out.Failure = result.Failure
+	out.TestKeys = result
+	return
+}

internal/engine/experiment/webconnectivity/httpget_test.go

@@ -0,0 +1,27 @@
+package webconnectivity_test
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+)
+
+func TestHTTPGet(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	ctx := context.Background()
+	r := webconnectivity.HTTPGet(ctx, webconnectivity.HTTPGetConfig{
+		Addresses: []string{"104.16.249.249", "104.16.248.249"},
+		Session:   newsession(t, false),
+		TargetURL: &url.URL{Scheme: "https", Host: "cloudflare-dns.com", Path: "/"},
+	})
+	if r.TestKeys.Failure != nil {
+		t.Fatal(*r.TestKeys.Failure)
+	}
+	if r.Failure != nil {
+		t.Fatal(*r.Failure)
+	}
+}

internal/engine/experiment/webconnectivity/internal/internal.go

@@ -0,0 +1,23 @@
+// Package internal contains internal code.
+package internal
+
+import "fmt"
+
+// StringPointerToString converts a string pointer to a string. When the
+// pointer is null, we return the "nil" string.
+func StringPointerToString(v *string) (out string) {
+	out = "nil"
+	if v != nil {
+		out = fmt.Sprintf("%+v", *v)
+	}
+	return
+}
+
+// BoolPointerToString is like StringPointerToString but for bool.
+func BoolPointerToString(v *bool) (out string) {
+	out = "nil"
+	if v != nil {
+		out = fmt.Sprintf("%+v", *v)
+	}
+	return
+}

internal/engine/experiment/webconnectivity/internal/internal_test.go

@@ -0,0 +1,31 @@
+package internal_test
+
+import (
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
+)
+
+func TestStringPointerToString(t *testing.T) {
+	s := "ANTANI"
+	if internal.StringPointerToString(&s) != s {
+		t.Fatal("unexpected result")
+	}
+	if internal.StringPointerToString(nil) != "nil" {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestBoolPointerToString(t *testing.T) {
+	v := true
+	if internal.BoolPointerToString(&v) != "true" {
+		t.Fatal("unexpected result")
+	}
+	v = false
+	if internal.BoolPointerToString(&v) != "false" {
+		t.Fatal("unexpected result")
+	}
+	if internal.BoolPointerToString(nil) != "nil" {
+		t.Fatal("unexpected result")
+	}
+}

internal/engine/experiment/webconnectivity/summary.go

@@ -0,0 +1,280 @@
+package webconnectivity
+
+import (
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+// The following set of status flags identifies in a more nuanced way the
+// reason why we say something is blocked, accessible, etc.
+//
+// This is an experimental implementation. The objective is to start using
+// it and learning from it, to eventually understand in which direction
+// the Web Connectivity experiment should evolve. For example, there are
+// a bunch of flags where our understandind is fuzzy or unclear.
+//
+// It also helps to write more precise unit and integration tests.
+const (
+	StatusSuccessSecure    = 1 << iota // success when using HTTPS
+	StatusSuccessCleartext             // success when using HTTP
+	StatusSuccessNXDOMAIN              // probe and control agree on NXDOMAIN
+
+	StatusAnomalyControlUnreachable // cannot access the control
+	StatusAnomalyControlFailure     // control failed for HTTP
+	StatusAnomalyDNS                // probe seems blocked by their DNS
+	StatusAnomalyHTTPDiff           // probe and control do not agree on HTTP features
+	StatusAnomalyConnect            // we saw an error when connecting
+	StatusAnomalyReadWrite          // we saw an error when doing I/O
+	StatusAnomalyUnknown            // we don't know when the error happened
+	StatusAnomalyTLSHandshake       // we think error was during TLS handshake
+
+	StatusExperimentDNS     // we noticed something in the DNS experiment
+	StatusExperimentConnect // ... in the connect experiment
+	StatusExperimentHTTP    // ... in the HTTP experiment
+
+	StatusBugNoRequests // this should never happen
+)
+
+// Summary contains the Web Connectivity summary.
+type Summary struct {
+	// Accessible is nil when the measurement failed, true if we do
+	// not think there was blocking, false in case of blocking.
+	Accessible *bool `json:"accessible"`
+
+	// BlockingReason indicates the cause of blocking when the Accessible
+	// variable is false. BlockingReason is meaningless otherwise.
+	//
+	// This is an intermediate variable used to compute Blocking, which
+	// is what OONI data consumers expect to see.
+	BlockingReason *string `json:"-"`
+
+	// Blocking implements the blocking variable as expected by OONI
+	// data consumers. See DetermineBlocking's docs.
+	Blocking interface{} `json:"blocking"`
+
+	// Status contains zero or more status flags. This is currently
+	// an experimental interface subject to change at any time.
+	Status int64 `json:"x_status"`
+}
+
+// DetermineBlocking returns the value of Summary.Blocking according to
+// the expectations of OONI data consumers (nil|false|string).
+//
+// Measurement Kit sets blocking to false when accessible is true. The spec
+// doesn't mention this possibility, as of 2019-08-20-001. Yet we implemented
+// it back in 2016, with little explanation <https://git.io/JJHOl>.
+//
+// We eventually managed to link such a change with the 0.3.4 release of
+// Measurement Kit <https://git.io/JJHOS>. This led us to find out the
+// related issue #867 <https://git.io/JJHOH>. From this issue it become
+// clear that the change on Measurement Kit was applied to mirror a change
+// implemented in OONI Probe Legacy <https://git.io/JJH3T>. In such a
+// change, determine_blocking() was modified to return False in case no
+// blocking was detected, to distinguish this case from the case where
+// there was an early failure in the experiment.
+//
+// Indeed, the OONI Android app uses the case where `blocking` is `null`
+// to flag failed tests. Instead, success is identified by `blocking` being
+// false and all other cases indicate anomaly <https://git.io/JJH3C>.
+//
+// Because of that, we must preserve the original behaviour.
+func DetermineBlocking(s Summary) interface{} {
+	if s.Accessible != nil && *s.Accessible == true {
+		return false
+	}
+	return s.BlockingReason
+}
+
+// Log logs the summary using the provided logger.
+func (s Summary) Log(logger model.Logger) {
+	logger.Infof("Blocking: %+v", internal.StringPointerToString(s.BlockingReason))
+	logger.Infof("Accessible: %+v", internal.BoolPointerToString(s.Accessible))
+}
+
+// Summarize computes the summary from the TestKeys.
+func Summarize(tk *TestKeys) (out Summary) {
+	// Make sure we correctly set out.Blocking's value.
+	defer func() {
+		out.Blocking = DetermineBlocking(out)
+	}()
+	var (
+		accessible   = true
+		inaccessible = false
+		dns          = "dns"
+		httpDiff     = "http-diff"
+		httpFailure  = "http-failure"
+		tcpIP        = "tcp_ip"
+	)
+	// If the measurement was for an HTTPS website and the HTTP experiment
+	// succeded, then either there is a compromised CA in our pool (which is
+	// certifi-go), or there is transparent proxying, or we are actually
+	// speaking with the legit server. We assume the latter. This applies
+	// also to cases in which we are redirected to HTTPS.
+	if len(tk.Requests) > 0 && tk.Requests[0].Failure == nil &&
+		strings.HasPrefix(tk.Requests[0].Request.URL, "https://") {
+		out.Accessible = &accessible
+		out.Status |= StatusSuccessSecure
+		return
+	}
+	// If we couldn't contact the control, we cannot do much more here.
+	if tk.ControlFailure != nil {
+		out.Status |= StatusAnomalyControlUnreachable
+		return
+	}
+	// If DNS failed with NXDOMAIN and the control DNS is consistent, then it
+	// means this website does not exist anymore.
+	if tk.DNSExperimentFailure != nil &&
+		*tk.DNSExperimentFailure == errorx.FailureDNSNXDOMAINError &&
+		tk.DNSConsistency != nil && *tk.DNSConsistency == DNSConsistent {
+		// TODO(bassosimone): MK flags this as accessible. This result is debateable. We
+		// are doing what MK does. But we most likely want to make it better later.
+		//
+		// See <https://github.com/ooni/probe-cli/v3/internal/engine/issues/579>.
+		out.Accessible = &accessible
+		out.Status |= StatusSuccessNXDOMAIN | StatusExperimentDNS
+		return
+	}
+	// Otherwise, if DNS failed with NXDOMAIN, it's DNS based blocking.
+	// TODO(bassosimone): do we wanna include other errors here? Like timeout?
+	if tk.DNSExperimentFailure != nil &&
+		*tk.DNSExperimentFailure == errorx.FailureDNSNXDOMAINError {
+		out.Accessible = &inaccessible
+		out.BlockingReason = &dns
+		out.Status |= StatusAnomalyDNS | StatusExperimentDNS
+		return
+	}
+	// If we tried to connect more than once and never succeded and we were
+	// able to measure DNS consistency, then we can conclude something.
+	if tk.TCPConnectAttempts > 0 && tk.TCPConnectSuccesses <= 0 && tk.DNSConsistency != nil {
+		out.Status |= StatusAnomalyConnect | StatusExperimentConnect
+		switch *tk.DNSConsistency {
+		case DNSConsistent:
+			// If the DNS is consistent, then it's TCP/IP blocking.
+			out.BlockingReason = &tcpIP
+			out.Accessible = &inaccessible
+		case DNSInconsistent:
+			// Otherwise, the culprit is the DNS.
+			out.BlockingReason = &dns
+			out.Accessible = &inaccessible
+			out.Status |= StatusAnomalyDNS
+		default:
+			// this case should not happen with this implementation
+			// so it's fine to leave this as unknown
+			out.Status |= StatusAnomalyUnknown
+		}
+		return
+	}
+	// If the control failed for HTTP it's not immediate for us to
+	// say anything specific on this measurement.
+	if tk.Control.HTTPRequest.Failure != nil {
+		out.Status |= StatusAnomalyControlFailure
+		return
+	}
+	// Likewise, if we don't have requests to examine, leave it.
+	if len(tk.Requests) < 1 {
+		out.Status |= StatusBugNoRequests
+		return
+	}
+	// If the HTTP measurement failed there could be a bunch of reasons
+	// why this occurred, because of HTTP redirects. Try to guess what
+	// could have been wrong by inspecting the error code.
+	if tk.Requests[0].Failure != nil {
+		out.Status |= StatusExperimentHTTP
+		switch *tk.Requests[0].Failure {
+		case errorx.FailureConnectionRefused:
+			// This is possibly because a subsequent connection to some
+			// other endpoint has been blocked. We call this http-failure
+			// because this is what MK would actually do.
+			out.BlockingReason = &httpFailure
+			out.Accessible = &inaccessible
+			out.Status |= StatusAnomalyConnect
+		case errorx.FailureConnectionReset:
+			// We don't currently support TLS failures and we don't have a
+			// way to know if it was during TLS or later. So, for now we are
+			// going to call this error condition an http-failure.
+			out.BlockingReason = &httpFailure
+			out.Accessible = &inaccessible
+			out.Status |= StatusAnomalyReadWrite
+		case errorx.FailureDNSNXDOMAINError:
+			// This is possibly because a subsequent resolution to
+			// some other domain name has been blocked.
+			out.BlockingReason = &dns
+			out.Accessible = &inaccessible
+			out.Status |= StatusAnomalyDNS
+		case errorx.FailureEOFError:
+			// We have seen this happening with TLS handshakes as well as
+			// sometimes with HTTP blocking. So http-failure.
+			out.BlockingReason = &httpFailure
+			out.Accessible = &inaccessible
+			out.Status |= StatusAnomalyReadWrite
+		case errorx.FailureGenericTimeoutError:
+			// Alas, here we don't know whether it's connect or whether it's
+			// perhaps the TLS handshake. So use the same classification used by MK.
+			out.BlockingReason = &httpFailure
+			out.Accessible = &inaccessible
+			out.Status |= StatusAnomalyUnknown
+		case errorx.FailureSSLInvalidHostname,
+			errorx.FailureSSLInvalidCertificate,
+			errorx.FailureSSLUnknownAuthority:
+			// We treat these three cases equally. Misconfiguration is a bit
+			// less likely since we also checked with the control. Since there
+			// is no TLS, for now we're going to call this http-failure.
+			out.BlockingReason = &httpFailure
+			out.Accessible = &inaccessible
+			out.Status |= StatusAnomalyTLSHandshake
+		default:
+			// We have not been able to classify the error. Could this perhaps be
+			// caused by a programmer's error? Let us be conservative.
+		}
+		// So, good that we have classified the error. Yet, how long is the
+		// redirect chain? If it's exactly one and we have determined that we
+		// should not trust the resolver, then let's bet on the DNS. If the
+		// chain is longer, for now better to be conservative. (I would argue
+		// that with a lying DNS that's likely the culprit, honestly.)
+		if out.BlockingReason != nil && len(tk.Requests) == 1 &&
+			tk.DNSConsistency != nil && *tk.DNSConsistency == DNSInconsistent {
+			out.BlockingReason = &dns
+			out.Status |= StatusAnomalyDNS
+		}
+		return
+	}
+	// So the HTTP request did not fail in the measurement and did not
+	// fail in the control as well, didn't it? Then, let us try to guess
+	// whether we've got the expected webpage after all. This set of
+	// conditions is adapted from MK v0.10.11.
+	if tk.StatusCodeMatch != nil && *tk.StatusCodeMatch {
+		if tk.BodyLengthMatch != nil && *tk.BodyLengthMatch {
+			out.Accessible = &accessible
+			out.Status |= StatusSuccessCleartext
+			return
+		}
+		if tk.HeadersMatch != nil && *tk.HeadersMatch {
+			out.Accessible = &accessible
+			out.Status |= StatusSuccessCleartext
+			return
+		}
+		if tk.TitleMatch != nil && *tk.TitleMatch {
+			out.Accessible = &accessible
+			out.Status |= StatusSuccessCleartext
+			return
+		}
+	}
+	// Set the status flag first
+	out.Status |= StatusAnomalyHTTPDiff
+	// It seems we didn't get the expected web page. What now? Well, if
+	// the DNS does not seem trustworthy, let us blame it.
+	if tk.DNSConsistency != nil && *tk.DNSConsistency == DNSInconsistent {
+		out.BlockingReason = &dns
+		out.Accessible = &inaccessible
+		out.Status |= StatusAnomalyDNS
+		return
+	}
+	// The only remaining conclusion seems that the web page we have got
+	// doesn't match what we were expecting.
+	out.BlockingReason = &httpDiff
+	out.Accessible = &inaccessible
+	return
+}

internal/engine/experiment/webconnectivity/summary_test.go

@@ -0,0 +1,459 @@
+package webconnectivity_test
+
+import (
+	"io"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestSummarize(t *testing.T) {
+	var (
+		genericFailure         = io.EOF.Error()
+		dns                    = "dns"
+		falseValue             = false
+		httpDiff               = "http-diff"
+		httpFailure            = "http-failure"
+		nilstring              *string
+		probeConnectionRefused = errorx.FailureConnectionRefused
+		probeConnectionReset   = errorx.FailureConnectionReset
+		probeEOFError          = errorx.FailureEOFError
+		probeNXDOMAIN          = errorx.FailureDNSNXDOMAINError
+		probeTimeout           = errorx.FailureGenericTimeoutError
+		probeSSLInvalidHost    = errorx.FailureSSLInvalidHostname
+		probeSSLInvalidCert    = errorx.FailureSSLInvalidCertificate
+		probeSSLUnknownAuth    = errorx.FailureSSLUnknownAuthority
+		tcpIP                  = "tcp_ip"
+		trueValue              = true
+	)
+	type args struct {
+		tk *webconnectivity.TestKeys
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantOut webconnectivity.Summary
+	}{{
+		name: "with an HTTPS request with no failure",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Request: archival.HTTPRequest{
+						URL: "https://www.kernel.org/",
+					},
+					Failure: nil,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       false,
+			Accessible:     &trueValue,
+			Status:         webconnectivity.StatusSuccessSecure,
+		},
+	}, {
+		name: "with failure in contacting the control",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				ControlFailure: &genericFailure,
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       nilstring,
+			Accessible:     nil,
+			Status:         webconnectivity.StatusAnomalyControlUnreachable,
+		},
+	}, {
+		name: "with non-existing website",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				DNSExperimentFailure: &probeNXDOMAIN,
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: &webconnectivity.DNSConsistent,
+				},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       false,
+			Accessible:     &trueValue,
+			Status: webconnectivity.StatusSuccessNXDOMAIN |
+				webconnectivity.StatusExperimentDNS,
+		},
+	}, {
+		name: "with NXDOMAIN measured only by the probe",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				DNSExperimentFailure: &probeNXDOMAIN,
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: &webconnectivity.DNSInconsistent,
+				},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &dns,
+			Blocking:       &dns,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusAnomalyDNS |
+				webconnectivity.StatusExperimentDNS,
+		},
+	}, {
+		name: "with TCP total failure and consistent DNS",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: &webconnectivity.DNSConsistent,
+				},
+				TCPConnectAttempts:  7,
+				TCPConnectSuccesses: 0,
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &tcpIP,
+			Blocking:       &tcpIP,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusAnomalyConnect |
+				webconnectivity.StatusExperimentConnect,
+		},
+	}, {
+		name: "with TCP total failure and inconsistent DNS",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: &webconnectivity.DNSInconsistent,
+				},
+				TCPConnectAttempts:  7,
+				TCPConnectSuccesses: 0,
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &dns,
+			Blocking:       &dns,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusAnomalyConnect |
+				webconnectivity.StatusExperimentConnect |
+				webconnectivity.StatusAnomalyDNS,
+		},
+	}, {
+		name: "with TCP total failure and unexpected DNS consistency",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: func() *string {
+						s := "ANTANI"
+						return &s
+					}(),
+				},
+				TCPConnectAttempts:  7,
+				TCPConnectSuccesses: 0,
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       nilstring,
+			Accessible:     nil,
+			Status: webconnectivity.StatusAnomalyConnect |
+				webconnectivity.StatusExperimentConnect |
+				webconnectivity.StatusAnomalyUnknown,
+		},
+	}, {
+		name: "with failed control HTTP request",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Control: webconnectivity.ControlResponse{
+					HTTPRequest: webconnectivity.ControlHTTPRequestResult{
+						Failure: &genericFailure,
+					},
+				},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       nilstring,
+			Accessible:     nil,
+			Status:         webconnectivity.StatusAnomalyControlFailure,
+		},
+	}, {
+		name: "with less that one request entry",
+		args: args{
+			tk: &webconnectivity.TestKeys{},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       nilstring,
+			Accessible:     nil,
+			Status:         webconnectivity.StatusBugNoRequests,
+		},
+	}, {
+		name: "with connection refused",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Failure: &probeConnectionRefused,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpFailure,
+			Blocking:       &httpFailure,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyConnect,
+		},
+	}, {
+		name: "with connection reset",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Failure: &probeConnectionReset,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpFailure,
+			Blocking:       &httpFailure,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyReadWrite,
+		},
+	}, {
+		name: "with NXDOMAIN",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Failure: &probeNXDOMAIN,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &dns,
+			Blocking:       &dns,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyDNS,
+		},
+	}, {
+		name: "with EOF",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Failure: &probeEOFError,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpFailure,
+			Blocking:       &httpFailure,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyReadWrite,
+		},
+	}, {
+		name: "with timeout",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Failure: &probeTimeout,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpFailure,
+			Blocking:       &httpFailure,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyUnknown,
+		},
+	}, {
+		name: "with SSL invalid hostname",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Failure: &probeSSLInvalidHost,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpFailure,
+			Blocking:       &httpFailure,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyTLSHandshake,
+		},
+	}, {
+		name: "with SSL invalid cert",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Failure: &probeSSLInvalidCert,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpFailure,
+			Blocking:       &httpFailure,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyTLSHandshake,
+		},
+	}, {
+		name: "with SSL unknown auth",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				Requests: []archival.RequestEntry{{
+					Failure: &probeSSLUnknownAuth,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpFailure,
+			Blocking:       &httpFailure,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyTLSHandshake,
+		},
+	}, {
+		name: "with SSL unknown auth _and_ untrustworthy DNS",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: &webconnectivity.DNSInconsistent,
+				},
+				Requests: []archival.RequestEntry{{
+					Failure: &probeSSLUnknownAuth,
+				}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &dns,
+			Blocking:       &dns,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyTLSHandshake |
+				webconnectivity.StatusAnomalyDNS,
+		},
+	}, {
+		name: "with SSL unknown auth _and_ untrustworthy DNS _and_ a longer chain",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: &webconnectivity.DNSInconsistent,
+				},
+				Requests: []archival.RequestEntry{{
+					Failure: &probeSSLUnknownAuth,
+				}, {}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpFailure,
+			Blocking:       &httpFailure,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusExperimentHTTP |
+				webconnectivity.StatusAnomalyTLSHandshake,
+		},
+	}, {
+		name: "with status code and body length matching",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
+					StatusCodeMatch: &trueValue,
+					BodyLengthMatch: &trueValue,
+				},
+				Requests: []archival.RequestEntry{{}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       falseValue,
+			Accessible:     &trueValue,
+			Status:         webconnectivity.StatusSuccessCleartext,
+		},
+	}, {
+		name: "with status code and headers matching",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
+					StatusCodeMatch: &trueValue,
+					HeadersMatch:    &trueValue,
+				},
+				Requests: []archival.RequestEntry{{}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       falseValue,
+			Accessible:     &trueValue,
+			Status:         webconnectivity.StatusSuccessCleartext,
+		},
+	}, {
+		name: "with status code and title matching",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
+					StatusCodeMatch: &trueValue,
+					TitleMatch:      &trueValue,
+				},
+				Requests: []archival.RequestEntry{{}},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: nil,
+			Blocking:       falseValue,
+			Accessible:     &trueValue,
+			Status:         webconnectivity.StatusSuccessCleartext,
+		},
+	}, {
+		name: "with suspect http-diff and inconsistent DNS",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
+					StatusCodeMatch: &falseValue,
+					TitleMatch:      &trueValue,
+				},
+				Requests: []archival.RequestEntry{{}},
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: &webconnectivity.DNSInconsistent,
+				},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &dns,
+			Blocking:       &dns,
+			Accessible:     &falseValue,
+			Status: webconnectivity.StatusAnomalyHTTPDiff |
+				webconnectivity.StatusAnomalyDNS,
+		},
+	}, {
+		name: "with suspect http-diff and consistent DNS",
+		args: args{
+			tk: &webconnectivity.TestKeys{
+				HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
+					StatusCodeMatch: &falseValue,
+					TitleMatch:      &trueValue,
+				},
+				Requests: []archival.RequestEntry{{}},
+				DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
+					DNSConsistency: &webconnectivity.DNSConsistent,
+				},
+			},
+		},
+		wantOut: webconnectivity.Summary{
+			BlockingReason: &httpDiff,
+			Blocking:       &httpDiff,
+			Accessible:     &falseValue,
+			Status:         webconnectivity.StatusAnomalyHTTPDiff,
+		},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotOut := webconnectivity.Summarize(tt.args.tk)
+			if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}