chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/experiment/dash/collect.go

@@ -0,0 +1,57 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type collectDeps interface {
+	HTTPClient() *http.Client
+	JSONMarshal(v interface{}) ([]byte, error)
+	Logger() model.Logger
+	NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
+	ReadAll(r io.Reader) ([]byte, error)
+	Scheme() string
+	UserAgent() string
+}
+
+func collect(ctx context.Context, fqdn, authorization string,
+	results []clientResults, deps collectDeps) error {
+	data, err := deps.JSONMarshal(results)
+	if err != nil {
+		return err
+	}
+	deps.Logger().Debugf("dash: body: %s", string(data))
+	var URL url.URL
+	URL.Scheme = deps.Scheme()
+	URL.Host = fqdn
+	URL.Path = collectPath
+	req, err := deps.NewHTTPRequest("POST", URL.String(), bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", deps.UserAgent())
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", authorization)
+	resp, err := deps.HTTPClient().Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != 200 {
+		return errHTTPRequestFailed
+	}
+	defer resp.Body.Close()
+	data, err = deps.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	deps.Logger().Debugf("dash: body: %s", string(data))
+	var serverResults []serverResults
+	return json.Unmarshal(data, &serverResults)
+}

internal/engine/experiment/dash/collect_test.go

@@ -0,0 +1,113 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestCollectJSONMarshalError(t *testing.T) {
+	expected := errors.New("mocked error")
+	deps := FakeDeps{jsonMarshalErr: expected}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectNewHTTPRequestFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	deps := FakeDeps{newHTTPRequestErr: expected}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectHTTPClientDoFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{err: expected}
+	deps := FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+		Header: http.Header{},
+		URL:    &url.URL{},
+	}}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectInternalError(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{StatusCode: 500}}
+	deps := FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+		Header: http.Header{},
+		URL:    &url.URL{},
+	}}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, errHTTPRequestFailed) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectReadAllFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllErr: expected,
+	}
+	err := collect(context.Background(), "", "", nil, deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectInvalidJSON(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte("["),
+	}
+	err := collect(context.Background(), "", "", nil, deps)
+	if err == nil || !strings.HasSuffix(err.Error(), "unexpected end of JSON input") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCollectSuccess(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte("[]"),
+	}
+	err := collect(context.Background(), "", "", nil, deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

internal/engine/experiment/dash/dash.go

@@ -0,0 +1,307 @@
+// Package dash implements the DASH network experiment.
+//
+// Spec: https://github.com/ooni/spec/blob/master/nettests/ts-021-dash.md
+package dash
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"runtime"
+	"time"
+
+	"github.com/montanaflynn/stats"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+const (
+	defaultTimeout = 120 * time.Second
+	magicVersion   = "0.008000000"
+	testName       = "dash"
+	testVersion    = "0.12.0"
+	totalStep      = 15.0
+)
+
+var (
+	errServerBusy        = errors.New("dash: server busy; try again later")
+	errHTTPRequestFailed = errors.New("dash: request failed")
+)
+
+// Config contains the experiment config.
+type Config struct{}
+
+// Simple contains the experiment total summary
+type Simple struct {
+	ConnectLatency  float64 `json:"connect_latency"`
+	MedianBitrate   int64   `json:"median_bitrate"`
+	MinPlayoutDelay float64 `json:"min_playout_delay"`
+}
+
+// ServerInfo contains information on the selected server
+//
+// This is currently an extension to the DASH specification
+// until the data format of the new mlab locate is clear.
+type ServerInfo struct {
+	Hostname string `json:"hostname"`
+	Site     string `json:"site,omitempty"`
+}
+
+// TestKeys contains the test keys
+type TestKeys struct {
+	Server       ServerInfo      `json:"server"`
+	Simple       Simple          `json:"simple"`
+	Failure      *string         `json:"failure"`
+	ReceiverData []clientResults `json:"receiver_data"`
+}
+
+type runner struct {
+	callbacks  model.ExperimentCallbacks
+	httpClient *http.Client
+	saver      *trace.Saver
+	sess       model.ExperimentSession
+	tk         *TestKeys
+}
+
+func (r runner) HTTPClient() *http.Client {
+	return r.httpClient
+}
+
+func (r runner) JSONMarshal(v interface{}) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+func (r runner) Logger() model.Logger {
+	return r.sess.Logger()
+}
+
+func (r runner) NewHTTPRequest(meth, url string, body io.Reader) (*http.Request, error) {
+	return http.NewRequest(meth, url, body)
+}
+
+func (r runner) ReadAll(reader io.Reader) ([]byte, error) {
+	return ioutil.ReadAll(reader)
+}
+
+func (r runner) Scheme() string {
+	return "https"
+}
+
+func (r runner) UserAgent() string {
+	return r.sess.UserAgent()
+}
+
+func (r runner) loop(ctx context.Context, numIterations int64) error {
+	locateResult, err := locate(ctx, r)
+	if err != nil {
+		return err
+	}
+	r.tk.Server = ServerInfo{
+		Hostname: locateResult.FQDN,
+		Site:     locateResult.Site,
+	}
+	fqdn := locateResult.FQDN
+	r.callbacks.OnProgress(0.0, fmt.Sprintf("streaming: server: %s", fqdn))
+	negotiateResp, err := negotiate(ctx, fqdn, r)
+	if err != nil {
+		return err
+	}
+	if err := r.measure(ctx, fqdn, negotiateResp, numIterations); err != nil {
+		return err
+	}
+	// TODO(bassosimone): it seems we're not saving the server data?
+	err = collect(ctx, fqdn, negotiateResp.Authorization, r.tk.ReceiverData, r)
+	if err != nil {
+		return err
+	}
+	return r.tk.analyze()
+}
+
+func (r runner) measure(
+	ctx context.Context, fqdn string, negotiateResp negotiateResponse,
+	numIterations int64) error {
+	// Note: according to a comment in MK sources 3000 kbit/s was the
+	// minimum speed recommended by Netflix for SD quality in 2017.
+	//
+	// See: <https://help.netflix.com/en/node/306>.
+	const initialBitrate = 3000
+	current := clientResults{
+		ElapsedTarget: 2,
+		Platform:      runtime.GOOS,
+		Rate:          initialBitrate,
+		RealAddress:   negotiateResp.RealAddress,
+		Version:       magicVersion,
+	}
+	var (
+		begin       = time.Now()
+		connectTime float64
+		total       int64
+	)
+	for current.Iteration < numIterations {
+		result, err := download(ctx, downloadConfig{
+			authorization: negotiateResp.Authorization,
+			begin:         begin,
+			currentRate:   current.Rate,
+			deps:          r,
+			elapsedTarget: current.ElapsedTarget,
+			fqdn:          fqdn,
+		})
+		if err != nil {
+			// Implementation note: ndt7 controls the connection much
+			// more than us and it can tell whether an error occurs when
+			// connecting or later. We cannot say that very precisely
+			// because, in principle, we may reconnect. So we always
+			// return error here. This comment is being introduced so
+			// that we don't do https://github.com/ooni/probe-cli/v3/internal/engine/pull/526
+			// again, because that isn't accurate.
+			return err
+		}
+		current.Elapsed = result.elapsed
+		current.Received = result.received
+		current.RequestTicks = result.requestTicks
+		current.Timestamp = result.timestamp
+		current.ServerURL = result.serverURL
+		// Read the events so far and possibly update our measurement
+		// of the latest connect time. We should have one sample in most
+		// cases, because the connection should be persistent.
+		for _, ev := range r.saver.Read() {
+			if ev.Name == errorx.ConnectOperation {
+				connectTime = ev.Duration.Seconds()
+			}
+		}
+		current.ConnectTime = connectTime
+		r.tk.ReceiverData = append(r.tk.ReceiverData, current)
+		total += current.Received
+		avgspeed := 8 * float64(total) / time.Now().Sub(begin).Seconds()
+		percentage := float64(current.Iteration) / float64(numIterations)
+		message := fmt.Sprintf("streaming: speed: %s", humanizex.SI(avgspeed, "bit/s"))
+		r.callbacks.OnProgress(percentage, message)
+		current.Iteration++
+		speed := float64(current.Received) / float64(current.Elapsed)
+		speed *= 8.0    // to bits per second
+		speed /= 1000.0 // to kbit/s
+		current.Rate = int64(speed)
+	}
+	return nil
+}
+
+func (tk *TestKeys) analyze() error {
+	var (
+		rates          []float64
+		frameReadyTime float64
+		playTime       float64
+	)
+	for _, results := range tk.ReceiverData {
+		rates = append(rates, float64(results.Rate))
+		// Same in all samples if we're using a single connection
+		tk.Simple.ConnectLatency = results.ConnectTime
+		// Rationale: first segment plays when it arrives. Subsequent segments
+		// would play in ElapsedTarget seconds. However, will play when they
+		// arrive. Stall is the time we need to wait for a frame to arrive with
+		// the video stopped and the spinning icon.
+		frameReadyTime += results.Elapsed
+		if playTime == 0.0 {
+			playTime += frameReadyTime
+		} else {
+			playTime += float64(results.ElapsedTarget)
+		}
+		stall := frameReadyTime - playTime
+		if stall > tk.Simple.MinPlayoutDelay {
+			tk.Simple.MinPlayoutDelay = stall
+		}
+	}
+	median, err := stats.Median(rates)
+	tk.Simple.MedianBitrate = int64(median)
+	return err
+}
+
+func (r runner) do(ctx context.Context) error {
+	defer r.callbacks.OnProgress(1, "streaming: done")
+	const numIterations = 15
+	err := r.loop(ctx, numIterations)
+	if err != nil {
+		s := err.Error()
+		r.tk.Failure = &s
+		// fallthrough
+	}
+	return err
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config Config
+}
+
+// ExperimentName implements model.ExperimentMeasurer.ExperimentName.
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements model.ExperimentMeasurer.ExperimentVersion.
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements model.ExperimentMeasurer.Run.
+func (m Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	tk := new(TestKeys)
+	measurement.TestKeys = tk
+	saver := &trace.Saver{}
+	httpClient := &http.Client{
+		Transport: netx.NewHTTPTransport(netx.Config{
+			ContextByteCounting: true,
+			DialSaver:           saver,
+			Logger:              sess.Logger(),
+		}),
+	}
+	defer httpClient.CloseIdleConnections()
+	r := runner{
+		callbacks:  callbacks,
+		httpClient: httpClient,
+		saver:      saver,
+		sess:       sess,
+		tk:         tk,
+	}
+	ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
+	defer cancel()
+	return r.do(ctx)
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	Latency   float64 `json:"connect_latency"`
+	Bitrate   float64 `json:"median_bitrate"`
+	Delay     float64 `json:"min_playout_delay"`
+	IsAnomaly bool    `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	sk.Latency = tk.Simple.ConnectLatency
+	sk.Bitrate = float64(tk.Simple.MedianBitrate)
+	sk.Delay = tk.Simple.MinPlayoutDelay
+	return sk, nil
+}

internal/engine/experiment/dash/dash_test.go

@@ -0,0 +1,328 @@
+package dash
+
+import (
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/montanaflynn/stats"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+func TestRunnerLoopLocateFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: FakeHTTPTransport{
+				err: expected,
+			},
+		},
+		saver: new(trace.Saver),
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerLoopNegotiateFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: &FakeHTTPTransportStack{
+				all: []FakeHTTPTransport{
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"fqdn": "ams01.measurementlab.net"}`)),
+							StatusCode: 200,
+						},
+					},
+					{err: expected},
+				},
+			},
+		},
+		saver: new(trace.Saver),
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerLoopMeasureFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: &FakeHTTPTransportStack{
+				all: []FakeHTTPTransport{
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"fqdn": "ams01.measurementlab.net"}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"authorization": "xx", "unchoked": 1}`)),
+							StatusCode: 200,
+						},
+					},
+					{err: expected},
+				},
+			},
+		},
+		saver: new(trace.Saver),
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerLoopCollectFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	saver := new(trace.Saver)
+	saver.Write(trace.Event{Name: errorx.ConnectOperation, Duration: 150 * time.Millisecond})
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: &FakeHTTPTransportStack{
+				all: []FakeHTTPTransport{
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"fqdn": "ams01.measurementlab.net"}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"authorization": "xx", "unchoked": 1}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body:       ioutil.NopCloser(strings.NewReader(`1234567`)),
+							StatusCode: 200,
+						},
+					},
+					{err: expected},
+				},
+			},
+		},
+		saver: saver,
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestRunnerLoopSuccess(t *testing.T) {
+	saver := new(trace.Saver)
+	saver.Write(trace.Event{Name: errorx.ConnectOperation, Duration: 150 * time.Millisecond})
+	r := runner{
+		callbacks: model.NewPrinterCallbacks(log.Log),
+		httpClient: &http.Client{
+			Transport: &FakeHTTPTransportStack{
+				all: []FakeHTTPTransport{
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"fqdn": "ams01.measurementlab.net"}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body: ioutil.NopCloser(strings.NewReader(
+								`{"authorization": "xx", "unchoked": 1}`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body:       ioutil.NopCloser(strings.NewReader(`1234567`)),
+							StatusCode: 200,
+						},
+					},
+					{
+						resp: &http.Response{
+							Body:       ioutil.NopCloser(strings.NewReader(`[]`)),
+							StatusCode: 200,
+						},
+					},
+				},
+			},
+		},
+		saver: saver,
+		sess: &mockable.Session{
+			MockableLogger: log.Log,
+		},
+		tk: new(TestKeys),
+	}
+	err := r.loop(context.Background(), 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestTestKeysAnalyzeWithNoData(t *testing.T) {
+	tk := &TestKeys{}
+	err := tk.analyze()
+	if !errors.Is(err, stats.EmptyInputErr) {
+		t.Fatal("expected an error here")
+	}
+}
+
+func TestTestKeysAnalyzeMedian(t *testing.T) {
+	tk := &TestKeys{
+		ReceiverData: []clientResults{
+			{
+				Rate: 1,
+			},
+			{
+				Rate: 2,
+			},
+			{
+				Rate: 3,
+			},
+		},
+	}
+	err := tk.analyze()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Simple.MedianBitrate != 2 {
+		t.Fatal("unexpected median value")
+	}
+}
+
+func TestTestKeysAnalyzeMinPlayoutDelay(t *testing.T) {
+	tk := &TestKeys{
+		ReceiverData: []clientResults{
+			{
+				ElapsedTarget: 2,
+				Elapsed:       1.4,
+			},
+			{
+				ElapsedTarget: 2,
+				Elapsed:       3.0,
+			},
+			{
+				ElapsedTarget: 2,
+				Elapsed:       1.8,
+			},
+		},
+	}
+	err := tk.analyze()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Simple.MinPlayoutDelay < 0.99 || tk.Simple.MinPlayoutDelay > 1.01 {
+		t.Fatal("unexpected min-playout-delay value")
+	}
+}
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	if measurer.ExperimentName() != "dash" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.12.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestMeasureWithCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // cause failure
+	measurement := new(model.Measurement)
+	m := &Measurer{}
+	err := m.Run(
+		ctx,
+		&mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("unexpected error value")
+	}
+	sk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := sk.(SummaryKeys); !ok {
+		t.Fatal("invalid type for summary keys")
+	}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysGood(t *testing.T) {
+	measurement := &model.Measurement{TestKeys: &TestKeys{Simple: Simple{
+		ConnectLatency:  1234,
+		MedianBitrate:   123,
+		MinPlayoutDelay: 12,
+	}}}
+	m := &Measurer{}
+	osk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sk := osk.(SummaryKeys)
+	if sk.Latency != 1234 {
+		t.Fatal("invalid latency")
+	}
+	if sk.Bitrate != 123 {
+		t.Fatal("invalid bitrate")
+	}
+	if sk.Delay != 12 {
+		t.Fatal("invalid delay")
+	}
+	if sk.IsAnomaly {
+		t.Fatal("invalid isAnomaly")
+	}
+}

internal/engine/experiment/dash/download.go

@@ -0,0 +1,74 @@
+package dash
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+type downloadDeps interface {
+	HTTPClient() *http.Client
+	NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
+	ReadAll(r io.Reader) ([]byte, error)
+	Scheme() string
+	UserAgent() string
+}
+
+type downloadConfig struct {
+	authorization string
+	begin         time.Time
+	currentRate   int64
+	deps          downloadDeps
+	elapsedTarget int64
+	fqdn          string
+}
+
+type downloadResult struct {
+	elapsed      float64
+	received     int64
+	requestTicks float64
+	serverURL    string
+	timestamp    int64
+}
+
+func download(ctx context.Context, config downloadConfig) (downloadResult, error) {
+	nbytes := (config.currentRate * 1000 * config.elapsedTarget) >> 3
+	var URL url.URL
+	URL.Scheme = config.deps.Scheme()
+	URL.Host = config.fqdn
+	URL.Path = fmt.Sprintf("%s%d", downloadPath, nbytes)
+	req, err := config.deps.NewHTTPRequest("GET", URL.String(), nil)
+	var result downloadResult
+	if err != nil {
+		return result, err
+	}
+	result.serverURL = URL.String()
+	req.Header.Set("User-Agent", config.deps.UserAgent())
+	req.Header.Set("Authorization", config.authorization)
+	savedTicks := time.Now()
+	resp, err := config.deps.HTTPClient().Do(req.WithContext(ctx))
+	if err != nil {
+		return result, err
+	}
+	if resp.StatusCode != 200 {
+		return result, errHTTPRequestFailed
+	}
+	defer resp.Body.Close()
+	data, err := config.deps.ReadAll(resp.Body)
+	if err != nil {
+		return result, err
+	}
+	// Implementation note: MK contains a comment that says that Neubot uses
+	// the elapsed time since when we start receiving the response but it
+	// turns out that Neubot and MK do the same. So, we do what they do. At
+	// the same time, we are currently not able to include the overhead that
+	// is caused by HTTP headers etc. So, we're a bit less precise.
+	result.elapsed = time.Now().Sub(savedTicks).Seconds()
+	result.received = int64(len(data))
+	result.requestTicks = savedTicks.Sub(config.begin).Seconds()
+	result.timestamp = time.Now().Unix()
+	return result, nil
+}

internal/engine/experiment/dash/download_test.go

@@ -0,0 +1,104 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+func TestDownloadNewHTTPRequestFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	_, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{newHTTPRequestErr: expected},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadHTTPClientDoFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{err: expected}
+	_, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		}},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadInternalError(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{StatusCode: 500}}
+	_, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		}},
+	})
+	if !errors.Is(err, errHTTPRequestFailed) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadReadAllFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	_, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{
+			httpTransport: txp,
+			newHTTPRequestResult: &http.Request{
+				Header: http.Header{},
+				URL:    &url.URL{},
+			},
+			readAllErr: expected,
+		},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDownloadSuccess(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	result, err := download(context.Background(), downloadConfig{
+		deps: FakeDeps{
+			httpTransport: txp,
+			newHTTPRequestResult: &http.Request{
+				Header: http.Header{},
+				URL:    &url.URL{},
+			},
+			readAllResult: []byte("[]"),
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.elapsed <= 0 {
+		t.Fatal("invalid elapsed")
+	}
+	if result.received <= 0 {
+		t.Fatal("invalid received")
+	}
+	if result.requestTicks <= 0 {
+		t.Fatal("invalid requestTicks")
+	}
+	if result.serverURL == "" {
+		t.Fatal("invalid serverURL")
+	}
+	if result.timestamp <= 0 {
+		t.Fatal("invalid timestamp")
+	}
+}

internal/engine/experiment/dash/fake_test.go

@@ -0,0 +1,69 @@
+package dash
+
+import (
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type FakeDeps struct {
+	httpTransport        http.RoundTripper
+	jsonMarshalErr       error
+	jsonMarshalResult    []byte
+	newHTTPRequestErr    error
+	newHTTPRequestResult *http.Request
+	readAllErr           error
+	readAllResult        []byte
+}
+
+func (d FakeDeps) HTTPClient() *http.Client {
+	return &http.Client{Transport: d.httpTransport}
+}
+
+func (d FakeDeps) JSONMarshal(v interface{}) ([]byte, error) {
+	return d.jsonMarshalResult, d.jsonMarshalErr
+}
+
+func (d FakeDeps) Logger() model.Logger {
+	return log.Log
+}
+
+func (d FakeDeps) NewHTTPRequest(
+	method string, url string, body io.Reader) (*http.Request, error) {
+	return d.newHTTPRequestResult, d.newHTTPRequestErr
+}
+
+func (d FakeDeps) ReadAll(r io.Reader) ([]byte, error) {
+	return d.readAllResult, d.readAllErr
+}
+
+func (d FakeDeps) Scheme() string {
+	return "https"
+}
+
+func (d FakeDeps) UserAgent() string {
+	return "miniooni/0.1.0-dev"
+}
+
+type FakeHTTPTransport struct {
+	err  error
+	resp *http.Response
+}
+
+func (txp FakeHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	time.Sleep(10 * time.Microsecond)
+	return txp.resp, txp.err
+}
+
+type FakeHTTPTransportStack struct {
+	all []FakeHTTPTransport
+}
+
+func (txp *FakeHTTPTransportStack) RoundTrip(req *http.Request) (*http.Response, error) {
+	frame := txp.all[0]
+	txp.all = txp.all[1:]
+	return frame.RoundTrip(req)
+}

internal/engine/experiment/dash/locate.go

@@ -0,0 +1,20 @@
+package dash
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mlablocate"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type locateDeps interface {
+	HTTPClient() *http.Client
+	Logger() model.Logger
+	UserAgent() string
+}
+
+func locate(ctx context.Context, deps locateDeps) (mlablocate.Result, error) {
+	return mlablocate.NewClient(
+		deps.HTTPClient(), deps.Logger(), deps.UserAgent()).Query(ctx, "neubot")
+}

internal/engine/experiment/dash/model.go

@@ -0,0 +1,47 @@
+package dash
+
+// clientResults contains the results measured by the client. This data
+// structure is sent to the server in the collection phase.
+//
+// All the fields listed here are part of the original specification
+// of DASH, except ServerURL, added in MK v0.10.6.
+type clientResults struct {
+	ConnectTime     float64 `json:"connect_time"`
+	DeltaSysTime    float64 `json:"delta_sys_time"`
+	DeltaUserTime   float64 `json:"delta_user_time"`
+	Elapsed         float64 `json:"elapsed"`
+	ElapsedTarget   int64   `json:"elapsed_target"`
+	InternalAddress string  `json:"internal_address"`
+	Iteration       int64   `json:"iteration"`
+	Platform        string  `json:"platform"`
+	Rate            int64   `json:"rate"`
+	RealAddress     string  `json:"real_address"`
+	Received        int64   `json:"received"`
+	RemoteAddress   string  `json:"remote_address"`
+	RequestTicks    float64 `json:"request_ticks"`
+	ServerURL       string  `json:"server_url"`
+	Timestamp       int64   `json:"timestamp"`
+	UUID            string  `json:"uuid"`
+	Version         string  `json:"version"`
+}
+
+// serverResults contains the server results. This data structure is sent
+// to the client during the collection phase of DASH.
+type serverResults struct {
+	Iteration int64   `json:"iteration"`
+	Ticks     float64 `json:"ticks"`
+	Timestamp int64   `json:"timestamp"`
+}
+
+// negotiateRequest contains the request of negotiation
+type negotiateRequest struct {
+	DASHRates []int64 `json:"dash_rates"`
+}
+
+// negotiateResponse contains the response of negotiation
+type negotiateResponse struct {
+	Authorization string `json:"authorization"`
+	QueuePos      int64  `json:"queue_pos"`
+	RealAddress   string `json:"real_address"`
+	Unchoked      int    `json:"unchoked"`
+}

internal/engine/experiment/dash/negotiate.go

@@ -0,0 +1,68 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type negotiateDeps interface {
+	HTTPClient() *http.Client
+	JSONMarshal(v interface{}) ([]byte, error)
+	Logger() model.Logger
+	NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
+	ReadAll(r io.Reader) ([]byte, error)
+	Scheme() string
+	UserAgent() string
+}
+
+func negotiate(
+	ctx context.Context, fqdn string, deps negotiateDeps) (negotiateResponse, error) {
+	var negotiateResp negotiateResponse
+	data, err := deps.JSONMarshal(negotiateRequest{DASHRates: defaultRates})
+	if err != nil {
+		return negotiateResp, err
+	}
+	deps.Logger().Debugf("dash: body: %s", string(data))
+	var URL url.URL
+	URL.Scheme = deps.Scheme()
+	URL.Host = fqdn
+	URL.Path = negotiatePath
+	req, err := deps.NewHTTPRequest("POST", URL.String(), bytes.NewReader(data))
+	if err != nil {
+		return negotiateResp, err
+	}
+	req.Header.Set("User-Agent", deps.UserAgent())
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "")
+	resp, err := deps.HTTPClient().Do(req.WithContext(ctx))
+	if err != nil {
+		return negotiateResp, err
+	}
+	if resp.StatusCode != 200 {
+		return negotiateResp, errHTTPRequestFailed
+	}
+	defer resp.Body.Close()
+	data, err = deps.ReadAll(resp.Body)
+	if err != nil {
+		return negotiateResp, err
+	}
+	deps.Logger().Debugf("dash: body: %s", string(data))
+	err = json.Unmarshal(data, &negotiateResp)
+	if err != nil {
+		return negotiateResp, err
+	}
+	// Implementation oddity: Neubot is using an integer rather than a
+	// boolean for the unchoked, with obvious semantics. I wonder why
+	// I choose an integer over a boolean, given that Python does have
+	// support for booleans. I don't remember 🤷.
+	if negotiateResp.Authorization == "" || negotiateResp.Unchoked == 0 {
+		return negotiateResp, errServerBusy
+	}
+	return negotiateResp, nil
+}

internal/engine/experiment/dash/negotiate_test.go

@@ -0,0 +1,178 @@
+package dash
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestNegotiateJSONMarshalError(t *testing.T) {
+	expected := errors.New("mocked error")
+	deps := FakeDeps{jsonMarshalErr: expected}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateNewHTTPRequestFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	deps := FakeDeps{newHTTPRequestErr: expected}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateHTTPClientDoFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{err: expected}
+	deps := FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+		Header: http.Header{},
+		URL:    &url.URL{},
+	}}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateInternalError(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{StatusCode: 500}}
+	deps := FakeDeps{httpTransport: txp, newHTTPRequestResult: &http.Request{
+		Header: http.Header{},
+		URL:    &url.URL{},
+	}}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, errHTTPRequestFailed) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateReadAllFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllErr: expected,
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateInvalidJSON(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte("["),
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if err == nil || !strings.HasSuffix(err.Error(), "unexpected end of JSON input") {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateServerBusyFirstCase(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte(`{"authorization": ""}`),
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, errServerBusy) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateServerBusyThirdCase(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte(`{}`),
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if !errors.Is(err, errServerBusy) {
+		t.Fatal("not the error we expected")
+	}
+	if result.Authorization != "" || result.Unchoked != 0 {
+		t.Fatal("unexpected result")
+	}
+}
+
+func TestNegotiateSuccess(t *testing.T) {
+	txp := FakeHTTPTransport{resp: &http.Response{
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		StatusCode: 200,
+	}}
+	deps := FakeDeps{
+		httpTransport: txp,
+		newHTTPRequestResult: &http.Request{
+			Header: http.Header{},
+			URL:    &url.URL{},
+		},
+		readAllResult: []byte(`{"authorization": "xx", "unchoked": 1}`),
+	}
+	result, err := negotiate(context.Background(), "", deps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Authorization != "xx" || result.Unchoked != 1 {
+		t.Fatal("invalid result")
+	}
+}

internal/engine/experiment/dash/spec.go

@@ -0,0 +1,26 @@
+package dash
+
+const (
+	// currentServerSchemaVersion is the version of the server schema that
+	// will be adopted by this implementation. Version 3 is the one that is
+	// Neubot uses. We needed to bump the version because Web100 is not on
+	// M-Lab anymore and hence we need to make a breaking change.
+	currentServerSchemaVersion = 4
+
+	// negotiatePath is the URL path used to negotiate
+	negotiatePath = "/negotiate/dash"
+
+	// downloadPath is the URL path used to request DASH segments. You can
+	// append to this path an integer indicating how many bytes you would like
+	// the server to send you as part of the next chunk.
+	downloadPath = "/dash/download/"
+
+	// collectPath is the URL path used to collect
+	collectPath = "/collect/dash"
+)
+
+// defaultRates contains the default DASH rates in kbit/s.
+var defaultRates = []int64{
+	100, 150, 200, 250, 300, 400, 500, 700, 900, 1200, 1500, 2000,
+	2500, 3000, 4000, 5000, 6000, 7000, 10000, 20000,
+}