chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/probeservices/README.md

@@ -0,0 +1,8 @@
+# Package github.com/ooni/probe-engine/probeservices
+
+This package contains code to contact OONI probe services.
+
+The probe services are HTTPS endpoints distributed across a bunch of data
+centres implementing a bunch of OONI APIs. When started, OONI will benchmark
+the available probe services and select the fastest one. Eventually all the
+possible OONI APIs will run as probe services.

internal/engine/probeservices/benchselect.go

@@ -0,0 +1,148 @@
+package probeservices
+
+import (
+	"context"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// Default returns the default probe services
+func Default() []model.Service {
+	return []model.Service{{
+		Address: "https://ps1.ooni.io",
+		Type:    "https",
+	}, {
+		Address: "https://ps2.ooni.io",
+		Type:    "https",
+	}, {
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}}
+}
+
+// SortEndpoints gives priority to https, then cloudfronted, then onion.
+func SortEndpoints(in []model.Service) (out []model.Service) {
+	for _, entry := range in {
+		if entry.Type == "https" {
+			out = append(out, entry)
+		}
+	}
+	for _, entry := range in {
+		if entry.Type == "cloudfront" {
+			out = append(out, entry)
+		}
+	}
+	for _, entry := range in {
+		if entry.Type == "onion" {
+			out = append(out, entry)
+		}
+	}
+	return
+}
+
+// OnlyHTTPS returns the HTTPS endpoints only.
+func OnlyHTTPS(in []model.Service) (out []model.Service) {
+	for _, entry := range in {
+		if entry.Type == "https" {
+			out = append(out, entry)
+		}
+	}
+	return
+}
+
+// OnlyFallbacks returns the fallback endpoints only.
+func OnlyFallbacks(in []model.Service) (out []model.Service) {
+	for _, entry := range SortEndpoints(in) {
+		if entry.Type != "https" {
+			out = append(out, entry)
+		}
+	}
+	return
+}
+
+// Candidate is a candidate probe service.
+type Candidate struct {
+	// Duration is the time it took to access the service.
+	Duration time.Duration
+
+	// Err indicates whether the service works.
+	Err error
+
+	// Endpoint is the service endpoint.
+	Endpoint model.Service
+
+	// TestHelpers contains the data returned by the endpoint.
+	TestHelpers map[string][]model.Service
+}
+
+func (c *Candidate) try(ctx context.Context, sess Session) {
+	client, err := NewClient(sess, c.Endpoint)
+	if err != nil {
+		c.Err = err
+		return
+	}
+	start := time.Now()
+	testhelpers, err := client.GetTestHelpers(ctx)
+	c.Duration = time.Now().Sub(start)
+	c.Err = err
+	c.TestHelpers = testhelpers
+	sess.Logger().Debugf("probe services: %+v: %+v %s", c.Endpoint, err, c.Duration)
+}
+
+func try(ctx context.Context, sess Session, svc model.Service) *Candidate {
+	candidate := &Candidate{Endpoint: svc}
+	candidate.try(ctx, sess)
+	return candidate
+}
+
+// TryAll tries all the input services using the provided context and session. It
+// returns a list containing information on each candidate that was tried. We will
+// try all the HTTPS candidates first. So, the beginning of the list will contain
+// all of them, and for each of them you will know whether it worked (by checking the
+// Err field) and how fast it was (by checking the Duration field). You should pick
+// the fastest one that worked. If none of them works, then TryAll will subsequently
+// attempt with all the available fallbacks, and return at the first success. In
+// such case, you will see a list of N failing HTTPS candidates, followed by a single
+// successful fallback candidate (e.g. cloudfronted). If all candidates fail, you
+// see in output a list containing all entries where Err is not nil.
+func TryAll(ctx context.Context, sess Session, in []model.Service) (out []*Candidate) {
+	var found bool
+	for _, svc := range OnlyHTTPS(in) {
+		candidate := try(ctx, sess, svc)
+		out = append(out, candidate)
+		if candidate.Err == nil {
+			found = true
+		}
+	}
+	if !found {
+		for _, svc := range OnlyFallbacks(in) {
+			candidate := try(ctx, sess, svc)
+			out = append(out, candidate)
+			if candidate.Err == nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// SelectBest selects the best among the candidates. If there is no
+// suitable candidate, then this function returns nil.
+func SelectBest(candidates []*Candidate) (selected *Candidate) {
+	for _, e := range candidates {
+		if e.Err != nil {
+			continue
+		}
+		if selected == nil {
+			selected = e
+			continue
+		}
+		if selected.Duration > e.Duration {
+			selected = e
+			continue
+		}
+	}
+	return
+}

internal/engine/probeservices/bouncer.go

@@ -0,0 +1,14 @@
+package probeservices
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// GetTestHelpers is like GetCollectors but for test helpers.
+func (c Client) GetTestHelpers(
+	ctx context.Context) (output map[string][]model.Service, err error) {
+	err = c.Client.GetJSON(ctx, "/api/v1/test-helpers", &output)
+	return
+}

internal/engine/probeservices/bouncer_test.go

@@ -0,0 +1,16 @@
+package probeservices_test
+
+import (
+	"context"
+	"testing"
+)
+
+func TestGetTestHelpers(t *testing.T) {
+	testhelpers, err := newclient().GetTestHelpers(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(testhelpers) <= 1 {
+		t.Fatal("no returned test helpers?!")
+	}
+}

internal/engine/probeservices/checkin.go

@@ -0,0 +1,23 @@
+package probeservices
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type checkInResult struct {
+	Tests model.CheckInInfo `json:"tests"`
+	V     int               `json:"v"`
+}
+
+// CheckIn function is called by probes asking if there are tests to be run
+// The config argument contains the mandatory settings.
+// Returns the list of tests to run and the URLs, on success, or an explanatory error, in case of failure.
+func (c Client) CheckIn(ctx context.Context, config model.CheckInConfig) (*model.CheckInInfo, error) {
+	var response checkInResult
+	if err := c.Client.PostJSON(ctx, "/api/v1/check-in", config, &response); err != nil {
+		return nil, err
+	}
+	return &response.Tests, nil
+}

internal/engine/probeservices/checkin_test.go

@@ -0,0 +1,72 @@
+package probeservices_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestCheckInSuccess(t *testing.T) {
+	client := newclient()
+	client.BaseURL = "https://ams-pg-test.ooni.org"
+	config := model.CheckInConfig{
+		Charging:        true,
+		OnWiFi:          true,
+		Platform:        "android",
+		ProbeASN:        "AS12353",
+		ProbeCC:         "PT",
+		RunType:         "timed",
+		SoftwareName:    "ooniprobe-android",
+		SoftwareVersion: "2.7.1",
+		WebConnectivity: model.CheckInConfigWebConnectivity{
+			CategoryCodes: []string{"NEWS", "CULTR"},
+		},
+	}
+	ctx := context.Background()
+	result, err := client.CheckIn(ctx, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result == nil || result.WebConnectivity == nil {
+		t.Fatal("got nil result or WebConnectivity")
+	}
+	if result.WebConnectivity.ReportID == "" {
+		t.Fatal("ReportID is empty")
+	}
+	if len(result.WebConnectivity.URLs) < 1 {
+		t.Fatal("unexpected number of URLs")
+	}
+	for _, entry := range result.WebConnectivity.URLs {
+		if entry.CategoryCode != "NEWS" && entry.CategoryCode != "CULTR" {
+			t.Fatalf("unexpected category code: %+v", entry)
+		}
+	}
+}
+
+func TestCheckInFailure(t *testing.T) {
+	client := newclient()
+	client.BaseURL = "https://\t\t\t/" // cause test to fail
+	config := model.CheckInConfig{
+		Charging:        true,
+		OnWiFi:          true,
+		Platform:        "android",
+		ProbeASN:        "AS12353",
+		ProbeCC:         "PT",
+		RunType:         "timed",
+		SoftwareName:    "ooniprobe-android",
+		SoftwareVersion: "2.7.1",
+		WebConnectivity: model.CheckInConfigWebConnectivity{
+			CategoryCodes: []string{"NEWS", "CULTR"},
+		},
+	}
+	ctx := context.Background()
+	result, err := client.CheckIn(ctx, config)
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+	if result != nil {
+		t.Fatal("results?!")
+	}
+}

internal/engine/probeservices/checkreportid.go

@@ -0,0 +1,29 @@
+package probeservices
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+)
+
+type checkReportIDResponse struct {
+	Found bool `json:"found"`
+}
+
+// CheckReportID checks whether the given ReportID exists.
+func (c Client) CheckReportID(ctx context.Context, reportID string) (bool, error) {
+	query := url.Values{}
+	query.Add("report_id", reportID)
+	var response checkReportIDResponse
+	err := (httpx.Client{
+		BaseURL:    c.BaseURL,
+		HTTPClient: c.HTTPClient,
+		Logger:     c.Logger,
+		UserAgent:  c.UserAgent,
+	}).GetJSONWithQuery(ctx, "/api/_/check_report_id", query, &response)
+	if err != nil {
+		return false, err
+	}
+	return response.Found, nil
+}

internal/engine/probeservices/checkreportid_test.go

@@ -0,0 +1,61 @@
+package probeservices_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+)
+
+func TestCheckReportIDWorkingAsIntended(t *testing.T) {
+	client := probeservices.Client{
+		Client: httpx.Client{
+			BaseURL:    "https://ams-pg.ooni.org/",
+			HTTPClient: http.DefaultClient,
+			Logger:     log.Log,
+			UserAgent:  "miniooni/0.1.0-dev",
+		},
+		LoginCalls:    atomicx.NewInt64(),
+		RegisterCalls: atomicx.NewInt64(),
+		StateFile:     probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
+	}
+	reportID := `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`
+	ctx := context.Background()
+	found, err := client.CheckReportID(ctx, reportID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if found != true {
+		t.Fatal("unexpected found value")
+	}
+}
+
+func TestCheckReportIDWorkingWithCancelledContext(t *testing.T) {
+	client := probeservices.Client{
+		Client: httpx.Client{
+			BaseURL:    "https://ams-pg.ooni.org/",
+			HTTPClient: http.DefaultClient,
+			Logger:     log.Log,
+			UserAgent:  "miniooni/0.1.0-dev",
+		},
+		LoginCalls:    atomicx.NewInt64(),
+		RegisterCalls: atomicx.NewInt64(),
+		StateFile:     probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
+	}
+	reportID := `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	found, err := client.CheckReportID(ctx, reportID)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if found != false {
+		t.Fatal("unexpected found value")
+	}
+}

internal/engine/probeservices/collector.go

@@ -0,0 +1,214 @@
+package probeservices
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+const (
+	// DefaultDataFormatVersion is the default data format version.
+	//
+	// See https://github.com/ooni/spec/tree/master/data-formats#history.
+	DefaultDataFormatVersion = "0.2.0"
+
+	// DefaultFormat is the default format
+	DefaultFormat = "json"
+)
+
+var (
+	// ErrUnsupportedDataFormatVersion indicates that the user provided
+	// in input a data format version that we do not support.
+	ErrUnsupportedDataFormatVersion = errors.New("Unsupported data format version")
+
+	// ErrUnsupportedFormat indicates that the format is not supported.
+	ErrUnsupportedFormat = errors.New("Unsupported format")
+
+	// ErrJSONFormatNotSupported indicates that the collector we're using
+	// does not support the JSON report format.
+	ErrJSONFormatNotSupported = errors.New("JSON format not supported")
+)
+
+// ReportTemplate is the template for opening a report
+type ReportTemplate struct {
+	// DataFormatVersion is unconditionally set to DefaultDataFormatVersion
+	// and you don't need to be concerned about it.
+	DataFormatVersion string `json:"data_format_version"`
+
+	// Format is unconditionally set to `json` and you don't need
+	// to be concerned about it.
+	Format string `json:"format"`
+
+	// ProbeASN is the probe's autonomous system number (e.g. `AS1234`)
+	ProbeASN string `json:"probe_asn"`
+
+	// ProbeCC is the probe's country code (e.g. `IT`)
+	ProbeCC string `json:"probe_cc"`
+
+	// SoftwareName is the app name (e.g. `measurement-kit`)
+	SoftwareName string `json:"software_name"`
+
+	// SoftwareVersion is the app version (e.g. `0.9.1`)
+	SoftwareVersion string `json:"software_version"`
+
+	// TestName is the test name (e.g. `ndt`)
+	TestName string `json:"test_name"`
+
+	// TestStartTime contains the test start time
+	TestStartTime string `json:"test_start_time"`
+
+	// TestVersion is the test version (e.g. `1.0.1`)
+	TestVersion string `json:"test_version"`
+}
+
+// NewReportTemplate creates a new ReportTemplate from a Measurement.
+func NewReportTemplate(m *model.Measurement) ReportTemplate {
+	return ReportTemplate{
+		DataFormatVersion: DefaultDataFormatVersion,
+		Format:            DefaultFormat,
+		ProbeASN:          m.ProbeASN,
+		ProbeCC:           m.ProbeCC,
+		SoftwareName:      m.SoftwareName,
+		SoftwareVersion:   m.SoftwareVersion,
+		TestName:          m.TestName,
+		TestStartTime:     m.TestStartTime,
+		TestVersion:       m.TestVersion,
+	}
+}
+
+type collectorOpenResponse struct {
+	ID               string   `json:"report_id"`
+	SupportedFormats []string `json:"supported_formats"`
+}
+
+type reportChan struct {
+	// ID is the report ID
+	ID string
+
+	// client is the client that was used.
+	client Client
+
+	// tmpl is the template used when opening this report.
+	tmpl ReportTemplate
+}
+
+// OpenReport opens a new report.
+func (c Client) OpenReport(ctx context.Context, rt ReportTemplate) (ReportChannel, error) {
+	if rt.DataFormatVersion != DefaultDataFormatVersion {
+		return nil, ErrUnsupportedDataFormatVersion
+	}
+	if rt.Format != DefaultFormat {
+		return nil, ErrUnsupportedFormat
+	}
+	var cor collectorOpenResponse
+	if err := c.Client.PostJSON(ctx, "/report", rt, &cor); err != nil {
+		return nil, err
+	}
+	for _, format := range cor.SupportedFormats {
+		if format == "json" {
+			return &reportChan{ID: cor.ID, client: c, tmpl: rt}, nil
+		}
+	}
+	return nil, ErrJSONFormatNotSupported
+}
+
+type collectorUpdateRequest struct {
+	// Format is the data format
+	Format string `json:"format"`
+
+	// Content is the actual report
+	Content interface{} `json:"content"`
+}
+
+type collectorUpdateResponse struct {
+	// ID is the measurement ID
+	ID string `json:"measurement_id"`
+}
+
+// CanSubmit returns true whether the provided measurement belongs to
+// this report, false otherwise. We say that a given measurement belongs
+// to this report if its report template matches the report's one.
+func (r reportChan) CanSubmit(m *model.Measurement) bool {
+	return reflect.DeepEqual(NewReportTemplate(m), r.tmpl)
+}
+
+// SubmitMeasurement submits a measurement belonging to the report
+// to the OONI collector. On success, we will modify the measurement
+// such that it contains the report ID for which it has been
+// submitted. Otherwise, we'll set the report ID to the empty
+// string, so that you know which measurements weren't submitted.
+func (r reportChan) SubmitMeasurement(ctx context.Context, m *model.Measurement) error {
+	var updateResponse collectorUpdateResponse
+	m.ReportID = r.ID
+	err := r.client.Client.PostJSON(
+		ctx, fmt.Sprintf("/report/%s", r.ID), collectorUpdateRequest{
+			Format:  "json",
+			Content: m,
+		}, &updateResponse,
+	)
+	if err != nil {
+		m.ReportID = ""
+		return err
+	}
+	return nil
+}
+
+// ReportID returns the report ID.
+func (r reportChan) ReportID() string {
+	return r.ID
+}
+
+// ReportChannel is a channel through which one could submit measurements
+// belonging to the same report. The Report struct belongs to this interface.
+type ReportChannel interface {
+	CanSubmit(m *model.Measurement) bool
+	ReportID() string
+	SubmitMeasurement(ctx context.Context, m *model.Measurement) error
+}
+
+var _ ReportChannel = &reportChan{}
+
+// ReportOpener is any struct that is able to open a new ReportChannel. The
+// Client struct belongs to this interface.
+type ReportOpener interface {
+	OpenReport(ctx context.Context, rt ReportTemplate) (ReportChannel, error)
+}
+
+var _ ReportOpener = Client{}
+
+// Submitter is an abstraction allowing you to submit arbitrary measurements
+// to a given OONI backend. This implementation will take care of opening
+// reports when needed as well as of closing reports when needed. Nonetheless
+// you need to remember to call its Close method when done, because there is
+// likely an open report that has not been closed yet.
+type Submitter struct {
+	channel ReportChannel
+	logger  model.Logger
+	mu      sync.Mutex
+	opener  ReportOpener
+}
+
+// NewSubmitter creates a new Submitter instance.
+func NewSubmitter(opener ReportOpener, logger model.Logger) *Submitter {
+	return &Submitter{opener: opener, logger: logger}
+}
+
+// Submit submits the current measurement to the OONI backend created using
+// the ReportOpener passed to the constructor.
+func (sub *Submitter) Submit(ctx context.Context, m *model.Measurement) error {
+	var err error
+	sub.mu.Lock()
+	defer sub.mu.Unlock()
+	if sub.channel == nil || !sub.channel.CanSubmit(m) {
+		sub.channel, err = sub.opener.OpenReport(ctx, NewReportTemplate(m))
+		if err != nil {
+			return err
+		}
+		sub.logger.Infof("New reportID: %s", sub.channel.ReportID())
+	}
+	return sub.channel.SubmitMeasurement(ctx, m)
+}

internal/engine/probeservices/collector_test.go

@@ -0,0 +1,459 @@
+package probeservices_test
+
+import (
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+)
+
+type fakeTestKeys struct {
+	Failure *string `json:"failure"`
+}
+
+func makeMeasurement(rt probeservices.ReportTemplate, ID string) model.Measurement {
+	return model.Measurement{
+		DataFormatVersion:    probeservices.DefaultDataFormatVersion,
+		ID:                   "bdd20d7a-bba5-40dd-a111-9863d7908572",
+		MeasurementRuntime:   5.0565230846405,
+		MeasurementStartTime: "2018-11-01 15:33:20",
+		ProbeIP:              "1.2.3.4",
+		ProbeASN:             rt.ProbeASN,
+		ProbeCC:              rt.ProbeCC,
+		ReportID:             ID,
+		ResolverASN:          "AS15169",
+		ResolverIP:           "8.8.8.8",
+		ResolverNetworkName:  "Google LLC",
+		SoftwareName:         rt.SoftwareName,
+		SoftwareVersion:      rt.SoftwareVersion,
+		TestKeys:             fakeTestKeys{Failure: nil},
+		TestName:             rt.TestName,
+		TestStartTime:        rt.TestStartTime,
+		TestVersion:          rt.TestVersion,
+	}
+}
+
+func TestNewReportTemplate(t *testing.T) {
+	m := &model.Measurement{
+		ProbeASN:        "AS117",
+		ProbeCC:         "IT",
+		SoftwareName:    "ooniprobe-engine",
+		SoftwareVersion: "0.1.0",
+		TestName:        "dummy",
+		TestStartTime:   "2019-10-28 12:51:06",
+		TestVersion:     "0.1.0",
+	}
+	rt := probeservices.NewReportTemplate(m)
+	expect := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS117",
+		ProbeCC:           "IT",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	if diff := cmp.Diff(expect, rt); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestReportLifecycle(t *testing.T) {
+	ctx := context.Background()
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	report, err := client.OpenReport(ctx, template)
+	if err != nil {
+		t.Fatal(err)
+	}
+	measurement := makeMeasurement(template, report.ReportID())
+	if report.CanSubmit(&measurement) != true {
+		t.Fatal("report should be able to submit this measurement")
+	}
+	if err = report.SubmitMeasurement(ctx, &measurement); err != nil {
+		t.Fatal(err)
+	}
+	if measurement.ReportID != report.ReportID() {
+		t.Fatal("report ID mismatch")
+	}
+}
+
+func TestReportLifecycleWrongExperiment(t *testing.T) {
+	ctx := context.Background()
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	report, err := client.OpenReport(ctx, template)
+	if err != nil {
+		t.Fatal(err)
+	}
+	measurement := makeMeasurement(template, report.ReportID())
+	measurement.TestName = "antani"
+	if report.CanSubmit(&measurement) != false {
+		t.Fatal("report should not be able to submit this measurement")
+	}
+}
+
+func TestOpenReportInvalidDataFormatVersion(t *testing.T) {
+	ctx := context.Background()
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: "0.1.0",
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	report, err := client.OpenReport(ctx, template)
+	if !errors.Is(err, probeservices.ErrUnsupportedDataFormatVersion) {
+		t.Fatal("not the error we expected")
+	}
+	if report != nil {
+		t.Fatal("expected a nil report here")
+	}
+}
+
+func TestOpenReportInvalidFormat(t *testing.T) {
+	ctx := context.Background()
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            "yaml",
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	report, err := client.OpenReport(ctx, template)
+	if !errors.Is(err, probeservices.ErrUnsupportedFormat) {
+		t.Fatal("not the error we expected")
+	}
+	if report != nil {
+		t.Fatal("expected a nil report here")
+	}
+}
+
+func TestJSONAPIClientCreateFailure(t *testing.T) {
+	ctx := context.Background()
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	client.BaseURL = "\t" // breaks the URL parser
+	report, err := client.OpenReport(ctx, template)
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+	if report != nil {
+		t.Fatal("expected a nil report here")
+	}
+}
+
+func TestOpenResponseNoJSONSupport(t *testing.T) {
+	server := httptest.NewServer(
+		http.HandlerFunc(func(writer http.ResponseWriter, req *http.Request) {
+			writer.Write([]byte(`{"ID":"abc","supported_formats":["yaml"]}`))
+		}),
+	)
+	defer server.Close()
+	ctx := context.Background()
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	client.BaseURL = server.URL
+	report, err := client.OpenReport(ctx, template)
+	if !errors.Is(err, probeservices.ErrJSONFormatNotSupported) {
+		t.Fatal("expected an error here")
+	}
+	if report != nil {
+		t.Fatal("expected a nil report here")
+	}
+}
+
+func TestEndToEnd(t *testing.T) {
+	server := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.RequestURI == "/report" {
+				w.Write([]byte(`{"report_id":"_id","supported_formats":["json"]}`))
+				return
+			}
+			if r.RequestURI == "/report/_id" {
+				data, err := ioutil.ReadAll(r.Body)
+				if err != nil {
+					panic(err)
+				}
+				sdata, err := ioutil.ReadFile("../testdata/collector-expected.jsonl")
+				if err != nil {
+					panic(err)
+				}
+				if diff := cmp.Diff(data, sdata); diff != "" {
+					panic(diff)
+				}
+				w.Write([]byte(`{"measurement_id":"e00c584e6e9e5326"}`))
+				return
+			}
+			if r.RequestURI == "/report/_id/close" {
+				w.Write([]byte(`{}`))
+				return
+			}
+			panic(r.RequestURI)
+		}),
+	)
+	defer server.Close()
+	ctx := context.Background()
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2018-11-01 15:33:17",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	client.BaseURL = server.URL
+	report, err := client.OpenReport(ctx, template)
+	if err != nil {
+		t.Fatal(err)
+	}
+	measurement := makeMeasurement(template, report.ReportID())
+	if err = report.SubmitMeasurement(ctx, &measurement); err != nil {
+		t.Fatal(err)
+	}
+}
+
+type RecordingReportChannel struct {
+	tmpl probeservices.ReportTemplate
+	m    []*model.Measurement
+	mu   sync.Mutex
+}
+
+func (rrc *RecordingReportChannel) CanSubmit(m *model.Measurement) bool {
+	return reflect.DeepEqual(probeservices.NewReportTemplate(m), rrc.tmpl)
+}
+
+func (rrc *RecordingReportChannel) SubmitMeasurement(ctx context.Context, m *model.Measurement) error {
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+	rrc.mu.Lock()
+	defer rrc.mu.Unlock()
+	rrc.m = append(rrc.m, m)
+	return nil
+}
+
+func (rrc *RecordingReportChannel) Close(ctx context.Context) error {
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+	rrc.mu.Lock()
+	defer rrc.mu.Unlock()
+	return nil
+}
+
+func (rrc *RecordingReportChannel) ReportID() string {
+	return ""
+}
+
+type RecordingReportOpener struct {
+	channels []*RecordingReportChannel
+	mu       sync.Mutex
+}
+
+func (rro *RecordingReportOpener) OpenReport(
+	ctx context.Context, rt probeservices.ReportTemplate,
+) (probeservices.ReportChannel, error) {
+	if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
+	rrc := &RecordingReportChannel{tmpl: rt}
+	rro.mu.Lock()
+	defer rro.mu.Unlock()
+	rro.channels = append(rro.channels, rrc)
+	return rrc, nil
+}
+
+func TestOpenReportCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately abort
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	report, err := client.OpenReport(ctx, template)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if report != nil {
+		t.Fatal("expected nil report here")
+	}
+}
+
+func TestSubmitMeasurementCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	template := probeservices.ReportTemplate{
+		DataFormatVersion: probeservices.DefaultDataFormatVersion,
+		Format:            probeservices.DefaultFormat,
+		ProbeASN:          "AS0",
+		ProbeCC:           "ZZ",
+		SoftwareName:      "ooniprobe-engine",
+		SoftwareVersion:   "0.1.0",
+		TestName:          "dummy",
+		TestStartTime:     "2019-10-28 12:51:06",
+		TestVersion:       "0.1.0",
+	}
+	client := newclient()
+	report, err := client.OpenReport(ctx, template)
+	if err != nil {
+		t.Fatal(err)
+	}
+	measurement := makeMeasurement(template, report.ReportID())
+	if report.CanSubmit(&measurement) != true {
+		t.Fatal("report should be able to submit this measurement")
+	}
+	cancel() // cause submission to fail
+	err = report.SubmitMeasurement(ctx, &measurement)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if measurement.ReportID != "" {
+		t.Fatal("report ID should be empty here")
+	}
+}
+
+func makeMeasurementWithoutTemplate(failure, testName string) *model.Measurement {
+	return &model.Measurement{
+		DataFormatVersion:    probeservices.DefaultDataFormatVersion,
+		ID:                   "bdd20d7a-bba5-40dd-a111-9863d7908572",
+		MeasurementRuntime:   5.0565230846405,
+		MeasurementStartTime: "2018-11-01 15:33:20",
+		ProbeIP:              "1.2.3.4",
+		ProbeASN:             "AS123",
+		ProbeCC:              "IT",
+		ReportID:             "",
+		ResolverASN:          "AS15169",
+		ResolverIP:           "8.8.8.8",
+		ResolverNetworkName:  "Google LLC",
+		SoftwareName:         "miniooni",
+		SoftwareVersion:      "0.1.0-dev",
+		TestKeys:             fakeTestKeys{Failure: &failure},
+		TestName:             testName,
+		TestStartTime:        "2018-11-01 15:33:17",
+		TestVersion:          "0.1.0",
+	}
+}
+
+func TestSubmitterLifecyle(t *testing.T) {
+	rro := &RecordingReportOpener{}
+	submitter := probeservices.NewSubmitter(rro, log.Log)
+	ctx := context.Background()
+	m1 := makeMeasurementWithoutTemplate("antani", "example")
+	if err := submitter.Submit(ctx, m1); err != nil {
+		t.Fatal(err)
+	}
+	m2 := makeMeasurementWithoutTemplate("mascetti", "example")
+	if err := submitter.Submit(ctx, m2); err != nil {
+		t.Fatal(err)
+	}
+	m3 := makeMeasurementWithoutTemplate("antani", "example_extended")
+	if err := submitter.Submit(ctx, m3); err != nil {
+		t.Fatal(err)
+	}
+	if len(rro.channels) != 2 {
+		t.Fatal("unexpected number of channels")
+	}
+	if len(rro.channels[0].m) != 2 {
+		t.Fatal("unexpected number of measurements in first channel")
+	}
+	if len(rro.channels[1].m) != 1 {
+		t.Fatal("unexpected number of measurements in second channel")
+	}
+}
+
+func TestSubmitterCannotOpenNewChannel(t *testing.T) {
+	rro := &RecordingReportOpener{}
+	submitter := probeservices.NewSubmitter(rro, log.Log)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	m1 := makeMeasurementWithoutTemplate("antani", "example")
+	if err := submitter.Submit(ctx, m1); !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	m2 := makeMeasurementWithoutTemplate("mascetti", "example")
+	if err := submitter.Submit(ctx, m2); !errors.Is(err, context.Canceled) {
+		t.Fatal(err)
+	}
+	m3 := makeMeasurementWithoutTemplate("antani", "example_extended")
+	if err := submitter.Submit(ctx, m3); !errors.Is(err, context.Canceled) {
+		t.Fatal(err)
+	}
+	if len(rro.channels) != 0 {
+		t.Fatal("unexpected number of channels")
+	}
+}

internal/engine/probeservices/login.go

@@ -0,0 +1,38 @@
+package probeservices
+
+import (
+	"context"
+	"time"
+)
+
+// LoginCredentials contains the login credentials
+type LoginCredentials struct {
+	ClientID string `json:"username"`
+	Password string `json:"password"`
+}
+
+// LoginAuth contains authentication info
+type LoginAuth struct {
+	Expire time.Time `json:"expire"`
+	Token  string    `json:"token"`
+}
+
+// MaybeLogin performs login if necessary
+func (c Client) MaybeLogin(ctx context.Context) error {
+	state := c.StateFile.Get()
+	if state.Auth() != nil {
+		return nil // we're already good
+	}
+	creds := state.Credentials()
+	if creds == nil {
+		return ErrNotRegistered
+	}
+	c.LoginCalls.Add(1)
+	var auth LoginAuth
+	if err := c.Client.PostJSON(ctx, "/api/v1/login", *creds, &auth); err != nil {
+		return err
+	}
+	state.Expire = auth.Expire
+	state.Token = auth.Token
+	return c.StateFile.Set(state)
+}

internal/engine/probeservices/login_test.go

@@ -0,0 +1,73 @@
+package probeservices_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
+)
+
+func TestMaybeLogin(t *testing.T) {
+	t.Run("when we already have a token", func(t *testing.T) {
+		clnt := newclient()
+		state := probeservices.State{
+			Expire: time.Now().Add(time.Hour),
+			Token:  "xx-xxx-x-xxxx",
+		}
+		if err := clnt.StateFile.Set(state); err != nil {
+			t.Fatal(err)
+		}
+		ctx := context.Background()
+		if err := clnt.MaybeLogin(ctx); err != nil {
+			t.Fatal(err)
+		}
+	})
+	t.Run("when we have already registered", func(t *testing.T) {
+		clnt := newclient()
+		state := probeservices.State{
+			// Explicitly empty to clarify what this test does
+		}
+		if err := clnt.StateFile.Set(state); err != nil {
+			t.Fatal(err)
+		}
+		ctx := context.Background()
+		if err := clnt.MaybeLogin(ctx); err == nil {
+			t.Fatal("expected an error here")
+		}
+	})
+	t.Run("when the API call fails", func(t *testing.T) {
+		clnt := newclient()
+		clnt.BaseURL = "\t\t\t" // causes the code to fail
+		state := probeservices.State{
+			ClientID: "xx-xxx-x-xxxx",
+			Password: "xx",
+		}
+		if err := clnt.StateFile.Set(state); err != nil {
+			t.Fatal(err)
+		}
+		ctx := context.Background()
+		if err := clnt.MaybeLogin(ctx); err == nil {
+			t.Fatal("expected an error here")
+		}
+	})
+}
+
+func TestMaybeLoginIdempotent(t *testing.T) {
+	clnt := newclient()
+	ctx := context.Background()
+	metadata := testorchestra.MetadataFixture()
+	if err := clnt.MaybeRegister(ctx, metadata); err != nil {
+		t.Fatal(err)
+	}
+	if err := clnt.MaybeLogin(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if err := clnt.MaybeLogin(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if clnt.LoginCalls.Load() != 1 {
+		t.Fatal("called login API too many times")
+	}
+}

internal/engine/probeservices/measurementmeta.go

@@ -0,0 +1,67 @@
+package probeservices
+
+import (
+	"context"
+	"net/url"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+)
+
+// MeasurementMetaConfig contains configuration for GetMeasurementMeta.
+type MeasurementMetaConfig struct {
+	// ReportID is the mandatory report ID.
+	ReportID string
+
+	// Full indicates whether we also want the full measurement body.
+	Full bool
+
+	// Input is the optional input.
+	Input string
+}
+
+// MeasurementMeta contains measurement metadata.
+type MeasurementMeta struct {
+	// Fields returned by the API server whenever we are
+	// calling /api/v1/measurement_meta.
+	Anomaly              bool      `json:"anomaly"`
+	CategoryCode         string    `json:"category_code"`
+	Confirmed            bool      `json:"confirmed"`
+	Failure              bool      `json:"failure"`
+	Input                *string   `json:"input"`
+	MeasurementStartTime time.Time `json:"measurement_start_time"`
+	ProbeASN             int64     `json:"probe_asn"`
+	ProbeCC              string    `json:"probe_cc"`
+	ReportID             string    `json:"report_id"`
+	Scores               string    `json:"scores"`
+	TestName             string    `json:"test_name"`
+	TestStartTime        time.Time `json:"test_start_time"`
+
+	// This field is only included if the user has specified
+	// the config.Full option, otherwise it's empty.
+	RawMeasurement string `json:"raw_measurement"`
+}
+
+// GetMeasurementMeta returns meta information about a measurement.
+func (c Client) GetMeasurementMeta(
+	ctx context.Context, config MeasurementMetaConfig) (*MeasurementMeta, error) {
+	query := url.Values{}
+	query.Add("report_id", config.ReportID)
+	if config.Input != "" {
+		query.Add("input", config.Input)
+	}
+	if config.Full {
+		query.Add("full", "true")
+	}
+	var response MeasurementMeta
+	err := (httpx.Client{
+		BaseURL:    c.BaseURL,
+		HTTPClient: c.HTTPClient,
+		Logger:     c.Logger,
+		UserAgent:  c.UserAgent,
+	}).GetJSONWithQuery(ctx, "/api/v1/measurement_meta", query, &response)
+	if err != nil {
+		return nil, err
+	}
+	return &response, nil
+}

internal/engine/probeservices/measurementmeta_test.go

@@ -0,0 +1,111 @@
+package probeservices_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+)
+
+func TestGetMeasurementMetaWorkingAsIntended(t *testing.T) {
+	client := probeservices.Client{
+		Client: httpx.Client{
+			BaseURL:    "https://ams-pg.ooni.org/",
+			HTTPClient: http.DefaultClient,
+			Logger:     log.Log,
+			UserAgent:  "miniooni/0.1.0-dev",
+		},
+		LoginCalls:    atomicx.NewInt64(),
+		RegisterCalls: atomicx.NewInt64(),
+		StateFile:     probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
+	}
+	config := probeservices.MeasurementMetaConfig{
+		ReportID: `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`,
+		Full:     true,
+		Input:    `https://www.example.org`,
+	}
+	ctx := context.Background()
+	mmeta, err := client.GetMeasurementMeta(ctx, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mmeta.Anomaly != false {
+		t.Fatal("unexpected anomaly value")
+	}
+	if mmeta.CategoryCode != "" {
+		t.Fatal("unexpected category code value")
+	}
+	if mmeta.Confirmed != false {
+		t.Fatal("unexpected confirmed value")
+	}
+	if mmeta.Failure != true {
+		// TODO(bassosimone): this field seems wrong
+		t.Fatal("unexpected failure value")
+	}
+	if mmeta.Input == nil || *mmeta.Input != config.Input {
+		t.Fatal("unexpected input value")
+	}
+	if mmeta.MeasurementStartTime.String() != "2020-12-09 05:22:25 +0000 UTC" {
+		t.Fatal("unexpected measurement start time value")
+	}
+	if mmeta.ProbeASN != 30722 {
+		t.Fatal("unexpected probe asn value")
+	}
+	if mmeta.ProbeCC != "IT" {
+		t.Fatal("unexpected probe cc value")
+	}
+	if mmeta.ReportID != config.ReportID {
+		t.Fatal("unexpected report id value")
+	}
+	// TODO(bassosimone): we could better this check
+	var scores interface{}
+	if err := json.Unmarshal([]byte(mmeta.Scores), &scores); err != nil {
+		t.Fatalf("cannot parse scores value: %+v", err)
+	}
+	if mmeta.TestName != "urlgetter" {
+		t.Fatal("unexpected test name value")
+	}
+	if mmeta.TestStartTime.String() != "2020-12-09 05:22:25 +0000 UTC" {
+		t.Fatal("unexpected test start time value")
+	}
+	// TODO(bassosimone): we could better this check
+	var rawmeas interface{}
+	if err := json.Unmarshal([]byte(mmeta.RawMeasurement), &rawmeas); err != nil {
+		t.Fatalf("cannot parse raw measurement: %+v", err)
+	}
+}
+
+func TestGetMeasurementMetaWorkingWithCancelledContext(t *testing.T) {
+	client := probeservices.Client{
+		Client: httpx.Client{
+			BaseURL:    "https://ams-pg.ooni.org/",
+			HTTPClient: http.DefaultClient,
+			Logger:     log.Log,
+			UserAgent:  "miniooni/0.1.0-dev",
+		},
+		LoginCalls:    atomicx.NewInt64(),
+		RegisterCalls: atomicx.NewInt64(),
+		StateFile:     probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
+	}
+	config := probeservices.MeasurementMetaConfig{
+		ReportID: `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`,
+		Full:     true,
+		Input:    `https://www.example.org`,
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	mmeta, err := client.GetMeasurementMeta(ctx, config)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if mmeta != nil {
+		t.Fatal("we expected a nil mmeta here")
+	}
+}

internal/engine/probeservices/metadata.go

@@ -0,0 +1,52 @@
+package probeservices
+
+// Metadata contains metadata about a probe. This message is
+// included into a bunch of messages sent to orchestra.
+type Metadata struct {
+	AvailableBandwidth string   `json:"available_bandwidth,omitempty"`
+	DeviceToken        string   `json:"device_token,omitempty"`
+	Language           string   `json:"language,omitempty"`
+	NetworkType        string   `json:"network_type,omitempty"`
+	Platform           string   `json:"platform"`
+	ProbeASN           string   `json:"probe_asn"`
+	ProbeCC            string   `json:"probe_cc"`
+	ProbeFamily        string   `json:"probe_family,omitempty"`
+	ProbeTimezone      string   `json:"probe_timezone,omitempty"`
+	SoftwareName       string   `json:"software_name"`
+	SoftwareVersion    string   `json:"software_version"`
+	SupportedTests     []string `json:"supported_tests"`
+}
+
+// Valid returns true if metadata is valid, false otherwise. Metadata is
+// considered valid if all the mandatory fields are not empty. If a field
+// is marked `json:",omitempty"` in the structure definition, then it's
+// for sure mandatory. The "device_token" field is mandatory only if the
+// platform is "ios" or "android", because there's currently no device
+// token that we know of for desktop devices.
+func (m Metadata) Valid() bool {
+	if m.ProbeCC == "" {
+		return false
+	}
+	if m.ProbeASN == "" {
+		return false
+	}
+	if m.Platform == "" {
+		return false
+	}
+	if m.SoftwareName == "" {
+		return false
+	}
+	if m.SoftwareVersion == "" {
+		return false
+	}
+	if len(m.SupportedTests) < 1 {
+		return false
+	}
+	switch m.Platform {
+	case "ios", "android":
+		if m.DeviceToken == "" {
+			return false
+		}
+	}
+	return true
+}

internal/engine/probeservices/metadata_test.go

@@ -0,0 +1,106 @@
+package probeservices_test
+
+import (
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+)
+
+func TestValid(t *testing.T) {
+	t.Run("fail on probe_cc", func(t *testing.T) {
+		var m probeservices.Metadata
+		if m.Valid() != false {
+			t.Fatal("expected false here")
+		}
+	})
+	t.Run("fail on probe_asn", func(t *testing.T) {
+		m := probeservices.Metadata{
+			ProbeCC: "IT",
+		}
+		if m.Valid() != false {
+			t.Fatal("expected false here")
+		}
+	})
+	t.Run("fail on platform", func(t *testing.T) {
+		m := probeservices.Metadata{
+			ProbeCC:  "IT",
+			ProbeASN: "AS1234",
+		}
+		if m.Valid() != false {
+			t.Fatal("expected false here")
+		}
+	})
+	t.Run("fail on software_name", func(t *testing.T) {
+		m := probeservices.Metadata{
+			ProbeCC:  "IT",
+			ProbeASN: "AS1234",
+			Platform: "linux",
+		}
+		if m.Valid() != false {
+			t.Fatal("expected false here")
+		}
+	})
+	t.Run("fail on software_version", func(t *testing.T) {
+		m := probeservices.Metadata{
+			ProbeCC:      "IT",
+			ProbeASN:     "AS1234",
+			Platform:     "linux",
+			SoftwareName: "miniooni",
+		}
+		if m.Valid() != false {
+			t.Fatal("expected false here")
+		}
+	})
+	t.Run("fail on supported_tests", func(t *testing.T) {
+		m := probeservices.Metadata{
+			ProbeCC:         "IT",
+			ProbeASN:        "AS1234",
+			Platform:        "linux",
+			SoftwareName:    "miniooni",
+			SoftwareVersion: "0.1.0-dev",
+		}
+		if m.Valid() != false {
+			t.Fatal("expected false here")
+		}
+	})
+	t.Run("fail on missing device_token", func(t *testing.T) {
+		m := probeservices.Metadata{
+			ProbeCC:         "IT",
+			ProbeASN:        "AS1234",
+			Platform:        "ios",
+			SoftwareName:    "miniooni",
+			SoftwareVersion: "0.1.0-dev",
+			SupportedTests:  []string{"web_connectivity"},
+		}
+		if m.Valid() != false {
+			t.Fatal("expected false here")
+		}
+	})
+	t.Run("success for desktop", func(t *testing.T) {
+		m := probeservices.Metadata{
+			ProbeCC:         "IT",
+			ProbeASN:        "AS1234",
+			Platform:        "linux",
+			SoftwareName:    "miniooni",
+			SoftwareVersion: "0.1.0-dev",
+			SupportedTests:  []string{"web_connectivity"},
+		}
+		if m.Valid() != true {
+			t.Fatal("expected true here")
+		}
+	})
+	t.Run("success for mobile", func(t *testing.T) {
+		m := probeservices.Metadata{
+			DeviceToken:     "xx-xxx-xx-xxxx",
+			ProbeCC:         "IT",
+			ProbeASN:        "AS1234",
+			Platform:        "android",
+			SoftwareName:    "miniooni",
+			SoftwareVersion: "0.1.0-dev",
+			SupportedTests:  []string{"web_connectivity"},
+		}
+		if m.Valid() != true {
+			t.Fatal("expected true here")
+		}
+	})
+}

internal/engine/probeservices/probeservices.go

@@ -0,0 +1,129 @@
+// Package probeservices contains code to contact OONI probe services.
+//
+// The probe services are HTTPS endpoints distributed across a bunch of data
+// centres implementing a bunch of OONI APIs. When started, OONI will benchmark
+// the available probe services and select the fastest one. Eventually all the
+// possible OONI APIs will run as probe services.
+//
+// This package implements the following APIs:
+//
+// 1. v2.0.0 of the OONI bouncer specification defined
+// in https://github.com/ooni/spec/blob/master/backends/bk-004-bouncer;
+//
+// 2. v2.0.0 of the OONI collector specification defined
+// in https://github.com/ooni/spec/blob/master/backends/bk-003-collector.md;
+//
+// 3. most of the OONI orchestra API: login, register, fetch URLs for
+// the Web Connectivity experiment, input for Tor and Psiphon.
+//
+// Orchestra is a set of OONI APIs for probe orchestration. We currently mainly
+// using it for fetching inputs for the tor, psiphon, and web experiments.
+//
+// In addition, this package also contains code to benchmark the available
+// probe services, discard non working ones, select the fastest.
+package probeservices
+
+import (
+	"errors"
+	"net/http"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+var (
+	// ErrUnsupportedEndpoint indicates that we don't support this endpoint type.
+	ErrUnsupportedEndpoint = errors.New("probe services: unsupported endpoint type")
+
+	// ErrUnsupportedCloudFrontAddress indicates that we don't support this
+	// cloudfront address (e.g. wrong scheme, presence of port).
+	ErrUnsupportedCloudFrontAddress = errors.New(
+		"probe services: unsupported cloud front address",
+	)
+
+	// ErrNotRegistered indicates that the probe is not registered
+	// with the OONI orchestra backend.
+	ErrNotRegistered = errors.New("not registered")
+
+	// ErrNotLoggedIn indicates that we are not logged in
+	ErrNotLoggedIn = errors.New("not logged in")
+
+	// ErrInvalidMetadata indicates that the metadata is not valid
+	ErrInvalidMetadata = errors.New("invalid metadata")
+)
+
+// Session is how this package sees a Session.
+type Session interface {
+	DefaultHTTPClient() *http.Client
+	KeyValueStore() model.KeyValueStore
+	Logger() model.Logger
+	ProxyURL() *url.URL
+	UserAgent() string
+}
+
+// Client is a client for the OONI probe services API.
+type Client struct {
+	httpx.Client
+	LoginCalls    *atomicx.Int64
+	RegisterCalls *atomicx.Int64
+	StateFile     StateFile
+}
+
+// GetCredsAndAuth is an utility function that returns the credentials with
+// which we are registered and the token with which we're logged in. If we're
+// not registered or not logged in, an error is returned instead.
+func (c Client) GetCredsAndAuth() (*LoginCredentials, *LoginAuth, error) {
+	state := c.StateFile.Get()
+	creds := state.Credentials()
+	if creds == nil {
+		return nil, nil, ErrNotRegistered
+	}
+	auth := state.Auth()
+	if auth == nil {
+		return nil, nil, ErrNotLoggedIn
+	}
+	return creds, auth, nil
+}
+
+// NewClient creates a new client for the specified probe services endpoint. This
+// function fails, e.g., we don't support the specified endpoint.
+func NewClient(sess Session, endpoint model.Service) (*Client, error) {
+	client := &Client{
+		Client: httpx.Client{
+			BaseURL:    endpoint.Address,
+			HTTPClient: sess.DefaultHTTPClient(),
+			Logger:     sess.Logger(),
+			ProxyURL:   sess.ProxyURL(),
+			UserAgent:  sess.UserAgent(),
+		},
+		LoginCalls:    atomicx.NewInt64(),
+		RegisterCalls: atomicx.NewInt64(),
+		StateFile:     NewStateFile(sess.KeyValueStore()),
+	}
+	switch endpoint.Type {
+	case "https":
+		return client, nil
+	case "cloudfront":
+		// Do the cloudfronting dance. The front must appear inside of the
+		// URL, so that we use it for DNS resolution and SNI. The real domain
+		// must instead appear inside of the Host header.
+		URL, err := url.Parse(client.BaseURL)
+		if err != nil {
+			return nil, err
+		}
+		if URL.Scheme != "https" || URL.Host != URL.Hostname() {
+			return nil, ErrUnsupportedCloudFrontAddress
+		}
+		client.Client.Host = URL.Hostname()
+		URL.Host = endpoint.Front
+		client.BaseURL = URL.String()
+		if _, err := url.Parse(client.BaseURL); err != nil {
+			return nil, err
+		}
+		return client, nil
+	default:
+		return nil, ErrUnsupportedEndpoint
+	}
+}

internal/engine/probeservices/probeservices_test.go

@@ -0,0 +1,627 @@
+package probeservices_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
+)
+
+func newclient() *probeservices.Client {
+	client, err := probeservices.NewClient(
+		&mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		model.Service{
+			Address: "https://ams-pg-test.ooni.org/",
+			Type:    "https",
+		},
+	)
+	if err != nil {
+		panic(err) // so fail the test
+	}
+	return client
+}
+
+func TestNewClientHTTPS(t *testing.T) {
+	client, err := probeservices.NewClient(
+		&mockable.Session{}, model.Service{
+			Address: "https://x.org",
+			Type:    "https",
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if client.BaseURL != "https://x.org" {
+		t.Fatal("not the URL we expected")
+	}
+}
+
+func TestNewClientUnsupportedEndpoint(t *testing.T) {
+	client, err := probeservices.NewClient(
+		&mockable.Session{}, model.Service{
+			Address: "https://x.org",
+			Type:    "onion",
+		})
+	if !errors.Is(err, probeservices.ErrUnsupportedEndpoint) {
+		t.Fatal("not the error we expected")
+	}
+	if client != nil {
+		t.Fatal("expected nil client here")
+	}
+}
+
+func TestNewClientCloudfrontInvalidURL(t *testing.T) {
+	client, err := probeservices.NewClient(
+		&mockable.Session{}, model.Service{
+			Address: "\t\t\t",
+			Type:    "cloudfront",
+		})
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+	if client != nil {
+		t.Fatal("expected nil client here")
+	}
+}
+
+func TestNewClientCloudfrontInvalidURLScheme(t *testing.T) {
+	client, err := probeservices.NewClient(
+		&mockable.Session{}, model.Service{
+			Address: "http://x.org",
+			Type:    "cloudfront",
+		})
+	if !errors.Is(err, probeservices.ErrUnsupportedCloudFrontAddress) {
+		t.Fatal("not the error we expected")
+	}
+	if client != nil {
+		t.Fatal("expected nil client here")
+	}
+}
+
+func TestNewClientCloudfrontInvalidURLWithPort(t *testing.T) {
+	client, err := probeservices.NewClient(
+		&mockable.Session{}, model.Service{
+			Address: "https://x.org:54321",
+			Type:    "cloudfront",
+		})
+	if !errors.Is(err, probeservices.ErrUnsupportedCloudFrontAddress) {
+		t.Fatal("not the error we expected")
+	}
+	if client != nil {
+		t.Fatal("expected nil client here")
+	}
+}
+
+func TestNewClientCloudfrontInvalidFront(t *testing.T) {
+	client, err := probeservices.NewClient(
+		&mockable.Session{}, model.Service{
+			Address: "https://x.org",
+			Type:    "cloudfront",
+			Front:   "\t\t\t",
+		})
+	if err == nil || !strings.HasSuffix(err.Error(), `invalid URL escape "%09"`) {
+		t.Fatal("not the error we expected")
+	}
+	if client != nil {
+		t.Fatal("expected nil client here")
+	}
+}
+
+func TestNewClientCloudfrontGood(t *testing.T) {
+	client, err := probeservices.NewClient(
+		&mockable.Session{}, model.Service{
+			Address: "https://x.org",
+			Type:    "cloudfront",
+			Front:   "google.com",
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if client.BaseURL != "https://google.com" {
+		t.Fatal("not the BaseURL we expected")
+	}
+	if client.Host != "x.org" {
+		t.Fatal("not the Host we expected")
+	}
+}
+
+func TestCloudfront(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	client, err := probeservices.NewClient(
+		&mockable.Session{}, model.Service{
+			Address: "https://meek.azureedge.net",
+			Type:    "cloudfront",
+			Front:   "ajax.aspnetcdn.com",
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+	req, err := http.NewRequest("GET", client.BaseURL, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Host = client.Host
+	resp, err := http.DefaultTransport.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		t.Fatal("unexpected status code")
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != "I’m just a happy little web server.\n" {
+		t.Fatal("unexpected response body")
+	}
+}
+
+func TestDefaultProbeServicesWorkAsIntended(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	for _, e := range probeservices.Default() {
+		client, err := probeservices.NewClient(&mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		}, e)
+		if err != nil {
+			t.Fatal(err)
+		}
+		testhelpers, err := client.GetTestHelpers(context.Background())
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(testhelpers) < 1 {
+			t.Fatal("no test helpers?!")
+		}
+	}
+}
+
+func TestSortEndpoints(t *testing.T) {
+	in := []model.Service{{
+		Type:    "onion",
+		Address: "httpo://jehhrikjjqrlpufu.onion",
+	}, {
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}, {
+		Type:    "https",
+		Address: "https://ams-ps2.ooni.nu:443",
+	}}
+	expect := []model.Service{{
+		Type:    "https",
+		Address: "https://ams-ps2.ooni.nu:443",
+	}, {
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}, {
+		Type:    "onion",
+		Address: "httpo://jehhrikjjqrlpufu.onion",
+	}}
+	out := probeservices.SortEndpoints(in)
+	diff := cmp.Diff(out, expect)
+	if diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestOnlyHTTPS(t *testing.T) {
+	in := []model.Service{{
+		Type:    "onion",
+		Address: "httpo://jehhrikjjqrlpufu.onion",
+	}, {
+		Type:    "https",
+		Address: "https://ams-ps-nonexistent.ooni.io",
+	}, {
+		Type:    "https",
+		Address: "https://hkg-ps-nonexistent.ooni.io",
+	}, {
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}, {
+		Type:    "https",
+		Address: "https://mia-ps-nonexistent.ooni.io",
+	}}
+	expect := []model.Service{{
+		Type:    "https",
+		Address: "https://ams-ps-nonexistent.ooni.io",
+	}, {
+		Type:    "https",
+		Address: "https://hkg-ps-nonexistent.ooni.io",
+	}, {
+		Type:    "https",
+		Address: "https://mia-ps-nonexistent.ooni.io",
+	}}
+	out := probeservices.OnlyHTTPS(in)
+	diff := cmp.Diff(out, expect)
+	if diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestOnlyFallbacks(t *testing.T) {
+	// put onion first so we also verify that we sort the endpoints
+	in := []model.Service{{
+		Type:    "onion",
+		Address: "httpo://jehhrikjjqrlpufu.onion",
+	}, {
+		Type:    "https",
+		Address: "https://ams-ps-nonexistent.ooni.io",
+	}, {
+		Type:    "https",
+		Address: "https://hkg-ps-nonexistent.ooni.io",
+	}, {
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}, {
+		Type:    "https",
+		Address: "https://mia-ps-nonexistent.ooni.io",
+	}}
+	expect := []model.Service{{
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}, {
+		Type:    "onion",
+		Address: "httpo://jehhrikjjqrlpufu.onion",
+	}}
+	out := probeservices.OnlyFallbacks(in)
+	diff := cmp.Diff(out, expect)
+	if diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestTryAllCanceledContext(t *testing.T) {
+	// put onion first so we also verify that we sort the endpoints
+	in := []model.Service{{
+		Type:    "onion",
+		Address: "httpo://jehhrikjjqrlpufu.onion",
+	}, {
+		Type:    "https",
+		Address: "https://ams-ps-nonexistent.ooni.io",
+	}, {
+		Type:    "https",
+		Address: "https://hkg-ps-nonexistent.ooni.io",
+	}, {
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}, {
+		Type:    "https",
+		Address: "https://mia-ps-nonexistent.ooni.io",
+	}}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // immediately cancel and cause every attempt to fail
+	sess := &mockable.Session{
+		MockableHTTPClient: http.DefaultClient,
+		MockableLogger:     log.Log,
+	}
+	out := probeservices.TryAll(ctx, sess, in)
+	if len(out) != 5 {
+		t.Fatal("invalid number of entries")
+	}
+	//
+	if out[0].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if !errors.Is(out[0].Err, context.Canceled) {
+		t.Fatal("invalid error")
+	}
+	if out[0].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[0].Endpoint.Address != "https://ams-ps-nonexistent.ooni.io" {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	if out[1].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if !errors.Is(out[1].Err, context.Canceled) {
+		t.Fatal("invalid error")
+	}
+	if out[1].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[1].Endpoint.Address != "https://hkg-ps-nonexistent.ooni.io" {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	if out[2].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if !errors.Is(out[2].Err, context.Canceled) {
+		t.Fatal("invalid error")
+	}
+	if out[2].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[2].Endpoint.Address != "https://mia-ps-nonexistent.ooni.io" {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	if out[3].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if !errors.Is(out[3].Err, context.Canceled) {
+		t.Fatal("invalid error")
+	}
+	if out[3].Endpoint.Type != "cloudfront" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[3].Endpoint.Front != "dkyhjv0wpi2dk.cloudfront.net" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[3].Endpoint.Address != "https://dkyhjv0wpi2dk.cloudfront.net" {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	// Note: here duration may be zero because the endpoint is not supported
+	// and so we don't basically do anything. But it also may be nonzero since
+	// we also run tests in the cloud, which is slower than my desktop. So, I
+	// have not written a specific test concerning out[4].Duration.
+	if !errors.Is(out[4].Err, probeservices.ErrUnsupportedEndpoint) {
+		t.Fatal("invalid error")
+	}
+	if out[4].Endpoint.Type != "onion" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[4].Endpoint.Address != "httpo://jehhrikjjqrlpufu.onion" {
+		t.Fatal("invalid endpoint type")
+	}
+}
+
+func TestTryAllIntegrationWeRaceForFastestHTTPS(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	const pattern = "^https://ps[1-4].ooni.io$"
+	// put onion first so we also verify that we sort the endpoints
+	in := []model.Service{{
+		Type:    "onion",
+		Address: "httpo://jehhrikjjqrlpufu.onion",
+	}, {
+		Type:    "https",
+		Address: "https://ps1.ooni.io",
+	}, {
+		Type:    "https",
+		Address: "https://ps2.ooni.io",
+	}, {
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}, {
+		Type:    "https",
+		Address: "https://ps3.ooni.io",
+	}}
+	sess := &mockable.Session{
+		MockableHTTPClient: http.DefaultClient,
+		MockableLogger:     log.Log,
+	}
+	out := probeservices.TryAll(context.Background(), sess, in)
+	if len(out) != 3 {
+		t.Fatal("invalid number of entries")
+	}
+	//
+	if out[0].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if out[0].Err != nil {
+		t.Fatal("invalid error")
+	}
+	if out[0].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if ok, _ := regexp.MatchString(pattern, out[0].Endpoint.Address); !ok {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	if out[1].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if out[1].Err != nil {
+		t.Fatal("invalid error")
+	}
+	if out[1].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if ok, _ := regexp.MatchString(pattern, out[1].Endpoint.Address); !ok {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	if out[2].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if out[2].Err != nil {
+		t.Fatal("invalid error")
+	}
+	if out[2].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if ok, _ := regexp.MatchString(pattern, out[2].Endpoint.Address); !ok {
+		t.Fatal("invalid endpoint type")
+	}
+}
+
+func TestTryAllIntegrationWeFallback(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	// put onion first so we also verify that we sort the endpoints
+	in := []model.Service{{
+		Type:    "onion",
+		Address: "httpo://jehhrikjjqrlpufu.onion",
+	}, {
+		Type:    "https",
+		Address: "https://ps-nonexistent.ooni.io",
+	}, {
+		Type:    "https",
+		Address: "https://hkg-ps-nonexistent.ooni.nu",
+	}, {
+		Front:   "dkyhjv0wpi2dk.cloudfront.net",
+		Type:    "cloudfront",
+		Address: "https://dkyhjv0wpi2dk.cloudfront.net",
+	}, {
+		Type:    "https",
+		Address: "https://mia-ps2-nonexistent.ooni.nu",
+	}}
+	sess := &mockable.Session{
+		MockableHTTPClient: http.DefaultClient,
+		MockableLogger:     log.Log,
+	}
+	out := probeservices.TryAll(context.Background(), sess, in)
+	if len(out) != 4 {
+		t.Fatal("invalid number of entries")
+	}
+	//
+	if out[0].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if !strings.HasSuffix(out[0].Err.Error(), "no such host") {
+		t.Fatal("invalid error")
+	}
+	if out[0].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[0].Endpoint.Address != "https://ps-nonexistent.ooni.io" {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	if out[1].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if !strings.HasSuffix(out[1].Err.Error(), "no such host") {
+		t.Fatal("invalid error")
+	}
+	if out[1].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[1].Endpoint.Address != "https://hkg-ps-nonexistent.ooni.nu" {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	if out[2].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if !strings.HasSuffix(out[2].Err.Error(), "no such host") {
+		t.Fatal("invalid error")
+	}
+	if out[2].Endpoint.Type != "https" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[2].Endpoint.Address != "https://mia-ps2-nonexistent.ooni.nu" {
+		t.Fatal("invalid endpoint type")
+	}
+	//
+	if out[3].Duration <= 0 {
+		t.Fatal("invalid duration")
+	}
+	if out[3].Err != nil {
+		t.Fatal("invalid error")
+	}
+	if out[3].Endpoint.Type != "cloudfront" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[3].Endpoint.Address != "https://dkyhjv0wpi2dk.cloudfront.net" {
+		t.Fatal("invalid endpoint type")
+	}
+	if out[3].Endpoint.Front != "dkyhjv0wpi2dk.cloudfront.net" {
+		t.Fatal("invalid front")
+	}
+}
+
+func TestSelectBestEmptyInput(t *testing.T) {
+	if out := probeservices.SelectBest(nil); out != nil {
+		t.Fatal("expected nil output here")
+	}
+}
+
+func TestSelectBestOnlyFailures(t *testing.T) {
+	in := []*probeservices.Candidate{{
+		Duration: 10 * time.Millisecond,
+		Err:      io.EOF,
+	}}
+	if out := probeservices.SelectBest(in); out != nil {
+		t.Fatal("expected nil output here")
+	}
+}
+
+func TestSelectBestSelectsTheFastest(t *testing.T) {
+	in := []*probeservices.Candidate{{
+		Duration: 10 * time.Millisecond,
+		Endpoint: model.Service{
+			Address: "https://ps1.ooni.io",
+			Type:    "https",
+		},
+	}, {
+		Duration: 4 * time.Millisecond,
+		Endpoint: model.Service{
+			Address: "https://ps2.ooni.io",
+			Type:    "https",
+		},
+	}, {
+		Duration: 7 * time.Millisecond,
+		Endpoint: model.Service{
+			Address: "https://ps3.ooni.io",
+			Type:    "https",
+		},
+	}, {
+		Duration: 11 * time.Millisecond,
+		Endpoint: model.Service{
+			Address: "https://ps4.ooni.io",
+			Type:    "https",
+		},
+	}}
+	expected := &probeservices.Candidate{
+		Duration: 4 * time.Millisecond,
+		Endpoint: model.Service{
+			Address: "https://ps2.ooni.io",
+			Type:    "https",
+		},
+	}
+	out := probeservices.SelectBest(in)
+	if diff := cmp.Diff(out, expected); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestGetCredsAndAuthNotLoggedIn(t *testing.T) {
+	clnt := newclient()
+	if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
+		t.Fatal(err)
+	}
+	creds, auth, err := clnt.GetCredsAndAuth()
+	if !errors.Is(err, probeservices.ErrNotLoggedIn) {
+		t.Fatal("not the error we expected")
+	}
+	if creds != nil {
+		t.Fatal("expected nil creds here")
+	}
+	if auth != nil {
+		t.Fatal("expected nil auth here")
+	}
+}

internal/engine/probeservices/psiphon.go

@@ -0,0 +1,17 @@
+package probeservices
+
+import (
+	"context"
+	"fmt"
+)
+
+// FetchPsiphonConfig fetches psiphon config from authenticated OONI orchestra.
+func (c Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) {
+	_, auth, err := c.GetCredsAndAuth()
+	if err != nil {
+		return nil, err
+	}
+	client := c.Client
+	client.Authorization = fmt.Sprintf("Bearer %s", auth.Token)
+	return client.FetchResource(ctx, "/api/v1/test-list/psiphon-config")
+}

internal/engine/probeservices/psiphon_test.go

@@ -0,0 +1,46 @@
+package probeservices_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
+)
+
+func TestFetchPsiphonConfig(t *testing.T) {
+	clnt := newclient()
+	if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
+		t.Fatal(err)
+	}
+	if err := clnt.MaybeLogin(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	data, err := clnt.FetchPsiphonConfig(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	var config interface{}
+	if err := json.Unmarshal(data, &config); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFetchPsiphonConfigNotRegistered(t *testing.T) {
+	clnt := newclient()
+	state := probeservices.State{
+		// Explicitly empty so the test is more clear
+	}
+	if err := clnt.StateFile.Set(state); err != nil {
+		t.Fatal(err)
+	}
+	data, err := clnt.FetchPsiphonConfig(context.Background())
+	if !errors.Is(err, probeservices.ErrNotRegistered) {
+		t.Fatal("expected an error here")
+	}
+	if data != nil {
+		t.Fatal("expected nil data here")
+	}
+}

internal/engine/probeservices/register.go

@@ -0,0 +1,40 @@
+package probeservices
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
+)
+
+type registerRequest struct {
+	Metadata
+	Password string `json:"password"`
+}
+
+type registerResult struct {
+	ClientID string `json:"client_id"`
+}
+
+// MaybeRegister registers this client if not already registered
+func (c Client) MaybeRegister(ctx context.Context, metadata Metadata) error {
+	if !metadata.Valid() {
+		return ErrInvalidMetadata
+	}
+	state := c.StateFile.Get()
+	if state.Credentials() != nil {
+		return nil // we're already good
+	}
+	c.RegisterCalls.Add(1)
+	pwd := randx.Letters(64)
+	req := &registerRequest{
+		Metadata: metadata,
+		Password: pwd,
+	}
+	var resp registerResult
+	if err := c.Client.PostJSON(ctx, "/api/v1/register", req, &resp); err != nil {
+		return err
+	}
+	state.ClientID = resp.ClientID
+	state.Password = pwd
+	return c.StateFile.Set(state)
+}

internal/engine/probeservices/register_test.go

@@ -0,0 +1,63 @@
+package probeservices_test
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
+)
+
+func TestMaybeRegister(t *testing.T) {
+	t.Run("when metadata is not valid", func(t *testing.T) {
+		clnt := newclient()
+		ctx := context.Background()
+		var metadata probeservices.Metadata
+		err := clnt.MaybeRegister(ctx, metadata)
+		if !errors.Is(err, probeservices.ErrInvalidMetadata) {
+			t.Fatal("expected an error here")
+		}
+	})
+	t.Run("when we have already registered", func(t *testing.T) {
+		clnt := newclient()
+		state := probeservices.State{
+			ClientID: "xx-xxx-x-xxxx",
+			Password: "xx",
+		}
+		if err := clnt.StateFile.Set(state); err != nil {
+			t.Fatal(err)
+		}
+		ctx := context.Background()
+		metadata := testorchestra.MetadataFixture()
+		if err := clnt.MaybeRegister(ctx, metadata); err != nil {
+			t.Fatal(err)
+		}
+	})
+	t.Run("when the API call fails", func(t *testing.T) {
+		clnt := newclient()
+		clnt.BaseURL = "\t\t\t" // makes it fail
+		ctx := context.Background()
+		metadata := testorchestra.MetadataFixture()
+		err := clnt.MaybeRegister(ctx, metadata)
+		if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+			t.Fatal("expected an error here")
+		}
+	})
+}
+
+func TestMaybeRegisterIdempotent(t *testing.T) {
+	clnt := newclient()
+	ctx := context.Background()
+	metadata := testorchestra.MetadataFixture()
+	if err := clnt.MaybeRegister(ctx, metadata); err != nil {
+		t.Fatal(err)
+	}
+	if err := clnt.MaybeRegister(ctx, metadata); err != nil {
+		t.Fatal(err)
+	}
+	if clnt.RegisterCalls.Load() != 1 {
+		t.Fatal("called register API too many times")
+	}
+}

internal/engine/probeservices/statefile.go

@@ -0,0 +1,87 @@
+package probeservices
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// State is the state stored inside the state file
+type State struct {
+	ClientID string
+	Expire   time.Time
+	Password string
+	Token    string
+}
+
+// Auth returns an authentication structure, if possible, otherwise
+// it returns nil, meaning that you should login again.
+func (s State) Auth() *LoginAuth {
+	if s.Token == "" {
+		return nil
+	}
+	if time.Now().Add(30 * time.Second).After(s.Expire) {
+		return nil
+	}
+	return &LoginAuth{Expire: s.Expire, Token: s.Token}
+}
+
+// Credentials returns login credentials, if possible, otherwise it
+// returns nil, meaning that you should create an account.
+func (s State) Credentials() *LoginCredentials {
+	if s.ClientID == "" {
+		return nil
+	}
+	if s.Password == "" {
+		return nil
+	}
+	return &LoginCredentials{ClientID: s.ClientID, Password: s.Password}
+}
+
+// StateFile is the orchestra state file. It is backed by
+// a generic key-value store configured by the user.
+type StateFile struct {
+	Store model.KeyValueStore
+	key   string
+}
+
+// NewStateFile creates a new state file backed by a key-value store
+func NewStateFile(kvstore model.KeyValueStore) StateFile {
+	return StateFile{key: "orchestra.state", Store: kvstore}
+}
+
+// SetMockable is a mockable version of Set
+func (sf StateFile) SetMockable(s State, mf func(interface{}) ([]byte, error)) error {
+	data, err := mf(s)
+	if err != nil {
+		return err
+	}
+	return sf.Store.Set(sf.key, data)
+}
+
+// Set saves the current state on the key-value store.
+func (sf StateFile) Set(s State) error {
+	return sf.SetMockable(s, json.Marshal)
+}
+
+// GetMockable is a mockable version of Get
+func (sf StateFile) GetMockable(sfget func(string) ([]byte, error),
+	unmarshal func([]byte, interface{}) error) (State, error) {
+	value, err := sfget(sf.key)
+	if err != nil {
+		return State{}, err
+	}
+	var state State
+	if err := unmarshal(value, &state); err != nil {
+		return State{}, err
+	}
+	return state, nil
+}
+
+// Get returns the current state. In case of any error with the
+// underlying key-value store, we return an empty state.
+func (sf StateFile) Get() (state State) {
+	state, _ = sf.GetMockable(sf.Store.Get, json.Unmarshal)
+	return
+}

internal/engine/probeservices/statefile_test.go

@@ -0,0 +1,153 @@
+package probeservices_test
+
+import (
+	"encoding/json"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+)
+
+func TestStateAuth(t *testing.T) {
+	t.Run("with no Token", func(t *testing.T) {
+		state := probeservices.State{Expire: time.Now().Add(10 * time.Hour)}
+		if state.Auth() != nil {
+			t.Fatal("expected nil here")
+		}
+	})
+	t.Run("with expired Token", func(t *testing.T) {
+		state := probeservices.State{
+			Expire: time.Now().Add(-1 * time.Hour),
+			Token:  "xx-x-xxx-xx",
+		}
+		if state.Auth() != nil {
+			t.Fatal("expected nil here")
+		}
+	})
+	t.Run("with good Token", func(t *testing.T) {
+		state := probeservices.State{
+			Expire: time.Now().Add(10 * time.Hour),
+			Token:  "xx-x-xxx-xx",
+		}
+		if state.Auth() == nil {
+			t.Fatal("expected valid auth here")
+		}
+	})
+}
+
+func TestStateCredentials(t *testing.T) {
+	t.Run("with no ClientID", func(t *testing.T) {
+		state := probeservices.State{}
+		if state.Credentials() != nil {
+			t.Fatal("expected nil here")
+		}
+	})
+	t.Run("with no Password", func(t *testing.T) {
+		state := probeservices.State{
+			ClientID: "xx-x-xxx-xx",
+		}
+		if state.Credentials() != nil {
+			t.Fatal("expected nil here")
+		}
+	})
+	t.Run("with all good", func(t *testing.T) {
+		state := probeservices.State{
+			ClientID: "xx-x-xxx-xx",
+			Password: "xx",
+		}
+		if state.Credentials() == nil {
+			t.Fatal("expected valid auth here")
+		}
+	})
+}
+
+func TestStateFileMemoryIntegration(t *testing.T) {
+	// Does the StateFile have the property that we can write
+	// values into it and then read again the same files?
+	sf := probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore())
+	s := probeservices.State{
+		Expire:   time.Now(),
+		Password: "xy",
+		Token:    "abc",
+		ClientID: "xx",
+	}
+	if err := sf.Set(s); err != nil {
+		t.Fatal(err)
+	}
+	os := sf.Get()
+	diff := cmp.Diff(s, os)
+	if diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestStateFileSetMarshalError(t *testing.T) {
+	sf := probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore())
+	s := probeservices.State{
+		Expire:   time.Now(),
+		Password: "xy",
+		Token:    "abc",
+		ClientID: "xx",
+	}
+	expected := errors.New("mocked error")
+	failingfunc := func(v interface{}) ([]byte, error) {
+		return nil, expected
+	}
+	if err := sf.SetMockable(s, failingfunc); !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestStateFileGetKVStoreGetError(t *testing.T) {
+	sf := probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore())
+	expected := errors.New("mocked error")
+	failingfunc := func(string) ([]byte, error) {
+		return nil, expected
+	}
+	s, err := sf.GetMockable(failingfunc, json.Unmarshal)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if s.ClientID != "" {
+		t.Fatal("unexpected ClientID field")
+	}
+	if !s.Expire.IsZero() {
+		t.Fatal("unexpected Expire field")
+	}
+	if s.Password != "" {
+		t.Fatal("unexpected Password field")
+	}
+	if s.Token != "" {
+		t.Fatal("unexpected Token field")
+	}
+}
+
+func TestStateFileGetUnmarshalError(t *testing.T) {
+	sf := probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore())
+	if err := sf.Set(probeservices.State{}); err != nil {
+		t.Fatal(err)
+	}
+	expected := errors.New("mocked error")
+	failingfunc := func([]byte, interface{}) error {
+		return expected
+	}
+	s, err := sf.GetMockable(sf.Store.Get, failingfunc)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if s.ClientID != "" {
+		t.Fatal("unexpected ClientID field")
+	}
+	if !s.Expire.IsZero() {
+		t.Fatal("unexpected Expire field")
+	}
+	if s.Password != "" {
+		t.Fatal("unexpected Password field")
+	}
+	if s.Token != "" {
+		t.Fatal("unexpected Token field")
+	}
+}

internal/engine/probeservices/testorchestra/testorchestra.go

@@ -0,0 +1,19 @@
+// Package testorchestra helps with testing the OONI orchestra API.
+package testorchestra
+
+import "github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+
+// MetadataFixture returns a valid metadata struct. This is mostly
+// useful for testing. (We should see if we can make this private.)
+func MetadataFixture() probeservices.Metadata {
+	return probeservices.Metadata{
+		Platform:        "linux",
+		ProbeASN:        "AS15169",
+		ProbeCC:         "US",
+		SoftwareName:    "miniooni",
+		SoftwareVersion: "0.1.0-dev",
+		SupportedTests: []string{
+			"web_connectivity",
+		},
+	}
+}

internal/engine/probeservices/tor.go

@@ -0,0 +1,24 @@
+package probeservices
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// FetchTorTargets returns the targets for the tor experiment.
+func (c Client) FetchTorTargets(ctx context.Context, cc string) (result map[string]model.TorTarget, err error) {
+	_, auth, err := c.GetCredsAndAuth()
+	if err != nil {
+		return nil, err
+	}
+	client := c.Client
+	client.Authorization = fmt.Sprintf("Bearer %s", auth.Token)
+	query := url.Values{}
+	query.Add("country_code", cc)
+	err = client.GetJSONWithQuery(
+		ctx, "/api/v1/test-list/tor-targets", query, &result)
+	return
+}

internal/engine/probeservices/tor_test.go

@@ -0,0 +1,82 @@
+package probeservices_test
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
+)
+
+func TestFetchTorTargets(t *testing.T) {
+	clnt := newclient()
+	if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
+		t.Fatal(err)
+	}
+	if err := clnt.MaybeLogin(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if data == nil || len(data) <= 0 {
+		t.Fatal("invalid data")
+	}
+}
+
+func TestFetchTorTargetsNotRegistered(t *testing.T) {
+	clnt := newclient()
+	state := probeservices.State{
+		// Explicitly empty so the test is more clear
+	}
+	if err := clnt.StateFile.Set(state); err != nil {
+		t.Fatal(err)
+	}
+	data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if data != nil {
+		t.Fatal("expected nil data here")
+	}
+}
+
+type FetchTorTargetsHTTPTransport struct {
+	Response *http.Response
+}
+
+func (clnt *FetchTorTargetsHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	resp, err := http.DefaultTransport.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+	if req.URL.Path == "/api/v1/test-list/tor-targets" {
+		clnt.Response = resp
+	}
+	return resp, err
+}
+
+func TestFetchTorTargetsSetsQueryString(t *testing.T) {
+	clnt := newclient()
+	txp := new(FetchTorTargetsHTTPTransport)
+	clnt.HTTPClient.Transport = txp
+	if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
+		t.Fatal(err)
+	}
+	if err := clnt.MaybeLogin(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if data == nil || len(data) <= 0 {
+		t.Fatal("invalid data")
+	}
+	requestURL := txp.Response.Request.URL
+	if requestURL.Query().Get("country_code") != "ZZ" {
+		t.Fatal("invalid country code")
+	}
+}

internal/engine/probeservices/urls.go

@@ -0,0 +1,36 @@
+package probeservices
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type urlListResult struct {
+	Results []model.URLInfo `json:"results"`
+}
+
+// FetchURLList fetches the list of URLs used by WebConnectivity. The config
+// argument contains the optional settings. Returns the list of URLs, on success,
+// or an explanatory error, in case of failure.
+func (c Client) FetchURLList(ctx context.Context, config model.URLListConfig) ([]model.URLInfo, error) {
+	query := url.Values{}
+	if config.CountryCode != "" {
+		query.Set("country_code", config.CountryCode)
+	}
+	if config.Limit > 0 {
+		query.Set("limit", fmt.Sprintf("%d", config.Limit))
+	}
+	if len(config.Categories) > 0 {
+		query.Set("category_codes", strings.Join(config.Categories, ","))
+	}
+	var response urlListResult
+	err := c.Client.GetJSONWithQuery(ctx, "/api/v1/test-list/urls", query, &response)
+	if err != nil {
+		return nil, err
+	}
+	return response.Results, nil
+}

internal/engine/probeservices/urls_test.go

@@ -0,0 +1,50 @@
+package probeservices_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestFetchURLListSuccess(t *testing.T) {
+	client := newclient()
+	client.BaseURL = "https://ams-pg-test.ooni.org"
+	config := model.URLListConfig{
+		Categories:  []string{"NEWS", "CULTR"},
+		CountryCode: "IT",
+		Limit:       17,
+	}
+	ctx := context.Background()
+	result, err := client.FetchURLList(ctx, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result) != 17 {
+		t.Fatal("unexpected number of results")
+	}
+	for _, entry := range result {
+		if entry.CategoryCode != "NEWS" && entry.CategoryCode != "CULTR" {
+			t.Fatalf("unexpected category code: %+v", entry)
+		}
+	}
+}
+
+func TestFetchURLListFailure(t *testing.T) {
+	client := newclient()
+	client.BaseURL = "https://\t\t\t/" // cause test to fail
+	config := model.URLListConfig{
+		Categories:  []string{"NEWS", "CULTR"},
+		CountryCode: "IT",
+		Limit:       17,
+	}
+	ctx := context.Background()
+	result, err := client.FetchURLList(ctx, config)
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+	if len(result) != 0 {
+		t.Fatal("results?!")
+	}
+}