chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/model/README.md

@@ -0,0 +1,3 @@
+# Package github.com/ooni/probe-engine/model
+
+Shared data structures and interfaces.

internal/engine/model/checkinconfig.go

@@ -0,0 +1,19 @@
+package model
+
+// CheckInConfigWebConnectivity is the configuration for the WebConnectivity test
+type CheckInConfigWebConnectivity struct {
+	CategoryCodes []string `json:"category_codes"` // CategoryCodes is an array of category codes
+}
+
+// CheckInConfig contains configuration for calling the checkin API.
+type CheckInConfig struct {
+	Charging        bool                         `json:"charging"`         // Charging indicate if the phone is actually charging
+	OnWiFi          bool                         `json:"on_wifi"`          // OnWiFi indicate if the phone is actually connected to a WiFi network
+	Platform        string                       `json:"platform"`         // Platform of the probe
+	ProbeASN        string                       `json:"probe_asn"`        // ProbeASN is the probe country code
+	ProbeCC         string                       `json:"probe_cc"`         // ProbeCC is the probe country code
+	RunType         string                       `json:"run_type"`         // RunType
+	SoftwareName    string                       `json:"software_name"`    // SoftwareName of the probe
+	SoftwareVersion string                       `json:"software_version"` // SoftwareVersion of the probe
+	WebConnectivity CheckInConfigWebConnectivity `json:"web_connectivity"` // WebConnectivity class contain an array of categories
+}

internal/engine/model/checkininfo.go

@@ -0,0 +1,12 @@
+package model
+
+// CheckInInfoWebConnectivity contains the array of URLs returned by the checkin API
+type CheckInInfoWebConnectivity struct {
+	ReportID string    `json:"report_id"`
+	URLs     []URLInfo `json:"urls"`
+}
+
+// CheckInInfo contains the return test objects from the checkin API
+type CheckInInfo struct {
+	WebConnectivity *CheckInInfoWebConnectivity `json:"web_connectivity"`
+}

internal/engine/model/experiment.go

@@ -0,0 +1,75 @@
+package model
+
+import (
+	"context"
+	"net/http"
+)
+
+// ExperimentOrchestraClient is the experiment's view of
+// a client for querying the OONI orchestra API.
+type ExperimentOrchestraClient interface {
+	FetchPsiphonConfig(ctx context.Context) ([]byte, error)
+	FetchTorTargets(ctx context.Context, cc string) (map[string]TorTarget, error)
+	FetchURLList(ctx context.Context, config URLListConfig) ([]URLInfo, error)
+}
+
+// ExperimentSession is the experiment's view of a session.
+type ExperimentSession interface {
+	ASNDatabasePath() string
+	GetTestHelpersByName(name string) ([]Service, bool)
+	DefaultHTTPClient() *http.Client
+	Logger() Logger
+	NewOrchestraClient(ctx context.Context) (ExperimentOrchestraClient, error)
+	ProbeCC() string
+	ResolverIP() string
+	TempDir() string
+	TorArgs() []string
+	TorBinary() string
+	UserAgent() string
+}
+
+// ExperimentCallbacks contains experiment event-handling callbacks
+type ExperimentCallbacks interface {
+	// OnProgress provides information about an experiment progress.
+	OnProgress(percentage float64, message string)
+}
+
+// PrinterCallbacks is the default event handler
+type PrinterCallbacks struct {
+	Logger
+}
+
+// NewPrinterCallbacks returns a new default callback handler
+func NewPrinterCallbacks(logger Logger) PrinterCallbacks {
+	return PrinterCallbacks{Logger: logger}
+}
+
+// OnProgress provides information about an experiment progress.
+func (d PrinterCallbacks) OnProgress(percentage float64, message string) {
+	d.Logger.Infof("[%5.1f%%] %s", percentage*100, message)
+}
+
+// ExperimentMeasurer is the interface that allows to run a
+// measurement for a specific experiment.
+type ExperimentMeasurer interface {
+	// ExperimentName returns the experiment name.
+	ExperimentName() string
+
+	// ExperimentVersion returns the experiment version.
+	ExperimentVersion() string
+
+	// Run runs the experiment with the specified context, session,
+	// measurement, and experiment calbacks. This method should only
+	// return an error in case the experiment could not run (e.g.,
+	// a required input is missing). Otherwise, the code should just
+	// set the relevant OONI error inside of the measurmeent and
+	// return nil. This is important because the caller may not submit
+	// the measurement if this method returns an error.
+	Run(
+		ctx context.Context, sess ExperimentSession,
+		measurement *Measurement, callbacks ExperimentCallbacks,
+	) error
+
+	// GetSummaryKeys returns summary keys expected by ooni/probe-cli.
+	GetSummaryKeys(*Measurement) (interface{}, error)
+}

internal/engine/model/experiment_test.go

@@ -0,0 +1,13 @@
+package model_test
+
+import (
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestPrinterCallbacksCallbacks(t *testing.T) {
+	printer := model.NewPrinterCallbacks(log.Log)
+	printer.OnProgress(0.4, "progress")
+}

internal/engine/model/keyvaluestore.go

@@ -0,0 +1,7 @@
+package model
+
+// KeyValueStore is a key-value store used by the session.
+type KeyValueStore interface {
+	Get(key string) (value []byte, err error)
+	Set(key string, value []byte) (err error)
+}

internal/engine/model/logger.go

@@ -0,0 +1,40 @@
+package model
+
+// Logger defines the common interface that a logger should have. It is
+// out of the box compatible with `log.Log` in `apex/log`.
+type Logger interface {
+	// Debug emits a debug message.
+	Debug(msg string)
+
+	// Debugf formats and emits a debug message.
+	Debugf(format string, v ...interface{})
+
+	// Info emits an informational message.
+	Info(msg string)
+
+	// Infof format and emits an informational message.
+	Infof(format string, v ...interface{})
+
+	// Warn emits a warning message.
+	Warn(msg string)
+
+	// Warnf formats and emits a warning message.
+	Warnf(format string, v ...interface{})
+}
+
+// DiscardLogger is a logger that discards its input
+var DiscardLogger Logger = logDiscarder{}
+
+type logDiscarder struct{}
+
+func (logDiscarder) Debug(msg string) {}
+
+func (logDiscarder) Debugf(format string, v ...interface{}) {}
+
+func (logDiscarder) Info(msg string) {}
+
+func (logDiscarder) Infof(format string, v ...interface{}) {}
+
+func (logDiscarder) Warn(msg string) {}
+
+func (logDiscarder) Warnf(format string, v ...interface{}) {}

internal/engine/model/measurement.go

@@ -0,0 +1,126 @@
+package model
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// MeasurementTarget is the target of a OONI measurement.
+type MeasurementTarget string
+
+// MarshalJSON serializes the MeasurementTarget.
+func (t MeasurementTarget) MarshalJSON() ([]byte, error) {
+	if t == "" {
+		return json.Marshal(nil)
+	}
+	return json.Marshal(string(t))
+}
+
+// Measurement is a OONI measurement.
+//
+// This structure is compatible with the definition of the base data format in
+// https://github.com/ooni/spec/blob/master/data-formats/df-000-base.md.
+type Measurement struct {
+	// Annotations contains results annotations
+	Annotations map[string]string `json:"annotations,omitempty"`
+
+	// DataFormatVersion is the version of the data format
+	DataFormatVersion string `json:"data_format_version"`
+
+	// Extensions contains information about the extensions included
+	// into the test_keys of this measurement.
+	Extensions map[string]int64 `json:"extensions,omitempty"`
+
+	// ID is the locally generated measurement ID
+	ID string `json:"id,omitempty"`
+
+	// Input is the measurement input
+	Input MeasurementTarget `json:"input"`
+
+	// InputHashes contains input hashes
+	InputHashes []string `json:"input_hashes,omitempty"`
+
+	// MeasurementStartTime is the time when the measurement started
+	MeasurementStartTime string `json:"measurement_start_time"`
+
+	// MeasurementStartTimeSaved is the moment in time when we
+	// started the measurement. This is not included into the JSON
+	// and is only used within probe-engine as a "zero" time.
+	MeasurementStartTimeSaved time.Time `json:"-"`
+
+	// Options contains command line options
+	Options []string `json:"options,omitempty"`
+
+	// ProbeASN contains the probe autonomous system number
+	ProbeASN string `json:"probe_asn"`
+
+	// ProbeCC contains the probe country code
+	ProbeCC string `json:"probe_cc"`
+
+	// ProbeCity contains the probe city
+	ProbeCity string `json:"probe_city,omitempty"`
+
+	// ProbeIP contains the probe IP
+	ProbeIP string `json:"probe_ip,omitempty"`
+
+	// ProbeNetworkName contains the probe network name
+	ProbeNetworkName string `json:"probe_network_name"`
+
+	// ReportID contains the report ID
+	ReportID string `json:"report_id"`
+
+	// ResolverASN is the ASN of the resolver
+	ResolverASN string `json:"resolver_asn"`
+
+	// ResolverIP is the resolver IP
+	ResolverIP string `json:"resolver_ip"`
+
+	// ResolverNetworkName is the network name of the resolver.
+	ResolverNetworkName string `json:"resolver_network_name"`
+
+	// SoftwareName contains the software name
+	SoftwareName string `json:"software_name"`
+
+	// SoftwareVersion contains the software version
+	SoftwareVersion string `json:"software_version"`
+
+	// TestHelpers contains the test helpers. It seems this structure is more
+	// complex than we would like. In particular, using a map from string to
+	// string does not fit into the web_connectivity use case. Hence, for now
+	// we're going to represent this using interface{}. In going forward we
+	// may probably want to have more uniform test helpers.
+	TestHelpers map[string]interface{} `json:"test_helpers,omitempty"`
+
+	// TestKeys contains the real test result. This field is opaque because
+	// each experiment will insert here a different structure.
+	TestKeys interface{} `json:"test_keys"`
+
+	// TestName contains the test name
+	TestName string `json:"test_name"`
+
+	// MeasurementRuntime contains the measurement runtime. The JSON name
+	// is test_runtime because this is the name expected by the OONI backend
+	// even though that name is clearly a misleading one.
+	MeasurementRuntime float64 `json:"test_runtime"`
+
+	// TestStartTime contains the test start time
+	TestStartTime string `json:"test_start_time"`
+
+	// TestVersion contains the test version
+	TestVersion string `json:"test_version"`
+}
+
+// AddAnnotations adds the annotations from input to m.Annotations.
+func (m *Measurement) AddAnnotations(input map[string]string) {
+	for key, value := range input {
+		m.AddAnnotation(key, value)
+	}
+}
+
+// AddAnnotation adds a single annotations to m.Annotations.
+func (m *Measurement) AddAnnotation(key, value string) {
+	if m.Annotations == nil {
+		m.Annotations = make(map[string]string)
+	}
+	m.Annotations[key] = value
+}

internal/engine/model/model.go

@@ -0,0 +1,7 @@
+// Package model defines shared data structures and interfaces.
+package model
+
+const (
+	// DefaultProbeIP is the default probe IP.
+	DefaultProbeIP = "127.0.0.1"
+)

internal/engine/model/model_test.go

@@ -0,0 +1,223 @@
+package model_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestMeasurementTargetMarshalJSON(t *testing.T) {
+	var mt model.MeasurementTarget
+	data, err := json.Marshal(mt)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != "null" {
+		t.Fatal("unexpected serialization")
+	}
+	mt = "xx"
+	data, err = json.Marshal(mt)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != `"xx"` {
+		t.Fatal("unexpected serialization")
+	}
+}
+
+type fakeTestKeys struct {
+	ClientResolver string `json:"client_resolver"`
+	Body           string `json:"body"`
+}
+
+func TestAddAnnotations(t *testing.T) {
+	m := &model.Measurement{}
+	m.AddAnnotations(map[string]string{
+		"foo": "bar",
+		"f":   "b",
+	})
+	m.AddAnnotations(map[string]string{
+		"foobar": "bar",
+		"f":      "b",
+	})
+	if len(m.Annotations) != 3 {
+		t.Fatal("unexpected number of annotations")
+	}
+	if m.Annotations["foo"] != "bar" {
+		t.Fatal("unexpected annotation")
+	}
+	if m.Annotations["f"] != "b" {
+		t.Fatal("unexpected annotation")
+	}
+	if m.Annotations["foobar"] != "bar" {
+		t.Fatal("unexpected annotation")
+	}
+}
+
+type makeMeasurementConfig struct {
+	ProbeIP             string
+	ProbeASN            string
+	ProbeNetworkName    string
+	ProbeCC             string
+	ResolverIP          string
+	ResolverNetworkName string
+	ResolverASN         string
+}
+
+func makeMeasurement(config makeMeasurementConfig) model.Measurement {
+	return model.Measurement{
+		DataFormatVersion:    "0.3.0",
+		ID:                   "bdd20d7a-bba5-40dd-a111-9863d7908572",
+		MeasurementStartTime: "2018-11-01 15:33:20",
+		ProbeIP:              config.ProbeIP,
+		ProbeASN:             config.ProbeASN,
+		ProbeNetworkName:     config.ProbeNetworkName,
+		ProbeCC:              config.ProbeCC,
+		ReportID:             "",
+		ResolverIP:           config.ResolverIP,
+		ResolverNetworkName:  config.ResolverNetworkName,
+		ResolverASN:          config.ResolverASN,
+		SoftwareName:         "probe-engine",
+		SoftwareVersion:      "0.1.0",
+		TestKeys: &fakeTestKeys{
+			ClientResolver: "91.80.37.104",
+			Body: fmt.Sprintf(`
+				<HTML><HEAD><TITLE>Your IP is %s</TITLE></HEAD>
+				<BODY><P>Hey you, I see your IP and it's %s!</P></BODY>
+			`, config.ProbeIP, config.ProbeIP),
+		},
+		TestName:           "dummy",
+		MeasurementRuntime: 5.0565230846405,
+		TestStartTime:      "2018-11-01 15:33:17",
+		TestVersion:        "0.1.0",
+	}
+}
+
+func TestScrubWeAreScrubbing(t *testing.T) {
+	config := makeMeasurementConfig{
+		ProbeIP:             "130.192.91.211",
+		ProbeASN:            "AS137",
+		ProbeCC:             "IT",
+		ProbeNetworkName:    "Vodafone Italia S.p.A.",
+		ResolverIP:          "8.8.8.8",
+		ResolverNetworkName: "Google LLC",
+		ResolverASN:         "AS12345",
+	}
+	m := makeMeasurement(config)
+	if err := m.Scrub(config.ProbeIP); err != nil {
+		t.Fatal(err)
+	}
+	if m.ProbeASN != config.ProbeASN {
+		t.Fatal("ProbeASN has been scrubbed")
+	}
+	if m.ProbeCC != config.ProbeCC {
+		t.Fatal("ProbeCC has been scrubbed")
+	}
+	if m.ProbeIP == config.ProbeIP {
+		t.Fatal("ProbeIP HAS NOT been scrubbed")
+	}
+	if m.ProbeNetworkName != config.ProbeNetworkName {
+		t.Fatal("ProbeNetworkName has been scrubbed")
+	}
+	if m.ResolverIP != config.ResolverIP {
+		t.Fatal("ResolverIP has been scrubbed")
+	}
+	if m.ResolverNetworkName != config.ResolverNetworkName {
+		t.Fatal("ResolverNetworkName has been scrubbed")
+	}
+	if m.ResolverASN != config.ResolverASN {
+		t.Fatal("ResolverASN has been scrubbed")
+	}
+	data, err := json.Marshal(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Count(data, []byte(config.ProbeIP)) != 0 {
+		t.Fatal("ProbeIP not fully redacted")
+	}
+}
+
+func TestScrubNoScrubbingRequired(t *testing.T) {
+	config := makeMeasurementConfig{
+		ProbeIP:             "130.192.91.211",
+		ProbeASN:            "AS137",
+		ProbeCC:             "IT",
+		ProbeNetworkName:    "Vodafone Italia S.p.A.",
+		ResolverIP:          "8.8.8.8",
+		ResolverNetworkName: "Google LLC",
+		ResolverASN:         "AS12345",
+	}
+	m := makeMeasurement(config)
+	m.TestKeys.(*fakeTestKeys).Body = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
+	if err := m.Scrub(config.ProbeIP); err != nil {
+		t.Fatal(err)
+	}
+	if m.ProbeASN != config.ProbeASN {
+		t.Fatal("ProbeASN has been scrubbed")
+	}
+	if m.ProbeCC != config.ProbeCC {
+		t.Fatal("ProbeCC has been scrubbed")
+	}
+	if m.ProbeIP == config.ProbeIP {
+		t.Fatal("ProbeIP HAS NOT been scrubbed")
+	}
+	if m.ProbeNetworkName != config.ProbeNetworkName {
+		t.Fatal("ProbeNetworkName has been scrubbed")
+	}
+	if m.ResolverIP != config.ResolverIP {
+		t.Fatal("ResolverIP has been scrubbed")
+	}
+	if m.ResolverNetworkName != config.ResolverNetworkName {
+		t.Fatal("ResolverNetworkName has been scrubbed")
+	}
+	if m.ResolverASN != config.ResolverASN {
+		t.Fatal("ResolverASN has been scrubbed")
+	}
+	data, err := json.Marshal(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Count(data, []byte(model.Scrubbed)) > 0 {
+		t.Fatal("We should not see any scrubbing")
+	}
+}
+
+func TestScrubInvalidIP(t *testing.T) {
+	m := &model.Measurement{
+		ProbeASN: "AS1234",
+		ProbeCC:  "IT",
+	}
+	err := m.Scrub("") // invalid IP
+	if !errors.Is(err, model.ErrInvalidProbeIP) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestScrubMarshalError(t *testing.T) {
+	expected := errors.New("mocked error")
+	m := &model.Measurement{
+		ProbeASN: "AS1234",
+		ProbeCC:  "IT",
+	}
+	err := m.MaybeRewriteTestKeys(
+		"8.8.8.8", func(v interface{}) ([]byte, error) {
+			return nil, expected
+		})
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestDiscardLoggerWorksAsIntended(t *testing.T) {
+	logger := model.DiscardLogger
+	logger.Debug("foo")
+	logger.Debugf("%s", "foo")
+	logger.Info("foo")
+	logger.Infof("%s", "foo")
+	logger.Warn("foo")
+	logger.Warnf("%s", "foo")
+}

internal/engine/model/scrub.go

@@ -0,0 +1,49 @@
+package model
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"net"
+)
+
+// ErrInvalidProbeIP indicates that we're dealing with a string that
+// is not the valid serialization of an IP address.
+var ErrInvalidProbeIP = errors.New("model: invalid probe IP")
+
+// Scrub scrubs the probeIP out of the measurement.
+func (m *Measurement) Scrub(probeIP string) (err error) {
+	// We now behave like we can share everything except the
+	// probe IP, which we instead cannot ever share
+	m.ProbeIP = DefaultProbeIP
+	return m.MaybeRewriteTestKeys(probeIP, json.Marshal)
+}
+
+// Scrubbed is the string that replaces IP addresses.
+const Scrubbed = `[scrubbed]`
+
+// MaybeRewriteTestKeys is the function called by Scrub that
+// ensures that m's serialization doesn't include the IP
+func (m *Measurement) MaybeRewriteTestKeys(
+	currentIP string, marshal func(interface{}) ([]byte, error)) error {
+	if net.ParseIP(currentIP) == nil {
+		return ErrInvalidProbeIP
+	}
+	data, err := marshal(m.TestKeys)
+	if err != nil {
+		return err
+	}
+	// The check using Count is to save an unnecessary copy performed by
+	// ReplaceAll when there are no matches into the body. This is what
+	// we would like the common case to be, meaning that the code has done
+	// its job correctly and has not leaked the IP.
+	bpip := []byte(currentIP)
+	if bytes.Count(data, bpip) <= 0 {
+		return nil
+	}
+	data = bytes.ReplaceAll(data, bpip, []byte(Scrubbed))
+	// We add an annotation such that hopefully later we can measure the
+	// number of cases where we failed to sanitize properly.
+	m.AddAnnotation("_probe_engine_sanitize_test_keys", "true")
+	return json.Unmarshal(data, &m.TestKeys)
+}

internal/engine/model/service.go

@@ -0,0 +1,17 @@
+package model
+
+// Service describes a backend service.
+//
+// The fields of this struct have the meaning described in v2.0.0 of the OONI
+// bouncer specification defined by
+// https://github.com/ooni/spec/blob/master/backends/bk-004-bouncer.md.
+type Service struct {
+	// Address is the address of the server.
+	Address string `json:"address"`
+
+	// Type is the type of the service.
+	Type string `json:"type"`
+
+	// Front is the front to use with "cloudfront" type entries.
+	Front string `json:"front,omitempty"`
+}

internal/engine/model/tortarget.go

@@ -0,0 +1,21 @@
+package model
+
+// TorTarget is a target for the tor experiment.
+type TorTarget struct {
+	// Address is the address of the target.
+	Address string `json:"address"`
+
+	// Name is the name of the target.
+	Name string `json:"name"`
+
+	// Params contains optional params for, e.g., pluggable transports.
+	Params map[string][]string `json:"params"`
+
+	// Protocol is the protocol to use with the target.
+	Protocol string `json:"protocol"`
+
+	// Source is the source from which we fetched this specific
+	// target. Whenever the source is non-empty, we will treat
+	// this specific target as a private target.
+	Source string `json:"source"`
+}

internal/engine/model/urlinfo.go

@@ -0,0 +1,8 @@
+package model
+
+// URLInfo contains info on a test lists URL
+type URLInfo struct {
+	CategoryCode string `json:"category_code"`
+	CountryCode  string `json:"country_code"`
+	URL          string `json:"url"`
+}

internal/engine/model/urllistconfig.go

@@ -0,0 +1,8 @@
+package model
+
+// URLListConfig contains configuration for fetching the URL list.
+type URLListConfig struct {
+	Categories  []string // Categories to query for (empty means all)
+	CountryCode string   // CountryCode is the optional country code
+	Limit       int64    // Max number of URLs (<= 0 means no limit)
+}