chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/experiment/run/dnscheck.go

@@ -0,0 +1,27 @@
+package run
+
+import (
+	"context"
+	"sync"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type dnsCheckMain struct {
+	Endpoints *dnscheck.Endpoints
+	mu        sync.Mutex
+}
+
+func (m *dnsCheckMain) do(ctx context.Context, input StructuredInput,
+	sess model.ExperimentSession, measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks) error {
+	exp := dnscheck.Measurer{
+		Config:    input.DNSCheck,
+		Endpoints: m.Endpoints,
+	}
+	measurement.TestName = exp.ExperimentName()
+	measurement.TestVersion = exp.ExperimentVersion()
+	measurement.Input = model.MeasurementTarget(input.Input)
+	return exp.Run(ctx, sess, measurement, callbacks)
+}

internal/engine/experiment/run/run.go

@@ -0,0 +1,79 @@
+// Package run contains code to run other experiments.
+//
+// This code is currently alpha.
+package run
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// Config contains settings.
+type Config struct{}
+
+// Measurer runs the measurement.
+type Measurer struct{}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName.
+func (Measurer) ExperimentName() string {
+	return "run"
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (Measurer) ExperimentVersion() string {
+	return "0.2.0"
+}
+
+// StructuredInput contains structured input for this experiment.
+type StructuredInput struct {
+	// Annotations contains extra annotations to add to the
+	// final measurement.
+	Annotations map[string]string `json:"annotations"`
+
+	// DNSCheck contains settings for the dnscheck experiment.
+	DNSCheck dnscheck.Config `json:"dnscheck"`
+
+	// URLGetter contains settings for the urlgetter experiment.
+	URLGetter urlgetter.Config `json:"urlgetter"`
+
+	// Name is the name of the experiment to run.
+	Name string `json:"name"`
+
+	// Input is the input for this experiment.
+	Input string `json:"input"`
+}
+
+// Run implements ExperimentMeasurer.ExperimentVersion.
+func (Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	var input StructuredInput
+	if err := json.Unmarshal([]byte(measurement.Input), &input); err != nil {
+		return err
+	}
+	exprun, found := table[input.Name]
+	if !found {
+		return fmt.Errorf("no such experiment: %s", input.Name)
+	}
+	measurement.AddAnnotations(input.Annotations)
+	return exprun.do(ctx, input, sess, measurement, callbacks)
+}
+
+// GetSummaryKeys implements ExperimentMeasurer.GetSummaryKeys
+func (Measurer) GetSummaryKeys(*model.Measurement) (interface{}, error) {
+	// TODO(bassosimone): we could extend this interface to call the
+	// specific GetSummaryKeys of the experiment we're running.
+	return dnscheck.SummaryKeys{IsAnomaly: false}, nil
+}
+
+// NewExperimentMeasurer creates a new model.ExperimentMeasurer
+// implementing the run experiment.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{}
+}

internal/engine/experiment/run/run_test.go

@@ -0,0 +1,107 @@
+package run_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/run"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+func TestExperimentNameAndVersion(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	if measurer.ExperimentName() != "run" {
+		t.Error("unexpected experiment name")
+	}
+	if measurer.ExperimentVersion() != "0.2.0" {
+		t.Error("unexpected experiment version")
+	}
+}
+
+func TestRunDNSCheckWithCancelledContext(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	input := `{"name": "dnscheck", "input": "https://dns.google/dns-query"}`
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	// TODO(bassosimone): here we could improve the tests by checking
+	// whether the result makes sense for a cancelled context.
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := measurement.TestKeys.(*dnscheck.TestKeys); !ok {
+		t.Fatal("invalid type for test keys")
+	}
+	sk, err := measurer.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rsk, ok := sk.(dnscheck.SummaryKeys)
+	if !ok {
+		t.Fatal("cannot convert summary keys to specific type")
+	}
+	if rsk.IsAnomaly != false {
+		t.Fatal("unexpected IsAnomaly value")
+	}
+}
+
+func TestRunURLGetterWithCancelledContext(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	input := `{"name": "urlgetter", "input": "https://google.com"}`
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err == nil {
+		t.Fatal(err)
+	}
+	if len(measurement.Extensions) != 6 {
+		t.Fatal("not the expected number of extensions")
+	}
+	tk, ok := measurement.TestKeys.(*urlgetter.TestKeys)
+	if !ok {
+		t.Fatal("invalid type for test keys")
+	}
+	if len(tk.DNSCache) != 0 {
+		t.Fatal("not the DNSCache value we expected")
+	}
+}
+
+func TestRunWithInvalidJSON(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	input := `{"name": }`
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	ctx := context.Background()
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err == nil || err.Error() != "invalid character '}' looking for beginning of value" {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+}
+
+func TestRunWithUnknownExperiment(t *testing.T) {
+	measurer := run.NewExperimentMeasurer(run.Config{})
+	input := `{"name": "antani"}`
+	measurement := new(model.Measurement)
+	measurement.Input = model.MeasurementTarget(input)
+	ctx := context.Background()
+	sess := &mockable.Session{MockableLogger: log.Log}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err == nil || err.Error() != "no such experiment: antani" {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+}

internal/engine/experiment/run/table.go

@@ -0,0 +1,26 @@
+package run
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type experimentMain interface {
+	do(ctx context.Context, input StructuredInput,
+		sess model.ExperimentSession, measurement *model.Measurement,
+		callbacks model.ExperimentCallbacks) error
+}
+
+var table = map[string]experimentMain{
+	// TODO(bassosimone): before extending run to support more than
+	// single experiment, we need to handle the case in which we are
+	// including different experiments into the same report ID.
+	// Probably, the right way to implement this functionality is to
+	// use proveservices.Submitter to submit reports.
+	"dnscheck": &dnsCheckMain{
+		Endpoints: &dnscheck.Endpoints{},
+	},
+	"urlgetter": &urlGetterMain{},
+}

internal/engine/experiment/run/urlgetter.go

@@ -0,0 +1,22 @@
+package run
+
+import (
+	"context"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+type urlGetterMain struct {}
+
+func (m *urlGetterMain) do(ctx context.Context, input StructuredInput,
+	sess model.ExperimentSession, measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks) error {
+	exp := urlgetter.Measurer{
+		Config: input.URLGetter,
+	}
+	measurement.TestName = exp.ExperimentName()
+	measurement.TestVersion = exp.ExperimentVersion()
+	measurement.Input = model.MeasurementTarget(input.Input)
+	return exp.Run(ctx, sess, measurement, callbacks)
+}