feat(torsf): experiment that bootstraps tor using snowflake (#387)

The current implementation assumes the user has already installed tor
on the current system. If tor is not present, the experiment fails.

This is meant to be the first version of this experiment.

We are going to add more functionality in subsequent revisions of
this experiment, once we've collected more feedback.

Reference issue: https://github.com/ooni/probe/issues/1565.

Here's the spec PR: https://github.com/ooni/spec/pull/218.

Here's the issue tracking future work: https://github.com/ooni/probe/issues/1686

.gitignore

@@ -17,6 +17,8 @@
 /ooniprobe_checksums.txt.asc
 /ooniprobe.exe
 /probe-cli.cov
+/ptxclient
+/ptxclient.exe
 /*.tar.gz
 /testdata/gotmp
 /*.zip

internal/engine/allexperiments.go

@@ -20,6 +20,7 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram"
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool"
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tor"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf"
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp"
@@ -289,6 +290,18 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
 		}
 	},
 
+	"torsf": func(session *Session) *ExperimentBuilder {
+		return &ExperimentBuilder{
+			build: func(config interface{}) *Experiment {
+				return NewExperiment(session, torsf.NewExperimentMeasurer(
+					*config.(*torsf.Config),
+				))
+			},
+			config:      &torsf.Config{},
+			inputPolicy: InputNone,
+		}
+	},
+
 	"urlgetter": func(session *Session) *ExperimentBuilder {
 		return &ExperimentBuilder{
 			build: func(config interface{}) *Experiment {

internal/engine/experiment/torsf/integration_test.go

@@ -0,0 +1,40 @@
+package torsf_test
+
+import (
+	"context"
+	"io/ioutil"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"golang.org/x/sys/execabs"
+)
+
+func TestRunWithExistingTor(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	path, err := execabs.LookPath("tor")
+	if err != nil {
+		t.Skip("there is no tor executable installed")
+	}
+	t.Log("found tor in path:", path)
+	tempdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("using this tempdir", tempdir)
+	m := torsf.NewExperimentMeasurer(torsf.Config{})
+	ctx := context.Background()
+	measurement := &model.Measurement{}
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	sess := &mockable.Session{
+		MockableLogger:  log.Log,
+		MockableTempDir: tempdir,
+	}
+	if err = m.Run(ctx, sess, measurement, callbacks); err != nil {
+		t.Fatal(err)
+	}
+}

internal/engine/experiment/torsf/torsf.go

@@ -0,0 +1,173 @@
+// Package torsf contains the torsf experiment. This experiment
+// measures the bootstrapping of tor using snowflake.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-030-torsf.md
+package torsf
+
+import (
+	"context"
+	"path"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+	"github.com/ooni/probe-cli/v3/internal/ptx"
+	"github.com/ooni/probe-cli/v3/internal/tunnel"
+)
+
+// testVersion is the tor experiment version.
+const testVersion = "0.1.0"
+
+// Config contains the experiment config.
+type Config struct{}
+
+// TestKeys contains the experiment's result.
+type TestKeys struct {
+	// BootstrapTime contains the bootstrap time on success.
+	BootstrapTime float64 `json:"bootstrap_time"`
+
+	// Failure contains the failure string or nil.
+	Failure *string `json:"failure"`
+}
+
+// Measurer performs the measurement.
+type Measurer struct {
+	// config contains the experiment settings.
+	config Config
+
+	// mockStartListener is an optional function that allows us to override
+	// the function we actually use to start the ptx listener.
+	mockStartListener func() error
+
+	// mockStartTunnel is an optional function that allows us to override the
+	// default tunnel.Start function used to start a tunnel.
+	mockStartTunnel func(ctx context.Context, config *tunnel.Config) (tunnel.Tunnel, error)
+}
+
+// ExperimentName implements model.ExperimentMeasurer.ExperimentName.
+func (m *Measurer) ExperimentName() string {
+	return "torsf"
+}
+
+// ExperimentVersion implements model.ExperimentMeasurer.ExperimentVersion.
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// registerExtensions registers the extensions used by this experiment.
+func (m *Measurer) registerExtensions(measurement *model.Measurement) {
+	// currently none
+}
+
+// Run runs the experiment with the specified context, session,
+// measurement, and experiment calbacks. This method should only
+// return an error in case the experiment could not run (e.g.,
+// a required input is missing). Otherwise, the code should just
+// set the relevant OONI error inside of the measurement and
+// return nil. This is important because the caller may not submit
+// the measurement if this method returns an error.
+func (m *Measurer) Run(
+	ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
+) error {
+	m.registerExtensions(measurement)
+	testkeys := &TestKeys{}
+	measurement.TestKeys = testkeys
+	start := time.Now()
+	const maxRuntime = 300 * time.Second
+	ctx, cancel := context.WithTimeout(ctx, maxRuntime)
+	defer cancel()
+	errch := make(chan error)
+	ticker := time.NewTicker(250 * time.Millisecond)
+	defer ticker.Stop()
+	go m.run(ctx, sess, testkeys, errch)
+	for {
+		select {
+		case err := <-errch:
+			callbacks.OnProgress(1.0, "torsf experiment is finished")
+			return err
+		case <-ticker.C:
+			progress := time.Since(start).Seconds() / maxRuntime.Seconds()
+			callbacks.OnProgress(progress, "torsf experiment is running")
+		}
+	}
+}
+
+// run runs the bootstrap. This function ONLY returns an error when
+// there has been a fundamental error starting the test. This behavior
+// follows the expectations for the ExperimentMeasurer.Run method.
+func (m *Measurer) run(ctx context.Context,
+	sess model.ExperimentSession, testkeys *TestKeys, errch chan<- error) {
+	sfdialer := &ptx.SnowflakeDialer{}
+	ptl := &ptx.Listener{
+		PTDialer: sfdialer,
+		Logger:   sess.Logger(),
+	}
+	if err := m.startListener(ptl.Start); err != nil {
+		testkeys.Failure = archival.NewFailure(err)
+		// This error condition mostly means "I could not open a local
+		// listening port", which strikes as fundamental failure.
+		errch <- err
+		return
+	}
+	defer ptl.Stop()
+	tun, err := m.startTunnel()(ctx, &tunnel.Config{
+		Name:      "tor",
+		Session:   sess,
+		TunnelDir: path.Join(sess.TempDir(), "torsf"),
+		Logger:    sess.Logger(),
+		TorArgs: []string{
+			"UseBridges", "1",
+			"ClientTransportPlugin", ptl.AsClientTransportPluginArgument(),
+			"Bridge", sfdialer.AsBridgeArgument(),
+		},
+	})
+	if err != nil {
+		// Note: archival.NewFailure scrubs IP addresses
+		testkeys.Failure = archival.NewFailure(err)
+		// This error condition means we could not bootstrap with snowflake
+		// for $reasons, so the experiment didn't fail, rather it did record
+		// that something prevented snowflake from running.
+		errch <- nil
+		return
+	}
+	defer tun.Stop()
+	testkeys.BootstrapTime = tun.BootstrapTime().Seconds()
+	errch <- nil
+}
+
+// startListener either calls f or mockStartListener depending
+// on whether mockStartListener is nil or not.
+func (m *Measurer) startListener(f func() error) error {
+	if m.mockStartListener != nil {
+		return m.mockStartListener()
+	}
+	return f()
+}
+
+// startTunnel returns the proper function to start a tunnel.
+func (m *Measurer) startTunnel() func(
+	ctx context.Context, config *tunnel.Config) (tunnel.Tunnel, error) {
+	if m.mockStartTunnel != nil {
+		return m.mockStartTunnel
+	}
+	return tunnel.Start
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return &Measurer{config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return &SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/torsf/torsf_test.go

@@ -0,0 +1,167 @@
+package torsf
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/tunnel"
+)
+
+func TestExperimentNameAndVersion(t *testing.T) {
+	m := NewExperimentMeasurer(Config{})
+	if m.ExperimentName() != "torsf" {
+		t.Fatal("invalid experiment name")
+	}
+	if m.ExperimentVersion() != "0.1.0" {
+		t.Fatal("invalid experiment version")
+	}
+}
+
+// mockedTunnel is a mocked tunnel.
+type mockedTunnel struct {
+	bootstrapTime time.Duration
+	proxyURL      *url.URL
+}
+
+// BootstrapTime implements Tunnel.BootstrapTime.
+func (mt *mockedTunnel) BootstrapTime() time.Duration {
+	return mt.bootstrapTime
+}
+
+// SOCKS5ProxyURL implements Tunnel.SOCKS5ProxyURL.
+func (mt *mockedTunnel) SOCKS5ProxyURL() *url.URL {
+	return mt.proxyURL
+}
+
+// Stop implements Tunnel.Stop.
+func (mt *mockedTunnel) Stop() {
+	// nothing
+}
+
+func TestSuccessWithMockedTunnelStart(t *testing.T) {
+	bootstrapTime := 400 * time.Millisecond
+	m := &Measurer{
+		config: Config{},
+		mockStartTunnel: func(ctx context.Context, config *tunnel.Config) (tunnel.Tunnel, error) {
+			// run for some time so we also exercise printing progress.
+			time.Sleep(bootstrapTime)
+			return &mockedTunnel{
+				bootstrapTime: time.Duration(bootstrapTime),
+			}, nil
+		},
+	}
+	ctx := context.Background()
+	measurement := &model.Measurement{}
+	sess := &mockable.Session{}
+	callbacks := &model.PrinterCallbacks{
+		Logger: log.Log,
+	}
+	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*TestKeys)
+	if tk.BootstrapTime != bootstrapTime.Seconds() {
+		t.Fatal("unexpected bootstrap time")
+	}
+}
+
+func TestFailureToStartTunnel(t *testing.T) {
+	expected := errors.New("mocked error")
+	m := &Measurer{
+		config: Config{},
+		mockStartTunnel: func(ctx context.Context, config *tunnel.Config) (tunnel.Tunnel, error) {
+			return nil, expected
+		},
+	}
+	ctx := context.Background()
+	measurement := &model.Measurement{}
+	sess := &mockable.Session{}
+	callbacks := &model.PrinterCallbacks{
+		Logger: log.Log,
+	}
+	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*TestKeys)
+	if tk.BootstrapTime != 0 {
+		t.Fatal("unexpected bootstrap time")
+	}
+	if tk.Failure == nil {
+		t.Fatal("unexpectedly nil failure string")
+	}
+	if *tk.Failure != "unknown_failure: mocked error" {
+		t.Fatal("unexpected failure string", *tk.Failure)
+	}
+}
+
+func TestFailureToStartPTXListener(t *testing.T) {
+	expected := errors.New("mocked error")
+	m := &Measurer{
+		config: Config{},
+		mockStartListener: func() error {
+			return expected
+		},
+	}
+	ctx := context.Background()
+	measurement := &model.Measurement{}
+	sess := &mockable.Session{}
+	callbacks := &model.PrinterCallbacks{
+		Logger: log.Log,
+	}
+	if err := m.Run(ctx, sess, measurement, callbacks); !errors.Is(err, expected) {
+		t.Fatal("not the error we expected", err)
+	}
+	tk := measurement.TestKeys.(*TestKeys)
+	if tk.BootstrapTime != 0 {
+		t.Fatal("unexpected bootstrap time")
+	}
+	if tk.Failure == nil {
+		t.Fatal("unexpectedly nil failure string")
+	}
+	if *tk.Failure != "unknown_failure: mocked error" {
+		t.Fatal("unexpected failure string", *tk.Failure)
+	}
+}
+
+func TestStartWithCancelledContext(t *testing.T) {
+	m := &Measurer{config: Config{}}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	measurement := &model.Measurement{}
+	sess := &mockable.Session{}
+	callbacks := &model.PrinterCallbacks{
+		Logger: log.Log,
+	}
+	if err := m.Run(ctx, sess, measurement, callbacks); err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*TestKeys)
+	if tk.BootstrapTime != 0 {
+		t.Fatal("unexpected bootstrap time")
+	}
+	if tk.Failure == nil {
+		t.Fatal("unexpected nil failure")
+	}
+	if *tk.Failure != "interrupted" {
+		t.Fatal("unexpected failure string", *tk.Failure)
+	}
+}
+
+func TestGetSummaryKeys(t *testing.T) {
+	measurement := &model.Measurement{}
+	m := &Measurer{}
+	sk, err := m.GetSummaryKeys(measurement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rsk := sk.(*SummaryKeys)
+	if rsk.IsAnomaly {
+		t.Fatal("expected no anomaly here")
+	}
+}

internal/engine/model/experiment.go

@@ -55,7 +55,7 @@ type ExperimentMeasurer interface {
 	// measurement, and experiment calbacks. This method should only
 	// return an error in case the experiment could not run (e.g.,
 	// a required input is missing). Otherwise, the code should just
-	// set the relevant OONI error inside of the measurmeent and
+	// set the relevant OONI error inside of the measurement and
 	// return nil. This is important because the caller may not submit
 	// the measurement if this method returns an error.
 	Run(