chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/libminiooni/.gitignore

@@ -0,0 +1 @@
+/report.jsonl

internal/engine/libminiooni/README.md

@@ -0,0 +1,14 @@
+# Package github.com/ooni/probe-engine/libminiooni
+
+Package libminiooni implements the cmd/miniooni CLI. Miniooni is our
+experimental client used for research and QA testing.
+
+This CLI has CLI options that do not conflict with Measurement Kit
+v0.10.x CLI options. There are some options conflict with the legacy
+OONI Probe CLI options. Perfect backwards compatibility is not a
+design goal for miniooni. Rather, we aim to have as little conflict
+as possible such that we can run side by side QA checks.
+
+We extracted this package from cmd/miniooni to allow us to further
+integrate the miniooni CLI into other binaries (see for example the
+code at github.com/bassosimone/aladdin).

internal/engine/libminiooni/libminiooni.go

@@ -0,0 +1,442 @@
+// Package libminiooni implements the cmd/miniooni CLI. Miniooni is our
+// experimental client used for research and QA testing.
+//
+// This CLI has CLI options that do not conflict with Measurement Kit
+// v0.10.x CLI options. There are some options conflict with the legacy
+// OONI Probe CLI options. Perfect backwards compatibility is not a
+// design goal for miniooni. Rather, we aim to have as little conflict
+// as possible such that we can run side by side QA checks.
+//
+// We extracted this package from cmd/miniooni to allow us to further
+// integrate the miniooni CLI into other binaries (see for example the
+// code at github.com/bassosimone/aladdin).
+package libminiooni
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/apex/log"
+	engine "github.com/ooni/probe-cli/v3/internal/engine"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
+	"github.com/ooni/probe-cli/v3/internal/engine/version"
+	"github.com/pborman/getopt/v2"
+)
+
+// Options contains the options you can set from the CLI.
+type Options struct {
+	Annotations      []string
+	ExtraOptions     []string
+	HomeDir          string
+	Inputs           []string
+	InputFilePaths   []string
+	NoJSON           bool
+	NoCollector      bool
+	ProbeServicesURL string
+	Proxy            string
+	Random           bool
+	ReportFile       string
+	SelfCensorSpec   string
+	TorArgs          []string
+	TorBinary        string
+	Tunnel           string
+	Verbose          bool
+	Yes              bool
+}
+
+const (
+	softwareName    = "miniooni"
+	softwareVersion = version.Version
+)
+
+var (
+	globalOptions Options
+	startTime     = time.Now()
+)
+
+func init() {
+	getopt.FlagLong(
+		&globalOptions.Annotations, "annotation", 'A', "Add annotaton", "KEY=VALUE",
+	)
+	getopt.FlagLong(
+		&globalOptions.ExtraOptions, "option", 'O',
+		"Pass an option to the experiment", "KEY=VALUE",
+	)
+	getopt.FlagLong(
+		&globalOptions.InputFilePaths, "input-file", 'f',
+		"Path to input file to supply test-dependent input. File must contain one input per line.", "PATH",
+	)
+	getopt.FlagLong(
+		&globalOptions.HomeDir, "home", 0,
+		"Force specific home directory", "PATH",
+	)
+	getopt.FlagLong(
+		&globalOptions.Inputs, "input", 'i',
+		"Add test-dependent input to the test input", "INPUT",
+	)
+	getopt.FlagLong(
+		&globalOptions.NoJSON, "no-json", 'N', "Disable writing to disk",
+	)
+	getopt.FlagLong(
+		&globalOptions.NoCollector, "no-collector", 'n', "Don't use a collector",
+	)
+	getopt.FlagLong(
+		&globalOptions.ProbeServicesURL, "probe-services", 0,
+		"Set the URL of the probe-services instance you want to use", "URL",
+	)
+	getopt.FlagLong(
+		&globalOptions.Proxy, "proxy", 0, "Set the proxy URL", "URL",
+	)
+	getopt.FlagLong(
+		&globalOptions.Random, "random", 0, "Randomize inputs",
+	)
+	getopt.FlagLong(
+		&globalOptions.ReportFile, "reportfile", 'o',
+		"Set the report file path", "PATH",
+	)
+	getopt.FlagLong(
+		&globalOptions.SelfCensorSpec, "self-censor-spec", 0,
+		"Enable and configure self censorship", "JSON",
+	)
+	getopt.FlagLong(
+		&globalOptions.TorArgs, "tor-args", 0,
+		"Extra args for tor binary (may be specified multiple times)",
+	)
+	getopt.FlagLong(
+		&globalOptions.TorBinary, "tor-binary", 0,
+		"Specify path to a specific tor binary",
+	)
+	getopt.FlagLong(
+		&globalOptions.Tunnel, "tunnel", 0,
+		"Name of the tunnel to use (one of `tor`, `psiphon`)",
+	)
+	getopt.FlagLong(
+		&globalOptions.Verbose, "verbose", 'v', "Increase verbosity",
+	)
+	getopt.FlagLong(
+		&globalOptions.Yes, "yes", 0, "I accept the risk of running OONI",
+	)
+}
+
+func fatalWithString(msg string) {
+	panic(msg)
+}
+
+func fatalIfFalse(cond bool, msg string) {
+	if !cond {
+		panic(msg)
+	}
+}
+
+// Main is the main function of miniooni. This function parses the command line
+// options and uses a global state. Use MainWithConfiguration if you want to avoid
+// using any global state and relying on command line options.
+//
+// This function will panic in case of a fatal error. It is up to you that
+// integrate this function to either handle the panic of ignore it.
+func Main() {
+	getopt.Parse()
+	fatalIfFalse(len(getopt.Args()) == 1, "Missing experiment name")
+	MainWithConfiguration(getopt.Arg(0), globalOptions)
+}
+
+func split(s string) (string, string, error) {
+	v := strings.SplitN(s, "=", 2)
+	if len(v) != 2 {
+		return "", "", errors.New("invalid key-value pair")
+	}
+	return v[0], v[1], nil
+}
+
+func fatalOnError(err error, msg string) {
+	if err != nil {
+		log.WithError(err).Warn(msg)
+		panic(msg)
+	}
+}
+
+func warnOnError(err error, msg string) {
+	if err != nil {
+		log.WithError(err).Warn(msg)
+	}
+}
+
+func mustMakeMap(input []string) (output map[string]string) {
+	output = make(map[string]string)
+	for _, opt := range input {
+		key, value, err := split(opt)
+		fatalOnError(err, "cannot split key-value pair")
+		output[key] = value
+	}
+	return
+}
+
+func mustParseURL(URL string) *url.URL {
+	rv, err := url.Parse(URL)
+	fatalOnError(err, "cannot parse URL")
+	return rv
+}
+
+type logHandler struct {
+	io.Writer
+}
+
+func (h *logHandler) HandleLog(e *log.Entry) (err error) {
+	s := fmt.Sprintf("[%14.6f] <%s> %s", time.Since(startTime).Seconds(), e.Level, e.Message)
+	if len(e.Fields) > 0 {
+		s += fmt.Sprintf(": %+v", e.Fields)
+	}
+	s += "\n"
+	_, err = h.Writer.Write([]byte(s))
+	return
+}
+
+// See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da
+func gethomedir(optionsHome string) string {
+	if optionsHome != "" {
+		return optionsHome
+	}
+	if runtime.GOOS == "windows" {
+		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+		if home == "" {
+			home = os.Getenv("USERPROFILE")
+		}
+		return home
+	}
+	if runtime.GOOS == "linux" {
+		home := os.Getenv("XDG_CONFIG_HOME")
+		if home != "" {
+			return home
+		}
+		// fallthrough
+	}
+	return os.Getenv("HOME")
+}
+
+const riskOfRunningOONI = `
+Do you consent to OONI Probe data collection?
+
+OONI Probe collects evidence of internet censorship and measures
+network performance:
+ 
+- OONI Probe will likely test objectionable sites and services;
+ 
+- Anyone monitoring your internet activity (such as a government
+or Internet provider) may be able to tell that you are using OONI Probe;
+ 
+- The network data you collect will be published automatically
+unless you use miniooni's -n command line flag.
+ 
+To learn more, see https://ooni.org/about/risks/.
+
+If you're onboard, re-run the same command and add the --yes flag, to
+indicate that you understand the risks. This will create an empty file
+named 'consent' in $HOME/.miniooni, meaning that we know you opted in
+and we will not ask you this question again.
+
+`
+
+func canOpen(filepath string) bool {
+	stat, err := os.Stat(filepath)
+	return err == nil && stat.Mode().IsRegular()
+}
+
+func maybeWriteConsentFile(yes bool, filepath string) (err error) {
+	if yes {
+		err = ioutil.WriteFile(filepath, []byte("\n"), 0644)
+	}
+	return
+}
+
+// MainWithConfiguration is the miniooni main with a specific configuration
+// represented by the experiment name and the current options.
+//
+// This function will panic in case of a fatal error. It is up to you that
+// integrate this function to either handle the panic of ignore it.
+func MainWithConfiguration(experimentName string, currentOptions Options) {
+	ctx := context.Background()
+
+	extraOptions := mustMakeMap(currentOptions.ExtraOptions)
+	annotations := mustMakeMap(currentOptions.Annotations)
+
+	err := selfcensor.MaybeEnable(currentOptions.SelfCensorSpec)
+	fatalOnError(err, "cannot parse --self-censor-spec argument")
+
+	logger := &log.Logger{Level: log.InfoLevel, Handler: &logHandler{Writer: os.Stderr}}
+	if currentOptions.Verbose {
+		logger.Level = log.DebugLevel
+	}
+	if currentOptions.ReportFile == "" {
+		currentOptions.ReportFile = "report.jsonl"
+	}
+	log.Log = logger
+
+	homeDir := gethomedir(currentOptions.HomeDir)
+	fatalIfFalse(homeDir != "", "home directory is empty")
+	miniooniDir := path.Join(homeDir, ".miniooni")
+	assetsDir := path.Join(miniooniDir, "assets")
+	err = os.MkdirAll(assetsDir, 0700)
+	fatalOnError(err, "cannot create assets directory")
+	log.Debugf("miniooni state directory: %s", miniooniDir)
+
+	consentFile := path.Join(miniooniDir, "informed")
+	fatalOnError(maybeWriteConsentFile(currentOptions.Yes, consentFile),
+		"cannot write informed consent file")
+	fatalIfFalse(canOpen(consentFile), riskOfRunningOONI)
+	log.Info("miniooni home directory: $HOME/.miniooni")
+
+	var proxyURL *url.URL
+	if currentOptions.Proxy != "" {
+		proxyURL = mustParseURL(currentOptions.Proxy)
+	}
+
+	kvstore2dir := filepath.Join(miniooniDir, "kvstore2")
+	kvstore, err := engine.NewFileSystemKVStore(kvstore2dir)
+	fatalOnError(err, "cannot create kvstore2 directory")
+
+	config := engine.SessionConfig{
+		AssetsDir:       assetsDir,
+		KVStore:         kvstore,
+		Logger:          logger,
+		ProxyURL:        proxyURL,
+		SoftwareName:    softwareName,
+		SoftwareVersion: softwareVersion,
+		TorArgs:         currentOptions.TorArgs,
+		TorBinary:       currentOptions.TorBinary,
+	}
+	if currentOptions.ProbeServicesURL != "" {
+		config.AvailableProbeServices = []model.Service{{
+			Address: currentOptions.ProbeServicesURL,
+			Type:    "https",
+		}}
+	}
+
+	sess, err := engine.NewSession(config)
+	fatalOnError(err, "cannot create measurement session")
+	defer func() {
+		sess.Close()
+		log.Infof("whole session: recv %s, sent %s",
+			humanizex.SI(sess.KibiBytesReceived()*1024, "byte"),
+			humanizex.SI(sess.KibiBytesSent()*1024, "byte"),
+		)
+	}()
+	log.Debugf("miniooni temporary directory: %s", sess.TempDir())
+
+	err = sess.MaybeStartTunnel(context.Background(), currentOptions.Tunnel)
+	fatalOnError(err, "cannot start session tunnel")
+
+	log.Info("Looking up OONI backends; please be patient...")
+	err = sess.MaybeLookupBackends()
+	fatalOnError(err, "cannot lookup OONI backends")
+	log.Info("Looking up your location; please be patient...")
+	err = sess.MaybeLookupLocation()
+	fatalOnError(err, "cannot lookup your location")
+	log.Debugf("- IP: %s", sess.ProbeIP())
+	log.Infof("- country: %s", sess.ProbeCC())
+	log.Infof("- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString())
+	log.Infof("- resolver's IP: %s", sess.ResolverIP())
+	log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(),
+		sess.ResolverASNString())
+
+	builder, err := sess.NewExperimentBuilder(experimentName)
+	fatalOnError(err, "cannot create experiment builder")
+
+	inputLoader := engine.NewInputLoader(engine.InputLoaderConfig{
+		StaticInputs: currentOptions.Inputs,
+		SourceFiles:  currentOptions.InputFilePaths,
+		InputPolicy:  builder.InputPolicy(),
+		Session:      sess,
+		URLLimit:     17,
+	})
+	inputs, err := inputLoader.Load(context.Background())
+	fatalOnError(err, "cannot load inputs")
+
+	if currentOptions.Random {
+		rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+		rnd.Shuffle(len(inputs), func(i, j int) {
+			inputs[i], inputs[j] = inputs[j], inputs[i]
+		})
+	}
+
+	err = builder.SetOptionsGuessType(extraOptions)
+	fatalOnError(err, "cannot parse extraOptions")
+
+	experiment := builder.NewExperiment()
+	defer func() {
+		log.Infof("experiment: recv %s, sent %s",
+			humanizex.SI(experiment.KibiBytesReceived()*1024, "byte"),
+			humanizex.SI(experiment.KibiBytesSent()*1024, "byte"),
+		)
+	}()
+
+	submitter, err := engine.NewSubmitter(ctx, engine.SubmitterConfig{
+		Enabled: currentOptions.NoCollector == false,
+		Session: sess,
+		Logger:  log.Log,
+	})
+	fatalOnError(err, "cannot create submitter")
+
+	saver, err := engine.NewSaver(engine.SaverConfig{
+		Enabled:    currentOptions.NoJSON == false,
+		Experiment: experiment,
+		FilePath:   currentOptions.ReportFile,
+		Logger:     log.Log,
+	})
+	fatalOnError(err, "cannot create saver")
+
+	inputProcessor := engine.InputProcessor{
+		Annotations: annotations,
+		Experiment: &experimentWrapper{
+			child: engine.NewInputProcessorExperimentWrapper(experiment),
+			total: len(inputs),
+		},
+		Inputs:  inputs,
+		Options: currentOptions.ExtraOptions,
+		Saver:   engine.NewInputProcessorSaverWrapper(saver),
+		Submitter: submitterWrapper{
+			child: engine.NewInputProcessorSubmitterWrapper(submitter),
+		},
+	}
+	err = inputProcessor.Run(ctx)
+	fatalOnError(err, "inputProcessor.Run failed")
+}
+
+type experimentWrapper struct {
+	child engine.InputProcessorExperimentWrapper
+	total int
+}
+
+func (ew *experimentWrapper) MeasureWithContext(
+	ctx context.Context, idx int, input string) (*model.Measurement, error) {
+	if input != "" {
+		log.Infof("[%d/%d] running with input: %s", idx+1, ew.total, input)
+	}
+	measurement, err := ew.child.MeasureWithContext(ctx, idx, input)
+	warnOnError(err, "measurement failed")
+	// policy: we do not stop the loop if the measurement fails
+	return measurement, nil
+}
+
+type submitterWrapper struct {
+	child engine.InputProcessorSubmitterWrapper
+}
+
+func (sw submitterWrapper) Submit(ctx context.Context, idx int, m *model.Measurement) error {
+	err := sw.child.Submit(ctx, idx, m)
+	warnOnError(err, "submitting measurement failed")
+	// policy: we do not stop the loop if measurement submission fails
+	return nil
+}

internal/engine/libminiooni/libminiooni_integration_test.go

@@ -0,0 +1,16 @@
+package libminiooni_test
+
+import (
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/libminiooni"
+)
+
+func TestSimple(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	libminiooni.MainWithConfiguration("example", libminiooni.Options{
+		Yes: true,
+	})
+}