fix: import path should be github.com/ooni/probe-cli/v3 (#200)

See https://github.com/ooni/probe/issues/1335#issuecomment-771499511

cmd/ooniprobe/internal/autorun/autorun.go

@@ -0,0 +1,49 @@
+// Package autorun contains code to manage automatic runs
+package autorun
+
+import "sync"
+
+const (
+	// StatusScheduled indicates that OONI is scheduled to run
+	// periodically in the background.
+	StatusScheduled = "scheduled"
+
+	// StatusStopped indicates that OONI is not scheduled to
+	// run periodically in the background.
+	StatusStopped = "stopped"
+
+	// StatusRunning indicates that OONI is currently
+	// running in the background.
+	StatusRunning = "running"
+)
+
+// Manager manages automatic runs
+type Manager interface {
+	LogShow() error
+	LogStream() error
+	Start() error
+	Status() (string, error)
+	Stop() error
+}
+
+var (
+	registry map[string]Manager
+	mtx      sync.Mutex
+)
+
+func register(platform string, manager Manager) {
+	defer mtx.Unlock()
+	mtx.Lock()
+	if registry == nil {
+		registry = make(map[string]Manager)
+	}
+	registry[platform] = manager
+}
+
+// Get gets the specified autorun manager. This function
+// returns nil if no autorun manager exists.
+func Get(platform string) Manager {
+	defer mtx.Unlock()
+	mtx.Lock()
+	return registry[platform]
+}

cmd/ooniprobe/internal/autorun/autorun_darwin.go

@@ -0,0 +1,178 @@
+package autorun
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+	"github.com/ooni/probe-engine/cmd/jafar/shellx"
+	"golang.org/x/sys/unix"
+)
+
+type managerDarwin struct{}
+
+var (
+	plistPath     = os.ExpandEnv("$HOME/Library/LaunchAgents/org.ooni.cli.plist")
+	domainTarget  = fmt.Sprintf("gui/%d", os.Getuid())
+	serviceTarget = fmt.Sprintf("%s/org.ooni.cli", domainTarget)
+)
+
+var plistTemplate = `
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>
+    <string>org.ooni.cli</string>
+    <key>KeepAlive</key>
+    <false/>
+    <key>RunAtLoad</key>
+    <true/>
+    <key>ProgramArguments</key>
+    <array>
+        <string>{{ .Executable }}</string>
+        <string>--log-handler=syslog</string>
+        <string>run</string>
+        <string>unattended</string>
+    </array>
+    <key>StartInterval</key>
+    <integer>3600</integer>
+</dict>
+</plist>
+`
+
+func runQuiteQuietly(name string, arg ...string) error {
+	log.Infof("exec: %s %s", name, strings.Join(arg, " "))
+	return shellx.RunQuiet(name, arg...)
+}
+
+func darwinVersionMajor() (int, error) {
+	out, err := exec.Command("uname", "-r").Output()
+	if err != nil {
+		return 0, err
+	}
+	v := bytes.Split(out, []byte("."))
+	if len(v) != 3 {
+		return 0, errors.New("cannot split version")
+	}
+	major, err := strconv.Atoi(string(v[0]))
+	if err != nil {
+		return 0, err
+	}
+	return major, nil
+}
+
+var errNotImplemented = errors.New(
+	"autorun: command not implemented in this version of macOS")
+
+func (managerDarwin) LogShow() error {
+	major, _ := darwinVersionMajor()
+	if major < 20 /* macOS 11.0 Big Sur */ {
+		return errNotImplemented
+	}
+	return shellx.Run("log", "show", "--info", "--debug",
+		"--process", "ooniprobe", "--style", "compact")
+}
+
+func (managerDarwin) LogStream() error {
+	return shellx.Run("log", "stream", "--style", "compact", "--level",
+		"debug", "--process", "ooniprobe")
+}
+
+func (managerDarwin) mustNotHavePlist() error {
+	log.Infof("exec: test -f %s && already_registered()", plistPath)
+	if utils.FileExists(plistPath) {
+		// This is not atomic. Do we need atomicity here?
+		return errors.New("autorun: service already registered")
+	}
+	return nil
+}
+
+func (managerDarwin) writePlist() error {
+	executable, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	var out bytes.Buffer
+	t := template.Must(template.New("plist").Parse(plistTemplate))
+	in := struct{ Executable string }{Executable: executable}
+	if err := t.Execute(&out, in); err != nil {
+		return err
+	}
+	log.Infof("exec: writePlist(%s)", plistPath)
+	return ioutil.WriteFile(plistPath, out.Bytes(), 0644)
+}
+
+func (managerDarwin) start() error {
+	if err := runQuiteQuietly("launchctl", "enable", serviceTarget); err != nil {
+		return err
+	}
+	return runQuiteQuietly("launchctl", "bootstrap", domainTarget, plistPath)
+}
+
+func (m managerDarwin) Start() error {
+	operations := []func() error{m.mustNotHavePlist, m.writePlist, m.start}
+	for _, op := range operations {
+		if err := op(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (managerDarwin) stop() error {
+	var failure *exec.ExitError
+	err := runQuiteQuietly("launchctl", "bootout", serviceTarget)
+	if errors.As(err, &failure) && failure.ExitCode() == int(unix.ESRCH) {
+		err = nil
+	}
+	return err
+}
+
+func (managerDarwin) removeFile() error {
+	log.Infof("exec: rm -f %s", plistPath)
+	err := os.Remove(plistPath)
+	if errors.Is(err, unix.ENOENT) {
+		err = nil
+	}
+	return err
+}
+
+func (m managerDarwin) Stop() error {
+	operations := []func() error{m.stop, m.removeFile}
+	for _, op := range operations {
+		if err := op(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m managerDarwin) Status() (string, error) {
+	err := runQuiteQuietly("launchctl", "kill", "SIGINFO", serviceTarget)
+	var failure *exec.ExitError
+	if errors.As(err, &failure) {
+		switch failure.ExitCode() {
+		case int(unix.ESRCH):
+			return StatusScheduled, nil
+		case 113: // exit code when there's no plist
+			return StatusStopped, nil
+		}
+	}
+	if err != nil {
+		return "", fmt.Errorf("autorun: unexpected error: %w", err)
+	}
+	return StatusRunning, nil
+}
+
+func init() {
+	register("darwin", managerDarwin{})
+}

cmd/ooniprobe/internal/cli/app/app.go

@@ -0,0 +1,20 @@
+package app
+
+import (
+	"os"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/version"
+)
+
+// Run the app. This is the main app entry point
+func Run() {
+	root.Cmd.Version(version.Version)
+	_, err := root.Cmd.Parse(os.Args[1:])
+	if err != nil {
+		log.WithError(err).Error("failure in main command")
+		os.Exit(2)
+	}
+	return
+}

cmd/ooniprobe/internal/cli/autorun/autorun.go

@@ -0,0 +1,95 @@
+package autorun
+
+import (
+	"errors"
+	"runtime"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/autorun"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/onboard"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+)
+
+var errNotImplemented = errors.New("autorun: not implemented on this platform")
+
+func init() {
+	cmd := root.Command("autorun", "Run automatic tests in the background")
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		probe, err := root.Init()
+		if err != nil {
+			log.Errorf("%s", err)
+			return err
+		}
+		if err := onboard.MaybeOnboarding(probe); err != nil {
+			log.WithError(err).Error("failed to perform onboarding")
+			return err
+		}
+		return nil
+	})
+
+	start := cmd.Command("start", "Start running automatic tests in the background")
+	start.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		if err := svc.Start(); err != nil {
+			return err
+		}
+		log.Info("hint: use 'ooniprobe autorun log stream' to follow logs")
+		return nil
+	})
+
+	stop := cmd.Command("stop", "Stop running automatic tests in the background")
+	stop.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		return svc.Stop()
+	})
+
+	logCmd := cmd.Command("log", "Access background runs logs")
+	stream := logCmd.Command("stream", "Stream background runs logs")
+	stream.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		return svc.LogStream()
+	})
+
+	show := logCmd.Command("show", "Show background runs logs")
+	show.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		return svc.LogShow()
+	})
+
+	status := cmd.Command("status", "Shows autorun instance status")
+	status.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		out, err := svc.Status()
+		if err != nil {
+			return err
+		}
+		log.Infof("status: %s", out)
+		switch out {
+		case autorun.StatusRunning:
+			log.Info("hint: use 'ooniprobe autorun stop' to stop")
+			log.Info("hint: use 'ooniprobe autorun log stream' to follow logs")
+		case autorun.StatusScheduled:
+			log.Info("hint: use 'ooniprobe autorun stop' to stop")
+			log.Info("hint: use 'ooniprobe autorun log show' to see previous logs")
+		case autorun.StatusStopped:
+			log.Info("hint: use 'ooniprobe autorun start' to start")
+		}
+		return nil
+	})
+}

cmd/ooniprobe/internal/cli/geoip/geoip.go

@@ -0,0 +1,57 @@
+package geoip
+
+import (
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
+)
+
+func init() {
+	cmd := root.Command("geoip", "Perform a geoip lookup")
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		return dogeoip(defaultconfig)
+	})
+}
+
+type dogeoipconfig struct {
+	Logger       log.Interface
+	NewProbeCLI  func() (ooni.ProbeCLI, error)
+	SectionTitle func(string)
+}
+
+var defaultconfig = dogeoipconfig{
+	Logger:       log.Log,
+	NewProbeCLI:  root.NewProbeCLI,
+	SectionTitle: output.SectionTitle,
+}
+
+func dogeoip(config dogeoipconfig) error {
+	config.SectionTitle("GeoIP lookup")
+	probeCLI, err := config.NewProbeCLI()
+	if err != nil {
+		return err
+	}
+
+	engine, err := probeCLI.NewProbeEngine()
+	if err != nil {
+		return err
+	}
+	defer engine.Close()
+
+	err = engine.MaybeLookupLocation()
+	if err != nil {
+		return err
+	}
+
+	config.Logger.WithFields(log.Fields{
+		"type":         "table",
+		"asn":          engine.ProbeASNString(),
+		"network_name": engine.ProbeNetworkName(),
+		"country_code": engine.ProbeCC(),
+		"ip":           engine.ProbeIP(),
+	}).Info("Looked up your location")
+
+	return nil
+}

cmd/ooniprobe/internal/cli/geoip/geoip_test.go

@@ -0,0 +1,134 @@
+package geoip
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/oonitest"
+)
+
+func TestNewProbeCLIFailed(t *testing.T) {
+	fo := &oonitest.FakeOutput{}
+	expected := errors.New("mocked error")
+	err := dogeoip(dogeoipconfig{
+		SectionTitle: fo.SectionTitle,
+		NewProbeCLI: func() (ooni.ProbeCLI, error) {
+			return nil, expected
+		},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if len(fo.FakeSectionTitle) != 1 {
+		t.Fatal("invalid section title list size")
+	}
+	if fo.FakeSectionTitle[0] != "GeoIP lookup" {
+		t.Fatal("unexpected string")
+	}
+}
+
+func TestNewProbeEngineFailed(t *testing.T) {
+	fo := &oonitest.FakeOutput{}
+	expected := errors.New("mocked error")
+	cli := &oonitest.FakeProbeCLI{
+		FakeProbeEngineErr: expected,
+	}
+	err := dogeoip(dogeoipconfig{
+		SectionTitle: fo.SectionTitle,
+		NewProbeCLI: func() (ooni.ProbeCLI, error) {
+			return cli, nil
+		},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if len(fo.FakeSectionTitle) != 1 {
+		t.Fatal("invalid section title list size")
+	}
+	if fo.FakeSectionTitle[0] != "GeoIP lookup" {
+		t.Fatal("unexpected string")
+	}
+}
+
+func TestMaybeLookupLocationFailed(t *testing.T) {
+	fo := &oonitest.FakeOutput{}
+	expected := errors.New("mocked error")
+	engine := &oonitest.FakeProbeEngine{
+		FakeMaybeLookupLocation: expected,
+	}
+	cli := &oonitest.FakeProbeCLI{
+		FakeProbeEnginePtr: engine,
+	}
+	err := dogeoip(dogeoipconfig{
+		SectionTitle: fo.SectionTitle,
+		NewProbeCLI: func() (ooni.ProbeCLI, error) {
+			return cli, nil
+		},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if len(fo.FakeSectionTitle) != 1 {
+		t.Fatal("invalid section title list size")
+	}
+	if fo.FakeSectionTitle[0] != "GeoIP lookup" {
+		t.Fatal("unexpected string")
+	}
+}
+
+func TestMaybeLookupLocationSuccess(t *testing.T) {
+	fo := &oonitest.FakeOutput{}
+	engine := &oonitest.FakeProbeEngine{
+		FakeProbeASNString:   "AS30722",
+		FakeProbeCC:          "IT",
+		FakeProbeNetworkName: "Vodafone Italia S.p.A.",
+		FakeProbeIP:          "130.25.90.216",
+	}
+	cli := &oonitest.FakeProbeCLI{
+		FakeProbeEnginePtr: engine,
+	}
+	handler := &oonitest.FakeLoggerHandler{}
+	err := dogeoip(dogeoipconfig{
+		SectionTitle: fo.SectionTitle,
+		NewProbeCLI: func() (ooni.ProbeCLI, error) {
+			return cli, nil
+		},
+		Logger: &log.Logger{
+			Handler: handler,
+			Level:   log.DebugLevel,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fo.FakeSectionTitle) != 1 {
+		t.Fatal("invalid section title list size")
+	}
+	if fo.FakeSectionTitle[0] != "GeoIP lookup" {
+		t.Fatal("unexpected string")
+	}
+	if len(handler.FakeEntries) != 1 {
+		t.Fatal("invalid number of written entries")
+	}
+	entry := handler.FakeEntries[0]
+	if entry.Level != log.InfoLevel {
+		t.Fatal("invalid log level")
+	}
+	if entry.Message != "Looked up your location" {
+		t.Fatal("invalid .Message")
+	}
+	if entry.Fields["asn"].(string) != "AS30722" {
+		t.Fatal("invalid asn")
+	}
+	if entry.Fields["country_code"].(string) != "IT" {
+		t.Fatal("invalid asn")
+	}
+	if entry.Fields["network_name"].(string) != "Vodafone Italia S.p.A." {
+		t.Fatal("invalid asn")
+	}
+	if entry.Fields["ip"].(string) != "130.25.90.216" {
+		t.Fatal("invalid asn")
+	}
+}

cmd/ooniprobe/internal/cli/info/info.go

@@ -0,0 +1,36 @@
+package info
+
+import (
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+)
+
+func init() {
+	cmd := root.Command("info", "Display information about OONI Probe")
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		return doinfo(defaultconfig)
+	})
+}
+
+type doinfoconfig struct {
+	Logger      log.Interface
+	NewProbeCLI func() (ooni.ProbeCLI, error)
+}
+
+var defaultconfig = doinfoconfig{
+	Logger:      log.Log,
+	NewProbeCLI: root.NewProbeCLI,
+}
+
+func doinfo(config doinfoconfig) error {
+	probeCLI, err := config.NewProbeCLI()
+	if err != nil {
+		config.Logger.Errorf("%s", err)
+		return err
+	}
+	config.Logger.WithFields(log.Fields{"path": probeCLI.Home()}).Info("Home")
+	config.Logger.WithFields(log.Fields{"path": probeCLI.TempDir()}).Info("TempDir")
+	return nil
+}

cmd/ooniprobe/internal/cli/info/info_test.go

@@ -0,0 +1,80 @@
+package info
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/oonitest"
+)
+
+func TestNewProbeCLIFailed(t *testing.T) {
+	expected := errors.New("mocked error")
+	handler := &oonitest.FakeLoggerHandler{}
+	err := doinfo(doinfoconfig{
+		NewProbeCLI: func() (ooni.ProbeCLI, error) {
+			return nil, expected
+		},
+		Logger: &log.Logger{
+			Handler: handler,
+			Level:   log.DebugLevel,
+		},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if len(handler.FakeEntries) != 1 {
+		t.Fatal("invalid number of log entries")
+	}
+	entry := handler.FakeEntries[0]
+	if entry.Level != log.ErrorLevel {
+		t.Fatal("invalid log level")
+	}
+	if entry.Message != "mocked error" {
+		t.Fatal("invalid .Message")
+	}
+}
+
+func TestSuccess(t *testing.T) {
+	handler := &oonitest.FakeLoggerHandler{}
+	cli := &oonitest.FakeProbeCLI{
+		FakeHome:    "fakehome",
+		FakeTempDir: "faketempdir",
+	}
+	err := doinfo(doinfoconfig{
+		NewProbeCLI: func() (ooni.ProbeCLI, error) {
+			return cli, nil
+		},
+		Logger: &log.Logger{
+			Handler: handler,
+			Level:   log.DebugLevel,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(handler.FakeEntries) != 2 {
+		t.Fatal("invalid number of log entries")
+	}
+	entry := handler.FakeEntries[0]
+	if entry.Level != log.InfoLevel {
+		t.Fatal("invalid log level")
+	}
+	if entry.Message != "Home" {
+		t.Fatal("invalid .Message")
+	}
+	if entry.Fields["path"].(string) != "fakehome" {
+		t.Fatal("invalid path")
+	}
+	entry = handler.FakeEntries[1]
+	if entry.Level != log.InfoLevel {
+		t.Fatal("invalid log level")
+	}
+	if entry.Message != "TempDir" {
+		t.Fatal("invalid .Message")
+	}
+	if entry.Fields["path"].(string) != "faketempdir" {
+		t.Fatal("invalid path")
+	}
+}

cmd/ooniprobe/internal/cli/list/list.go

@@ -0,0 +1,128 @@
+package list
+
+import (
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
+)
+
+func init() {
+	cmd := root.Command("list", "List results")
+	resultID := cmd.Arg("id", "the id of the result to list measurements for").Int64()
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		probeCLI, err := root.Init()
+		if err != nil {
+			log.WithError(err).Error("failed to initialize root context")
+			return err
+		}
+		if *resultID > 0 {
+			measurements, err := database.ListMeasurements(probeCLI.DB(), *resultID)
+			if err != nil {
+				log.WithError(err).Error("failed to list measurements")
+				return err
+			}
+			msmtSummary := output.MeasurementSummaryData{
+				TotalCount:         0,
+				AnomalyCount:       0,
+				DataUsageUp:        0.0,
+				DataUsageDown:      0.0,
+				TotalRuntime:       0,
+				ASN:                0,
+				NetworkName:        "",
+				NetworkCountryCode: "ZZ",
+			}
+			isFirst := true
+			isLast := false
+			for idx, msmt := range measurements {
+				if idx > 0 {
+					isFirst = false
+				}
+				if idx == len(measurements)-1 {
+					isLast = true
+				}
+				// We assume that since these are summary level information the first
+				// item will contain the information necessary.
+				if isFirst {
+					msmtSummary.TotalRuntime = msmt.Result.Runtime
+					msmtSummary.DataUsageUp = msmt.DataUsageUp
+					msmtSummary.DataUsageDown = msmt.DataUsageDown
+					msmtSummary.NetworkName = msmt.NetworkName
+					msmtSummary.NetworkCountryCode = msmt.Network.CountryCode
+					msmtSummary.ASN = msmt.ASN
+					msmtSummary.StartTime = msmt.Measurement.StartTime
+				}
+				if msmt.IsAnomaly.Bool == true {
+					msmtSummary.AnomalyCount++
+				}
+				msmtSummary.TotalCount++
+				output.MeasurementItem(msmt, isFirst, isLast)
+			}
+			output.MeasurementSummary(msmtSummary)
+		} else {
+			doneResults, incompleteResults, err := database.ListResults(probeCLI.DB())
+			if err != nil {
+				log.WithError(err).Error("failed to list results")
+				return err
+			}
+			if len(incompleteResults) > 0 {
+				output.SectionTitle("Incomplete results")
+			}
+			for idx, result := range incompleteResults {
+				output.ResultItem(output.ResultItemData{
+					ID:                      result.Result.ID,
+					Index:                   idx,
+					TotalCount:              len(incompleteResults),
+					Name:                    result.TestGroupName,
+					StartTime:               result.StartTime,
+					NetworkName:             result.Network.NetworkName,
+					Country:                 result.Network.CountryCode,
+					ASN:                     result.Network.ASN,
+					MeasurementCount:        0,
+					MeasurementAnomalyCount: 0,
+					TestKeys:                "{}", // FIXME this used to be Summary we probably need to use a list now
+					Done:                    result.IsDone,
+					DataUsageUp:             result.DataUsageUp,
+					DataUsageDown:           result.DataUsageDown,
+				})
+			}
+			resultSummary := output.ResultSummaryData{}
+			netCount := make(map[uint]int)
+			output.SectionTitle("Results")
+			for idx, result := range doneResults {
+				totalCount, anmlyCount, err := database.GetMeasurementCounts(probeCLI.DB(), result.Result.ID)
+				if err != nil {
+					log.WithError(err).Error("failed to list measurement counts")
+				}
+				testKeys, err := database.GetResultTestKeys(probeCLI.DB(), result.Result.ID)
+				if err != nil {
+					log.WithError(err).Error("failed to get testKeys")
+				}
+				output.ResultItem(output.ResultItemData{
+					ID:                      result.Result.ID,
+					Index:                   idx,
+					TotalCount:              len(doneResults),
+					Name:                    result.TestGroupName,
+					StartTime:               result.StartTime,
+					NetworkName:             result.Network.NetworkName,
+					Country:                 result.Network.CountryCode,
+					ASN:                     result.Network.ASN,
+					TestKeys:                testKeys,
+					MeasurementCount:        totalCount,
+					MeasurementAnomalyCount: anmlyCount,
+					Done:                    result.IsDone,
+					DataUsageUp:             result.DataUsageUp,
+					DataUsageDown:           result.DataUsageDown,
+				})
+				resultSummary.TotalTests++
+				netCount[result.Network.ASN]++
+				resultSummary.TotalDataUsageUp += result.DataUsageUp
+				resultSummary.TotalDataUsageDown += result.DataUsageDown
+			}
+			resultSummary.TotalNetworks = int64(len(netCount))
+			output.ResultSummary(resultSummary)
+		}
+		return nil
+	})
+}

cmd/ooniprobe/internal/cli/onboard/onboard.go

@@ -0,0 +1,189 @@
+package onboard
+
+import (
+	"fmt"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/fatih/color"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
+	"github.com/pkg/errors"
+	"gopkg.in/AlecAivazis/survey.v1"
+)
+
+// Onboarding start the interactive onboarding procedure
+func Onboarding(config *config.Config) error {
+	output.SectionTitle("What is OONI Probe?")
+
+	fmt.Println()
+	output.Paragraph("Your tool for detecting internet censorship!")
+	fmt.Println()
+	output.Paragraph("OONI Probe checks whether your provider blocks access to sites and services. Run OONI Probe to collect evidence of internet censorship and to measure your network performance.")
+	fmt.Println()
+	err := output.PressEnterToContinue("Press 'Enter' to continue...")
+	if err != nil {
+		return err
+	}
+
+	output.SectionTitle("Heads Up")
+	fmt.Println()
+	output.Bullet("Anyone monitoring your internet activity (such as your government or ISP) may be able to see that you are running OONI Probe.")
+	fmt.Println()
+	output.Bullet("The network data you will collect will automatically be published (unless you opt-out in the settings).")
+	fmt.Println()
+	output.Bullet("You may test objectionable sites.")
+	fmt.Println()
+	output.Bullet("Read the documentation to learn more.")
+	fmt.Println()
+	err = output.PressEnterToContinue("Press 'Enter' to continue...")
+	if err != nil {
+		return err
+	}
+
+	output.SectionTitle("Pop Quiz!")
+	output.Paragraph("")
+	answer := ""
+	quiz1 := &survey.Select{
+		Message: "Anyone monitoring my internet activity may be able to see that I am running OONI Probe.",
+		Options: []string{"true", "false"},
+		Default: "true",
+	}
+	if err := survey.AskOne(quiz1, &answer, nil); err != nil {
+		return err
+	}
+	if answer != "true" {
+		output.Paragraph(color.RedString("Actually..."))
+		output.Paragraph("OONI Probe is not a privacy tool. Therefore, anyone monitoring your internet activity may be able to see which software you are running.")
+	} else {
+		output.Paragraph(color.BlueString("Good job!"))
+	}
+	answer = ""
+	quiz2 := &survey.Select{
+		Message: "The network data I will collect will automatically be published (unless I opt-out in the settings).",
+		Options: []string{"true", "false"},
+		Default: "true",
+	}
+	if err := survey.AskOne(quiz2, &answer, nil); err != nil {
+		return err
+	}
+	if answer != "true" {
+		output.Paragraph(color.RedString("Actually..."))
+		output.Paragraph("The network data you will collect will automatically be published to increase transparency of internet censorship (unless you opt-out in the settings).")
+	} else {
+		output.Paragraph(color.BlueString("Well done!"))
+	}
+
+	changeDefaults := false
+	prompt := &survey.Confirm{
+		Message: "Do you want to change the default settings?",
+		Default: false,
+	}
+	if err := survey.AskOne(prompt, &changeDefaults, nil); err != nil {
+		return err
+	}
+
+	settings := struct {
+		IncludeIP        bool
+		IncludeNetwork   bool
+		UploadResults    bool
+		SendCrashReports bool
+	}{}
+	settings.IncludeIP = false
+	settings.IncludeNetwork = true
+	settings.UploadResults = true
+	settings.SendCrashReports = true
+
+	if changeDefaults == true {
+		var qs = []*survey.Question{
+			{
+				Name:   "IncludeIP",
+				Prompt: &survey.Confirm{Message: "Should we include your IP?"},
+			},
+			{
+				Name: "IncludeNetwork",
+				Prompt: &survey.Confirm{
+					Message: "Can we include your network name?",
+					Default: true,
+				},
+			},
+			{
+				Name: "UploadResults",
+				Prompt: &survey.Confirm{
+					Message: "Can we upload your results?",
+					Default: true,
+				},
+			},
+			{
+				Name: "SendCrashReports",
+				Prompt: &survey.Confirm{
+					Message: "Can we send crash reports to OONI?",
+					Default: true,
+				},
+			},
+		}
+
+		if err := survey.Ask(qs, &settings); err != nil {
+			log.WithError(err).Error("there was an error in parsing your responses")
+			return err
+		}
+	}
+
+	config.Lock()
+	config.InformedConsent = true
+	config.Advanced.SendCrashReports = settings.SendCrashReports
+	config.Sharing.UploadResults = settings.UploadResults
+	config.Unlock()
+
+	if err := config.Write(); err != nil {
+		log.WithError(err).Error("failed to write config file")
+		return err
+	}
+	return nil
+}
+
+// MaybeOnboarding will run the onboarding process only if the informed consent
+// config option is set to false
+func MaybeOnboarding(probe *ooni.Probe) error {
+	if probe.Config().InformedConsent == false {
+		if probe.IsBatch() == true {
+			return errors.New("cannot run onboarding in batch mode")
+		}
+		if err := Onboarding(probe.Config()); err != nil {
+			return errors.Wrap(err, "onboarding")
+		}
+	}
+	return nil
+}
+
+func init() {
+	cmd := root.Command("onboard", "Starts the onboarding process")
+
+	yes := cmd.Flag("yes", "Answer yes to all the onboarding questions.").Bool()
+
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		probe, err := root.Init()
+		if err != nil {
+			return err
+		}
+
+		if *yes == true {
+			probe.Config().Lock()
+			probe.Config().InformedConsent = true
+			probe.Config().Unlock()
+
+			if err := probe.Config().Write(); err != nil {
+				log.WithError(err).Error("failed to write config file")
+				return err
+			}
+			return nil
+		}
+		if probe.IsBatch() == true {
+			return errors.New("cannot do onboarding in batch mode")
+		}
+
+		return Onboarding(probe.Config())
+	})
+}

cmd/ooniprobe/internal/cli/reset/reset.go

@@ -0,0 +1,37 @@
+package reset
+
+import (
+	"os"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+)
+
+func init() {
+	cmd := root.Command("reset", "Cleanup an old or experimental installation")
+	force := cmd.Flag("force", "Force deleting the OONI Home").Bool()
+
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		ctx, err := root.Init()
+		if err != nil {
+			log.WithError(err).Error("failed to init root context")
+			return err
+		}
+		// We need to first the DB otherwise the DB will be rewritten on close when
+		// we delete the home directory.
+		err = ctx.DB().Close()
+		if err != nil {
+			log.WithError(err).Error("failed to close the DB")
+			return err
+		}
+		if *force == true {
+			os.RemoveAll(ctx.Home())
+			log.Infof("Deleted %s", ctx.Home())
+		} else {
+			log.Infof("Run with --force to delete %s", ctx.Home())
+		}
+
+		return nil
+	})
+}

cmd/ooniprobe/internal/cli/rm/rm.go

@@ -0,0 +1,94 @@
+package rm
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	survey "gopkg.in/AlecAivazis/survey.v1"
+	db "upper.io/db.v3"
+	"upper.io/db.v3/lib/sqlbuilder"
+)
+
+func deleteAll(sess sqlbuilder.Database, skipInteractive bool) error {
+	if skipInteractive == false {
+		answer := ""
+		confirm := &survey.Select{
+			Message: fmt.Sprintf("Are you sure you wish to delete ALL results"),
+			Options: []string{"true", "false"},
+			Default: "false",
+		}
+		survey.AskOne(confirm, &answer, nil)
+		if answer == "false" {
+			return errors.New("canceled by user")
+		}
+	}
+	doneResults, incompleteResults, err := database.ListResults(sess)
+	if err != nil {
+		log.WithError(err).Error("failed to list results")
+		return err
+	}
+	cnt := 0
+	for _, result := range incompleteResults {
+		err = database.DeleteResult(sess, result.Result.ID)
+		if err == db.ErrNoMoreRows {
+			log.WithError(err).Errorf("failed to delete result #%d", result.Result.ID)
+		}
+		cnt++
+	}
+	for _, result := range doneResults {
+		err = database.DeleteResult(sess, result.Result.ID)
+		if err == db.ErrNoMoreRows {
+			log.WithError(err).Errorf("failed to delete result #%d", result.Result.ID)
+		}
+		cnt++
+	}
+	log.Infof("Deleted #%d measurements", cnt)
+	return nil
+}
+
+func init() {
+	cmd := root.Command("rm", "Delete a result")
+	yes := cmd.Flag("yes", "Skip interactive prompt").Bool()
+	all := cmd.Flag("all", "Delete all measurements").Bool()
+
+	resultID := cmd.Arg("id", "the id of the result to delete").Int64()
+
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		ctx, err := root.Init()
+		if err != nil {
+			log.Errorf("%s", err)
+			return err
+		}
+
+		if *all == true {
+			return deleteAll(ctx.DB(), *yes)
+		}
+
+		if *yes == true {
+			err = database.DeleteResult(ctx.DB(), *resultID)
+			if err == db.ErrNoMoreRows {
+				return errors.New("result not found")
+			}
+			return err
+		}
+		answer := ""
+		confirm := &survey.Select{
+			Message: fmt.Sprintf("Are you sure you wish to delete the result #%d", *resultID),
+			Options: []string{"true", "false"},
+			Default: "false",
+		}
+		survey.AskOne(confirm, &answer, nil)
+		if answer == "false" {
+			return errors.New("canceled by user")
+		}
+		err = database.DeleteResult(ctx.DB(), *resultID)
+		if err == db.ErrNoMoreRows {
+			return errors.New("result not found")
+		}
+		return err
+	})
+}

cmd/ooniprobe/internal/cli/root/root.go

@@ -0,0 +1,94 @@
+package root
+
+import (
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/log/handlers/batch"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/log/handlers/cli"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/log/handlers/syslog"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/version"
+)
+
+// Cmd is the root command
+var Cmd = kingpin.New("ooniprobe", "")
+
+// Command is syntax sugar for defining sub-commands
+var Command = Cmd.Command
+
+// Init should be called by all subcommand that care to have a ooni.Context instance
+var Init func() (*ooni.Probe, error)
+
+// NewProbeCLI is like Init but returns a ooni.ProbeCLI instead.
+func NewProbeCLI() (ooni.ProbeCLI, error) {
+	probeCLI, err := Init()
+	if err != nil {
+		return nil, err
+	}
+	return probeCLI, nil
+}
+
+func init() {
+	configPath := Cmd.Flag("config", "Set a custom config file path").Short('c').String()
+
+	isVerbose := Cmd.Flag("verbose", "Enable verbose log output.").Short('v').Bool()
+	isBatch := Cmd.Flag("batch", "Enable batch command line usage.").Bool()
+	logHandler := Cmd.Flag(
+		"log-handler", "Set the desired log handler (one of: batch, cli, syslog)",
+	).String()
+
+	softwareName := Cmd.Flag(
+		"software-name", "Override application name",
+	).Default("ooniprobe-cli").String()
+	softwareVersion := Cmd.Flag(
+		"software-version", "Override the application version",
+	).Default(version.Version).String()
+
+	Cmd.PreAction(func(ctx *kingpin.ParseContext) error {
+		// TODO(bassosimone): we need to properly deprecate --batch
+		// in favour of more granular command line flags.
+		if *isBatch && *logHandler != "" {
+			log.Fatal("cannot specify --batch and --log-handler together")
+		}
+		if *isBatch {
+			*logHandler = "batch"
+		}
+		switch *logHandler {
+		case "batch":
+			log.SetHandler(batch.Default)
+		case "cli", "":
+			log.SetHandler(cli.Default)
+		case "syslog":
+			log.SetHandler(syslog.Default)
+		default:
+			log.Fatalf("unknown --log-handler: %s", *logHandler)
+		}
+		if *isVerbose {
+			log.SetLevel(log.DebugLevel)
+			log.Debugf("ooni version %s", version.Version)
+		}
+
+		Init = func() (*ooni.Probe, error) {
+			var err error
+
+			homePath, err := utils.GetOONIHome()
+			if err != nil {
+				return nil, err
+			}
+
+			probe := ooni.NewProbe(*configPath, homePath)
+			err = probe.Init(*softwareName, *softwareVersion)
+			if err != nil {
+				return nil, err
+			}
+			if *isBatch {
+				probe.SetIsBatch(true)
+			}
+
+			return probe, nil
+		}
+
+		return nil
+	})
+}

cmd/ooniprobe/internal/cli/run/run.go

@@ -0,0 +1,97 @@
+package run
+
+import (
+	"runtime"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/fatih/color"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/onboard"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/nettests"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+)
+
+func init() {
+	cmd := root.Command("run", "Run a test group or OONI Run link")
+	noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool()
+
+	var probe *ooni.Probe
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		var err error
+		probe, err = root.Init()
+		if err != nil {
+			log.Errorf("%s", err)
+			return err
+		}
+		if err = onboard.MaybeOnboarding(probe); err != nil {
+			log.WithError(err).Error("failed to perform onboarding")
+			return err
+		}
+		if *noCollector == true {
+			probe.Config().Sharing.UploadResults = false
+		}
+		return nil
+	})
+
+	functionalRun := func(pred func(name string, gr nettests.Group) bool) error {
+		for name, group := range nettests.All {
+			if pred(name, group) != true {
+				continue
+			}
+			log.Infof("Running %s tests", color.BlueString(name))
+			conf := nettests.RunGroupConfig{GroupName: name, Probe: probe}
+			if err := nettests.RunGroup(conf); err != nil {
+				log.WithError(err).Errorf("failed to run %s", name)
+			}
+		}
+		return nil
+	}
+
+	genRunWithGroupName := func(targetName string) func(*kingpin.ParseContext) error {
+		return func(*kingpin.ParseContext) error {
+			return functionalRun(func(groupName string, gr nettests.Group) bool {
+				return groupName == targetName
+			})
+		}
+	}
+
+	websitesCmd := cmd.Command("websites", "")
+	inputFile := websitesCmd.Flag("input-file", "File containing input URLs").Strings()
+	input := websitesCmd.Flag("input", "Test the specified URL").Strings()
+	websitesCmd.Action(func(_ *kingpin.ParseContext) error {
+		log.Infof("Running %s tests", color.BlueString("websites"))
+		return nettests.RunGroup(nettests.RunGroupConfig{
+			GroupName:  "websites",
+			Probe:      probe,
+			InputFiles: *inputFile,
+			Inputs:     *input,
+		})
+	})
+
+	easyRuns := []string{"im", "performance", "circumvention", "middlebox"}
+	for _, name := range easyRuns {
+		cmd.Command(name, "").Action(genRunWithGroupName(name))
+	}
+
+	unattendedCmd := cmd.Command("unattended", "")
+	unattendedCmd.Action(func(_ *kingpin.ParseContext) error {
+		if runtime.GOOS == "darwin" {
+			// Until we have enabled the check-in API we're called every
+			// hour on darwin and we need to self throttle.
+			// TODO(bassosimone): switch to check-in and remove this hack.
+			const veryFew = 10
+			probe.Config().Nettests.WebsitesURLLimit = veryFew
+		}
+		return functionalRun(func(name string, gr nettests.Group) bool {
+			return gr.UnattendedOK == true
+		})
+	})
+
+	allCmd := cmd.Command("all", "").Default()
+	allCmd.Action(func(_ *kingpin.ParseContext) error {
+		return functionalRun(func(name string, gr nettests.Group) bool {
+			return true
+		})
+	})
+}

cmd/ooniprobe/internal/cli/show/show.go

@@ -0,0 +1,28 @@
+package nettest
+
+import (
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
+)
+
+func init() {
+	cmd := root.Command("show", "Show a specific measurement")
+	msmtID := cmd.Arg("id", "the id of the measurement to show").Required().Int64()
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		ctx, err := root.Init()
+		if err != nil {
+			log.WithError(err).Error("failed to initialize root context")
+			return err
+		}
+		msmt, err := database.GetMeasurementJSON(ctx.DB(), *msmtID)
+		if err != nil {
+			log.Errorf("error: %v", err)
+			return err
+		}
+		output.MeasurementJSON(msmt)
+		return nil
+	})
+}

cmd/ooniprobe/internal/cli/upload/upload.go

@@ -0,0 +1,17 @@
+package upload
+
+import (
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+)
+
+func init() {
+	cmd := root.Command("upload", "Upload a specific measurement")
+
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		log.Info("Uploading")
+		log.Error("this function is not implemented")
+		return nil
+	})
+}

cmd/ooniprobe/internal/cli/version/version.go

@@ -0,0 +1,17 @@
+package version
+
+import (
+	"fmt"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/version"
+)
+
+func init() {
+	cmd := root.Command("version", "Show version.")
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		fmt.Println(version.Version)
+		return nil
+	})
+}

cmd/ooniprobe/internal/config/parser.go

@@ -0,0 +1,100 @@
+package config
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"sync"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/crashreport"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+	"github.com/pkg/errors"
+)
+
+// ConfigVersion is the current version of the config
+const ConfigVersion = 1
+
+// ReadConfig reads the configuration from the path
+func ReadConfig(path string) (*Config, error) {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := ParseConfig(b)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing config")
+	}
+	c.path = path
+	return c, err
+}
+
+// ParseConfig returns config from JSON bytes.
+func ParseConfig(b []byte) (*Config, error) {
+	var c Config
+
+	if err := json.Unmarshal(b, &c); err != nil {
+		return nil, errors.Wrap(err, "parsing json")
+	}
+
+	home, err := utils.GetOONIHome()
+	if err != nil {
+		return nil, err
+	}
+	c.path = utils.ConfigPath(home)
+
+	if c.Advanced.SendCrashReports == false {
+		log.Info("Disabling crash reporting.")
+		crashreport.Disabled = true
+	}
+
+	return &c, nil
+}
+
+// Config for the OONI Probe installation
+type Config struct {
+	// Private settings
+	Comment         string `json:"_"`
+	Version         int64  `json:"_version"`
+	InformedConsent bool   `json:"_informed_consent"`
+
+	Sharing  Sharing  `json:"sharing"`
+	Nettests Nettests `json:"nettests"`
+	Advanced Advanced `json:"advanced"`
+
+	mutex sync.Mutex
+	path  string
+}
+
+// Write the config file in json to the path
+func (c *Config) Write() error {
+	c.Lock()
+	defer c.Unlock()
+	configJSON, _ := json.MarshalIndent(c, "", "  ")
+	if c.path == "" {
+		return errors.New("config file path is empty")
+	}
+	if err := ioutil.WriteFile(c.path, configJSON, 0644); err != nil {
+		return errors.Wrap(err, "writing config JSON")
+	}
+	return nil
+}
+
+// Lock acquires the write mutex
+func (c *Config) Lock() {
+	c.mutex.Lock()
+}
+
+// Unlock releases the write mutex
+func (c *Config) Unlock() {
+	c.mutex.Unlock()
+}
+
+// MaybeMigrate checks the current config version and the config file on disk
+// and if necessary performs and upgrade of the configuration file.
+func (c *Config) MaybeMigrate() error {
+	if c.Version < ConfigVersion {
+		return c.Write()
+	}
+	return nil
+}

cmd/ooniprobe/internal/config/parser_test.go

@@ -0,0 +1,97 @@
+package config
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func getShasum(path string) (string, error) {
+	hasher := sha256.New()
+
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	if _, err := io.Copy(hasher, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
+func TestParseConfig(t *testing.T) {
+	config, err := ReadConfig("testdata/valid-config.json")
+	if err != nil {
+		t.Error(err)
+	}
+	if config.Sharing.UploadResults != true {
+		t.Fatal("not the expected value for UploadResults")
+	}
+}
+
+func TestUpdateConfig(t *testing.T) {
+	tmpFile, err := ioutil.TempFile(os.TempDir(), "ooniconfig-")
+	if err != nil {
+		t.Error(err)
+	}
+	configPath := tmpFile.Name()
+	defer os.Remove(configPath)
+
+	data, err := ioutil.ReadFile("testdata/config-v0.json")
+	if err != nil {
+		t.Error(err)
+	}
+	err = ioutil.WriteFile(configPath, data, 0644)
+	if err != nil {
+		t.Error(err)
+	}
+
+	origShasum, err := getShasum(configPath)
+	if err != nil {
+		t.Error(err)
+	}
+	config, err := ReadConfig(configPath)
+	if err != nil {
+		t.Error(err)
+	}
+	origUploadResults := config.Sharing.UploadResults
+	origInformedConsent := config.InformedConsent
+	if err != nil {
+		t.Error(err)
+	}
+
+	config.MaybeMigrate()
+	migratedShasum, err := getShasum(configPath)
+	if err != nil {
+		t.Error(err)
+	}
+	if migratedShasum == origShasum {
+		t.Fatal("the config was not migrated")
+	}
+
+	newConfig, err := ReadConfig(configPath)
+	if err != nil {
+		t.Error(err)
+	}
+	if newConfig.Sharing.UploadResults != origUploadResults {
+		t.Error("UploadResults differs")
+	}
+	if newConfig.InformedConsent != origInformedConsent {
+		t.Error("InformedConsent differs")
+	}
+
+	// Check that the config file stays the same if it's already the most up to
+	// date version
+	config.MaybeMigrate()
+	finalShasum, err := getShasum(configPath)
+	if err != nil {
+		t.Error(err)
+	}
+	if migratedShasum != finalShasum {
+		t.Fatal("the config was migrated again")
+	}
+}

cmd/ooniprobe/internal/config/settings.go

@@ -0,0 +1,50 @@
+package config
+
+var websiteCategories = []string{
+	"ALDR",
+	"ANON",
+	"COMM",
+	"COMT",
+	"CTRL",
+	"CULTR",
+	"DATE",
+	"ECON",
+	"ENV",
+	"FILE",
+	"GAME",
+	"GMB",
+	"GOVT",
+	"GRP",
+	"HACK",
+	"HATE",
+	"HOST",
+	"HUMR",
+	"IGO",
+	"LGBT",
+	"MILX",
+	"MMED",
+	"NEWS",
+	"POLR",
+	"PORN",
+	"PROV",
+	"PUBH",
+	"REL",
+	"SRCH",
+	"XED",
+}
+
+// Sharing settings
+type Sharing struct {
+	UploadResults bool `json:"upload_results"`
+}
+
+// Advanced settings
+type Advanced struct {
+	SendCrashReports bool `json:"send_crash_reports"`
+}
+
+// Nettests related settings
+type Nettests struct {
+	WebsitesURLLimit             int64    `json:"websites_url_limit"`
+	WebsitesEnabledCategoryCodes []string `json:"websites_enabled_category_codes"`
+}

cmd/ooniprobe/internal/config/testdata/config-v0.json

@@ -0,0 +1,62 @@
+{
+  "_version": 0,
+  "_informed_consent": true,
+  "_is_beta": true,
+  "auto_update": true,
+  "sharing": {
+    "include_ip": false,
+    "include_asn": true,
+    "include_country": true,
+    "include_gps": true,
+    "upload_results": true
+  },
+  "notifications": {
+    "enabled": true,
+    "notify_on_test_completion": true,
+    "notify_on_news": false
+  },
+  "automated_testing": {
+    "enabled": false,
+    "enabled_tests": [
+      "web-connectivity",
+      "facebook-messenger",
+      "whatsapp",
+      "telegram",
+      "dash",
+      "ndt",
+      "http-invalid-request-line",
+      "http-header-field-manipulation"
+    ],
+    "monthly_allowance": "300MB"
+  },
+  "test_settings": {
+    "websites": {
+      "enabled_categories": []
+    },
+    "instant_messaging": {
+      "enabled_tests": [
+        "facebook-messenger",
+        "whatsapp",
+        "telegram"
+      ]
+    },
+    "performance": {
+      "ndt_server": "auto",
+      "ndt_server_port": "auto",
+      "dash_server": "auto",
+      "dash_server_port": "auto"
+    },
+    "middlebox": {
+      "enabled_tests": [
+        "http-invalid-request-line",
+        "http-header-field-manipulation"
+      ]
+    }
+  },
+  "advanced": {
+    "use_domain_fronting": false,
+    "send_crash_reports": true,
+    "collector_url": "",
+    "bouncer_url": "https://bouncer.ooni.io"
+  }
+}

cmd/ooniprobe/internal/config/testdata/valid-config.json

@@ -0,0 +1,13 @@
+{
+  "_version": 1,
+  "_informed_consent": false,
+  "sharing": {
+    "upload_results": true
+  },
+  "nettests": {
+    "websites_url_limit": 0
+  },
+  "advanced": {
+    "send_crash_reports": true
+  }
+}

cmd/ooniprobe/internal/crashreport/crashreport.go

@@ -0,0 +1,62 @@
+package crashreport
+
+import (
+	"github.com/apex/log"
+	"github.com/getsentry/raven-go"
+)
+
+// Disabled flag is used to globally disable crash reporting and make all the
+// crash reporting logic a no-op.
+var Disabled = false
+
+var client *raven.Client
+
+// CapturePanic is a wrapper around raven.CapturePanic that becomes a noop if
+// `Disabled` is set to true.
+func CapturePanic(f func(), tags map[string]string) (interface{}, string) {
+	if Disabled == true {
+		return nil, ""
+	}
+	return client.CapturePanic(f, tags)
+}
+
+// CapturePanicAndWait is a wrapper around raven.CapturePanicAndWait that becomes a noop if
+// `Disabled` is set to true.
+func CapturePanicAndWait(f func(), tags map[string]string) (interface{}, string) {
+	if Disabled == true {
+		return nil, ""
+	}
+	return client.CapturePanicAndWait(f, tags)
+}
+
+// CaptureError is a wrapper around raven.CaptureError
+func CaptureError(err error, tags map[string]string) string {
+	if Disabled == true {
+		return ""
+	}
+	return client.CaptureError(err, tags)
+}
+
+// CaptureErrorAndWait is a wrapper around raven.CaptureErrorAndWait
+func CaptureErrorAndWait(err error, tags map[string]string) string {
+	if Disabled == true {
+		return ""
+	}
+	return client.CaptureErrorAndWait(err, tags)
+}
+
+// Wait will block on sending messages to the sentry server
+func Wait() {
+	if Disabled == false {
+		log.Info("sending exception backtrace")
+		client.Wait()
+	}
+}
+
+func init() {
+	var err error
+	client, err = raven.NewClient("https://cb4510e090f64382ac371040c19b2258:8448daeebfa643c289ef398f8645980b@sentry.io/1234954", nil)
+	if err != nil {
+		log.WithError(err).Error("failed to create a raven client")
+	}
+}

cmd/ooniprobe/internal/database/actions.go

@@ -0,0 +1,355 @@
+package database
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"reflect"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+	"github.com/pkg/errors"
+	db "upper.io/db.v3"
+	"upper.io/db.v3/lib/sqlbuilder"
+)
+
+// ListMeasurements given a result ID
+func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementURLNetwork, error) {
+	measurements := []MeasurementURLNetwork{}
+	req := sess.Select(
+		db.Raw("networks.*"),
+		db.Raw("urls.*"),
+		db.Raw("measurements.*"),
+		db.Raw("results.*"),
+	).From("results").
+		Join("measurements").On("results.result_id = measurements.result_id").
+		Join("networks").On("results.network_id = networks.network_id").
+		LeftJoin("urls").On("urls.url_id = measurements.url_id").
+		OrderBy("measurements.measurement_start_time").
+		Where("results.result_id = ?", resultID)
+	if err := req.All(&measurements); err != nil {
+		log.Errorf("failed to run query %s: %v", req.String(), err)
+		return measurements, err
+	}
+	return measurements, nil
+}
+
+// GetMeasurementJSON returns a map[string]interface{} given a database and a measurementID
+func GetMeasurementJSON(sess sqlbuilder.Database, measurementID int64) (map[string]interface{}, error) {
+	var (
+		measurement MeasurementURLNetwork
+		msmtJSON    map[string]interface{}
+	)
+	req := sess.Select(
+		db.Raw("urls.*"),
+		db.Raw("measurements.*"),
+	).From("measurements").
+		LeftJoin("urls").On("urls.url_id = measurements.url_id").
+		Where("measurements.measurement_id= ?", measurementID)
+	if err := req.One(&measurement); err != nil {
+		log.Errorf("failed to run query %s: %v", req.String(), err)
+		return nil, err
+	}
+	if measurement.IsUploaded {
+		// TODO(bassosimone): this should be a function exposed by probe-engine
+		reportID := measurement.Measurement.ReportID.String
+		measurementURL := &url.URL{
+			Scheme: "https",
+			Host:   "api.ooni.io",
+			Path:   "/api/v1/raw_measurement",
+		}
+		query := url.Values{}
+		query.Add("report_id", reportID)
+		if measurement.URL.URL.Valid == true {
+			query.Add("input", measurement.URL.URL.String)
+		}
+		measurementURL.RawQuery = query.Encode()
+		log.Debugf("using %s", measurementURL.String())
+		resp, err := http.Get(measurementURL.String())
+		if err != nil {
+			log.Errorf("failed to fetch the measurement %s %s", reportID, measurement.URL.URL.String)
+			return nil, err
+		}
+		defer resp.Body.Close()
+		if err := json.NewDecoder(resp.Body).Decode(&msmtJSON); err != nil {
+			log.Error("failed to unmarshal the measurement_json")
+			return nil, err
+		}
+		return msmtJSON, nil
+	}
+	// MeasurementFilePath might be NULL because the measurement from a
+	// 3.0.0-beta install
+	if measurement.Measurement.MeasurementFilePath.Valid == false {
+		log.Error("invalid measurement_file_path")
+		log.Error("backup your OONI_HOME and run `ooniprobe reset`")
+		return nil, errors.New("cannot access measurement file")
+	}
+	measurementFilePath := measurement.Measurement.MeasurementFilePath.String
+	b, err := ioutil.ReadFile(measurementFilePath)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(b, &msmtJSON); err != nil {
+		log.Error("failed to unmarshal the measurement_json")
+		log.Error("backup your OONI_HOME and run `ooniprobe reset`")
+		return nil, err
+	}
+	return msmtJSON, nil
+}
+
+// GetResultTestKeys returns a list of TestKeys for a given result
+func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error) {
+	res := sess.Collection("measurements").Find("result_id", resultID)
+	defer res.Close()
+
+	var (
+		msmt Measurement
+		tk   PerformanceTestKeys
+	)
+	for res.Next(&msmt) {
+		// We only really care about performance keys.
+		// Note: since even in case of failure we still initialise an empty struct,
+		// it could be that these keys come out as initializes with the default
+		// values.
+		// XXX we may want to change this behaviour by adding `omitempty` to the
+		// struct definition.
+		if msmt.TestName != "ndt" && msmt.TestName != "dash" {
+			return "{}", nil
+		}
+		if err := json.Unmarshal([]byte(msmt.TestKeys), &tk); err != nil {
+			log.WithError(err).Error("failed to parse testKeys")
+			return "{}", err
+		}
+	}
+	b, err := json.Marshal(tk)
+	if err != nil {
+		log.WithError(err).Error("failed to serialize testKeys")
+		return "{}", err
+	}
+	return string(b), nil
+}
+
+// GetMeasurementCounts returns the number of anomalous and total measurement for a given result
+func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) {
+	var (
+		totalCount uint64
+		anmlyCount uint64
+		err        error
+	)
+	col := sess.Collection("measurements")
+
+	// XXX these two queries can be done with a single query
+	totalCount, err = col.Find("result_id", resultID).
+		Count()
+	if err != nil {
+		log.WithError(err).Error("failed to get total count")
+		return totalCount, anmlyCount, err
+	}
+
+	anmlyCount, err = col.Find("result_id", resultID).
+		And(db.Cond{"is_anomaly": true}).Count()
+	if err != nil {
+		log.WithError(err).Error("failed to get anmly count")
+		return totalCount, anmlyCount, err
+	}
+
+	log.Debugf("counts: %d, %d, %d", resultID, totalCount, anmlyCount)
+	return totalCount, anmlyCount, err
+}
+
+// ListResults return the list of results
+func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) {
+	doneResults := []ResultNetwork{}
+	incompleteResults := []ResultNetwork{}
+	req := sess.Select(
+		db.Raw("networks.*"),
+		db.Raw("results.*"),
+	).From("results").
+		Join("networks").On("results.network_id = networks.network_id").
+		OrderBy("results.result_start_time")
+	if err := req.Where("result_is_done = true").All(&doneResults); err != nil {
+		return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
+	}
+	if err := req.Where("result_is_done = false").All(&incompleteResults); err != nil {
+		return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
+	}
+	return doneResults, incompleteResults, nil
+}
+
+// DeleteResult will delete a particular result and the relative measurement on
+// disk.
+func DeleteResult(sess sqlbuilder.Database, resultID int64) error {
+	var result Result
+	res := sess.Collection("results").Find("result_id", resultID)
+	if err := res.One(&result); err != nil {
+		if err == db.ErrNoMoreRows {
+			return err
+		}
+		log.WithError(err).Error("error in obtaining the result")
+		return err
+	}
+	if err := res.Delete(); err != nil {
+		log.WithError(err).Error("failed to delete the result directory")
+		return err
+	}
+
+	os.RemoveAll(result.MeasurementDir)
+	return nil
+}
+
+// CreateMeasurement writes the measurement to the database a returns a pointer
+// to the Measurement
+func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testName string, measurementDir string, idx int, resultID int64, urlID sql.NullInt64) (*Measurement, error) {
+	// TODO we should look into generating this file path in a more robust way.
+	// If there are two identical test_names in the same test group there is
+	// going to be a clash of test_name
+	msmtFilePath := filepath.Join(measurementDir, fmt.Sprintf("msmt-%s-%d.json", testName, idx))
+	msmt := Measurement{
+		ReportID:            reportID,
+		TestName:            testName,
+		ResultID:            resultID,
+		MeasurementFilePath: sql.NullString{String: msmtFilePath, Valid: true},
+		URLID:               urlID,
+		IsFailed:            false,
+		IsDone:              false,
+		// XXX Do we want to have this be part of something else?
+		StartTime: time.Now().UTC(),
+		TestKeys:  "",
+	}
+
+	newID, err := sess.Collection("measurements").Insert(msmt)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating measurement")
+	}
+	msmt.ID = newID.(int64)
+	return &msmt, nil
+}
+
+// CreateResult writes the Result to the database a returns a pointer
+// to the Result
+func CreateResult(sess sqlbuilder.Database, homePath string, testGroupName string, networkID int64) (*Result, error) {
+	startTime := time.Now().UTC()
+
+	p, err := utils.MakeResultsDir(homePath, testGroupName, startTime)
+	if err != nil {
+		return nil, err
+	}
+
+	result := Result{
+		TestGroupName: testGroupName,
+		StartTime:     startTime,
+		NetworkID:     networkID,
+	}
+	result.MeasurementDir = p
+	log.Debugf("Creating result %v", result)
+
+	newID, err := sess.Collection("results").Insert(result)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating result")
+	}
+	result.ID = newID.(int64)
+	return &result, nil
+}
+
+// CreateNetwork will create a new network in the network table
+func CreateNetwork(sess sqlbuilder.Database, loc enginex.LocationProvider) (*Network, error) {
+	network := Network{
+		ASN:         loc.ProbeASN(),
+		CountryCode: loc.ProbeCC(),
+		NetworkName: loc.ProbeNetworkName(),
+		// On desktop we consider it to always be wifi
+		NetworkType: "wifi",
+		IP:          loc.ProbeIP(),
+	}
+	newID, err := sess.Collection("networks").Insert(network)
+	if err != nil {
+		return nil, err
+	}
+
+	network.ID = newID.(int64)
+	return &network, nil
+}
+
+// CreateOrUpdateURL will create a new URL entry to the urls table if it doesn't
+// exists, otherwise it will update the category code of the one already in
+// there.
+func CreateOrUpdateURL(sess sqlbuilder.Database, urlStr string, categoryCode string, countryCode string) (int64, error) {
+	var url URL
+
+	tx, err := sess.NewTx(nil)
+	if err != nil {
+		log.WithError(err).Error("failed to create transaction")
+		return 0, err
+	}
+	res := tx.Collection("urls").Find(
+		db.Cond{"url": urlStr, "url_country_code": countryCode},
+	)
+	err = res.One(&url)
+
+	if err == db.ErrNoMoreRows {
+		url = URL{
+			URL:          sql.NullString{String: urlStr, Valid: true},
+			CategoryCode: sql.NullString{String: categoryCode, Valid: true},
+			CountryCode:  sql.NullString{String: countryCode, Valid: true},
+		}
+		newID, insErr := tx.Collection("urls").Insert(url)
+		if insErr != nil {
+			log.Error("Failed to insert into the URLs table")
+			return 0, insErr
+		}
+		url.ID = sql.NullInt64{Int64: newID.(int64), Valid: true}
+	} else if err != nil {
+		log.WithError(err).Error("Failed to get single result")
+		return 0, err
+	} else {
+		url.CategoryCode = sql.NullString{String: categoryCode, Valid: true}
+		res.Update(url)
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		log.WithError(err).Error("Failed to write to the URL table")
+		return 0, err
+	}
+
+	log.Debugf("returning url %d", url.ID.Int64)
+
+	return url.ID.Int64, nil
+}
+
+// AddTestKeys writes the summary to the measurement
+func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error {
+	var (
+		isAnomaly      bool
+		isAnomalyValid bool
+	)
+	tkBytes, err := json.Marshal(tk)
+	if err != nil {
+		log.WithError(err).Error("failed to serialize summary")
+	}
+
+	// This is necessary so that we can extract from the the opaque testKeys just
+	// the IsAnomaly field of bool type.
+	// Maybe generics are not so bad after-all, heh golang?
+	isAnomalyValue := reflect.ValueOf(tk).FieldByName("IsAnomaly")
+	if isAnomalyValue.IsValid() == true && isAnomalyValue.Kind() == reflect.Bool {
+		isAnomaly = isAnomalyValue.Bool()
+		isAnomalyValid = true
+	}
+	msmt.TestKeys = string(tkBytes)
+	msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: isAnomalyValid}
+
+	err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
+	if err != nil {
+		log.WithError(err).Error("failed to update measurement")
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}

cmd/ooniprobe/internal/database/actions_test.go

@@ -0,0 +1,363 @@
+package database
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	db "upper.io/db.v3"
+)
+
+type locationInfo struct {
+	asn         uint
+	countryCode string
+	ip          string
+	networkName string
+	resolverIP  string
+}
+
+func (lp *locationInfo) ProbeASN() uint {
+	return lp.asn
+}
+
+func (lp *locationInfo) ProbeASNString() string {
+	return fmt.Sprintf("AS%d", lp.asn)
+}
+
+func (lp *locationInfo) ProbeCC() string {
+	return lp.countryCode
+}
+
+func (lp *locationInfo) ProbeIP() string {
+	return lp.ip
+}
+
+func (lp *locationInfo) ProbeNetworkName() string {
+	return lp.networkName
+}
+
+func (lp *locationInfo) ResolverIP() string {
+	return lp.resolverIP
+}
+
+func TestMeasurementWorkflow(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	tmpdir, err := ioutil.TempDir("", "oonitest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	location := locationInfo{
+		asn:         0,
+		countryCode: "IT",
+		networkName: "Unknown",
+	}
+	network, err := CreateNetwork(sess, &location)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reportID := sql.NullString{String: "", Valid: false}
+	testName := "antani"
+	resultID := result.ID
+	msmtFilePath := tmpdir
+	urlID := sql.NullInt64{Int64: 0, Valid: false}
+
+	m1, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var m2 Measurement
+	err = sess.Collection("measurements").Find("measurement_id", m1.ID).One(&m2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m2.ResultID != m1.ResultID {
+		t.Error("result_id mismatch")
+	}
+
+	done, incomplete, err := ListResults(sess)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(incomplete) != 1 {
+		t.Error("there should be 1 incomplete measurement")
+	}
+	if len(done) != 0 {
+		t.Error("there should be 0 done measurements")
+	}
+
+	msmts, err := ListMeasurements(sess, resultID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if msmts[0].Network.NetworkType != "wifi" {
+		t.Error("network_type should be wifi")
+	}
+}
+
+func TestDeleteResult(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	tmpdir, err := ioutil.TempDir("", "oonitest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	location := locationInfo{
+		asn:         0,
+		countryCode: "IT",
+		networkName: "Unknown",
+	}
+	network, err := CreateNetwork(sess, &location)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reportID := sql.NullString{String: "", Valid: false}
+	testName := "antani"
+	resultID := result.ID
+	msmtFilePath := tmpdir
+	urlID := sql.NullInt64{Int64: 0, Valid: false}
+
+	m1, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var m2 Measurement
+	err = sess.Collection("measurements").Find("measurement_id", m1.ID).One(&m2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m2.ResultID != m1.ResultID {
+		t.Error("result_id mismatch")
+	}
+
+	err = DeleteResult(sess, resultID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	totalResults, err := sess.Collection("results").Find().Count()
+	if err != nil {
+		t.Fatal(err)
+	}
+	totalMeasurements, err := sess.Collection("measurements").Find().Count()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if totalResults != 0 {
+		t.Fatal("results should be zero")
+	}
+	if totalMeasurements != 0 {
+		t.Fatal("measurements should be zero")
+	}
+
+	err = DeleteResult(sess, 20)
+	if err != db.ErrNoMoreRows {
+		t.Fatal(err)
+	}
+}
+
+func TestNetworkCreate(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	l1 := locationInfo{
+		asn:         2,
+		countryCode: "IT",
+		networkName: "Antaninet",
+	}
+
+	l2 := locationInfo{
+		asn:         3,
+		countryCode: "IT",
+		networkName: "Fufnet",
+	}
+
+	_, err = CreateNetwork(sess, &l1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = CreateNetwork(sess, &l2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
+func TestURLCreation(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID1, err := CreateOrUpdateURL(sess, "https://google.com", "GMB", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID2, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID3, err := CreateOrUpdateURL(sess, "https://facebook.com", "GRP", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID4, err := CreateOrUpdateURL(sess, "https://facebook.com", "GMP", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID5, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if newID2 != newID1 {
+		t.Error("inserting the same URL with different category code should produce the same result")
+	}
+
+	if newID3 == newID1 {
+		t.Error("inserting different URL should produce different ids")
+	}
+
+	if newID4 != newID3 {
+		t.Error("inserting the same URL with different category code should produce the same result")
+	}
+
+	if newID5 != newID1 {
+		t.Error("the ID of google should still be the same")
+	}
+}
+
+func TestPerformanceTestKeys(t *testing.T) {
+	var tk PerformanceTestKeys
+
+	ndtS := "{\"download\":100.0,\"upload\":20.0,\"ping\":2.2}"
+	dashS := "{\"median_bitrate\":102.0}"
+	if err := json.Unmarshal([]byte(ndtS), &tk); err != nil {
+		t.Fatal("failed to parse ndtS")
+	}
+	if err := json.Unmarshal([]byte(dashS), &tk); err != nil {
+		t.Fatal("failed to parse dashS")
+	}
+	if tk.Bitrate != 102.0 {
+		t.Fatalf("error Bitrate %f", tk.Bitrate)
+	}
+	if tk.Download != 100.0 {
+		t.Fatalf("error Download %f", tk.Download)
+	}
+}
+
+func TestGetMeasurementJSON(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	tmpdir, err := ioutil.TempDir("", "oonitest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	location := locationInfo{
+		asn:         0,
+		countryCode: "IT",
+		networkName: "Unknown",
+	}
+	network, err := CreateNetwork(sess, &location)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reportID := sql.NullString{String: "20210111T085144Z_ndt_RU_3216_n1_qMVnP0PTX7ObUSmD", Valid: true}
+	testName := "antani"
+	resultID := result.ID
+	msmtFilePath := tmpdir
+	urlID := sql.NullInt64{Int64: 0, Valid: false}
+
+	msmt, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	msmt.IsUploaded = true
+	err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tk, err := GetMeasurementJSON(sess, msmt.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk["probe_asn"] != "AS3216" {
+		t.Error("inconsistent measurement downloaded")
+	}
+}

cmd/ooniprobe/internal/database/database.go

@@ -0,0 +1,47 @@
+package database
+
+import (
+	"database/sql"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/bindata"
+	migrate "github.com/rubenv/sql-migrate"
+	"upper.io/db.v3/lib/sqlbuilder"
+	"upper.io/db.v3/sqlite"
+)
+
+// RunMigrations runs the database migrations
+func RunMigrations(db *sql.DB) error {
+	log.Debugf("running migrations")
+	migrations := &migrate.AssetMigrationSource{
+		Asset:    bindata.Asset,
+		AssetDir: bindata.AssetDir,
+		Dir:      "data/migrations",
+	}
+	n, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up)
+	if err != nil {
+		return err
+	}
+	log.Debugf("performed %d migrations", n)
+	return nil
+}
+
+// Connect to the database
+func Connect(path string) (db sqlbuilder.Database, err error) {
+	settings := sqlite.ConnectionURL{
+		Database: path,
+		Options:  map[string]string{"_foreign_keys": "1"},
+	}
+	sess, err := sqlite.Open(settings)
+	if err != nil {
+		log.WithError(err).Error("failed to open the DB")
+		return nil, err
+	}
+
+	err = RunMigrations(sess.Driver().(*sql.DB))
+	if err != nil {
+		log.WithError(err).Error("failed to run DB migration")
+		return nil, err
+	}
+	return sess, err
+}

cmd/ooniprobe/internal/database/database_test.go

@@ -0,0 +1,32 @@
+package database
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/apex/log"
+)
+
+func TestConnect(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Error(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Error(err)
+	}
+
+	colls, err := sess.Collections()
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(colls) < 1 {
+		log.Fatal("missing tables")
+	}
+
+}

cmd/ooniprobe/internal/database/models.go

@@ -0,0 +1,150 @@
+package database
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/pkg/errors"
+	"upper.io/db.v3/lib/sqlbuilder"
+)
+
+// ResultNetwork is used to represent the structure made from the JOIN
+// between the results and networks tables.
+type ResultNetwork struct {
+	Result  `db:",inline"`
+	Network `db:",inline"`
+}
+
+// MeasurementURLNetwork is used for the JOIN between Measurement and URL
+type MeasurementURLNetwork struct {
+	Measurement `db:",inline"`
+	Network     `db:",inline"`
+	Result      `db:",inline"`
+	URL         `db:",inline"`
+}
+
+// Network represents a network tested by the user
+type Network struct {
+	ID          int64  `db:"network_id,omitempty"`
+	NetworkName string `db:"network_name"`
+	NetworkType string `db:"network_type"`
+	IP          string `db:"ip"`
+	ASN         uint   `db:"asn"`
+	CountryCode string `db:"network_country_code"`
+}
+
+// URL represents URLs from the testing lists
+type URL struct {
+	ID           sql.NullInt64  `db:"url_id,omitempty"`
+	URL          sql.NullString `db:"url"`
+	CategoryCode sql.NullString `db:"category_code"`
+	CountryCode  sql.NullString `db:"url_country_code"`
+}
+
+// Measurement model
+type Measurement struct {
+	ID               int64          `db:"measurement_id,omitempty"`
+	TestName         string         `db:"test_name"`
+	StartTime        time.Time      `db:"measurement_start_time"`
+	Runtime          float64        `db:"measurement_runtime"` // Fractional number of seconds
+	IsDone           bool           `db:"measurement_is_done"`
+	IsUploaded       bool           `db:"measurement_is_uploaded"`
+	IsFailed         bool           `db:"measurement_is_failed"`
+	FailureMsg       sql.NullString `db:"measurement_failure_msg,omitempty"`
+	IsUploadFailed   bool           `db:"measurement_is_upload_failed"`
+	UploadFailureMsg sql.NullString `db:"measurement_upload_failure_msg,omitempty"`
+	IsRerun          bool           `db:"measurement_is_rerun"`
+	ReportID         sql.NullString `db:"report_id,omitempty"`
+	URLID            sql.NullInt64  `db:"url_id,omitempty"` // Used to reference URL
+	MeasurementID    sql.NullInt64  `db:"collector_measurement_id,omitempty"`
+	IsAnomaly        sql.NullBool   `db:"is_anomaly,omitempty"`
+	// FIXME we likely want to support JSON. See: https://github.com/upper/db/issues/462
+	TestKeys            string         `db:"test_keys"`
+	ResultID            int64          `db:"result_id"`
+	ReportFilePath      sql.NullString `db:"report_file_path,omitempty"`
+	MeasurementFilePath sql.NullString `db:"measurement_file_path,omitempty"`
+}
+
+// Result model
+type Result struct {
+	ID             int64     `db:"result_id,omitempty"`
+	TestGroupName  string    `db:"test_group_name"`
+	StartTime      time.Time `db:"result_start_time"`
+	NetworkID      int64     `db:"network_id"`     // Used to include a Network
+	Runtime        float64   `db:"result_runtime"` // Runtime is expressed in fractional seconds
+	IsViewed       bool      `db:"result_is_viewed"`
+	IsDone         bool      `db:"result_is_done"`
+	DataUsageUp    float64   `db:"result_data_usage_up"`
+	DataUsageDown  float64   `db:"result_data_usage_down"`
+	MeasurementDir string    `db:"measurement_dir"`
+}
+
+// PerformanceTestKeys is the result summary for a performance test
+type PerformanceTestKeys struct {
+	Upload   float64 `json:"upload"`
+	Download float64 `json:"download"`
+	Ping     float64 `json:"ping"`
+	Bitrate  float64 `json:"median_bitrate"`
+}
+
+// Finished marks the result as done and sets the runtime
+func (r *Result) Finished(sess sqlbuilder.Database) error {
+	if r.IsDone == true || r.Runtime != 0 {
+		return errors.New("Result is already finished")
+	}
+	r.Runtime = time.Now().UTC().Sub(r.StartTime).Seconds()
+	r.IsDone = true
+
+	err := sess.Collection("results").Find("result_id", r.ID).Update(r)
+	if err != nil {
+		return errors.Wrap(err, "updating finished result")
+	}
+	return nil
+}
+
+// Failed writes the error string to the measurement
+func (m *Measurement) Failed(sess sqlbuilder.Database, failure string) error {
+	m.FailureMsg = sql.NullString{String: failure, Valid: true}
+	m.IsFailed = true
+	err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
+	if err != nil {
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}
+
+// Done marks the measurement as completed
+func (m *Measurement) Done(sess sqlbuilder.Database) error {
+	runtime := time.Now().UTC().Sub(m.StartTime)
+	m.Runtime = runtime.Seconds()
+	m.IsDone = true
+
+	err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
+	if err != nil {
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}
+
+// UploadFailed writes the error string for the upload failure to the measurement
+func (m *Measurement) UploadFailed(sess sqlbuilder.Database, failure string) error {
+	m.UploadFailureMsg = sql.NullString{String: failure, Valid: true}
+	m.IsUploaded = false
+
+	err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
+	if err != nil {
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}
+
+// UploadSucceeded writes the error string for the upload failure to the measurement
+func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error {
+	m.IsUploaded = true
+
+	err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
+	if err != nil {
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}

cmd/ooniprobe/internal/enginex/enginex.go

@@ -0,0 +1,22 @@
+// Package enginex contains ooni/probe-engine extensions.
+package enginex
+
+import (
+	"github.com/apex/log"
+)
+
+// Logger is the logger used by the engine.
+var Logger = log.WithFields(log.Fields{
+	"type": "engine",
+})
+
+// LocationProvider is an interface that returns the current location. The
+// github.com/ooni/probe-engine/session.Session implements it.
+type LocationProvider interface {
+	ProbeASN() uint
+	ProbeASNString() string
+	ProbeCC() string
+	ProbeIP() string
+	ProbeNetworkName() string
+	ResolverIP() string
+}

cmd/ooniprobe/internal/log/handlers/batch/batch.go

@@ -0,0 +1,33 @@
+package batch
+
+import (
+	j "encoding/json"
+	"io"
+	"os"
+	"sync"
+
+	"github.com/apex/log"
+)
+
+// Default handler outputting to stderr.
+var Default = New(os.Stderr)
+
+// Handler implementation.
+type Handler struct {
+	*j.Encoder
+	mu sync.Mutex
+}
+
+// New handler.
+func New(w io.Writer) *Handler {
+	return &Handler{
+		Encoder: j.NewEncoder(w),
+	}
+}
+
+// HandleLog implements log.Handler.
+func (h *Handler) HandleLog(e *log.Entry) error {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.Encoder.Encode(e)
+}

cmd/ooniprobe/internal/log/handlers/cli/cli.go

@@ -0,0 +1,173 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/fatih/color"
+	colorable "github.com/mattn/go-colorable"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+)
+
+// Default handler outputting to stderr.
+var Default = New(os.Stdout)
+
+// start time.
+var start = time.Now()
+
+var bold = color.New(color.Bold)
+
+// Colors mapping.
+var Colors = [...]*color.Color{
+	log.DebugLevel: color.New(color.FgWhite),
+	log.InfoLevel:  color.New(color.FgBlue),
+	log.WarnLevel:  color.New(color.FgYellow),
+	log.ErrorLevel: color.New(color.FgRed),
+	log.FatalLevel: color.New(color.FgRed),
+}
+
+// Strings mapping.
+var Strings = [...]string{
+	log.DebugLevel: "•",
+	log.InfoLevel:  "•",
+	log.WarnLevel:  "•",
+	log.ErrorLevel: "⨯",
+	log.FatalLevel: "⨯",
+}
+
+// Handler implementation.
+type Handler struct {
+	mu      sync.Mutex
+	Writer  io.Writer
+	Padding int
+}
+
+// New handler.
+func New(w io.Writer) *Handler {
+	if f, ok := w.(*os.File); ok {
+		return &Handler{
+			Writer:  colorable.NewColorable(f),
+			Padding: 3,
+		}
+	}
+
+	return &Handler{
+		Writer:  w,
+		Padding: 3,
+	}
+}
+
+func logSectionTitle(w io.Writer, f log.Fields) error {
+	colWidth := 24
+
+	title := f.Get("title").(string)
+	fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth+2)+"┓\n")
+	fmt.Fprintf(w, "┃ %s ┃\n", utils.RightPad(title, colWidth))
+	fmt.Fprintf(w, "┗"+strings.Repeat("━", colWidth+2)+"┛\n")
+	return nil
+}
+
+func logTable(w io.Writer, f log.Fields) error {
+	color := color.New(color.FgBlue)
+
+	names := f.Names()
+
+	var lines []string
+	colWidth := 0
+	for _, name := range names {
+		if name == "type" {
+			continue
+		}
+		line := fmt.Sprintf("%s: %s", color.Sprint(name), f.Get(name))
+		lineLength := utils.EscapeAwareRuneCountInString(line)
+		lines = append(lines, line)
+		if colWidth < lineLength {
+			colWidth = lineLength
+		}
+	}
+
+	fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth+2)+"┓\n")
+	for _, line := range lines {
+		fmt.Fprintf(w, "┃ %s ┃\n",
+			utils.RightPad(line, colWidth),
+		)
+	}
+	fmt.Fprintf(w, "┗"+strings.Repeat("━", colWidth+2)+"┛\n")
+	return nil
+}
+
+// TypedLog is used for handling special "typed" logs to the CLI
+func (h *Handler) TypedLog(t string, e *log.Entry) error {
+	switch t {
+	case "engine":
+		fmt.Fprintf(h.Writer, "[engine] %s\n", e.Message)
+		return nil
+	case "progress":
+		perc := e.Fields.Get("percentage").(float64) * 100
+		eta := e.Fields.Get("eta").(float64)
+		var etaMessage string
+		if eta >= 0 {
+			etaMessage = fmt.Sprintf("(%ss left)", bold.Sprintf("%.2f", eta))
+		}
+		s := fmt.Sprintf("   %s %-25s %s",
+			bold.Sprintf("%.2f%%", perc),
+			e.Message, etaMessage)
+		fmt.Fprint(h.Writer, s)
+		fmt.Fprintln(h.Writer)
+		return nil
+	case "table":
+		return logTable(h.Writer, e.Fields)
+	case "measurement_item":
+		return logMeasurementItem(h.Writer, e.Fields)
+	case "measurement_json":
+		return logMeasurementJSON(h.Writer, e.Fields)
+	case "measurement_summary":
+		return logMeasurementSummary(h.Writer, e.Fields)
+	case "result_item":
+		return logResultItem(h.Writer, e.Fields)
+	case "result_summary":
+		return logResultSummary(h.Writer, e.Fields)
+	case "section_title":
+		return logSectionTitle(h.Writer, e.Fields)
+	default:
+		return h.DefaultLog(e)
+	}
+}
+
+// DefaultLog is the default way of printing out logs
+func (h *Handler) DefaultLog(e *log.Entry) error {
+	color := Colors[e.Level]
+	level := Strings[e.Level]
+	names := e.Fields.Names()
+
+	s := color.Sprintf("%s %-25s", bold.Sprintf("%*s", h.Padding+1, level), e.Message)
+	for _, name := range names {
+		if name == "source" {
+			continue
+		}
+		s += fmt.Sprintf(" %s=%v", color.Sprint(name), e.Fields.Get(name))
+	}
+
+	fmt.Fprint(h.Writer, s)
+	fmt.Fprintln(h.Writer)
+
+	return nil
+}
+
+// HandleLog implements log.Handler.
+func (h *Handler) HandleLog(e *log.Entry) error {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	t, isTyped := e.Fields["type"].(string)
+	if isTyped {
+		return h.TypedLog(t, e)
+	}
+
+	return h.DefaultLog(e)
+}

cmd/ooniprobe/internal/log/handlers/cli/measurements.go

@@ -0,0 +1,142 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+)
+
+func statusIcon(ok bool) string {
+	if ok {
+		return "✓"
+	}
+	return "❌"
+}
+
+func logTestKeys(w io.Writer, testKeys string) error {
+	colWidth := 24
+
+	var out bytes.Buffer
+	if err := json.Indent(&out, []byte(testKeys), "", " "); err != nil {
+		return err
+	}
+
+	testKeysLines := strings.Split(string(out.Bytes()), "\n")
+	if len(testKeysLines) > 1 {
+		testKeysLines = testKeysLines[1 : len(testKeysLines)-1]
+		testKeysLines[0] = "{" + testKeysLines[0][1:]
+		testKeysLines[len(testKeysLines)-1] = testKeysLines[len(testKeysLines)-1] + "}"
+	}
+	for _, line := range testKeysLines {
+		fmt.Fprintf(w, fmt.Sprintf("│ %s │\n",
+			utils.RightPad(line, colWidth*2)))
+	}
+	return nil
+}
+
+func logMeasurementItem(w io.Writer, f log.Fields) error {
+	colWidth := 24
+
+	rID := f.Get("id").(int64)
+	testName := f.Get("test_name").(string)
+
+	// We currently don't use these fields in the view
+	//testGroupName := f.Get("test_group_name").(string)
+	//networkName := f.Get("network_name").(string)
+	//asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("network_country_code").(string))
+	testKeys := f.Get("test_keys").(string)
+
+	isAnomaly := f.Get("is_anomaly").(bool)
+	isFailed := f.Get("is_failed").(bool)
+	isUploaded := f.Get("is_uploaded").(bool)
+	url := f.Get("url").(string)
+	urlCategoryCode := f.Get("url_category_code").(string)
+
+	isFirst := f.Get("is_first").(bool)
+	isLast := f.Get("is_last").(bool)
+	if isFirst {
+		fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth*2+2)+"┓\n")
+	} else {
+		fmt.Fprintf(w, "┢"+strings.Repeat("━", colWidth*2+2)+"┪\n")
+	}
+
+	anomalyStr := fmt.Sprintf("ok: %s", statusIcon(!isAnomaly))
+	uploadStr := fmt.Sprintf("uploaded: %s", statusIcon(isUploaded))
+	failureStr := fmt.Sprintf("success: %s", statusIcon(!isFailed))
+
+	fmt.Fprintf(w, fmt.Sprintf("│ %s │\n",
+		utils.RightPad(
+			fmt.Sprintf("#%d", rID), colWidth*2)))
+
+	if url != "" {
+		fmt.Fprintf(w, fmt.Sprintf("│ %s │\n",
+			utils.RightPad(
+				fmt.Sprintf("%s (%s)", url, urlCategoryCode), colWidth*2)))
+	}
+
+	fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
+		utils.RightPad(testName, colWidth),
+		utils.RightPad(anomalyStr, colWidth)))
+
+	fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
+		utils.RightPad(failureStr, colWidth),
+		utils.RightPad(uploadStr, colWidth)))
+
+	if testKeys != "" {
+		if err := logTestKeys(w, testKeys); err != nil {
+			return err
+		}
+	}
+
+	if isLast {
+		fmt.Fprintf(w, "└┬────────────────────────────────────────────────┬┘\n")
+	}
+	return nil
+}
+
+func logMeasurementSummary(w io.Writer, f log.Fields) error {
+	colWidth := 12
+
+	totalCount := f.Get("total_count").(int64)
+	anomalyCount := f.Get("anomaly_count").(int64)
+	totalRuntime := f.Get("total_runtime").(float64)
+	dataUp := f.Get("data_usage_up").(float64)
+	dataDown := f.Get("data_usage_down").(float64)
+
+	startTime := f.Get("start_time").(time.Time)
+
+	asn := f.Get("asn").(uint)
+	countryCode := f.Get("network_country_code").(string)
+	networkName := f.Get("network_name").(string)
+
+	fmt.Fprintf(w, " │ %s  │\n",
+		utils.RightPad(startTime.Format(time.RFC822), (colWidth+3)*3),
+	)
+	fmt.Fprintf(w, " │ %s  │\n",
+		utils.RightPad(fmt.Sprintf("AS%d, %s (%s)", asn, networkName, countryCode), (colWidth+3)*3),
+	)
+	fmt.Fprintf(w, " │ %s   %s   %s │\n",
+		utils.RightPad(fmt.Sprintf("%.2fs", totalRuntime), colWidth),
+		utils.RightPad(fmt.Sprintf("%d/%d anmls", anomalyCount, totalCount), colWidth),
+		utils.RightPad(fmt.Sprintf("⬆ %s  ⬇ %s", formatSize(dataUp), formatSize(dataDown)), colWidth+4))
+	fmt.Fprintf(w, " └────────────────────────────────────────────────┘\n")
+
+	return nil
+}
+
+func logMeasurementJSON(w io.Writer, f log.Fields) error {
+	m := f.Get("measurement_json").(map[string]interface{})
+
+	json, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(w, string(json))
+	return nil
+}

cmd/ooniprobe/internal/log/handlers/cli/progress/progress.go

@@ -0,0 +1,126 @@
+// Package progress provides a simple terminal progress bar.
+package progress
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"io"
+	"math"
+	"strings"
+)
+
+// Bar is a progress bar.
+type Bar struct {
+	StartDelimiter string  // StartDelimiter for the bar ("|").
+	EndDelimiter   string  // EndDelimiter for the bar ("|").
+	Filled         string  // Filled section representation ("█").
+	Empty          string  // Empty section representation ("░")
+	Total          float64 // Total value.
+	Width          int     // Width of the bar.
+
+	value float64
+	tmpl  *template.Template
+	text  string
+}
+
+// New returns a new bar with the given total.
+func New(total float64) *Bar {
+	b := &Bar{
+		StartDelimiter: "|",
+		EndDelimiter:   "|",
+		Filled:         "█",
+		Empty:          "░",
+		Total:          total,
+		Width:          60,
+	}
+
+	b.Template(`{{.Percent | printf "%3.0f"}}% {{.Bar}} {{.Text}}`)
+
+	return b
+}
+
+// NewInt returns a new bar with the given total.
+func NewInt(total int) *Bar {
+	return New(float64(total))
+}
+
+// Text sets the text value.
+func (b *Bar) Text(s string) {
+	b.text = s
+}
+
+// Value sets the value.
+func (b *Bar) Value(n float64) {
+	if n > b.Total {
+		panic("Bar update value cannot be greater than the total")
+	}
+	b.value = n
+}
+
+// ValueInt sets the value.
+func (b *Bar) ValueInt(n int) {
+	b.Value(float64(n))
+}
+
+// Percent returns the percentage
+func (b *Bar) percent() float64 {
+	return (b.value / b.Total) * 100
+}
+
+// Bar returns the progress bar string.
+func (b *Bar) bar() string {
+	p := b.value / b.Total
+	filled := math.Ceil(float64(b.Width) * p)
+	empty := math.Floor(float64(b.Width) - filled)
+	s := b.StartDelimiter
+	s += strings.Repeat(b.Filled, int(filled))
+	s += strings.Repeat(b.Empty, int(empty))
+	s += b.EndDelimiter
+	return s
+}
+
+// String returns the progress bar.
+func (b *Bar) String() string {
+	var buf bytes.Buffer
+
+	data := struct {
+		Value          float64
+		Total          float64
+		Percent        float64
+		StartDelimiter string
+		EndDelimiter   string
+		Bar            string
+		Text           string
+	}{
+		Value:          b.value,
+		Text:           b.text,
+		StartDelimiter: b.StartDelimiter,
+		EndDelimiter:   b.EndDelimiter,
+		Percent:        b.percent(),
+		Bar:            b.bar(),
+	}
+
+	if err := b.tmpl.Execute(&buf, data); err != nil {
+		panic(err)
+	}
+
+	return buf.String()
+}
+
+// WriteTo writes the progress bar to w.
+func (b *Bar) WriteTo(w io.Writer) (int64, error) {
+	s := fmt.Sprintf("\r   %s ", b.String())
+	_, err := io.WriteString(w, s)
+	return int64(len(s)), err
+}
+
+// Template for rendering. This method will panic if the template fails to parse.
+func (b *Bar) Template(s string) {
+	t, err := template.New("").Parse(s)
+	if err != nil {
+		panic(err)
+	}
+
+	b.tmpl = t
+}

cmd/ooniprobe/internal/log/handlers/cli/results.go

@@ -0,0 +1,154 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+)
+
+func formatSpeed(speed float64) string {
+	if speed < 1000 {
+		return fmt.Sprintf("%.2f Kbit/s", speed)
+	} else if speed < 1000*1000 {
+		return fmt.Sprintf("%.2f Mbit/s", float32(speed)/1000)
+	} else if speed < 1000*1000*1000 {
+		return fmt.Sprintf("%.2f Gbit/s", float32(speed)/(1000*1000))
+	}
+	// WTF, you crazy?
+	return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000))
+}
+
+func formatSize(size float64) string {
+	if size < 1024 {
+		return fmt.Sprintf("%.1fK", size)
+	} else if size < 1024*1024 {
+		return fmt.Sprintf("%.1fM", size/1024.0)
+	} else if size < 1024*1024*1024 {
+		return fmt.Sprintf("%.1fG", size/(1024.0*1024.0))
+	}
+	// WTF, you crazy?
+	return fmt.Sprintf("%.1fT", size/(1024*1024*1024))
+}
+
+var summarizers = map[string]func(uint64, uint64, string) []string{
+	"websites": func(totalCount uint64, anomalyCount uint64, ss string) []string {
+		return []string{
+			fmt.Sprintf("%d tested", totalCount),
+			fmt.Sprintf("%d blocked", anomalyCount),
+			"",
+		}
+	},
+	"performance": func(totalCount uint64, anomalyCount uint64, ss string) []string {
+		var tk database.PerformanceTestKeys
+		if err := json.Unmarshal([]byte(ss), &tk); err != nil {
+			return nil
+		}
+		return []string{
+			fmt.Sprintf("Download: %s", formatSpeed(tk.Download)),
+			fmt.Sprintf("Upload: %s", formatSpeed(tk.Upload)),
+			fmt.Sprintf("Ping: %.2fms", tk.Ping),
+		}
+	},
+	"im": func(totalCount uint64, anomalyCount uint64, ss string) []string {
+		return []string{
+			fmt.Sprintf("%d tested", totalCount),
+			fmt.Sprintf("%d blocked", anomalyCount),
+			"",
+		}
+	},
+	"middlebox": func(totalCount uint64, anomalyCount uint64, ss string) []string {
+		return []string{
+			fmt.Sprintf("Detected: %v", anomalyCount > 0),
+			"",
+			"",
+		}
+	},
+	"circumvention": func(totalCount uint64, anomalyCount uint64, ss string) []string {
+		return []string{
+			fmt.Sprintf("%d tested", totalCount),
+			fmt.Sprintf("%d blocked", anomalyCount),
+			"",
+		}
+	},
+}
+
+func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string {
+	return summarizers[name](totalCount, anomalyCount, ss)
+}
+
+func logResultItem(w io.Writer, f log.Fields) error {
+	colWidth := 24
+	rID := f.Get("id").(int64)
+	name := f.Get("name").(string)
+	isDone := f.Get("is_done").(bool)
+	startTime := f.Get("start_time").(time.Time)
+	networkName := f.Get("network_name").(string)
+	asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("network_country_code").(string))
+	//runtime := f.Get("runtime").(float64)
+	//dataUsageUp := f.Get("dataUsageUp").(int64)
+	//dataUsageDown := f.Get("dataUsageDown").(int64)
+	index := f.Get("index").(int)
+	totalCount := f.Get("total_count").(int)
+	if index == 0 {
+		fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth*2+2)+"┓\n")
+	} else {
+		fmt.Fprintf(w, "┢"+strings.Repeat("━", colWidth*2+2)+"┪\n")
+	}
+
+	firstRow := utils.RightPad(fmt.Sprintf("#%d - %s", rID, startTime.Format(time.RFC822)), colWidth*2)
+	fmt.Fprintf(w, "┃ "+firstRow+" ┃\n")
+	fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n")
+
+	summary := makeSummary(name,
+		f.Get("measurement_count").(uint64),
+		f.Get("measurement_anomaly_count").(uint64),
+		f.Get("test_keys").(string))
+
+	fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
+		utils.RightPad(name, colWidth),
+		utils.RightPad(summary[0], colWidth)))
+	fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
+		utils.RightPad(networkName, colWidth),
+		utils.RightPad(summary[1], colWidth)))
+	fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
+		utils.RightPad(asn, colWidth),
+		utils.RightPad(summary[2], colWidth)))
+
+	if index == totalCount-1 {
+		if isDone == true {
+			fmt.Fprintf(w, "└┬──────────────┬─────────────┬───────────────────┬┘\n")
+		} else {
+			// We want the incomplete section to not have a footer
+			fmt.Fprintf(w, "└──────────────────────────────────────────────────┘\n")
+		}
+	}
+	return nil
+}
+
+func logResultSummary(w io.Writer, f log.Fields) error {
+
+	networks := f.Get("total_networks").(int64)
+	tests := f.Get("total_tests").(int64)
+	dataUp := f.Get("total_data_usage_up").(float64)
+	dataDown := f.Get("total_data_usage_down").(float64)
+	if tests == 0 {
+		fmt.Fprintf(w, "No results\n")
+		fmt.Fprintf(w, "Try running:\n")
+		fmt.Fprintf(w, "  ooniprobe run websites\n")
+		return nil
+	}
+	//              └┬──────────────┬─────────────┬───────────────┬
+	fmt.Fprintf(w, " │ %s │ %s │ %s │\n",
+		utils.RightPad(fmt.Sprintf("%d tests", tests), 12),
+		utils.RightPad(fmt.Sprintf("%d nets", networks), 11),
+		utils.RightPad(fmt.Sprintf("⬆ %s  ⬇ %s", formatSize(dataUp), formatSize(dataDown)), 17))
+	fmt.Fprintf(w, " └──────────────┴─────────────┴───────────────────┘\n")
+
+	return nil
+}

cmd/ooniprobe/internal/log/handlers/syslog/syslog.c

@@ -0,0 +1,39 @@
+#ifndef _WIN32
+#include <syslog.h>
+#endif
+
+void ooniprobe_openlog(void) {
+#ifndef _WIN32
+    (void)openlog("ooniprobe", LOG_PID, LOG_USER);
+#endif
+}
+
+void ooniprobe_log_debug(const char *message) {
+#ifndef _WIN32
+    (void)syslog(LOG_DEBUG, "%s", message);
+#endif
+}
+
+void ooniprobe_log_info(const char *message) {
+#ifndef _WIN32
+    (void)syslog(LOG_INFO, "%s", message);
+#endif
+}
+
+void ooniprobe_log_warning(const char *message) {
+#ifndef _WIN32
+    (void)syslog(LOG_WARNING, "%s", message);
+#endif
+}
+
+void ooniprobe_log_err(const char *message) {
+#ifndef _WIN32
+    (void)syslog(LOG_ERR, "%s", message);
+#endif
+}
+
+void ooniprobe_log_crit(const char *message) {
+#ifndef _WIN32
+    (void)syslog(LOG_CRIT, "%s", message);
+#endif
+}

cmd/ooniprobe/internal/log/handlers/syslog/syslog.go

@@ -0,0 +1,53 @@
+// Package syslog contains a syslog handler.
+//
+// We use this handler on macOS systems to log messages
+// when ooniprobe is running in the background.
+package syslog
+
+import (
+	"fmt"
+	"unsafe"
+
+	"github.com/apex/log"
+)
+
+/*
+#include<stdlib.h>
+
+void ooniprobe_openlog(void);
+void ooniprobe_log_debug(const char *message);
+void ooniprobe_log_info(const char *message);
+void ooniprobe_log_warning(const char *message);
+void ooniprobe_log_err(const char *message);
+void ooniprobe_log_crit(const char *message);
+*/
+import "C"
+
+// Default is the handler that emits logs with syslog
+var Default log.Handler = newhandler()
+
+type handler struct{}
+
+func newhandler() handler {
+	C.ooniprobe_openlog()
+	return handler{}
+}
+
+func (h handler) HandleLog(e *log.Entry) error {
+	message := fmt.Sprintf("%s %+v", e.Message, e.Fields)
+	cstr := C.CString(message)
+	defer C.free(unsafe.Pointer(cstr))
+	switch e.Level {
+	case log.DebugLevel:
+		C.ooniprobe_log_debug(cstr)
+	case log.InfoLevel:
+		C.ooniprobe_log_info(cstr)
+	case log.WarnLevel:
+		C.ooniprobe_log_warning(cstr)
+	case log.ErrorLevel:
+		C.ooniprobe_log_err(cstr)
+	default:
+		C.ooniprobe_log_crit(cstr)
+	}
+	return nil
+}

cmd/ooniprobe/internal/nettests/dash.go

@@ -0,0 +1,14 @@
+package nettests
+
+// Dash test implementation
+type Dash struct {
+}
+
+// Run starts the test
+func (d Dash) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder("dash")
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/facebook_messenger.go

@@ -0,0 +1,16 @@
+package nettests
+
+// FacebookMessenger test implementation
+type FacebookMessenger struct {
+}
+
+// Run starts the test
+func (h FacebookMessenger) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"facebook_messenger",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/groups.go

@@ -0,0 +1,52 @@
+package nettests
+
+// Group is a group of nettests
+type Group struct {
+	Label        string
+	Nettests     []Nettest
+	UnattendedOK bool
+}
+
+// All contains all the nettests that can be run by the user
+var All = map[string]Group{
+	"websites": {
+		Label: "Websites",
+		Nettests: []Nettest{
+			WebConnectivity{},
+		},
+		UnattendedOK: true,
+	},
+	"performance": {
+		Label: "Performance",
+		Nettests: []Nettest{
+			Dash{},
+			NDT{},
+		},
+	},
+	"middlebox": {
+		Label: "Middleboxes",
+		Nettests: []Nettest{
+			HTTPInvalidRequestLine{},
+			HTTPHeaderFieldManipulation{},
+		},
+		UnattendedOK: true,
+	},
+	"im": {
+		Label: "Instant Messaging",
+		Nettests: []Nettest{
+			FacebookMessenger{},
+			Telegram{},
+			WhatsApp{},
+		},
+		UnattendedOK: true,
+	},
+	"circumvention": {
+		Label: "Circumvention Tools",
+		Nettests: []Nettest{
+			Psiphon{},
+			RiseupVPN{},
+			Tor{},
+		},
+		UnattendedOK: true,
+	},
+}

cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go

@@ -0,0 +1,16 @@
+package nettests
+
+// HTTPHeaderFieldManipulation test implementation
+type HTTPHeaderFieldManipulation struct {
+}
+
+// Run starts the test
+func (h HTTPHeaderFieldManipulation) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"http_header_field_manipulation",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/http_invalid_request_line.go

@@ -0,0 +1,16 @@
+package nettests
+
+// HTTPInvalidRequestLine test implementation
+type HTTPInvalidRequestLine struct {
+}
+
+// Run starts the test
+func (h HTTPInvalidRequestLine) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"http_invalid_request_line",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/ndt.go

@@ -0,0 +1,15 @@
+package nettests
+
+// NDT test implementation. We use v7 of NDT since 2020-03-12.
+type NDT struct {
+}
+
+// Run starts the test
+func (n NDT) Run(ctl *Controller) error {
+	// Since 2020-03-18 probe-engine exports v7 as "ndt".
+	builder, err := ctl.Session.NewExperimentBuilder("ndt")
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/nettests.go

@@ -0,0 +1,219 @@
+package nettests
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/fatih/color"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
+	engine "github.com/ooni/probe-engine"
+	"github.com/ooni/probe-engine/model"
+	"github.com/pkg/errors"
+)
+
+// Nettest interface. Every Nettest should implement this.
+type Nettest interface {
+	Run(*Controller) error
+}
+
+// NewController creates a nettest controller
+func NewController(
+	nt Nettest, probe *ooni.Probe, res *database.Result, sess *engine.Session) *Controller {
+	return &Controller{
+		Probe:   probe,
+		nt:      nt,
+		res:     res,
+		Session: sess,
+	}
+}
+
+// Controller is passed to the run method of every Nettest
+// each nettest instance has one controller
+type Controller struct {
+	Probe       *ooni.Probe
+	Session     *engine.Session
+	res         *database.Result
+	nt          Nettest
+	ntCount     int
+	ntIndex     int
+	ntStartTime time.Time // used to calculate the eta
+	msmts       map[int64]*database.Measurement
+	inputIdxMap map[int64]int64 // Used to map mk idx to database id
+
+	// InputFiles optionally contains the names of the input
+	// files to read inputs from (only for nettests that take
+	// inputs, of course)
+	InputFiles []string
+
+	// Inputs contains inputs to be tested. These are specified
+	// using the command line using the --input flag.
+	Inputs []string
+
+	// numInputs is the total number of inputs
+	numInputs int
+
+	// curInputIdx is the current input index
+	curInputIdx int
+}
+
+// SetInputIdxMap is used to set the mapping of index into input. This mapping
+// is used to reference, for example, a particular URL based on the index inside
+// of the input list and the index of it in the database.
+func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error {
+	c.inputIdxMap = inputIdxMap
+	return nil
+}
+
+// SetNettestIndex is used to set the current nettest index and total nettest
+// count to compute a different progress percentage.
+func (c *Controller) SetNettestIndex(i, n int) {
+	c.ntCount = n
+	c.ntIndex = i
+}
+
+// Run runs the selected nettest using the related experiment
+// with the specified inputs.
+//
+// This function will continue to run in most cases but will
+// immediately halt if something's wrong with the file system.
+func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) error {
+	// This will configure the controller as handler for the callbacks
+	// called by ooni/probe-engine/experiment.Experiment.
+	builder.SetCallbacks(model.ExperimentCallbacks(c))
+	c.numInputs = len(inputs)
+	exp := builder.NewExperiment()
+	defer func() {
+		c.res.DataUsageDown += exp.KibiBytesReceived()
+		c.res.DataUsageUp += exp.KibiBytesSent()
+	}()
+
+	c.msmts = make(map[int64]*database.Measurement)
+
+	// These values are shared by every measurement
+	var reportID sql.NullString
+	resultID := c.res.ID
+
+	log.Debug(color.RedString("status.queued"))
+	log.Debug(color.RedString("status.started"))
+
+	if c.Probe.Config().Sharing.UploadResults {
+		if err := exp.OpenReport(); err != nil {
+			log.Debugf(
+				"%s: %s", color.RedString("failure.report_create"), err.Error(),
+			)
+		} else {
+			log.Debugf(color.RedString("status.report_create"))
+			reportID = sql.NullString{String: exp.ReportID(), Valid: true}
+		}
+	}
+
+	c.ntStartTime = time.Now()
+	for idx, input := range inputs {
+		if c.Probe.IsTerminated() == true {
+			log.Debug("isTerminated == true, breaking the input loop")
+			break
+		}
+		c.curInputIdx = idx // allow for precise progress
+		idx64 := int64(idx)
+		log.Debug(color.RedString("status.measurement_start"))
+		var urlID sql.NullInt64
+		if c.inputIdxMap != nil {
+			urlID = sql.NullInt64{Int64: c.inputIdxMap[idx64], Valid: true}
+		}
+
+		msmt, err := database.CreateMeasurement(
+			c.Probe.DB(), reportID, exp.Name(), c.res.MeasurementDir, idx, resultID, urlID,
+		)
+		if err != nil {
+			return errors.Wrap(err, "failed to create measurement")
+		}
+		c.msmts[idx64] = msmt
+
+		if input != "" {
+			c.OnProgress(0, fmt.Sprintf("processing input: %s", input))
+		}
+		measurement, err := exp.Measure(input)
+		if err != nil {
+			log.WithError(err).Debug(color.RedString("failure.measurement"))
+			if err := c.msmts[idx64].Failed(c.Probe.DB(), err.Error()); err != nil {
+				return errors.Wrap(err, "failed to mark measurement as failed")
+			}
+			// Even with a failed measurement, we want to continue. We want to
+			// record and submit the information we have. Saving the information
+			// is useful for local inspection. Submitting it is useful to us to
+			// undertsand what went wrong (censorship? bug? anomaly?).
+		}
+
+		saveToDisk := true
+		if c.Probe.Config().Sharing.UploadResults {
+			// Implementation note: SubmitMeasurement will fail here if we did fail
+			// to open the report but we still want to continue. There will be a
+			// bit of a spew in the logs, perhaps, but stopping seems less efficient.
+			if err := exp.SubmitAndUpdateMeasurement(measurement); err != nil {
+				log.Debug(color.RedString("failure.measurement_submission"))
+				if err := c.msmts[idx64].UploadFailed(c.Probe.DB(), err.Error()); err != nil {
+					return errors.Wrap(err, "failed to mark upload as failed")
+				}
+			} else if err := c.msmts[idx64].UploadSucceeded(c.Probe.DB()); err != nil {
+				return errors.Wrap(err, "failed to mark upload as succeeded")
+			} else {
+				// Everything went OK, don't save to disk
+				saveToDisk = false
+			}
+		}
+		// We only save the measurement to disk if we failed to upload the measurement
+		if saveToDisk == true {
+			if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil {
+				return errors.Wrap(err, "failed to save measurement on disk")
+			}
+		}
+
+		if err := c.msmts[idx64].Done(c.Probe.DB()); err != nil {
+			return errors.Wrap(err, "failed to mark measurement as done")
+		}
+
+		// We're not sure whether it's enough to log the error or we should
+		// instead also mark the measurement as failed. Strictly speaking this
+		// is an inconsistency between the code that generate the measurement
+		// and the code that process the measurement. We do have some data
+		// but we're not gonna have a summary. To be reconsidered.
+		tk, err := exp.GetSummaryKeys(measurement)
+		if err != nil {
+			log.WithError(err).Error("failed to obtain testKeys")
+			continue
+		}
+		log.Debugf("Fetching: %d %v", idx, c.msmts[idx64])
+		if err := database.AddTestKeys(c.Probe.DB(), c.msmts[idx64], tk); err != nil {
+			return errors.Wrap(err, "failed to add test keys to summary")
+		}
+	}
+
+	log.Debugf("status.end")
+	return nil
+}
+
+// OnProgress should be called when a new progress event is available.
+func (c *Controller) OnProgress(perc float64, msg string) {
+	log.Debugf("OnProgress: %f - %s", perc, msg)
+	var eta float64
+	eta = -1.0
+	if c.numInputs > 1 {
+		// make the percentage relative to the current input over all inputs
+		floor := (float64(c.curInputIdx) / float64(c.numInputs))
+		step := 1.0 / float64(c.numInputs)
+		perc = floor + perc*step
+		if c.curInputIdx > 0 {
+			eta = (time.Now().Sub(c.ntStartTime).Seconds() / float64(c.curInputIdx)) * float64(c.numInputs-c.curInputIdx)
+		}
+	}
+	if c.ntCount > 0 {
+		// make the percentage relative to the current nettest over all nettests
+		perc = float64(c.ntIndex)/float64(c.ntCount) + perc/float64(c.ntCount)
+	}
+	key := fmt.Sprintf("%T", c.nt)
+	output.Progress(key, perc, eta, msg)
+}

cmd/ooniprobe/internal/nettests/nettests_test.go

@@ -0,0 +1,52 @@
+package nettests
+
+import (
+	"io/ioutil"
+	"path"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils/shutil"
+)
+
+func newOONIProbe(t *testing.T) *ooni.Probe {
+	homePath, err := ioutil.TempDir("", "ooniprobetests")
+	if err != nil {
+		t.Fatal(err)
+	}
+	configPath := path.Join(homePath, "config.json")
+	testingConfig := path.Join("..", "..", "testdata", "testing-config.json")
+	shutil.Copy(testingConfig, configPath, false)
+	probe := ooni.NewProbe(configPath, homePath)
+	swName := "ooniprobe-cli-tests"
+	swVersion := "3.0.0-alpha"
+	err = probe.Init(swName, swVersion)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return probe
+}
+
+func TestCreateContext(t *testing.T) {
+	newOONIProbe(t)
+}
+
+func TestRun(t *testing.T) {
+	probe := newOONIProbe(t)
+	sess, err := probe.NewSession()
+	if err != nil {
+		t.Fatal(err)
+	}
+	network, err := database.CreateNetwork(probe.DB(), sess)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res, err := database.CreateResult(probe.DB(), probe.Home(), "middlebox", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	nt := HTTPInvalidRequestLine{}
+	ctl := NewController(nt, probe, res, sess)
+	nt.Run(ctl)
+}

cmd/ooniprobe/internal/nettests/psiphon.go

@@ -0,0 +1,16 @@
+package nettests
+
+// Psiphon test implementation
+type Psiphon struct {
+}
+
+// Run starts the test
+func (h Psiphon) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"psiphon",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/riseupvpn.go

@@ -0,0 +1,17 @@
+package nettests
+
+// RiseupVPN test implementation
+type RiseupVPN struct {
+}
+
+// Run starts the test
+func (h RiseupVPN) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"riseupvpn",
+	)
+	if err != nil {
+		return err
+	}
+
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/run.go

@@ -0,0 +1,82 @@
+package nettests
+
+import (
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"github.com/pkg/errors"
+)
+
+// RunGroupConfig contains the settings for running a nettest group.
+type RunGroupConfig struct {
+	GroupName  string
+	Probe      *ooni.Probe
+	InputFiles []string
+	Inputs     []string
+}
+
+// RunGroup runs a group of nettests according to the specified config.
+func RunGroup(config RunGroupConfig) error {
+	if config.Probe.IsTerminated() == true {
+		log.Debugf("context is terminated, stopping runNettestGroup early")
+		return nil
+	}
+
+	sess, err := config.Probe.NewSession()
+	if err != nil {
+		log.WithError(err).Error("Failed to create a measurement session")
+		return err
+	}
+	defer sess.Close()
+
+	err = sess.MaybeLookupLocation()
+	if err != nil {
+		log.WithError(err).Error("Failed to lookup the location of the probe")
+		return err
+	}
+	network, err := database.CreateNetwork(config.Probe.DB(), sess)
+	if err != nil {
+		log.WithError(err).Error("Failed to create the network row")
+		return err
+	}
+	if err := sess.MaybeLookupBackends(); err != nil {
+		log.WithError(err).Warn("Failed to discover OONI backends")
+		return err
+	}
+
+	group, ok := All[config.GroupName]
+	if !ok {
+		log.Errorf("No test group named %s", config.GroupName)
+		return errors.New("invalid test group name")
+	}
+	log.Debugf("Running test group %s", group.Label)
+
+	result, err := database.CreateResult(
+		config.Probe.DB(), config.Probe.Home(), config.GroupName, network.ID)
+	if err != nil {
+		log.Errorf("DB result error: %s", err)
+		return err
+	}
+
+	config.Probe.ListenForSignals()
+	config.Probe.MaybeListenForStdinClosed()
+	for i, nt := range group.Nettests {
+		if config.Probe.IsTerminated() == true {
+			log.Debugf("context is terminated, stopping group.Nettests early")
+			break
+		}
+		log.Debugf("Running test %T", nt)
+		ctl := NewController(nt, config.Probe, result, sess)
+		ctl.InputFiles = config.InputFiles
+		ctl.Inputs = config.Inputs
+		ctl.SetNettestIndex(i, len(group.Nettests))
+		if err = nt.Run(ctl); err != nil {
+			log.WithError(err).Errorf("Failed to run %s", group.Label)
+		}
+	}
+
+	if err = result.Finished(config.Probe.DB()); err != nil {
+		return err
+	}
+	return nil
+}

cmd/ooniprobe/internal/nettests/telegram.go

@@ -0,0 +1,16 @@
+package nettests
+
+// Telegram test implementation
+type Telegram struct {
+}
+
+// Run starts the test
+func (h Telegram) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"telegram",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/tor.go

@@ -0,0 +1,16 @@
+package nettests
+
+// Tor test implementation
+type Tor struct {
+}
+
+// Run starts the test
+func (h Tor) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"tor",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/web_connectivity.go

@@ -0,0 +1,61 @@
+package nettests
+
+import (
+	"context"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	engine "github.com/ooni/probe-engine"
+)
+
+func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
+	inputloader := engine.NewInputLoader(engine.InputLoaderConfig{
+		InputPolicy:   engine.InputOrQueryTestLists,
+		Session:       ctl.Session,
+		SourceFiles:   ctl.InputFiles,
+		StaticInputs:  ctl.Inputs,
+		URLCategories: categories,
+		URLLimit:      limit,
+	})
+	testlist, err := inputloader.Load(context.Background())
+	var urls []string
+	urlIDMap := make(map[int64]int64)
+	if err != nil {
+		return nil, nil, err
+	}
+	for idx, url := range testlist {
+		log.Debugf("Going over URL %d", idx)
+		urlID, err := database.CreateOrUpdateURL(
+			ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode,
+		)
+		if err != nil {
+			log.Error("failed to add to the URL table")
+			return nil, nil, err
+		}
+		log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
+		urlIDMap[int64(idx)] = urlID
+		urls = append(urls, url.URL)
+	}
+	return urls, urlIDMap, nil
+}
+
+// WebConnectivity test implementation
+type WebConnectivity struct {
+}
+
+// Run starts the test
+func (n WebConnectivity) Run(ctl *Controller) error {
+	log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
+	urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesURLLimit, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
+	if err != nil {
+		return err
+	}
+	ctl.SetInputIdxMap(urlIDMap)
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"web_connectivity",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, urls)
+}

cmd/ooniprobe/internal/nettests/whatsapp.go

@@ -0,0 +1,16 @@
+package nettests
+
+// WhatsApp test implementation
+type WhatsApp struct {
+}
+
+// Run starts the test
+func (h WhatsApp) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder(
+		"whatsapp",
+	)
+	if err != nil {
+		return err
+	}
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/ooni/ooni.go

@@ -0,0 +1,284 @@
+package ooni
+
+import (
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"sync/atomic"
+	"syscall"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/bindata"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+	engine "github.com/ooni/probe-engine"
+	"github.com/pkg/errors"
+	"upper.io/db.v3/lib/sqlbuilder"
+)
+
+// ProbeCLI is the OONI Probe CLI context.
+type ProbeCLI interface {
+	Config() *config.Config
+	DB() sqlbuilder.Database
+	IsBatch() bool
+	Home() string
+	TempDir() string
+	NewProbeEngine() (ProbeEngine, error)
+}
+
+// ProbeEngine is an instance of the OONI Probe engine.
+type ProbeEngine interface {
+	Close() error
+	MaybeLookupLocation() error
+	ProbeASNString() string
+	ProbeCC() string
+	ProbeIP() string
+	ProbeNetworkName() string
+}
+
+// Probe contains the ooniprobe CLI context.
+type Probe struct {
+	config  *config.Config
+	db      sqlbuilder.Database
+	isBatch bool
+
+	home    string
+	tempDir string
+
+	dbPath     string
+	configPath string
+
+	// We need to use a int32 in order to use the atomic.AddInt32/LoadInt32
+	// operations to ensure consistent reads of the variables. We do not use
+	// a 64 bit integer here because that may lead to crashes with 32 bit
+	// OSes as documented in https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
+	isTerminatedAtomicInt int32
+
+	softwareName    string
+	softwareVersion string
+}
+
+// SetIsBatch sets the value of isBatch.
+func (p *Probe) SetIsBatch(v bool) {
+	p.isBatch = v
+}
+
+// IsBatch returns whether we're running in batch mode.
+func (p *Probe) IsBatch() bool {
+	return p.isBatch
+}
+
+// Config returns the configuration
+func (p *Probe) Config() *config.Config {
+	return p.config
+}
+
+// DB returns the database we're using
+func (p *Probe) DB() sqlbuilder.Database {
+	return p.db
+}
+
+// Home returns the home directory.
+func (p *Probe) Home() string {
+	return p.home
+}
+
+// TempDir returns the temporary directory.
+func (p *Probe) TempDir() string {
+	return p.tempDir
+}
+
+// IsTerminated checks to see if the isTerminatedAtomicInt is set to a non zero
+// value and therefore we have received the signal to shutdown the running test
+func (p *Probe) IsTerminated() bool {
+	i := atomic.LoadInt32(&p.isTerminatedAtomicInt)
+	return i != 0
+}
+
+// Terminate interrupts the running context
+func (p *Probe) Terminate() {
+	atomic.AddInt32(&p.isTerminatedAtomicInt, 1)
+}
+
+// ListenForSignals will listen for SIGINT and SIGTERM. When it receives those
+// signals it will set isTerminatedAtomicInt to non-zero, which will cleanly
+// shutdown the test logic.
+//
+// TODO refactor this to use a cancellable context.Context instead of a bool
+// flag, probably as part of: https://github.com/ooni/probe-cli/issues/45
+func (p *Probe) ListenForSignals() {
+	s := make(chan os.Signal, 1)
+	signal.Notify(s, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-s
+		log.Info("caught a stop signal, shutting down cleanly")
+		p.Terminate()
+	}()
+}
+
+// MaybeListenForStdinClosed will treat any error on stdin just
+// like SIGTERM if and only if
+//
+//     os.Getenv("OONI_STDIN_EOF_IMPLIES_SIGTERM") == "true"
+//
+// When this feature is enabled, a collateral effect is that we swallow
+// whatever is passed to us on the standard input.
+//
+// See https://github.com/ooni/probe-cli/pull/111 for more info
+// regarding the design of this functionality.
+//
+// TODO refactor this to use a cancellable context.Context instead of a bool
+// flag, probably as part of: https://github.com/ooni/probe-cli/issues/45
+func (p *Probe) MaybeListenForStdinClosed() {
+	if os.Getenv("OONI_STDIN_EOF_IMPLIES_SIGTERM") != "true" {
+		return
+	}
+	go func() {
+		defer p.Terminate()
+		defer log.Info("stdin closed, shutting down cleanly")
+		b := make([]byte, 1<<10)
+		for {
+			if _, err := os.Stdin.Read(b); err != nil {
+				return
+			}
+		}
+	}()
+}
+
+// Init the OONI manager
+func (p *Probe) Init(softwareName, softwareVersion string) error {
+	var err error
+
+	if err = MaybeInitializeHome(p.home); err != nil {
+		return err
+	}
+
+	if p.configPath != "" {
+		log.Debugf("Reading config file from %s", p.configPath)
+		p.config, err = config.ReadConfig(p.configPath)
+	} else {
+		log.Debug("Reading default config file")
+		p.config, err = InitDefaultConfig(p.home)
+	}
+	if err != nil {
+		return err
+	}
+	if err = p.config.MaybeMigrate(); err != nil {
+		return errors.Wrap(err, "migrating config")
+	}
+
+	p.dbPath = utils.DBDir(p.home, "main")
+	log.Debugf("Connecting to database sqlite3://%s", p.dbPath)
+	db, err := database.Connect(p.dbPath)
+	if err != nil {
+		return err
+	}
+	p.db = db
+
+	tempDir, err := ioutil.TempDir("", "ooni")
+	if err != nil {
+		return errors.Wrap(err, "creating TempDir")
+	}
+	p.tempDir = tempDir
+
+	p.softwareName = softwareName
+	p.softwareVersion = softwareVersion
+	return nil
+}
+
+// NewSession creates a new ooni/probe-engine session using the
+// current configuration inside the context. The caller must close
+// the session when done using it, by calling sess.Close().
+func (p *Probe) NewSession() (*engine.Session, error) {
+	kvstore, err := engine.NewFileSystemKVStore(
+		utils.EngineDir(p.home),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating engine's kvstore")
+	}
+	return engine.NewSession(engine.SessionConfig{
+		AssetsDir:       utils.AssetsDir(p.home),
+		KVStore:         kvstore,
+		Logger:          enginex.Logger,
+		SoftwareName:    p.softwareName,
+		SoftwareVersion: p.softwareVersion,
+		TempDir:         p.tempDir,
+	})
+}
+
+// NewProbeEngine creates a new ProbeEngine instance.
+func (p *Probe) NewProbeEngine() (ProbeEngine, error) {
+	sess, err := p.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	return sess, nil
+}
+
+// NewProbe creates a new probe instance.
+func NewProbe(configPath string, homePath string) *Probe {
+	return &Probe{
+		home:                  homePath,
+		config:                &config.Config{},
+		configPath:            configPath,
+		isTerminatedAtomicInt: 0,
+	}
+}
+
+// MaybeInitializeHome does the setup for a new OONI Home
+func MaybeInitializeHome(home string) error {
+	for _, d := range utils.RequiredDirs(home) {
+		if _, e := os.Stat(d); e != nil {
+			if err := os.MkdirAll(d, 0700); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// InitDefaultConfig reads the config from common locations or creates it if
+// missing.
+func InitDefaultConfig(home string) (*config.Config, error) {
+	var (
+		err        error
+		c          *config.Config
+		configPath = utils.ConfigPath(home)
+	)
+
+	c, err = config.ReadConfig(configPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Debugf("writing default config to %s", configPath)
+			var data []byte
+			data, err = bindata.Asset("data/default-config.json")
+			if err != nil {
+				return nil, err
+			}
+			if err = ioutil.WriteFile(configPath, data, 0644); err != nil {
+				return nil, err
+			}
+			// If the user did the informed consent procedure in
+			// probe-legacy, migrate it over.
+			if utils.DidLegacyInformedConsent() {
+				c, err := config.ReadConfig(configPath)
+				if err != nil {
+					return nil, err
+				}
+				c.Lock()
+				c.InformedConsent = true
+				c.Unlock()
+				if err := c.Write(); err != nil {
+					return nil, err
+				}
+			}
+
+			return InitDefaultConfig(home)
+		}
+		return nil, err
+	}
+
+	return c, nil
+}

cmd/ooniprobe/internal/ooni/ooni_test.go

@@ -0,0 +1,29 @@
+package ooni
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+func TestInit(t *testing.T) {
+	ooniHome, err := ioutil.TempDir("", "oonihome")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(ooniHome)
+
+	probe := NewProbe("", ooniHome)
+	swName := "ooniprobe-cli-tests"
+	swVersion := "3.0.0-alpha"
+	if err := probe.Init(swName, swVersion); err != nil {
+		t.Error(err)
+		t.Fatal("failed to init the context")
+	}
+
+	configPath := path.Join(ooniHome, "config.json")
+	if _, err := os.Stat(configPath); os.IsNotExist(err) {
+		t.Fatal("config file was not created")
+	}
+}

cmd/ooniprobe/internal/oonitest/oonitest.go

@@ -0,0 +1,126 @@
+// Package oonitest contains code used for testing.
+package oonitest
+
+import (
+	"sync"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
+	"upper.io/db.v3/lib/sqlbuilder"
+)
+
+// FakeOutput allows to fake the output package.
+type FakeOutput struct {
+	FakeSectionTitle []string
+	mu               sync.Mutex
+}
+
+// SectionTitle writes the section title.
+func (fo *FakeOutput) SectionTitle(s string) {
+	fo.mu.Lock()
+	defer fo.mu.Unlock()
+	fo.FakeSectionTitle = append(fo.FakeSectionTitle, s)
+}
+
+// FakeProbeCLI fakes ooni.ProbeCLI
+type FakeProbeCLI struct {
+	FakeConfig         *config.Config
+	FakeDB             sqlbuilder.Database
+	FakeIsBatch        bool
+	FakeHome           string
+	FakeTempDir        string
+	FakeProbeEnginePtr ooni.ProbeEngine
+	FakeProbeEngineErr error
+}
+
+// Config implements ProbeCLI.Config
+func (cli *FakeProbeCLI) Config() *config.Config {
+	return cli.FakeConfig
+}
+
+// DB implements ProbeCLI.DB
+func (cli *FakeProbeCLI) DB() sqlbuilder.Database {
+	return cli.FakeDB
+}
+
+// IsBatch implements ProbeCLI.IsBatch
+func (cli *FakeProbeCLI) IsBatch() bool {
+	return cli.FakeIsBatch
+}
+
+// Home implements ProbeCLI.Home
+func (cli *FakeProbeCLI) Home() string {
+	return cli.FakeHome
+}
+
+// TempDir implements ProbeCLI.TempDir
+func (cli *FakeProbeCLI) TempDir() string {
+	return cli.FakeTempDir
+}
+
+// NewProbeEngine implements ProbeCLI.NewProbeEngine
+func (cli *FakeProbeCLI) NewProbeEngine() (ooni.ProbeEngine, error) {
+	return cli.FakeProbeEnginePtr, cli.FakeProbeEngineErr
+}
+
+var _ ooni.ProbeCLI = &FakeProbeCLI{}
+
+// FakeProbeEngine fakes ooni.ProbeEngine
+type FakeProbeEngine struct {
+	FakeClose               error
+	FakeMaybeLookupLocation error
+	FakeProbeASNString      string
+	FakeProbeCC             string
+	FakeProbeIP             string
+	FakeProbeNetworkName    string
+}
+
+// Close implements ProbeEngine.Close
+func (eng *FakeProbeEngine) Close() error {
+	return eng.FakeClose
+}
+
+// MaybeLookupLocation implements ProbeEngine.MaybeLookupLocation
+func (eng *FakeProbeEngine) MaybeLookupLocation() error {
+	return eng.FakeMaybeLookupLocation
+}
+
+// ProbeASNString implements ProbeEngine.ProbeASNString
+func (eng *FakeProbeEngine) ProbeASNString() string {
+	return eng.FakeProbeASNString
+}
+
+// ProbeCC implements ProbeEngine.ProbeCC
+func (eng *FakeProbeEngine) ProbeCC() string {
+	return eng.FakeProbeCC
+}
+
+// ProbeIP implements ProbeEngine.ProbeIP
+func (eng *FakeProbeEngine) ProbeIP() string {
+	return eng.FakeProbeIP
+}
+
+// ProbeNetworkName implements ProbeEngine.ProbeNetworkName
+func (eng *FakeProbeEngine) ProbeNetworkName() string {
+	return eng.FakeProbeNetworkName
+}
+
+var _ ooni.ProbeEngine = &FakeProbeEngine{}
+
+// FakeLoggerHandler fakes apex.log.Handler.
+type FakeLoggerHandler struct {
+	FakeEntries []*log.Entry
+	FakeErr     error
+	mu          sync.Mutex
+}
+
+// HandleLog implements Handler.HandleLog.
+func (handler *FakeLoggerHandler) HandleLog(entry *log.Entry) error {
+	handler.mu.Lock()
+	defer handler.mu.Unlock()
+	handler.FakeEntries = append(handler.FakeEntries, entry)
+	return handler.FakeErr
+}
+
+var _ log.Handler = &FakeLoggerHandler{}

cmd/ooniprobe/internal/output/output.go

@@ -0,0 +1,170 @@
+package output
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+)
+
+// MeasurementJSON prints the JSON of a measurement
+func MeasurementJSON(j map[string]interface{}) {
+	log.WithFields(log.Fields{
+		"type":             "measurement_json",
+		"measurement_json": j,
+	}).Info("Measurement JSON")
+}
+
+// Progress logs a progress type event
+func Progress(key string, perc float64, eta float64, msg string) {
+	log.WithFields(log.Fields{
+		"type":       "progress",
+		"key":        key,
+		"percentage": perc,
+		"eta":        eta,
+	}).Info(msg)
+}
+
+type MeasurementSummaryData struct {
+	TotalRuntime       float64
+	TotalCount         int64
+	AnomalyCount       int64
+	DataUsageUp        float64
+	DataUsageDown      float64
+	ASN                uint
+	NetworkName        string
+	NetworkCountryCode string
+	StartTime          time.Time
+}
+
+func MeasurementSummary(msmt MeasurementSummaryData) {
+	log.WithFields(log.Fields{
+		"type":                 "measurement_summary",
+		"total_runtime":        msmt.TotalRuntime,
+		"total_count":          msmt.TotalCount,
+		"anomaly_count":        msmt.AnomalyCount,
+		"data_usage_down":      msmt.DataUsageDown,
+		"data_usage_up":        msmt.DataUsageUp,
+		"asn":                  msmt.ASN,
+		"network_country_code": msmt.NetworkCountryCode,
+		"network_name":         msmt.NetworkName,
+		"start_time":           msmt.StartTime,
+	}).Info("measurement summary")
+}
+
+// MeasurementItem logs a progress type event
+func MeasurementItem(msmt database.MeasurementURLNetwork, isFirst bool, isLast bool) {
+	log.WithFields(log.Fields{
+		"type":     "measurement_item",
+		"is_first": isFirst,
+		"is_last":  isLast,
+
+		"id":                    msmt.Measurement.ID,
+		"test_name":             msmt.TestName,
+		"test_group_name":       msmt.Result.TestGroupName,
+		"start_time":            msmt.Measurement.StartTime,
+		"test_keys":             msmt.TestKeys,
+		"network_country_code":  msmt.Network.CountryCode,
+		"network_name":          msmt.Network.NetworkName,
+		"asn":                   msmt.Network.ASN,
+		"runtime":               msmt.Measurement.Runtime,
+		"url":                   msmt.URL.URL.String,
+		"url_category_code":     msmt.URL.CategoryCode.String,
+		"url_country_code":      msmt.URL.CountryCode.String,
+		"is_anomaly":            msmt.IsAnomaly.Bool,
+		"is_uploaded":           msmt.IsUploaded,
+		"is_upload_failed":      msmt.IsUploadFailed,
+		"upload_failure_msg":    msmt.UploadFailureMsg.String,
+		"is_failed":             msmt.IsFailed,
+		"failure_msg":           msmt.FailureMsg.String,
+		"is_done":               msmt.Measurement.IsDone,
+		"report_file_path":      msmt.ReportFilePath.String,
+		"measurement_file_path": msmt.MeasurementFilePath.String,
+	}).Info("measurement")
+}
+
+// ResultItemData is the metadata about a result
+type ResultItemData struct {
+	ID                      int64
+	Name                    string
+	StartTime               time.Time
+	TestKeys                string
+	MeasurementCount        uint64
+	MeasurementAnomalyCount uint64
+	Runtime                 float64
+	Country                 string
+	NetworkName             string
+	ASN                     uint
+	Done                    bool
+	DataUsageDown           float64
+	DataUsageUp             float64
+	Index                   int
+	TotalCount              int
+}
+
+// ResultItem logs a progress type event
+func ResultItem(result ResultItemData) {
+	log.WithFields(log.Fields{
+		"type":                      "result_item",
+		"id":                        result.ID,
+		"name":                      result.Name,
+		"start_time":                result.StartTime,
+		"test_keys":                 result.TestKeys,
+		"measurement_count":         result.MeasurementCount,
+		"measurement_anomaly_count": result.MeasurementAnomalyCount,
+		"network_country_code":      result.Country,
+		"network_name":              result.NetworkName,
+		"asn":                       result.ASN,
+		"runtime":                   result.Runtime,
+		"is_done":                   result.Done,
+		"data_usage_down":           result.DataUsageDown,
+		"data_usage_up":             result.DataUsageUp,
+		"index":                     result.Index,
+		"total_count":               result.TotalCount,
+	}).Info("result item")
+}
+
+type ResultSummaryData struct {
+	TotalTests         int64
+	TotalDataUsageUp   float64
+	TotalDataUsageDown float64
+	TotalNetworks      int64
+}
+
+func ResultSummary(result ResultSummaryData) {
+	log.WithFields(log.Fields{
+		"type":                  "result_summary",
+		"total_tests":           result.TotalTests,
+		"total_data_usage_up":   result.TotalDataUsageUp,
+		"total_data_usage_down": result.TotalDataUsageDown,
+		"total_networks":        result.TotalNetworks,
+	}).Info("result summary")
+}
+
+// SectionTitle is the title of a section
+func SectionTitle(text string) {
+	log.WithFields(log.Fields{
+		"type":  "section_title",
+		"title": text,
+	}).Info(text)
+}
+
+func Paragraph(text string) {
+	const width = 80
+	fmt.Println(utils.WrapString(text, width))
+}
+
+func Bullet(text string) {
+	const width = 80
+	fmt.Printf("• %s\n", utils.WrapString(text, width))
+}
+
+func PressEnterToContinue(text string) error {
+	fmt.Print(text)
+	_, err := bufio.NewReader(os.Stdin).ReadBytes('\n')
+	return err
+}

cmd/ooniprobe/internal/utils/homedir/homedir.go

@@ -0,0 +1,211 @@
+package homedir
+
+// Stolen from: https://github.com/puma/puma-dev/blob/master/homedir/homedir.go
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+// DisableCache will disable caching of the home directory. Caching is enabled
+// by default.
+var DisableCache bool
+
+var homedirCache string
+var cacheLock sync.RWMutex
+
+// ErrNoHomeDir when no home dir could be found
+var ErrNoHomeDir = errors.New("no home directory available")
+
+// Dir returns the home directory for the executing user.
+//
+// This uses an OS-specific method for discovering the home directory.
+// An error is returned if a home directory cannot be detected.
+func Dir() (string, error) {
+	if !DisableCache {
+		cacheLock.RLock()
+		cached := homedirCache
+		cacheLock.RUnlock()
+		if cached != "" {
+			return cached, nil
+		}
+	}
+
+	cacheLock.Lock()
+	defer cacheLock.Unlock()
+
+	var result string
+	var err error
+	switch runtime.GOOS {
+	case "windows":
+		result, err = dirWindows()
+	case "darwin":
+		result, err = dirDarwin()
+	default:
+		// Unix-like system, so just assume Unix
+		result, err = dirUnix()
+	}
+
+	if err != nil {
+		return "", err
+	}
+
+	homedirCache = result
+	return result, nil
+}
+
+// Expand expands the path to include the home directory if the path
+// is prefixed with `~`. If it isn't prefixed with `~`, the path is
+// returned as-is.
+func Expand(path string) (string, error) {
+	if len(path) == 0 {
+		return path, nil
+	}
+
+	if path[0] != '~' {
+		return path, nil
+	}
+
+	if len(path) > 1 && path[1] != '/' && path[1] != '\\' {
+		return "", errors.New("cannot expand user-specific home dir")
+	}
+
+	dir, err := Dir()
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(dir, path[1:]), nil
+}
+
+func MustExpand(path string) string {
+	str, err := Expand(path)
+	if err != nil {
+		panic(err)
+	}
+
+	return str
+}
+
+func dirDarwin() (string, error) {
+	// First prefer the HOME environmental variable
+	if home := os.Getenv("HOME"); home != "" {
+		return home, nil
+	}
+
+	var stdout bytes.Buffer
+
+	// If that fails, try OS specific commands
+	cmd := exec.Command("sh", "-c", `dscl -q . -read /Users/"$(whoami)" NFSHomeDirectory | sed 's/^[^ ]*: //'`)
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		result := strings.TrimSpace(stdout.String())
+		if result != "" {
+			return result, nil
+		}
+	}
+
+	// try the shell
+	stdout.Reset()
+	cmd = exec.Command("sh", "-c", "cd && pwd")
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		result := strings.TrimSpace(stdout.String())
+		if result != "" {
+			return result, nil
+		}
+	}
+
+	// try to figure out the user and check the default location
+	stdout.Reset()
+	cmd = exec.Command("whoami")
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		user := strings.TrimSpace(stdout.String())
+
+		path := "/Users/" + user
+
+		stat, err := os.Stat(path)
+		if err == nil && stat.IsDir() {
+			return path, nil
+		}
+	}
+
+	return "", ErrNoHomeDir
+}
+
+func dirUnix() (string, error) {
+	// First prefer the HOME environmental variable
+	if home := os.Getenv("HOME"); home != "" {
+		return home, nil
+	}
+
+	var stdout bytes.Buffer
+
+	// If that fails, try OS specific commands
+	cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid()))
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		if passwd := strings.TrimSpace(stdout.String()); passwd != "" {
+			// username:password:uid:gid:gecos:home:shell
+			passwdParts := strings.SplitN(passwd, ":", 7)
+			if len(passwdParts) > 5 {
+				return passwdParts[5], nil
+			}
+		}
+	}
+
+	// If all else fails, try the shell
+	stdout.Reset()
+	cmd = exec.Command("sh", "-c", "cd && pwd")
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		result := strings.TrimSpace(stdout.String())
+		if result == "" {
+			return "", errors.New("blank output when reading home directory")
+		}
+	}
+
+	// try to figure out the user and check the default location
+	stdout.Reset()
+	cmd = exec.Command("whoami")
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		user := strings.TrimSpace(stdout.String())
+
+		path := "/home/" + user
+
+		stat, err := os.Stat(path)
+		if err == nil && stat.IsDir() {
+			return path, nil
+		}
+	}
+
+	return "", ErrNoHomeDir
+}
+
+func dirWindows() (string, error) {
+	// First prefer the HOME environmental variable
+	if home := os.Getenv("HOME"); home != "" {
+		return home, nil
+	}
+
+	drive := os.Getenv("HOMEDRIVE")
+	path := os.Getenv("HOMEPATH")
+	home := drive + path
+	if drive == "" || path == "" {
+		home = os.Getenv("USERPROFILE")
+	}
+	if home == "" {
+		return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank")
+	}
+
+	return home, nil
+}

cmd/ooniprobe/internal/utils/paths.go

@@ -0,0 +1,98 @@
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils/homedir"
+)
+
+// RequiredDirs returns the required ooni home directories
+func RequiredDirs(home string) []string {
+	requiredDirs := []string{}
+	requiredSubdirs := []string{"assets", "db", "msmts"}
+	for _, d := range requiredSubdirs {
+		requiredDirs = append(requiredDirs, filepath.Join(home, d))
+	}
+	return requiredDirs
+}
+
+// ConfigPath returns the default path to the config file
+func ConfigPath(home string) string {
+	return filepath.Join(home, "config.json")
+}
+
+// AssetsDir returns the assets data dir for the given OONI Home
+func AssetsDir(home string) string {
+	return filepath.Join(home, "assets")
+}
+
+// EngineDir returns the directory where ooni/probe-engine should
+// store its private data given a specific OONI Home.
+func EngineDir(home string) string {
+	return filepath.Join(home, "engine")
+}
+
+// DBDir returns the database dir for the given name
+func DBDir(home string, name string) string {
+	return filepath.Join(home, "db", fmt.Sprintf("%s.sqlite3", name))
+}
+
+// FileExists returns true if the specified path exists and is a
+// regular file.
+func FileExists(path string) bool {
+	stat, err := os.Stat(path)
+	return err == nil && stat.Mode().IsRegular()
+}
+
+// ResultTimestamp is a windows friendly timestamp
+const ResultTimestamp = "2006-01-02T150405.999999999Z0700"
+
+// MakeResultsDir creates and returns a directory for the result
+func MakeResultsDir(home string, name string, ts time.Time) (string, error) {
+	p := filepath.Join(home, "msmts",
+		fmt.Sprintf("%s-%s", name, ts.Format(ResultTimestamp)))
+
+	// If the path already exists, this is a problem. It should not clash, because
+	// we are using nanosecond precision for the starttime.
+	if _, e := os.Stat(p); e == nil {
+		return "", errors.New("results path already exists")
+	}
+	err := os.MkdirAll(p, 0700)
+	if err != nil {
+		return "", err
+	}
+	return p, nil
+}
+
+// GetOONIHome returns the path to the OONI Home
+func GetOONIHome() (string, error) {
+	if ooniHome := os.Getenv("OONI_HOME"); ooniHome != "" {
+		return ooniHome, nil
+	}
+
+	home, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+
+	path := filepath.Join(home, ".ooniprobe")
+	return path, nil
+}
+
+// DidLegacyInformedConsent checks if the user did the informed consent procedure in probe-legacy
+func DidLegacyInformedConsent() bool {
+	home, err := homedir.Dir()
+	if err != nil {
+		return false
+	}
+
+	path := filepath.Join(filepath.Join(home, ".ooni"), "initialized")
+	if FileExists(path) {
+		return true
+	}
+	return false
+}

cmd/ooniprobe/internal/utils/shutil/shutil.go

@@ -0,0 +1,319 @@
+package shutil
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+type SameFileError struct {
+	Src string
+	Dst string
+}
+
+func (e SameFileError) Error() string {
+	return fmt.Sprintf("%s and %s are the same file", e.Src, e.Dst)
+}
+
+type SpecialFileError struct {
+	File     string
+	FileInfo os.FileInfo
+}
+
+func (e SpecialFileError) Error() string {
+	return fmt.Sprintf("`%s` is a named pipe", e.File)
+}
+
+type NotADirectoryError struct {
+	Src string
+}
+
+func (e NotADirectoryError) Error() string {
+	return fmt.Sprintf("`%s` is not a directory", e.Src)
+}
+
+type AlreadyExistsError struct {
+	Dst string
+}
+
+func (e AlreadyExistsError) Error() string {
+	return fmt.Sprintf("`%s` already exists", e.Dst)
+}
+
+func samefile(src string, dst string) bool {
+	srcInfo, _ := os.Stat(src)
+	dstInfo, _ := os.Stat(dst)
+	return os.SameFile(srcInfo, dstInfo)
+}
+
+func specialfile(fi os.FileInfo) bool {
+	return (fi.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe
+}
+
+func stringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}
+
+func IsSymlink(fi os.FileInfo) bool {
+	return (fi.Mode() & os.ModeSymlink) == os.ModeSymlink
+}
+
+// Copy data from src to dst
+//
+// If followSymlinks is not set and src is a symbolic link, a
+// new symlink will be created instead of copying the file it points
+// to.
+func CopyFile(src, dst string, followSymlinks bool) error {
+	if samefile(src, dst) {
+		return &SameFileError{src, dst}
+	}
+
+	// Make sure src exists and neither are special files
+	srcStat, err := os.Lstat(src)
+	if err != nil {
+		return err
+	}
+	if specialfile(srcStat) {
+		return &SpecialFileError{src, srcStat}
+	}
+
+	dstStat, err := os.Stat(dst)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	} else if err == nil {
+		if specialfile(dstStat) {
+			return &SpecialFileError{dst, dstStat}
+		}
+	}
+
+	// If we don't follow symlinks and it's a symlink, just link it and be done
+	if !followSymlinks && IsSymlink(srcStat) {
+		return os.Symlink(src, dst)
+	}
+
+	// If we are a symlink, follow it
+	if IsSymlink(srcStat) {
+		src, err = os.Readlink(src)
+		if err != nil {
+			return err
+		}
+		srcStat, err = os.Stat(src)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Do the actual copy
+	fsrc, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer fsrc.Close()
+
+	fdst, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer fdst.Close()
+
+	size, err := io.Copy(fdst, fsrc)
+	if err != nil {
+		return err
+	}
+
+	if size != srcStat.Size() {
+		return fmt.Errorf("%s: %d/%d copied", src, size, srcStat.Size())
+	}
+
+	return nil
+}
+
+// Copy mode bits from src to dst.
+//
+// If followSymlinks is false, symlinks aren't followed if and only
+// if both `src` and `dst` are symlinks. If `lchmod` isn't available
+// and both are symlinks this does nothing. (I don't think lchmod is
+// available in Go)
+func CopyMode(src, dst string, followSymlinks bool) error {
+	srcStat, err := os.Lstat(src)
+	if err != nil {
+		return err
+	}
+
+	dstStat, err := os.Lstat(dst)
+	if err != nil {
+		return err
+	}
+
+	// They are both symlinks and we can't change mode on symlinks.
+	if !followSymlinks && IsSymlink(srcStat) && IsSymlink(dstStat) {
+		return nil
+	}
+
+	// Atleast one is not a symlink, get the actual file stats
+	srcStat, _ = os.Stat(src)
+	err = os.Chmod(dst, srcStat.Mode())
+	return err
+}
+
+// Copy data and mode bits ("cp src dst"). Return the file's destination.
+//
+// The destination may be a directory.
+//
+// If followSymlinks is false, symlinks won't be followed. This
+// resembles GNU's "cp -P src dst".
+//
+// If source and destination are the same file, a SameFileError will be
+// rased.
+func Copy(src, dst string, followSymlinks bool) (string, error) {
+	dstInfo, err := os.Stat(dst)
+
+	if err == nil && dstInfo.Mode().IsDir() {
+		dst = filepath.Join(dst, filepath.Base(src))
+	}
+
+	if err != nil && !os.IsNotExist(err) {
+		return dst, err
+	}
+
+	err = CopyFile(src, dst, followSymlinks)
+	if err != nil {
+		return dst, err
+	}
+
+	err = CopyMode(src, dst, followSymlinks)
+	if err != nil {
+		return dst, err
+	}
+
+	return dst, nil
+}
+
+type CopyTreeOptions struct {
+	Symlinks               bool
+	IgnoreDanglingSymlinks bool
+	CopyFunction           func(string, string, bool) (string, error)
+	Ignore                 func(string, []os.FileInfo) []string
+}
+
+// Recursively copy a directory tree.
+//
+// The destination directory must not already exist.
+//
+// If the optional Symlinks flag is true, symbolic links in the
+// source tree result in symbolic links in the destination tree; if
+// it is false, the contents of the files pointed to by symbolic
+// links are copied. If the file pointed by the symlink doesn't
+// exist, an error will be returned.
+//
+// You can set the optional IgnoreDanglingSymlinks flag to true if you
+// want to silence this error. Notice that this has no effect on
+// platforms that don't support os.Symlink.
+//
+// The optional ignore argument is a callable. If given, it
+// is called with the `src` parameter, which is the directory
+// being visited by CopyTree(), and `names` which is the list of
+// `src` contents, as returned by ioutil.ReadDir():
+//
+//   callable(src, entries) -> ignoredNames
+//
+// Since CopyTree() is called recursively, the callable will be
+// called once for each directory that is copied. It returns a
+// list of names relative to the `src` directory that should
+// not be copied.
+//
+// The optional copyFunction argument is a callable that will be used
+// to copy each file. It will be called with the source path and the
+// destination path as arguments. By default, Copy() is used, but any
+// function that supports the same signature (like Copy2() when it
+// exists) can be used.
+func CopyTree(src, dst string, options *CopyTreeOptions) error {
+	if options == nil {
+		options = &CopyTreeOptions{Symlinks: false,
+			Ignore:                 nil,
+			CopyFunction:           Copy,
+			IgnoreDanglingSymlinks: false}
+	}
+
+	srcFileInfo, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if !srcFileInfo.IsDir() {
+		return &NotADirectoryError{src}
+	}
+
+	_, err = os.Open(dst)
+	if !os.IsNotExist(err) {
+		return &AlreadyExistsError{dst}
+	}
+
+	entries, err := ioutil.ReadDir(src)
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(dst, srcFileInfo.Mode())
+	if err != nil {
+		return err
+	}
+
+	ignoredNames := []string{}
+	if options.Ignore != nil {
+		ignoredNames = options.Ignore(src, entries)
+	}
+
+	for _, entry := range entries {
+		if stringInSlice(entry.Name(), ignoredNames) {
+			continue
+		}
+		srcPath := filepath.Join(src, entry.Name())
+		dstPath := filepath.Join(dst, entry.Name())
+
+		entryFileInfo, err := os.Lstat(srcPath)
+		if err != nil {
+			return err
+		}
+
+		// Deal with symlinks
+		if IsSymlink(entryFileInfo) {
+			linkTo, err := os.Readlink(srcPath)
+			if err != nil {
+				return err
+			}
+			if options.Symlinks {
+				os.Symlink(linkTo, dstPath)
+				//CopyStat(srcPath, dstPath, false)
+			} else {
+				// ignore dangling symlink if flag is on
+				_, err = os.Stat(linkTo)
+				if os.IsNotExist(err) && options.IgnoreDanglingSymlinks {
+					continue
+				}
+				_, err = options.CopyFunction(srcPath, dstPath, false)
+				if err != nil {
+					return err
+				}
+			}
+		} else if entryFileInfo.IsDir() {
+			err = CopyTree(srcPath, dstPath, options)
+			if err != nil {
+				return err
+			}
+		} else {
+			_, err = options.CopyFunction(srcPath, dstPath, false)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

cmd/ooniprobe/internal/utils/util_test.go

@@ -0,0 +1,18 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/fatih/color"
+)
+
+func TestEscapeAwareRuneCountInString(t *testing.T) {
+	var bold = color.New(color.Bold)
+	var myColor = color.New(color.FgBlue)
+
+	s := myColor.Sprintf("•ABC%s%s", bold.Sprintf("DEF"), "\x1B[00;38;5;244m\x1B[m\x1B[00;38;5;33mGHI\x1B[0m")
+	count := EscapeAwareRuneCountInString(s)
+	if count != 10 {
+		t.Errorf("Count was incorrect, got: %d, want: %d.", count, 10)
+	}
+}

cmd/ooniprobe/internal/utils/utils.go

@@ -0,0 +1,117 @@
+package utils
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/fatih/color"
+)
+
+// Log outputs a log message.
+func Log(msg string, v ...interface{}) {
+	fmt.Printf("     %s\n", color.CyanString(msg, v...))
+}
+
+// Fatal error
+func Fatal(err error) {
+	fmt.Fprintf(os.Stderr, "\n     %s %s\n\n", color.RedString("Error:"), err)
+	os.Exit(1)
+}
+
+// Finds the ansi escape sequences (like colors)
+// Taken from: https://github.com/chalk/ansi-regex/blob/d9d806ecb45d899cf43408906a4440060c5c50e5/index.js
+var ansiEscapes = regexp.MustCompile(`[\x1B\x9B][[\]()#;?]*` +
+	`(?:(?:(?:[a-zA-Z\d]*(?:;[a-zA-Z\\d]*)*)?\x07)` +
+	`|(?:(?:\d{1,4}(?:;\d{0,4})*)?[\dA-PRZcf-ntqry=><~]))`)
+
+// EscapeAwareRuneCountInString counts the number of runes in a
+// string taking into account escape sequences.
+func EscapeAwareRuneCountInString(s string) int {
+	n := utf8.RuneCountInString(s)
+	for _, sm := range ansiEscapes.FindAllString(s, -1) {
+		n -= utf8.RuneCountInString(sm)
+	}
+	return n
+}
+
+// RightPadd adds right padding in from of a string
+func RightPad(str string, length int) string {
+	c := length - EscapeAwareRuneCountInString(str)
+	if c < 0 {
+		c = 0
+	}
+	return str + strings.Repeat(" ", c)
+}
+
+// WrapString wraps the given string within lim width in characters.
+//
+// Wrapping is currently naive and only happens at white-space. A future
+// version of the library will implement smarter wrapping. This means that
+// pathological cases can dramatically reach past the limit, such as a very
+// long word.
+// This is taken from: https://github.com/mitchellh/go-wordwrap/tree/f253961a26562056904822f2a52d4692347db1bd
+func WrapString(s string, lim uint) string {
+	// Initialize a buffer with a slightly larger size to account for breaks
+	init := make([]byte, 0, len(s))
+	buf := bytes.NewBuffer(init)
+
+	var current uint
+	var wordBuf, spaceBuf bytes.Buffer
+
+	for _, char := range s {
+		if char == '\n' {
+			if wordBuf.Len() == 0 {
+				if current+uint(spaceBuf.Len()) > lim {
+					current = 0
+				} else {
+					current += uint(spaceBuf.Len())
+					spaceBuf.WriteTo(buf)
+				}
+				spaceBuf.Reset()
+			} else {
+				current += uint(spaceBuf.Len() + wordBuf.Len())
+				spaceBuf.WriteTo(buf)
+				spaceBuf.Reset()
+				wordBuf.WriteTo(buf)
+				wordBuf.Reset()
+			}
+			buf.WriteRune(char)
+			current = 0
+		} else if unicode.IsSpace(char) {
+			if spaceBuf.Len() == 0 || wordBuf.Len() > 0 {
+				current += uint(spaceBuf.Len() + wordBuf.Len())
+				spaceBuf.WriteTo(buf)
+				spaceBuf.Reset()
+				wordBuf.WriteTo(buf)
+				wordBuf.Reset()
+			}
+
+			spaceBuf.WriteRune(char)
+		} else {
+
+			wordBuf.WriteRune(char)
+
+			if current+uint(spaceBuf.Len()+wordBuf.Len()) > lim && uint(wordBuf.Len()) < lim {
+				buf.WriteRune('\n')
+				current = 0
+				spaceBuf.Reset()
+			}
+		}
+	}
+
+	if wordBuf.Len() == 0 {
+		if current+uint(spaceBuf.Len()) <= lim {
+			spaceBuf.WriteTo(buf)
+		}
+	} else {
+		spaceBuf.WriteTo(buf)
+		wordBuf.WriteTo(buf)
+	}
+
+	return buf.String()
+}

cmd/ooniprobe/internal/version/version.go

@@ -0,0 +1,7 @@
+// Package version contains version information
+package version
+
+const (
+	// Version is the software version
+	Version = "3.5.0-alpha"
+)