refactor: move more commands to internal/cmd (#207)

* refactor: move more commands to internal/cmd

Part of https://github.com/ooni/probe/issues/1335.

We would like all commands to be at the same level of engine
rather than inside engine (now that we can do it).

* fix: update .gitignore

* refactor: also move jafar outside engine

* We should be good now?

internal/cmd/README.md

@@ -0,0 +1,4 @@
+# Directory github.com/ooni/probe-cli/internal/cmd
+
+This directory contains the source code for the CLI tools we build
+but we don't want to expose to the outside world.

internal/cmd/apitool/README.md

@@ -0,0 +1,8 @@
+# apitool
+
+This directory contains a tool to fetch measurements. This tool is
+intended to sporadically fetch measurements, not for batch downloading.
+
+Please, see https://ooni.org/data for information pertaining how to
+access OONI data in bulk. Please see https://explorer.ooni.org if your
+intent is to navigate and explore OONI data.

internal/cmd/apitool/main.go

@@ -0,0 +1,115 @@
+// Command apitool is a simple tool to fetch individual OONI measurements.
+//
+// This tool IS NOT intended for batch downloading.
+//
+// Please, see https://ooni.org/data for information pertaining how to
+// access OONI data in bulk. Please see https://explorer.ooni.org if your
+// intent is to navigate and explore OONI data
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
+	"github.com/ooni/probe-cli/v3/internal/engine/version"
+)
+
+func newclient() probeservices.Client {
+	txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
+	ua := fmt.Sprintf("apitool/%s ooniprobe-engine/%s", version.Version, version.Version)
+	return probeservices.Client{
+		Client: httpx.Client{
+			BaseURL:    "https://ams-pg.ooni.org/",
+			HTTPClient: &http.Client{Transport: txp},
+			Logger:     log.Log,
+			UserAgent:  ua,
+		},
+		LoginCalls:    atomicx.NewInt64(),
+		RegisterCalls: atomicx.NewInt64(),
+		StateFile:     probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
+	}
+}
+
+var osExit = os.Exit
+
+func fatalOnError(err error, message string) {
+	if err != nil {
+		log.WithError(err).Error(message)
+		osExit(1) // overridable from tests
+	}
+}
+
+var (
+	debug    = flag.Bool("v", false, "Enable verbose mode")
+	input    = flag.String("input", "", "Input of the measurement")
+	mode     = flag.String("mode", "", "One of: check, meta, raw")
+	reportid = flag.String("report-id", "", "Report ID of the measurement")
+)
+
+var logmap = map[bool]log.Level{
+	true:  log.DebugLevel,
+	false: log.InfoLevel,
+}
+
+func main() {
+	flag.Parse()
+	log.SetLevel(logmap[*debug])
+	client := newclient()
+	switch *mode {
+	case "check":
+		check(client)
+	case "meta":
+		meta(client)
+	case "raw":
+		raw(client)
+	default:
+		fatalOnError(fmt.Errorf("invalid -mode flag value: %s", *mode), "usage error")
+	}
+}
+
+func check(c probeservices.Client) {
+	found, err := c.CheckReportID(context.Background(), *reportid)
+	fatalOnError(err, "c.CheckReportID failed")
+	fmt.Printf("%+v\n", found)
+}
+
+func meta(c probeservices.Client) {
+	pprint(mmeta(c, false))
+}
+
+func raw(c probeservices.Client) {
+	m := mmeta(c, true)
+	rm := []byte(m.RawMeasurement)
+	var opaque interface{}
+	err := json.Unmarshal(rm, &opaque)
+	fatalOnError(err, "json.Unmarshal failed")
+	pprint(opaque)
+}
+
+func pprint(opaque interface{}) {
+	data, err := json.MarshalIndent(opaque, "", "  ")
+	fatalOnError(err, "json.MarshalIndent failed")
+	fmt.Printf("%s\n", data)
+}
+
+func mmeta(c probeservices.Client, full bool) *probeservices.MeasurementMeta {
+	config := probeservices.MeasurementMetaConfig{
+		ReportID: *reportid,
+		Full:     full,
+		Input:    *input,
+	}
+	ctx := context.Background()
+	m, err := c.GetMeasurementMeta(ctx, config)
+	fatalOnError(err, "client.GetMeasurementMeta failed")
+	return m
+}

internal/cmd/apitool/main_test.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+func init() {
+	*reportid = `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`
+	*input = `https://www.example.org`
+}
+
+func TestCheck(t *testing.T) {
+	*mode = "check"
+	main()
+}
+
+func TestRaw(t *testing.T) {
+	*mode = "raw"
+	main()
+}
+
+func TestMeta(t *testing.T) {
+	*mode = "meta"
+	main()
+}
+
+func TestInvalidMode(t *testing.T) {
+	defer func() {
+		if recover() == nil {
+			t.Fatal("the code did not panic")
+		}
+	}()
+	osExit = func(code int) {
+		panic(fmt.Errorf("%d", code))
+	}
+	*mode = "antani"
+	main()
+}

internal/cmd/jafar/.gitignore

@@ -0,0 +1 @@
+/badproxy.pem

internal/cmd/jafar/README.md

@@ -0,0 +1,258 @@
+# Jafar
+
+> We stepped up the game of simulating censorship upgrading from the
+> evil genius to the evil grand vizier.
+
+Jafar is a censorship simulation tool used for testing OONI. It builds on
+any system but it really on works on Linux.
+
+## Building
+
+We use Go >= 1.14. Jafar also needs the C library headers,
+iptables installed, and root permissions.
+
+With Linux Alpine edge, you can compile Jafar with:
+
+```bash
+apk add go git musl-dev iptables
+go build -v .
+```
+
+Otherwise, using Docker:
+
+```bash
+docker build -t jafar-runner .
+docker run -it --privileged -v`pwd`:/jafar -w/jafar jafar-runner
+go build -v .
+```
+
+## Usage
+
+You need to run Jafar as root. You can get a complete list
+of all flags using `./jafar -help`. Jafar is composed of modules. Each
+module is controllable via flags. We describe modules below.
+
+### main
+
+The main module starts all the other modules. If you don't provide the
+`-main-command <command>` flag, the code will run until interrupted. If
+instead you use the `-main-command` flag, you can specify a command to
+run inside the censored environment. In such case, the main module
+will exit when the specified command terminates. Note that the main
+module will propagate the child exit code, if the child fails.
+
+The command can also include arguments. Make sure you quote the arguments
+such that your shell passes the whole string to the specified option, as
+in `-main-command 'ls -lha'`. This will execute the `ls -lha` command line
+inside the censored Jafar context. You can also combine that with quoting
+and variables interpolation, e.g., `-main-command "echo '$USER is the
+walrus'"`. The `$USER` variable will be expanded by your shell. Assuming
+your user name is `paul`, then Jafar will lex the main command as `echo
+"paul is the walrus"` and will execute it.
+
+Use the `-main-user <username>` flag to select the user to use for
+running child commands. By default, we use the `nobody` user for this
+purpose. We implement this feature using `sudo`, therefore you need
+to make sure that `sudo` is installed.
+
+### iptables
+
+The iptables module is only available on Linux. It exports these flags:
+
+```bash
+  -iptables-drop-ip value
+        Drop traffic to the specified IP address
+  -iptables-drop-keyword-hex value
+        Drop traffic containing the specified hex keyword
+  -iptables-drop-keyword value
+        Drop traffic containing the specified keyword
+  -iptables-hijack-dns-to string
+        Hijack all DNS UDP traffic to the specified endpoint
+  -iptables-hijack-https-to string
+        Hijack all HTTPS traffic to the specified endpoint
+  -iptables-hijack-http-to string
+        Hijack all HTTP traffic to the specified endpoint
+  -iptables-reset-ip value
+        Reset TCP/IP traffic to the specified IP address
+  -iptables-reset-keyword-hex value
+        Reset TCP/IP traffic containing the specified hex keyword
+  -iptables-reset-keyword value
+        Reset TCP/IP traffic containing the specified keyword
+```
+
+The difference between `drop` and `reset` is that in the former case
+a packet is dropped, in the latter case a RST is sent.
+
+The difference between `ip` and `keyword` flags is that the former
+match an outgoing IP, the latter uses DPI.
+
+The `drop` and `reset` rules allow you to simulate, respectively, when
+operations timeout and when a connection cannot be established (with
+`reset` and `ip`) or is reset after a keyword is seen (with `keyword`).
+
+Hijacking DNS traffic is useful, for example, to redirect all DNS UDP
+traffic from the box to the `dns-proxy` module.
+
+Hijacking HTTP and HTTPS traffic actually hijacks based on ports rather
+than on DPI. As a known bug, when hijacking HTTP or HTTPS traffic, we
+do not hijack traffic owned by root. This is because Jafar runs as root
+and therefore its traffic must not match the hijack rule.
+
+When matching keywords, the simplest option is to use ASCII strings as
+in `-iptables-drop-keyword ooni`. However, you can also specify a sequence
+of hex bytes, as in `-iptables-drop-keyword-hex |6f 6f 6e 69|`.
+
+Note that with `-iptables-drop-keyword`, DNS queries containing such
+keyword will fail returning `EPERM`. For a more realistic approach to
+dropping specific DNS packets, combine DNS traffic hijacking with
+`-dns-proxy-ignore`, to "drop" packets at the DNS proxy.
+
+### dns-proxy (aka resolver)
+
+The DNS proxy or resolver allows to manipulate DNS. Unless you use DNS
+hijacking, you will need to configure your application explicitly to use
+the proxy with application specific command line flags.
+
+```bash
+  -dns-proxy-address string
+        Address where the DNS proxy should listen (default "127.0.0.1:53")
+  -dns-proxy-block value
+        Register keyword triggering NXDOMAIN censorship
+  -dns-proxy-hijack value
+        Register keyword triggering redirection to 127.0.0.1
+  -dns-proxy-ignore value
+        Register keyword causing the proxy to ignore the query
+```
+
+The `-dns-proxy-address` flag controls the endpoint where the proxy is
+listening.
+
+The `-dns-proxy-block` tells the resolver that every incoming request whose
+query contains the specifed string shall receive an `NXDOMAIN` reply.
+
+The `-dns-proxy-hijack` is similar but instead lies and returns to the
+client that the requested domain is at `127.0.0.1`. This is an opportunity
+to redirect traffic to the HTTP and TLS proxies.
+
+The `-dns-proxy-ignore` is similar but instead just ignores the query.
+
+### http-proxy
+
+The HTTP proxy is an HTTP proxy that may refuse to forward some
+specific requests. It's controlled by these flags:
+
+```bash
+  -http-proxy-address string
+        Address where the HTTP proxy should listen (default "127.0.0.1:80")
+  -http-proxy-block value
+        Register keyword triggering HTTP 451 censorship
+```
+
+The `-http-proxy-address` flag has the same semantics it has for the DNS
+proxy.
+
+The `-http-proxy-block` flag tells the proxy that it should return a `451`
+response for every request whose `Host` contains the specified string.
+
+### tls-proxy
+
+TLS proxy is a proxy that routes traffic to specific servers depending
+on their SNI value. It is controlled by the following flags:
+
+```bash
+  -tls-proxy-address string
+        Address where the HTTP proxy should listen (default "127.0.0.1:443")
+  -tls-proxy-block value
+        Register keyword triggering TLS censorship
+```
+
+The `-tls-proxy-address` flags has the same semantics it has for the DNS
+proxy.
+
+The `-tls-proxy-block` specifies which string or strings should cause the
+proxy to return an internal-erorr alert when the incoming ClientHello's SNI
+contains one of the strings provided with this option.
+
+### bad-proxy
+
+```bash
+  -bad-proxy-address string
+        Address where to listen for TCP connections (default "127.0.0.1:7117")
+  -bad-proxy-address-tls string
+        Address where to listen for TLS connections (default "127.0.0.1:4114")
+  -bad-proxy-tls-output-ca string
+        File where to write the CA used by the bad proxy (default "badproxy.pem")
+```
+
+The bad proxy is a proxy that reads some bytes from any incoming connection
+and then closes the connection without replying anything. This simulates a
+proxy that is not working properly, hence the name of the module.
+
+When connecting using TLS, the above behaviour happens after the handshake.
+
+We write the CA on the file specified using `-bad-proxy-tls-output-ca` such that
+tools like curl(1) can use such CA to avoid TLS handshake errors. The code will
+generate on the fly a certificate for the provided SNI. Not providing any SNI in
+the client Hello message will cause the TLS handshake to fail.
+
+### uncensored
+
+```bash
+  -uncensored-resolver-url string
+    	URL of an hopefully uncensored resolver (default "dot://1.1.1.1:853")
+```
+
+The HTTP, DNS, and TLS proxies need to resolve domain names. If you setup DNS
+censorship, they may be affected as well. To avoid this issue, we use a different
+resolver for them, which by default is `dot://1.1.1.1:853`. You can change such
+default by using the `-uncensored-resolver-url` command line flag. The input
+URL is `<transport>://<domain>[:<port>][/<path>]`. Here are some examples:
+
+* `system:///` uses the system resolver (i.e. `getaddrinfo`)
+* `udp://8.8.8.8:53` uses DNS over UDP
+* `tcp://8.8.8.8:53` used DNS over TCP
+* `dot://8.8.8.8:853` uses DNS over TLS
+* `https://dns.google/dns-query` uses DNS over HTTPS
+
+So, for example, if you are using Jafar to censor `1.1.1.1:853`, then you
+most likely want to use `-uncensored-resolver-url`.
+
+## Examples
+
+Block `play.google.com` with RST injection, force DNS traffic to use the our
+DNS proxy, and force it to censor `play.google.com` with `NXDOMAIN`.
+
+```bash
+# ./jafar -iptables-reset-keyword play.google.com \
+          -iptables-hijack-dns-to 127.0.0.1:5353  \
+          -dns-proxy-address 127.0.0.1:5353       \
+          -dns-proxy-block play.google.com
+```
+
+Force all traffic through the HTTP and TLS proxy and use them to censor
+`play.google.com` using HTTP 451 and responding with TLS alerts:
+
+```bash
+# ./jafar -iptables-hijack-dns-to 127.0.0.1:5353 \
+          -dns-proxy-address 127.0.0.1:5353      \
+          -dns-proxy-hijack play.google.com      \
+          -http-proxy-block play.google.com      \
+          -tls-proxy-block play.google.com
+```
+
+Run `ping` in a censored environment:
+
+```bash
+# ./jafar -iptables-drop-ip 8.8.8.8 -main-command 'ping -c3 8.8.8.8'
+```
+
+Run `curl` in a censored environment where it cannot connect to
+`play.google.com` using `https`:
+
+```bash
+# ./jafar -iptables-hijack-https-to 127.0.0.1:443         \
+          -tls-proxy-block play.google.com                \
+          -main-command 'curl -Lv http://play.google.com'
+```
+
+For more usage examples, see `../../testjafar.bash`.

internal/cmd/jafar/badproxy/badproxy.go

@@ -0,0 +1,113 @@
+// Package badproxy implements misbehaving proxies. We have a single
+// CensoringProxy that exports two misbehaving endpoints. Each endpoint
+// implements a different proxy-censorsing technique. The first one
+// reads some bytes from the connection then closes the connection. The
+// other instead replies with a self signed x509 certificate.
+package badproxy
+
+import (
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"io"
+	"io/ioutil"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/google/martian/v3/mitm"
+)
+
+// CensoringProxy is a proxy that does not behave correctly.
+type CensoringProxy struct {
+	mitmNewAuthority func(
+		name string, organization string,
+		validity time.Duration,
+	) (*x509.Certificate, *rsa.PrivateKey, error)
+
+	mitmNewConfig func(
+		ca *x509.Certificate, privateKey interface{},
+	) (*mitm.Config, error)
+
+	tlsListen func(
+		network string, laddr string, config *tls.Config,
+	) (net.Listener, error)
+}
+
+// NewCensoringProxy creates a new instance of a misbehaving proxy.
+func NewCensoringProxy() *CensoringProxy {
+	return &CensoringProxy{
+		mitmNewAuthority: mitm.NewAuthority,
+		mitmNewConfig:    mitm.NewConfig,
+		tlsListen:        tls.Listen,
+	}
+}
+
+func (p *CensoringProxy) serve(conn net.Conn) {
+	deadline := time.Now().Add(250 * time.Millisecond)
+	conn.SetDeadline(deadline)
+	// To simulate the case where the proxy isn't willing to forward our
+	// traffic, we close the connection (1) right after the handshake for
+	// TLS connections and (2) reasonably after we've received the HTTP
+	// request for cleartext connections. This may break in several cases
+	// but is good enough approximation of these bad proxies for now.
+	if tlsconn, ok := conn.(*tls.Conn); ok {
+		tlsconn.Handshake()
+	} else {
+		const maxread = 1 << 17
+		reader := io.LimitReader(conn, maxread)
+		ioutil.ReadAll(reader)
+	}
+	conn.Close()
+}
+
+func (p *CensoringProxy) run(listener net.Listener) {
+	for {
+		conn, err := listener.Accept()
+		if err != nil && strings.Contains(
+			err.Error(), "use of closed network connection") {
+			return
+		}
+		if err == nil {
+			// It's difficult to make accept fail, so restructure
+			// the code such that we enter into the happy path
+			go p.serve(conn)
+		}
+	}
+}
+
+// Start starts the misbehaving proxy for TCP. This endpoint will read some
+// bytes from the request and then close the connection. This behaviour is
+// implemented by a bunch of censoring proxy around the world. Usually such
+// proxies only close the connection with offending SNIs/Host headers.
+func (p *CensoringProxy) Start(address string) (net.Listener, error) {
+	listener, err := net.Listen("tcp", address)
+	if err != nil {
+		return nil, err
+	}
+	go p.run(listener)
+	return listener, nil
+}
+
+// StartTLS starts the misbehaving proxy for TLS. This endpoint will return
+// to the client a self signed certificate. Thus, it models the case where a
+// MITM forces users to accept a rogue certificate. After sending such a
+// certificate, this proxy will close the TCP connection.
+func (p *CensoringProxy) StartTLS(address string) (net.Listener, *x509.Certificate, error) {
+	cert, privkey, err := p.mitmNewAuthority(
+		"jafar", "OONI", 24*time.Hour,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+	config, err := p.mitmNewConfig(cert, privkey)
+	if err != nil {
+		return nil, nil, err
+	}
+	listener, err := p.tlsListen("tcp", address, config.TLS())
+	if err != nil {
+		return nil, nil, err
+	}
+	go p.run(listener)
+	return listener, cert, nil
+}

internal/cmd/jafar/badproxy/badproxy_test.go

@@ -0,0 +1,157 @@
+package badproxy
+
+import (
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/google/martian/v3/mitm"
+)
+
+func TestCleartext(t *testing.T) {
+	listener := newproxy(t)
+	checkdial(t, listener.Addr().String(), nil, net.Dial)
+	killproxy(t, listener)
+}
+
+func TestTLS(t *testing.T) {
+	listener := newproxytls(t)
+	checkdial(t, listener.Addr().String(), nil,
+		func(network, address string) (net.Conn, error) {
+			conn, err := tls.Dial(network, address, &tls.Config{
+				InsecureSkipVerify: true,
+				ServerName:         "antani.local",
+			})
+			if err != nil {
+				return nil, err
+			}
+			if err = conn.Handshake(); err != nil {
+				conn.Close()
+				return nil, err
+			}
+			return conn, nil
+		})
+	killproxy(t, listener)
+}
+
+func TestListenError(t *testing.T) {
+	proxy := NewCensoringProxy()
+	listener, err := proxy.Start("8.8.8.8:80")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if listener != nil {
+		t.Fatal("expected nil listener here")
+	}
+}
+
+func TestStarTLS(t *testing.T) {
+	expected := errors.New("mocked error")
+
+	t.Run("when we cannot create a new authority", func(t *testing.T) {
+		proxy := NewCensoringProxy()
+		proxy.mitmNewAuthority = func(
+			name string, organization string,
+			validity time.Duration,
+		) (*x509.Certificate, *rsa.PrivateKey, error) {
+			return nil, nil, expected
+		}
+		cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
+		if !errors.Is(err, expected) {
+			t.Fatal("not the error we expected")
+		}
+		if cert != nil {
+			t.Fatal("expected nil cert")
+		}
+		if privkey != nil {
+			t.Fatal("expected nil privkey")
+		}
+	})
+
+	t.Run("when we cannot create a new config", func(t *testing.T) {
+		proxy := NewCensoringProxy()
+		proxy.mitmNewConfig = func(
+			ca *x509.Certificate, privateKey interface{},
+		) (*mitm.Config, error) {
+			return nil, expected
+		}
+		cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
+		if !errors.Is(err, expected) {
+			t.Fatal("not the error we expected")
+		}
+		if cert != nil {
+			t.Fatal("expected nil cert")
+		}
+		if privkey != nil {
+			t.Fatal("expected nil privkey")
+		}
+	})
+
+	t.Run("when we cannot listen", func(t *testing.T) {
+		proxy := NewCensoringProxy()
+		proxy.tlsListen = func(
+			network string, laddr string, config *tls.Config,
+		) (net.Listener, error) {
+			return nil, expected
+		}
+		cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
+		if !errors.Is(err, expected) {
+			t.Fatal("not the error we expected")
+		}
+		if cert != nil {
+			t.Fatal("expected nil cert")
+		}
+		if privkey != nil {
+			t.Fatal("expected nil privkey")
+		}
+	})
+}
+
+func newproxy(t *testing.T) net.Listener {
+	proxy := NewCensoringProxy()
+	listener, err := proxy.Start("127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return listener
+}
+
+func newproxytls(t *testing.T) net.Listener {
+	proxy := NewCensoringProxy()
+	listener, _, err := proxy.StartTLS("127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return listener
+}
+
+func killproxy(t *testing.T, listener net.Listener) {
+	err := listener.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func checkdial(
+	t *testing.T, proxyAddr string, expectErr error,
+	dial func(network, address string) (net.Conn, error),
+) {
+	conn, err := dial("tcp", proxyAddr)
+	if err != expectErr {
+		t.Fatal("not the result we expected")
+	}
+	if conn == nil && expectErr == nil {
+		t.Fatal("expected actionable conn")
+	}
+	if conn != nil && expectErr != nil {
+		t.Fatal("expected nil conn")
+	}
+	if conn != nil {
+		conn.Write([]byte("123454321"))
+		conn.Close()
+	}
+}

internal/cmd/jafar/flagx/stringarray.go

@@ -0,0 +1,46 @@
+// Package flagx contains extensions for the standard library
+// flag package. The code is adapted from github.com/m-lab/go and more
+// specifically from <https://git.io/JJ8UA>. This file is licensed under
+// version 2.0 of the Apache License <https://git.io/JJ8Ux>.
+package flagx
+
+import (
+	"fmt"
+	"strings"
+)
+
+// StringArray is a new flag type. It appends the flag parameter to an
+// `[]string` allowing the parameter to be specified multiple times or using ","
+// separated items. Unlike other Flag types, the default argument should almost
+// always be the empty array, because there is no way to remove an element, only
+// to add one.
+type StringArray []string
+
+// Get retrieves the value contained in the flag.
+func (sa StringArray) Get() interface{} {
+	return sa
+}
+
+// Set accepts a string parameter and appends it to the associated StringArray.
+// Set attempts to split the given string on commas "," and appends each element
+// to the StringArray.
+func (sa *StringArray) Set(s string) error {
+	f := strings.Split(s, ",")
+	*sa = append(*sa, f...)
+	return nil
+}
+
+// String reports the StringArray as a Go value.
+func (sa StringArray) String() string {
+	return fmt.Sprintf("%#v", []string(sa))
+}
+
+// Contains returns true when the given value equals one of the StringArray values.
+func (sa StringArray) Contains(value string) bool {
+	for _, v := range sa {
+		if v == value {
+			return true
+		}
+	}
+	return false
+}

internal/cmd/jafar/flagx/stringarray_test.go

@@ -0,0 +1,73 @@
+package flagx_test
+
+// The code in this file is adapted from github.com/m-lab/go and more
+// specifically from <https://git.io/JJ8UA>. This file is licensed under
+// version 2.0 of the Apache License <https://git.io/JJ8Ux>.
+
+import (
+	"flag"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/flagx"
+)
+
+func TestStringArray(t *testing.T) {
+	tests := []struct {
+		name     string
+		args     []string
+		expt     flagx.StringArray
+		repr     string
+		contains string
+		wantErr  bool
+	}{
+		{
+			name:     "okay",
+			args:     []string{"a", "b"},
+			expt:     flagx.StringArray{"a", "b"},
+			repr:     `[]string{"a", "b"}`,
+			contains: "b",
+		},
+		{
+			name:     "okay-split-commas",
+			args:     []string{"a", "b", "c,d"},
+			expt:     flagx.StringArray{"a", "b", "c", "d"},
+			repr:     `[]string{"a", "b", "c", "d"}`,
+			contains: "d",
+		},
+		{
+			name:     "empty",
+			args:     []string{},
+			expt:     flagx.StringArray{},
+			repr:     `[]string{}`,
+			contains: "a",
+			wantErr:  true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sa := &flagx.StringArray{}
+			for i := range tt.args {
+				if err := sa.Set(tt.args[i]); err != nil {
+					t.Errorf("StringArray.Set() error = %v, want nil", err)
+				}
+			}
+			v := (sa.Get().(flagx.StringArray))
+			if diff := cmp.Diff(v, tt.expt); diff != "" {
+				t.Errorf("StringArray.Get() unexpected differences %v", diff)
+			}
+			if tt.repr != sa.String() {
+				t.Errorf("StringArray.String() want = %q, got %q", tt.repr, sa.String())
+			}
+			if sa.Contains(tt.contains) == tt.wantErr {
+				t.Errorf("StringArray.Contains() want = %q, got %t", tt.repr, sa.Contains(tt.contains))
+			}
+		})
+	}
+}
+
+// Successful compilation of this function means that StringArray implements the
+// flag.Getter interface. The function need not be called.
+func assertFlagGetterStringArray(b flagx.StringArray) {
+	func(in flag.Getter) {}(&b)
+}

internal/cmd/jafar/httpproxy/httpproxy.go

@@ -0,0 +1,80 @@
+// Package httpproxy contains a censoring HTTP proxy. This proxy will
+// vet all the traffic and reply with 451 responses for a configurable
+// set of offending Host headers in incoming requests.
+package httpproxy
+
+import (
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+const product = "jafar/0.1.0"
+
+// CensoringProxy is a censoring HTTP proxy
+type CensoringProxy struct {
+	keywords  []string
+	transport http.RoundTripper
+}
+
+// NewCensoringProxy creates a new CensoringProxy instance using
+// the specified list of keywords to censor. keywords is the list
+// of keywords that trigger censorship if any of them appears in
+// the Host header of a request. dnsNetwork and dnsAddress are
+// settings to configure the upstream, non censored DNS.
+func NewCensoringProxy(
+	keywords []string, uncensored netx.HTTPRoundTripper,
+) *CensoringProxy {
+	return &CensoringProxy{keywords: keywords, transport: uncensored}
+}
+
+var blockpage = []byte(`<html><head>
+  <title>451 Unavailable For Legal Reasons</title>
+</head><body>
+  <center><h1>451 Unavailable For Legal Reasons</h1></center>
+  <p>This content is not available in your jurisdiction.</p>
+</body></html>
+`)
+
+// ServeHTTP serves HTTP requests
+func (p *CensoringProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Implementation note: use Via header to detect in a loose way
+	// requests originated by us and directed to us
+	if r.Header.Get("Via") != "" || r.Host == "" {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	for _, pattern := range p.keywords {
+		if strings.Contains(r.Host, pattern) {
+			w.WriteHeader(http.StatusUnavailableForLegalReasons)
+			w.Write(blockpage)
+			return
+		}
+	}
+	r.Header.Add("Via", product) // see above
+	proxy := httputil.NewSingleHostReverseProxy(&url.URL{
+		Host:   r.Host,
+		Scheme: "http",
+	})
+	proxy.ModifyResponse = func(resp *http.Response) error {
+		resp.Header.Add("Via", product) // see above
+		return nil
+	}
+	proxy.Transport = p.transport
+	proxy.ServeHTTP(w, r)
+}
+
+// Start starts the censoring proxy.
+func (p *CensoringProxy) Start(address string) (*http.Server, net.Addr, error) {
+	server := &http.Server{Handler: p}
+	listener, err := net.Listen("tcp", address)
+	if err != nil {
+		return nil, nil, err
+	}
+	go server.Serve(listener)
+	return server, listener.Addr(), nil
+}

internal/cmd/jafar/httpproxy/httpproxy_test.go

@@ -0,0 +1,130 @@
+package httpproxy
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
+)
+
+func TestPass(t *testing.T) {
+	server, addr := newproxy(t, "ooni.io")
+	// We're filtering ooni.io, so we expect example.com to pass
+	// through the proxy with 200 and we also expect to see the
+	// Via header in the responses we receive, of course.
+	checkrequest(t, addr.String(), "example.com", 200, true)
+	killproxy(t, server)
+}
+
+func TestBlock(t *testing.T) {
+	server, addr := newproxy(t, "ooni.io")
+	// Here we're filtering any domain containing ooni.io, so we
+	// expect the proxy to send 451 without actually proxing, thus
+	// there should not be any Via header in the output.
+	checkrequest(t, addr.String(), "mia-ps.ooni.io", 451, false)
+	killproxy(t, server)
+}
+
+func TestLoop(t *testing.T) {
+	server, addr := newproxy(t, "ooni.io")
+	// Here we're forcing the proxy to connect to itself. It does
+	// does that and recognizes itself because of the Via header
+	// being set in the request generated by the connection to itself,
+	// which should cause a 400. The response should have the Via
+	// header set because the 400 is received by the connection that
+	// this code has made to the proxy.
+	checkrequest(t, addr.String(), addr.String(), 400, true)
+	killproxy(t, server)
+}
+
+func TestListenError(t *testing.T) {
+	proxy := NewCensoringProxy([]string{""}, uncensored.DefaultClient)
+	server, addr, err := proxy.Start("8.8.8.8:80")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if server != nil {
+		t.Fatal("expected nil server here")
+	}
+	if addr != nil {
+		t.Fatal("expected nil addr here")
+	}
+}
+
+func newproxy(t *testing.T, blocked string) (*http.Server, net.Addr) {
+	proxy := NewCensoringProxy([]string{blocked}, uncensored.DefaultClient)
+	server, addr, err := proxy.Start("127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return server, addr
+}
+
+func killproxy(t *testing.T, server *http.Server) {
+	err := server.Shutdown(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func checkrequest(
+	t *testing.T, proxyAddr, host string,
+	expectStatus int, expectVia bool,
+) {
+	req, err := http.NewRequest("GET", "http://"+proxyAddr, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Host = host
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != expectStatus {
+		t.Fatal("unexpected value of status code")
+	}
+	t.Log(resp)
+	values, _ := resp.Header["Via"]
+	var foundProduct bool
+	for _, value := range values {
+		if value == product {
+			foundProduct = true
+		}
+	}
+	if foundProduct && !expectVia {
+		t.Fatal("unexpectedly found Via header")
+	}
+	if !foundProduct && expectVia {
+		t.Fatal("Via header not found")
+	}
+	proxiedData, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if expectStatus == 200 {
+		checkbody(t, proxiedData, host)
+	}
+}
+
+func checkbody(t *testing.T, proxiedData []byte, host string) {
+	resp, err := http.Get("http://" + host)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != 200 {
+		t.Fatal("unexpected status code")
+	}
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Equal(data, proxiedData) == false {
+		t.Fatal("body mismatch")
+	}
+}

internal/cmd/jafar/iptables/iptables.go

@@ -0,0 +1,98 @@
+// Package iptables contains code for managing firewall rules. This package
+// really only works reliably on Linux. In all other systems the functionality
+// in here is just a set of stubs returning errors.
+package iptables
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+)
+
+type shell interface {
+	createChains() error
+	dropIfDestinationEquals(ip string) error
+	rstIfDestinationEqualsAndIsTCP(ip string) error
+	dropIfContainsKeywordHex(keyword string) error
+	dropIfContainsKeyword(keyword string) error
+	rstIfContainsKeywordHexAndIsTCP(keyword string) error
+	rstIfContainsKeywordAndIsTCP(keyword string) error
+	hijackDNS(address string) error
+	hijackHTTPS(address string) error
+	hijackHTTP(address string) error
+	waive() error
+}
+
+// CensoringPolicy implements a censoring policy.
+type CensoringPolicy struct {
+	DropIPs            []string // drop IP traffic to these IPs
+	DropKeywordsHex    []string // drop IP packets with these hex keywords
+	DropKeywords       []string // drop IP packets with these keywords
+	HijackDNSAddress   string   // where to hijack DNS to
+	HijackHTTPSAddress string   // where to hijack HTTPS to
+	HijackHTTPAddress  string   // where to hijack HTTP to
+	ResetIPs           []string // RST TCP/IP traffic to these IPs
+	ResetKeywordsHex   []string // RST TCP/IP flows with these hex keywords
+	ResetKeywords      []string // RST TCP/IP flows with these keywords
+	sh                 shell
+}
+
+// NewCensoringPolicy returns a new censoring policy.
+func NewCensoringPolicy() *CensoringPolicy {
+	return &CensoringPolicy{
+		sh: newShell(),
+	}
+}
+
+// Apply applies the censorship policy
+func (c *CensoringPolicy) Apply() (err error) {
+	defer func() {
+		if recover() != nil {
+			// JUST KNOW WE'VE BEEN HERE
+		}
+	}()
+	err = c.sh.createChains()
+	runtimex.PanicOnError(err, "c.sh.createChains failed")
+	// Implementation note: we want the RST rules to be first such
+	// that we end up enforcing them before the drop rules.
+	for _, keyword := range c.ResetKeywordsHex {
+		err = c.sh.rstIfContainsKeywordHexAndIsTCP(keyword)
+		runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordHexAndIsTCP failed")
+	}
+	for _, keyword := range c.ResetKeywords {
+		err = c.sh.rstIfContainsKeywordAndIsTCP(keyword)
+		runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordAndIsTCP failed")
+	}
+	for _, ip := range c.ResetIPs {
+		err = c.sh.rstIfDestinationEqualsAndIsTCP(ip)
+		runtimex.PanicOnError(err, "c.sh.rstIfDestinationEqualsAndIsTCP failed")
+	}
+	for _, keyword := range c.DropKeywordsHex {
+		err = c.sh.dropIfContainsKeywordHex(keyword)
+		runtimex.PanicOnError(err, "c.sh.dropIfContainsKeywordHex failed")
+	}
+	for _, keyword := range c.DropKeywords {
+		err = c.sh.dropIfContainsKeyword(keyword)
+		runtimex.PanicOnError(err, "c.sh.dropIfContainsKeyword failed")
+	}
+	for _, ip := range c.DropIPs {
+		err = c.sh.dropIfDestinationEquals(ip)
+		runtimex.PanicOnError(err, "c.sh.dropIfDestinationEquals failed")
+	}
+	if c.HijackDNSAddress != "" {
+		err = c.sh.hijackDNS(c.HijackDNSAddress)
+		runtimex.PanicOnError(err, "c.sh.hijackDNS failed")
+	}
+	if c.HijackHTTPSAddress != "" {
+		err = c.sh.hijackHTTPS(c.HijackHTTPSAddress)
+		runtimex.PanicOnError(err, "c.sh.hijackHTTPS failed")
+	}
+	if c.HijackHTTPAddress != "" {
+		err = c.sh.hijackHTTP(c.HijackHTTPAddress)
+		runtimex.PanicOnError(err, "c.sh.hijackHTTP failed")
+	}
+	return
+}
+
+// Waive removes any censorship policy
+func (c *CensoringPolicy) Waive() error {
+	return c.sh.waive()
+}

internal/cmd/jafar/iptables/iptables_integration_test.go

@@ -0,0 +1,345 @@
+package iptables
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os/exec"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
+)
+
+func init() {
+	log.SetLevel(log.ErrorLevel)
+}
+
+func newCensoringPolicy() *CensoringPolicy {
+	policy := NewCensoringPolicy()
+	policy.Waive() // start over to allow for repeated tests on failure
+	return policy
+}
+
+func TestCannotApplyPolicy(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	policy.DropIPs = []string{"antani"}
+	if err := policy.Apply(); err == nil {
+		t.Fatal("expected an error here")
+	}
+}
+
+func TestCreateChainsError(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	// you should not be able to apply the policy when there is
+	// already a policy, you need to waive it first
+	if err := policy.Apply(); err == nil {
+		t.Fatal("expected an error here")
+	}
+}
+
+func TestDropIP(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	policy.DropIPs = []string{"1.1.1.1"}
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", "1.1.1.1:853")
+	if err == nil {
+		t.Fatalf("expected an error here")
+	}
+	if err.Error() != "dial tcp 1.1.1.1:853: i/o timeout" {
+		t.Fatal("unexpected error occurred")
+	}
+	if conn != nil {
+		t.Fatal("expected nil connection here")
+	}
+}
+
+func TestDropKeyword(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	policy.DropKeywords = []string{"ooni.io"}
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	req, err := http.NewRequest("GET", "http://www.ooni.io", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if !strings.HasSuffix(err.Error(), "context deadline exceeded") {
+		t.Fatal("unexpected error occurred")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+}
+
+func TestDropKeywordHex(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	policy.DropKeywordsHex = []string{"|6f 6f 6e 69|"}
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	req, err := http.NewRequest("GET", "http://www.ooni.io", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	// the error we see with GitHub Actions is different from the error
+	// we see when testing locally on Fedora
+	if !strings.HasSuffix(err.Error(), "operation not permitted") &&
+		!strings.HasSuffix(err.Error(), "Temporary failure in name resolution") &&
+		!strings.HasSuffix(err.Error(), "no such host") {
+		t.Fatalf("unexpected error occurred: %+v", err)
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+}
+
+func TestResetIP(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	policy.ResetIPs = []string{"1.1.1.1"}
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	conn, err := (&net.Dialer{}).Dial("tcp", "1.1.1.1:853")
+	if err == nil {
+		t.Fatalf("expected an error here")
+	}
+	if err.Error() != "dial tcp 1.1.1.1:853: connect: connection refused" {
+		t.Fatal("unexpected error occurred")
+	}
+	if conn != nil {
+		t.Fatal("expected nil connection here")
+	}
+}
+
+func TestResetKeyword(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	policy.ResetKeywords = []string{"ooni.io"}
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	resp, err := http.Get("http://www.ooni.io")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if strings.Contains(err.Error(), "read: connection reset by peer") == false {
+		t.Fatal("unexpected error occurred")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+}
+
+func TestResetKeywordHex(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	policy.ResetKeywordsHex = []string{"|6f 6f 6e 69|"}
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	resp, err := http.Get("http://www.ooni.io")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if strings.Contains(err.Error(), "read: connection reset by peer") == false {
+		t.Fatal("unexpected error occurred")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+}
+
+func TestHijackDNS(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	resolver := resolver.NewCensoringResolver(
+		[]string{"ooni.io"}, nil, nil,
+		uncensored.Must(uncensored.NewClient("dot://1.1.1.1:853")),
+	)
+	server, err := resolver.Start("127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer server.Shutdown()
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	policy.HijackDNSAddress = server.PacketConn.LocalAddr().String()
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	addrs, err := net.LookupHost("www.ooni.io")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if strings.Contains(err.Error(), "no such host") == false {
+		t.Fatal("unexpected error occurred")
+	}
+	if addrs != nil {
+		t.Fatal("expected nil addrs here")
+	}
+}
+
+func TestHijackHTTP(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	// Implementation note: this test is complicated by the fact
+	// that we are running as root and so we're whitelisted.
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(451)
+		}),
+	)
+	defer server.Close()
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	pu, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	policy.HijackHTTPAddress = pu.Host
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	err = shellx.Run("sudo", "-u", "nobody", "--",
+		"curl", "-sf", "http://example.com")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	var exitErr *exec.ExitError
+	if !errors.As(err, &exitErr) {
+		t.Fatal("not the error type we expected")
+	}
+	if exitErr.ExitCode() != 22 {
+		t.Fatal("not the exit code we expected")
+	}
+}
+
+func TestHijackHTTPS(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("not implemented on this platform")
+	}
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	// Implementation note: this test is complicated by the fact
+	// that we are running as root and so we're whitelisted.
+	server := httptest.NewTLSServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(451)
+		}),
+	)
+	defer server.Close()
+	policy := newCensoringPolicy()
+	defer policy.Waive()
+	pu, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	policy.HijackHTTPSAddress = pu.Host
+	if err := policy.Apply(); err != nil {
+		t.Fatal(err)
+	}
+	err = shellx.Run("sudo", "-u", "nobody", "--",
+		"curl", "-sf", "https://example.com")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	t.Log(err)
+	var exitErr *exec.ExitError
+	if !errors.As(err, &exitErr) {
+		t.Fatal("not the error type we expected")
+	}
+	if exitErr.ExitCode() != 60 {
+		t.Fatal("not the exit code we expected")
+	}
+}

internal/cmd/jafar/iptables/iptables_linux.go

@@ -0,0 +1,117 @@
+// +build linux
+
+package iptables
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
+)
+
+type linuxShell struct{}
+
+func (s *linuxShell) createChains() (err error) {
+	defer func() {
+		if recover() != nil {
+			// JUST KNOW WE'VE BEEN HERE
+		}
+	}()
+	err = shellx.Run("sudo", "iptables", "-N", "JAFAR_INPUT")
+	runtimex.PanicOnError(err, "cannot create JAFAR_INPUT chain")
+	err = shellx.Run("sudo", "iptables", "-N", "JAFAR_OUTPUT")
+	runtimex.PanicOnError(err, "cannot create JAFAR_OUTPUT chain")
+	err = shellx.Run("sudo", "iptables", "-t", "nat", "-N", "JAFAR_NAT_OUTPUT")
+	runtimex.PanicOnError(err, "cannot create JAFAR_NAT_OUTPUT chain")
+	err = shellx.Run("sudo", "iptables", "-I", "OUTPUT", "-j", "JAFAR_OUTPUT")
+	runtimex.PanicOnError(err, "cannot insert jump to JAFAR_OUTPUT")
+	err = shellx.Run("sudo", "iptables", "-I", "INPUT", "-j", "JAFAR_INPUT")
+	runtimex.PanicOnError(err, "cannot insert jump to JAFAR_INPUT")
+	err = shellx.Run("sudo", "iptables", "-t", "nat", "-I", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT")
+	runtimex.PanicOnError(err, "cannot insert jump to JAFAR_NAT_OUTPUT")
+	return nil
+}
+
+func (s *linuxShell) dropIfDestinationEquals(ip string) error {
+	return shellx.Run("sudo", "iptables", "-A", "JAFAR_OUTPUT", "-d", ip, "-j", "DROP")
+}
+
+func (s *linuxShell) rstIfDestinationEqualsAndIsTCP(ip string) error {
+	return shellx.Run(
+		"sudo", "iptables", "-A", "JAFAR_OUTPUT", "--proto", "tcp", "-d", ip,
+		"-j", "REJECT", "--reject-with", "tcp-reset",
+	)
+}
+
+func (s *linuxShell) dropIfContainsKeywordHex(keyword string) error {
+	return shellx.Run(
+		"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp",
+		"--hex-string", keyword, "-j", "DROP",
+	)
+}
+
+func (s *linuxShell) dropIfContainsKeyword(keyword string) error {
+	return shellx.Run(
+		"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp",
+		"--string", keyword, "-j", "DROP",
+	)
+}
+
+func (s *linuxShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error {
+	return shellx.Run(
+		"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo",
+		"kmp", "--hex-string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset",
+	)
+}
+
+func (s *linuxShell) rstIfContainsKeywordAndIsTCP(keyword string) error {
+	return shellx.Run(
+		"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo",
+		"kmp", "--string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset",
+	)
+}
+
+func (s *linuxShell) hijackDNS(address string) error {
+	// Hijack any DNS query, like the Vodafone station does when using the
+	// secure network feature. Our transparent proxies will use DoT, in order
+	// to bypass this restriction and avoid routing loop.
+	return shellx.Run(
+		"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "udp",
+		"--dport", "53", "-j", "DNAT", "--to", address,
+	)
+}
+
+func (s *linuxShell) hijackHTTPS(address string) error {
+	// We need to whitelist root otherwise the traffic sent by Jafar
+	// itself will match the rule and loop.
+	return shellx.Run(
+		"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp",
+		"--dport", "443", "-m", "owner", "!", "--uid-owner", "0",
+		"-j", "DNAT", "--to", address,
+	)
+}
+
+func (s *linuxShell) hijackHTTP(address string) error {
+	// We need to whitelist root otherwise the traffic sent by Jafar
+	// itself will match the rule and loop.
+	return shellx.Run(
+		"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp",
+		"--dport", "80", "-m", "owner", "!", "--uid-owner", "0",
+		"-j", "DNAT", "--to", address,
+	)
+}
+
+func (s *linuxShell) waive() error {
+	shellx.RunQuiet("sudo", "iptables", "-D", "OUTPUT", "-j", "JAFAR_OUTPUT")
+	shellx.RunQuiet("sudo", "iptables", "-D", "INPUT", "-j", "JAFAR_INPUT")
+	shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-D", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT")
+	shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_INPUT")
+	shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_INPUT")
+	shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_OUTPUT")
+	shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_OUTPUT")
+	shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-F", "JAFAR_NAT_OUTPUT")
+	shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-X", "JAFAR_NAT_OUTPUT")
+	return nil
+}
+
+func newShell() *linuxShell {
+	return &linuxShell{}
+}

internal/cmd/jafar/iptables/iptables_unsupported.go

@@ -0,0 +1,45 @@
+// +build !linux
+
+package iptables
+
+import "errors"
+
+type otherwiseShell struct{}
+
+func (*otherwiseShell) createChains() error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) dropIfDestinationEquals(ip string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) rstIfDestinationEqualsAndIsTCP(ip string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) dropIfContainsKeywordHex(keyword string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) dropIfContainsKeyword(keyword string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) rstIfContainsKeywordAndIsTCP(keyword string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) hijackDNS(address string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) hijackHTTPS(address string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) hijackHTTP(address string) error {
+	return errors.New("not implemented")
+}
+func (*otherwiseShell) waive() error {
+	return errors.New("not implemented")
+}
+
+func newShell() *otherwiseShell {
+	return &otherwiseShell{}
+}

internal/cmd/jafar/main.go

@@ -0,0 +1,286 @@
+// Jafar is a censorship simulation tool used for testing OONI.
+package main
+
+import (
+	"encoding/pem"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"github.com/apex/log"
+	"github.com/apex/log/handlers/cli"
+	"github.com/miekg/dns"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/badproxy"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/flagx"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/httpproxy"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/iptables"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/tlsproxy"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
+	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
+)
+
+var (
+	badProxyAddress     *string
+	badProxyAddressTLS  *string
+	badProxyTLSOutputCA *string
+
+	dnsProxyAddress *string
+	dnsProxyBlock   flagx.StringArray
+	dnsProxyHijack  flagx.StringArray
+	dnsProxyIgnore  flagx.StringArray
+
+	httpProxyAddress *string
+	httpProxyBlock   flagx.StringArray
+
+	iptablesDropIP          flagx.StringArray
+	iptablesDropKeywordHex  flagx.StringArray
+	iptablesDropKeyword     flagx.StringArray
+	iptablesHijackDNSTo     *string
+	iptablesHijackHTTPSTo   *string
+	iptablesHijackHTTPTo    *string
+	iptablesResetIP         flagx.StringArray
+	iptablesResetKeywordHex flagx.StringArray
+	iptablesResetKeyword    flagx.StringArray
+
+	mainCh      chan os.Signal
+	mainCommand *string
+	mainUser    *string
+
+	tag *string
+
+	tlsProxyAddress *string
+	tlsProxyBlock   flagx.StringArray
+
+	uncensoredResolverURL *string
+)
+
+func init() {
+	// badProxy
+	badProxyAddress = flag.String(
+		"bad-proxy-address", "127.0.0.1:7117",
+		"Address where to listen for TCP connections",
+	)
+	badProxyAddressTLS = flag.String(
+		"bad-proxy-address-tls", "127.0.0.1:4114",
+		"Address where to listen for TLS connections",
+	)
+	badProxyTLSOutputCA = flag.String(
+		"bad-proxy-tls-output-ca", "badproxy.pem",
+		"File where to write the CA used by the bad proxy",
+	)
+
+	// dnsProxy
+	dnsProxyAddress = flag.String(
+		"dns-proxy-address", "127.0.0.1:53",
+		"Address where the DNS proxy should listen",
+	)
+	flag.Var(
+		&dnsProxyBlock, "dns-proxy-block",
+		"Register keyword triggering NXDOMAIN censorship",
+	)
+	flag.Var(
+		&dnsProxyHijack, "dns-proxy-hijack",
+		"Register keyword triggering redirection to 127.0.0.1",
+	)
+	flag.Var(
+		&dnsProxyIgnore, "dns-proxy-ignore",
+		"Register keyword causing the proxy to ignore the query",
+	)
+
+	// httpProxy
+	httpProxyAddress = flag.String(
+		"http-proxy-address", "127.0.0.1:80",
+		"Address where the HTTP proxy should listen",
+	)
+	flag.Var(
+		&httpProxyBlock, "http-proxy-block",
+		"Register keyword triggering HTTP 451 censorship",
+	)
+
+	// iptables
+	flag.Var(
+		&iptablesDropIP, "iptables-drop-ip",
+		"Drop traffic to the specified IP address",
+	)
+	flag.Var(
+		&iptablesDropKeywordHex, "iptables-drop-keyword-hex",
+		"Drop traffic containing the specified keyword in hex",
+	)
+	flag.Var(
+		&iptablesDropKeyword, "iptables-drop-keyword",
+		"Drop traffic containing the specified keyword",
+	)
+	iptablesHijackDNSTo = flag.String(
+		"iptables-hijack-dns-to", "",
+		"Hijack all DNS UDP traffic to the specified endpoint",
+	)
+	iptablesHijackHTTPSTo = flag.String(
+		"iptables-hijack-https-to", "",
+		"Hijack all HTTPS traffic to the specified endpoint",
+	)
+	iptablesHijackHTTPTo = flag.String(
+		"iptables-hijack-http-to", "",
+		"Hijack all HTTP traffic to the specified endpoint",
+	)
+	flag.Var(
+		&iptablesResetIP, "iptables-reset-ip",
+		"Reset TCP/IP traffic to the specified IP address",
+	)
+	flag.Var(
+		&iptablesResetKeywordHex, "iptables-reset-keyword-hex",
+		"Reset TCP/IP traffic containing the specified keyword in hex",
+	)
+	flag.Var(
+		&iptablesResetKeyword, "iptables-reset-keyword",
+		"Reset TCP/IP traffic containing the specified keyword",
+	)
+
+	// main
+	mainCh = make(chan os.Signal, 1)
+	signal.Notify(
+		mainCh, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT,
+	)
+	mainCommand = flag.String("main-command", "", "Optional command to execute")
+	mainUser = flag.String("main-user", "nobody", "Run command as user")
+
+	// tag
+	tag = flag.String("tag", "", "Add tag to a specific run")
+
+	// tlsProxy
+	tlsProxyAddress = flag.String(
+		"tls-proxy-address", "127.0.0.1:443",
+		"Address where the HTTP proxy should listen",
+	)
+	flag.Var(
+		&tlsProxyBlock, "tls-proxy-block",
+		"Register keyword triggering TLS censorship",
+	)
+
+	// uncensored
+	uncensoredResolverURL = flag.String(
+		"uncensored-resolver-url", "dot://1.1.1.1:853",
+		"URL of an hopefully uncensored resolver",
+	)
+}
+
+func badProxyStart() net.Listener {
+	proxy := badproxy.NewCensoringProxy()
+	listener, err := proxy.Start(*badProxyAddress)
+	runtimex.PanicOnError(err, "proxy.Start failed")
+	return listener
+}
+
+func badProxyStartTLS() net.Listener {
+	proxy := badproxy.NewCensoringProxy()
+	listener, cert, err := proxy.StartTLS(*badProxyAddressTLS)
+	runtimex.PanicOnError(err, "proxy.StartTLS failed")
+	err = ioutil.WriteFile(*badProxyTLSOutputCA, pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: cert.Raw,
+	}), 0644)
+	runtimex.PanicOnError(err, "ioutil.WriteFile failed")
+	return listener
+}
+
+func dnsProxyStart(uncensored *uncensored.Client) *dns.Server {
+	proxy := resolver.NewCensoringResolver(
+		dnsProxyBlock, dnsProxyHijack, dnsProxyIgnore, uncensored,
+	)
+	server, err := proxy.Start(*dnsProxyAddress)
+	runtimex.PanicOnError(err, "proxy.Start failed")
+	return server
+}
+
+func httpProxyStart(uncensored *uncensored.Client) *http.Server {
+	proxy := httpproxy.NewCensoringProxy(httpProxyBlock, uncensored)
+	server, _, err := proxy.Start(*httpProxyAddress)
+	runtimex.PanicOnError(err, "proxy.Start failed")
+	return server
+}
+
+func iptablesStart() *iptables.CensoringPolicy {
+	policy := iptables.NewCensoringPolicy()
+	// For robustness waive the policy so we start afresh
+	policy.Waive()
+	policy.DropIPs = iptablesDropIP
+	policy.DropKeywordsHex = iptablesDropKeywordHex
+	policy.DropKeywords = iptablesDropKeyword
+	policy.HijackDNSAddress = *iptablesHijackDNSTo
+	policy.HijackHTTPSAddress = *iptablesHijackHTTPSTo
+	policy.HijackHTTPAddress = *iptablesHijackHTTPTo
+	policy.ResetIPs = iptablesResetIP
+	policy.ResetKeywordsHex = iptablesResetKeywordHex
+	policy.ResetKeywords = iptablesResetKeyword
+	err := policy.Apply()
+	runtimex.PanicOnError(err, "policy.Apply failed")
+	return policy
+}
+
+func tlsProxyStart(uncensored *uncensored.Client) net.Listener {
+	proxy := tlsproxy.NewCensoringProxy(tlsProxyBlock, uncensored)
+	listener, err := proxy.Start(*tlsProxyAddress)
+	runtimex.PanicOnError(err, "proxy.Start failed")
+	return listener
+}
+
+func newUncensoredClient() *uncensored.Client {
+	clnt, err := uncensored.NewClient(*uncensoredResolverURL)
+	runtimex.PanicOnError(err, "uncensored.NewClient failed")
+	return clnt
+}
+
+func mustx(err error, message string, osExit func(int)) {
+	if err != nil {
+		var (
+			exitcode = 1
+			exiterr  *exec.ExitError
+		)
+		if errors.As(err, &exiterr) {
+			exitcode = exiterr.ExitCode()
+		}
+		log.Errorf("%s", message)
+		osExit(exitcode)
+	}
+}
+
+func main() {
+	flag.Parse()
+	// TODO(bassosimone): we may want a verbose flag
+	log.SetLevel(log.InfoLevel)
+	log.SetHandler(cli.Default)
+	log.Infof("jafar command line: [%s]", strings.Join(os.Args, ", "))
+	log.Infof("jafar tag: %s", *tag)
+	uncensoredClient := newUncensoredClient()
+	defer uncensoredClient.CloseIdleConnections()
+	badlistener := badProxyStart()
+	defer badlistener.Close()
+	badtlslistener := badProxyStartTLS()
+	defer badtlslistener.Close()
+	dnsproxy := dnsProxyStart(uncensoredClient)
+	defer dnsproxy.Shutdown()
+	httpproxy := httpProxyStart(uncensoredClient)
+	defer httpproxy.Close()
+	tlslistener := tlsProxyStart(uncensoredClient)
+	defer tlslistener.Close()
+	policy := iptablesStart()
+	var err error
+	if *mainCommand != "" {
+		err = shellx.RunCommandline(fmt.Sprintf(
+			"sudo -u '%s' -- %s", *mainUser, *mainCommand,
+		))
+	} else {
+		<-mainCh
+	}
+	policy.Waive()
+	mustx(err, "subcommand failed", os.Exit)
+}

internal/cmd/jafar/main_test.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"errors"
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/iptables"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
+)
+
+func ensureWeStartOverWithIPTables() {
+	iptables.NewCensoringPolicy().Waive()
+}
+
+func TestNoCommand(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("skip test on non Linux systems")
+	}
+	ensureWeStartOverWithIPTables()
+	*dnsProxyAddress = "127.0.0.1:0"
+	*httpProxyAddress = "127.0.0.1:0"
+	*tlsProxyAddress = "127.0.0.1:0"
+	go func() {
+		mainCh <- os.Interrupt
+	}()
+	main()
+}
+
+func TestWithCommand(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("skip test on non Linux systems")
+	}
+	ensureWeStartOverWithIPTables()
+	*dnsProxyAddress = "127.0.0.1:0"
+	*httpProxyAddress = "127.0.0.1:0"
+	*tlsProxyAddress = "127.0.0.1:0"
+	*mainCommand = "whoami"
+	defer func() {
+		*mainCommand = ""
+	}()
+	main()
+}
+
+func TestMustx(t *testing.T) {
+	t.Run("with no error", func(t *testing.T) {
+		var called int
+		mustx(nil, "", func(int) {
+			called++
+		})
+		if called != 0 {
+			t.Fatal("should not happen")
+		}
+	})
+	t.Run("with non-exit-code error", func(t *testing.T) {
+		var (
+			called   int
+			exitcode int
+		)
+		mustx(errors.New("antani"), "", func(ec int) {
+			called++
+			exitcode = ec
+		})
+		if called != 1 {
+			t.Fatal("not called?!")
+		}
+		if exitcode != 1 {
+			t.Fatal("unexpected exitcode value")
+		}
+	})
+	t.Run("with exit-code error", func(t *testing.T) {
+		var (
+			called   int
+			exitcode int
+		)
+		err := shellx.Run("curl", "-sf", "") // cause exitcode == 3
+		mustx(err, "", func(ec int) {
+			called++
+			exitcode = ec
+		})
+		if called != 1 {
+			t.Fatal("not called?!")
+		}
+		if exitcode != 3 {
+			t.Fatal("unexpected exitcode value")
+		}
+	})
+}

internal/cmd/jafar/resolver/resolver.go

@@ -0,0 +1,130 @@
+// Package resolver contains a censoring DNS resolver. Most queries are
+// answered without censorship, but selected queries could either be
+// discarded or replied to with a bogon or NXDOMAIN answer.
+package resolver
+
+import (
+	"context"
+	"net"
+	"strings"
+
+	"github.com/miekg/dns"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+// CensoringResolver is a censoring resolver.
+type CensoringResolver struct {
+	blocked    []string
+	hijacked   []string
+	ignored    []string
+	lookupHost func(ctx context.Context, host string) ([]string, error)
+}
+
+// NewCensoringResolver creates a new CensoringResolver instance using
+// the specified list of keywords to censor. blocked is the list of
+// keywords that trigger NXDOMAIN if they appear in a query. hijacked
+// is similar but redirects to 127.0.0.1, where the transparent HTTP
+// and TLS proxies will pick them up. dnsNetwork and dnsAddress are the
+// settings to configure the upstream, non censored DNS.
+func NewCensoringResolver(
+	blocked, hijacked, ignored []string, uncensored netx.Resolver,
+) *CensoringResolver {
+	return &CensoringResolver{
+		blocked:    blocked,
+		hijacked:   hijacked,
+		ignored:    ignored,
+		lookupHost: uncensored.LookupHost,
+	}
+}
+
+func (r *CensoringResolver) roundtrip(rw dns.ResponseWriter, req *dns.Msg) {
+	name := req.Question[0].Name
+	addrs, err := r.lookupHost(context.Background(), name)
+	var ips []net.IP
+	if err == nil {
+		for _, addr := range addrs {
+			if ip := net.ParseIP(addr); ip != nil {
+				ips = append(ips, ip)
+			}
+		}
+	}
+	r.reply(rw, req, ips)
+}
+
+func (r *CensoringResolver) reply(
+	rw dns.ResponseWriter, req *dns.Msg, ips []net.IP,
+) {
+	m := new(dns.Msg)
+	m.Compress = true
+	m.MsgHdr.RecursionAvailable = true
+	m.SetReply(req)
+	for _, ip := range ips {
+		ipv6 := strings.Contains(ip.String(), ":")
+		if !ipv6 && req.Question[0].Qtype == dns.TypeA {
+			m.Answer = append(m.Answer, &dns.A{
+				Hdr: dns.RR_Header{
+					Name:   req.Question[0].Name,
+					Rrtype: dns.TypeA,
+					Class:  dns.ClassINET,
+					Ttl:    0,
+				},
+				A: ip,
+			})
+		}
+	}
+	if m.Answer == nil {
+		m.SetRcode(req, dns.RcodeNameError)
+	}
+	rw.WriteMsg(m)
+}
+
+func (r *CensoringResolver) failure(rw dns.ResponseWriter, req *dns.Msg) {
+	m := new(dns.Msg)
+	m.Compress = true
+	m.MsgHdr.RecursionAvailable = true
+	m.SetRcode(req, dns.RcodeServerFailure)
+	rw.WriteMsg(m)
+}
+
+// ServeDNS serves a DNS request
+func (r *CensoringResolver) ServeDNS(rw dns.ResponseWriter, req *dns.Msg) {
+	if len(req.Question) < 1 {
+		r.failure(rw, req)
+		return
+	}
+	name := req.Question[0].Name
+	for _, pattern := range r.blocked {
+		if strings.Contains(name, pattern) {
+			r.reply(rw, req, nil)
+			return
+		}
+	}
+	for _, pattern := range r.hijacked {
+		if strings.Contains(name, pattern) {
+			r.reply(rw, req, []net.IP{net.IPv4(127, 0, 0, 1)})
+			return
+		}
+	}
+	for _, pattern := range r.ignored {
+		if strings.Contains(name, pattern) {
+			return
+		}
+	}
+	r.roundtrip(rw, req)
+}
+
+// Start starts the DNS resolver
+func (r *CensoringResolver) Start(address string) (*dns.Server, error) {
+	packetconn, err := net.ListenPacket("udp", address)
+	if err != nil {
+		return nil, err
+	}
+	server := &dns.Server{
+		Addr:       address,
+		Handler:    r,
+		Net:        "udp",
+		PacketConn: packetconn,
+	}
+	go server.ActivateAndServe()
+	return server, nil
+}

internal/cmd/jafar/resolver/resolver_test.go

@@ -0,0 +1,173 @@
+package resolver
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/miekg/dns"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
+)
+
+func TestPass(t *testing.T) {
+	server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
+	checkrequest(t, server, "example.com", "success", nil)
+	killserver(t, server)
+}
+
+func TestBlock(t *testing.T) {
+	server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
+	checkrequest(t, server, "mia-ps.ooni.io", "blocked", nil)
+	killserver(t, server)
+}
+
+func TestRedirect(t *testing.T) {
+	server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
+	checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", nil)
+	killserver(t, server)
+}
+
+func TestIgnore(t *testing.T) {
+	server := newresolver(t, nil, nil, []string{"ooni.nu"})
+	iotimeout := "i/o timeout"
+	checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", &iotimeout)
+	killserver(t, server)
+}
+
+func TestLookupFailure(t *testing.T) {
+	server := newresolver(t, nil, nil, nil)
+	// we should receive same response as when we're blocked
+	checkrequest(t, server, "example.antani", "blocked", nil)
+	killserver(t, server)
+}
+
+func TestFailureNoQuestion(t *testing.T) {
+	resolver := NewCensoringResolver(
+		nil, nil, nil, uncensored.DefaultClient,
+	)
+	resolver.ServeDNS(&fakeResponseWriter{t: t}, new(dns.Msg))
+}
+
+func TestListenFailure(t *testing.T) {
+	resolver := NewCensoringResolver(
+		nil, nil, nil, uncensored.DefaultClient,
+	)
+	server, err := resolver.Start("8.8.8.8:53")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if server != nil {
+		t.Fatal("expected nil server here")
+	}
+}
+
+func newresolver(t *testing.T, blocked, hijacked, ignored []string) *dns.Server {
+	resolver := NewCensoringResolver(
+		blocked, hijacked, ignored,
+		// using faster dns because dot here causes miekg/dns's
+		// dns.Exchange to timeout and I don't want more complexity
+		uncensored.Must(uncensored.NewClient("system:///")),
+	)
+	server, err := resolver.Start("127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return server
+}
+
+func killserver(t *testing.T, server *dns.Server) {
+	err := server.Shutdown()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func checkrequest(
+	t *testing.T, server *dns.Server, host string, expectStatus string,
+	expectErrorSuffix *string,
+) {
+	address := server.PacketConn.LocalAddr().String()
+	query := newquery(host)
+	reply, err := dns.Exchange(query, address)
+	if err != nil {
+		if expectErrorSuffix != nil &&
+			strings.HasSuffix(err.Error(), *expectErrorSuffix) {
+			return
+		}
+		t.Fatal(err)
+	}
+	switch expectStatus {
+	case "success":
+		checksuccess(t, reply)
+	case "hijacked":
+		checkhijacked(t, reply)
+	case "blocked":
+		checkblocked(t, reply)
+	default:
+		panic("unexpected value")
+	}
+}
+
+func checksuccess(t *testing.T, reply *dns.Msg) {
+	if reply.Rcode != dns.RcodeSuccess {
+		t.Fatal("unexpected rcode")
+	}
+	if len(reply.Answer) < 1 {
+		t.Fatal("too few answers")
+	}
+	for _, answer := range reply.Answer {
+		if rr, ok := answer.(*dns.A); ok {
+			if rr.A.String() == "127.0.0.1" {
+				t.Fatal("unexpected hijacked response here")
+			}
+		}
+	}
+}
+
+func checkhijacked(t *testing.T, reply *dns.Msg) {
+	if reply.Rcode != dns.RcodeSuccess {
+		t.Fatal("unexpected rcode")
+	}
+	if len(reply.Answer) < 1 {
+		t.Fatal("too few answers")
+	}
+	for _, answer := range reply.Answer {
+		if rr, ok := answer.(*dns.A); ok {
+			if rr.A.String() != "127.0.0.1" {
+				t.Fatal("unexpected non-hijacked response here")
+			}
+		}
+	}
+}
+
+func checkblocked(t *testing.T, reply *dns.Msg) {
+	if reply.Rcode != dns.RcodeNameError {
+		t.Fatal("unexpected rcode")
+	}
+	if len(reply.Answer) >= 1 {
+		t.Fatal("too many answers")
+	}
+}
+
+func newquery(name string) *dns.Msg {
+	query := new(dns.Msg)
+	query.Id = dns.Id()
+	query.RecursionDesired = true
+	query.Question = append(query.Question, dns.Question{
+		Name:   dns.Fqdn(name),
+		Qclass: dns.ClassINET,
+		Qtype:  dns.TypeA,
+	})
+	return query
+}
+
+type fakeResponseWriter struct {
+	dns.ResponseWriter
+	t *testing.T
+}
+
+func (rw *fakeResponseWriter) WriteMsg(m *dns.Msg) error {
+	if m.Rcode != dns.RcodeServerFailure {
+		rw.t.Fatal("unexpected rcode")
+	}
+	return nil
+}

internal/cmd/jafar/tlsproxy/tlsproxy.go

@@ -0,0 +1,194 @@
+// Package tlsproxy contains a censoring TLS proxy. Most traffic is passed
+// through using the SNI to choose the hostname to connect to. Specific offending
+// SNIs are censored by returning a TLS alert to the client.
+package tlsproxy
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net"
+	"strings"
+	"sync"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+// CensoringProxy is a censoring TLS proxy
+type CensoringProxy struct {
+	keywords []string
+	dial     func(network, address string) (net.Conn, error)
+}
+
+// NewCensoringProxy creates a new CensoringProxy instance using
+// the specified list of keywords to censor. keywords is the list
+// of keywords that trigger censorship if any of them appears in
+// the SNII record of a ClientHello. dnsNetwork and dnsAddress are
+// settings to configure the upstream, non censored DNS.
+func NewCensoringProxy(
+	keywords []string, uncensored netx.Dialer,
+) *CensoringProxy {
+	return &CensoringProxy{
+		keywords: keywords,
+		dial: func(network, address string) (net.Conn, error) {
+			return uncensored.DialContext(context.Background(), network, address)
+		},
+	}
+}
+
+// handshakeReader is a hack to perform the initial part of the
+// TLS handshake so to know the SNI and then replay the bytes of
+// this initial part of the handshake with the server.
+type handshakeReader struct {
+	net.Conn
+	incoming []byte
+}
+
+// Read saves the initial bytes of the handshake such that later
+// we can replay the handshake with the real TLS server.
+func (c *handshakeReader) Read(b []byte) (int, error) {
+	count, err := c.Conn.Read(b)
+	if err == nil {
+		c.incoming = append(c.incoming, b[:count]...)
+	}
+	return count, err
+}
+
+// Write prevents writing on the real connection
+func (c *handshakeReader) Write(b []byte) (int, error) {
+	return 0, errors.New("cannot write on this connection")
+}
+
+// forward forwards left traffic to right
+func forward(wg *sync.WaitGroup, left, right net.Conn) {
+	data := make([]byte, 1<<18)
+	for {
+		n, err := left.Read(data)
+		if err != nil {
+			break
+		}
+		if _, err = right.Write(data[:n]); err != nil {
+			break
+		}
+	}
+	wg.Done()
+}
+
+// reset closes the connection with a RST segment
+func reset(conn net.Conn) {
+	if tc, ok := conn.(*net.TCPConn); ok {
+		tc.SetLinger(0)
+	}
+	conn.Close()
+}
+
+// alertclose sends a TLS alert and then closes the connection
+func alertclose(conn net.Conn) {
+	alertdata := []byte{
+		21, // alert
+		3,  // version[0]
+		3,  // version[1]
+		0,  // length[0]
+		2,  // length[1]
+		2,  // fatal
+		80, // internal error
+	}
+	conn.Write(alertdata)
+	conn.Close()
+}
+
+// getsni attempts the handshakeReader hack to obtain the SNI by reading
+// the beginning of the TLS handshake. On success a nonempty SNI string
+// is returned. Otherwise we cannot distinguish between the absence of a
+// SNI and any other reading network error that may have occurred.
+func getsni(conn *handshakeReader) string {
+	var (
+		sni   string
+		mutex sync.Mutex // just for safety
+	)
+	tls.Server(conn, &tls.Config{
+		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			mutex.Lock()
+			sni = info.ServerName
+			mutex.Unlock()
+			return nil, errors.New("tlsproxy: we can't really continue handshake")
+		},
+	}).Handshake()
+	return sni
+}
+
+func (p *CensoringProxy) connectingToMyself(conn net.Conn) bool {
+	local := conn.LocalAddr().String()
+	localAddr, _, localErr := net.SplitHostPort(local)
+	remote := conn.RemoteAddr().String()
+	remoteAddr, _, remoteErr := net.SplitHostPort(remote)
+	return localErr != nil || remoteErr != nil || localAddr == remoteAddr
+}
+
+// handle implements the TLS SNI proxy
+func (p *CensoringProxy) handle(clientconn net.Conn) {
+	hr := &handshakeReader{Conn: clientconn}
+	sni := getsni(hr)
+	if sni == "" {
+		log.Warn("tlsproxy: network failure or SNI not provided")
+		reset(clientconn)
+		return
+	}
+	for _, pattern := range p.keywords {
+		if strings.Contains(sni, pattern) {
+			log.Warnf("tlsproxy: reject SNI by policy: %s", sni)
+			alertclose(clientconn)
+			return
+		}
+	}
+	serverconn, err := p.dial("tcp", net.JoinHostPort(sni, "443"))
+	if err != nil {
+		log.WithError(err).Warn("tlsproxy: p.dial failed")
+		alertclose(clientconn)
+		return
+	}
+	if p.connectingToMyself(serverconn) {
+		log.Warn("tlsproxy: connecting to myself")
+		alertclose(clientconn)
+		return
+	}
+	if _, err := serverconn.Write(hr.incoming); err != nil {
+		log.WithError(err).Warn("tlsproxy: serverconn.Write failed")
+		alertclose(clientconn)
+		return
+	}
+	log.Debugf("tlsproxy: routing for %s", sni)
+	defer clientconn.Close()
+	defer serverconn.Close()
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go forward(&wg, clientconn, serverconn)
+	go forward(&wg, serverconn, clientconn)
+	wg.Wait()
+}
+
+func (p *CensoringProxy) run(listener net.Listener) {
+	for {
+		conn, err := listener.Accept()
+		if err != nil && strings.Contains(
+			err.Error(), "use of closed network connection") {
+			return
+		}
+		if err == nil {
+			// It's difficult to make accept fail, so restructure
+			// the code such that we enter into the happy path
+			go p.handle(conn)
+		}
+	}
+}
+
+// Start starts the censoring proxy.
+func (p *CensoringProxy) Start(address string) (net.Listener, error) {
+	listener, err := net.Listen("tcp", address)
+	if err != nil {
+		return nil, err
+	}
+	go p.run(listener)
+	return listener, nil
+}

internal/cmd/jafar/tlsproxy/tlsproxy_test.go

@@ -0,0 +1,183 @@
+package tlsproxy
+
+import (
+	"crypto/tls"
+	"errors"
+	"net"
+	"sync"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
+)
+
+func TestPass(t *testing.T) {
+	listener := newproxy(t, "ooni.io")
+	checkdialtls(t, listener.Addr().String(), true, &tls.Config{
+		ServerName: "example.com",
+	})
+	killproxy(t, listener)
+}
+
+func TestBlock(t *testing.T) {
+	listener := newproxy(t, "ooni.io")
+	checkdialtls(t, listener.Addr().String(), false, &tls.Config{
+		ServerName: "mia-ps.ooni.io",
+	})
+	killproxy(t, listener)
+}
+
+func TestNoSNI(t *testing.T) {
+	listener := newproxy(t, "ooni.io")
+	checkdialtls(t, listener.Addr().String(), false, &tls.Config{
+		ServerName: "",
+	})
+	killproxy(t, listener)
+}
+
+func TestInvalidDomain(t *testing.T) {
+	listener := newproxy(t, "ooni.io")
+	checkdialtls(t, listener.Addr().String(), false, &tls.Config{
+		ServerName: "antani.local",
+	})
+	killproxy(t, listener)
+}
+
+func TestFailHandshake(t *testing.T) {
+	listener := newproxy(t, "ooni.io")
+	checkdialtls(t, listener.Addr().String(), false, &tls.Config{
+		ServerName: "expired.badssl.com",
+	})
+	killproxy(t, listener)
+}
+
+func TestFailConnectingToSelf(t *testing.T) {
+	proxy := &CensoringProxy{
+		dial: func(network string, address string) (net.Conn, error) {
+			return &mockedConnWriteError{}, nil
+		},
+	}
+	listener, err := proxy.Start("127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if listener == nil {
+		t.Fatal("expected non nil listener here")
+	}
+	checkdialtls(t, listener.Addr().String(), false, &tls.Config{
+		ServerName: "www.google.com",
+	})
+	killproxy(t, listener)
+}
+
+func TestFailWriteAfterConnect(t *testing.T) {
+	proxy := &CensoringProxy{
+		dial: func(network string, address string) (net.Conn, error) {
+			return &mockedConnWriteError{
+				// must be different or it refuses connecting to self
+				localIP:  net.IPv4(127, 0, 0, 1),
+				remoteIP: net.IPv4(127, 0, 0, 2),
+			}, nil
+		},
+	}
+	listener, err := proxy.Start("127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if listener == nil {
+		t.Fatal("expected non nil listener here")
+	}
+	checkdialtls(t, listener.Addr().String(), false, &tls.Config{
+		ServerName: "www.google.com",
+	})
+	killproxy(t, listener)
+}
+
+func TestListenError(t *testing.T) {
+	proxy := NewCensoringProxy(
+		[]string{""}, uncensored.DefaultClient,
+	)
+	listener, err := proxy.Start("8.8.8.8:80")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if listener != nil {
+		t.Fatal("expected nil listener here")
+	}
+}
+
+func newproxy(t *testing.T, blocked string) net.Listener {
+	proxy := NewCensoringProxy(
+		[]string{blocked}, uncensored.DefaultClient,
+	)
+	listener, err := proxy.Start("127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return listener
+}
+
+func killproxy(t *testing.T, listener net.Listener) {
+	err := listener.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func checkdialtls(
+	t *testing.T, proxyAddr string, expectSuccess bool, config *tls.Config,
+) {
+	conn, err := tls.Dial("tcp", proxyAddr, config)
+	if err != nil && expectSuccess {
+		t.Fatal(err)
+	}
+	if err == nil && !expectSuccess {
+		t.Fatal("expected failure here")
+	}
+	if conn == nil && expectSuccess {
+		t.Fatal("expected actionable conn")
+	}
+	if conn != nil && !expectSuccess {
+		t.Fatal("expected nil conn")
+	}
+	if conn != nil {
+		conn.Close()
+	}
+}
+
+type mockedConnWriteError struct {
+	net.Conn
+	localIP  net.IP
+	remoteIP net.IP
+}
+
+func (c *mockedConnWriteError) Write(b []byte) (int, error) {
+	return 0, errors.New("cannot write sorry")
+}
+
+func (c *mockedConnWriteError) LocalAddr() net.Addr {
+	return &net.TCPAddr{
+		IP: c.localIP,
+	}
+}
+
+func (c *mockedConnWriteError) RemoteAddr() net.Addr {
+	return &net.TCPAddr{
+		IP: c.remoteIP,
+	}
+}
+
+func TestForwardWriteError(t *testing.T) {
+	var wg sync.WaitGroup
+	wg.Add(1)
+	forward(&wg, &mockedConnReadOkay{}, &mockedConnWriteError{})
+}
+
+type mockedConnReadOkay struct {
+	net.Conn
+	localIP  net.IP
+	remoteIP net.IP
+}
+
+func (c *mockedConnReadOkay) Read(b []byte) (int, error) {
+	return len(b), nil
+}

internal/cmd/jafar/uncensored/uncensored.go

@@ -0,0 +1,86 @@
+// Package uncensored contains code used by Jafar to evade its own
+// censorship efforts by taking alternate routes.
+package uncensored
+
+import (
+	"context"
+	"net"
+	"net/http"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+// Client is DNS, HTTP, and TCP client.
+type Client struct {
+	dnsClient     *netx.DNSClient
+	httpTransport netx.HTTPRoundTripper
+	dialer        netx.Dialer
+}
+
+// NewClient creates a new Client.
+func NewClient(resolverURL string) (*Client, error) {
+	configuration, err := urlgetter.Configurer{
+		Config: urlgetter.Config{
+			ResolverURL: resolverURL,
+		},
+		Logger: log.Log,
+	}.NewConfiguration()
+	if err != nil {
+		return nil, err
+	}
+	return &Client{
+		dnsClient:     &configuration.DNSClient,
+		httpTransport: netx.NewHTTPTransport(configuration.HTTPConfig),
+		dialer:        netx.NewDialer(configuration.HTTPConfig),
+	}, nil
+}
+
+// Must panics if it's not possible to create a Client. Usually you should
+// use it like `uncensored.Must(uncensored.NewClient(URL))`.
+func Must(client *Client, err error) *Client {
+	runtimex.PanicOnError(err, "cannot create uncensored client")
+	return client
+}
+
+// DefaultClient is the default client for DNS, HTTP, and TCP.
+var DefaultClient = Must(NewClient(""))
+
+var _ netx.Resolver = DefaultClient
+
+// Address implements netx.Resolver.Address
+func (c *Client) Address() string {
+	return c.dnsClient.Address()
+}
+
+// LookupHost implements netx.Resolver.LookupHost
+func (c *Client) LookupHost(ctx context.Context, domain string) ([]string, error) {
+	return c.dnsClient.LookupHost(ctx, domain)
+}
+
+// Network implements netx.Resolver.Network
+func (c *Client) Network() string {
+	return c.dnsClient.Network()
+}
+
+var _ netx.Dialer = DefaultClient
+
+// DialContext implements netx.Dialer.DialContext
+func (c *Client) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	return c.dialer.DialContext(ctx, network, address)
+}
+
+var _ netx.HTTPRoundTripper = DefaultClient
+
+// CloseIdleConnections implement netx.HTTPRoundTripper.CloseIdleConnections
+func (c *Client) CloseIdleConnections() {
+	c.dnsClient.CloseIdleConnections()
+	c.httpTransport.CloseIdleConnections()
+}
+
+// RoundTrip implement netx.HTTPRoundTripper.RoundTrip
+func (c *Client) RoundTrip(req *http.Request) (*http.Response, error) {
+	return c.httpTransport.RoundTrip(req)
+}

internal/cmd/jafar/uncensored/uncensored_test.go

@@ -0,0 +1,75 @@
+package uncensored
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+func TestGood(t *testing.T) {
+	client, err := NewClient("dot://1.1.1.1:853")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.CloseIdleConnections()
+	if client.Address() != "1.1.1.1:853" {
+		t.Fatal("invalid address")
+	}
+	if client.Network() != "dot" {
+		t.Fatal("invalid network")
+	}
+	ctx := context.Background()
+	addrs, err := client.LookupHost(ctx, "dns.google")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var quad8, two8two4 bool
+	for _, addr := range addrs {
+		quad8 = quad8 || (addr == "8.8.8.8")
+		two8two4 = two8two4 || (addr == "8.8.4.4")
+	}
+	if quad8 != true && two8two4 != true {
+		t.Fatal("invalid response")
+	}
+	conn, err := client.DialContext(ctx, "tcp", "8.8.8.8:853")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	resp, err := client.RoundTrip(&http.Request{
+		Method: "GET",
+		URL: &url.URL{
+			Scheme: "https",
+			Host:   "www.google.com",
+			Path:   "/humans.txt",
+		},
+		Header: http.Header{},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		t.Fatal("invalid status-code")
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.HasPrefix(data, []byte("Google is built by a large team")) {
+		t.Fatal("not the expected body")
+	}
+}
+
+func TestNewClientFailure(t *testing.T) {
+	clnt, err := NewClient("antani:///")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if clnt != nil {
+		t.Fatal("expected nil client here")
+	}
+}

internal/cmd/oohelper/README.md

@@ -0,0 +1,4 @@
+# oohelper
+
+This directory contains the source code of a simple client
+for the Web Connectivity test helper.

internal/cmd/oohelper/internal/client.go

@@ -0,0 +1,138 @@
+package internal
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/version"
+)
+
+type (
+	// CtrlResponse is the type of response returned by the test helper.
+	CtrlResponse = webconnectivity.ControlResponse
+
+	// ctrlRequest is the type of the request sent to the test helper.
+	ctrlRequest = webconnectivity.ControlRequest
+)
+
+// The following errors may be returned by this implementation.
+var (
+	ErrHTTPStatusCode          = errors.New("oohelper: http status code indicates failure")
+	ErrUnsupportedURLScheme    = errors.New("oohelper: unsupported URL scheme")
+	ErrUnsupportedExplicitPort = errors.New("oohelper: unsupported explicit port")
+	ErrEmptyURL                = errors.New("oohelper: empty server and/or target URL")
+	ErrInvalidURL              = errors.New("oohelper: cannot parse URL")
+	ErrCannotCreateRequest     = errors.New("oohelper: cannot create HTTP request")
+	ErrCannotParseJSONReply    = errors.New("oohelper: cannot parse JSON reply")
+)
+
+// OOClient is a client for the OONI Web Connectivity test helper.
+type OOClient struct {
+	// HTTPClient is the HTTP client to use.
+	HTTPClient *http.Client
+
+	// Resolver is the resolver to user.
+	Resolver netx.Resolver
+}
+
+// OOConfig contains configuration for the client.
+type OOConfig struct {
+	// ServerURL is the URL of the test helper server.
+	ServerURL string
+
+	// TargetURL is the URL that we want to measure.
+	TargetURL string
+}
+
+// MakeTCPEndpoints constructs the list of TCP endpoints to send
+// to the Web Connectivity test helper.
+func MakeTCPEndpoints(URL *url.URL, addrs []string) ([]string, error) {
+	var (
+		port string
+		out  []string
+	)
+	if URL.Host != URL.Hostname() {
+		return nil, ErrUnsupportedExplicitPort
+	}
+	switch URL.Scheme {
+	case "https":
+		port = "443"
+	case "http":
+		port = "80"
+	default:
+		return nil, ErrUnsupportedURLScheme
+	}
+	for _, addr := range addrs {
+		out = append(out, net.JoinHostPort(addr, port))
+	}
+	return out, nil
+}
+
+// Do sends a measurement request to the Web Connectivity test
+// helper and receives the corresponding response.
+func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, error) {
+	if config.TargetURL == "" || config.ServerURL == "" {
+		return nil, ErrEmptyURL
+	}
+	targetURL, err := url.Parse(config.TargetURL)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrInvalidURL, err.Error())
+	}
+	addrs, err := oo.Resolver.LookupHost(ctx, targetURL.Hostname())
+	if err != nil {
+		return nil, err
+	}
+	endpoints, err := MakeTCPEndpoints(targetURL, addrs)
+	if err != nil {
+		return nil, err
+	}
+	creq := ctrlRequest{
+		HTTPRequest: config.TargetURL,
+		HTTPRequestHeaders: map[string][]string{
+			"Accept":          {httpheader.Accept()},
+			"Accept-Language": {httpheader.AcceptLanguage()},
+			"User-Agent":      {httpheader.UserAgent()},
+		},
+		TCPConnect: endpoints,
+	}
+	data, err := json.Marshal(creq)
+	runtimex.PanicOnError(err, "oohelper: cannot marshal control request")
+	log.Debugf("out: %s", string(data))
+	req, err := http.NewRequestWithContext(ctx, "POST", config.ServerURL, bytes.NewReader(data))
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrCannotCreateRequest, err.Error())
+	}
+	req.Header.Add("user-agent", fmt.Sprintf(
+		"oohelper/%s ooniprobe-engine/%s", version.Version, version.Version,
+	))
+	req.Header.Add("content-type", "application/json")
+	resp, err := oo.HTTPClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, ErrHTTPStatusCode
+	}
+	data, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var cresp CtrlResponse
+	if err := json.Unmarshal(data, &cresp); err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrCannotParseJSONReply, err.Error())
+	}
+	return &cresp, nil
+}

internal/cmd/oohelper/internal/client_test.go

@@ -0,0 +1,338 @@
+package internal_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/url"
+	"reflect"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal"
+)
+
+func TestMakeTCPEndpoints(t *testing.T) {
+	type args struct {
+		URL   *url.URL
+		addrs []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+		err  error
+	}{{
+		name: "with host != hostname",
+		args: args{URL: &url.URL{Host: "127.0.0.1:8080"}},
+		err:  internal.ErrUnsupportedExplicitPort,
+	}, {
+		name: "with unsupported URL scheme",
+		args: args{URL: &url.URL{Host: "127.0.0.1", Scheme: "imap"}},
+		err:  internal.ErrUnsupportedURLScheme,
+	}, {
+		name: "with http scheme",
+		args: args{
+			URL:   &url.URL{Host: "www.kernel.org", Scheme: "http"},
+			addrs: []string{"1.1.1.1", "2.2.2.2", "::1"},
+		},
+		want: []string{"1.1.1.1:80", "2.2.2.2:80", "[::1]:80"},
+	}, {
+		name: "with https scheme",
+		args: args{
+			URL:   &url.URL{Host: "www.kernel.org", Scheme: "https"},
+			addrs: []string{"1.1.1.1", "2.2.2.2", "::1"},
+		},
+		want: []string{"1.1.1.1:443", "2.2.2.2:443", "[::1]:443"},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := internal.MakeTCPEndpoints(tt.args.URL, tt.args.addrs)
+			if !errors.Is(err, tt.err) {
+				t.Errorf("MakeTCPEndpoints() error = %v, wantErr %v", err, tt.err)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("MakeTCPEndpoints() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestOOClientDoWithEmptyTargetURL(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{}
+	clnt := internal.OOClient{}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrEmptyURL) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithEmptyServerURL(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{TargetURL: "http://www.example.com"}
+	clnt := internal.OOClient{}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrEmptyURL) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithInvalidTargetURL(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{TargetURL: "\t", ServerURL: "https://wcth.ooni.io"}
+	clnt := internal.OOClient{}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrInvalidURL) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithResolverFailure(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverThatFails(),
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrNotFound) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithUnsupportedExplicitPort(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com:8080",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrUnsupportedExplicitPort) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithInvalidServerURL(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "\t",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrCannotCreateRequest) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithRoundTripError(t *testing.T) {
+	expected := errors.New("mocked error")
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{
+			Transport: internal.FakeTransport{Err: expected},
+		},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithInvalidStatusCode(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{Transport: internal.FakeTransport{
+			Resp: &http.Response{
+				StatusCode: 400,
+				Body:       &internal.FakeBody{},
+			},
+		}},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrHTTPStatusCode) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithBodyReadError(t *testing.T) {
+	expected := errors.New("mocked error")
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{Transport: internal.FakeTransport{
+			Resp: &http.Response{
+				StatusCode: 200,
+				Body: &internal.FakeBody{
+					Err: expected,
+				},
+			},
+		}},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithInvalidJSON(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{Transport: internal.FakeTransport{
+			Resp: &http.Response{
+				StatusCode: 200,
+				Body: &internal.FakeBody{
+					Data: []byte("{"),
+				},
+			},
+		}},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrCannotParseJSONReply) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+const goodresponse = `{
+    "tcp_connect": {
+        "172.217.21.68:80": {
+            "status": true,
+            "failure": null
+        }
+    },
+    "http_request": {
+        "body_length": 207878,
+        "failure": null,
+        "title": "Google",
+        "headers": {
+            "Content-Type": "text/html"
+        },
+        "status_code": 200
+    },
+    "dns": {
+        "failure": null,
+        "addrs": [
+            "172.217.17.68"
+        ]
+    }
+}`
+
+func TestOOClientDoWithParseableJSON(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{Transport: internal.FakeTransport{
+			Resp: &http.Response{
+				StatusCode: 200,
+				Body: &internal.FakeBody{
+					Data: []byte(goodresponse),
+				},
+			},
+		}},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, nil) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp.DNS.Failure != nil {
+		t.Fatal("unexpected Failure value")
+	}
+	if len(cresp.DNS.Addrs) != 1 {
+		t.Fatal("unexpected number of DNS entries")
+	}
+	if cresp.DNS.Addrs[0] != "172.217.17.68" {
+		t.Fatal("unexpected DNS addrs [0]")
+	}
+	if cresp.HTTPRequest.BodyLength != 207878 {
+		t.Fatal("invalid http body length")
+	}
+	if cresp.HTTPRequest.Failure != nil {
+		t.Fatal("invalid http failure")
+	}
+	if cresp.HTTPRequest.Title != "Google" {
+		t.Fatal("invalid http title")
+	}
+	if len(cresp.HTTPRequest.Headers) != 1 {
+		t.Fatal("invalid http headers length")
+	}
+	if cresp.HTTPRequest.Headers["Content-Type"] != "text/html" {
+		t.Fatal("invalid http content-type header")
+	}
+	if cresp.HTTPRequest.StatusCode != 200 {
+		t.Fatal("invalid http status code")
+	}
+	if len(cresp.TCPConnect) != 1 {
+		t.Fatal("invalid tcp connect length")
+	}
+	entry, ok := cresp.TCPConnect["172.217.21.68:80"]
+	if !ok {
+		t.Fatal("cannot find expected TCP connect entry")
+	}
+	if entry.Status != true {
+		t.Fatal("unexpected TCP connect entry status")
+	}
+	if entry.Failure != nil {
+		t.Fatal("unexpected TCP connect entry failure value")
+	}
+}

internal/cmd/oohelper/internal/fake_test.go

@@ -0,0 +1,102 @@
+package internal
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+type FakeResolver struct {
+	NumFailures *atomicx.Int64
+	Err         error
+	Result      []string
+}
+
+func NewFakeResolverThatFails() FakeResolver {
+	return FakeResolver{NumFailures: atomicx.NewInt64(), Err: ErrNotFound}
+}
+
+func NewFakeResolverWithResult(r []string) FakeResolver {
+	return FakeResolver{NumFailures: atomicx.NewInt64(), Result: r}
+}
+
+var ErrNotFound = &net.DNSError{
+	Err: "no such host",
+}
+
+func (c FakeResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
+	time.Sleep(10 * time.Microsecond)
+	if c.Err != nil {
+		if c.NumFailures != nil {
+			c.NumFailures.Add(1)
+		}
+		return nil, c.Err
+	}
+	return c.Result, nil
+}
+
+func (c FakeResolver) Network() string {
+	return "fake"
+}
+
+func (c FakeResolver) Address() string {
+	return ""
+}
+
+var _ netx.Resolver = FakeResolver{}
+
+type FakeTransport struct {
+	Err  error
+	Func func(*http.Request) (*http.Response, error)
+	Resp *http.Response
+}
+
+func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	time.Sleep(10 * time.Microsecond)
+	if txp.Func != nil {
+		return txp.Func(req)
+	}
+	if req.Body != nil {
+		ioutil.ReadAll(req.Body)
+		req.Body.Close()
+	}
+	if txp.Err != nil {
+		return nil, txp.Err
+	}
+	txp.Resp.Request = req // non thread safe but it doesn't matter
+	return txp.Resp, nil
+}
+
+func (txp FakeTransport) CloseIdleConnections() {}
+
+var _ netx.HTTPRoundTripper = FakeTransport{}
+
+type FakeBody struct {
+	Data []byte
+	Err  error
+}
+
+func (fb *FakeBody) Read(p []byte) (int, error) {
+	time.Sleep(10 * time.Microsecond)
+	if fb.Err != nil {
+		return 0, fb.Err
+	}
+	if len(fb.Data) <= 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, fb.Data)
+	fb.Data = fb.Data[n:]
+	return n, nil
+}
+
+func (fb *FakeBody) Close() error {
+	return nil
+}
+
+var _ io.ReadCloser = &FakeBody{}

internal/cmd/oohelper/oohelper.go

@@ -0,0 +1,48 @@
+// Command oohelper contains a simple command line
+// client for the Web Connectivity test helper.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+var (
+	ctx, cancel = context.WithCancel(context.Background())
+	debug       = flag.Bool("debug", false, "Toggle debug mode")
+	httpClient  *http.Client
+	resolver    netx.Resolver
+	server      = flag.String("server", "https://wcth.ooni.io/", "URL of the test helper")
+	target      = flag.String("target", "", "Target URL for the test helper")
+)
+
+func init() {
+	txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
+	httpClient = &http.Client{Transport: txp}
+	resolver = netx.NewResolver(netx.Config{Logger: log.Log})
+}
+
+func main() {
+	logmap := map[bool]log.Level{
+		true:  log.DebugLevel,
+		false: log.InfoLevel,
+	}
+	flag.Parse()
+	log.SetLevel(logmap[*debug])
+	clnt := internal.OOClient{HTTPClient: httpClient, Resolver: resolver}
+	config := internal.OOConfig{TargetURL: *target, ServerURL: *server}
+	defer cancel()
+	cresp, err := clnt.Do(ctx, config)
+	runtimex.PanicOnError(err, "client.Do failed")
+	data, err := json.MarshalIndent(cresp, "", "    ")
+	runtimex.PanicOnError(err, "json.MarshalIndent failed")
+	fmt.Printf("%s\n", string(data))
+}

internal/cmd/oohelper/oohelper_test.go

@@ -0,0 +1,8 @@
+package main
+
+import "testing"
+
+func TestSmoke(t *testing.T) {
+	*target = "http://www.example.com"
+	main()
+}

internal/cmd/oohelperd/README.md

@@ -0,0 +1,4 @@
+# oohelperd
+
+This directory contains the source code of the Web
+Connectivity test helper written in Go.

internal/cmd/oohelperd/internal/dns.go

@@ -0,0 +1,32 @@
+package internal
+
+import (
+	"context"
+	"sync"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+)
+
+// newfailure is a convenience shortcut to save typing
+var newfailure = archival.NewFailure
+
+// CtrlDNSResult is the result of the DNS check performed by
+// the Web Connectivity test helper.
+type CtrlDNSResult = webconnectivity.ControlDNSResult
+
+// DNSConfig configures the DNS check.
+type DNSConfig struct {
+	Domain   string
+	Out      chan CtrlDNSResult
+	Resolver netx.Resolver
+	Wg       *sync.WaitGroup
+}
+
+// DNSDo performs the DNS check.
+func DNSDo(ctx context.Context, config *DNSConfig) {
+	defer config.Wg.Done()
+	addrs, err := config.Resolver.LookupHost(ctx, config.Domain)
+	config.Out <- CtrlDNSResult{Failure: newfailure(err), Addrs: addrs}
+}

internal/cmd/oohelperd/internal/fake_test.go

@@ -0,0 +1,127 @@
+package internal
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+type FakeResolver struct {
+	NumFailures *atomicx.Int64
+	Err         error
+	Result      []string
+}
+
+func NewFakeResolverThatFails() FakeResolver {
+	return FakeResolver{NumFailures: atomicx.NewInt64(), Err: ErrNotFound}
+}
+
+func NewFakeResolverWithResult(r []string) FakeResolver {
+	return FakeResolver{NumFailures: atomicx.NewInt64(), Result: r}
+}
+
+var ErrNotFound = &net.DNSError{
+	Err: "no such host",
+}
+
+func (c FakeResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
+	time.Sleep(10 * time.Microsecond)
+	if c.Err != nil {
+		if c.NumFailures != nil {
+			c.NumFailures.Add(1)
+		}
+		return nil, c.Err
+	}
+	return c.Result, nil
+}
+
+func (c FakeResolver) Network() string {
+	return "fake"
+}
+
+func (c FakeResolver) Address() string {
+	return ""
+}
+
+var _ netx.Resolver = FakeResolver{}
+
+type FakeTransport struct {
+	Err  error
+	Func func(*http.Request) (*http.Response, error)
+	Resp *http.Response
+}
+
+func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	time.Sleep(10 * time.Microsecond)
+	if txp.Func != nil {
+		return txp.Func(req)
+	}
+	if req.Body != nil {
+		ioutil.ReadAll(req.Body)
+		req.Body.Close()
+	}
+	if txp.Err != nil {
+		return nil, txp.Err
+	}
+	txp.Resp.Request = req // non thread safe but it doesn't matter
+	return txp.Resp, nil
+}
+
+func (txp FakeTransport) CloseIdleConnections() {}
+
+var _ netx.HTTPRoundTripper = FakeTransport{}
+
+type FakeBody struct {
+	Data []byte
+	Err  error
+}
+
+func (fb *FakeBody) Read(p []byte) (int, error) {
+	time.Sleep(10 * time.Microsecond)
+	if fb.Err != nil {
+		return 0, fb.Err
+	}
+	if len(fb.Data) <= 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, fb.Data)
+	fb.Data = fb.Data[n:]
+	return n, nil
+}
+
+func (fb *FakeBody) Close() error {
+	return nil
+}
+
+var _ io.ReadCloser = &FakeBody{}
+
+type FakeResponseWriter struct {
+	Body       [][]byte
+	HeaderMap  http.Header
+	StatusCode int
+}
+
+func NewFakeResponseWriter() *FakeResponseWriter {
+	return &FakeResponseWriter{HeaderMap: make(http.Header)}
+}
+
+func (frw *FakeResponseWriter) Header() http.Header {
+	return frw.HeaderMap
+}
+
+func (frw *FakeResponseWriter) Write(b []byte) (int, error) {
+	frw.Body = append(frw.Body, b)
+	return len(b), nil
+}
+
+func (frw *FakeResponseWriter) WriteHeader(statusCode int) {
+	frw.StatusCode = statusCode
+}
+
+var _ http.ResponseWriter = &FakeResponseWriter{}

internal/cmd/oohelperd/internal/http.go

@@ -0,0 +1,67 @@
+package internal
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+)
+
+// CtrlHTTPResponse is the result of the HTTP check performed by
+// the Web Connectivity test helper.
+type CtrlHTTPResponse = webconnectivity.ControlHTTPRequestResult
+
+// HTTPConfig configures the HTTP check.
+type HTTPConfig struct {
+	Client            *http.Client
+	Headers           map[string][]string
+	MaxAcceptableBody int64
+	Out               chan CtrlHTTPResponse
+	URL               string
+	Wg                *sync.WaitGroup
+}
+
+// HTTPDo performs the HTTP check.
+func HTTPDo(ctx context.Context, config *HTTPConfig) {
+	defer config.Wg.Done()
+	req, err := http.NewRequestWithContext(ctx, "GET", config.URL, nil)
+	if err != nil {
+		config.Out <- CtrlHTTPResponse{Failure: newfailure(err)}
+		return
+	}
+	// The original test helper failed with extra headers while here
+	// we're implementing (for now?) a more liberal approach.
+	for k, vs := range config.Headers {
+		switch strings.ToLower(k) {
+		case "user-agent":
+		case "accept":
+		case "accept-language":
+			for _, v := range vs {
+				req.Header.Add(k, v)
+			}
+		}
+	}
+	resp, err := config.Client.Do(req)
+	if err != nil {
+		config.Out <- CtrlHTTPResponse{Failure: newfailure(err)}
+		return
+	}
+	defer resp.Body.Close()
+	headers := make(map[string]string)
+	for k := range resp.Header {
+		headers[k] = resp.Header.Get(k)
+	}
+	reader := &io.LimitedReader{R: resp.Body, N: config.MaxAcceptableBody}
+	data, err := ioutil.ReadAll(reader)
+	config.Out <- CtrlHTTPResponse{
+		BodyLength: int64(len(data)),
+		Failure:    newfailure(err),
+		StatusCode: int64(resp.StatusCode),
+		Headers:    headers,
+		Title:      webconnectivity.GetTitle(string(data)),
+	}
+}

internal/cmd/oohelperd/internal/http_test.go

@@ -0,0 +1,59 @@
+package internal_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
+)
+
+func TestHTTPDoWithInvalidURL(t *testing.T) {
+	ctx := context.Background()
+	wg := new(sync.WaitGroup)
+	httpch := make(chan internal.CtrlHTTPResponse, 1)
+	wg.Add(1)
+	go internal.HTTPDo(ctx, &internal.HTTPConfig{
+		Client:            http.DefaultClient,
+		Headers:           nil,
+		MaxAcceptableBody: 1 << 24,
+		Out:               httpch,
+		URL:               "http://[::1]aaaa",
+		Wg:                wg,
+	})
+	// wait for measurement steps to complete
+	wg.Wait()
+	resp := <-httpch
+	if resp.Failure == nil || !strings.HasSuffix(*resp.Failure, `invalid port "aaaa" after host`) {
+		t.Fatal("not the failure we expected")
+	}
+}
+
+func TestHTTPDoWithHTTPTransportFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	ctx := context.Background()
+	wg := new(sync.WaitGroup)
+	httpch := make(chan internal.CtrlHTTPResponse, 1)
+	wg.Add(1)
+	go internal.HTTPDo(ctx, &internal.HTTPConfig{
+		Client: &http.Client{
+			Transport: internal.FakeTransport{
+				Err: expected,
+			},
+		},
+		Headers:           nil,
+		MaxAcceptableBody: 1 << 24,
+		Out:               httpch,
+		URL:               "http://www.x.org",
+		Wg:                wg,
+	})
+	// wait for measurement steps to complete
+	wg.Wait()
+	resp := <-httpch
+	if resp.Failure == nil || !strings.HasSuffix(*resp.Failure, "mocked error") {
+		t.Fatal("not the error we expected")
+	}
+}

internal/cmd/oohelperd/internal/internal.go

@@ -0,0 +1,61 @@
+package internal
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/version"
+)
+
+// Handler implements the Web Connectivity test helper HTTP API.
+type Handler struct {
+	Client            *http.Client
+	Dialer            netx.Dialer
+	MaxAcceptableBody int64
+	Resolver          netx.Resolver
+}
+
+func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.Header().Add("Server", fmt.Sprintf(
+		"oohelperd/%s ooniprobe-engine/%s", version.Version, version.Version,
+	))
+	if req.Method != "POST" {
+		w.WriteHeader(400)
+		return
+	}
+	if req.Header.Get("content-type") != "application/json" {
+		w.WriteHeader(400)
+		return
+	}
+	reader := &io.LimitedReader{R: req.Body, N: h.MaxAcceptableBody}
+	data, err := ioutil.ReadAll(reader)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	var creq CtrlRequest
+	if err := json.Unmarshal(data, &creq); err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	measureConfig := MeasureConfig{
+		Client:            h.Client,
+		Dialer:            h.Dialer,
+		MaxAcceptableBody: h.MaxAcceptableBody,
+		Resolver:          h.Resolver,
+	}
+	cresp, err := Measure(req.Context(), measureConfig, &creq)
+	if err != nil {
+		w.WriteHeader(400)
+		return
+	}
+	// We assume that the following call cannot fail because it's a
+	// clearly serializable data structure.
+	data, _ = json.Marshal(cresp)
+	w.Header().Add("Content-Type", "application/json")
+	w.Write(data)
+}

internal/cmd/oohelperd/internal/internal_test.go

@@ -0,0 +1,160 @@
+package internal_test
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
+)
+
+const simplerequest = `{
+	"http_request": "https://dns.google",
+	"http_request_headers": {
+	  "Accept": [
+		"*/*"
+	  ],
+	  "Accept-Language": [
+		"en-US;q=0.8,en;q=0.5"
+	  ],
+	  "User-Agent": [
+		"Mozilla/5.0"
+	  ]
+	},
+	"tcp_connect": [
+	  "8.8.8.8:443"
+	]
+}`
+
+const requestWithoutDomainName = `{
+	"http_request": "https://8.8.8.8",
+	"http_request_headers": {
+	  "Accept": [
+		"*/*"
+	  ],
+	  "Accept-Language": [
+		"en-US;q=0.8,en;q=0.5"
+	  ],
+	  "User-Agent": [
+		"Mozilla/5.0"
+	  ]
+	},
+	"tcp_connect": [
+	  "8.8.8.8:443"
+	]
+}`
+
+func TestWorkingAsIntended(t *testing.T) {
+	handler := internal.Handler{
+		Client:            http.DefaultClient,
+		Dialer:            new(net.Dialer),
+		MaxAcceptableBody: 1 << 24,
+		Resolver:          resolver.SystemResolver{},
+	}
+	srv := httptest.NewServer(handler)
+	defer srv.Close()
+	type expectationSpec struct {
+		name            string
+		reqMethod       string
+		reqContentType  string
+		reqBody         string
+		respStatusCode  int
+		respContentType string
+		parseBody       bool
+	}
+	expectations := []expectationSpec{{
+		name:           "check for invalid method",
+		reqMethod:      "GET",
+		respStatusCode: 400,
+	}, {
+		name:           "check for invalid content-type",
+		reqMethod:      "POST",
+		respStatusCode: 400,
+	}, {
+		name:           "check for invalid request body",
+		reqMethod:      "POST",
+		reqContentType: "application/json",
+		reqBody:        "{",
+		respStatusCode: 400,
+	}, {
+		name:           "with measurement failure",
+		reqMethod:      "POST",
+		reqContentType: "application/json",
+		reqBody:        `{"http_request": "http://[::1]aaaa"}`,
+		respStatusCode: 400,
+	}, {
+		name:            "with reasonably good request",
+		reqMethod:       "POST",
+		reqContentType:  "application/json",
+		reqBody:         simplerequest,
+		respStatusCode:  200,
+		respContentType: "application/json",
+		parseBody:       true,
+	}, {
+		name:            "when there's no domain name in the request",
+		reqMethod:       "POST",
+		reqContentType:  "application/json",
+		reqBody:         requestWithoutDomainName,
+		respStatusCode:  200,
+		respContentType: "application/json",
+		parseBody:       true,
+	}}
+	for _, expect := range expectations {
+		t.Run(expect.name, func(t *testing.T) {
+			body := strings.NewReader(expect.reqBody)
+			req, err := http.NewRequest(expect.reqMethod, srv.URL, body)
+			if err != nil {
+				t.Fatalf("%s: %+v", expect.name, err)
+			}
+			if expect.reqContentType != "" {
+				req.Header.Add("content-type", expect.reqContentType)
+			}
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				t.Fatalf("%s: %+v", expect.name, err)
+			}
+			defer resp.Body.Close()
+			if resp.StatusCode != expect.respStatusCode {
+				t.Fatalf("unexpected status code: %+v", resp.StatusCode)
+			}
+			if v := resp.Header.Get("content-type"); v != expect.respContentType {
+				t.Fatalf("unexpected content-type: %s", v)
+			}
+			data, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !expect.parseBody {
+				return
+			}
+			var v interface{}
+			if err := json.Unmarshal(data, &v); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func TestHandlerWithRequestBodyReadingError(t *testing.T) {
+	expected := errors.New("mocked error")
+	handler := internal.Handler{MaxAcceptableBody: 1 << 24}
+	rw := internal.NewFakeResponseWriter()
+	req := &http.Request{
+		Method: "POST",
+		Header: map[string][]string{
+			"Content-Type":   {"application/json"},
+			"Content-Length": {"2048"},
+		},
+		Body: &internal.FakeBody{Err: expected},
+	}
+	handler.ServeHTTP(rw, req)
+	if rw.StatusCode != 400 {
+		t.Fatal("unexpected status code")
+	}
+}

internal/cmd/oohelperd/internal/measure.go

@@ -0,0 +1,88 @@
+package internal
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/url"
+	"sync"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+type (
+	// CtrlRequest is the request sent to the test helper
+	CtrlRequest = webconnectivity.ControlRequest
+
+	// CtrlResponse is the response from the test helper
+	CtrlResponse = webconnectivity.ControlResponse
+)
+
+// MeasureConfig contains configuration for Measure.
+type MeasureConfig struct {
+	Client            *http.Client
+	Dialer            netx.Dialer
+	MaxAcceptableBody int64
+	Resolver          netx.Resolver
+}
+
+// Measure performs the measurement described by the request and
+// returns the corresponding response or an error.
+func Measure(ctx context.Context, config MeasureConfig, creq *CtrlRequest) (*CtrlResponse, error) {
+	// parse input for correctness
+	URL, err := url.Parse(creq.HTTPRequest)
+	if err != nil {
+		return nil, err
+	}
+	// dns: start
+	wg := new(sync.WaitGroup)
+	dnsch := make(chan CtrlDNSResult, 1)
+	if net.ParseIP(URL.Hostname()) == nil {
+		wg.Add(1)
+		go DNSDo(ctx, &DNSConfig{
+			Domain:   URL.Hostname(),
+			Out:      dnsch,
+			Resolver: config.Resolver,
+			Wg:       wg,
+		})
+	}
+	// tcpconnect: start
+	tcpconnch := make(chan TCPResultPair, len(creq.TCPConnect))
+	for _, endpoint := range creq.TCPConnect {
+		wg.Add(1)
+		go TCPDo(ctx, &TCPConfig{
+			Dialer:   config.Dialer,
+			Endpoint: endpoint,
+			Out:      tcpconnch,
+			Wg:       wg,
+		})
+	}
+	// http: start
+	httpch := make(chan CtrlHTTPResponse, 1)
+	wg.Add(1)
+	go HTTPDo(ctx, &HTTPConfig{
+		Client:            config.Client,
+		Headers:           creq.HTTPRequestHeaders,
+		MaxAcceptableBody: config.MaxAcceptableBody,
+		Out:               httpch,
+		URL:               creq.HTTPRequest,
+		Wg:                wg,
+	})
+	// wait for measurement steps to complete
+	wg.Wait()
+	// assemble response
+	cresp := new(CtrlResponse)
+	select {
+	case cresp.DNS = <-dnsch:
+	default:
+		// we land here when there's no domain name
+	}
+	cresp.HTTPRequest = <-httpch
+	cresp.TCPConnect = make(map[string]CtrlTCPResult)
+	for len(cresp.TCPConnect) < len(creq.TCPConnect) {
+		tcpconn := <-tcpconnch
+		cresp.TCPConnect[tcpconn.Endpoint] = tcpconn.Result
+	}
+	return cresp, nil
+}

internal/cmd/oohelperd/internal/tcpconnect.go

@@ -0,0 +1,42 @@
+package internal
+
+import (
+	"context"
+	"sync"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+// CtrlTCPResult is the result of the TCP check performed by the test helper.
+type CtrlTCPResult = webconnectivity.ControlTCPConnectResult
+
+// TCPResultPair contains the endpoint and the corresponding result.
+type TCPResultPair struct {
+	Endpoint string
+	Result   CtrlTCPResult
+}
+
+// TCPConfig configures the TCP connect check.
+type TCPConfig struct {
+	Dialer   netx.Dialer
+	Endpoint string
+	Out      chan TCPResultPair
+	Wg       *sync.WaitGroup
+}
+
+// TCPDo performs the TCP check.
+func TCPDo(ctx context.Context, config *TCPConfig) {
+	defer config.Wg.Done()
+	conn, err := config.Dialer.DialContext(ctx, "tcp", config.Endpoint)
+	if conn != nil {
+		conn.Close()
+	}
+	config.Out <- TCPResultPair{
+		Endpoint: config.Endpoint,
+		Result: CtrlTCPResult{
+			Failure: newfailure(err),
+			Status:  err == nil,
+		},
+	}
+}

internal/cmd/oohelperd/oohelperd.go

@@ -0,0 +1,67 @@
+// Command oohelperd contains the Web Connectivity test helper.
+package main
+
+import (
+	"context"
+	"flag"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+const maxAcceptableBody = 1 << 24
+
+var (
+	dialer    netx.Dialer
+	endpoint  = flag.String("endpoint", ":8080", "Endpoint where to listen")
+	httpx     *http.Client
+	resolver  netx.Resolver
+	srvcancel context.CancelFunc
+	srvctx    context.Context
+	srvwg     = new(sync.WaitGroup)
+)
+
+func init() {
+	srvctx, srvcancel = context.WithCancel(context.Background())
+	dialer = netx.NewDialer(netx.Config{Logger: log.Log})
+	txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
+	httpx = &http.Client{Transport: txp}
+	resolver = netx.NewResolver(netx.Config{Logger: log.Log})
+}
+
+func shutdown(srv *http.Server) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	srv.Shutdown(ctx)
+}
+
+func main() {
+	logmap := map[bool]log.Level{
+		true:  log.DebugLevel,
+		false: log.InfoLevel,
+	}
+	debug := flag.Bool("debug", false, "Toggle debug mode")
+	flag.Parse()
+	log.SetLevel(logmap[*debug])
+	testableMain()
+}
+
+func testableMain() {
+	mux := http.NewServeMux()
+	mux.Handle("/", internal.Handler{
+		Client:            httpx,
+		Dialer:            dialer,
+		MaxAcceptableBody: maxAcceptableBody,
+		Resolver:          resolver,
+	})
+	srv := &http.Server{Addr: *endpoint, Handler: mux}
+	srvwg.Add(1)
+	go srv.ListenAndServe()
+	<-srvctx.Done()
+	shutdown(srv)
+	srvwg.Done()
+}

internal/cmd/oohelperd/oohelperd_test.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestSmoke(t *testing.T) {
+	// Just check whether we can start and then tear down the server, so
+	// we have coverage of this code and when we see that some lines aren't
+	// covered we know these are genuine places where we're not testing
+	// the code rather than just places like this simple main.
+	go testableMain()
+	srvcancel()  // kills the listener
+	srvwg.Wait() // joined
+}