chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/netx/httptransport/bytecounter.go

@@ -0,0 +1,73 @@
+package httptransport
+
+import (
+	"io"
+	"net/http"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
+)
+
+// ByteCountingTransport is a RoundTripper that counts bytes.
+type ByteCountingTransport struct {
+	RoundTripper
+	Counter *bytecounter.Counter
+}
+
+// RoundTrip implements RoundTripper.RoundTrip
+func (txp ByteCountingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.Body != nil {
+		req.Body = byteCountingBody{
+			ReadCloser: req.Body, Account: txp.Counter.CountBytesSent}
+	}
+	txp.estimateRequestMetadata(req)
+	resp, err := txp.RoundTripper.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+	txp.estimateResponseMetadata(resp)
+	resp.Body = byteCountingBody{
+		ReadCloser: resp.Body, Account: txp.Counter.CountBytesReceived}
+	return resp, nil
+}
+
+func (txp ByteCountingTransport) estimateRequestMetadata(req *http.Request) {
+	txp.Counter.CountBytesSent(len(req.Method))
+	txp.Counter.CountBytesSent(len(req.URL.String()))
+	for key, values := range req.Header {
+		for _, value := range values {
+			txp.Counter.CountBytesSent(len(key))
+			txp.Counter.CountBytesSent(len(": "))
+			txp.Counter.CountBytesSent(len(value))
+			txp.Counter.CountBytesSent(len("\r\n"))
+		}
+	}
+	txp.Counter.CountBytesSent(len("\r\n"))
+}
+
+func (txp ByteCountingTransport) estimateResponseMetadata(resp *http.Response) {
+	txp.Counter.CountBytesReceived(len(resp.Status))
+	for key, values := range resp.Header {
+		for _, value := range values {
+			txp.Counter.CountBytesReceived(len(key))
+			txp.Counter.CountBytesReceived(len(": "))
+			txp.Counter.CountBytesReceived(len(value))
+			txp.Counter.CountBytesReceived(len("\r\n"))
+		}
+	}
+	txp.Counter.CountBytesReceived(len("\r\n"))
+}
+
+type byteCountingBody struct {
+	io.ReadCloser
+	Account func(int)
+}
+
+func (r byteCountingBody) Read(p []byte) (int, error) {
+	count, err := r.ReadCloser.Read(p)
+	if count > 0 {
+		r.Account(count)
+	}
+	return count, err
+}
+
+var _ RoundTripper = ByteCountingTransport{}

internal/engine/netx/httptransport/bytecounter_test.go

@@ -0,0 +1,128 @@
+package httptransport_test
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
+)
+
+func TestByteCounterFailure(t *testing.T) {
+	counter := bytecounter.New()
+	txp := httptransport.ByteCountingTransport{
+		Counter: counter,
+		RoundTripper: httptransport.FakeTransport{
+			Err: io.EOF,
+		},
+	}
+	client := &http.Client{Transport: txp}
+	req, err := http.NewRequest(
+		"POST", "https://www.google.com", strings.NewReader("AAAAAA"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("User-Agent", "antani-browser/1.0.0")
+	resp, err := client.Do(req)
+	if !errors.Is(err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+	if counter.Sent.Load() != 68 {
+		t.Fatal("expected around 68 bytes sent")
+	}
+	if counter.Received.Load() != 0 {
+		t.Fatal("expected zero bytes received")
+	}
+}
+
+func TestByteCounterSuccess(t *testing.T) {
+	counter := bytecounter.New()
+	txp := httptransport.ByteCountingTransport{
+		Counter: counter,
+		RoundTripper: httptransport.FakeTransport{
+			Resp: &http.Response{
+				Body: ioutil.NopCloser(strings.NewReader("1234567")),
+				Header: http.Header{
+					"Server": []string{"antani/0.1.0"},
+				},
+				Status:     "200 OK",
+				StatusCode: http.StatusOK,
+			},
+		},
+	}
+	client := &http.Client{Transport: txp}
+	req, err := http.NewRequest(
+		"POST", "https://www.google.com", strings.NewReader("AAAAAA"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("User-Agent", "antani-browser/1.0.0")
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp.Body.Close()
+	if string(data) != "1234567" {
+		t.Fatal("expected a different body here")
+	}
+	if counter.Sent.Load() != 68 {
+		t.Fatal("expected around 68 bytes sent")
+	}
+	if counter.Received.Load() != 37 {
+		t.Fatal("expected zero around 37 bytes received")
+	}
+}
+
+func TestByteCounterSuccessWithEOF(t *testing.T) {
+	counter := bytecounter.New()
+	txp := httptransport.ByteCountingTransport{
+		Counter: counter,
+		RoundTripper: httptransport.FakeTransport{
+			Resp: &http.Response{
+				Body: bodyReaderWithEOF{},
+				Header: http.Header{
+					"Server": []string{"antani/0.1.0"},
+				},
+				Status:     "200 OK",
+				StatusCode: http.StatusOK,
+			},
+		},
+	}
+	client := &http.Client{Transport: txp}
+	resp, err := client.Get("https://www.google.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp.Body.Close()
+	if string(data) != "A" {
+		t.Fatal("expected a different body here")
+	}
+}
+
+type bodyReaderWithEOF struct{}
+
+func (bodyReaderWithEOF) Read(p []byte) (int, error) {
+	if len(p) < 1 {
+		panic("should not happen")
+	}
+	p[0] = 'A'
+	return 1, io.EOF // we want code to be robust to this
+}
+func (bodyReaderWithEOF) Close() error {
+	return nil
+}

internal/engine/netx/httptransport/fake_test.go

@@ -0,0 +1,56 @@
+package httptransport
+
+import (
+	"context"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"time"
+)
+
+type FakeDialer struct {
+	Conn net.Conn
+	Err  error
+}
+
+func (d FakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	time.Sleep(10 * time.Microsecond)
+	return d.Conn, d.Err
+}
+
+type FakeTransport struct {
+	Err  error
+	Func func(*http.Request) (*http.Response, error)
+	Resp *http.Response
+}
+
+func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	time.Sleep(10 * time.Microsecond)
+	if txp.Func != nil {
+		return txp.Func(req)
+	}
+	if req.Body != nil {
+		ioutil.ReadAll(req.Body)
+		req.Body.Close()
+	}
+	if txp.Err != nil {
+		return nil, txp.Err
+	}
+	txp.Resp.Request = req // non thread safe but it doesn't matter
+	return txp.Resp, nil
+}
+
+func (txp FakeTransport) CloseIdleConnections() {}
+
+type FakeBody struct {
+	Err error
+}
+
+func (fb FakeBody) Read(p []byte) (int, error) {
+	time.Sleep(10 * time.Microsecond)
+	return 0, fb.Err
+}
+
+func (fb FakeBody) Close() error {
+	return nil
+}

internal/engine/netx/httptransport/http3transport.go

@@ -0,0 +1,43 @@
+package httptransport
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+
+	"github.com/lucas-clemente/quic-go"
+	"github.com/lucas-clemente/quic-go/http3"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
+)
+
+// QUICWrapperDialer is a QUICDialer that wraps a ContextDialer
+// This is necessary because the http3 RoundTripper does not support a DialContext method.
+type QUICWrapperDialer struct {
+	Dialer quicdialer.ContextDialer
+}
+
+// Dial implements QUICDialer.Dial
+func (d QUICWrapperDialer) Dial(network, host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
+	return d.Dialer.DialContext(context.Background(), network, host, tlsCfg, cfg)
+}
+
+// HTTP3Transport is a httptransport.RoundTripper using the http3 protocol.
+type HTTP3Transport struct {
+	http3.RoundTripper
+}
+
+// CloseIdleConnections closes all the connections opened by this transport.
+func (t *HTTP3Transport) CloseIdleConnections() {
+	t.RoundTripper.Close()
+}
+
+// NewHTTP3Transport creates a new HTTP3Transport instance.
+func NewHTTP3Transport(config Config) RoundTripper {
+	txp := &HTTP3Transport{}
+	txp.QuicConfig = &quic.Config{}
+	txp.TLSClientConfig = config.TLSConfig
+	txp.Dial = config.QUICDialer.Dial
+	return txp
+}
+
+var _ RoundTripper = &http.Transport{}

internal/engine/netx/httptransport/http3transport_test.go

@@ -0,0 +1,157 @@
+package httptransport_test
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/lucas-clemente/quic-go"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
+)
+
+type MockQUICDialer struct{}
+
+func (d MockQUICDialer) Dial(network, host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
+	return quic.DialAddrEarly(host, tlsCfg, cfg)
+}
+
+type MockSNIQUICDialer struct {
+	namech chan string
+}
+
+func (d MockSNIQUICDialer) Dial(network, host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
+	d.namech <- tlsCfg.ServerName
+	return quic.DialAddrEarly(host, tlsCfg, cfg)
+}
+
+type MockCertQUICDialer struct {
+	certch chan *x509.CertPool
+}
+
+func (d MockCertQUICDialer) Dial(network, host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
+	d.certch <- tlsCfg.RootCAs
+	return quic.DialAddrEarly(host, tlsCfg, cfg)
+}
+
+func TestHTTP3TransportSNI(t *testing.T) {
+	namech := make(chan string, 1)
+	sni := "sni.org"
+	txp := httptransport.NewHTTP3Transport(httptransport.Config{
+		Dialer: selfcensor.SystemDialer{}, QUICDialer: MockSNIQUICDialer{namech: namech}, TLSConfig: &tls.Config{ServerName: sni}})
+	req, err := http.NewRequest("GET", "https://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if err == nil {
+		t.Fatal("expected error here")
+	}
+	if resp != nil {
+		t.Fatal("expected nil resp here")
+	}
+	if !strings.Contains(err.Error(), "certificate is valid for www.google.com, not "+sni) {
+		t.Fatal("unexpected error type", err)
+	}
+	servername := <-namech
+	if servername != sni {
+		t.Fatal("unexpected server name", servername)
+	}
+}
+
+func TestHTTP3TransportSNINoVerify(t *testing.T) {
+	namech := make(chan string, 1)
+	sni := "sni.org"
+	txp := httptransport.NewHTTP3Transport(httptransport.Config{
+		Dialer: selfcensor.SystemDialer{}, QUICDialer: MockSNIQUICDialer{namech: namech}, TLSConfig: &tls.Config{ServerName: sni, InsecureSkipVerify: true}})
+	req, err := http.NewRequest("GET", "https://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if err != nil {
+		t.Fatalf("unexpected error: %+v", err)
+	}
+	if resp == nil {
+		t.Fatal("unexpected nil resp")
+	}
+	servername := <-namech
+	if servername != sni {
+		t.Fatal("unexpected server name", servername)
+	}
+}
+
+func TestHTTP3TransportCABundle(t *testing.T) {
+	certch := make(chan *x509.CertPool, 1)
+	certpool := x509.NewCertPool()
+	txp := httptransport.NewHTTP3Transport(httptransport.Config{
+		Dialer: selfcensor.SystemDialer{}, QUICDialer: MockCertQUICDialer{certch: certch}, TLSConfig: &tls.Config{RootCAs: certpool}})
+	req, err := http.NewRequest("GET", "https://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if err == nil {
+		t.Fatal("expected error here")
+	}
+	if resp != nil {
+		t.Fatal("expected nil resp here")
+	}
+	// since the certificate pool is empty, the unknown authority error should be thrown
+	if !strings.Contains(err.Error(), "certificate signed by unknown authority") {
+		t.Fatal("unexpected error type")
+	}
+	certs := <-certch
+	if certs != certpool {
+		t.Fatal("not the certpool we expected")
+	}
+
+}
+
+func TestUnitHTTP3TransportSuccess(t *testing.T) {
+	txp := httptransport.NewHTTP3Transport(httptransport.Config{
+		Dialer: selfcensor.SystemDialer{}, QUICDialer: MockQUICDialer{}})
+
+	req, err := http.NewRequest("GET", "https://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("unexpected nil response here")
+	}
+	if resp.StatusCode != 200 {
+		t.Fatal("HTTP statuscode should be 200 OK", resp.StatusCode)
+	}
+}
+
+func TestUnitHTTP3TransportFailure(t *testing.T) {
+	txp := httptransport.NewHTTP3Transport(httptransport.Config{
+		Dialer: selfcensor.SystemDialer{}, QUICDialer: MockQUICDialer{}})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // so that the request immediately fails
+	req, err := http.NewRequestWithContext(ctx, "GET", "https://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if err == nil {
+		t.Fatal("expected error here")
+	}
+	// context.Canceled error occurs if the test host supports QUIC
+	// timeout error ("Handshake did not complete in time") occurs if the test host does not support QUIC
+	if !(errors.Is(err, context.Canceled) || strings.HasSuffix(err.Error(), "Handshake did not complete in time")) {
+		t.Fatal("not the error we expected", err)
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+}

internal/engine/netx/httptransport/httptransport.go

@@ -0,0 +1,47 @@
+// Package httptransport contains HTTP transport extensions.
+package httptransport
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"net/http"
+
+	"github.com/lucas-clemente/quic-go"
+)
+
+// Config contains the configuration required for constructing an HTTP transport
+type Config struct {
+	Dialer     Dialer
+	QUICDialer QUICDialer
+	TLSDialer  TLSDialer
+	TLSConfig  *tls.Config
+}
+
+// Dialer is the definition of dialer assumed by this package.
+type Dialer interface {
+	DialContext(ctx context.Context, network, address string) (net.Conn, error)
+}
+
+// TLSDialer is the definition of a TLS dialer assumed by this package.
+type TLSDialer interface {
+	DialTLSContext(ctx context.Context, network, address string) (net.Conn, error)
+}
+
+// QUICDialer is the definition of dialer for QUIC assumed by this package.
+type QUICDialer interface {
+	Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error)
+}
+
+// RoundTripper is the definition of http.RoundTripper used by this package.
+type RoundTripper interface {
+	RoundTrip(req *http.Request) (*http.Response, error)
+	CloseIdleConnections()
+}
+
+// Resolver is the interface we expect from a resolver
+type Resolver interface {
+	LookupHost(ctx context.Context, hostname string) (addrs []string, err error)
+	Network() string
+	Address() string
+}

internal/engine/netx/httptransport/logging.go

@@ -0,0 +1,50 @@
+package httptransport
+
+import "net/http"
+
+// Logger is the logger assumed by this package
+type Logger interface {
+	Debugf(format string, v ...interface{})
+	Debug(message string)
+}
+
+// LoggingTransport is a logging transport
+type LoggingTransport struct {
+	RoundTripper
+	Logger Logger
+}
+
+// RoundTrip implements RoundTripper.RoundTrip
+func (txp LoggingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	host := req.Host
+	if host == "" {
+		host = req.URL.Host
+	}
+	req.Header.Set("Host", host) // anticipate what Go would do
+	return txp.logTrip(req)
+}
+
+func (txp LoggingTransport) logTrip(req *http.Request) (*http.Response, error) {
+	txp.Logger.Debugf("> %s %s", req.Method, req.URL.String())
+	for key, values := range req.Header {
+		for _, value := range values {
+			txp.Logger.Debugf("> %s: %s", key, value)
+		}
+	}
+	txp.Logger.Debug(">")
+	resp, err := txp.RoundTripper.RoundTrip(req)
+	if err != nil {
+		txp.Logger.Debugf("< %s", err)
+		return nil, err
+	}
+	txp.Logger.Debugf("< %d", resp.StatusCode)
+	for key, values := range resp.Header {
+		for _, value := range values {
+			txp.Logger.Debugf("< %s: %s", key, value)
+		}
+	}
+	txp.Logger.Debug("<")
+	return resp, nil
+}
+
+var _ RoundTripper = LoggingTransport{}

internal/engine/netx/httptransport/logging_test.go

@@ -0,0 +1,77 @@
+package httptransport_test
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
+)
+
+func TestLoggingFailure(t *testing.T) {
+	txp := httptransport.LoggingTransport{
+		Logger: log.Log,
+		RoundTripper: httptransport.FakeTransport{
+			Err: io.EOF,
+		},
+	}
+	client := &http.Client{Transport: txp}
+	resp, err := client.Get("https://www.google.com")
+	if !errors.Is(err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+}
+
+func TestLoggingFailureWithNoHostHeader(t *testing.T) {
+	txp := httptransport.LoggingTransport{
+		Logger: log.Log,
+		RoundTripper: httptransport.FakeTransport{
+			Err: io.EOF,
+		},
+	}
+	req := &http.Request{
+		Header: http.Header{},
+		URL: &url.URL{
+			Scheme: "https",
+			Host:   "www.google.com",
+			Path:   "/",
+		},
+	}
+	resp, err := txp.RoundTrip(req)
+	if !errors.Is(err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+}
+
+func TestLoggingSuccess(t *testing.T) {
+	txp := httptransport.LoggingTransport{
+		Logger: log.Log,
+		RoundTripper: httptransport.FakeTransport{
+			Resp: &http.Response{
+				Body: ioutil.NopCloser(strings.NewReader("")),
+				Header: http.Header{
+					"Server": []string{"antani/0.1.0"},
+				},
+				StatusCode: 200,
+			},
+		},
+	}
+	client := &http.Client{Transport: txp}
+	resp, err := client.Get("https://www.google.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+}

internal/engine/netx/httptransport/saver.go

@@ -0,0 +1,158 @@
+package httptransport
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptrace"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+// SaverPerformanceHTTPTransport is a RoundTripper that saves
+// performance events occurring during the round trip
+type SaverPerformanceHTTPTransport struct {
+	RoundTripper
+	Saver *trace.Saver
+}
+
+// RoundTrip implements RoundTripper.RoundTrip
+func (txp SaverPerformanceHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	tracep := httptrace.ContextClientTrace(req.Context())
+	if tracep == nil {
+		tracep = &httptrace.ClientTrace{
+			WroteHeaders: func() {
+				txp.Saver.Write(trace.Event{Name: "http_wrote_headers", Time: time.Now()})
+			},
+			WroteRequest: func(httptrace.WroteRequestInfo) {
+				txp.Saver.Write(trace.Event{Name: "http_wrote_request", Time: time.Now()})
+			},
+			GotFirstResponseByte: func() {
+				txp.Saver.Write(trace.Event{
+					Name: "http_first_response_byte", Time: time.Now()})
+			},
+		}
+		req = req.WithContext(httptrace.WithClientTrace(req.Context(), tracep))
+	}
+	return txp.RoundTripper.RoundTrip(req)
+}
+
+// SaverMetadataHTTPTransport is a RoundTripper that saves
+// events related to HTTP request and response metadata
+type SaverMetadataHTTPTransport struct {
+	RoundTripper
+	Saver     *trace.Saver
+	Transport string
+}
+
+// RoundTrip implements RoundTripper.RoundTrip
+func (txp SaverMetadataHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	txp.Saver.Write(trace.Event{
+		HTTPHeaders: req.Header,
+		HTTPMethod:  req.Method,
+		HTTPURL:     req.URL.String(),
+		Transport:   txp.Transport,
+		Name:        "http_request_metadata",
+		Time:        time.Now(),
+	})
+	resp, err := txp.RoundTripper.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+	txp.Saver.Write(trace.Event{
+		HTTPHeaders:    resp.Header,
+		HTTPStatusCode: resp.StatusCode,
+		Name:           "http_response_metadata",
+		Time:           time.Now(),
+	})
+	return resp, err
+}
+
+// SaverTransactionHTTPTransport is a RoundTripper that saves
+// events related to the HTTP transaction
+type SaverTransactionHTTPTransport struct {
+	RoundTripper
+	Saver *trace.Saver
+}
+
+// RoundTrip implements RoundTripper.RoundTrip
+func (txp SaverTransactionHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	txp.Saver.Write(trace.Event{
+		Name: "http_transaction_start",
+		Time: time.Now(),
+	})
+	resp, err := txp.RoundTripper.RoundTrip(req)
+	txp.Saver.Write(trace.Event{
+		Err:  err,
+		Name: "http_transaction_done",
+		Time: time.Now(),
+	})
+	return resp, err
+}
+
+// SaverBodyHTTPTransport is a RoundTripper that saves
+// body events occurring during the round trip
+type SaverBodyHTTPTransport struct {
+	RoundTripper
+	Saver        *trace.Saver
+	SnapshotSize int
+}
+
+// RoundTrip implements RoundTripper.RoundTrip
+func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	const defaultSnapSize = 1 << 17
+	snapsize := defaultSnapSize
+	if txp.SnapshotSize != 0 {
+		snapsize = txp.SnapshotSize
+	}
+	if req.Body != nil {
+		data, err := saverSnapRead(req.Body, snapsize)
+		if err != nil {
+			return nil, err
+		}
+		req.Body = saverCompose(data, req.Body)
+		txp.Saver.Write(trace.Event{
+			DataIsTruncated: len(data) >= snapsize,
+			Data:            data,
+			Name:            "http_request_body_snapshot",
+			Time:            time.Now(),
+		})
+	}
+	resp, err := txp.RoundTripper.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+	data, err := saverSnapRead(resp.Body, snapsize)
+	if err != nil {
+		resp.Body.Close()
+		return nil, err
+	}
+	resp.Body = saverCompose(data, resp.Body)
+	txp.Saver.Write(trace.Event{
+		DataIsTruncated: len(data) >= snapsize,
+		Data:            data,
+		Name:            "http_response_body_snapshot",
+		Time:            time.Now(),
+	})
+	return resp, nil
+}
+
+func saverSnapRead(r io.ReadCloser, snapsize int) ([]byte, error) {
+	return ioutil.ReadAll(io.LimitReader(r, int64(snapsize)))
+}
+
+func saverCompose(data []byte, r io.ReadCloser) io.ReadCloser {
+	return saverReadCloser{Closer: r, Reader: io.MultiReader(bytes.NewReader(data), r)}
+}
+
+type saverReadCloser struct {
+	io.Closer
+	io.Reader
+}
+
+var _ RoundTripper = SaverPerformanceHTTPTransport{}
+var _ RoundTripper = SaverMetadataHTTPTransport{}
+var _ RoundTripper = SaverBodyHTTPTransport{}
+var _ RoundTripper = SaverTransactionHTTPTransport{}

internal/engine/netx/httptransport/saver_test.go

@@ -0,0 +1,429 @@
+package httptransport_test
+
+import (
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
+)
+
+func TestSaverPerformanceNoMultipleEvents(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	saver := &trace.Saver{}
+	// register twice - do we see events twice?
+	txp := httptransport.SaverPerformanceHTTPTransport{
+		RoundTripper: http.DefaultTransport.(*http.Transport),
+		Saver:        saver,
+	}
+	txp = httptransport.SaverPerformanceHTTPTransport{
+		RoundTripper: txp,
+		Saver:        saver,
+	}
+	req, err := http.NewRequest("GET", "https://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if err != nil {
+		t.Fatal("not the error we expected")
+	}
+	if resp == nil {
+		t.Fatal("expected non nil response here")
+	}
+	ev := saver.Read()
+	// we should specifically see the events not attached to any
+	// context being submitted twice. This is fine because they are
+	// explicit, while the context is implicit and hence leads to
+	// more subtle bugs. For example, this happens when you measure
+	// every event and combine HTTP with DoH.
+	if len(ev) != 3 {
+		t.Fatal("expected three events")
+	}
+	expected := []string{
+		"http_wrote_headers",       // measured with context
+		"http_wrote_request",       // measured with context
+		"http_first_response_byte", // measured with context
+	}
+	for i := 0; i < len(expected); i++ {
+		if ev[i].Name != expected[i] {
+			t.Fatal("unexpected event name")
+		}
+	}
+}
+
+func TestSaverMetadataSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	saver := &trace.Saver{}
+	txp := httptransport.SaverMetadataHTTPTransport{
+		RoundTripper: http.DefaultTransport.(*http.Transport),
+		Saver:        saver,
+	}
+	req, err := http.NewRequest("GET", "https://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Add("User-Agent", "miniooni/0.1.0-dev")
+	resp, err := txp.RoundTrip(req)
+	if err != nil {
+		t.Fatal("not the error we expected")
+	}
+	if resp == nil {
+		t.Fatal("expected non nil response here")
+	}
+	ev := saver.Read()
+	if len(ev) != 2 {
+		t.Fatal("expected two events")
+	}
+	//
+	if ev[0].HTTPMethod != "GET" {
+		t.Fatal("unexpected Method")
+	}
+	if len(ev[0].HTTPHeaders) <= 0 {
+		t.Fatal("unexpected Headers")
+	}
+	if ev[0].HTTPURL != "https://www.google.com" {
+		t.Fatal("unexpected URL")
+	}
+	if ev[0].Name != "http_request_metadata" {
+		t.Fatal("unexpected Name")
+	}
+	if !ev[0].Time.Before(time.Now()) {
+		t.Fatal("unexpected Time")
+	}
+	//
+	if ev[1].HTTPStatusCode != 200 {
+		t.Fatal("unexpected StatusCode")
+	}
+	if len(ev[1].HTTPHeaders) <= 0 {
+		t.Fatal("unexpected Headers")
+	}
+	if ev[1].Name != "http_response_metadata" {
+		t.Fatal("unexpected Name")
+	}
+	if !ev[1].Time.After(ev[0].Time) {
+		t.Fatal("unexpected Time")
+	}
+}
+
+func TestSaverMetadataFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	saver := &trace.Saver{}
+	txp := httptransport.SaverMetadataHTTPTransport{
+		RoundTripper: httptransport.FakeTransport{
+			Err: expected,
+		},
+		Saver: saver,
+	}
+	req, err := http.NewRequest("GET", "http://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Add("User-Agent", "miniooni/0.1.0-dev")
+	resp, err := txp.RoundTrip(req)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+	ev := saver.Read()
+	if len(ev) != 1 {
+		t.Fatal("expected one event")
+	}
+	if ev[0].HTTPMethod != "GET" {
+		t.Fatal("unexpected Method")
+	}
+	if len(ev[0].HTTPHeaders) <= 0 {
+		t.Fatal("unexpected Headers")
+	}
+	if ev[0].HTTPURL != "http://www.google.com" {
+		t.Fatal("unexpected URL")
+	}
+	if ev[0].Name != "http_request_metadata" {
+		t.Fatal("unexpected Name")
+	}
+	if !ev[0].Time.Before(time.Now()) {
+		t.Fatal("unexpected Time")
+	}
+}
+
+func TestSaverTransactionSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	saver := &trace.Saver{}
+	txp := httptransport.SaverTransactionHTTPTransport{
+		RoundTripper: http.DefaultTransport.(*http.Transport),
+		Saver:        saver,
+	}
+	req, err := http.NewRequest("GET", "https://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if err != nil {
+		t.Fatal("not the error we expected")
+	}
+	if resp == nil {
+		t.Fatal("expected non nil response here")
+	}
+	ev := saver.Read()
+	if len(ev) != 2 {
+		t.Fatal("expected two events")
+	}
+	//
+	if ev[0].Name != "http_transaction_start" {
+		t.Fatal("unexpected Name")
+	}
+	if !ev[0].Time.Before(time.Now()) {
+		t.Fatal("unexpected Time")
+	}
+	//
+	if ev[1].Err != nil {
+		t.Fatal("unexpected Err")
+	}
+	if ev[1].Name != "http_transaction_done" {
+		t.Fatal("unexpected Name")
+	}
+	if !ev[1].Time.After(ev[0].Time) {
+		t.Fatal("unexpected Time")
+	}
+}
+
+func TestSaverTransactionFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	saver := &trace.Saver{}
+	txp := httptransport.SaverTransactionHTTPTransport{
+		RoundTripper: httptransport.FakeTransport{
+			Err: expected,
+		},
+		Saver: saver,
+	}
+	req, err := http.NewRequest("GET", "http://www.google.com", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response here")
+	}
+	ev := saver.Read()
+	if len(ev) != 2 {
+		t.Fatal("expected two events")
+	}
+	if ev[0].Name != "http_transaction_start" {
+		t.Fatal("unexpected Name")
+	}
+	if !ev[0].Time.Before(time.Now()) {
+		t.Fatal("unexpected Time")
+	}
+	if ev[1].Name != "http_transaction_done" {
+		t.Fatal("unexpected Name")
+	}
+	if !errors.Is(ev[1].Err, expected) {
+		t.Fatal("unexpected Err")
+	}
+	if !ev[1].Time.After(ev[0].Time) {
+		t.Fatal("unexpected Time")
+	}
+}
+
+func TestSaverBodySuccess(t *testing.T) {
+	saver := new(trace.Saver)
+	txp := httptransport.SaverBodyHTTPTransport{
+		RoundTripper: httptransport.FakeTransport{
+			Func: func(req *http.Request) (*http.Response, error) {
+				data, err := ioutil.ReadAll(req.Body)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if string(data) != "deadbeef" {
+					t.Fatal("invalid data")
+				}
+				return &http.Response{
+					StatusCode: 501,
+					Body:       ioutil.NopCloser(strings.NewReader("abad1dea")),
+				}, nil
+			},
+		},
+		SnapshotSize: 4,
+		Saver:        saver,
+	}
+	body := strings.NewReader("deadbeef")
+	req, err := http.NewRequest("POST", "http://x.org/y", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != 501 {
+		t.Fatal("unexpected status code")
+	}
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != "abad1dea" {
+		t.Fatal("unexpected body")
+	}
+	ev := saver.Read()
+	if len(ev) != 2 {
+		t.Fatal("unexpected number of events")
+	}
+	if string(ev[0].Data) != "dead" {
+		t.Fatal("invalid Data")
+	}
+	if ev[0].DataIsTruncated != true {
+		t.Fatal("invalid DataIsTruncated")
+	}
+	if ev[0].Name != "http_request_body_snapshot" {
+		t.Fatal("invalid Name")
+	}
+	if ev[0].Time.After(time.Now()) {
+		t.Fatal("invalid Time")
+	}
+	if string(ev[1].Data) != "abad" {
+		t.Fatal("invalid Data")
+	}
+	if ev[1].DataIsTruncated != true {
+		t.Fatal("invalid DataIsTruncated")
+	}
+	if ev[1].Name != "http_response_body_snapshot" {
+		t.Fatal("invalid Name")
+	}
+	if ev[1].Time.Before(ev[0].Time) {
+		t.Fatal("invalid Time")
+	}
+}
+
+func TestSaverBodyRequestReadError(t *testing.T) {
+	saver := new(trace.Saver)
+	txp := httptransport.SaverBodyHTTPTransport{
+		RoundTripper: httptransport.FakeTransport{
+			Func: func(req *http.Request) (*http.Response, error) {
+				panic("should not be called")
+			},
+		},
+		SnapshotSize: 4,
+		Saver:        saver,
+	}
+	expected := errors.New("mocked error")
+	body := httptransport.FakeBody{Err: expected}
+	req, err := http.NewRequest("POST", "http://x.org/y", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response")
+	}
+	ev := saver.Read()
+	if len(ev) != 0 {
+		t.Fatal("unexpected number of events")
+	}
+}
+
+func TestSaverBodyRoundTripError(t *testing.T) {
+	saver := new(trace.Saver)
+	expected := errors.New("mocked error")
+	txp := httptransport.SaverBodyHTTPTransport{
+		RoundTripper: httptransport.FakeTransport{
+			Err: expected,
+		},
+		SnapshotSize: 4,
+		Saver:        saver,
+	}
+	body := strings.NewReader("deadbeef")
+	req, err := http.NewRequest("POST", "http://x.org/y", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response")
+	}
+	ev := saver.Read()
+	if len(ev) != 1 {
+		t.Fatal("unexpected number of events")
+	}
+	if string(ev[0].Data) != "dead" {
+		t.Fatal("invalid Data")
+	}
+	if ev[0].DataIsTruncated != true {
+		t.Fatal("invalid DataIsTruncated")
+	}
+	if ev[0].Name != "http_request_body_snapshot" {
+		t.Fatal("invalid Name")
+	}
+	if ev[0].Time.After(time.Now()) {
+		t.Fatal("invalid Time")
+	}
+}
+
+func TestSaverBodyResponseReadError(t *testing.T) {
+	saver := new(trace.Saver)
+	expected := errors.New("mocked error")
+	txp := httptransport.SaverBodyHTTPTransport{
+		RoundTripper: httptransport.FakeTransport{
+			Func: func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					StatusCode: 200,
+					Body: httptransport.FakeBody{
+						Err: expected,
+					},
+				}, nil
+			},
+		},
+		SnapshotSize: 4,
+		Saver:        saver,
+	}
+	body := strings.NewReader("deadbeef")
+	req, err := http.NewRequest("POST", "http://x.org/y", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if resp != nil {
+		t.Fatal("expected nil response")
+	}
+	ev := saver.Read()
+	if len(ev) != 1 {
+		t.Fatal("unexpected number of events")
+	}
+	if string(ev[0].Data) != "dead" {
+		t.Fatal("invalid Data")
+	}
+	if ev[0].DataIsTruncated != true {
+		t.Fatal("invalid DataIsTruncated")
+	}
+	if ev[0].Name != "http_request_body_snapshot" {
+		t.Fatal("invalid Name")
+	}
+	if ev[0].Time.After(time.Now()) {
+		t.Fatal("invalid Time")
+	}
+}

internal/engine/netx/httptransport/system.go

@@ -0,0 +1,24 @@
+package httptransport
+
+import (
+	"net/http"
+)
+
+// NewSystemTransport creates a new "system" HTTP transport. That is a transport
+// using the Go standard library with custom dialer and TLS dialer.
+func NewSystemTransport(config Config) RoundTripper {
+	txp := http.DefaultTransport.(*http.Transport).Clone()
+	txp.DialContext = config.Dialer.DialContext
+	txp.DialTLSContext = config.TLSDialer.DialTLSContext
+	// Better for Cloudflare DNS and also better because we have less
+	// noisy events and we can better understand what happened.
+	txp.MaxConnsPerHost = 1
+	// The following (1) reduces the number of headers that Go will
+	// automatically send for us and (2) ensures that we always receive
+	// back the true headers, such as Content-Length. This change is
+	// functional to OONI's goal of observing the network.
+	txp.DisableCompression = true
+	return txp
+}
+
+var _ RoundTripper = &http.Transport{}

internal/engine/netx/httptransport/useragent.go

@@ -0,0 +1,19 @@
+package httptransport
+
+import "net/http"
+
+// UserAgentTransport is a transport that ensures that we always
+// set an OONI specific default User-Agent header.
+type UserAgentTransport struct {
+	RoundTripper
+}
+
+// RoundTrip implements RoundTripper.RoundTrip
+func (txp UserAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.Header.Get("User-Agent") == "" {
+		req.Header.Set("User-Agent", "miniooni/0.1.0-dev")
+	}
+	return txp.RoundTripper.RoundTrip(req)
+}
+
+var _ RoundTripper = UserAgentTransport{}

internal/engine/netx/httptransport/useragent_test.go

@@ -0,0 +1,51 @@
+package httptransport_test
+
+import (
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
+)
+
+func TestUserAgentWithDefault(t *testing.T) {
+	txp := httptransport.UserAgentTransport{
+		RoundTripper: httptransport.FakeTransport{
+			Resp: &http.Response{StatusCode: 200},
+		},
+	}
+	req := &http.Request{URL: &url.URL{
+		Scheme: "https",
+		Host:   "www.google.com",
+		Path:   "/",
+	}}
+	req.Header = http.Header{}
+	resp, err := txp.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.Request.Header.Get("User-Agent") != "miniooni/0.1.0-dev" {
+		t.Fatal("not the User-Agent we expected")
+	}
+}
+
+func TestUserAgentWithExplicitValue(t *testing.T) {
+	txp := httptransport.UserAgentTransport{
+		RoundTripper: httptransport.FakeTransport{
+			Resp: &http.Response{StatusCode: 200},
+		},
+	}
+	req := &http.Request{URL: &url.URL{
+		Scheme: "https",
+		Host:   "www.google.com",
+		Path:   "/",
+	}}
+	req.Header = http.Header{"User-Agent": []string{"antani-client/0.1.1"}}
+	resp, err := txp.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.Request.Header.Get("User-Agent") != "antani-client/0.1.1" {
+		t.Fatal("not the User-Agent we expected")
+	}
+}