chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/experiment/tlstool/internal/dialer.go

@@ -0,0 +1,29 @@
+package internal
+
+import (
+	"context"
+	"net"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+// Dialer creates net.Conn instances where (1) we delay writes if
+// a delay is configured and (2) we split outgoing buffers if there
+// is a configured splitter function.
+type Dialer struct {
+	netx.Dialer
+	Delay    time.Duration
+	Splitter func([]byte) [][]byte
+}
+
+// DialContext implements netx.Dialer.DialContext.
+func (d Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	conn, err := d.Dialer.DialContext(ctx, network, address)
+	if err != nil {
+		return nil, err
+	}
+	conn = SleeperWriter{Conn: conn, Delay: d.Delay}
+	conn = SplitterWriter{Conn: conn, Splitter: d.Splitter}
+	return conn, nil
+}

internal/engine/experiment/tlstool/internal/dialer_test.go

@@ -0,0 +1,56 @@
+package internal_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+)
+
+func TestDialerFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	dialer := internal.Dialer{Dialer: internal.FakeDialer{
+		Err: expected,
+	}}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "8.8.8.8:853")
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn here")
+	}
+}
+
+func TestDialerSuccess(t *testing.T) {
+	splitter := func([]byte) [][]byte {
+		return nil // any value is fine we just a need a splitter != nil here
+	}
+	innerconn := &internal.FakeConn{}
+	dialer := internal.Dialer{
+		Delay:    12345,
+		Dialer:   internal.FakeDialer{Conn: innerconn},
+		Splitter: splitter,
+	}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "8.8.8.8:853")
+	if err != nil {
+		t.Fatal(err)
+	}
+	sconn, ok := conn.(internal.SplitterWriter)
+	if !ok {
+		t.Fatal("the outer connection is not a splitter")
+	}
+	if sconn.Splitter == nil {
+		t.Fatal("not the splitter we expected")
+	}
+	dconn, ok := sconn.Conn.(internal.SleeperWriter)
+	if !ok {
+		t.Fatal("the inner connection is not a sleeper")
+	}
+	if dconn.Delay != 12345 {
+		t.Fatal("invalid delay")
+	}
+	if dconn.Conn != innerconn {
+		t.Fatal("invalid inner connection")
+	}
+}

internal/engine/experiment/tlstool/internal/fake_test.go

@@ -0,0 +1,73 @@
+package internal
+
+import (
+	"context"
+	"io"
+	"net"
+	"time"
+)
+
+type FakeDialer struct {
+	Conn net.Conn
+	Err  error
+}
+
+func (d FakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	time.Sleep(10 * time.Microsecond)
+	return d.Conn, d.Err
+}
+
+type FakeConn struct {
+	ReadError             error
+	ReadData              []byte
+	SetDeadlineError      error
+	SetReadDeadlineError  error
+	SetWriteDeadlineError error
+	WriteData             [][]byte
+	WriteError            error
+}
+
+func (c *FakeConn) Read(b []byte) (int, error) {
+	if len(c.ReadData) > 0 {
+		n := copy(b, c.ReadData)
+		c.ReadData = c.ReadData[n:]
+		return n, nil
+	}
+	if c.ReadError != nil {
+		return 0, c.ReadError
+	}
+	return 0, io.EOF
+}
+
+func (c *FakeConn) Write(b []byte) (n int, err error) {
+	if c.WriteError != nil {
+		return 0, c.WriteError
+	}
+	c.WriteData = append(c.WriteData, b)
+	n = len(b)
+	return
+}
+
+func (*FakeConn) Close() (err error) {
+	return
+}
+
+func (*FakeConn) LocalAddr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+func (*FakeConn) RemoteAddr() net.Addr {
+	return &net.TCPAddr{}
+}
+
+func (c *FakeConn) SetDeadline(t time.Time) (err error) {
+	return c.SetDeadlineError
+}
+
+func (c *FakeConn) SetReadDeadline(t time.Time) (err error) {
+	return c.SetReadDeadlineError
+}
+
+func (c *FakeConn) SetWriteDeadline(t time.Time) (err error) {
+	return c.SetWriteDeadlineError
+}

internal/engine/experiment/tlstool/internal/internal.go

@@ -0,0 +1,57 @@
+// Package internal contains the implementation of tlstool.
+package internal
+
+import (
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+// DialerConfig contains the config for creating a dialer
+type DialerConfig struct {
+	Dialer netx.Dialer
+	Delay  time.Duration
+	SNI    string
+}
+
+// NewSNISplitterDialer creates a new dialer that splits
+// outgoing messages such that the SNI should end up being
+// splitted into different TCP segments.
+func NewSNISplitterDialer(config DialerConfig) Dialer {
+	return Dialer{
+		Dialer: config.Dialer,
+		Delay:  config.Delay,
+		Splitter: func(b []byte) [][]byte {
+			return SNISplitter(b, []byte(config.SNI))
+		},
+	}
+}
+
+// NewThriceSplitterDialer creates a new dialer that splits
+// outgoing messages in three parts according to the circumvention
+// technique described by Kevin Boch in the Internet Measurement
+// Village 2020 <https://youtu.be/ksojSRFLbBM?t=1140>.
+func NewThriceSplitterDialer(config DialerConfig) Dialer {
+	return Dialer{
+		Dialer:   config.Dialer,
+		Delay:    config.Delay,
+		Splitter: Splitter84rest,
+	}
+}
+
+// NewRandomSplitterDialer creates a new dialer that splits
+// the SNI like the fixed splitting schema used by outline. See
+// github.com/Jigsaw-Code/outline-go-tun2socks.
+func NewRandomSplitterDialer(config DialerConfig) Dialer {
+	return Dialer{
+		Dialer:   config.Dialer,
+		Delay:    config.Delay,
+		Splitter: Splitter3264rand,
+	}
+}
+
+// NewVanillaDialer creates a new vanilla dialer that does
+// nothing and is used to establish a baseline.
+func NewVanillaDialer(config DialerConfig) Dialer {
+	return Dialer{Dialer: config.Dialer}
+}

internal/engine/experiment/tlstool/internal/internal_test.go

@@ -0,0 +1,40 @@
+package internal_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+var config = internal.DialerConfig{
+	Dialer: netx.NewDialer(netx.Config{}),
+	Delay:  10,
+	SNI:    "dns.google",
+}
+
+func dial(t *testing.T, d netx.Dialer) {
+	td := netx.NewTLSDialer(netx.Config{Dialer: d})
+	conn, err := td.DialTLSContext(context.Background(), "tcp", "dns.google:853")
+	if err != nil {
+		t.Fatal(err)
+	}
+	conn.Close()
+}
+
+func TestNewSNISplitterDialer(t *testing.T) {
+	dial(t, internal.NewSNISplitterDialer(config))
+}
+
+func TestNewThriceSplitterDialer(t *testing.T) {
+	dial(t, internal.NewThriceSplitterDialer(config))
+}
+
+func TestNewRandomSplitterDialer(t *testing.T) {
+	dial(t, internal.NewRandomSplitterDialer(config))
+}
+
+func TestNewVanillaDialer(t *testing.T) {
+	dial(t, internal.NewVanillaDialer(config))
+}

internal/engine/experiment/tlstool/internal/splitter.go

@@ -0,0 +1,67 @@
+package internal
+
+import (
+	"bytes"
+	"math/rand"
+	"time"
+)
+
+// SNISplitter splits input such that SNI is splitted across
+// a bunch of different output buffers.
+func SNISplitter(input []byte, sni []byte) (output [][]byte) {
+	idx := bytes.Index(input, sni)
+	if idx < 0 {
+		output = append(output, input)
+		return
+	}
+	output = append(output, input[:idx])
+	// TODO(bassosimone): splitting every three bytes causes
+	// a bunch of Unicode chatacters (e.g., in Chinese) to be
+	// sent as part of the same segment. Is that OK?
+	const segmentsize = 3
+	var buf []byte
+	for _, chr := range input[idx : idx+len(sni)] {
+		buf = append(buf, chr)
+		if len(buf) == segmentsize {
+			output = append(output, buf)
+			buf = nil
+		}
+	}
+	if len(buf) > 0 {
+		output = append(output, buf)
+		buf = nil
+	}
+	output = append(output, input[idx+len(sni):])
+	return
+}
+
+// Splitter84rest segments the specified buffer into three
+// sub-buffers containing respectively 8 bytes, 4 bytes, and
+// the rest of the buffer. This segment technique has been
+// described by Kevin Bock during the Internet Measurements
+// Village 2020: https://youtu.be/ksojSRFLbBM?t=1140.
+func Splitter84rest(input []byte) (output [][]byte) {
+	if len(input) <= 12 {
+		output = append(output, input)
+		return
+	}
+	output = append(output, input[:8])
+	output = append(output, input[8:12])
+	output = append(output, input[12:])
+	return
+}
+
+// Splitter3264rand splits the specified buffer at a random
+// offset between 32 and 64 bytes. This is the methodology used
+// by github.com/Jigsaw-Code/outline-go-tun2socks.
+func Splitter3264rand(input []byte) (output [][]byte) {
+	if len(input) <= 64 {
+		output = append(output, input)
+		return
+	}
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+	offset := rnd.Intn(32) + 32
+	output = append(output, input[:offset])
+	output = append(output, input[offset:])
+	return
+}

internal/engine/experiment/tlstool/internal/splitter_test.go

@@ -0,0 +1,143 @@
+package internal_test
+
+import (
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
+)
+
+func TestSplitter84restSmall(t *testing.T) {
+	input := []byte("1111222")
+	output := internal.Splitter84rest(input)
+	if len(output) != 1 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != "1111222" {
+		t.Fatal("invalid output[0]")
+	}
+}
+
+func TestSplitter84restGood(t *testing.T) {
+	input := []byte("1111222233334")
+	output := internal.Splitter84rest(input)
+	if len(output) != 3 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != "11112222" {
+		t.Fatal("invalid output[0]")
+	}
+	if string(output[1]) != "3333" {
+		t.Fatal("invalid output[1]")
+	}
+	if string(output[2]) != "4" {
+		t.Fatal("invalid output[2]")
+	}
+}
+
+func TestSplitter3264randSmall(t *testing.T) {
+	input := randx.Letters(64)
+	output := internal.Splitter3264rand([]byte(input))
+	if len(output) != 1 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != input {
+		t.Fatal("invalid output[0]")
+	}
+}
+
+func TestSplitter3264Works(t *testing.T) {
+	input := randx.Letters(65)
+	output := internal.Splitter3264rand([]byte(input))
+	for i := 0; i < 32; i++ {
+		if len(output) != 2 {
+			t.Fatal("invalid output length")
+		}
+		if len(output[0]) < 32 || len(output[0]) > 64 {
+			t.Fatal("invalid output[0] length")
+		}
+	}
+}
+
+func TestSNISplitterEasyCase(t *testing.T) {
+	input := []byte("11112222334555foo.barbar.deadbeef.com6777778888")
+	sni := []byte("barbar.deadbeef.com")
+	output := internal.SNISplitter(input, sni)
+	if len(output) != 9 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != "11112222334555foo." {
+		t.Fatal("invalid output[0]")
+	}
+	if string(output[1]) != "bar" {
+		t.Fatal("invalid output[1]")
+	}
+	if string(output[2]) != "bar" {
+		t.Fatal("invalid output[2]")
+	}
+	if string(output[3]) != ".de" {
+		t.Fatal("invalid output[3]")
+	}
+	if string(output[4]) != "adb" {
+		t.Fatal("invalid output[4]")
+	}
+	if string(output[5]) != "eef" {
+		t.Fatal("invalid output[5]")
+	}
+	if string(output[6]) != ".co" {
+		t.Fatal("invalid output[6]")
+	}
+	if string(output[7]) != "m" {
+		t.Fatal("invalid output[7]")
+	}
+	if string(output[8]) != "6777778888" {
+		t.Fatal("invalid output[8]")
+	}
+}
+
+func TestSNISplitterNoMatch(t *testing.T) {
+	input := []byte("11112222334555foo.barbar.deadbeef.com6777778888")
+	sni := []byte("www.google.com")
+	output := internal.SNISplitter(input, sni)
+	if len(output) != 1 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != string(input) {
+		t.Fatal("invalid output[0]")
+	}
+}
+
+func TestSNISplitterWithUnicode(t *testing.T) {
+	input := []byte("11112222334555你好世界.com6777778888")
+	sni := []byte("你好世界.com")
+	output := internal.SNISplitter(input, sni)
+	t.Log(string(output[2]))
+	t.Log(output)
+	if len(output) != 8 {
+		t.Fatal("invalid output length")
+	}
+	if string(output[0]) != "11112222334555" {
+		t.Fatal("invalid output[0]")
+	}
+	if string(output[1]) != "你" {
+		t.Fatal("invalid output[1]")
+	}
+	if string(output[2]) != "好" {
+		t.Fatal("invalid output[2]")
+	}
+	if string(output[3]) != "世" {
+		t.Fatal("invalid output[3]")
+	}
+	if string(output[4]) != "界" {
+		t.Fatal("invalid output[4]")
+	}
+	if string(output[5]) != ".co" {
+		t.Fatal("invalid output[5]")
+	}
+	if string(output[6]) != "m" {
+		t.Fatal("invalid output[6]")
+	}
+	if string(output[7]) != "6777778888" {
+		t.Fatal("invalid output[7]")
+	}
+}

internal/engine/experiment/tlstool/internal/writer.go

@@ -0,0 +1,57 @@
+package internal
+
+import (
+	"net"
+	"time"
+)
+
+// SleeperWriter is a net.Conn that optionally sleeps for the
+// specified delay before posting each write.
+type SleeperWriter struct {
+	net.Conn
+	Delay time.Duration
+}
+
+func (c SleeperWriter) Write(b []byte) (int, error) {
+	<-time.After(c.Delay)
+	return c.Conn.Write(b)
+}
+
+// SplitterWriter is a writer that splits every outgoing buffer
+// according to the rules specified by the Splitter.
+//
+// Caveat
+//
+// The TLS ClientHello may be retransmitted if the server is
+// requesting us to restart the negotiation. Therefore, it is
+// not safe to just run the splitting once. Since this code
+// is meant to investigate TLS blocking, that's fine.
+type SplitterWriter struct {
+	net.Conn
+	Splitter func([]byte) [][]byte
+}
+
+// Write implements net.Conn.Write
+func (c SplitterWriter) Write(b []byte) (int, error) {
+	if c.Splitter != nil {
+		return Writev(c.Conn, c.Splitter(b))
+	}
+	return c.Conn.Write(b)
+}
+
+// Writev writes all the vectors inside datalist using the specified
+// conn. Returns either an error or the number of bytes sent. Note
+// that this function skips any empty entry in datalist.
+func Writev(conn net.Conn, datalist [][]byte) (int, error) {
+	var total int
+	for _, data := range datalist {
+		if len(data) > 0 {
+			count, err := conn.Write(data)
+			if err != nil {
+				return 0, err
+			}
+			total += count
+		}
+	}
+	return total, nil
+}

internal/engine/experiment/tlstool/internal/writer_test.go

@@ -0,0 +1,164 @@
+package internal_test
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal"
+)
+
+func TestSleeperWriterWorksAsIntended(t *testing.T) {
+	origconn := &internal.FakeConn{}
+	const outdata = "deadbeefbadidea"
+	conn := internal.SleeperWriter{
+		Conn:  origconn,
+		Delay: 1 * time.Second,
+	}
+	before := time.Now()
+	count, err := conn.Write([]byte(outdata))
+	elapsed := time.Since(before)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != len(outdata) {
+		t.Fatal("unexpected count")
+	}
+	if len(origconn.WriteData) != 1 {
+		t.Fatal("wrong length of written data queue")
+	}
+	if string(origconn.WriteData[0]) != outdata {
+		t.Fatal("we did not write the right data")
+	}
+	if elapsed < 750*time.Millisecond {
+		t.Fatalf("unexpected elapsed time: %+v", elapsed)
+	}
+}
+
+func TestSplitterWriterNoSplitSuccess(t *testing.T) {
+	innerconn := &internal.FakeConn{}
+	conn := internal.SplitterWriter{Conn: innerconn}
+	const data = "deadbeef"
+	count, err := conn.Write([]byte(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != len(data) {
+		t.Fatal("invalid count")
+	}
+	if len(innerconn.WriteData) != 1 {
+		t.Fatal("invalid data queue")
+	}
+	if string(innerconn.WriteData[0]) != data {
+		t.Fatal("invalid written data")
+	}
+}
+
+func TestSplitterWriterNoSplitFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	innerconn := &internal.FakeConn{WriteError: expected}
+	conn := internal.SplitterWriter{Conn: innerconn}
+	const data = "deadbeef"
+	count, err := conn.Write([]byte(data))
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if count != 0 {
+		t.Fatal("invalid count")
+	}
+	if len(innerconn.WriteData) != 0 {
+		t.Fatal("invalid data queue")
+	}
+}
+
+func TestSplitterWriterSplitSuccess(t *testing.T) {
+	innerconn := &internal.FakeConn{}
+	conn := internal.SplitterWriter{
+		Conn: innerconn,
+		Splitter: func(b []byte) [][]byte {
+			return [][]byte{
+				b[:2], b[2:],
+			}
+		},
+	}
+	const data = "deadbeef"
+	count, err := conn.Write([]byte(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != len(data) {
+		t.Fatal("invalid count")
+	}
+	if len(innerconn.WriteData) != 2 {
+		t.Fatal("invalid data queue")
+	}
+	if string(innerconn.WriteData[0]) != "de" {
+		t.Fatal("invalid written data[0]")
+	}
+	if string(innerconn.WriteData[1]) != "adbeef" {
+		t.Fatal("invalid written data[1]")
+	}
+}
+
+func TestSplitterWriterSplitFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	innerconn := &internal.FakeConn{WriteError: expected}
+	conn := internal.SplitterWriter{
+		Conn: innerconn,
+		Splitter: func(b []byte) [][]byte {
+			return [][]byte{
+				b[:2], b[2:],
+			}
+		},
+	}
+	const data = "deadbeef"
+	count, err := conn.Write([]byte(data))
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if count != 0 {
+		t.Fatal("invalid count")
+	}
+	if len(innerconn.WriteData) != 0 {
+		t.Fatal("invalid data queue")
+	}
+}
+
+func TestWritevWorksWithAlsoEmptyData(t *testing.T) {
+	conn := &internal.FakeConn{}
+	datalist := [][]byte{
+		[]byte("deadbeef"),
+		[]byte(""),
+		[]byte("dead"),
+		nil,
+		[]byte("badidea"),
+		nil,
+	}
+	count, err := internal.Writev(conn, datalist)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != 19 {
+		t.Fatal("invalid number of bytes written")
+	}
+}
+
+func TestWritevFailsAsIntended(t *testing.T) {
+	expected := errors.New("mocked error")
+	conn := &internal.FakeConn{WriteError: expected}
+	datalist := [][]byte{
+		[]byte("deadbeef"),
+		[]byte(""),
+		[]byte("dead"),
+		nil,
+		[]byte("badidea"),
+		nil,
+	}
+	count, err := internal.Writev(conn, datalist)
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if count != 0 {
+		t.Fatal("invalid number of bytes written")
+	}
+}