refactor: start pivoting netx (#396)

What do I mean by pivoting? Netx is currently organized by row:

```
               | dialer | quicdialer | resolver | ...
 saving        |        |            |          | ...
 errorwrapping |        |            |          | ...
 logging       |        |            |          | ...
 mocking/sys   |        |            |          | ...
```

Every row needs to implement saving, errorwrapping, logging, mocking (or
adapting to the system or to some underlying library).

This causes cross package dependencies and, in turn, complexity. For
example, we need the `trace` package for supporting saving.

And `dialer`, `quickdialer`, et al. need to depend on such a package.

The same goes for errorwrapping.

This arrangement further complicates testing. For example, I am
currently working on https://github.com/ooni/probe/issues/1505 and
I realize it need to repeat integration tests in multiple places.

Let's say instead we pivot the above matrix as follows:

```
             | saving | errorwrapping | logging | ...
 dialer      |        |               |         | ...
 quicdialer  |        |               |         | ...
 logging     |        |               |         | ...
 mocking/sys |        |               |         | ...
 ...
```

In this way, now every row contains everything related to a specific
action to perform. We can now share code without relying on extra
support packages. What's more, we can write tests and, judding from
the way in which things are made, it seems we only need integration
testing in `errorwrapping` because it's where data quality matters
whereas, in all other cases, unit testing is fine.

I am going, therefore, to proceed with these changes and "pivot"
`netx`. Hopefully, it won't be too painful.

internal/netxlite/dialer.go

@@ -0,0 +1,86 @@
+package netxlite
+
+import (
+	"context"
+	"net"
+	"time"
+)
+
+// Dialer establishes network connections.
+type Dialer interface {
+	// DialContext behaves like net.Dialer.DialContext.
+	DialContext(ctx context.Context, network, address string) (net.Conn, error)
+}
+
+// DefaultDialer is the Dialer we use by default.
+var DefaultDialer = &net.Dialer{
+	Timeout:   15 * time.Second,
+	KeepAlive: 15 * time.Second,
+}
+
+var _ Dialer = DefaultDialer
+
+// DialerResolver is a dialer that uses the configured Resolver to resolver a
+// domain name to IP addresses, and the configured Dialer to connect.
+type DialerResolver struct {
+	// Dialer is the underlying Dialer.
+	Dialer Dialer
+
+	// Resolver is the underlying Resolver.
+	Resolver Resolver
+}
+
+var _ Dialer = &DialerResolver{}
+
+// DialContext implements Dialer.DialContext.
+func (d *DialerResolver) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	onlyhost, onlyport, err := net.SplitHostPort(address)
+	if err != nil {
+		return nil, err
+	}
+	var addrs []string
+	addrs, err = d.lookupHost(ctx, onlyhost)
+	if err != nil {
+		return nil, err
+	}
+	// TODO(bassosimone): here we should be using multierror rather
+	// than just calling ReduceErrors. We are not ready to do that
+	// yet, though. To do that, we need first to modify nettests so
+	// that we actually avoid dialing when measuring.
+	var errorslist []error
+	for _, addr := range addrs {
+		target := net.JoinHostPort(addr, onlyport)
+		conn, err := d.Dialer.DialContext(ctx, network, target)
+		if err == nil {
+			return conn, nil
+		}
+		errorslist = append(errorslist, err)
+	}
+	return nil, ReduceErrors(errorslist)
+}
+
+// lookupHost performs a domain name resolution.
+func (d *DialerResolver) lookupHost(ctx context.Context, hostname string) ([]string, error) {
+	if net.ParseIP(hostname) != nil {
+		return []string{hostname}, nil
+	}
+	return d.Resolver.LookupHost(ctx, hostname)
+}
+
+// DialerLogger is a Dialer with logging
+type DialerLogger struct {
+	Dialer
+	Logger Logger
+}
+
+var _ Dialer = &DialerLogger{}
+
+// DialContext implements Dialer.DialContext
+func (d *DialerLogger) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	d.Logger.Debugf("dial %s/%s...", address, network)
+	start := time.Now()
+	conn, err := d.Dialer.DialContext(ctx, network, address)
+	stop := time.Now()
+	d.Logger.Debugf("dial %s/%s... %+v in %s", address, network, err, stop.Sub(start))
+	return conn, err
+}

internal/netxlite/dialer_test.go

@@ -0,0 +1,161 @@
+package netxlite
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/mockablex"
+)
+
+func TestDialerResolverNoPort(t *testing.T) {
+	dialer := &DialerResolver{Dialer: new(net.Dialer), Resolver: DefaultResolver}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "antani.ooni.nu")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if conn != nil {
+		t.Fatal("expected a nil conn here")
+	}
+}
+
+func TestDialerResolverLookupHostAddress(t *testing.T) {
+	dialer := &DialerResolver{Dialer: new(net.Dialer), Resolver: MockableResolver{
+		Err: errors.New("mocked error"),
+	}}
+	addrs, err := dialer.lookupHost(context.Background(), "1.1.1.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(addrs) != 1 || addrs[0] != "1.1.1.1" {
+		t.Fatal("not the result we expected")
+	}
+}
+
+func TestDialerResolverLookupHostFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	dialer := &DialerResolver{Dialer: new(net.Dialer), Resolver: MockableResolver{
+		Err: expected,
+	}}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "dns.google.com:853")
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn")
+	}
+}
+
+type MockableResolver struct {
+	Addresses []string
+	Err       error
+}
+
+func (r MockableResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
+	return r.Addresses, r.Err
+}
+
+func (r MockableResolver) Network() string {
+	return "mockable"
+}
+
+func (r MockableResolver) Address() string {
+	return ""
+}
+
+func TestDialerResolverDialForSingleIPFails(t *testing.T) {
+	dialer := &DialerResolver{Dialer: mockablex.Dialer{
+		MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
+			return nil, io.EOF
+		},
+	}, Resolver: DefaultResolver}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "1.1.1.1:853")
+	if !errors.Is(err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn")
+	}
+}
+
+func TestDialerResolverDialForManyIPFails(t *testing.T) {
+	dialer := &DialerResolver{
+		Dialer: mockablex.Dialer{
+			MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
+				return nil, io.EOF
+			},
+		}, Resolver: MockableResolver{
+			Addresses: []string{"1.1.1.1", "8.8.8.8"},
+		}}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "dot.dns:853")
+	if !errors.Is(err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn")
+	}
+}
+
+func TestDialerResolverDialForManyIPSuccess(t *testing.T) {
+	dialer := &DialerResolver{Dialer: mockablex.Dialer{
+		MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
+			return &mockablex.Conn{
+				MockClose: func() error {
+					return nil
+				},
+			}, nil
+		},
+	}, Resolver: MockableResolver{
+		Addresses: []string{"1.1.1.1", "8.8.8.8"},
+	}}
+	conn, err := dialer.DialContext(context.Background(), "tcp", "dot.dns:853")
+	if err != nil {
+		t.Fatal("expected nil error here")
+	}
+	if conn == nil {
+		t.Fatal("expected non-nil conn")
+	}
+	conn.Close()
+}
+
+func TestDialerLoggerFailure(t *testing.T) {
+	d := &DialerLogger{
+		Dialer: mockablex.Dialer{
+			MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
+				return nil, io.EOF
+			},
+		},
+		Logger: log.Log,
+	}
+	conn, err := d.DialContext(context.Background(), "tcp", "www.google.com:443")
+	if !errors.Is(err, io.EOF) {
+		t.Fatal("not the error we expected")
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn here")
+	}
+}
+
+func TestDefaultDialerWorks(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	conn, err := DefaultDialer.DialContext(ctx, "tcp", "8.8.8.8:853")
+	if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
+		t.Fatal("not the error we expected", err)
+	}
+	if conn != nil {
+		t.Fatal("expected nil conn here")
+	}
+}
+
+func TestDefaultDialerHasTimeout(t *testing.T) {
+	expected := 15 * time.Second
+	if DefaultDialer.Timeout != expected {
+		t.Fatal("unexpected timeout value")
+	}
+}

internal/netxlite/doc.go

@@ -0,0 +1,46 @@
+// Package netxlite contains network extensions.
+//
+// This package is the basic networking building block that you
+// should be using every time you need networking.
+//
+// Naming and history
+//
+// Previous versions of this package were called netx. Compared to such
+// versions this package is lightweight because it does not contain code
+// to perform the measurements, hence its name.
+//
+// Design
+//
+// We want to potentially be able to observe each low-level operation
+// separately, even though this is not done by this package. This is
+// the use case where we are performing measurements.
+//
+// We also want to be able to use this package in a more casual way
+// without having to compose each operation separately. This, instead, is
+// the use case where we're communicating with the OONI backend.
+//
+// We want to optionally provide detailed logging of every operation,
+// thus users can use `-v` to obtain OONI logs.
+//
+// We also want to mock any underlying dependency for testing.
+//
+// Operations
+//
+// This package implements the following operations:
+//
+// 1. establishing a TCP connection;
+//
+// 2. performing a domain name resolution;
+//
+// 3. performing the TLS handshake;
+//
+// 4. performing the QUIC handshake;
+//
+// 5. dialing with TCP, TLS, and QUIC (where in this context dialing
+// means combining domain name resolution and "connecting");
+//
+// 6. performing HTTP, HTTP2, and HTTP3 round trips.
+//
+// Operations 1, 2, 3, and 4 are used when we perform measurements,
+// while 5 and 6 are mostly used when speaking with our backend.
+package netxlite

internal/netxlite/legacy.go

@@ -0,0 +1,39 @@
+package netxlite
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+// ReduceErrors finds a known error in a list of errors since
+// it's probably most relevant.
+//
+// Deprecation warning
+//
+// Albeit still used, this function is now DEPRECATED.
+//
+// In perspective, we would like to transition to a scenario where
+// full dialing is NOT used for measurements and we return a multierror here.
+func ReduceErrors(errorslist []error) error {
+	if len(errorslist) == 0 {
+		return nil
+	}
+	// If we have a known error, let's consider this the real error
+	// since it's probably most relevant. Otherwise let's return the
+	// first considering that (1) local resolvers likely will give
+	// us IPv4 first and (2) also our resolver does that. So, in case
+	// the user has no IPv6 connectivity, an IPv6 error is going to
+	// appear later in the list of errors.
+	for _, err := range errorslist {
+		var wrapper *errorx.ErrWrapper
+		if errors.As(err, &wrapper) && !strings.HasPrefix(
+			err.Error(), "unknown_failure",
+		) {
+			return err
+		}
+	}
+	// TODO(bassosimone): handle this case in a better way
+	return errorslist[0]
+}

internal/netxlite/legacy_test.go

@@ -0,0 +1,49 @@
+package netxlite
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestReduceErrors(t *testing.T) {
+	t.Run("no errors", func(t *testing.T) {
+		result := ReduceErrors(nil)
+		if result != nil {
+			t.Fatal("wrong result")
+		}
+	})
+
+	t.Run("single error", func(t *testing.T) {
+		err := errors.New("mocked error")
+		result := ReduceErrors([]error{err})
+		if result != err {
+			t.Fatal("wrong result")
+		}
+	})
+
+	t.Run("multiple errors", func(t *testing.T) {
+		err1 := errors.New("mocked error #1")
+		err2 := errors.New("mocked error #2")
+		result := ReduceErrors([]error{err1, err2})
+		if result.Error() != "mocked error #1" {
+			t.Fatal("wrong result")
+		}
+	})
+
+	t.Run("multiple errors with meaningful ones", func(t *testing.T) {
+		err1 := errors.New("mocked error #1")
+		err2 := &errorx.ErrWrapper{
+			Failure: "unknown_failure: antani",
+		}
+		err3 := &errorx.ErrWrapper{
+			Failure: errorx.FailureConnectionRefused,
+		}
+		err4 := errors.New("mocked error #3")
+		result := ReduceErrors([]error{err1, err2, err3, err4})
+		if result.Error() != errorx.FailureConnectionRefused {
+			t.Fatal("wrong result")
+		}
+	})
+}

internal/netxlite/logger.go

@@ -0,0 +1,7 @@
+package netxlite
+
+// Logger is the interface we expect from a logger.
+type Logger interface {
+	// Debugf formats and emits a debug message.
+	Debugf(format string, v ...interface{})
+}

internal/netxlite/resolver.go

@@ -0,0 +1,78 @@
+package netxlite
+
+import (
+	"context"
+	"net"
+	"time"
+)
+
+// Resolver performs domain name resolutions.
+type Resolver interface {
+	// LookupHost behaves like net.Resolver.LookupHost.
+	LookupHost(ctx context.Context, hostname string) (addrs []string, err error)
+}
+
+// ResolverSystem is the system resolver.
+type ResolverSystem struct{}
+
+var _ Resolver = ResolverSystem{}
+
+// LookupHost implements Resolver.LookupHost.
+func (r ResolverSystem) LookupHost(ctx context.Context, hostname string) ([]string, error) {
+	return net.DefaultResolver.LookupHost(ctx, hostname)
+}
+
+// Network implements Resolver.Network.
+func (r ResolverSystem) Network() string {
+	return "system"
+}
+
+// Address implements Resolver.Address.
+func (r ResolverSystem) Address() string {
+	return ""
+}
+
+// DefaultResolver is the resolver we use by default.
+var DefaultResolver = ResolverSystem{}
+
+// ResolverLogger is a resolver that emits events
+type ResolverLogger struct {
+	Resolver
+	Logger Logger
+}
+
+var _ Resolver = ResolverLogger{}
+
+// LookupHost returns the IP addresses of a host
+func (r ResolverLogger) LookupHost(ctx context.Context, hostname string) ([]string, error) {
+	r.Logger.Debugf("resolve %s...", hostname)
+	start := time.Now()
+	addrs, err := r.Resolver.LookupHost(ctx, hostname)
+	stop := time.Now()
+	r.Logger.Debugf("resolve %s... (%+v, %+v) in %s", hostname, addrs, err, stop.Sub(start))
+	return addrs, err
+}
+
+type resolverNetworker interface {
+	Network() string
+}
+
+// Network implements Resolver.Network.
+func (r ResolverLogger) Network() string {
+	if rn, ok := r.Resolver.(resolverNetworker); ok {
+		return rn.Network()
+	}
+	return "logger"
+}
+
+type resolverAddresser interface {
+	Address() string
+}
+
+// Address implements Resolver.Address.
+func (r ResolverLogger) Address() string {
+	if ra, ok := r.Resolver.(resolverAddresser); ok {
+		return ra.Address()
+	}
+	return ""
+}

internal/netxlite/resolver_test.go

@@ -0,0 +1,56 @@
+package netxlite
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/apex/log"
+)
+
+func TestResolverSystemLookupHost(t *testing.T) {
+	r := ResolverSystem{}
+	if r.Network() != "system" {
+		t.Fatal("invalid Network")
+	}
+	if r.Address() != "" {
+		t.Fatal("invalid Address")
+	}
+	addrs, err := r.LookupHost(context.Background(), "dns.google.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if addrs == nil {
+		t.Fatal("expected non-nil result here")
+	}
+}
+
+func TestResolverLoggerWithFailure(t *testing.T) {
+	r := ResolverLogger{
+		Logger:   log.Log,
+		Resolver: DefaultResolver,
+	}
+	if r.Network() != "system" {
+		t.Fatal("invalid Network")
+	}
+	if r.Address() != "" {
+		t.Fatal("invalid Address")
+	}
+	addrs, err := r.LookupHost(context.Background(), "nonexistent.antani")
+	if err == nil {
+		t.Fatal("expected an error here")
+	}
+	if addrs != nil {
+		t.Fatal("expected nil addr here")
+	}
+}
+
+func TestResolverLoggerDefaultNetworkAddress(t *testing.T) {
+	r := &ResolverLogger{Logger: log.Log, Resolver: &net.Resolver{}}
+	if r.Network() != "logger" {
+		t.Fatal("invalid Network")
+	}
+	if r.Address() != "" {
+		t.Fatal("invalid Address")
+	}
+}