c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: start pivoting netx (#396)

Browse Source 
What do I mean by pivoting? Netx is currently organized by row:

```
               | dialer | quicdialer | resolver | ...
 saving        |        |            |          | ...
 errorwrapping |        |            |          | ...
 logging       |        |            |          | ...
 mocking/sys   |        |            |          | ...
```

Every row needs to implement saving, errorwrapping, logging, mocking (or
adapting to the system or to some underlying library).

This causes cross package dependencies and, in turn, complexity. For
example, we need the `trace` package for supporting saving.

And `dialer`, `quickdialer`, et al. need to depend on such a package.

The same goes for errorwrapping.

This arrangement further complicates testing. For example, I am
currently working on https://github.com/ooni/probe/issues/1505 and
I realize it need to repeat integration tests in multiple places.

Let's say instead we pivot the above matrix as follows:

```
             | saving | errorwrapping | logging | ...
 dialer      |        |               |         | ...
 quicdialer  |        |               |         | ...
 logging     |        |               |         | ...
 mocking/sys |        |               |         | ...
 ...
```

In this way, now every row contains everything related to a specific
action to perform. We can now share code without relying on extra
support packages. What's more, we can write tests and, judding from
the way in which things are made, it seems we only need integration
testing in `errorwrapping` because it's where data quality matters
whereas, in all other cases, unit testing is fine.

I am going, therefore, to proceed with these changes and "pivot"
`netx`. Hopefully, it won't be too painful.
This commit is contained in:
Simone Basso
2021-06-23 15:53:12 +02:00
committed by GitHub
parent c74c94d616
commit 8a0beee808
27 changed files with 457 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/netx/dialer/dialer.go
+4 -3
View File
@@ -6,6 +6,7 @@ import (
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// Dialer establishes network connections.
@@ -64,10 +65,10 @@ type Config struct {
// New creates a new Dialer from the specified config and resolver.
func New(config *Config, resolver Resolver) Dialer {
var d Dialer = systemDialer
var d Dialer = netxlite.DefaultDialer
d = &errorWrapperDialer{Dialer: d}
if config.Logger != nil {
d = &loggingDialer{Dialer: d, Logger: config.Logger}
d = &netxlite.DialerLogger{Dialer: d, Logger: config.Logger}
}
if config.DialSaver != nil {
d = &saverDialer{Dialer: d, Saver: config.DialSaver}
@@ -75,7 +76,7 @@ func New(config *Config, resolver Resolver) Dialer {
if config.ReadWriteSaver != nil {
d = &saverConnDialer{Dialer: d, Saver: config.ReadWriteSaver}
}
d = &dnsDialer{Resolver: resolver, Dialer: d}
d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d}
d = &proxyDialer{ProxyURL: config.ProxyURL, Dialer: d}
if config.ContextByteCounting {
d = &byteCounterDialer{Dialer: d}
internal/engine/netx/dialer/dialer_test.go
+3 -2
View File
@@ -7,6 +7,7 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestNewCreatesTheExpectedChain(t *testing.T) {
@@ -30,7 +31,7 @@ func TestNewCreatesTheExpectedChain(t *testing.T) {
if !ok {
t.Fatal("not a proxyDialer")
}
dnsd, ok := pd.Dialer.(*dnsDialer)
dnsd, ok := pd.Dialer.(*netxlite.DialerResolver)
if !ok {
t.Fatal("not a dnsDialer")
}
@@ -42,7 +43,7 @@ func TestNewCreatesTheExpectedChain(t *testing.T) {
if !ok {
t.Fatal("not a saverDialer")
}
ld, ok := sd.Dialer.(*loggingDialer)
ld, ok := sd.Dialer.(*netxlite.DialerLogger)
if !ok {
t.Fatal("not a loggingDialer")
}
internal/engine/netx/dialer/dns.go
-71
View File
@@ -1,71 +0,0 @@
package dialer
import (
"context"
"errors"
"net"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
// dnsDialer is a dialer that uses the configured Resolver to resolver a
// domain name to IP addresses, and the configured Dialer to connect.
type dnsDialer struct {
Dialer
Resolver Resolver
}
// DialContext implements Dialer.DialContext.
func (d *dnsDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
onlyhost, onlyport, err := net.SplitHostPort(address)
if err != nil {
return nil, err
}
var addrs []string
addrs, err = d.lookupHost(ctx, onlyhost)
if err != nil {
return nil, err
}
var errorslist []error
for _, addr := range addrs {
target := net.JoinHostPort(addr, onlyport)
conn, err := d.Dialer.DialContext(ctx, network, target)
if err == nil {
return conn, nil
}
errorslist = append(errorslist, err)
}
return nil, ReduceErrors(errorslist)
}
// ReduceErrors finds a known error in a list of errors since it's probably most relevant.
func ReduceErrors(errorslist []error) error {
if len(errorslist) == 0 {
return nil
}
// If we have a known error, let's consider this the real error
// since it's probably most relevant. Otherwise let's return the
// first considering that (1) local resolvers likely will give
// us IPv4 first and (2) also our resolver does that. So, in case
// the user has no IPv6 connectivity, an IPv6 error is going to
// appear later in the list of errors.
for _, err := range errorslist {
var wrapper *errorx.ErrWrapper
if errors.As(err, &wrapper) && !strings.HasPrefix(
err.Error(), "unknown_failure",
) {
return err
}
}
// TODO(bassosimone): handle this case in a better way
return errorslist[0]
}
// lookupHost performs a domain name resolution.
func (d *dnsDialer) lookupHost(ctx context.Context, hostname string) ([]string, error) {
if net.ParseIP(hostname) != nil {
return []string{hostname}, nil
}
return d.Resolver.LookupHost(ctx, hostname)
}
internal/engine/netx/dialer/dns_test.go
-155
View File
@@ -1,155 +0,0 @@
package dialer
import (
"context"
"errors"
"io"
"net"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/mockablex"
)
func TestDNSDialerNoPort(t *testing.T) {
dialer := &dnsDialer{Dialer: new(net.Dialer), Resolver: new(net.Resolver)}
conn, err := dialer.DialContext(context.Background(), "tcp", "antani.ooni.nu")
if err == nil {
t.Fatal("expected an error here")
}
if conn != nil {
t.Fatal("expected a nil conn here")
}
}
func TestDNSDialerLookupHostAddress(t *testing.T) {
dialer := &dnsDialer{Dialer: new(net.Dialer), Resolver: MockableResolver{
Err: errors.New("mocked error"),
}}
addrs, err := dialer.lookupHost(context.Background(), "1.1.1.1")
if err != nil {
t.Fatal(err)
}
if len(addrs) != 1 || addrs[0] != "1.1.1.1" {
t.Fatal("not the result we expected")
}
}
func TestDNSDialerLookupHostFailure(t *testing.T) {
expected := errors.New("mocked error")
dialer := &dnsDialer{Dialer: new(net.Dialer), Resolver: MockableResolver{
Err: expected,
}}
conn, err := dialer.DialContext(context.Background(), "tcp", "dns.google.com:853")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if conn != nil {
t.Fatal("expected nil conn")
}
}
type MockableResolver struct {
Addresses []string
Err error
}
func (r MockableResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
return r.Addresses, r.Err
}
func TestDNSDialerDialForSingleIPFails(t *testing.T) {
dialer := &dnsDialer{Dialer: mockablex.Dialer{
MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
return nil, io.EOF
},
}, Resolver: new(net.Resolver)}
conn, err := dialer.DialContext(context.Background(), "tcp", "1.1.1.1:853")
if !errors.Is(err, io.EOF) {
t.Fatal("not the error we expected")
}
if conn != nil {
t.Fatal("expected nil conn")
}
}
func TestDNSDialerDialForManyIPFails(t *testing.T) {
dialer := &dnsDialer{
Dialer: mockablex.Dialer{
MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
return nil, io.EOF
},
}, Resolver: MockableResolver{
Addresses: []string{"1.1.1.1", "8.8.8.8"},
}}
conn, err := dialer.DialContext(context.Background(), "tcp", "dot.dns:853")
if !errors.Is(err, io.EOF) {
t.Fatal("not the error we expected")
}
if conn != nil {
t.Fatal("expected nil conn")
}
}
func TestDNSDialerDialForManyIPSuccess(t *testing.T) {
dialer := &dnsDialer{Dialer: mockablex.Dialer{
MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
return &mockablex.Conn{
MockClose: func() error {
return nil
},
}, nil
},
}, Resolver: MockableResolver{
Addresses: []string{"1.1.1.1", "8.8.8.8"},
}}
conn, err := dialer.DialContext(context.Background(), "tcp", "dot.dns:853")
if err != nil {
t.Fatal("expected nil error here")
}
if conn == nil {
t.Fatal("expected non-nil conn")
}
conn.Close()
}
func TestReduceErrors(t *testing.T) {
t.Run("no errors", func(t *testing.T) {
result := ReduceErrors(nil)
if result != nil {
t.Fatal("wrong result")
}
})
t.Run("single error", func(t *testing.T) {
err := errors.New("mocked error")
result := ReduceErrors([]error{err})
if result != err {
t.Fatal("wrong result")
}
})
t.Run("multiple errors", func(t *testing.T) {
err1 := errors.New("mocked error #1")
err2 := errors.New("mocked error #2")
result := ReduceErrors([]error{err1, err2})
if result.Error() != "mocked error #1" {
t.Fatal("wrong result")
}
})
t.Run("multiple errors with meaningful ones", func(t *testing.T) {
err1 := errors.New("mocked error #1")
err2 := &errorx.ErrWrapper{
Failure: "unknown_failure: antani",
}
err3 := &errorx.ErrWrapper{
Failure: errorx.FailureConnectionRefused,
}
err4 := errors.New("mocked error #3")
result := ReduceErrors([]error{err1, err2, err3, err4})
if result.Error() != errorx.FailureConnectionRefused {
t.Fatal("wrong result")
}
})
}
internal/engine/netx/dialer/logging.go
-23
View File
@@ -1,23 +0,0 @@
package dialer
import (
"context"
"net"
"time"
)
// loggingDialer is a Dialer with logging
type loggingDialer struct {
Dialer
Logger Logger
}
// DialContext implements Dialer.DialContext
func (d *loggingDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
d.Logger.Debugf("dial %s/%s...", address, network)
start := time.Now()
conn, err := d.Dialer.DialContext(ctx, network, address)
stop := time.Now()
d.Logger.Debugf("dial %s/%s... %+v in %s", address, network, err, stop.Sub(start))
return conn, err
}
internal/engine/netx/dialer/logging_test.go
-30
View File
@@ -1,30 +0,0 @@
package dialer
import (
"context"
"errors"
"io"
"net"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/netx/mockablex"
)
func TestLoggingDialerFailure(t *testing.T) {
d := &loggingDialer{
Dialer: mockablex.Dialer{
MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
return nil, io.EOF
},
},
Logger: log.Log,
}
conn, err := d.DialContext(context.Background(), "tcp", "www.google.com:443")
if !errors.Is(err, io.EOF) {
t.Fatal("not the error we expected")
}
if conn != nil {
t.Fatal("expected nil conn here")
}
}
internal/engine/netx/dialer/system.go
-12
View File
@@ -1,12 +0,0 @@
package dialer
import (
"net"
"time"
)
// systemDialer is the underlying net.Dialer.
var systemDialer = &net.Dialer{
Timeout: 15 * time.Second,
KeepAlive: 15 * time.Second,
}
internal/engine/netx/dialer/system_test.go
-28
View File
@@ -1,28 +0,0 @@
package dialer
import (
"strings"
"testing"
"time"
"github.com/ooni/psiphon/oopsi/golang.org/x/net/context"
)
func TestSystemDialerWorks(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
conn, err := systemDialer.DialContext(ctx, "tcp", "8.8.8.8:853")
if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
t.Fatal("not the error we expected", err)
}
if conn != nil {
t.Fatal("expected nil conn here")
}
}
func TestSystemDialerHasTimeout(t *testing.T) {
expected := 15 * time.Second
if systemDialer.Timeout != expected {
t.Fatal("unexpected timeout value")
}
}