chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
2021-02-02 12:05:47 +01:00
committed by GitHub
parent b1ce300c8d
commit d57c78bc71
535 changed files with 66182 additions and 23 deletions
@@ -0,0 +1,14 @@
// +build !go1.15
package quicdialer
import (
"crypto/tls"
"github.com/lucas-clemente/quic-go"
)
// ConnectionState returns the ConnectionState of a QUIC Session.
func ConnectionState(sess quic.EarlySession) tls.ConnectionState {
return tls.ConnectionState{}
}
@@ -0,0 +1,14 @@
// +build go1.15
package quicdialer
import (
"crypto/tls"
"github.com/lucas-clemente/quic-go"
)
// ConnectionState returns the ConnectionState of a QUIC Session.
func ConnectionState(sess quic.EarlySession) tls.ConnectionState {
return sess.ConnectionState().ConnectionState
}
+59
View File
@@ -0,0 +1,59 @@
package quicdialer
import (
"context"
"crypto/tls"
"net"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/dialid"
"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
)
// DNSDialer is a dialer that uses the configured Resolver to resolve a
// domain name to IP addresses
type DNSDialer struct {
Dialer ContextDialer
Resolver Resolver
}
// DialContext implements ContextDialer.DialContext
func (d DNSDialer) DialContext(
ctx context.Context, network, host string,
tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
onlyhost, onlyport, err := net.SplitHostPort(host)
if err != nil {
return nil, err
}
// TODO(kelmenhorst): Should this be somewhere else?
// failure if tlsCfg is nil but that should not happen
if tlsCfg.ServerName == "" {
tlsCfg.ServerName = onlyhost
}
ctx = dialid.WithDialID(ctx)
var addrs []string
addrs, err = d.LookupHost(ctx, onlyhost)
if err != nil {
return nil, err
}
var errorslist []error
for _, addr := range addrs {
target := net.JoinHostPort(addr, onlyport)
sess, err := d.Dialer.DialContext(
ctx, network, target, tlsCfg, cfg)
if err == nil {
return sess, nil
}
errorslist = append(errorslist, err)
}
// TODO(bassosimone): maybe ReduceErrors could be in netx/internal.
return nil, dialer.ReduceErrors(errorslist)
}
// LookupHost implements Resolver.LookupHost
func (d DNSDialer) LookupHost(ctx context.Context, hostname string) ([]string, error) {
if net.ParseIP(hostname) != nil {
return []string{hostname}, nil
}
return d.Resolver.LookupHost(ctx, hostname)
}
+142
View File
@@ -0,0 +1,142 @@
package quicdialer_test
import (
"context"
"crypto/tls"
"errors"
"net"
"strconv"
"strings"
"testing"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
)
type MockableResolver struct {
Addresses []string
Err error
}
func (r MockableResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
return r.Addresses, r.Err
}
func TestDNSDialerSuccess(t *testing.T) {
tlsConf := &tls.Config{NextProtos: []string{"h3-29"}}
dialer := quicdialer.DNSDialer{
Resolver: new(net.Resolver), Dialer: quicdialer.SystemDialer{}}
sess, err := dialer.DialContext(
context.Background(), "udp", "www.google.com:443",
tlsConf, &quic.Config{})
if err != nil {
t.Fatal("unexpected error", err)
}
if sess == nil {
t.Fatal("non nil sess expected")
}
}
func TestDNSDialerNoPort(t *testing.T) {
tlsConf := &tls.Config{NextProtos: []string{"h3-29"}}
dialer := quicdialer.DNSDialer{
Resolver: new(net.Resolver), Dialer: quicdialer.SystemDialer{}}
sess, err := dialer.DialContext(
context.Background(), "udp", "www.google.com",
tlsConf, &quic.Config{})
if err == nil {
t.Fatal("expected an error here")
}
if sess != nil {
t.Fatal("expected a nil sess here")
}
if err.Error() != "address www.google.com: missing port in address" {
t.Fatal("not the error we expected")
}
}
func TestDNSDialerLookupHostAddress(t *testing.T) {
dialer := quicdialer.DNSDialer{Resolver: MockableResolver{
Err: errors.New("mocked error"),
}}
addrs, err := dialer.LookupHost(context.Background(), "1.1.1.1")
if err != nil {
t.Fatal(err)
}
if len(addrs) != 1 || addrs[0] != "1.1.1.1" {
t.Fatal("not the result we expected")
}
}
func TestDNSDialerLookupHostFailure(t *testing.T) {
tlsConf := &tls.Config{NextProtos: []string{"h3-29"}}
expected := errors.New("mocked error")
dialer := quicdialer.DNSDialer{Resolver: MockableResolver{
Err: expected,
}}
sess, err := dialer.DialContext(
context.Background(), "udp", "dns.google.com:853",
tlsConf, &quic.Config{})
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if sess != nil {
t.Fatal("expected nil sess")
}
}
func TestDNSDialerInvalidPort(t *testing.T) {
tlsConf := &tls.Config{NextProtos: []string{"h3-29"}}
dialer := quicdialer.DNSDialer{
Resolver: new(net.Resolver), Dialer: quicdialer.SystemDialer{}}
sess, err := dialer.DialContext(
context.Background(), "udp", "www.google.com:0",
tlsConf, &quic.Config{})
if err == nil {
t.Fatal("expected an error here")
}
if sess != nil {
t.Fatal("expected nil sess")
}
if !strings.HasSuffix(err.Error(), "sendto: invalid argument") &&
!strings.HasSuffix(err.Error(), "sendto: can't assign requested address") {
t.Fatal("not the error we expected")
}
}
func TestDNSDialerInvalidPortSyntax(t *testing.T) {
tlsConf := &tls.Config{NextProtos: []string{"h3-29"}}
dialer := quicdialer.DNSDialer{
Resolver: new(net.Resolver), Dialer: quicdialer.SystemDialer{}}
sess, err := dialer.DialContext(
context.Background(), "udp", "www.google.com:port",
tlsConf, &quic.Config{})
if err == nil {
t.Fatal("expected an error here")
}
if sess != nil {
t.Fatal("expected nil sess")
}
if !errors.Is(err, strconv.ErrSyntax) {
t.Fatal("not the error we expected")
}
}
func TestDNSDialerDialEarlyFails(t *testing.T) {
tlsConf := &tls.Config{NextProtos: []string{"h3-29"}}
expected := errors.New("mocked DialEarly error")
dialer := quicdialer.DNSDialer{
Resolver: new(net.Resolver), Dialer: MockDialer{Err: expected}}
sess, err := dialer.DialContext(
context.Background(), "udp", "www.google.com:443",
tlsConf, &quic.Config{})
if err == nil {
t.Fatal("expected an error here")
}
if sess != nil {
t.Fatal("expected nil sess")
}
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
}
@@ -0,0 +1,34 @@
package quicdialer
import (
"context"
"crypto/tls"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/dialid"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
// ErrorWrapperDialer is a dialer that performs quic err wrapping
type ErrorWrapperDialer struct {
Dialer ContextDialer
}
// DialContext implements ContextDialer.DialContext
func (d ErrorWrapperDialer) DialContext(
ctx context.Context, network string, host string,
tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
dialID := dialid.ContextDialID(ctx)
sess, err := d.Dialer.DialContext(ctx, network, host, tlsCfg, cfg)
err = errorx.SafeErrWrapperBuilder{
// ConnID does not make any sense if we've failed and the error
// does not make any sense (and is nil) if we succeded.
DialID: dialID,
Error: err,
Operation: errorx.QUICHandshakeOperation,
}.MaybeBuild()
if err != nil {
return nil, err
}
return sess, nil
}
@@ -0,0 +1,61 @@
package quicdialer_test
import (
"context"
"crypto/tls"
"errors"
"io"
"testing"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/dialid"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
)
func TestErrorWrapperFailure(t *testing.T) {
ctx := dialid.WithDialID(context.Background())
d := quicdialer.ErrorWrapperDialer{
Dialer: MockDialer{Sess: nil, Err: io.EOF}}
sess, err := d.DialContext(
ctx, "udp", "www.google.com:443", &tls.Config{}, &quic.Config{})
if sess != nil {
t.Fatal("expected a nil sess here")
}
errorWrapperCheckErr(t, err, errorx.QUICHandshakeOperation)
}
func errorWrapperCheckErr(t *testing.T, err error, op string) {
if !errors.Is(err, io.EOF) {
t.Fatal("expected another error here")
}
var errWrapper *errorx.ErrWrapper
if !errors.As(err, &errWrapper) {
t.Fatal("cannot cast to ErrWrapper")
}
if errWrapper.DialID == 0 {
t.Fatal("unexpected DialID")
}
if errWrapper.Operation != op {
t.Fatal("unexpected Operation")
}
if errWrapper.Failure != errorx.FailureEOFError {
t.Fatal("unexpected failure")
}
}
func TestErrorWrapperSuccess(t *testing.T) {
ctx := dialid.WithDialID(context.Background())
tlsConf := &tls.Config{
NextProtos: []string{"h3-29"},
ServerName: "www.google.com",
}
d := quicdialer.ErrorWrapperDialer{Dialer: quicdialer.SystemDialer{}}
sess, err := d.DialContext(ctx, "udp", "216.58.212.164:443", tlsConf, &quic.Config{})
if err != nil {
t.Fatal(err)
}
if sess == nil {
t.Fatal("expected non-nil sess here")
}
}
@@ -0,0 +1,26 @@
package quicdialer
import (
"context"
"crypto/tls"
"github.com/lucas-clemente/quic-go"
)
// ContextDialer is a dialer for QUIC using Context.
type ContextDialer interface {
// Note: assumes that tlsCfg and cfg are not nil.
DialContext(ctx context.Context, network, host string,
tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error)
}
// Dialer dials QUIC connections.
type Dialer interface {
// Note: assumes that tlsCfg and cfg are not nil.
Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error)
}
// Resolver is the interface we expect from a resolver.
type Resolver interface {
LookupHost(ctx context.Context, hostname string) (addrs []string, err error)
}
+62
View File
@@ -0,0 +1,62 @@
package quicdialer
import (
"context"
"crypto/tls"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/internal/tlsx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
)
// HandshakeSaver saves events occurring during the handshake
type HandshakeSaver struct {
Saver *trace.Saver
Dialer ContextDialer
}
// DialContext implements ContextDialer.DialContext
func (h HandshakeSaver) DialContext(ctx context.Context, network string,
host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
start := time.Now()
// TODO(bassosimone): in the future we probably want to also save
// information about what versions we're willing to accept.
h.Saver.Write(trace.Event{
Address: host,
Name: "quic_handshake_start",
NoTLSVerify: tlsCfg.InsecureSkipVerify,
Proto: network,
TLSNextProtos: tlsCfg.NextProtos,
TLSServerName: tlsCfg.ServerName,
Time: start,
})
sess, err := h.Dialer.DialContext(ctx, network, host, tlsCfg, cfg)
stop := time.Now()
if err != nil {
h.Saver.Write(trace.Event{
Duration: stop.Sub(start),
Err: err,
Name: "quic_handshake_done",
NoTLSVerify: tlsCfg.InsecureSkipVerify,
TLSNextProtos: tlsCfg.NextProtos,
TLSServerName: tlsCfg.ServerName,
Time: stop,
})
return nil, err
}
state := ConnectionState(sess)
h.Saver.Write(trace.Event{
Duration: stop.Sub(start),
Name: "quic_handshake_done",
NoTLSVerify: tlsCfg.InsecureSkipVerify,
TLSCipherSuite: tlsx.CipherSuiteString(state.CipherSuite),
TLSNegotiatedProto: state.NegotiatedProtocol,
TLSNextProtos: tlsCfg.NextProtos,
TLSPeerCerts: trace.PeerCerts(state, err),
TLSServerName: tlsCfg.ServerName,
TLSVersion: tlsx.VersionString(state.Version),
Time: stop,
})
return sess, nil
}
@@ -0,0 +1,118 @@
package quicdialer_test
import (
"context"
"crypto/tls"
"reflect"
"strings"
"testing"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
)
type MockDialer struct {
Dialer quicdialer.ContextDialer
Sess quic.EarlySession
Err error
}
func (d MockDialer) DialContext(ctx context.Context, network, host string,
tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
if d.Dialer != nil {
return d.Dialer.DialContext(ctx, network, host, tlsCfg, cfg)
}
return d.Sess, d.Err
}
func TestHandshakeSaverSuccess(t *testing.T) {
nextprotos := []string{"h3-29"}
servername := "www.google.com"
tlsConf := &tls.Config{
NextProtos: nextprotos,
ServerName: servername,
}
saver := &trace.Saver{}
dlr := quicdialer.HandshakeSaver{
Dialer: quicdialer.SystemDialer{},
Saver: saver,
}
sess, err := dlr.DialContext(context.Background(), "udp",
"216.58.212.164:443", tlsConf, &quic.Config{})
if err != nil {
t.Fatal("unexpected error", err)
}
if sess == nil {
t.Fatal("unexpected nil sess")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("unexpected number of events")
}
if ev[0].Name != "quic_handshake_start" {
t.Fatal("unexpected Name")
}
if ev[0].TLSServerName != "www.google.com" {
t.Fatal("unexpected TLSServerName")
}
if !reflect.DeepEqual(ev[0].TLSNextProtos, nextprotos) {
t.Fatal("unexpected TLSNextProtos")
}
if ev[0].Time.After(time.Now()) {
t.Fatal("unexpected Time")
}
if ev[1].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[1].Err != nil {
t.Fatal("unexpected Err", ev[1].Err)
}
if ev[1].Name != "quic_handshake_done" {
t.Fatal("unexpected Name")
}
if !reflect.DeepEqual(ev[1].TLSNextProtos, nextprotos) {
t.Fatal("unexpected TLSNextProtos")
}
if ev[1].TLSServerName != "www.google.com" {
t.Fatal("unexpected TLSServerName")
}
if ev[1].Time.Before(ev[0].Time) {
t.Fatal("unexpected Time")
}
}
func TestHandshakeSaverHostNameError(t *testing.T) {
nextprotos := []string{"h3-29"}
servername := "wrong.host.badssl.com"
tlsConf := &tls.Config{
NextProtos: nextprotos,
ServerName: servername,
}
saver := &trace.Saver{}
dlr := quicdialer.HandshakeSaver{
Dialer: quicdialer.SystemDialer{},
Saver: saver,
}
sess, err := dlr.DialContext(context.Background(), "udp",
"216.58.212.164:443", tlsConf, &quic.Config{})
if err == nil {
t.Fatal("expected an error here")
}
if sess != nil {
t.Fatal("expected nil sess here")
}
for _, ev := range saver.Read() {
if ev.Name != "quic_handshake_done" {
continue
}
if ev.NoTLSVerify == true {
t.Fatal("expected NoTLSVerify to be false")
}
if !strings.Contains(ev.Err.Error(),
"certificate is valid for www.google.com, not "+servername) {
t.Fatal("unexpected error", ev.Err)
}
}
}
+87
View File
@@ -0,0 +1,87 @@
package quicdialer
import (
"context"
"crypto/tls"
"errors"
"net"
"strconv"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
)
// SystemDialer is the basic dialer for QUIC
type SystemDialer struct {
// Saver saves read/write events on the underlying UDP
// connection. (Implementation note: we need it here since
// this is the only part in the codebase that is able to
// observe the underlying UDP connection.)
Saver *trace.Saver
}
// DialContext implements ContextDialer.DialContext
func (d SystemDialer) DialContext(ctx context.Context, network string,
host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
onlyhost, onlyport, err := net.SplitHostPort(host)
port, err := strconv.Atoi(onlyport)
if err != nil {
return nil, err
}
ip := net.ParseIP(onlyhost)
if ip == nil {
// TODO(kelmenhorst): write test for this error condition.
return nil, errors.New("quicdialer: invalid IP representation")
}
udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
var pconn net.PacketConn = udpConn
if d.Saver != nil {
pconn = saverUDPConn{UDPConn: udpConn, saver: d.Saver}
}
udpAddr := &net.UDPAddr{IP: ip, Port: port, Zone: ""}
return quic.DialEarlyContext(ctx, pconn, udpAddr, host, tlsCfg, cfg)
}
type saverUDPConn struct {
*net.UDPConn
saver *trace.Saver
}
func (c saverUDPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
start := time.Now()
count, err := c.UDPConn.WriteTo(p, addr)
stop := time.Now()
c.saver.Write(trace.Event{
Address: addr.String(),
Data: p[:count],
Duration: stop.Sub(start),
Err: err,
NumBytes: count,
Name: errorx.WriteToOperation,
Time: stop,
})
return count, err
}
func (c saverUDPConn) ReadMsgUDP(b, oob []byte) (int, int, int, *net.UDPAddr, error) {
start := time.Now()
n, oobn, flags, addr, err := c.UDPConn.ReadMsgUDP(b, oob)
stop := time.Now()
var data []byte
if n > 0 {
data = b[:n]
}
c.saver.Write(trace.Event{
Address: addr.String(),
Data: data,
Duration: stop.Sub(start),
Err: err,
NumBytes: n,
Name: errorx.ReadFromOperation,
Time: stop,
})
return n, oobn, flags, addr, err
}
@@ -0,0 +1,75 @@
package quicdialer_test
import (
"context"
"crypto/tls"
"testing"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
)
func TestSystemDialerInvalidIPFailure(t *testing.T) {
tlsConf := &tls.Config{
NextProtos: []string{"h3-29"},
ServerName: "www.google.com",
}
saver := &trace.Saver{}
systemdialer := quicdialer.SystemDialer{
Saver: saver,
}
sess, err := systemdialer.DialContext(context.Background(), "udp", "a.b.c.d:0", tlsConf, &quic.Config{})
if err == nil {
t.Fatal("expected an error here")
}
if sess != nil {
t.Fatal("expected nil sess here")
}
if err.Error() != "quicdialer: invalid IP representation" {
t.Fatal("expected another error here")
}
}
func TestSystemDialerSuccessWithReadWrite(t *testing.T) {
// This is the most common use case for collecting reads, writes
tlsConf := &tls.Config{
NextProtos: []string{"h3-29"},
ServerName: "www.google.com",
}
saver := &trace.Saver{}
systemdialer := quicdialer.SystemDialer{Saver: saver}
_, err := systemdialer.DialContext(context.Background(), "udp",
"216.58.212.164:443", tlsConf, &quic.Config{})
if err != nil {
t.Fatal(err)
}
ev := saver.Read()
if len(ev) < 2 {
t.Fatal("unexpected number of events")
}
last := len(ev) - 1
for idx := 1; idx < last; idx++ {
if ev[idx].Data == nil {
t.Fatal("unexpected Data")
}
if ev[idx].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[idx].Err != nil {
t.Fatal("unexpected Err")
}
if ev[idx].NumBytes <= 0 {
t.Fatal("unexpected NumBytes")
}
switch ev[idx].Name {
case errorx.ReadFromOperation, errorx.WriteToOperation:
default:
t.Fatal("unexpected Name")
}
if ev[idx].Time.Before(ev[idx-1].Time) {
t.Fatal("unexpected Time")
}
}
}