diff --git a/internal/archival/dialer.go b/internal/archival/dialer.go new file mode 100644 index 0000000..4258ae4 --- /dev/null +++ b/internal/archival/dialer.go @@ -0,0 +1,86 @@ +package archival + +// +// Saves dial and net.Conn events +// + +import ( + "context" + "net" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// NetworkEvent contains a network event. This kind of events +// are generated by Dialer, QUICDialer, Conn, QUICConn. +type NetworkEvent struct { + Count int + Failure error + Finished time.Time + Network string + Operation string + RemoteAddr string + Started time.Time +} + +// DialContext dials with the given dialer with the given arguments +// and stores the dial result inside of this saver. +func (s *Saver) DialContext(ctx context.Context, + dialer model.Dialer, network, address string) (net.Conn, error) { + started := time.Now() + conn, err := dialer.DialContext(ctx, network, address) + s.appendNetworkEvent(&NetworkEvent{ + Count: 0, + Failure: err, + Finished: time.Now(), + Network: network, + Operation: netxlite.ConnectOperation, + RemoteAddr: address, + Started: started, + }) + return conn, err +} + +// Read reads from the given conn and stores the results in the saver. +func (s *Saver) Read(conn net.Conn, buf []byte) (int, error) { + network := conn.RemoteAddr().Network() + remoteAddr := conn.RemoteAddr().String() + started := time.Now() + count, err := conn.Read(buf) + s.appendNetworkEvent(&NetworkEvent{ + Count: count, + Failure: err, + Finished: time.Now(), + Network: network, + Operation: netxlite.ReadOperation, + RemoteAddr: remoteAddr, + Started: started, + }) + return count, err +} + +// Write writes to the given conn and stores the results into the saver. +func (s *Saver) Write(conn net.Conn, buf []byte) (int, error) { + network := conn.RemoteAddr().Network() + remoteAddr := conn.RemoteAddr().String() + started := time.Now() + count, err := conn.Write(buf) + s.appendNetworkEvent(&NetworkEvent{ + Count: count, + Failure: err, + Finished: time.Now(), + Network: network, + Operation: netxlite.WriteOperation, + RemoteAddr: remoteAddr, + Started: started, + }) + return count, err +} + +func (s *Saver) appendNetworkEvent(ev *NetworkEvent) { + s.mu.Lock() + s.trace.Network = append(s.trace.Network, ev) + s.mu.Unlock() +} diff --git a/internal/archival/dialer_test.go b/internal/archival/dialer_test.go new file mode 100644 index 0000000..9ebc246 --- /dev/null +++ b/internal/archival/dialer_test.go @@ -0,0 +1,281 @@ +package archival + +import ( + "context" + "errors" + "io" + "net" + "testing" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func TestSaverDialContext(t *testing.T) { + // newConn creates a new connection with the desired properties. + newConn := func(address string) net.Conn { + return &mocks.Conn{ + MockRemoteAddr: func() net.Addr { + return &mocks.Addr{ + MockString: func() string { + return address + }, + } + }, + MockClose: func() error { + return nil + }, + } + } + + // newDialer creates a dialer for testing. + newDialer := func(conn net.Conn, err error) model.Dialer { + return &mocks.Dialer{ + MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + time.Sleep(1 * time.Microsecond) + return conn, err + }, + } + } + + t.Run("on success", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + dialer := newDialer(newConn(mockedEndpoint), nil) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: 0, + ExpectedErr: nil, + ExpectedNetwork: "tcp", + ExpectedOp: netxlite.ConnectOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + ctx := context.Background() + conn, err := saver.DialContext(ctx, dialer, "tcp", mockedEndpoint) + if err != nil { + t.Fatal(err) + } + if conn == nil { + t.Fatal("expected non-nil conn") + } + conn.Close() + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + dialer := newDialer(nil, mockedError) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: 0, + ExpectedErr: mockedError, + ExpectedNetwork: "tcp", + ExpectedOp: netxlite.ConnectOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + ctx := context.Background() + conn, err := saver.DialContext(ctx, dialer, "tcp", mockedEndpoint) + if !errors.Is(err, mockedError) { + t.Fatal("unexpected err", err) + } + if conn != nil { + t.Fatal("expected nil conn") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSaverRead(t *testing.T) { + // newConn is a helper function for creating a new connection. + newConn := func(endpoint string, numBytes int, err error) net.Conn { + return &mocks.Conn{ + MockRead: func(b []byte) (int, error) { + time.Sleep(time.Microsecond) + return numBytes, err + }, + MockRemoteAddr: func() net.Addr { + return &mocks.Addr{ + MockString: func() string { + return endpoint + }, + MockNetwork: func() string { + return "tcp" + }, + } + }, + } + } + + t.Run("on success", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + const mockedNumBytes = 128 + conn := newConn(mockedEndpoint, mockedNumBytes, nil) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: mockedNumBytes, + ExpectedErr: nil, + ExpectedNetwork: "tcp", + ExpectedOp: netxlite.ReadOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + buf := make([]byte, 1024) + count, err := saver.Read(conn, buf) + if err != nil { + t.Fatal(err) + } + if count != mockedNumBytes { + t.Fatal("unexpected count") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + conn := newConn(mockedEndpoint, 0, mockedError) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: 0, + ExpectedErr: mockedError, + ExpectedNetwork: "tcp", + ExpectedOp: netxlite.ReadOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + buf := make([]byte, 1024) + count, err := saver.Read(conn, buf) + if !errors.Is(err, mockedError) { + t.Fatal("unexpected err", err) + } + if count != 0 { + t.Fatal("unexpected count") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSaverWrite(t *testing.T) { + // newConn is a helper function for creating a new connection. + newConn := func(endpoint string, numBytes int, err error) net.Conn { + return &mocks.Conn{ + MockWrite: func(b []byte) (int, error) { + time.Sleep(time.Microsecond) + return numBytes, err + }, + MockRemoteAddr: func() net.Addr { + return &mocks.Addr{ + MockString: func() string { + return endpoint + }, + MockNetwork: func() string { + return "tcp" + }, + } + }, + } + } + + t.Run("on success", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + const mockedNumBytes = 128 + conn := newConn(mockedEndpoint, mockedNumBytes, nil) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: mockedNumBytes, + ExpectedErr: nil, + ExpectedNetwork: "tcp", + ExpectedOp: netxlite.WriteOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + buf := make([]byte, 1024) + count, err := saver.Write(conn, buf) + if err != nil { + t.Fatal(err) + } + if count != mockedNumBytes { + t.Fatal("unexpected count") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + conn := newConn(mockedEndpoint, 0, mockedError) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: 0, + ExpectedErr: mockedError, + ExpectedNetwork: "tcp", + ExpectedOp: netxlite.WriteOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + buf := make([]byte, 1024) + count, err := saver.Write(conn, buf) + if !errors.Is(err, mockedError) { + t.Fatal("unexpected err", err) + } + if count != 0 { + t.Fatal("unexpected count") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) +} + +// SingleNetworkEventValidator expects to find a single +// network event inside of the saver and ensures that such +// an event contains the required field values. +type SingleNetworkEventValidator struct { + ExpectedCount int + ExpectedErr error + ExpectedNetwork string + ExpectedOp string + ExpectedEpnt string + Saver *Saver +} + +func (v *SingleNetworkEventValidator) Validate() error { + trace := v.Saver.MoveOutTrace() + if len(trace.Network) != 1 { + return errors.New("expected to see a single .Network event") + } + entry := trace.Network[0] + if entry.Count != v.ExpectedCount { + return errors.New("expected to see a different .Count") + } + if !errors.Is(entry.Failure, v.ExpectedErr) { + return errors.New("unexpected .Failure") + } + if !entry.Finished.After(entry.Started) { + return errors.New(".Finished should be after .Started") + } + if entry.Network != v.ExpectedNetwork { + return errors.New("invalid value for .Network") + } + if entry.Operation != v.ExpectedOp { + return errors.New("invalid value for .Operation") + } + if entry.RemoteAddr != v.ExpectedEpnt { + return errors.New("unexpected value for .RemoteAddr") + } + return nil +} diff --git a/internal/archival/doc.go b/internal/archival/doc.go new file mode 100644 index 0000000..690cafd --- /dev/null +++ b/internal/archival/doc.go @@ -0,0 +1,4 @@ +// Package implements a Saver type that saves network, TCP, DNS, +// and TLS events. Given a Saver, you can export a Trace. Given a +// Trace you can obtain data in the OONI archival data format. +package archival diff --git a/internal/archival/http.go b/internal/archival/http.go new file mode 100644 index 0000000..ff8890f --- /dev/null +++ b/internal/archival/http.go @@ -0,0 +1,107 @@ +package archival + +// +// Saves HTTP events +// + +import ( + "bytes" + "io" + "net/http" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// HTTPRoundTripEvent contains an HTTP round trip. +type HTTPRoundTripEvent struct { + Failure error + Finished time.Time + Method string + RequestHeaders http.Header + ResponseBody []byte + ResponseBodyIsTruncated bool + ResponseBodyLength int64 + ResponseHeaders http.Header + Started time.Time + StatusCode int64 + Transport string + URL string +} + +// HTTPRoundTrip performs the round trip with the given transport and +// the given arguments and saves the results into the saver. +// +// The maxBodySnapshotSize argument controls the maximum size of the +// body snapshot that we collect along with the HTTP round trip. +func (s *Saver) HTTPRoundTrip( + txp model.HTTPTransport, maxBodySnapshotSize int64, + req *http.Request) (*http.Response, error) { + started := time.Now() + resp, err := txp.RoundTrip(req) + rt := &HTTPRoundTripEvent{ + Failure: nil, // set later + Finished: time.Time{}, // set later + Method: req.Method, + RequestHeaders: s.cloneRequestHeaders(req), + ResponseBody: nil, // set later + ResponseBodyIsTruncated: false, + ResponseBodyLength: 0, + ResponseHeaders: nil, // set later + Started: started, + StatusCode: 0, // set later + Transport: txp.Network(), + URL: req.URL.String(), + } + if err != nil { + rt.Finished = time.Now() + rt.Failure = err + s.appendHTTPRoundTripEvent(rt) + return nil, err + } + rt.StatusCode = int64(resp.StatusCode) + rt.ResponseHeaders = resp.Header.Clone() + r := io.LimitReader(resp.Body, maxBodySnapshotSize) + body, err := netxlite.ReadAllContext(req.Context(), r) + if err != nil { + rt.Finished = time.Now() + rt.Failure = err + s.appendHTTPRoundTripEvent(rt) + return nil, err + } + resp.Body = &archivalHTTPTransportBody{ // allow for reading again the whole body + Reader: io.MultiReader(bytes.NewReader(body), resp.Body), + Closer: resp.Body, + } + rt.ResponseBody = body + rt.ResponseBodyLength = int64(len(body)) + rt.ResponseBodyIsTruncated = int64(len(body)) >= maxBodySnapshotSize + rt.Finished = time.Now() + s.appendHTTPRoundTripEvent(rt) + return resp, nil +} + +// cloneRequestHeaders ensure we include the Host header among the saved +// headers, which is what OONI should do, even though the Go transport is +// such that this header is added later when we're sending the request. +func (s *Saver) cloneRequestHeaders(req *http.Request) http.Header { + header := req.Header.Clone() + if req.Host != "" { + header.Set("Host", req.Host) + } else { + header.Set("Host", req.URL.Host) + } + return header +} + +type archivalHTTPTransportBody struct { + io.Reader + io.Closer +} + +func (s *Saver) appendHTTPRoundTripEvent(ev *HTTPRoundTripEvent) { + s.mu.Lock() + s.trace.HTTPRoundTrip = append(s.trace.HTTPRoundTrip, ev) + s.mu.Unlock() +} diff --git a/internal/archival/http_test.go b/internal/archival/http_test.go new file mode 100644 index 0000000..c08fd7b --- /dev/null +++ b/internal/archival/http_test.go @@ -0,0 +1,354 @@ +package archival + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func TestSaverHTTPRoundTrip(t *testing.T) { + // newHTTPTransport creates a new HTTP transport for testing. + newHTTPTransport := func(resp *http.Response, err error) model.HTTPTransport { + return &mocks.HTTPTransport{ + MockRoundTrip: func(req *http.Request) (*http.Response, error) { + return resp, err + }, + MockNetwork: func() string { + return "tcp" + }, + } + } + + // successFlowWithBody is a successful test case with possible body truncation. + successFlowWithBody := func(realBody []byte, maxBodySize int64) error { + // truncate the expected body if required + expectedBody := realBody + truncated := false + if int64(len(realBody)) > maxBodySize { + expectedBody = realBody[:maxBodySize] + truncated = true + } + // construct the saver and the validator + saver := NewSaver() + v := &SingleHTTPRoundTripValidator{ + ExpectFailure: nil, + ExpectMethod: "GET", + ExpectRequestHeaders: map[string][]string{ + "Host": {"127.0.0.1:8080"}, + "User-Agent": {"antani/1.0"}, + "X-Client-IP": {"130.192.91.211"}, + }, + ExpectResponseBody: expectedBody, + ExpectResponseBodyIsTruncated: truncated, + ExpectResponseBodyLength: int64(len(expectedBody)), + ExpectResponseHeaders: map[string][]string{ + "Server": {"antani/1.0"}, + "Content-Type": {"text/plain"}, + }, + ExpectStatusCode: 200, + ExpectTransport: "tcp", + ExpectURL: "http://127.0.0.1:8080/antani", + RealResponseBody: realBody, + Saver: saver, + } + // construct transport and perform the HTTP round trip + txp := newHTTPTransport(v.NewHTTPResponse(), nil) + resp, err := saver.HTTPRoundTrip(txp, maxBodySize, v.NewHTTPRequest()) + if err != nil { + return err + } + if resp == nil { + return errors.New("expected non-nil resp") + } + // ensure that we can still read the _full_ response body + ctx := context.Background() + data, err := netxlite.ReadAllContext(ctx, resp.Body) + if err != nil { + return err + } + if diff := cmp.Diff(realBody, data); diff != "" { + return errors.New(diff) + } + // validate the content of the trace + return v.Validate() + } + + t.Run("on success without truncation", func(t *testing.T) { + realBody := []byte("0xdeadbeef") + const maxBodySize = 1 << 20 + err := successFlowWithBody(realBody, maxBodySize) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("on success with truncation", func(t *testing.T) { + realBody := []byte("0xdeadbeef") + const maxBodySize = 4 + err := successFlowWithBody(realBody, maxBodySize) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure during round trip", func(t *testing.T) { + expectedError := netxlite.NewTopLevelGenericErrWrapper(netxlite.ECONNRESET) + const maxBodySize = 1 << 20 + saver := NewSaver() + v := &SingleHTTPRoundTripValidator{ + ExpectFailure: expectedError, + ExpectMethod: "GET", + ExpectRequestHeaders: map[string][]string{ + "Host": {"127.0.0.1:8080"}, + "User-Agent": {"antani/1.0"}, + "X-Client-IP": {"130.192.91.211"}, + }, + ExpectResponseBody: nil, + ExpectResponseBodyIsTruncated: false, + ExpectResponseBodyLength: 0, + ExpectResponseHeaders: nil, + ExpectStatusCode: 0, + ExpectTransport: "tcp", + ExpectURL: "http://127.0.0.1:8080/antani", + RealResponseBody: nil, + Saver: saver, + } + txp := newHTTPTransport(nil, expectedError) + resp, err := saver.HTTPRoundTrip(txp, maxBodySize, v.NewHTTPRequest()) + if !errors.Is(err, expectedError) { + t.Fatal(err) + } + if resp != nil { + t.Fatal("expected nil resp") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure reading body", func(t *testing.T) { + expectedError := netxlite.NewTopLevelGenericErrWrapper(netxlite.ECONNRESET) + const maxBodySize = 1 << 20 + saver := NewSaver() + v := &SingleHTTPRoundTripValidator{ + ExpectFailure: expectedError, + ExpectMethod: "GET", + ExpectRequestHeaders: map[string][]string{ + "Host": {"127.0.0.1:8080"}, + "User-Agent": {"antani/1.0"}, + "X-Client-IP": {"130.192.91.211"}, + }, + ExpectResponseBody: nil, + ExpectResponseBodyIsTruncated: false, + ExpectResponseBodyLength: 0, + ExpectResponseHeaders: map[string][]string{ + "Server": {"antani/1.0"}, + "Content-Type": {"text/plain"}, + }, + ExpectStatusCode: 200, + ExpectTransport: "tcp", + ExpectURL: "http://127.0.0.1:8080/antani", + RealResponseBody: nil, + Saver: saver, + } + resp := v.NewHTTPResponse() + // Hack the body so it returns a connection reset error + // after some useful piece of data. We do not see any + // body in the response or in the trace. We may possibly + // want to include all the body we could read into the + // trace in the future, but for now it seems fine to do + // exactly what the previous code was doing. + resp.Body = io.NopCloser(io.MultiReader( + bytes.NewReader([]byte("0xdeadbeef")), + &mocks.Reader{ + MockRead: func(b []byte) (int, error) { + return 0, expectedError + }, + }, + )) + txp := newHTTPTransport(resp, nil) + resp, err := saver.HTTPRoundTrip(txp, maxBodySize, v.NewHTTPRequest()) + if !errors.Is(err, expectedError) { + t.Fatal("unexpected err", err) + } + if resp != nil { + t.Fatal("expected nil resp") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("cloneRequestHeaders", func(t *testing.T) { + // doWithRequest is an helper function that creates a suitable + // round tripper and returns trace.HTTPRoundTrip[0] for inspection + doWithRequest := func(req *http.Request) (*HTTPRoundTripEvent, error) { + expect := errors.New("mocked err") + txp := newHTTPTransport(nil, expect) + saver := NewSaver() + const maxBodySize = 1 << 20 // irrelevant + resp, err := saver.HTTPRoundTrip(txp, maxBodySize, req) + if !errors.Is(err, expect) { + return nil, fmt.Errorf("unexpected error: %w", err) + } + if resp != nil { + return nil, errors.New("expected nil resp") + } + trace := saver.MoveOutTrace() + if len(trace.HTTPRoundTrip) != 1 { + return nil, errors.New("expected exactly one HTTPRoundTrip") + } + return trace.HTTPRoundTrip[0], nil + } + + t.Run("with req.URL.Host", func(t *testing.T) { + req, err := http.NewRequest("GET", "https://x.org/", nil) + if err != nil { + t.Fatal(err) + } + ev, err := doWithRequest(req) + if err != nil { + t.Fatal(err) + } + if ev.RequestHeaders.Get("Host") != "x.org" { + t.Fatal("unexpected request host") + } + }) + + t.Run("with req.Host", func(t *testing.T) { + req, err := http.NewRequest("GET", "https://x.org/", nil) + if err != nil { + t.Fatal(err) + } + req.Host = "google.com" + ev, err := doWithRequest(req) + if err != nil { + t.Fatal(err) + } + if ev.RequestHeaders.Get("Host") != "google.com" { + t.Fatal("unexpected request host") + } + }) + }) +} + +type SingleHTTPRoundTripValidator struct { + ExpectFailure error + ExpectMethod string + ExpectRequestHeaders http.Header + ExpectResponseBody []byte + ExpectResponseBodyIsTruncated bool + ExpectResponseBodyLength int64 + ExpectResponseHeaders http.Header + ExpectStatusCode int64 + ExpectTransport string + ExpectURL string + RealResponseBody []byte + Saver *Saver +} + +func (v *SingleHTTPRoundTripValidator) NewHTTPRequest() *http.Request { + parsedURL, err := url.Parse(v.ExpectURL) + runtimex.PanicOnError(err, "url.Parse should not fail here") + // The saving code clones the headers and adds the host header, which + // Go would instead add later. So, a realistic mock should not include + // such an header inside of the http.Request. + clonedHeaders := v.ExpectRequestHeaders.Clone() + clonedHeaders.Del("Host") + return &http.Request{ + Method: v.ExpectMethod, + URL: parsedURL, + Proto: "", + ProtoMajor: 0, + ProtoMinor: 0, + Header: clonedHeaders, + Body: nil, + GetBody: nil, + ContentLength: 0, + TransferEncoding: nil, + Close: false, + Host: "", + Form: nil, + PostForm: nil, + MultipartForm: nil, + Trailer: nil, + RemoteAddr: "", + RequestURI: "", + TLS: nil, + Cancel: nil, + Response: nil, + } +} + +func (v *SingleHTTPRoundTripValidator) NewHTTPResponse() *http.Response { + body := io.NopCloser(bytes.NewReader(v.RealResponseBody)) + return &http.Response{ + Status: http.StatusText(int(v.ExpectStatusCode)), + StatusCode: int(v.ExpectStatusCode), + Proto: "", + ProtoMajor: 0, + ProtoMinor: 0, + Header: v.ExpectResponseHeaders, + Body: body, + ContentLength: 0, + TransferEncoding: nil, + Close: false, + Uncompressed: false, + Trailer: nil, + Request: nil, + TLS: nil, + } +} + +func (v *SingleHTTPRoundTripValidator) Validate() error { + trace := v.Saver.MoveOutTrace() + if len(trace.HTTPRoundTrip) != 1 { + return errors.New("expected to see one event") + } + entry := trace.HTTPRoundTrip[0] + if !errors.Is(entry.Failure, v.ExpectFailure) { + return errors.New("unexpected .Failure") + } + if !entry.Finished.After(entry.Started) { + return errors.New(".Finished is not after .Started") + } + if entry.Method != v.ExpectMethod { + return errors.New("unexpected .Method") + } + if diff := cmp.Diff(v.ExpectRequestHeaders, entry.RequestHeaders); diff != "" { + return errors.New(diff) + } + if diff := cmp.Diff(v.ExpectResponseBody, entry.ResponseBody); diff != "" { + return errors.New(diff) + } + if entry.ResponseBodyIsTruncated != v.ExpectResponseBodyIsTruncated { + return errors.New("unexpected .ResponseBodyIsTruncated") + } + if entry.ResponseBodyLength != v.ExpectResponseBodyLength { + return errors.New("unexpected .ResponseBodyLength") + } + if diff := cmp.Diff(v.ExpectResponseHeaders, entry.ResponseHeaders); diff != "" { + return errors.New(diff) + } + if entry.StatusCode != v.ExpectStatusCode { + return errors.New("unexpected .StatusCode") + } + if entry.Transport != v.ExpectTransport { + return errors.New("unexpected .Transport") + } + if entry.URL != v.ExpectURL { + return errors.New("unexpected .URL") + } + return nil +} diff --git a/internal/archival/quic.go b/internal/archival/quic.go new file mode 100644 index 0000000..7cf3990 --- /dev/null +++ b/internal/archival/quic.go @@ -0,0 +1,95 @@ +package archival + +// +// Saves QUIC events. +// + +import ( + "context" + "crypto/tls" + "net" + "time" + + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// WriteTo performs WriteTo with the given pconn and saves the +// operation's results inside the saver. +func (s *Saver) WriteTo(pconn model.UDPLikeConn, buf []byte, addr net.Addr) (int, error) { + started := time.Now() + count, err := pconn.WriteTo(buf, addr) + s.appendNetworkEvent(&NetworkEvent{ + Count: count, + Failure: err, + Finished: time.Now(), + Network: addr.Network(), + Operation: netxlite.WriteToOperation, + RemoteAddr: addr.String(), + Started: started, + }) + return count, err +} + +// ReadFrom performs ReadFrom with the given pconn and saves the +// operation's results inside the saver. +func (s *Saver) ReadFrom(pconn model.UDPLikeConn, buf []byte) (int, net.Addr, error) { + started := time.Now() + count, addr, err := pconn.ReadFrom(buf) + s.appendNetworkEvent(&NetworkEvent{ + Count: count, + Failure: err, + Finished: time.Now(), + Network: "udp", // must be always set even on failure + Operation: netxlite.ReadFromOperation, + RemoteAddr: s.safeAddrString(addr), + Started: started, + }) + return count, addr, err +} + +func (s *Saver) safeAddrString(addr net.Addr) (out string) { + if addr != nil { + out = addr.String() + } + return +} + +// QUICDialContext dials a QUIC session using the given dialer +// and saves the results inside of the saver. +func (s *Saver) QUICDialContext(ctx context.Context, dialer model.QUICDialer, + network, address string, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlySession, error) { + started := time.Now() + var state tls.ConnectionState + sess, err := dialer.DialContext(ctx, network, address, tlsConfig, quicConfig) + if err == nil { + select { + case <-sess.HandshakeComplete().Done(): + state = sess.ConnectionState().TLS.ConnectionState + case <-ctx.Done(): + sess, err = nil, ctx.Err() + } + } + s.appendQUICHandshake(&QUICTLSHandshakeEvent{ + ALPN: tlsConfig.NextProtos, + CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), + Failure: err, + Finished: time.Now(), + NegotiatedProto: state.NegotiatedProtocol, + Network: "quic", + PeerCerts: s.tlsPeerCerts(err, &state), + RemoteAddr: address, + SNI: tlsConfig.ServerName, + SkipVerify: tlsConfig.InsecureSkipVerify, + Started: started, + TLSVersion: netxlite.TLSVersionString(state.Version), + }) + return sess, err +} + +func (s *Saver) appendQUICHandshake(ev *QUICTLSHandshakeEvent) { + s.mu.Lock() + s.trace.QUICHandshake = append(s.trace.QUICHandshake, ev) + s.mu.Unlock() +} diff --git a/internal/archival/quic_test.go b/internal/archival/quic_test.go new file mode 100644 index 0000000..f2237cc --- /dev/null +++ b/internal/archival/quic_test.go @@ -0,0 +1,463 @@ +package archival + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/lucas-clemente/quic-go" + "github.com/marten-seemann/qtls-go1-17" // it's annoying to depend on that + "github.com/ooni/probe-cli/v3/internal/fakefill" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func TestSaverWriteTo(t *testing.T) { + // newAddr creates an new net.Addr for testing. + newAddr := func(endpoint string) net.Addr { + return &mocks.Addr{ + MockString: func() string { + return endpoint + }, + MockNetwork: func() string { + return "udp" + }, + } + } + + // newConn is a helper function for creating a new connection. + newConn := func(numBytes int, err error) model.UDPLikeConn { + return &mocks.UDPLikeConn{ + MockWriteTo: func(p []byte, addr net.Addr) (int, error) { + time.Sleep(time.Microsecond) + return numBytes, err + }, + } + } + + t.Run("on success", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + const mockedNumBytes = 128 + addr := newAddr(mockedEndpoint) + conn := newConn(mockedNumBytes, nil) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: mockedNumBytes, + ExpectedErr: nil, + ExpectedNetwork: "udp", + ExpectedOp: netxlite.WriteToOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + buf := make([]byte, 1024) + count, err := saver.WriteTo(conn, buf, addr) + if err != nil { + t.Fatal(err) + } + if count != mockedNumBytes { + t.Fatal("invalid count") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + addr := newAddr(mockedEndpoint) + conn := newConn(0, mockedError) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: 0, + ExpectedErr: mockedError, + ExpectedNetwork: "udp", + ExpectedOp: netxlite.WriteToOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + buf := make([]byte, 1024) + count, err := saver.WriteTo(conn, buf, addr) + if !errors.Is(err, mockedError) { + t.Fatal("unexpected err", err) + } + if count != 0 { + t.Fatal("invalid count") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSaverReadFrom(t *testing.T) { + // newAddr creates an new net.Addr for testing. + newAddr := func(endpoint string) net.Addr { + return &mocks.Addr{ + MockString: func() string { + return endpoint + }, + MockNetwork: func() string { + return "udp" + }, + } + } + + // newConn is a helper function for creating a new connection. + newConn := func(numBytes int, addr net.Addr, err error) model.UDPLikeConn { + return &mocks.UDPLikeConn{ + MockReadFrom: func(p []byte) (int, net.Addr, error) { + time.Sleep(time.Microsecond) + return numBytes, addr, err + }, + } + } + + t.Run("on success", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + const mockedNumBytes = 128 + expectedAddr := newAddr(mockedEndpoint) + conn := newConn(mockedNumBytes, expectedAddr, nil) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: mockedNumBytes, + ExpectedErr: nil, + ExpectedNetwork: "udp", + ExpectedOp: netxlite.ReadFromOperation, + ExpectedEpnt: mockedEndpoint, + Saver: saver, + } + buf := make([]byte, 1024) + count, addr, err := saver.ReadFrom(conn, buf) + if err != nil { + t.Fatal(err) + } + if expectedAddr.Network() != addr.Network() { + t.Fatal("invalid addr.Network") + } + if expectedAddr.String() != addr.String() { + t.Fatal("invalid addr.String") + } + if count != mockedNumBytes { + t.Fatal("invalid count") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure", func(t *testing.T) { + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + conn := newConn(0, nil, mockedError) + saver := NewSaver() + v := &SingleNetworkEventValidator{ + ExpectedCount: 0, + ExpectedErr: mockedError, + ExpectedNetwork: "udp", + ExpectedOp: netxlite.ReadFromOperation, + ExpectedEpnt: "", + Saver: saver, + } + buf := make([]byte, 1024) + count, addr, err := saver.ReadFrom(conn, buf) + if !errors.Is(err, mockedError) { + t.Fatal(err) + } + if addr != nil { + t.Fatal("invalid addr") + } + if count != 0 { + t.Fatal("invalid count") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSaverQUICDialContext(t *testing.T) { + // newQUICDialer creates a new QUICDialer for testing. + newQUICDialer := func(sess quic.EarlySession, err error) model.QUICDialer { + return &mocks.QUICDialer{ + MockDialContext: func( + ctx context.Context, network, address string, tlsConfig *tls.Config, + quicConfig *quic.Config) (quic.EarlySession, error) { + time.Sleep(time.Microsecond) + return sess, err + }, + } + } + + // newQUICSession creates a new quic.EarlySession for testing. + newQUICSession := func(handshakeComplete context.Context, state tls.ConnectionState) quic.EarlySession { + return &mocks.QUICEarlySession{ + MockHandshakeComplete: func() context.Context { + return handshakeComplete + }, + MockConnectionState: func() quic.ConnectionState { + return quic.ConnectionState{ + TLS: qtls.ConnectionStateWith0RTT{ + ConnectionState: state, + }, + } + }, + MockCloseWithError: func(code quic.ApplicationErrorCode, reason string) error { + return nil + }, + } + } + + t.Run("on success", func(t *testing.T) { + handshakeCtx := context.Background() + handshakeCtx, handshakeCancel := context.WithCancel(handshakeCtx) + handshakeCancel() // simulate a completed handshake + const expectedNetwork = "udp" + const mockedEndpoint = "8.8.4.4:443" + saver := NewSaver() + var peerCerts [][]byte + ff := &fakefill.Filler{} + ff.Fill(&peerCerts) + if len(peerCerts) < 1 { + t.Fatal("did not fill peerCerts") + } + v := &SingleQUICTLSHandshakeValidator{ + ExpectedALPN: []string{"h3"}, + ExpectedSNI: "dns.google", + ExpectedSkipVerify: true, + // + ExpectedCipherSuite: tls.TLS_AES_128_GCM_SHA256, + ExpectedNegotiatedProtocol: "h3", + ExpectedPeerCerts: peerCerts, + ExpectedVersion: tls.VersionTLS13, + // + ExpectedNetwork: "quic", + ExpectedRemoteAddr: mockedEndpoint, + // + QUICConfig: &quic.Config{}, + // + ExpectedFailure: nil, + Saver: saver, + } + sess := newQUICSession(handshakeCtx, v.NewTLSConnectionState()) + dialer := newQUICDialer(sess, nil) + ctx := context.Background() + sess, err := saver.QUICDialContext(ctx, dialer, expectedNetwork, + mockedEndpoint, v.NewTLSConfig(), v.QUICConfig) + if err != nil { + t.Fatal(err) + } + if sess == nil { + t.Fatal("expected nil sess") + } + sess.CloseWithError(0, "") + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on handshake timeout", func(t *testing.T) { + handshakeCtx := context.Background() + handshakeCtx, handshakeCancel := context.WithCancel(handshakeCtx) + defer handshakeCancel() + const expectedNetwork = "udp" + const mockedEndpoint = "8.8.4.4:443" + saver := NewSaver() + v := &SingleQUICTLSHandshakeValidator{ + ExpectedALPN: []string{"h3"}, + ExpectedSNI: "dns.google", + ExpectedSkipVerify: true, + // + ExpectedCipherSuite: 0, + ExpectedNegotiatedProtocol: "", + ExpectedPeerCerts: nil, + ExpectedVersion: 0, + // + ExpectedNetwork: "quic", + ExpectedRemoteAddr: mockedEndpoint, + // + QUICConfig: &quic.Config{}, + // + ExpectedFailure: context.DeadlineExceeded, + Saver: saver, + } + sess := newQUICSession(handshakeCtx, tls.ConnectionState{}) + dialer := newQUICDialer(sess, nil) + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, time.Microsecond) + defer cancel() + sess, err := saver.QUICDialContext(ctx, dialer, expectedNetwork, + mockedEndpoint, v.NewTLSConfig(), v.QUICConfig) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("unexpected error") + } + if sess != nil { + t.Fatal("expected nil sess") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on other error", func(t *testing.T) { + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + const expectedNetwork = "udp" + const mockedEndpoint = "8.8.4.4:443" + saver := NewSaver() + v := &SingleQUICTLSHandshakeValidator{ + ExpectedALPN: []string{"h3"}, + ExpectedSNI: "dns.google", + ExpectedSkipVerify: true, + // + ExpectedCipherSuite: 0, + ExpectedNegotiatedProtocol: "", + ExpectedPeerCerts: nil, + ExpectedVersion: 0, + // + ExpectedNetwork: "quic", + ExpectedRemoteAddr: mockedEndpoint, + // + QUICConfig: &quic.Config{}, + // + ExpectedFailure: mockedError, + Saver: saver, + } + dialer := newQUICDialer(nil, mockedError) + ctx := context.Background() + sess, err := saver.QUICDialContext(ctx, dialer, expectedNetwork, + mockedEndpoint, v.NewTLSConfig(), v.QUICConfig) + if !errors.Is(err, mockedError) { + t.Fatal("unexpected error") + } + if sess != nil { + t.Fatal("expected nil sess") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + // TODO(bassosimone): here we're not testing the case in which + // the certificate is invalid for the required SNI. + // + // We need first to figure out whether this is what happens + // when we validate for QUIC in such cases. If that's the case + // indeed, then we can write the tests. + + t.Run("on x509.HostnameError", func(t *testing.T) { + t.Skip("test not implemented") + }) + + t.Run("on x509.UnknownAuthorityError", func(t *testing.T) { + t.Skip("test not implemented") + }) + + t.Run("on x509.CertificateInvalidError", func(t *testing.T) { + t.Skip("test not implemented") + }) +} + +type SingleQUICTLSHandshakeValidator struct { + // related to the tls.Config + ExpectedALPN []string + ExpectedSNI string + ExpectedSkipVerify bool + + // related to the tls.ConnectionState + ExpectedCipherSuite uint16 + ExpectedNegotiatedProtocol string + ExpectedPeerCerts [][]byte + ExpectedVersion uint16 + + // related to the mocked conn (TLS) / dial params (QUIC) + ExpectedNetwork string + ExpectedRemoteAddr string + + // tells us whether we're using QUIC + QUICConfig *quic.Config + + // other fields + ExpectedFailure error + Saver *Saver +} + +func (v *SingleQUICTLSHandshakeValidator) NewTLSConfig() *tls.Config { + return &tls.Config{ + NextProtos: v.ExpectedALPN, + ServerName: v.ExpectedSNI, + InsecureSkipVerify: v.ExpectedSkipVerify, + } +} + +func (v *SingleQUICTLSHandshakeValidator) NewTLSConnectionState() tls.ConnectionState { + var state tls.ConnectionState + if v.ExpectedCipherSuite != 0 { + state.CipherSuite = v.ExpectedCipherSuite + } + if v.ExpectedNegotiatedProtocol != "" { + state.NegotiatedProtocol = v.ExpectedNegotiatedProtocol + } + for _, cert := range v.ExpectedPeerCerts { + state.PeerCertificates = append(state.PeerCertificates, &x509.Certificate{ + Raw: cert, + }) + } + if v.ExpectedVersion != 0 { + state.Version = v.ExpectedVersion + } + return state +} + +func (v *SingleQUICTLSHandshakeValidator) Validate() error { + trace := v.Saver.MoveOutTrace() + var entries []*QUICTLSHandshakeEvent + if v.QUICConfig != nil { + entries = trace.QUICHandshake + } else { + entries = trace.TLSHandshake + } + if len(entries) != 1 { + return errors.New("expected to see a single entry") + } + entry := entries[0] + if diff := cmp.Diff(entry.ALPN, v.ExpectedALPN); diff != "" { + return errors.New(diff) + } + if entry.CipherSuite != netxlite.TLSCipherSuiteString(v.ExpectedCipherSuite) { + return errors.New("unexpected .CipherSuite") + } + if !errors.Is(entry.Failure, v.ExpectedFailure) { + return errors.New("unexpected .Failure") + } + if !entry.Finished.After(entry.Started) { + return errors.New(".Finished is not after .Started") + } + if entry.NegotiatedProto != v.ExpectedNegotiatedProtocol { + return errors.New("unexpected .NegotiatedProto") + } + if entry.Network != v.ExpectedNetwork { + return errors.New("unexpected .Network") + } + if diff := cmp.Diff(entry.PeerCerts, v.ExpectedPeerCerts); diff != "" { + return errors.New("unexpected .PeerCerts") + } + if entry.RemoteAddr != v.ExpectedRemoteAddr { + return errors.New("unexpected .RemoteAddr") + } + if entry.SNI != v.ExpectedSNI { + return errors.New("unexpected .ServerName") + } + if entry.SkipVerify != v.ExpectedSkipVerify { + return errors.New("unexpected .SkipVerify") + } + if entry.TLSVersion != netxlite.TLSVersionString(v.ExpectedVersion) { + return errors.New("unexpected .Version") + } + return nil +} diff --git a/internal/archival/resolver.go b/internal/archival/resolver.go new file mode 100644 index 0000000..7ad34e0 --- /dev/null +++ b/internal/archival/resolver.go @@ -0,0 +1,123 @@ +package archival + +// +// Saves DNS lookup events +// + +import ( + "context" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// DNSLookupEvent contains the results of a DNS lookup. +type DNSLookupEvent struct { + ALPNs []string + Addresses []string + Domain string + Failure error + Finished time.Time + LookupType string + ResolverAddress string + ResolverNetwork string + Started time.Time +} + +// LookupHost performs a host lookup with the given resolver +// and saves the results into the saver. +func (s *Saver) LookupHost(ctx context.Context, reso model.Resolver, domain string) ([]string, error) { + started := time.Now() + addrs, err := reso.LookupHost(ctx, domain) + s.appendLookupHostEvent(&DNSLookupEvent{ + ALPNs: nil, + Addresses: addrs, + Domain: domain, + Failure: err, + Finished: time.Now(), + LookupType: "getaddrinfo", + ResolverAddress: reso.Address(), + ResolverNetwork: reso.Network(), + Started: started, + }) + return addrs, err +} + +func (s *Saver) appendLookupHostEvent(ev *DNSLookupEvent) { + s.mu.Lock() + s.trace.DNSLookupHost = append(s.trace.DNSLookupHost, ev) + s.mu.Unlock() +} + +// LookupHTTPS performs an HTTPSSvc-record lookup using the given +// resolver and saves the results into the saver. +func (s *Saver) LookupHTTPS(ctx context.Context, reso model.Resolver, domain string) (*model.HTTPSSvc, error) { + started := time.Now() + https, err := reso.LookupHTTPS(ctx, domain) + s.appendLookupHTTPSEvent(&DNSLookupEvent{ + ALPNs: s.safeALPNs(https), + Addresses: s.safeAddresses(https), + Domain: domain, + Failure: err, + Finished: time.Now(), + LookupType: "https", + ResolverAddress: reso.Address(), + ResolverNetwork: reso.Network(), + Started: started, + }) + return https, err +} + +func (s *Saver) appendLookupHTTPSEvent(ev *DNSLookupEvent) { + s.mu.Lock() + s.trace.DNSLookupHTTPS = append(s.trace.DNSLookupHTTPS, ev) + s.mu.Unlock() +} + +func (s *Saver) safeALPNs(https *model.HTTPSSvc) (out []string) { + if https != nil { + out = https.ALPN + } + return +} + +func (s *Saver) safeAddresses(https *model.HTTPSSvc) (out []string) { + if https != nil { + out = append(out, https.IPv4...) + out = append(out, https.IPv6...) + } + return +} + +// DNSRoundTripEvent contains the result of a DNS round trip. +type DNSRoundTripEvent struct { + Address string + Failure error + Finished time.Time + Network string + Query []byte + Reply []byte + Started time.Time +} + +// DNSRoundTrip implements ArchivalSaver.DNSRoundTrip. +func (s *Saver) DNSRoundTrip(ctx context.Context, txp model.DNSTransport, query []byte) ([]byte, error) { + started := time.Now() + reply, err := txp.RoundTrip(ctx, query) + s.appendDNSRoundTripEvent(&DNSRoundTripEvent{ + Address: txp.Address(), + Failure: err, + Finished: time.Now(), + Network: txp.Network(), + Query: query, + Reply: reply, + Started: started, + }) + return reply, err +} + +func (s *Saver) appendDNSRoundTripEvent(ev *DNSRoundTripEvent) { + s.mu.Lock() + s.trace.DNSRoundTrip = append(s.trace.DNSRoundTrip, ev) + s.mu.Unlock() +} diff --git a/internal/archival/resolver_test.go b/internal/archival/resolver_test.go new file mode 100644 index 0000000..96d4807 --- /dev/null +++ b/internal/archival/resolver_test.go @@ -0,0 +1,347 @@ +package archival + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/fakefill" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func TestSaverLookupHost(t *testing.T) { + // newResolver helps to create a new resolver. + newResolver := func(addrs []string, err error) model.Resolver { + return &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return addrs, err + }, + MockAddress: func() string { + return "8.8.8.8:53" + }, + MockNetwork: func() string { + return "udp" + }, + } + } + + t.Run("on success", func(t *testing.T) { + const domain = "dns.google" + expectAddrs := []string{"8.8.8.8", "8.8.4.4"} + saver := NewSaver() + v := &SingleDNSLookupValidator{ + ExpectALPNs: nil, + ExpectAddrs: expectAddrs, + ExpectDomain: domain, + ExpectLookupType: "getaddrinfo", + ExpectFailure: nil, + ExpectResolverAddress: "8.8.8.8:53", + ExpectResolverNetwork: "udp", + Saver: saver, + } + reso := newResolver(expectAddrs, nil) + ctx := context.Background() + addrs, err := saver.LookupHost(ctx, reso, domain) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expectAddrs, addrs); diff != "" { + t.Fatal(diff) + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure", func(t *testing.T) { + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + const domain = "dns.google" + saver := NewSaver() + v := &SingleDNSLookupValidator{ + ExpectALPNs: nil, + ExpectAddrs: nil, + ExpectDomain: domain, + ExpectLookupType: "getaddrinfo", + ExpectFailure: mockedError, + ExpectResolverAddress: "8.8.8.8:53", + ExpectResolverNetwork: "udp", + Saver: saver, + } + reso := newResolver(nil, mockedError) + ctx := context.Background() + addrs, err := saver.LookupHost(ctx, reso, domain) + if !errors.Is(err, mockedError) { + t.Fatal("invalid err", err) + } + if len(addrs) != 0 { + t.Fatal("invalid addrs") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSaverLookupHTTPS(t *testing.T) { + // newResolver helps to create a new resolver. + newResolver := func(alpns, ipv4, ipv6 []string, err error) model.Resolver { + return &mocks.Resolver{ + MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) { + if alpns == nil && ipv4 == nil && ipv6 == nil { + return nil, err + } + return &model.HTTPSSvc{ + ALPN: alpns, + IPv4: ipv4, + IPv6: ipv6, + }, err + }, + MockAddress: func() string { + return "8.8.8.8:53" + }, + MockNetwork: func() string { + return "udp" + }, + } + } + + t.Run("on success", func(t *testing.T) { + const domain = "dns.google" + expectALPN := []string{"h3", "h2", "http/1.1"} + expectA := []string{"8.8.8.8", "8.8.4.4"} + expectAAAA := []string{"2001:4860:4860::8844"} + expectAddrs := append(expectA, expectAAAA...) + saver := NewSaver() + v := &SingleDNSLookupValidator{ + ExpectALPNs: expectALPN, + ExpectAddrs: expectAddrs, + ExpectDomain: domain, + ExpectLookupType: "https", + ExpectFailure: nil, + ExpectResolverAddress: "8.8.8.8:53", + ExpectResolverNetwork: "udp", + Saver: saver, + } + reso := newResolver(expectALPN, expectA, expectAAAA, nil) + ctx := context.Background() + https, err := saver.LookupHTTPS(ctx, reso, domain) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expectALPN, https.ALPN); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(expectA, https.IPv4); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(expectAAAA, https.IPv6); diff != "" { + t.Fatal(diff) + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure", func(t *testing.T) { + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + const domain = "dns.google" + saver := NewSaver() + v := &SingleDNSLookupValidator{ + ExpectALPNs: nil, + ExpectAddrs: nil, + ExpectDomain: domain, + ExpectLookupType: "https", + ExpectFailure: mockedError, + ExpectResolverAddress: "8.8.8.8:53", + ExpectResolverNetwork: "udp", + Saver: saver, + } + reso := newResolver(nil, nil, nil, mockedError) + ctx := context.Background() + https, err := saver.LookupHTTPS(ctx, reso, domain) + if !errors.Is(err, mockedError) { + t.Fatal("unexpected err", err) + } + if https != nil { + t.Fatal("expected nil https") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) +} + +type SingleDNSLookupValidator struct { + ExpectALPNs []string + ExpectAddrs []string + ExpectDomain string + ExpectLookupType string + ExpectFailure error + ExpectResolverAddress string + ExpectResolverNetwork string + Saver *Saver +} + +func (v *SingleDNSLookupValidator) Validate() error { + trace := v.Saver.MoveOutTrace() + var entries []*DNSLookupEvent + switch v.ExpectLookupType { + case "getaddrinfo": + entries = trace.DNSLookupHost + case "https": + entries = trace.DNSLookupHTTPS + default: + return errors.New("invalid v.ExpectLookupType") + } + if len(entries) != 1 { + return errors.New("expected a single entry") + } + entry := entries[0] + if diff := cmp.Diff(v.ExpectALPNs, entry.ALPNs); diff != "" { + return errors.New(diff) + } + if diff := cmp.Diff(v.ExpectAddrs, entry.Addresses); diff != "" { + return errors.New(diff) + } + if v.ExpectDomain != entry.Domain { + return errors.New("invalid .Domain value") + } + if !errors.Is(entry.Failure, v.ExpectFailure) { + return errors.New("invalid .Failure value") + } + if !entry.Finished.After(entry.Started) { + return errors.New(".Finished is not after .Started") + } + if entry.ResolverAddress != v.ExpectResolverAddress { + return errors.New("invalid .ResolverAddress value") + } + if entry.ResolverNetwork != v.ExpectResolverNetwork { + return errors.New("invalid .ResolverNetwork value") + } + return nil +} + +func TestSaverDNSRoundTrip(t *testing.T) { + // generateQueryAndReply generates a fake query and reply. + generateQueryAndReply := func() (query, reply []byte, err error) { + ff := &fakefill.Filler{} + ff.Fill(&query) + ff.Fill(&reply) + if len(query) < 1 || len(reply) < 1 { + return nil, nil, errors.New("did not generate query or reply") + } + return query, reply, nil + } + + // newDNSTransport creates a suitable DNSTransport. + newDNSTransport := func(reply []byte, err error) model.DNSTransport { + return &mocks.DNSTransport{ + MockRoundTrip: func(ctx context.Context, query []byte) ([]byte, error) { + return reply, err + }, + MockNetwork: func() string { + return "udp" + }, + MockAddress: func() string { + return "8.8.8.8:53" + }, + } + } + + t.Run("on success", func(t *testing.T) { + query, expectedReply, err := generateQueryAndReply() + if err != nil { + t.Fatal(err) + } + saver := NewSaver() + v := &SingleDNSRoundTripValidator{ + ExpectAddress: "8.8.8.8:53", + ExpectFailure: nil, + ExpectNetwork: "udp", + ExpectQuery: query, + ExpectReply: expectedReply, + Saver: saver, + } + ctx := context.Background() + txp := newDNSTransport(expectedReply, nil) + reply, err := saver.DNSRoundTrip(ctx, txp, query) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expectedReply, reply); diff != "" { + t.Fatal(diff) + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + t.Run("on failure", func(t *testing.T) { + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + query, _, err := generateQueryAndReply() + if err != nil { + t.Fatal(err) + } + saver := NewSaver() + v := &SingleDNSRoundTripValidator{ + ExpectAddress: "8.8.8.8:53", + ExpectFailure: mockedError, + ExpectNetwork: "udp", + ExpectQuery: query, + ExpectReply: nil, + Saver: saver, + } + ctx := context.Background() + txp := newDNSTransport(nil, mockedError) + reply, err := saver.DNSRoundTrip(ctx, txp, query) + if !errors.Is(err, mockedError) { + t.Fatal(err) + } + if len(reply) != 0 { + t.Fatal("unexpected reply") + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) +} + +type SingleDNSRoundTripValidator struct { + ExpectAddress string + ExpectFailure error + ExpectNetwork string + ExpectQuery []byte + ExpectReply []byte + Saver *Saver +} + +func (v *SingleDNSRoundTripValidator) Validate() error { + trace := v.Saver.MoveOutTrace() + if len(trace.DNSRoundTrip) != 1 { + return errors.New("expected a single entry") + } + entry := trace.DNSRoundTrip[0] + if v.ExpectAddress != entry.Address { + return errors.New("invalid .Address") + } + if !errors.Is(entry.Failure, v.ExpectFailure) { + return errors.New("invalid .Failure value") + } + if !entry.Finished.After(entry.Started) { + return errors.New(".Finished is not after .Started") + } + if v.ExpectNetwork != entry.Network { + return errors.New("invalid .Network value") + } + if diff := cmp.Diff(v.ExpectQuery, entry.Query); diff != "" { + return errors.New(diff) + } + if diff := cmp.Diff(v.ExpectReply, entry.Reply); diff != "" { + return errors.New(diff) + } + return nil +} diff --git a/internal/archival/saver.go b/internal/archival/saver.go new file mode 100644 index 0000000..2fcab4f --- /dev/null +++ b/internal/archival/saver.go @@ -0,0 +1,40 @@ +package archival + +// +// Saver implementation +// + +import ( + "sync" +) + +// Saver allows to save network, DNS, QUIC, TLS, HTTP events. +// +// You MUST use NewSaver to create a new instance. +type Saver struct { + // mu provides mutual exclusion. + mu sync.Mutex + + // trace is the current trace. + trace *Trace +} + +// NewSaver creates a new Saver instance. +// +// You MUST use this function to create a Saver. +func NewSaver() *Saver { + return &Saver{ + mu: sync.Mutex{}, + trace: &Trace{}, + } +} + +// MoveOutTrace moves the current trace out of the saver and +// creates a new empty trace inside it. +func (as *Saver) MoveOutTrace() *Trace { + as.mu.Lock() + t := as.trace + as.trace = &Trace{} + as.mu.Unlock() + return t +} diff --git a/internal/archival/saver_test.go b/internal/archival/saver_test.go new file mode 100644 index 0000000..0c1ed99 --- /dev/null +++ b/internal/archival/saver_test.go @@ -0,0 +1,37 @@ +package archival + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/fakefill" +) + +func TestSaverNewSaver(t *testing.T) { + saver := NewSaver() + if saver.trace == nil { + t.Fatal("expected non-nil trace here") + } +} + +func TestSaverMoveOutTrace(t *testing.T) { + saver := NewSaver() + var ev DNSRoundTripEvent + ff := &fakefill.Filler{} + ff.Fill(&ev) + if len(ev.Query) < 1 { + t.Fatal("did not fill") // be sure + } + saver.appendDNSRoundTripEvent(&ev) + trace := saver.MoveOutTrace() + if len(saver.trace.DNSRoundTrip) != 0 { + t.Fatal("expected zero length") + } + if len(trace.DNSRoundTrip) != 1 { + t.Fatal("expected one entry") + } + entry := trace.DNSRoundTrip[0] + if diff := cmp.Diff(&ev, entry); diff != "" { + t.Fatal(diff) + } +} diff --git a/internal/archival/tls.go b/internal/archival/tls.go new file mode 100644 index 0000000..cf2b3a5 --- /dev/null +++ b/internal/archival/tls.go @@ -0,0 +1,89 @@ +package archival + +// +// Saves TLS events +// + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "net" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// QUICTLSHandshakeEvent contains a QUIC or TLS handshake event. +type QUICTLSHandshakeEvent struct { + ALPN []string + CipherSuite string + Failure error + Finished time.Time + NegotiatedProto string + Network string + PeerCerts [][]byte + RemoteAddr string + SNI string + SkipVerify bool + Started time.Time + TLSVersion string +} + +// TLSHandshake performs a TLS handshake with the given handshaker +// and saves the results into the saver. +func (s *Saver) TLSHandshake(ctx context.Context, thx model.TLSHandshaker, + conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { + network := conn.RemoteAddr().Network() + remoteAddr := conn.RemoteAddr().String() + started := time.Now() + tconn, state, err := thx.Handshake(ctx, conn, config) + // Implementation note: state is an empty ConnectionState on failure + // so it's safe to access its fields also in that case + s.appendTLSHandshake(&QUICTLSHandshakeEvent{ + ALPN: config.NextProtos, + CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), + Failure: err, + Finished: time.Now(), + NegotiatedProto: state.NegotiatedProtocol, + Network: network, + PeerCerts: s.tlsPeerCerts(err, &state), + RemoteAddr: remoteAddr, + SNI: config.ServerName, + SkipVerify: config.InsecureSkipVerify, + Started: started, + TLSVersion: netxlite.TLSVersionString(state.Version), + }) + return tconn, state, err +} + +func (s *Saver) appendTLSHandshake(ev *QUICTLSHandshakeEvent) { + s.mu.Lock() + s.trace.TLSHandshake = append(s.trace.TLSHandshake, ev) + s.mu.Unlock() +} + +func (s *Saver) tlsPeerCerts(err error, state *tls.ConnectionState) (out [][]byte) { + var x509HostnameError x509.HostnameError + if errors.As(err, &x509HostnameError) { + // Test case: https://wrong.host.badssl.com/ + return [][]byte{x509HostnameError.Certificate.Raw} + } + var x509UnknownAuthorityError x509.UnknownAuthorityError + if errors.As(err, &x509UnknownAuthorityError) { + // Test case: https://self-signed.badssl.com/. This error has + // never been among the ones returned by MK. + return [][]byte{x509UnknownAuthorityError.Cert.Raw} + } + var x509CertificateInvalidError x509.CertificateInvalidError + if errors.As(err, &x509CertificateInvalidError) { + // Test case: https://expired.badssl.com/ + return [][]byte{x509CertificateInvalidError.Cert.Raw} + } + for _, cert := range state.PeerCertificates { + out = append(out, cert.Raw) + } + return +} diff --git a/internal/archival/tls_test.go b/internal/archival/tls_test.go new file mode 100644 index 0000000..ab9d602 --- /dev/null +++ b/internal/archival/tls_test.go @@ -0,0 +1,178 @@ +package archival + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/ooni/probe-cli/v3/internal/fakefill" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func TestSaverTLSHandshake(t *testing.T) { + // newTLSHandshaker helps with building a TLS handshaker + newTLSHandshaker := func(tlsConn net.Conn, state tls.ConnectionState, err error) model.TLSHandshaker { + return &mocks.TLSHandshaker{ + MockHandshake: func(ctx context.Context, tcpConn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { + time.Sleep(1 * time.Microsecond) + return tlsConn, state, err + }, + } + } + + // newTCPConn creates a suitable net.Conn + newTCPConn := func(address string) net.Conn { + return &mocks.Conn{ + MockRemoteAddr: func() net.Addr { + return &mocks.Addr{ + MockString: func() string { + return address + }, + MockNetwork: func() string { + return "tcp" + }, + } + }, + MockClose: func() error { + return nil + }, + } + } + + t.Run("on success", func(t *testing.T) { + const mockedEndpoint = "8.8.4.4:443" + var certs [][]byte + ff := &fakefill.Filler{} + ff.Fill(&certs) + if len(certs) < 1 { + t.Fatal("did not fill certs") + } + saver := NewSaver() + v := &SingleQUICTLSHandshakeValidator{ + ExpectedALPN: []string{"h2", "http/1.1"}, + ExpectedSNI: "dns.google", + ExpectedSkipVerify: true, + // + ExpectedCipherSuite: tls.TLS_AES_128_GCM_SHA256, + ExpectedNegotiatedProtocol: "h2", + ExpectedPeerCerts: certs, + ExpectedVersion: tls.VersionTLS12, + // + ExpectedNetwork: "tcp", + ExpectedRemoteAddr: mockedEndpoint, + // + QUICConfig: nil, // this is not QUIC + ExpectedFailure: nil, + Saver: saver, + } + expectedState := v.NewTLSConnectionState() + thx := newTLSHandshaker(newTCPConn(mockedEndpoint), expectedState, nil) + ctx := context.Background() + tcpConn := newTCPConn(mockedEndpoint) + conn, state, err := saver.TLSHandshake(ctx, thx, tcpConn, v.NewTLSConfig()) + if conn == nil { + t.Fatal("expected non-nil conn") + } + conn.Close() + if diff := cmp.Diff(expectedState, state, cmpopts.IgnoreUnexported(tls.ConnectionState{})); diff != "" { + t.Fatal(diff) + } + if err != nil { + t.Fatal(err) + } + if err := v.Validate(); err != nil { + t.Fatal(err) + } + }) + + // failureFlow is the flow we run on failure. + failureFlow := func(mockedError error, peerCerts [][]byte) error { + const mockedEndpoint = "8.8.4.4:443" + saver := NewSaver() + v := &SingleQUICTLSHandshakeValidator{ + ExpectedALPN: []string{"h2", "http/1.1"}, + ExpectedSNI: "dns.google", + ExpectedSkipVerify: true, + // + ExpectedCipherSuite: 0, + ExpectedNegotiatedProtocol: "", + ExpectedPeerCerts: peerCerts, + ExpectedVersion: 0, + // + ExpectedNetwork: "tcp", + ExpectedRemoteAddr: mockedEndpoint, + // + QUICConfig: nil, // this is not QUIC + ExpectedFailure: mockedError, + Saver: saver, + } + expectedState := v.NewTLSConnectionState() + thx := newTLSHandshaker(nil, expectedState, mockedError) + ctx := context.Background() + tcpConn := newTCPConn(mockedEndpoint) + conn, state, err := saver.TLSHandshake(ctx, thx, tcpConn, v.NewTLSConfig()) + if conn != nil { + return errors.New("expected nil conn") + } + if diff := cmp.Diff(expectedState, state, cmpopts.IgnoreUnexported(tls.ConnectionState{})); diff != "" { + return errors.New(diff) + } + if !errors.Is(err, mockedError) { + return fmt.Errorf("unexpected err: %w", err) + } + return v.Validate() + } + + t.Run("on generic failure", func(t *testing.T) { + mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF) + if err := failureFlow(mockedError, nil); err != nil { + t.Fatal(err) + } + }) + + t.Run("on x509.HostnameError", func(t *testing.T) { + var certificate []byte + ff := &fakefill.Filler{} + ff.Fill(&certificate) + mockedError := x509.HostnameError{ + Certificate: &x509.Certificate{Raw: certificate}, + } + if err := failureFlow(mockedError, [][]byte{certificate}); err != nil { + t.Fatal(err) + } + }) + + t.Run("on x509.UnknownAuthorityError", func(t *testing.T) { + var certificate []byte + ff := &fakefill.Filler{} + ff.Fill(&certificate) + mockedError := x509.UnknownAuthorityError{ + Cert: &x509.Certificate{Raw: certificate}, + } + if err := failureFlow(mockedError, [][]byte{certificate}); err != nil { + t.Fatal(err) + } + }) + + t.Run("on x509.CertificateInvalidError", func(t *testing.T) { + var certificate []byte + ff := &fakefill.Filler{} + ff.Fill(&certificate) + mockedError := x509.CertificateInvalidError{ + Cert: &x509.Certificate{Raw: certificate}, + } + if err := failureFlow(mockedError, [][]byte{certificate}); err != nil { + t.Fatal(err) + } + }) +} diff --git a/internal/archival/trace.go b/internal/archival/trace.go new file mode 100644 index 0000000..3c807ab --- /dev/null +++ b/internal/archival/trace.go @@ -0,0 +1,283 @@ +package archival + +import ( + "net" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/ooni/probe-cli/v3/internal/engine/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// +// Trace implementation +// + +// Trace contains the events. +type Trace struct { + // DNSLookupHTTPS contains DNSLookupHTTPS events. + DNSLookupHTTPS []*DNSLookupEvent + + // DNSLookupHost contains DNSLookupHost events. + DNSLookupHost []*DNSLookupEvent + + // DNSRoundTrip contains DNSRoundTrip events. + DNSRoundTrip []*DNSRoundTripEvent + + // HTTPRoundTrip contains HTTPRoundTrip round trip events. + HTTPRoundTrip []*HTTPRoundTripEvent + + // Network contains network events. + Network []*NetworkEvent + + // QUICHandshake contains QUICHandshake handshake events. + QUICHandshake []*QUICTLSHandshakeEvent + + // TLSHandshake contains TLSHandshake handshake events. + TLSHandshake []*QUICTLSHandshakeEvent +} + +func (t *Trace) newFailure(err error) (out *string) { + if err != nil { + s := err.Error() + out = &s + } + return +} + +// +// TCP connect +// + +// NewArchivalTCPConnectResultList builds a TCP connect list in the OONI archival +// data format out of the results saved inside the trace. +func (t *Trace) NewArchivalTCPConnectResultList(begin time.Time) (out []model.ArchivalTCPConnectResult) { + for _, ev := range t.Network { + if ev.Operation != netxlite.ConnectOperation || ev.Network != "tcp" { + continue + } + // We assume Go is passing us legit data structures + ip, sport, _ := net.SplitHostPort(ev.RemoteAddr) + iport, _ := strconv.Atoi(sport) + out = append(out, model.ArchivalTCPConnectResult{ + IP: ip, + Port: iport, + Status: model.ArchivalTCPConnectStatus{ + Blocked: nil, // Web Connectivity only, depends on the control + Failure: t.newFailure(ev.Failure), + Success: ev.Failure == nil, + }, + T: ev.Finished.Sub(begin).Seconds(), + }) + } + return +} + +// +// HTTP +// + +// NewArchivalHTTPRequestResultList builds an HTTP requests list in the OONI +// archival data format out of the results saved inside the trace. +// +// This function will sort the emitted list of requests such that the last +// request that happened in time is the first one to be emitted. If the +// measurement code performs related requests sequentially (which is a kinda a +// given because you cannot follow a redirect before reading the previous request), +// then the result is sorted how the OONI pipeline expects it to be. +func (t *Trace) NewArchivalHTTPRequestResultList(begin time.Time) (out []model.ArchivalHTTPRequestResult) { + for _, ev := range t.HTTPRoundTrip { + out = append(out, model.ArchivalHTTPRequestResult{ + Failure: t.newFailure(ev.Failure), + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalMaybeBinaryData{}, + BodyIsTruncated: false, + HeadersList: t.newHTTPHeadersList(ev.RequestHeaders), + Headers: t.newHTTPHeadersMap(ev.RequestHeaders), + Method: ev.Method, + Tor: model.ArchivalHTTPTor{}, + Transport: ev.Transport, + URL: ev.URL, + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalMaybeBinaryData{ + Value: string(ev.ResponseBody), + }, + BodyIsTruncated: ev.ResponseBodyIsTruncated, + Code: ev.StatusCode, + HeadersList: t.newHTTPHeadersList(ev.ResponseHeaders), + Headers: t.newHTTPHeadersMap(ev.ResponseHeaders), + Locations: ev.ResponseHeaders.Values("Location"), // safe with nil headers + }, + T: ev.Finished.Sub(begin).Seconds(), + }) + } + // Implementation note: historically OONI has always added + // the _last_ measurement in _first_ position. This has only + // been relevant for sequentially performed requests. For + // this purpose it feels okay to use T as the sorting key, + // since it's the time when we exited RoundTrip(). + sort.Slice(out, func(i, j int) bool { + return out[i].T > out[j].T + }) + return +} + +func (t *Trace) newHTTPHeadersList(source http.Header) (out []model.ArchivalHTTPHeader) { + for key, values := range source { + for _, value := range values { + out = append(out, model.ArchivalHTTPHeader{ + Key: key, + Value: model.ArchivalMaybeBinaryData{ + Value: value, + }, + }) + } + } + // Implementation note: we need to sort the keys to have + // stable testing since map iteration is random. + sort.Slice(out, func(i, j int) bool { + return out[i].Key < out[j].Key + }) + return +} + +func (t *Trace) newHTTPHeadersMap(source http.Header) (out map[string]model.ArchivalMaybeBinaryData) { + for key, values := range source { + for index, value := range values { + if index > 0 { + break // only the first entry + } + if out == nil { + out = make(map[string]model.ArchivalMaybeBinaryData) + } + out[key] = model.ArchivalMaybeBinaryData{Value: value} + } + } + return +} + +// +// DNS +// + +// NewArchivalDNSLookupResultList builds a DNS lookups list in the OONI +// archival data format out of the results saved inside the trace. +func (t *Trace) NewArchivalDNSLookupResultList(begin time.Time) (out []model.ArchivalDNSLookupResult) { + for _, ev := range t.DNSLookupHost { + out = append(out, model.ArchivalDNSLookupResult{ + Answers: t.gatherA(ev.Addresses), + Engine: ev.ResolverNetwork, + Failure: t.newFailure(ev.Failure), + Hostname: ev.Domain, + QueryType: "A", + ResolverHostname: nil, // legacy + ResolverPort: nil, // legacy + ResolverAddress: ev.ResolverAddress, + T: ev.Finished.Sub(begin).Seconds(), + }) + aaaa := t.gatherAAAA(ev.Addresses) + if len(aaaa) <= 0 && ev.Failure == nil { + // We don't have any AAAA results. Historically we do not + // create a record for AAAA with no results when A succeeded + continue + } + out = append(out, model.ArchivalDNSLookupResult{ + Answers: aaaa, + Engine: ev.ResolverNetwork, + Failure: t.newFailure(ev.Failure), + Hostname: ev.Domain, + QueryType: "AAAA", + ResolverHostname: nil, // legacy + ResolverPort: nil, // legacy + ResolverAddress: ev.ResolverAddress, + T: ev.Finished.Sub(begin).Seconds(), + }) + } + return +} + +func (t *Trace) gatherA(addrs []string) (out []model.ArchivalDNSAnswer) { + for _, addr := range addrs { + if strings.Contains(addr, ":") { + continue // it's AAAA so we need to skip it + } + answer := model.ArchivalDNSAnswer{AnswerType: "A"} + asn, org, _ := geolocate.LookupASN(addr) + answer.ASN = int64(asn) + answer.ASOrgName = org + answer.IPv4 = addr + out = append(out, answer) + } + return +} + +func (t *Trace) gatherAAAA(addrs []string) (out []model.ArchivalDNSAnswer) { + for _, addr := range addrs { + if !strings.Contains(addr, ":") { + continue // it's A so we need to skip it + } + answer := model.ArchivalDNSAnswer{AnswerType: "AAAA"} + asn, org, _ := geolocate.LookupASN(addr) + answer.ASN = int64(asn) + answer.ASOrgName = org + answer.IPv6 = addr + out = append(out, answer) + } + return +} + +// +// NetworkEvents +// + +// NewArchivalNetworkEventList builds a network events list in the OONI +// archival data format out of the results saved inside the trace. +func (t *Trace) NewArchivalNetworkEventList(begin time.Time) (out []model.ArchivalNetworkEvent) { + for _, ev := range t.Network { + out = append(out, model.ArchivalNetworkEvent{ + Address: ev.RemoteAddr, + Failure: t.newFailure(ev.Failure), + NumBytes: int64(ev.Count), + Operation: ev.Operation, + Proto: ev.Network, + T: ev.Finished.Sub(begin).Seconds(), + Tags: nil, + }) + } + return +} + +// +// TLS handshake +// + +// NewArchivalTLSHandshakeResultList builds a TLS handshakes list in the OONI +// archival data format out of the results saved inside the trace. +func (t *Trace) NewArchivalTLSHandshakeResultList(begin time.Time) (out []model.ArchivalTLSOrQUICHandshakeResult) { + for _, ev := range t.TLSHandshake { + out = append(out, model.ArchivalTLSOrQUICHandshakeResult{ + CipherSuite: ev.CipherSuite, + Failure: t.newFailure(ev.Failure), + NegotiatedProtocol: ev.NegotiatedProto, + NoTLSVerify: ev.SkipVerify, + PeerCertificates: t.makePeerCerts(ev.PeerCerts), + ServerName: ev.SNI, + T: ev.Finished.Sub(begin).Seconds(), + Tags: nil, + TLSVersion: ev.TLSVersion, + }) + } + return +} + +func (t *Trace) makePeerCerts(in [][]byte) (out []model.ArchivalMaybeBinaryData) { + for _, v := range in { + out = append(out, model.ArchivalMaybeBinaryData{Value: string(v)}) + } + return +} diff --git a/internal/archival/trace_test.go b/internal/archival/trace_test.go new file mode 100644 index 0000000..b02dd18 --- /dev/null +++ b/internal/archival/trace_test.go @@ -0,0 +1,921 @@ +package archival + +import ( + "errors" + "io" + "net/http" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// traceTime generates a time that is based off a fixed beginning +// in time, so we can easily compare times. +func traceTime(d int64) time.Time { + t := time.Date(2021, 01, 13, 14, 21, 59, 0, time.UTC) + return t.Add(time.Duration(d) * time.Millisecond) +} + +// deltaSinceTraceTime computes the delta since the original +// trace time expressed in floating point seconds. +func deltaSinceTraceTime(d int64) float64 { + return (time.Duration(d) * time.Millisecond).Seconds() +} + +// failureFromString converts a string to a failure. +func failureFromString(failure string) *string { + return &failure +} + +func TestTraceNewArchivalTCPConnectResultList(t *testing.T) { + type fields struct { + DNSLookupHTTPS []*DNSLookupEvent + DNSLookupHost []*DNSLookupEvent + DNSRoundTrip []*DNSRoundTripEvent + HTTPRoundTrip []*HTTPRoundTripEvent + Network []*NetworkEvent + QUICHandshake []*QUICTLSHandshakeEvent + TLSHandshake []*QUICTLSHandshakeEvent + } + type args struct { + begin time.Time + } + tests := []struct { + name string + fields fields + args args + wantOut []model.ArchivalTCPConnectResult + }{{ + name: "with empty trace", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: nil, + }, { + name: "we ignore I/O operations", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{{ + Count: 1024, + Failure: nil, + Finished: traceTime(2), + Network: "tcp", + Operation: netxlite.WriteOperation, + RemoteAddr: "8.8.8.8:443", + Started: traceTime(1), + }, { + Count: 4096, + Failure: nil, + Finished: traceTime(4), + Network: "tcp", + Operation: netxlite.ReadOperation, + RemoteAddr: "8.8.8.8:443", + Started: traceTime(3), + }}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: nil, + }, { + name: "we ignore UDP connect", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{{ + Count: 0, + Failure: nil, + Finished: traceTime(2), + Network: "udp", + Operation: netxlite.ConnectOperation, + RemoteAddr: "8.8.8.8:53", + Started: traceTime(1), + }}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: nil, + }, { + name: "with TCP connect success", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{{ + Count: 0, + Failure: nil, + Finished: traceTime(2), + Network: "tcp", + Operation: netxlite.ConnectOperation, + RemoteAddr: "8.8.8.8:443", + Started: traceTime(1), + }}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalTCPConnectResult{{ + IP: "8.8.8.8", + Port: 443, + Status: model.ArchivalTCPConnectStatus{ + Blocked: nil, + Failure: nil, + Success: true, + }, + T: deltaSinceTraceTime(2), + }}, + }, { + name: "with TCP connect failure", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{{ + Count: 0, + Failure: netxlite.NewTopLevelGenericErrWrapper(netxlite.ECONNREFUSED), + Finished: traceTime(2), + Network: "tcp", + Operation: netxlite.ConnectOperation, + RemoteAddr: "8.8.8.8:443", + Started: traceTime(1), + }}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalTCPConnectResult{{ + IP: "8.8.8.8", + Port: 443, + Status: model.ArchivalTCPConnectStatus{ + Blocked: nil, + Failure: failureFromString(netxlite.FailureConnectionRefused), + Success: false, + }, + T: deltaSinceTraceTime(2), + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr := &Trace{ + DNSLookupHTTPS: tt.fields.DNSLookupHTTPS, + DNSLookupHost: tt.fields.DNSLookupHost, + DNSRoundTrip: tt.fields.DNSRoundTrip, + HTTPRoundTrip: tt.fields.HTTPRoundTrip, + Network: tt.fields.Network, + QUICHandshake: tt.fields.QUICHandshake, + TLSHandshake: tt.fields.TLSHandshake, + } + gotOut := tr.NewArchivalTCPConnectResultList(tt.args.begin) + if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestTraceNewArchivalHTTPRequestResultList(t *testing.T) { + type fields struct { + DNSLookupHTTPS []*DNSLookupEvent + DNSLookupHost []*DNSLookupEvent + DNSRoundTrip []*DNSRoundTripEvent + HTTPRoundTrip []*HTTPRoundTripEvent + Network []*NetworkEvent + QUICHandshake []*QUICTLSHandshakeEvent + TLSHandshake []*QUICTLSHandshakeEvent + } + type args struct { + begin time.Time + } + tests := []struct { + name string + fields fields + args args + wantOut []model.ArchivalHTTPRequestResult + }{{ + name: "with empty trace", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: nil, + }, { + name: "with failure", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{{ + Failure: netxlite.NewTopLevelGenericErrWrapper(netxlite.ECONNRESET), + Finished: traceTime(2), + Method: "GET", + RequestHeaders: http.Header{ + "Accept": {"*/*"}, + "X-Cookie": {"A", "B", "C"}, + }, + ResponseBody: nil, + ResponseBodyIsTruncated: false, + ResponseBodyLength: 0, + ResponseHeaders: nil, + Started: traceTime(1), + StatusCode: 0, + Transport: "tcp", + URL: "http://x.org/", + }}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalHTTPRequestResult{{ + Failure: failureFromString(netxlite.FailureConnectionReset), + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalMaybeBinaryData{}, + BodyIsTruncated: false, + HeadersList: []model.ArchivalHTTPHeader{{ + Key: "Accept", + Value: model.ArchivalMaybeBinaryData{ + Value: "*/*", + }, + }, { + Key: "X-Cookie", + Value: model.ArchivalMaybeBinaryData{ + Value: "A", + }, + }, { + Key: "X-Cookie", + Value: model.ArchivalMaybeBinaryData{ + Value: "B", + }, + }, { + Key: "X-Cookie", + Value: model.ArchivalMaybeBinaryData{ + Value: "C", + }, + }}, + Headers: map[string]model.ArchivalMaybeBinaryData{ + "Accept": {Value: "*/*"}, + "X-Cookie": {Value: "A"}, + }, + Method: "GET", + Tor: model.ArchivalHTTPTor{}, + Transport: "tcp", + URL: "http://x.org/", + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalMaybeBinaryData{}, + BodyIsTruncated: false, + Code: 0, + HeadersList: nil, + Headers: nil, + Locations: nil, + }, + T: deltaSinceTraceTime(2), + }}, + }, { + name: "with success", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{{ + Failure: nil, + Finished: traceTime(2), + Method: "GET", + RequestHeaders: http.Header{ + "Accept": {"*/*"}, + "X-Cookie": {"A", "B", "C"}, + }, + ResponseBody: []byte("0xdeadbeef"), + ResponseBodyIsTruncated: true, + ResponseBodyLength: 10, + ResponseHeaders: http.Header{ + "Server": {"antani/1.0"}, + "X-Cookie-Reply": {"C", "D", "F"}, + "Location": {"https://x.org/", "https://x.org/robots.txt"}, + }, + Started: traceTime(1), + StatusCode: 302, + Transport: "tcp", + URL: "http://x.org/", + }}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalHTTPRequestResult{{ + Failure: nil, + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalMaybeBinaryData{}, + BodyIsTruncated: false, + HeadersList: []model.ArchivalHTTPHeader{{ + Key: "Accept", + Value: model.ArchivalMaybeBinaryData{ + Value: "*/*", + }, + }, { + Key: "X-Cookie", + Value: model.ArchivalMaybeBinaryData{ + Value: "A", + }, + }, { + Key: "X-Cookie", + Value: model.ArchivalMaybeBinaryData{ + Value: "B", + }, + }, { + Key: "X-Cookie", + Value: model.ArchivalMaybeBinaryData{ + Value: "C", + }, + }}, + Headers: map[string]model.ArchivalMaybeBinaryData{ + "Accept": {Value: "*/*"}, + "X-Cookie": {Value: "A"}, + }, + Method: "GET", + Tor: model.ArchivalHTTPTor{}, + Transport: "tcp", + URL: "http://x.org/", + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalMaybeBinaryData{ + Value: "0xdeadbeef", + }, + BodyIsTruncated: true, + Code: 302, + HeadersList: []model.ArchivalHTTPHeader{{ + Key: "Location", + Value: model.ArchivalMaybeBinaryData{ + Value: "https://x.org/", + }, + }, { + Key: "Location", + Value: model.ArchivalMaybeBinaryData{ + Value: "https://x.org/robots.txt", + }, + }, { + Key: "Server", + Value: model.ArchivalMaybeBinaryData{ + Value: "antani/1.0", + }, + }, { + Key: "X-Cookie-Reply", + Value: model.ArchivalMaybeBinaryData{ + Value: "C", + }, + }, { + Key: "X-Cookie-Reply", + Value: model.ArchivalMaybeBinaryData{ + Value: "D", + }, + }, { + Key: "X-Cookie-Reply", + Value: model.ArchivalMaybeBinaryData{ + Value: "F", + }, + }}, + Headers: map[string]model.ArchivalMaybeBinaryData{ + "Server": {Value: "antani/1.0"}, + "X-Cookie-Reply": {Value: "C"}, + "Location": {Value: "https://x.org/"}, + }, + Locations: []string{ + "https://x.org/", + "https://x.org/robots.txt", + }, + }, + T: deltaSinceTraceTime(2), + }}, + }, { + name: "The result is sorted by the value of T", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{{ + Failure: nil, + Finished: traceTime(3), + Method: "", + RequestHeaders: map[string][]string{}, + ResponseBody: []byte{}, + ResponseBodyIsTruncated: false, + ResponseBodyLength: 0, + ResponseHeaders: map[string][]string{}, + Started: time.Time{}, + StatusCode: 0, + Transport: "", + URL: "", + }, { + Failure: nil, + Finished: traceTime(2), + Method: "", + RequestHeaders: map[string][]string{}, + ResponseBody: []byte{}, + ResponseBodyIsTruncated: false, + ResponseBodyLength: 0, + ResponseHeaders: map[string][]string{}, + Started: time.Time{}, + StatusCode: 0, + Transport: "", + URL: "", + }, { + Failure: nil, + Finished: traceTime(5), + Method: "", + RequestHeaders: map[string][]string{}, + ResponseBody: []byte{}, + ResponseBodyIsTruncated: false, + ResponseBodyLength: 0, + ResponseHeaders: map[string][]string{}, + Started: time.Time{}, + StatusCode: 0, + Transport: "", + URL: "", + }}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalHTTPRequestResult{{ + Failure: nil, + Request: model.ArchivalHTTPRequest{}, + Response: model.ArchivalHTTPResponse{}, + T: deltaSinceTraceTime(5), + }, { + Failure: nil, + Request: model.ArchivalHTTPRequest{}, + Response: model.ArchivalHTTPResponse{}, + T: deltaSinceTraceTime(3), + }, { + Failure: nil, + Request: model.ArchivalHTTPRequest{}, + Response: model.ArchivalHTTPResponse{}, + T: deltaSinceTraceTime(2), + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr := &Trace{ + DNSLookupHTTPS: tt.fields.DNSLookupHTTPS, + DNSLookupHost: tt.fields.DNSLookupHost, + DNSRoundTrip: tt.fields.DNSRoundTrip, + HTTPRoundTrip: tt.fields.HTTPRoundTrip, + Network: tt.fields.Network, + QUICHandshake: tt.fields.QUICHandshake, + TLSHandshake: tt.fields.TLSHandshake, + } + gotOut := tr.NewArchivalHTTPRequestResultList(tt.args.begin) + if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestTraceNewArchivalDNSLookupResultList(t *testing.T) { + type fields struct { + DNSLookupHTTPS []*DNSLookupEvent + DNSLookupHost []*DNSLookupEvent + DNSRoundTrip []*DNSRoundTripEvent + HTTPRoundTrip []*HTTPRoundTripEvent + Network []*NetworkEvent + QUICHandshake []*QUICTLSHandshakeEvent + TLSHandshake []*QUICTLSHandshakeEvent + } + type args struct { + begin time.Time + } + tests := []struct { + name string + fields fields + args args + wantOut []model.ArchivalDNSLookupResult + }{{ + name: "with empty trace", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: nil, + }, { + name: "with NXDOMAIN failure", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{{ + ALPNs: nil, + Addresses: nil, + Domain: "example.com", + Failure: netxlite.NewTopLevelGenericErrWrapper(errors.New(netxlite.DNSNoSuchHostSuffix)), + Finished: traceTime(2), + LookupType: "", // not processed + ResolverAddress: "8.8.8.8:53", + ResolverNetwork: "udp", + Started: traceTime(1), + }}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalDNSLookupResult{{ + Answers: nil, + Engine: "udp", + Failure: failureFromString(netxlite.FailureDNSNXDOMAINError), + Hostname: "example.com", + QueryType: "A", + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "8.8.8.8:53", + T: deltaSinceTraceTime(2), + }, { + Answers: nil, + Engine: "udp", + Failure: failureFromString(netxlite.FailureDNSNXDOMAINError), + Hostname: "example.com", + QueryType: "AAAA", + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "8.8.8.8:53", + T: deltaSinceTraceTime(2), + }}, + }, { + name: "with success for A and AAAA", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{{ + ALPNs: nil, + Addresses: []string{ + "8.8.8.8", "8.8.4.4", "2001:4860:4860::8844", + }, + Domain: "dns.google", + Failure: nil, + Finished: traceTime(2), + LookupType: "", // not processed + ResolverAddress: "8.8.8.8:53", + ResolverNetwork: "udp", + Started: traceTime(1), + }}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalDNSLookupResult{{ + Answers: []model.ArchivalDNSAnswer{{ + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "A", + Hostname: "", + IPv4: "8.8.8.8", + IPv6: "", + TTL: nil, + }, { + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "A", + Hostname: "", + IPv4: "8.8.4.4", + IPv6: "", + TTL: nil, + }}, + Engine: "udp", + Failure: nil, + Hostname: "dns.google", + QueryType: "A", + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "8.8.8.8:53", + T: deltaSinceTraceTime(2), + }, { + Answers: []model.ArchivalDNSAnswer{{ + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "AAAA", + Hostname: "", + IPv4: "", + IPv6: "2001:4860:4860::8844", + TTL: nil, + }}, + Engine: "udp", + Failure: nil, + Hostname: "dns.google", + QueryType: "AAAA", + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "8.8.8.8:53", + T: deltaSinceTraceTime(2), + }}, + }, { + name: "when a domain has no AAAA addresses", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{{ + ALPNs: nil, + Addresses: []string{ + "8.8.8.8", "8.8.4.4", + }, + Domain: "dns.google", + Failure: nil, + Finished: traceTime(2), + LookupType: "", // not processed + ResolverAddress: "8.8.8.8:53", + ResolverNetwork: "udp", + Started: traceTime(1), + }}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalDNSLookupResult{{ + Answers: []model.ArchivalDNSAnswer{{ + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "A", + Hostname: "", + IPv4: "8.8.8.8", + IPv6: "", + TTL: nil, + }, { + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "A", + Hostname: "", + IPv4: "8.8.4.4", + IPv6: "", + TTL: nil, + }}, + Engine: "udp", + Failure: nil, + Hostname: "dns.google", + QueryType: "A", + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "8.8.8.8:53", + T: deltaSinceTraceTime(2), + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr := &Trace{ + DNSLookupHTTPS: tt.fields.DNSLookupHTTPS, + DNSLookupHost: tt.fields.DNSLookupHost, + DNSRoundTrip: tt.fields.DNSRoundTrip, + HTTPRoundTrip: tt.fields.HTTPRoundTrip, + Network: tt.fields.Network, + QUICHandshake: tt.fields.QUICHandshake, + TLSHandshake: tt.fields.TLSHandshake, + } + gotOut := tr.NewArchivalDNSLookupResultList(tt.args.begin) + if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestTraceNewArchivalNetworkEventList(t *testing.T) { + type fields struct { + DNSLookupHTTPS []*DNSLookupEvent + DNSLookupHost []*DNSLookupEvent + DNSRoundTrip []*DNSRoundTripEvent + HTTPRoundTrip []*HTTPRoundTripEvent + Network []*NetworkEvent + QUICHandshake []*QUICTLSHandshakeEvent + TLSHandshake []*QUICTLSHandshakeEvent + } + type args struct { + begin time.Time + } + tests := []struct { + name string + fields fields + args args + wantOut []model.ArchivalNetworkEvent + }{{ + name: "with empty trace", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: nil, + }, { + name: "we fill all the fields we should be filling", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{{ + Count: 1234, + Failure: netxlite.NewTopLevelGenericErrWrapper(io.EOF), + Finished: traceTime(2), + Network: "tcp", + Operation: netxlite.ReadOperation, + RemoteAddr: "8.8.8.8:443", + Started: traceTime(1), + }}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalNetworkEvent{{ + Address: "8.8.8.8:443", + Failure: failureFromString(netxlite.FailureEOFError), + NumBytes: 1234, + Operation: netxlite.ReadOperation, + Proto: "tcp", + T: deltaSinceTraceTime(2), + Tags: nil, + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr := &Trace{ + DNSLookupHTTPS: tt.fields.DNSLookupHTTPS, + DNSLookupHost: tt.fields.DNSLookupHost, + DNSRoundTrip: tt.fields.DNSRoundTrip, + HTTPRoundTrip: tt.fields.HTTPRoundTrip, + Network: tt.fields.Network, + QUICHandshake: tt.fields.QUICHandshake, + TLSHandshake: tt.fields.TLSHandshake, + } + gotOut := tr.NewArchivalNetworkEventList(tt.args.begin) + if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestTraceNewArchivalTLSHandshakeResultList(t *testing.T) { + type fields struct { + DNSLookupHTTPS []*DNSLookupEvent + DNSLookupHost []*DNSLookupEvent + DNSRoundTrip []*DNSRoundTripEvent + HTTPRoundTrip []*HTTPRoundTripEvent + Network []*NetworkEvent + QUICHandshake []*QUICTLSHandshakeEvent + TLSHandshake []*QUICTLSHandshakeEvent + } + type args struct { + begin time.Time + } + tests := []struct { + name string + fields fields + args args + wantOut []model.ArchivalTLSOrQUICHandshakeResult + }{{ + name: "with empty trace", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: nil, + }, { + name: "we fill all the fields we should be filling", + fields: fields{ + DNSLookupHTTPS: []*DNSLookupEvent{}, + DNSLookupHost: []*DNSLookupEvent{}, + DNSRoundTrip: []*DNSRoundTripEvent{}, + HTTPRoundTrip: []*HTTPRoundTripEvent{}, + Network: []*NetworkEvent{}, + QUICHandshake: []*QUICTLSHandshakeEvent{}, + TLSHandshake: []*QUICTLSHandshakeEvent{{ + ALPN: []string{"h2", "http/1.1"}, + CipherSuite: "TLS_AES_128_GCM_SHA256", + Failure: netxlite.NewTopLevelGenericErrWrapper(io.EOF), + Finished: traceTime(2), + NegotiatedProto: "h2", + Network: "tcp", + PeerCerts: [][]byte{ + []byte("deadbeef"), + []byte("xox"), + }, + RemoteAddr: "8.8.8.8:443", + SNI: "dns.google", + SkipVerify: true, + Started: traceTime(1), + TLSVersion: "TLSv1.3", + }}, + }, + args: args{ + begin: traceTime(0), + }, + wantOut: []model.ArchivalTLSOrQUICHandshakeResult{{ + CipherSuite: "TLS_AES_128_GCM_SHA256", + Failure: failureFromString(netxlite.FailureEOFError), + NegotiatedProtocol: "h2", + NoTLSVerify: true, + PeerCertificates: []model.ArchivalMaybeBinaryData{{ + Value: "deadbeef", + }, { + Value: "xox", + }}, + ServerName: "dns.google", + T: deltaSinceTraceTime(2), + Tags: nil, + TLSVersion: "TLSv1.3", + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr := &Trace{ + DNSLookupHTTPS: tt.fields.DNSLookupHTTPS, + DNSLookupHost: tt.fields.DNSLookupHost, + DNSRoundTrip: tt.fields.DNSRoundTrip, + HTTPRoundTrip: tt.fields.HTTPRoundTrip, + Network: tt.fields.Network, + QUICHandshake: tt.fields.QUICHandshake, + TLSHandshake: tt.fields.TLSHandshake, + } + gotOut := tr.NewArchivalTLSHandshakeResultList(tt.args.begin) + if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/internal/cmd/oohelper/internal/fake_test.go b/internal/cmd/oohelper/internal/fake_test.go index 4553664..12e536c 100644 --- a/internal/cmd/oohelper/internal/fake_test.go +++ b/internal/cmd/oohelper/internal/fake_test.go @@ -59,11 +59,16 @@ func (c FakeResolver) LookupHTTPS(ctx context.Context, domain string) (*model.HT var _ model.Resolver = FakeResolver{} type FakeTransport struct { + Name string Err error Func func(*http.Request) (*http.Response, error) Resp *http.Response } +func (txp FakeTransport) Network() string { + return txp.Name +} + func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { time.Sleep(10 * time.Microsecond) if txp.Func != nil { diff --git a/internal/cmd/oohelperd/internal/webconnectivity/fake_test.go b/internal/cmd/oohelperd/internal/webconnectivity/fake_test.go index d4e6e72..8df8aa4 100644 --- a/internal/cmd/oohelperd/internal/webconnectivity/fake_test.go +++ b/internal/cmd/oohelperd/internal/webconnectivity/fake_test.go @@ -59,11 +59,16 @@ func (c FakeResolver) LookupHTTPS(ctx context.Context, domain string) (*model.HT var _ model.Resolver = FakeResolver{} type FakeTransport struct { + Name string Err error Func func(*http.Request) (*http.Response, error) Resp *http.Response } +func (txp FakeTransport) Network() string { + return txp.Name +} + func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { time.Sleep(10 * time.Microsecond) if txp.Func != nil { diff --git a/internal/engine/experiment/dash/dash.go b/internal/engine/experiment/dash/dash.go index 7807bd5..70d6084 100644 --- a/internal/engine/experiment/dash/dash.go +++ b/internal/engine/experiment/dash/dash.go @@ -26,7 +26,7 @@ const ( magicVersion = "0.008000000" testName = "dash" testVersion = "0.13.0" - totalStep = 15.0 + totalStep = 15 ) var ( diff --git a/internal/engine/netx/httptransport/fake_test.go b/internal/engine/netx/httptransport/fake_test.go index 83c72a0..92acfb2 100644 --- a/internal/engine/netx/httptransport/fake_test.go +++ b/internal/engine/netx/httptransport/fake_test.go @@ -20,11 +20,16 @@ func (d FakeDialer) DialContext(ctx context.Context, network, address string) (n } type FakeTransport struct { + Name string Err error Func func(*http.Request) (*http.Response, error) Resp *http.Response } +func (txp FakeTransport) Network() string { + return txp.Name +} + func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { time.Sleep(10 * time.Microsecond) if txp.Func != nil { diff --git a/internal/engine/netx/httptransport/saver.go b/internal/engine/netx/httptransport/saver.go index ba179e0..1e23e91 100644 --- a/internal/engine/netx/httptransport/saver.go +++ b/internal/engine/netx/httptransport/saver.go @@ -45,8 +45,7 @@ func (txp SaverPerformanceHTTPTransport) RoundTrip(req *http.Request) (*http.Res // events related to HTTP request and response metadata type SaverMetadataHTTPTransport struct { model.HTTPTransport - Saver *trace.Saver - Transport string + Saver *trace.Saver } // RoundTrip implements RoundTripper.RoundTrip @@ -55,7 +54,7 @@ func (txp SaverMetadataHTTPTransport) RoundTrip(req *http.Request) (*http.Respon HTTPHeaders: txp.CloneHeaders(req), HTTPMethod: req.Method, HTTPURL: req.URL.String(), - Transport: txp.Transport, + Transport: txp.HTTPTransport.Network(), Name: "http_request_metadata", Time: time.Now(), }) diff --git a/internal/engine/netx/httptransport/saver_test.go b/internal/engine/netx/httptransport/saver_test.go index 7a93eb9..989bf36 100644 --- a/internal/engine/netx/httptransport/saver_test.go +++ b/internal/engine/netx/httptransport/saver_test.go @@ -12,6 +12,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -22,7 +23,7 @@ func TestSaverPerformanceNoMultipleEvents(t *testing.T) { saver := &trace.Saver{} // register twice - do we see events twice? txp := httptransport.SaverPerformanceHTTPTransport{ - HTTPTransport: http.DefaultTransport.(*http.Transport), + HTTPTransport: netxlite.NewHTTPTransportStdlib(model.DiscardLogger), Saver: saver, } txp = httptransport.SaverPerformanceHTTPTransport{ @@ -67,7 +68,7 @@ func TestSaverMetadataSuccess(t *testing.T) { } saver := &trace.Saver{} txp := httptransport.SaverMetadataHTTPTransport{ - HTTPTransport: http.DefaultTransport.(*http.Transport), + HTTPTransport: netxlite.NewHTTPTransportStdlib(model.DiscardLogger), Saver: saver, } req, err := http.NewRequest("GET", "https://www.google.com", nil) @@ -165,7 +166,7 @@ func TestSaverTransactionSuccess(t *testing.T) { } saver := &trace.Saver{} txp := httptransport.SaverTransactionHTTPTransport{ - HTTPTransport: http.DefaultTransport.(*http.Transport), + HTTPTransport: netxlite.NewHTTPTransportStdlib(model.DiscardLogger), Saver: saver, } req, err := http.NewRequest("GET", "https://www.google.com", nil) diff --git a/internal/engine/netx/httptransport/system.go b/internal/engine/netx/httptransport/system.go index 0d92fcb..690e825 100644 --- a/internal/engine/netx/httptransport/system.go +++ b/internal/engine/netx/httptransport/system.go @@ -24,7 +24,16 @@ func NewSystemTransport(config Config) model.HTTPTransport { // back the true headers, such as Content-Length. This change is // functional to OONI's goal of observing the network. txp.DisableCompression = true - return txp + return &SystemTransportWrapper{txp} } -var _ model.HTTPTransport = &http.Transport{} +// SystemTransportWrapper adapts *http.Transport to have the .Network method +type SystemTransportWrapper struct { + *http.Transport +} + +func (txp *SystemTransportWrapper) Network() string { + return "tcp" +} + +var _ model.HTTPTransport = &SystemTransportWrapper{} diff --git a/internal/engine/netx/netx.go b/internal/engine/netx/netx.go index 4be37f9..d0222c7 100644 --- a/internal/engine/netx/netx.go +++ b/internal/engine/netx/netx.go @@ -200,7 +200,6 @@ func NewHTTPTransport(config Config) model.HTTPTransport { txp := tInfo.Factory(httptransport.Config{ Dialer: config.Dialer, QUICDialer: config.QUICDialer, TLSDialer: config.TLSDialer, TLSConfig: config.TLSConfig}) - transport := tInfo.TransportName if config.ByteCounter != nil { txp = httptransport.ByteCountingTransport{ @@ -211,7 +210,7 @@ func NewHTTPTransport(config Config) model.HTTPTransport { } if config.HTTPSaver != nil { txp = httptransport.SaverMetadataHTTPTransport{ - HTTPTransport: txp, Saver: config.HTTPSaver, Transport: transport} + HTTPTransport: txp, Saver: config.HTTPSaver} txp = httptransport.SaverBodyHTTPTransport{ HTTPTransport: txp, Saver: config.HTTPSaver} txp = httptransport.SaverPerformanceHTTPTransport{ diff --git a/internal/engine/netx/netx_test.go b/internal/engine/netx/netx_test.go index 17117e7..0fc35fe 100644 --- a/internal/engine/netx/netx_test.go +++ b/internal/engine/netx/netx_test.go @@ -420,7 +420,7 @@ func TestNewTLSDialerWithNoTLSVerifyAndNoConfig(t *testing.T) { func TestNewVanilla(t *testing.T) { txp := netx.NewHTTPTransport(netx.Config{}) - if _, ok := txp.(*http.Transport); !ok { + if _, ok := txp.(*httptransport.SystemTransportWrapper); !ok { t.Fatal("not the transport we expected") } } @@ -480,7 +480,7 @@ func TestNewWithByteCounter(t *testing.T) { if bctxp.Counter != counter { t.Fatal("not the byte counter we expected") } - if _, ok := bctxp.HTTPTransport.(*http.Transport); !ok { + if _, ok := bctxp.HTTPTransport.(*httptransport.SystemTransportWrapper); !ok { t.Fatal("not the transport we expected") } } @@ -496,7 +496,7 @@ func TestNewWithLogger(t *testing.T) { if ltxp.Logger != log.Log { t.Fatal("not the logger we expected") } - if _, ok := ltxp.HTTPTransport.(*http.Transport); !ok { + if _, ok := ltxp.HTTPTransport.(*httptransport.SystemTransportWrapper); !ok { t.Fatal("not the transport we expected") } } @@ -534,7 +534,7 @@ func TestNewWithSaver(t *testing.T) { if smtxp.Saver != saver { t.Fatal("not the logger we expected") } - if _, ok := smtxp.HTTPTransport.(*http.Transport); !ok { + if _, ok := smtxp.HTTPTransport.(*httptransport.SystemTransportWrapper); !ok { t.Fatal("not the transport we expected") } } diff --git a/internal/model/mocks/addr.go b/internal/model/mocks/addr.go new file mode 100644 index 0000000..98e662e --- /dev/null +++ b/internal/model/mocks/addr.go @@ -0,0 +1,21 @@ +package mocks + +import "net" + +// Addr allows mocking net.Addr. +type Addr struct { + MockString func() string + MockNetwork func() string +} + +var _ net.Addr = &Addr{} + +// String calls MockString. +func (a *Addr) String() string { + return a.MockString() +} + +// Network calls MockNetwork. +func (a *Addr) Network() string { + return a.MockNetwork() +} diff --git a/internal/model/mocks/addr_test.go b/internal/model/mocks/addr_test.go new file mode 100644 index 0000000..ec629f8 --- /dev/null +++ b/internal/model/mocks/addr_test.go @@ -0,0 +1,27 @@ +package mocks + +import "testing" + +func TestAddr(t *testing.T) { + t.Run("String", func(t *testing.T) { + a := &Addr{ + MockString: func() string { + return "antani" + }, + } + if a.String() != "antani" { + t.Fatal("invalid result for String") + } + }) + + t.Run("Network", func(t *testing.T) { + a := &Addr{ + MockNetwork: func() string { + return "mascetti" + }, + } + if a.Network() != "mascetti" { + t.Fatal("invalid result for Network") + } + }) +} diff --git a/internal/model/mocks/dnstransport.go b/internal/model/mocks/dnstransport.go index 9e74845..ed32f09 100644 --- a/internal/model/mocks/dnstransport.go +++ b/internal/model/mocks/dnstransport.go @@ -4,7 +4,7 @@ import "context" // DNSTransport allows mocking dnsx.DNSTransport. type DNSTransport struct { - MockRoundTrip func(ctx context.Context, query []byte) (reply []byte, err error) + MockRoundTrip func(ctx context.Context, query []byte) ([]byte, error) MockRequiresPadding func() bool @@ -16,7 +16,7 @@ type DNSTransport struct { } // RoundTrip calls MockRoundTrip. -func (txp *DNSTransport) RoundTrip(ctx context.Context, query []byte) (reply []byte, err error) { +func (txp *DNSTransport) RoundTrip(ctx context.Context, query []byte) ([]byte, error) { return txp.MockRoundTrip(ctx, query) } diff --git a/internal/model/mocks/http.go b/internal/model/mocks/http.go index 4f4cf3e..4500f84 100644 --- a/internal/model/mocks/http.go +++ b/internal/model/mocks/http.go @@ -4,10 +4,16 @@ import "net/http" // HTTPTransport mocks netxlite.HTTPTransport. type HTTPTransport struct { + MockNetwork func() string MockRoundTrip func(req *http.Request) (*http.Response, error) MockCloseIdleConnections func() } +// Network calls MockNetwork. +func (txp *HTTPTransport) Network() string { + return txp.MockNetwork() +} + // RoundTrip calls MockRoundTrip. func (txp *HTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.MockRoundTrip(req) diff --git a/internal/model/mocks/quic.go b/internal/model/mocks/quic.go index 23fe9a4..bd87e54 100644 --- a/internal/model/mocks/quic.go +++ b/internal/model/mocks/quic.go @@ -139,8 +139,8 @@ func (s *QUICEarlySession) ReceiveMessage() ([]byte, error) { return s.MockReceiveMessage() } -// QUICUDPLikeConn is an UDP conn used by QUIC. -type QUICUDPLikeConn struct { +// UDPLikeConn is an UDP conn used by QUIC. +type UDPLikeConn struct { MockWriteTo func(p []byte, addr net.Addr) (int, error) MockClose func() error MockLocalAddr func() net.Addr @@ -148,59 +148,59 @@ type QUICUDPLikeConn struct { MockSetDeadline func(t time.Time) error MockSetReadDeadline func(t time.Time) error MockSetWriteDeadline func(t time.Time) error - MockReadFrom func(p []byte) (n int, addr net.Addr, err error) + MockReadFrom func(p []byte) (int, net.Addr, error) MockSyscallConn func() (syscall.RawConn, error) MockSetReadBuffer func(n int) error } -var _ model.UDPLikeConn = &QUICUDPLikeConn{} +var _ model.UDPLikeConn = &UDPLikeConn{} // WriteTo calls MockWriteTo. -func (c *QUICUDPLikeConn) WriteTo(p []byte, addr net.Addr) (int, error) { +func (c *UDPLikeConn) WriteTo(p []byte, addr net.Addr) (int, error) { return c.MockWriteTo(p, addr) } // Close calls MockClose. -func (c *QUICUDPLikeConn) Close() error { +func (c *UDPLikeConn) Close() error { return c.MockClose() } // LocalAddr calls MockLocalAddr. -func (c *QUICUDPLikeConn) LocalAddr() net.Addr { +func (c *UDPLikeConn) LocalAddr() net.Addr { return c.MockLocalAddr() } // RemoteAddr calls MockRemoteAddr. -func (c *QUICUDPLikeConn) RemoteAddr() net.Addr { +func (c *UDPLikeConn) RemoteAddr() net.Addr { return c.MockRemoteAddr() } // SetDeadline calls MockSetDeadline. -func (c *QUICUDPLikeConn) SetDeadline(t time.Time) error { +func (c *UDPLikeConn) SetDeadline(t time.Time) error { return c.MockSetDeadline(t) } // SetReadDeadline calls MockSetReadDeadline. -func (c *QUICUDPLikeConn) SetReadDeadline(t time.Time) error { +func (c *UDPLikeConn) SetReadDeadline(t time.Time) error { return c.MockSetReadDeadline(t) } // SetWriteDeadline calls MockSetWriteDeadline. -func (c *QUICUDPLikeConn) SetWriteDeadline(t time.Time) error { +func (c *UDPLikeConn) SetWriteDeadline(t time.Time) error { return c.MockSetWriteDeadline(t) } // ReadFrom calls MockReadFrom. -func (c *QUICUDPLikeConn) ReadFrom(b []byte) (int, net.Addr, error) { +func (c *UDPLikeConn) ReadFrom(b []byte) (int, net.Addr, error) { return c.MockReadFrom(b) } // SyscallConn calls MockSyscallConn. -func (c *QUICUDPLikeConn) SyscallConn() (syscall.RawConn, error) { +func (c *UDPLikeConn) SyscallConn() (syscall.RawConn, error) { return c.MockSyscallConn() } // SetReadBuffer calls MockSetReadBuffer. -func (c *QUICUDPLikeConn) SetReadBuffer(n int) error { +func (c *UDPLikeConn) SetReadBuffer(n int) error { return c.MockSetReadBuffer(n) } diff --git a/internal/model/mocks/quic_test.go b/internal/model/mocks/quic_test.go index 30066b0..687cf12 100644 --- a/internal/model/mocks/quic_test.go +++ b/internal/model/mocks/quic_test.go @@ -292,7 +292,7 @@ func TestQUICEarlySession(t *testing.T) { func TestQUICUDPLikeConn(t *testing.T) { t.Run("WriteTo", func(t *testing.T) { expected := errors.New("mocked error") - quc := &QUICUDPLikeConn{ + quc := &UDPLikeConn{ MockWriteTo: func(p []byte, addr net.Addr) (int, error) { return 0, expected }, @@ -310,7 +310,7 @@ func TestQUICUDPLikeConn(t *testing.T) { t.Run("ConnClose", func(t *testing.T) { expected := errors.New("mocked error") - quc := &QUICUDPLikeConn{ + quc := &UDPLikeConn{ MockClose: func() error { return expected }, @@ -326,7 +326,7 @@ func TestQUICUDPLikeConn(t *testing.T) { IP: net.IPv6loopback, Port: 1234, } - c := &QUICUDPLikeConn{ + c := &UDPLikeConn{ MockLocalAddr: func() net.Addr { return expected }, @@ -342,7 +342,7 @@ func TestQUICUDPLikeConn(t *testing.T) { IP: net.IPv6loopback, Port: 1234, } - c := &QUICUDPLikeConn{ + c := &UDPLikeConn{ MockRemoteAddr: func() net.Addr { return expected }, @@ -355,7 +355,7 @@ func TestQUICUDPLikeConn(t *testing.T) { t.Run("SetDeadline", func(t *testing.T) { expected := errors.New("mocked error") - c := &QUICUDPLikeConn{ + c := &UDPLikeConn{ MockSetDeadline: func(t time.Time) error { return expected }, @@ -368,7 +368,7 @@ func TestQUICUDPLikeConn(t *testing.T) { t.Run("SetReadDeadline", func(t *testing.T) { expected := errors.New("mocked error") - c := &QUICUDPLikeConn{ + c := &UDPLikeConn{ MockSetReadDeadline: func(t time.Time) error { return expected }, @@ -381,7 +381,7 @@ func TestQUICUDPLikeConn(t *testing.T) { t.Run("SetWriteDeadline", func(t *testing.T) { expected := errors.New("mocked error") - c := &QUICUDPLikeConn{ + c := &UDPLikeConn{ MockSetWriteDeadline: func(t time.Time) error { return expected }, @@ -394,7 +394,7 @@ func TestQUICUDPLikeConn(t *testing.T) { t.Run("ConnReadFrom", func(t *testing.T) { expected := errors.New("mocked error") - quc := &QUICUDPLikeConn{ + quc := &UDPLikeConn{ MockReadFrom: func(b []byte) (int, net.Addr, error) { return 0, nil, expected }, @@ -414,7 +414,7 @@ func TestQUICUDPLikeConn(t *testing.T) { t.Run("SyscallConn", func(t *testing.T) { expected := errors.New("mocked error") - quc := &QUICUDPLikeConn{ + quc := &UDPLikeConn{ MockSyscallConn: func() (syscall.RawConn, error) { return nil, expected }, @@ -430,7 +430,7 @@ func TestQUICUDPLikeConn(t *testing.T) { t.Run("SetReadBuffer", func(t *testing.T) { expected := errors.New("mocked error") - quc := &QUICUDPLikeConn{ + quc := &UDPLikeConn{ MockSetReadBuffer: func(n int) error { return expected }, diff --git a/internal/model/netx.go b/internal/model/netx.go index fa21482..9a6a2d2 100644 --- a/internal/model/netx.go +++ b/internal/model/netx.go @@ -107,6 +107,10 @@ type HTTPClient interface { // HTTPTransport is an http.Transport-like structure. type HTTPTransport interface { + // Network returns the network used by the transport, which + // should be one of "tcp" and "quic". + Network() string + // RoundTrip performs the HTTP round trip. RoundTrip(req *http.Request) (*http.Response, error) diff --git a/internal/netxlite/filtering/dns_test.go b/internal/netxlite/filtering/dns_test.go index 65b3dc6..94920cf 100644 --- a/internal/netxlite/filtering/dns_test.go +++ b/internal/netxlite/filtering/dns_test.go @@ -224,7 +224,7 @@ func TestDNSProxy(t *testing.T) { t.Run("ReadFrom failure after which we should continue", func(t *testing.T) { expected := errors.New("mocked error") p := &DNSProxy{} - conn := &mocks.QUICUDPLikeConn{ + conn := &mocks.UDPLikeConn{ MockReadFrom: func(p []byte) (n int, addr net.Addr, err error) { return 0, nil, expected }, @@ -238,7 +238,7 @@ func TestDNSProxy(t *testing.T) { t.Run("ReadFrom the connection is closed", func(t *testing.T) { expected := errors.New("use of closed network connection") p := &DNSProxy{} - conn := &mocks.QUICUDPLikeConn{ + conn := &mocks.UDPLikeConn{ MockReadFrom: func(p []byte) (n int, addr net.Addr, err error) { return 0, nil, expected }, @@ -251,7 +251,7 @@ func TestDNSProxy(t *testing.T) { t.Run("Unpack fails", func(t *testing.T) { p := &DNSProxy{} - conn := &mocks.QUICUDPLikeConn{ + conn := &mocks.UDPLikeConn{ MockReadFrom: func(p []byte) (n int, addr net.Addr, err error) { if len(p) < 4 { panic("buffer too small") @@ -268,7 +268,7 @@ func TestDNSProxy(t *testing.T) { t.Run("reply fails", func(t *testing.T) { p := &DNSProxy{} - conn := &mocks.QUICUDPLikeConn{ + conn := &mocks.UDPLikeConn{ MockReadFrom: func(p []byte) (n int, addr net.Addr, err error) { query := &dns.Msg{} query.Question = append(query.Question, dns.Question{}) @@ -300,7 +300,7 @@ func TestDNSProxy(t *testing.T) { return reply, nil }, } - conn := &mocks.QUICUDPLikeConn{ + conn := &mocks.UDPLikeConn{ MockReadFrom: func(p []byte) (n int, addr net.Addr, err error) { query := &dns.Msg{} query.Question = append(query.Question, dns.Question{}) diff --git a/internal/netxlite/filtering/tproxy_test.go b/internal/netxlite/filtering/tproxy_test.go index ead1a86..4d628f9 100644 --- a/internal/netxlite/filtering/tproxy_test.go +++ b/internal/netxlite/filtering/tproxy_test.go @@ -159,7 +159,7 @@ func TestTProxyQUIC(t *testing.T) { defer proxy.Close() var called bool proxy.listenUDP = func(network string, laddr *net.UDPAddr) (model.UDPLikeConn, error) { - return &mocks.QUICUDPLikeConn{ + return &mocks.UDPLikeConn{ MockWriteTo: func(p []byte, addr net.Addr) (int, error) { called = true return len(p), nil @@ -196,7 +196,7 @@ func TestTProxyQUIC(t *testing.T) { defer proxy.Close() var called bool proxy.listenUDP = func(network string, laddr *net.UDPAddr) (model.UDPLikeConn, error) { - return &mocks.QUICUDPLikeConn{ + return &mocks.UDPLikeConn{ MockWriteTo: func(p []byte, addr net.Addr) (int, error) { called = true return len(p), nil diff --git a/internal/netxlite/http.go b/internal/netxlite/http.go index de00b77..1f3fe2c 100644 --- a/internal/netxlite/http.go +++ b/internal/netxlite/http.go @@ -27,7 +27,7 @@ func (txp *httpTransportErrWrapper) RoundTrip(req *http.Request) (*http.Response // httpTransportLogger is an HTTPTransport with logging. type httpTransportLogger struct { // HTTPTransport is the underlying HTTP transport. - HTTPTransport model.HTTPTransport + model.HTTPTransport // Logger is the underlying logger. Logger model.DebugLogger @@ -140,12 +140,22 @@ func NewOOHTTPBaseTransport(dialer model.Dialer, tlsDialer model.TLSDialer) mode // Ensure we correctly forward CloseIdleConnections. return &httpTransportConnectionsCloser{ - HTTPTransport: &oohttp.StdlibTransport{Transport: txp}, + HTTPTransport: &stdlibTransport{&oohttp.StdlibTransport{Transport: txp}}, Dialer: dialer, TLSDialer: tlsDialer, } } +// stdlibTransport wraps oohttp.StdlibTransport to add .Network() +type stdlibTransport struct { + *oohttp.StdlibTransport +} + +// Network implements HTTPTransport.Network. +func (txp *stdlibTransport) Network() string { + return "tcp" +} + // WrapHTTPTransport creates an HTTPTransport using the given logger // and guarantees that returned errors are wrapped. // diff --git a/internal/netxlite/http3.go b/internal/netxlite/http3.go index 281e30f..5f8e807 100644 --- a/internal/netxlite/http3.go +++ b/internal/netxlite/http3.go @@ -39,6 +39,11 @@ type http3Transport struct { var _ model.HTTPTransport = &http3Transport{} +// Network implements HTTPTransport.Network. +func (txp *http3Transport) Network() string { + return "quic" +} + // RoundTrip implements HTTPTransport.RoundTrip. func (txp *http3Transport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.child.RoundTrip(req) diff --git a/internal/netxlite/http_test.go b/internal/netxlite/http_test.go index a160c1c..a64101d 100644 --- a/internal/netxlite/http_test.go +++ b/internal/netxlite/http_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/apex/log" - oohttp "github.com/ooni/oohttp" "github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/model/mocks" @@ -252,7 +251,7 @@ func TestNewHTTPTransport(t *testing.T) { if tlsWithReadTimeout.TLSDialer != td { t.Fatal("invalid tls dialer") } - stdlib := connectionsCloser.HTTPTransport.(*oohttp.StdlibTransport) + stdlib := connectionsCloser.HTTPTransport.(*stdlibTransport) if !stdlib.Transport.ForceAttemptHTTP2 { t.Fatal("invalid ForceAttemptHTTP2") } diff --git a/internal/netxlite/integration_test.go b/internal/netxlite/integration_test.go index 66115eb..60b2ddf 100644 --- a/internal/netxlite/integration_test.go +++ b/internal/netxlite/integration_test.go @@ -426,6 +426,9 @@ func TestMeasureWithQUICDialer(t *testing.T) { t.Skip("skip test in short mode") } + // TODO(bassosimone): here we're not testing the case in which + // the certificate is invalid for the required SNI. + // // Measurement conditions we care about: // diff --git a/internal/netxlite/quic_test.go b/internal/netxlite/quic_test.go index 8b9f2e9..ee1a0c7 100644 --- a/internal/netxlite/quic_test.go +++ b/internal/netxlite/quic_test.go @@ -475,7 +475,7 @@ func TestNewSingleUseQUICDialer(t *testing.T) { func TestQUICListenerErrWrapper(t *testing.T) { t.Run("Listen", func(t *testing.T) { t.Run("on success", func(t *testing.T) { - expectedConn := &mocks.QUICUDPLikeConn{} + expectedConn := &mocks.UDPLikeConn{} ql := &quicListenerErrWrapper{ QUICListener: &mocks.QUICListener{ MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { @@ -519,7 +519,7 @@ func TestQUICErrWrapperUDPLikeConn(t *testing.T) { expectedAddr := &net.UDPAddr{} p := make([]byte, 128) conn := &quicErrWrapperUDPLikeConn{ - UDPLikeConn: &mocks.QUICUDPLikeConn{ + UDPLikeConn: &mocks.UDPLikeConn{ MockReadFrom: func(p []byte) (n int, addr net.Addr, err error) { return len(p), expectedAddr, nil }, @@ -541,7 +541,7 @@ func TestQUICErrWrapperUDPLikeConn(t *testing.T) { p := make([]byte, 128) expectedErr := io.EOF conn := &quicErrWrapperUDPLikeConn{ - UDPLikeConn: &mocks.QUICUDPLikeConn{ + UDPLikeConn: &mocks.UDPLikeConn{ MockReadFrom: func(p []byte) (n int, addr net.Addr, err error) { return 0, nil, expectedErr }, @@ -564,7 +564,7 @@ func TestQUICErrWrapperUDPLikeConn(t *testing.T) { t.Run("on success", func(t *testing.T) { p := make([]byte, 128) conn := &quicErrWrapperUDPLikeConn{ - UDPLikeConn: &mocks.QUICUDPLikeConn{ + UDPLikeConn: &mocks.UDPLikeConn{ MockWriteTo: func(p []byte, addr net.Addr) (int, error) { return len(p), nil }, @@ -583,7 +583,7 @@ func TestQUICErrWrapperUDPLikeConn(t *testing.T) { p := make([]byte, 128) expectedErr := io.EOF conn := &quicErrWrapperUDPLikeConn{ - UDPLikeConn: &mocks.QUICUDPLikeConn{ + UDPLikeConn: &mocks.UDPLikeConn{ MockWriteTo: func(p []byte, addr net.Addr) (int, error) { return 0, expectedErr }, @@ -602,7 +602,7 @@ func TestQUICErrWrapperUDPLikeConn(t *testing.T) { t.Run("Close", func(t *testing.T) { t.Run("on success", func(t *testing.T) { conn := &quicErrWrapperUDPLikeConn{ - UDPLikeConn: &mocks.QUICUDPLikeConn{ + UDPLikeConn: &mocks.UDPLikeConn{ MockClose: func() error { return nil }, @@ -617,7 +617,7 @@ func TestQUICErrWrapperUDPLikeConn(t *testing.T) { t.Run("on failure", func(t *testing.T) { expectedErr := io.EOF conn := &quicErrWrapperUDPLikeConn{ - UDPLikeConn: &mocks.QUICUDPLikeConn{ + UDPLikeConn: &mocks.UDPLikeConn{ MockClose: func() error { return expectedErr },