diff --git a/internal/engine/legacy/netx/DESIGN.md b/internal/engine/legacy/netx/DESIGN.md deleted file mode 100644 index 6bdc364..0000000 --- a/internal/engine/legacy/netx/DESIGN.md +++ /dev/null @@ -1,400 +0,0 @@ -# OONI Network Extensions - -| Author | Simone Basso | -|--------------|--------------| -| Last-Updated | 2020-04-02 | -| Status | approved | - -## Introduction - -OONI experiments send and/or receive network traffic to -determine if there is blocking. We want the implementation -of OONI experiments to be as simple as possible. We also -_want to attribute errors to the major network or protocol -operation that caused them_. - -At the same time, _we want an experiment to collect as much -low-level data as possible_. For example, we want to know -whether and when the TLS handshake completed; what certificates -were provided by the server; what TLS version was selected; -and so forth. These bits of information are very useful -to analyze a measurement and better classify it. - -We also want to _automatically or manually run follow-up -measurements where we change some configuration properties -and repeat the measurement_. For example, we may want to -configure DNS over HTTPS (DoH) and then attempt to -fetch again an URL. Or we may want to detect whether -there is SNI bases blocking. This package allows us to -do that in other parts of probe-engine. - -## Rationale - -As we observed [ooni/probe-engine#13]( -https://github.com/ooni/probe-engine/issues/13), every -experiment consists of two separate phases: - -1. measurement gathering - -2. measurement analysis - -During measurement gathering, we perform specific actions -that cause network data to be sent and/or received. During -measurement analysis, we process the measurement on the -device. For some experiments (e.g., Web Connectivity), this -second phase also entails contacting OONI backend services -that provide data useful to complete the analysis. - -This package implements measurement gathering. The analysis -is performed by other packages in probe-engine. The core -design idea is to provide OONI-measurements-aware replacements -for Go standard library interfaces, e.g., the -`http.RoundTripper`. On top of that, we'll create all the -required interfaces to achive the measurement goals mentioned above. - -We are of course writing test templates in `probe-engine` -anyway, because we need additional abstraction, but we can -take advantage of the fact that the API exposed by this package -is stable by definition, because it mimics the stdlib. Also, -for many experiments we can collect information pertaining -to TCP, DNS, TLS, and HTTP with a single call to `netx`. - -This code used to live at `github.com/ooni/netx`. On 2020-03-02 -we merged github.com/ooni/netx@4f8d645bce6466bb into `probe-engine` -because it was more practical and enabled easier refactoring. - -## Definitions - -Consistently with Go's terminology, we define -_HTTP round trip_ the process where we get a request -to send; we find a suitable connection for sending -it, or we create one; we send headers and -possibly body; and we receive response headers. - -We also define _HTTP transaction_ the process starting -with an HTTP round trip and terminating by reading -the full response body. - -We define _netx replacement_ a Go struct of interface that -has the same interface of a Go standard library object -but additionally performs measurements. - -## Enhanced error handling - -This library MUST wrap `error` such that: - -1. we can classify all errors we care about; and - -2. we can map them to major operations. - -The `github.com/ooni/netx/modelx` MUST contain a wrapper for -Go `error` named `ErrWrapper` that is at least like: - -```Go -type ErrWrapper struct { - Failure string // error classification - Operation string // operation that caused error - WrappedErr error // the original error -} - -func (e *ErrWrapper) Error() string { - return e.Failure -} -``` - -Where `Failure` is one of the errors we care about, i.e.: - -- `connection_refused`: ECONNREFUSED -- `connection_reset`: ECONNRESET -- `dns_bogon_error`: detected bogon in DNS reply -- `dns_nxdomain_error`: NXDOMAIN in DNS reply -- `eof_error`: unexpected EOF on connection -- `generic_timeout_error`: some timer has expired -- `ssl_invalid_hostname`: certificate not valid for SNI -- `ssl_unknown_autority`: cannot find CA validating certificate -- `ssl_invalid_certificate`: e.g. certificate expired -- `unknown_failure `: any other error - -Note that we care about bogons in DNS replies because they are -often used to censor specific websites. - -And where `Operation` is one of: - -- `resolve`: domain name resolution -- `connect`: TCP connect -- `tls_handshake`: TLS handshake -- `http_round_trip`: reading/writing HTTP - -The code in this library MUST wrap returned errors such -that we can cast back to `ErrWrapper` during the analysis -phase, using Go 1.13 `errors` library as follows: - -```Go -var wrapper *modelx.ErrWrapper -if errors.As(err, &wrapper) == true { - // Do something with the error -} -``` - -## Netx replacements - -We want to provide netx replacements for the following -interfaces in the Go standard library: - -1. `http.RoundTripper` - -2. `http.Client` - -3. `net.Dialer` - -4. `net.Resolver` - -Accordingly, we'll define the following interfaces in -the `github.com/ooni/probe-engine/netx/modelx` package: - -```Go -type DNSResolver interface { - LookupHost(ctx context.Context, hostname string) ([]string, error) -} - -type Dialer interface { - Dial(network, address string) (net.Conn, error) - DialContext(ctx context.Context, network, address string) (net.Conn, error) -} - -type TLSDialer interface { - DialTLS(network, address string) (net.Conn, error) - DialTLSContext(ctx context.Context, network, address string) (net.Conn, error) -} -``` - -We won't need an interface for `http.RoundTripper` -because it is already an interface, so we'll just use it. - -Our replacements will implement these interfaces. - -Using an API compatible with Go's standard libary makes -it possible to use, say, our `net.Dialer` replacement with -other libraries. Both `http.Transport` and -`gorilla/websocket`'s `websocket.Dialer` have -functions like `Dial` and `DialContext` that can be -overriden. By overriding such function pointers, -we could use our replacements instead of the standard -libary, thus we could collect measurements while -using third party code to implement specific protocols. - -Also, using interfaces allows us to combine code -quite easily. For example, a resolver that detects -bogons is easily implemented as a wrapper around -another resolve that performs the real resolution. - -## Dispatching events - -The `github.com/ooni/netx/modelx` package will define -an handler for low level events as: - -```Go -type Handler interface { - OnMeasurement(Measurement) -} -``` - -We will provide a mechanism to bind a specific -handler to a `context.Context` such that the handler -will receive all the measurements caused by code -using such context. This mechanism is like: - -```Go -type MeasurementRoot struct { - Beginning time.Time // the "zero" time - Handler Handler // the handler to use -} -``` - -You will be able to assign a `MeasurementRoot` to -a context by using the following function: - -```Go -func WithMeasurementRoot( - ctx context.Context, root *MeasurementRoot) context.Context -``` - -which will return a clone of the original context -that uses the `MeasurementRoot`. Pass this context to -any method of our replacements to get measurements. - -Given such context, or a subcontext, you can get -back the original `MeasurementRoot` using: - -```Go -func ContextMeasurementRoot(ctx context.Context) *MeasurementRoot -``` - -which will return the context `MeasurementRoot` or -`nil` if none is set into the context. This is how our -internal code gets access to the `MeasurementRoot`. - -## Constructing and configuring replacements - -The `github.com/ooni/probe-engine/netx` package MUST provide an API such -that you can construct and configure a `net.Resolver` replacement -as follows: - -```Go -r, err := netx.NewResolverWithoutHandler(dnsNetwork, dnsAddress) -if err != nil { - log.Fatal("cannot configure specifc resolver") -} -var resolver modelx.DNSResolver = r -// now use resolver ... -``` - -where `DNSNetwork` and `DNSAddress` configure the type -of the resolver as follows: - -- when `DNSNetwork` is `""` or `"system"`, `DNSAddress` does -not matter and we use the system resolver - -- when `DNSNetwork` is `"udp"`, `DNSAddress` is the address -or domain name, with optional port, of the DNS server -(e.g., `8.8.8.8:53`) - -- when `DNSNetwork` is `"tcp"`, `DNSAddress` is the address -or domain name, with optional port, of the DNS server -(e.g., `8.8.8.8:53`) - -- when `DNSNetwork` is `"dot"`, `DNSAddress` is the address -or domain name, with optional port, of the DNS server -(e.g., `8.8.8.8:853`) - -- when `DNSNetwork` is `"doh"`, `DNSAddress` is the URL -of the DNS server (e.g. `https://cloudflare-dns.com/dns-query`) - -When the resolve is not the system one, we'll also be able -to emit events when performing resolution. Otherwise, we'll -just emit the `DNSResolveDone` event defined below. - -Any resolver returned by this function may be configured to return the -`dns_bogon_error` if any `LookupHost` lookup returns a bogon IP. - -The package will also contain this function: - -```Go -func ChainResolvers( - primary, secondary modelx.DNSResolver) modelx.DNSResolver -``` - -where you can create a new resolver where `secondary` will be -invoked whenever `primary` fails. This functionality allows -us to be more resilient and bypass automatically certain types -of censorship, e.g., a resolver returning a bogon. - -The `github.com/ooni/probe-engine/netx` package MUST also provide an API such -that you can construct and configure a `net.Dialer` replacement -as follows: - -```Go -d := netx.NewDialerWithoutHandler() -d.SetResolver(resolver) -d.ForceSpecificSNI("www.kernel.org") -d.SetCABundle("/etc/ssl/cert.pem") -d.ForceSkipVerify() -var dialer modelx.Dialer = d -// now use dialer -``` - -where `SetResolver` allows you to change the resolver, -`ForceSpecificSNI` forces the TLS dials to use such SNI -instead of using the provided domain, `SetCABundle` -allows to set a specific CA bundle, and `ForceSkipVerify` -allows to disable certificate verification. All these funcs -MUST NOT be invoked once you're using the dialer. - -The `github.com/ooni/probe-engine/netx` package MUST contain -code so that we can do: - -```Go -t := netx.NewHTTPTransportWithProxyFunc( - http.ProxyFromEnvironment, -) -t.SetResolver(resolver) -t.ForceSpecificSNI("www.kernel.org") -t.SetCABundle("/etc/ssl/cert.pem") -t.ForceSkipVerify() -var transport http.RoundTripper = t -// now use transport -``` - -where the functions have the same semantics as the -namesake functions described before and the same caveats. - -We also have syntactic sugar on top of that and legacy -methods, but this fully describes the design. - -## Structure of events - -The `github.com/ooni/probe-engine/netx/modelx` will contain the -definition of low-level events. We are interested in -knowing the following: - -1. the timing and result of each I/O operation. - -2. the timing of HTTP events occurring during the -lifecycle of an HTTP request. - -3. the timing and result of the TLS handshake including -the negotiated TLS version and other details such as -what certificates the server has provided. - -4. DNS events, e.g. queries and replies, generated -as part of using DoT and DoH. - -We will represent time as a `time.Duration` since the -beginning configured either in the context or when -constructing an object. The `modelx` package will also -define the `Measurement` event as follows: - -```Go -type Measurement struct { - Connect *ConnectEvent - HTTPConnectionReady *HTTPConnectionReadyEvent - HTTPRoundTripDone *HTTPRoundTripDoneEvent - ResolveDone *ResolveDoneEvent - TLSHandshakeDone *TLSHandshakeDoneEvent -} -``` - -The events above MUST always be present, but more -events will likely be available. The structure -will contain a pointer for every event that -we support. The events processing code will check -what pointer or pointers are not `nil` to known -which event or events have occurred. - -To simplify joining events together the following holds: - -1. when we're establishing a new connection there is a nonzero -`DialID` shared by `Connect` and `ResolveDone` - -2. a new connection has a nonzero `ConnID` that is emitted -as part of a successful `Connect` event - -3. during an HTTP transaction there is a nonzero `TransactionID` -shared by `HTTPConnectionReady` and `HTTPRoundTripDone` - -4. if the TLS handshake is invoked by HTTP code it will have a -nonzero `TrasactionID` otherwise a nonzero `ConnID` - -5. the `HTTPConnectionReady` will also see the `ConnID` - -6. when a transaction starts dialing, it will pass its -`TransactionID` to `ResolveDone` and `Connect` - -7. when we're dialing a connection for DoH, we pass the `DialID` -to the `HTTPConnectionReady` event as well - -Because of the following rules, it should always be possible -to bind together events. Also, we define more events than the -above, but they are ancillary to the above events. Also, the -main reason why `HTTPConnectionReady` is here is because it is -the event allowing to bind `ConnID` and `TransactionID`. diff --git a/internal/engine/legacy/netx/dialer.go b/internal/engine/legacy/netx/dialer.go deleted file mode 100644 index 8329052..0000000 --- a/internal/engine/legacy/netx/dialer.go +++ /dev/null @@ -1,203 +0,0 @@ -// Package netx contains OONI's net extensions. -package netx - -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "net" - "os" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/errorsx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/dialer" - "github.com/ooni/probe-cli/v3/internal/engine/netx/tlsdialer" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// Dialer performs measurements while dialing. -type Dialer struct { - Beginning time.Time - Handler modelx.Handler - Resolver modelx.DNSResolver - TLSConfig *tls.Config -} - -func newDialer(beginning time.Time, handler modelx.Handler) *Dialer { - return &Dialer{ - Beginning: beginning, - Handler: handler, - Resolver: newResolverSystem(), - TLSConfig: new(tls.Config), - } -} - -// NewDialer creates a new Dialer instance. -func NewDialer() *Dialer { - return newDialer(time.Now(), handlers.NoHandler) -} - -// Dial creates a TCP or UDP connection. See net.Dial docs. -func (d *Dialer) Dial(network, address string) (net.Conn, error) { - return d.DialContext(context.Background(), network, address) -} - -func maybeWithMeasurementRoot( - ctx context.Context, beginning time.Time, handler modelx.Handler, -) context.Context { - if modelx.ContextMeasurementRoot(ctx) != nil { - return ctx - } - return modelx.WithMeasurementRoot(ctx, &modelx.MeasurementRoot{ - Beginning: beginning, - Handler: handler, - }) -} - -// newDNSDialer creates a new DNS dialer using the following chain: -// -// - DNSDialer (topmost) -// - EmitterDialer -// - ErrorWrapperDialer -// - ByteCountingDialer -// - dialer.Default -// -// If you have others needs, manually build the chain you need. -func newDNSDialer(resolver dialer.Resolver) dialer.Dialer { - // Implementation note: we're wrapping the result of dialer.New - // on the outside, while previously we were puttting the - // EmitterDialer before the DNSDialer (see the above comment). - // - // Yet, this is fine because the only experiment which is - // using this code is tor, for which it doesn't matter. - // - // Also (and I am always scared to write this kind of - // comments), we should rewrite tor soon. - return &EmitterDialer{dialer.New(&dialer.Config{ - ContextByteCounting: true, - }, resolver)} -} - -// DialContext is like Dial but the context allows to interrupt a -// pending connection attempt at any time. -func (d *Dialer) DialContext( - ctx context.Context, network, address string, -) (conn net.Conn, err error) { - ctx = maybeWithMeasurementRoot(ctx, d.Beginning, d.Handler) - return newDNSDialer(d.Resolver).DialContext(ctx, network, address) -} - -// DialTLS is like Dial, but creates TLS connections. -func (d *Dialer) DialTLS(network, address string) (net.Conn, error) { - return d.DialTLSContext(context.Background(), network, address) -} - -// newTLSDialer creates a new TLSDialer using: -// -// - EmitterTLSHandshaker (topmost) -// - ErrorWrapperTLSHandshaker -// - TimeoutTLSHandshaker -// - SystemTLSHandshaker -// -// If you have others needs, manually build the chain you need. -func newTLSDialer(d dialer.Dialer, config *tls.Config) *netxlite.TLSDialerLegacy { - return &netxlite.TLSDialerLegacy{ - Config: config, - Dialer: netxlite.NewDialerLegacyAdapter(d), - TLSHandshaker: tlsdialer.EmitterTLSHandshaker{ - TLSHandshaker: &errorsx.ErrorWrapperTLSHandshaker{ - TLSHandshaker: &netxlite.TLSHandshakerConfigurable{}, - }, - }, - } -} - -// DialTLSContext is like DialTLS, but with context -func (d *Dialer) DialTLSContext( - ctx context.Context, network, address string, -) (net.Conn, error) { - ctx = maybeWithMeasurementRoot(ctx, d.Beginning, d.Handler) - return newTLSDialer( - newDNSDialer(d.Resolver), - d.TLSConfig, - ).DialTLSContext(ctx, network, address) -} - -// SetCABundle configures the dialer to use a specific CA bundle. This -// function is not goroutine safe. Make sure you call it before starting -// to use this specific dialer. -func (d *Dialer) SetCABundle(path string) error { - cert, err := os.ReadFile(path) - if err != nil { - return err - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(cert) { - return errors.New("AppendCertsFromPEM failed") - } - d.TLSConfig.RootCAs = pool - return nil -} - -// ForceSpecificSNI forces using a specific SNI. -func (d *Dialer) ForceSpecificSNI(sni string) error { - d.TLSConfig.ServerName = sni - return nil -} - -// ForceSkipVerify forces to skip certificate verification -func (d *Dialer) ForceSkipVerify() error { - d.TLSConfig.InsecureSkipVerify = true - return nil -} - -// ConfigureDNS configures the DNS resolver. The network argument -// selects the type of resolver. The address argument indicates the -// resolver address and depends on the network. -// -// This functionality is not goroutine safe. You should only change -// the DNS settings before starting to use the Dialer. -// -// The following is a list of all the possible network values: -// -// - "": behaves exactly like "system" -// -// - "system": this indicates that Go should use the system resolver -// and prevents us from seeing any DNS packet. The value of the -// address parameter is ignored when using "system". If you do -// not ConfigureDNS, this is the default resolver used. -// -// - "udp": indicates that we should send queries using UDP. In this -// case the address is a host, port UDP endpoint. -// -// - "tcp": like "udp" but we use TCP. -// -// - "dot": we use DNS over TLS (DoT). In this case the address is -// the domain name of the DoT server. -// -// - "doh": we use DNS over HTTPS (DoH). In this case the address is -// the URL of the DoH server. -// -// For example: -// -// d.ConfigureDNS("system", "") -// d.ConfigureDNS("udp", "8.8.8.8:53") -// d.ConfigureDNS("tcp", "8.8.8.8:53") -// d.ConfigureDNS("dot", "dns.quad9.net") -// d.ConfigureDNS("doh", "https://cloudflare-dns.com/dns-query") -func (d *Dialer) ConfigureDNS(network, address string) error { - r, err := newResolver(d.Beginning, d.Handler, network, address) - if err == nil { - d.Resolver = r - } - return err -} - -// SetResolver is a more flexible way of configuring a resolver -// that should perhaps be used instead of ConfigureDNS. -func (d *Dialer) SetResolver(r modelx.DNSResolver) { - d.Resolver = r -} diff --git a/internal/engine/legacy/netx/dialer_test.go b/internal/engine/legacy/netx/dialer_test.go deleted file mode 100644 index 186fdd8..0000000 --- a/internal/engine/legacy/netx/dialer_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package netx_test - -import ( - "crypto/x509" - "errors" - "testing" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" -) - -func TestDialerDial(t *testing.T) { - dialer := netx.NewDialer() - conn, err := dialer.Dial("tcp", "www.google.com:80") - if err != nil { - t.Fatal(err) - } - conn.Close() -} - -func TestDialerDialWithCustomResolver(t *testing.T) { - dialer := netx.NewDialer() - resolver, err := netx.NewResolver("tcp", "1.1.1.1:53") - if err != nil { - t.Fatal(err) - } - dialer.SetResolver(resolver) - conn, err := dialer.Dial("tcp", "www.google.com:80") - if err != nil { - t.Fatal(err) - } - conn.Close() -} - -func TestDialerDialWithConfigureDNS(t *testing.T) { - dialer := netx.NewDialer() - err := dialer.ConfigureDNS("tcp", "1.1.1.1:53") - if err != nil { - t.Fatal(err) - } - conn, err := dialer.Dial("tcp", "www.google.com:80") - if err != nil { - t.Fatal(err) - } - conn.Close() -} - -func TestDialerDialTLS(t *testing.T) { - dialer := netx.NewDialer() - conn, err := dialer.DialTLS("tcp", "www.google.com:443") - if err != nil { - t.Fatal(err) - } - conn.Close() -} - -func TestDialerDialTLSForceSkipVerify(t *testing.T) { - dialer := netx.NewDialer() - dialer.ForceSkipVerify() - conn, err := dialer.DialTLS("tcp", "self-signed.badssl.com:443") - if err != nil { - t.Fatal(err) - } - conn.Close() -} - -func TestDialerSetCABundleNonexisting(t *testing.T) { - dialer := netx.NewDialer() - err := dialer.SetCABundle("testdata/cacert-nonexistent.pem") - if err == nil { - t.Fatal("expected an error here") - } -} - -func TestDialerSetCABundleInvalid(t *testing.T) { - dialer := netx.NewDialer() - err := dialer.SetCABundle("testdata/cacert-invalid.pem") - if err == nil { - t.Fatal("expected an error here") - } -} - -func TestDialerSetCABundleWAI(t *testing.T) { - dialer := netx.NewDialer() - err := dialer.SetCABundle("testdata/cacert.pem") - if err != nil { - t.Fatal(err) - } - conn, err := dialer.DialTLS("tcp", "www.google.com:443") - if err == nil { - t.Fatal("expected an error here") - } - var target x509.UnknownAuthorityError - if errors.As(err, &target) == false { - t.Fatal("not the error we expected") - } - if conn != nil { - t.Fatal("expected a nil conn here") - } -} - -func TestDialerForceSpecificSNI(t *testing.T) { - dialer := netx.NewDialer() - err := dialer.ForceSpecificSNI("www.facebook.com") - if err != nil { - t.Fatal(err) - } - conn, err := dialer.DialTLS("tcp", "www.google.com:443") - if err == nil { - t.Fatal("expected an error here") - } - var target x509.HostnameError - if errors.As(err, &target) == false { - t.Fatal("not the error we expected") - } - if conn != nil { - t.Fatal("expected a nil connection here") - } -} diff --git a/internal/engine/legacy/netx/emitterdialer.go b/internal/engine/legacy/netx/emitterdialer.go deleted file mode 100644 index f005051..0000000 --- a/internal/engine/legacy/netx/emitterdialer.go +++ /dev/null @@ -1,94 +0,0 @@ -package netx - -import ( - "context" - "net" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/dialer" -) - -// EmitterDialer is a Dialer that emits events -type EmitterDialer struct { - dialer.Dialer -} - -// DialContext implements Dialer.DialContext -func (d EmitterDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - start := time.Now() - conn, err := d.Dialer.DialContext(ctx, network, address) - stop := time.Now() - root := modelx.ContextMeasurementRootOrDefault(ctx) - root.Handler.OnMeasurement(modelx.Measurement{ - Connect: &modelx.ConnectEvent{ - DurationSinceBeginning: stop.Sub(root.Beginning), - Error: err, - Network: network, - RemoteAddress: address, - SyscallDuration: stop.Sub(start), - }, - }) - if err != nil { - return nil, err - } - return EmitterConn{ - Conn: conn, - Beginning: root.Beginning, - Handler: root.Handler, - }, nil -} - -// EmitterConn is a net.Conn used to emit events -type EmitterConn struct { - net.Conn - Beginning time.Time - Handler modelx.Handler -} - -// Read implements net.Conn.Read -func (c EmitterConn) Read(b []byte) (n int, err error) { - start := time.Now() - n, err = c.Conn.Read(b) - stop := time.Now() - c.Handler.OnMeasurement(modelx.Measurement{ - Read: &modelx.ReadEvent{ - DurationSinceBeginning: stop.Sub(c.Beginning), - Error: err, - NumBytes: int64(n), - SyscallDuration: stop.Sub(start), - }, - }) - return -} - -// Write implements net.Conn.Write -func (c EmitterConn) Write(b []byte) (n int, err error) { - start := time.Now() - n, err = c.Conn.Write(b) - stop := time.Now() - c.Handler.OnMeasurement(modelx.Measurement{ - Write: &modelx.WriteEvent{ - DurationSinceBeginning: stop.Sub(c.Beginning), - Error: err, - NumBytes: int64(n), - SyscallDuration: stop.Sub(start), - }, - }) - return -} - -// Close implements net.Conn.Close -func (c EmitterConn) Close() (err error) { - start := time.Now() - err = c.Conn.Close() - stop := time.Now() - c.Handler.OnMeasurement(modelx.Measurement{ - Close: &modelx.CloseEvent{ - DurationSinceBeginning: stop.Sub(c.Beginning), - Error: err, - SyscallDuration: stop.Sub(start), - }, - }) - return -} diff --git a/internal/engine/legacy/netx/emitterdialer_test.go b/internal/engine/legacy/netx/emitterdialer_test.go deleted file mode 100644 index d0576d2..0000000 --- a/internal/engine/legacy/netx/emitterdialer_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package netx - -import ( - "context" - "errors" - "io" - "net" - "testing" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/model/mocks" -) - -func TestEmitterFailure(t *testing.T) { - ctx := context.Background() - saver := &handlers.SavingHandler{} - ctx = modelx.WithMeasurementRoot(ctx, &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: saver, - }) - d := EmitterDialer{Dialer: &mocks.Dialer{ - MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) { - return nil, io.EOF - }, - }} - conn, err := d.DialContext(ctx, "tcp", "www.google.com:443") - if !errors.Is(err, io.EOF) { - t.Fatal("not the error we expected") - } - if conn != nil { - t.Fatal("expected a nil conn here") - } - events := saver.Read() - if len(events) != 1 { - t.Fatal("unexpected number of events saved") - } - if events[0].Connect == nil { - t.Fatal("expected non nil Connect") - } - conninfo := events[0].Connect - emitterCheckConnectEventCommon(t, conninfo, io.EOF) -} - -func emitterCheckConnectEventCommon( - t *testing.T, conninfo *modelx.ConnectEvent, err error) { - if conninfo.DurationSinceBeginning == 0 { - t.Fatal("unexpected DurationSinceBeginning value") - } - if !errors.Is(conninfo.Error, err) { - t.Fatal("unexpected Error value") - } - if conninfo.Network != "tcp" { - t.Fatal("unexpected Network value") - } - if conninfo.RemoteAddress != "www.google.com:443" { - t.Fatal("unexpected Network value") - } - if conninfo.SyscallDuration == 0 { - t.Fatal("unexpected SyscallDuration value") - } -} - -func TestEmitterSuccess(t *testing.T) { - ctx := context.Background() - saver := &handlers.SavingHandler{} - ctx = modelx.WithMeasurementRoot(ctx, &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: saver, - }) - d := EmitterDialer{Dialer: &mocks.Dialer{ - MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) { - return &mocks.Conn{ - MockRead: func(b []byte) (int, error) { - return 0, io.EOF - }, - MockWrite: func(b []byte) (int, error) { - return 0, io.EOF - }, - MockClose: func() error { - return io.EOF - }, - MockLocalAddr: func() net.Addr { - return &net.TCPAddr{Port: 12345} - }, - }, nil - }, - }} - conn, err := d.DialContext(ctx, "tcp", "www.google.com:443") - if err != nil { - t.Fatal("we expected no error") - } - if conn == nil { - t.Fatal("expected a non-nil conn here") - } - conn.Read(nil) - conn.Write(nil) - conn.Close() - events := saver.Read() - if len(events) != 4 { - t.Fatal("unexpected number of events saved") - } - if events[0].Connect == nil { - t.Fatal("expected non nil Connect") - } - conninfo := events[0].Connect - emitterCheckConnectEventCommon(t, conninfo, nil) - if events[1].Read == nil { - t.Fatal("expected non nil Read") - } - emitterCheckReadEvent(t, events[1].Read) - if events[2].Write == nil { - t.Fatal("expected non nil Write") - } - emitterCheckWriteEvent(t, events[2].Write) - if events[3].Close == nil { - t.Fatal("expected non nil Close") - } - emitterCheckCloseEvent(t, events[3].Close) -} - -func emitterCheckReadEvent(t *testing.T, ev *modelx.ReadEvent) { - if ev.DurationSinceBeginning == 0 { - t.Fatal("unexpected DurationSinceBeginning") - } - if !errors.Is(ev.Error, io.EOF) { - t.Fatal("unexpected Error") - } - if ev.NumBytes != 0 { - t.Fatal("unexpected NumBytes") - } - if ev.SyscallDuration == 0 { - t.Fatal("unexpected SyscallDuration") - } -} - -func emitterCheckWriteEvent(t *testing.T, ev *modelx.WriteEvent) { - if ev.DurationSinceBeginning == 0 { - t.Fatal("unexpected DurationSinceBeginning") - } - if !errors.Is(ev.Error, io.EOF) { - t.Fatal("unexpected Error") - } - if ev.NumBytes != 0 { - t.Fatal("unexpected NumBytes") - } - if ev.SyscallDuration == 0 { - t.Fatal("unexpected SyscallDuration") - } -} - -func emitterCheckCloseEvent(t *testing.T, ev *modelx.CloseEvent) { - if ev.DurationSinceBeginning == 0 { - t.Fatal("unexpected DurationSinceBeginning") - } - if !errors.Is(ev.Error, io.EOF) { - t.Fatal("unexpected Error") - } - if ev.SyscallDuration == 0 { - t.Fatal("unexpected SyscallDuration") - } -} diff --git a/internal/engine/legacy/netx/handlers/handlers.go b/internal/engine/legacy/netx/handlers/handlers.go deleted file mode 100644 index 4ad7248..0000000 --- a/internal/engine/legacy/netx/handlers/handlers.go +++ /dev/null @@ -1,52 +0,0 @@ -// Package handlers contains default modelx.Handler handlers. -package handlers - -import ( - "encoding/json" - "fmt" - "sync" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -type stdoutHandler struct{} - -func (stdoutHandler) OnMeasurement(m modelx.Measurement) { - data, err := json.Marshal(m) - runtimex.PanicOnError(err, "unexpected json.Marshal failure") - fmt.Printf("%s\n", string(data)) -} - -// StdoutHandler is a Handler that logs on stdout. -var StdoutHandler stdoutHandler - -type noHandler struct{} - -func (noHandler) OnMeasurement(m modelx.Measurement) { -} - -// NoHandler is a Handler that does not print anything -var NoHandler noHandler - -// SavingHandler saves the events it receives. -type SavingHandler struct { - mu sync.Mutex - v []modelx.Measurement -} - -// OnMeasurement implements modelx.Handler.OnMeasurement -func (sh *SavingHandler) OnMeasurement(ev modelx.Measurement) { - sh.mu.Lock() - sh.v = append(sh.v, ev) - sh.mu.Unlock() -} - -// Read extracts the saved events -func (sh *SavingHandler) Read() []modelx.Measurement { - sh.mu.Lock() - v := sh.v - sh.v = nil - sh.mu.Unlock() - return v -} diff --git a/internal/engine/legacy/netx/handlers/handlers_test.go b/internal/engine/legacy/netx/handlers/handlers_test.go deleted file mode 100644 index f8c49a9..0000000 --- a/internal/engine/legacy/netx/handlers/handlers_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package handlers_test - -import ( - "testing" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" -) - -func TestGood(t *testing.T) { - handlers.NoHandler.OnMeasurement(modelx.Measurement{}) - handlers.StdoutHandler.OnMeasurement(modelx.Measurement{}) - saver := handlers.SavingHandler{} - saver.OnMeasurement(modelx.Measurement{}) - events := saver.Read() - if len(events) != 1 { - t.Fatal("invalid number of events") - } -} diff --git a/internal/engine/legacy/netx/http.go b/internal/engine/legacy/netx/http.go deleted file mode 100644 index c03e976..0000000 --- a/internal/engine/legacy/netx/http.go +++ /dev/null @@ -1,207 +0,0 @@ -package netx - -import ( - "net/http" - "net/url" - "time" - - errorsxlegacy "github.com/ooni/probe-cli/v3/internal/engine/legacy/errorsx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/oldhttptransport" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "golang.org/x/net/http2" -) - -// HTTPTransport performs single HTTP transactions and emits -// measurement events as they happen. -type HTTPTransport struct { - Beginning time.Time - Dialer *Dialer - Handler modelx.Handler - Transport *http.Transport - roundTripper http.RoundTripper -} - -func newHTTPTransport( - beginning time.Time, - handler modelx.Handler, - dialer *Dialer, - disableKeepAlives bool, - proxyFunc func(*http.Request) (*url.URL, error), -) *HTTPTransport { - baseTransport := &http.Transport{ - // The following values are copied from Go 1.12 docs and match - // what should be used by the default transport - ExpectContinueTimeout: 1 * time.Second, - IdleConnTimeout: 90 * time.Second, - MaxIdleConns: 100, - Proxy: proxyFunc, - TLSHandshakeTimeout: 10 * time.Second, - DisableKeepAlives: disableKeepAlives, - } - ooniTransport := oldhttptransport.New(baseTransport) - // Configure h2 and make sure that the custom TLSConfig we use for dialing - // is actually compatible with upgrading to h2. (This mainly means we - // need to make sure we include "h2" in the NextProtos array.) Because - // http2.ConfigureTransport only returns error when we have already - // configured http2, it is safe to ignore the return value. - http2.ConfigureTransport(baseTransport) - // Since we're not going to use our dialer for TLS, the main purpose of - // the following line is to make sure ForseSpecificSNI has impact on the - // config we are going to use when doing TLS. The code is as such since - // we used to force net/http through using dialer.DialTLS. - dialer.TLSConfig = baseTransport.TLSClientConfig - // Arrange the configuration such that we always use `dialer` for dialing - // cleartext connections. The net/http code will dial TLS connections. - baseTransport.DialContext = dialer.DialContext - // Better for Cloudflare DNS and also better because we have less - // noisy events and we can better understand what happened. - baseTransport.MaxConnsPerHost = 1 - // The following (1) reduces the number of headers that Go will - // automatically send for us and (2) ensures that we always receive - // back the true headers, such as Content-Length. This change is - // functional to OONI's goal of observing the network. - baseTransport.DisableCompression = true - return &HTTPTransport{ - Beginning: beginning, - Dialer: dialer, - Handler: handler, - Transport: baseTransport, - roundTripper: ooniTransport, - } -} - -// RoundTrip executes a single HTTP transaction, returning -// a Response for the provided Request. -func (t *HTTPTransport) RoundTrip( - req *http.Request, -) (resp *http.Response, err error) { - ctx := maybeWithMeasurementRoot(req.Context(), t.Beginning, t.Handler) - req = req.WithContext(ctx) - resp, err = t.roundTripper.RoundTrip(req) - // For safety wrap the error as modelx.HTTPRoundTripOperation but this - // will only be used if the error chain does not contain any - // other major operation failure. See netxlite.ErrWrapper. - err = errorsxlegacy.SafeErrWrapperBuilder{ - Error: err, - Operation: netxlite.HTTPRoundTripOperation, - }.MaybeBuild() - return resp, err -} - -// CloseIdleConnections closes the idle connections. -func (t *HTTPTransport) CloseIdleConnections() { - // Adapted from net/http code - type closeIdler interface { - CloseIdleConnections() - } - if tr, ok := t.roundTripper.(closeIdler); ok { - tr.CloseIdleConnections() - } -} - -// NewHTTPTransportWithProxyFunc creates a transport without any -// handler attached using the specified proxy func. -func NewHTTPTransportWithProxyFunc( - proxyFunc func(*http.Request) (*url.URL, error), -) *HTTPTransport { - return newHTTPTransport(time.Now(), handlers.NoHandler, NewDialer(), false, proxyFunc) -} - -// NewHTTPTransport creates a new HTTP transport. -func NewHTTPTransport() *HTTPTransport { - return NewHTTPTransportWithProxyFunc(http.ProxyFromEnvironment) -} - -// ConfigureDNS is exactly like netx.Dialer.ConfigureDNS. -func (t *HTTPTransport) ConfigureDNS(network, address string) error { - return t.Dialer.ConfigureDNS(network, address) -} - -// SetResolver is exactly like netx.Dialer.SetResolver. -func (t *HTTPTransport) SetResolver(r modelx.DNSResolver) { - t.Dialer.SetResolver(r) -} - -// SetCABundle internally calls netx.Dialer.SetCABundle and -// therefore it has the same caveats and limitations. -func (t *HTTPTransport) SetCABundle(path string) error { - return t.Dialer.SetCABundle(path) -} - -// ForceSpecificSNI forces using a specific SNI. -func (t *HTTPTransport) ForceSpecificSNI(sni string) error { - return t.Dialer.ForceSpecificSNI(sni) -} - -// ForceSkipVerify forces to skip certificate verification -func (t *HTTPTransport) ForceSkipVerify() error { - return t.Dialer.ForceSkipVerify() -} - -// HTTPClient is a replacement for http.HTTPClient. -type HTTPClient struct { - // HTTPClient is the underlying client. Pass this client to existing code - // that expects an *http.HTTPClient. For this reason we can't embed it. - HTTPClient *http.Client - - // Transport is the transport configured by NewClient to be used - // by the HTTPClient field. - Transport *HTTPTransport -} - -// NewHTTPClientWithProxyFunc creates a new client using the -// specified proxyFunc for handling proxying. -func NewHTTPClientWithProxyFunc( - proxyFunc func(*http.Request) (*url.URL, error), -) *HTTPClient { - transport := NewHTTPTransportWithProxyFunc(proxyFunc) - return &HTTPClient{ - HTTPClient: &http.Client{Transport: transport}, - Transport: transport, - } -} - -// NewHTTPClient creates a new client instance. -func NewHTTPClient() *HTTPClient { - return NewHTTPClientWithProxyFunc(http.ProxyFromEnvironment) -} - -// NewHTTPClientWithoutProxy creates a new client instance that -// does not use any kind of proxy. -func NewHTTPClientWithoutProxy() *HTTPClient { - return NewHTTPClientWithProxyFunc(nil) -} - -// ConfigureDNS internally calls netx.Dialer.ConfigureDNS and -// therefore it has the same caveats and limitations. -func (c *HTTPClient) ConfigureDNS(network, address string) error { - return c.Transport.ConfigureDNS(network, address) -} - -// SetResolver internally calls netx.Dialer.SetResolver -func (c *HTTPClient) SetResolver(r modelx.DNSResolver) { - c.Transport.SetResolver(r) -} - -// SetCABundle internally calls netx.Dialer.SetCABundle and -// therefore it has the same caveats and limitations. -func (c *HTTPClient) SetCABundle(path string) error { - return c.Transport.SetCABundle(path) -} - -// ForceSpecificSNI forces using a specific SNI. -func (c *HTTPClient) ForceSpecificSNI(sni string) error { - return c.Transport.ForceSpecificSNI(sni) -} - -// ForceSkipVerify forces to skip certificate verification -func (c *HTTPClient) ForceSkipVerify() error { - return c.Transport.ForceSkipVerify() -} - -// CloseIdleConnections closes the idle connections. -func (c *HTTPClient) CloseIdleConnections() { - c.Transport.CloseIdleConnections() -} diff --git a/internal/engine/legacy/netx/http_test.go b/internal/engine/legacy/netx/http_test.go deleted file mode 100644 index 1d97d0d..0000000 --- a/internal/engine/legacy/netx/http_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package netx_test - -import ( - "context" - "crypto/x509" - "errors" - "net" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func dowithclient(t *testing.T, client *netx.HTTPClient) { - defer client.CloseIdleConnections() - resp, err := client.HTTPClient.Get("https://www.google.com") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - _, err = netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } -} - -func TestHTTPClient(t *testing.T) { - client := netx.NewHTTPClient() - dowithclient(t, client) -} - -func TestHTTPClientAndTransport(t *testing.T) { - client := netx.NewHTTPClient() - client.Transport = netx.NewHTTPTransport() - dowithclient(t, client) -} - -func TestHTTPClientConfigureDNS(t *testing.T) { - client := netx.NewHTTPClientWithoutProxy() - err := client.ConfigureDNS("udp", "1.1.1.1:53") - if err != nil { - t.Fatal(err) - } - dowithclient(t, client) -} - -func TestHTTPClientSetResolver(t *testing.T) { - client := netx.NewHTTPClientWithoutProxy() - client.SetResolver(new(net.Resolver)) - dowithclient(t, client) -} - -func TestHTTPClientSetCABundle(t *testing.T) { - client := netx.NewHTTPClientWithoutProxy() - err := client.SetCABundle("testdata/cacert.pem") - if err != nil { - t.Fatal(err) - } - resp, err := client.HTTPClient.Get("https://www.google.com") - var target x509.UnknownAuthorityError - if errors.As(err, &target) == false { - t.Fatal("not the error we expected") - } - if resp != nil { - t.Fatal("expected a nil conn here") - } -} - -func TestHTTPClientForceSpecificSNI(t *testing.T) { - client := netx.NewHTTPClientWithoutProxy() - err := client.ForceSpecificSNI("www.facebook.com") - if err != nil { - t.Fatal(err) - } - resp, err := client.HTTPClient.Get("https://www.google.com") - var target x509.HostnameError - if errors.As(err, &target) == false { - t.Fatal("not the error we expected") - } - if resp != nil { - t.Fatal("expected a nil response here") - } -} - -func TestHTTPClientForceSkipVerify(t *testing.T) { - client := netx.NewHTTPClientWithoutProxy() - client.ForceSkipVerify() - resp, err := client.HTTPClient.Get("https://self-signed.badssl.com/") - if err != nil { - t.Fatal(err) - } - if resp == nil { - t.Fatal("expected non nil response here") - } -} - -func TestHTTPNewClientProxy(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(451) - })) - defer server.Close() - client := netx.NewHTTPClientWithoutProxy() - httpProxyTestMain(t, client.HTTPClient, 200) - client = netx.NewHTTPClientWithProxyFunc(func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }) - httpProxyTestMain(t, client.HTTPClient, 451) -} - -const httpProxyTestsURL = "http://explorer.ooni.org" - -func httpProxyTestMain(t *testing.T, client *http.Client, expect int) { - req, err := http.NewRequest("GET", httpProxyTestsURL, nil) - if err != nil { - t.Fatal(err) - } - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - _, err = netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - if resp.StatusCode != expect { - t.Fatal("unexpected status code") - } -} - -func TestHTTPTransportTimeout(t *testing.T) { - client := &http.Client{Transport: netx.NewHTTPTransport()} - req, err := http.NewRequest("GET", "https://www.google.com", nil) - if err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - req = req.WithContext(ctx) - resp, err := client.Do(req) - if err == nil { - t.Fatal("expected an error here") - } - if !strings.HasSuffix(err.Error(), netxlite.FailureGenericTimeoutError) { - t.Fatal("not the error we expected") - } - if resp != nil { - t.Fatal("expected nil resp here") - } -} - -func TestHTTPTransportFailure(t *testing.T) { - client := &http.Client{Transport: netx.NewHTTPTransport()} - // This fails the request because we attempt to speak cleartext HTTP with - // a server that instead is expecting TLS. - resp, err := client.Get("http://www.google.com:443") - if err == nil { - t.Fatal("expected an error here") - } - if resp != nil { - t.Fatal("expected a nil response here") - } - client.CloseIdleConnections() -} diff --git a/internal/engine/legacy/netx/modelx/modelx.go b/internal/engine/legacy/netx/modelx/modelx.go deleted file mode 100644 index 995d007..0000000 --- a/internal/engine/legacy/netx/modelx/modelx.go +++ /dev/null @@ -1,571 +0,0 @@ -// Package modelx contains the data modelx. -package modelx - -import ( - "context" - "crypto/tls" - "crypto/x509" - "math" - "net" - "net/http" - "net/url" - "time" - - "github.com/miekg/dns" -) - -// Measurement contains zero or more events. Do not assume that at any -// time a Measurement will only contain a single event. When a Measurement -// contains an event, the corresponding pointer is non nil. -// -// All events contain a time measurement, `DurationSinceBeginning`, that -// uses a monotonic clock and is relative to a preconfigured "zero". -type Measurement struct { - // DNS events - ResolveStart *ResolveStartEvent `json:",omitempty"` - DNSQuery *DNSQueryEvent `json:",omitempty"` - DNSReply *DNSReplyEvent `json:",omitempty"` - ResolveDone *ResolveDoneEvent `json:",omitempty"` - - // Syscalls - // - // Because they are syscalls, we don't split them in start/done pairs - // but we record the amount of time in which we were blocked. - Connect *ConnectEvent `json:",omitempty"` - Read *ReadEvent `json:",omitempty"` - Write *WriteEvent `json:",omitempty"` - Close *CloseEvent `json:",omitempty"` - - // TLS events - TLSHandshakeStart *TLSHandshakeStartEvent `json:",omitempty"` - TLSHandshakeDone *TLSHandshakeDoneEvent `json:",omitempty"` - - // HTTP roundtrip events - // - // A round trip starts when we need a connection to send a request - // and ends when we've got the response headers or an error. - HTTPRoundTripStart *HTTPRoundTripStartEvent `json:",omitempty"` - HTTPConnectionReady *HTTPConnectionReadyEvent `json:",omitempty"` - HTTPRequestHeader *HTTPRequestHeaderEvent `json:",omitempty"` - HTTPRequestHeadersDone *HTTPRequestHeadersDoneEvent `json:",omitempty"` - HTTPRequestDone *HTTPRequestDoneEvent `json:",omitempty"` - HTTPResponseStart *HTTPResponseStartEvent `json:",omitempty"` - HTTPRoundTripDone *HTTPRoundTripDoneEvent `json:",omitempty"` - - // HTTP body events - HTTPResponseBodyPart *HTTPResponseBodyPartEvent `json:",omitempty"` - HTTPResponseDone *HTTPResponseDoneEvent `json:",omitempty"` - - // Extension events. - // - // The purpose of these events is to give us some flexibility to - // experiment with message formats before blessing something as - // part of the official API of the library. The intent however is - // to avoid keeping something as an extension for a long time. - Extension *ExtensionEvent `json:",omitempty"` -} - -// CloseEvent is emitted when the CLOSE syscall returns. -type CloseEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error is the error returned by CLOSE. - Error error - - // SyscallDuration is the number of nanoseconds we were - // blocked waiting for the syscall to return. - SyscallDuration time.Duration -} - -// ConnectEvent is emitted when the CONNECT syscall returns. -type ConnectEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error is the error returned by CONNECT. - Error error - - // Network is the network we're dialing for, e.g. "tcp" - Network string - - // RemoteAddress is the remote IP address we're dialing for - RemoteAddress string - - // SyscallDuration is the number of nanoseconds we were - // blocked waiting for the syscall to return. - SyscallDuration time.Duration -} - -// DNSQueryEvent is emitted when we send a DNS query. -type DNSQueryEvent struct { - // Data is the raw data we're sending to the server. - Data []byte - - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Msg is the parsed message we're sending to the server. - Msg *dns.Msg `json:"-"` -} - -// DNSReplyEvent is emitted when we receive byte that are -// successfully parsed into a DNS reply. -type DNSReplyEvent struct { - // Data is the raw data we've received and parsed. - Data []byte - - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Msg is the received parsed message. - Msg *dns.Msg `json:"-"` -} - -// ExtensionEvent is emitted by a netx extension. -type ExtensionEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Key is the unique identifier of the event. A good rule of - // thumb is to use `${packageName}.${messageType}`. - Key string - - // Severity of the emitted message ("WARN", "INFO", "DEBUG") - Severity string - - // Value is the extension dependent message. This message - // has the only requirement of being JSON serializable. - Value interface{} -} - -// HTTPRoundTripStartEvent is emitted when the HTTP transport -// starts the HTTP "round trip". That is, when the transport -// receives from the HTTP client a request to sent. The round -// trip terminates when we receive headers. What we call the -// "transaction" here starts with this event and does not finish -// until we have also finished receiving the response body. -type HTTPRoundTripStartEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Method is the request method - Method string - - // URL is the request URL - URL string -} - -// HTTPConnectionReadyEvent is emitted when the HTTP transport has got -// a connection which is ready for sending the request. -type HTTPConnectionReadyEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration -} - -// HTTPRequestHeaderEvent is emitted when we have written a header, -// where written typically means just "buffered". -type HTTPRequestHeaderEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Key is the header key - Key string - - // Value is the value/values of this header. - Value []string -} - -// HTTPRequestHeadersDoneEvent is emitted when we have written, or more -// correctly, "buffered" all headers. -type HTTPRequestHeadersDoneEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Headers contain the original request headers. This is included - // here to make this event actionable without needing to join it with - // other events, i.e., to simplify logging. - Headers http.Header - - // Method is the original request method. This is here - // for the same reason of Headers. - Method string - - // URL is the original request URL. This is here - // for the same reason of Headers. We use an object - // rather than a string, because here you want to - // use specific subfields directly for logging. - URL *url.URL -} - -// HTTPRequestDoneEvent is emitted when we have sent the request -// body or there has been any failure in sending the request. -type HTTPRequestDoneEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error is non nil if we could not write the request headers or - // some specific part of the body. When this step of writing - // the request fails, of course the whole transaction will fail - // as well. This error however tells you that the issue was - // when sending the request, not when receiving the response. - Error error -} - -// HTTPResponseStartEvent is emitted when we receive the byte from -// the response on the wire. -type HTTPResponseStartEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration -} - -const defaultBodySnapSize int64 = 1 << 20 - -// ComputeBodySnapSize computes the body snap size. If snapSize is negative -// we return MaxInt64. If it's zero we return the default snap size. Otherwise -// the value of snapSize is returned. -func ComputeBodySnapSize(snapSize int64) int64 { - if snapSize < 0 { - snapSize = math.MaxInt64 - } else if snapSize == 0 { - snapSize = defaultBodySnapSize - } - return snapSize -} - -// HTTPRoundTripDoneEvent is emitted at the end of the round trip. Either -// we have an error, or a valid HTTP response. An error could be caused -// either by not being able to send the request or not being able to receive -// the response. Note that here errors are network/TLS/dialing errors or -// protocol violation errors. No status code will cause errors here. -type HTTPRoundTripDoneEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error is the overall result of the round trip. If non-nil, checking - // also the result of HTTPResponseDone helps to disambiguate whether the - // error was in sending the request or receiving the response. - Error error - - // RequestBodySnap contains a snap of the request body. We'll - // not read more than SnapSize bytes of the body. Because typically - // you control the request bodies that you send, perhaps think - // about saving them using other means. - RequestBodySnap []byte - - // RequestHeaders contain the original request headers. This is - // included here to make this event actionable without needing to - // join it with other events, as it's too important. - RequestHeaders http.Header - - // RequestMethod is the original request method. This is here - // for the same reason of RequestHeaders. - RequestMethod string - - // RequestURL is the original request URL. This is here - // for the same reason of RequestHeaders. - RequestURL string - - // ResponseBodySnap is like RequestBodySnap but for the response. You - // can still save the whole body by just reading it, if this - // is something that you need to do. We're using the snaps here - // mainly to log small stuff like DoH and redirects. - ResponseBodySnap []byte - - // ResponseHeaders contains the response headers if error is nil. - ResponseHeaders http.Header - - // ResponseProto contains the response protocol - ResponseProto string - - // ResponseStatusCode contains the HTTP status code if error is nil. - ResponseStatusCode int64 - - // MaxBodySnapSize is the maximum size of the bodies snapshot. - MaxBodySnapSize int64 -} - -// HTTPResponseBodyPartEvent is emitted after we have received -// a part of the response body, or an error reading it. Note that -// bytes read here does not necessarily match bytes returned by -// ReadEvent because of (1) transparent gzip decompression by Go, -// (2) HTTP overhead (headers and chunked body), (3) TLS. This -// is the reason why we also want to record the error here rather -// than just recording the error in ReadEvent. -// -// Note that you are not going to see this event if you do not -// drain the response body, which you're supposed to do, tho. -type HTTPResponseBodyPartEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error indicates whether we could not read a part of the body - Error error - - // Data is a reference to the body we've just read. - Data []byte -} - -// HTTPResponseDoneEvent is emitted after we have received the body, -// when the response body is being closed. -// -// Note that you are not going to see this event if you do not -// drain the response body, which you're supposed to do, tho. -type HTTPResponseDoneEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration -} - -// ReadEvent is emitted when the READ/RECV syscall returns. -type ReadEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error is the error returned by READ/RECV. - Error error - - // NumBytes is the number of bytes received, which may in - // principle also be nonzero on error. - NumBytes int64 - - // SyscallDuration is the number of nanoseconds we were - // blocked waiting for the syscall to return. - SyscallDuration time.Duration -} - -// ResolveStartEvent is emitted when we start resolving a domain name. -type ResolveStartEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Hostname is the domain name to resolve. - Hostname string - - // TransportNetwork is the network used by the DNS transport, which - // can be one of "doh", "dot", "tcp", "udp", or "system". - TransportNetwork string - - // TransportAddress is the address used by the DNS transport, which - // is of course relative to the TransportNetwork. - TransportAddress string -} - -// ResolveDoneEvent is emitted when we know the IP addresses of a -// specific domain name, or the resolution failed. -type ResolveDoneEvent struct { - // Addresses is the list of returned addresses (empty on error). - Addresses []string - - // ContainsBogons indicates whether Addresses contains one - // or more IP addresses that classify as bogons. - ContainsBogons bool - - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error is the result of the dial operation. - Error error - - // Hostname is the domain name to resolve. - Hostname string - - // TransportNetwork is the network used by the DNS transport, which - // can be one of "doh", "dot", "tcp", "udp", or "system". - TransportNetwork string - - // TransportAddress is the address used by the DNS transport, which - // is of course relative to the TransportNetwork. - TransportAddress string -} - -// X509Certificate is an x.509 certificate. -type X509Certificate struct { - // Data contains the certificate bytes in DER format. - Data []byte -} - -// TLSConnectionState contains the TLS connection state. -type TLSConnectionState struct { - CipherSuite uint16 - NegotiatedProtocol string - PeerCertificates []X509Certificate - Version uint16 -} - -// NewTLSConnectionState creates a new TLSConnectionState. -func NewTLSConnectionState(s tls.ConnectionState) TLSConnectionState { - return TLSConnectionState{ - CipherSuite: s.CipherSuite, - NegotiatedProtocol: s.NegotiatedProtocol, - PeerCertificates: SimplifyCerts(s.PeerCertificates), - Version: s.Version, - } -} - -// SimplifyCerts simplifies a certificate chain for archival -func SimplifyCerts(in []*x509.Certificate) (out []X509Certificate) { - for _, cert := range in { - out = append(out, X509Certificate{ - Data: cert.Raw, - }) - } - return -} - -// TLSHandshakeStartEvent is emitted when the TLS handshake starts. -type TLSHandshakeStartEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // SNI is the SNI used when we force a specific SNI. - SNI string -} - -// TLSHandshakeDoneEvent is emitted when conn.Handshake returns. -type TLSHandshakeDoneEvent struct { - // ConnectionState is the TLS connection state. Depending on the - // error type, some fields may have little meaning. - ConnectionState TLSConnectionState - - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error is the result of the TLS handshake. - Error error -} - -// WriteEvent is emitted when the WRITE/SEND syscall returns. -type WriteEvent struct { - // DurationSinceBeginning is the number of nanoseconds since - // the time configured as the "zero" time. - DurationSinceBeginning time.Duration - - // Error is the error returned by WRITE/SEND. - Error error - - // NumBytes is the number of bytes sent, which may in - // principle also be nonzero on error. - NumBytes int64 - - // SyscallDuration is the number of nanoseconds we were - // blocked waiting for the syscall to return. - SyscallDuration time.Duration -} - -// Handler handles measurement events. -type Handler interface { - // OnMeasurement is called when an event occurs. There will be no - // events after the code that is using the modified Dialer, Transport, - // or Client is returned. OnMeasurement may be called by background - // goroutines and OnMeasurement calls may happen concurrently. - OnMeasurement(Measurement) -} - -// DNSResolver is a DNS resolver. The *net.Resolver used by Go implements -// this interface, but other implementations are possible. -type DNSResolver interface { - // LookupHost resolves a hostname to a list of IP addresses. - LookupHost(ctx context.Context, hostname string) (addrs []string, err error) -} - -// Dialer is a dialer for network connections. -type Dialer interface { - // Dial dials a new connection - Dial(network, address string) (net.Conn, error) - - // DialContext is like Dial but with context - DialContext(ctx context.Context, network, address string) (net.Conn, error) -} - -// TLSDialer is a dialer for TLS connections. -type TLSDialer interface { - // DialTLS dials a new TLS connection - DialTLS(network, address string) (net.Conn, error) - - // DialTLSContext is like DialTLS but with context - DialTLSContext(ctx context.Context, network, address string) (net.Conn, error) -} - -// MeasurementRoot is the measurement root. -// -// If you attach this to a context, we'll use it rather than using -// the beginning and hndler configured with resolvers, dialers, HTTP -// clients, and HTTP transports. By attaching a measurement root to -// a context, you can naturally split events by HTTP round trip. -type MeasurementRoot struct { - // Beginning is the "zero" used to compute the elapsed time. - Beginning time.Time - - // Handler is the handler that will handle events. - Handler Handler - - // MaxBodySnapSize is the maximum size after which we'll stop - // reading request and response bodies. They will of course - // be fully transmitted, but we'll save only MaxBodySnapSize - // bytes as part of the event stream. If this value is negative, - // we use math.MaxInt64. If the value is zero, we use a - // reasonable large value. Otherwise, we'll use this value. - MaxBodySnapSize int64 -} - -type measurementRootContextKey struct{} - -type dummyHandler struct{} - -func (*dummyHandler) OnMeasurement(Measurement) {} - -// ContextMeasurementRoot returns the MeasurementRoot configured in the -// provided context, or a nil pointer, if not set. -func ContextMeasurementRoot(ctx context.Context) *MeasurementRoot { - root, _ := ctx.Value(measurementRootContextKey{}).(*MeasurementRoot) - return root -} - -// ContextMeasurementRootOrDefault returns the MeasurementRoot configured in -// the provided context, or a working, dummy, MeasurementRoot otherwise. -func ContextMeasurementRootOrDefault(ctx context.Context) *MeasurementRoot { - root := ContextMeasurementRoot(ctx) - if root == nil { - root = &MeasurementRoot{ - Beginning: time.Now(), - Handler: &dummyHandler{}, - } - } - return root -} - -// WithMeasurementRoot returns a copy of the context with the -// configured MeasurementRoot set. Panics if the provided root -// is a nil pointer, like httptrace.WithClientTrace. -// -// Merging more than one root is not supported. Setting again -// the root is just going to replace the original root. -func WithMeasurementRoot( - ctx context.Context, root *MeasurementRoot, -) context.Context { - if root == nil { - panic("nil measurement root") - } - return context.WithValue( - ctx, measurementRootContextKey{}, root, - ) -} diff --git a/internal/engine/legacy/netx/modelx/modelx_test.go b/internal/engine/legacy/netx/modelx/modelx_test.go deleted file mode 100644 index 77dd37e..0000000 --- a/internal/engine/legacy/netx/modelx/modelx_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package modelx - -import ( - "context" - "crypto/tls" - "errors" - "math" - "testing" - "time" - - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestNewTLSConnectionState(t *testing.T) { - conn, err := tls.Dial("tcp", "www.google.com:443", nil) - if err != nil { - t.Fatal(err) - } - state := NewTLSConnectionState(conn.ConnectionState()) - if len(state.PeerCertificates) < 1 { - t.Fatal("too few certificates") - } - if state.Version < tls.VersionSSL30 || state.Version > 0x0304 /*tls.VersionTLS13*/ { - t.Fatal("unexpected TLS version") - } -} - -func TestMeasurementRoot(t *testing.T) { - ctx := context.Background() - if ContextMeasurementRoot(ctx) != nil { - t.Fatal("unexpected value for ContextMeasurementRoot") - } - if ContextMeasurementRootOrDefault(ctx) == nil { - t.Fatal("unexpected value ContextMeasurementRootOrDefault") - } - handler := &dummyHandler{} - root := &MeasurementRoot{ - Handler: handler, - Beginning: time.Time{}, - } - ctx = WithMeasurementRoot(ctx, root) - v := ContextMeasurementRoot(ctx) - if v != root { - t.Fatal("unexpected ContextMeasurementRoot value") - } - v = ContextMeasurementRootOrDefault(ctx) - if v != root { - t.Fatal("unexpected ContextMeasurementRoot value") - } -} - -func TestMeasurementRootWithMeasurementRootPanic(t *testing.T) { - defer func() { - if recover() == nil { - t.Fatal("expected panic") - } - }() - ctx := context.Background() - _ = WithMeasurementRoot(ctx, nil) -} - -func TestErrWrapperPublicAPI(t *testing.T) { - child := errors.New("mocked error") - wrapper := &netxlite.ErrWrapper{ - Failure: "moobar", - WrappedErr: child, - } - if wrapper.Error() != "moobar" { - t.Fatal("The Error() method is misbehaving") - } - if wrapper.Unwrap() != child { - t.Fatal("The Unwrap() method is misbehaving") - } -} - -func TestComputeBodySnapSize(t *testing.T) { - if ComputeBodySnapSize(-1) != math.MaxInt64 { - t.Fatal("unexpected result") - } - if ComputeBodySnapSize(0) != defaultBodySnapSize { - t.Fatal("unexpected result") - } - if ComputeBodySnapSize(127) != 127 { - t.Fatal("unexpected result") - } -} diff --git a/internal/engine/legacy/netx/oldhttptransport/bodytracer.go b/internal/engine/legacy/netx/oldhttptransport/bodytracer.go deleted file mode 100644 index 44695b2..0000000 --- a/internal/engine/legacy/netx/oldhttptransport/bodytracer.go +++ /dev/null @@ -1,77 +0,0 @@ -package oldhttptransport - -import ( - "io" - "net/http" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" -) - -// BodyTracer performs single HTTP transactions and emits -// measurement events as they happen. -type BodyTracer struct { - Transport http.RoundTripper -} - -// NewBodyTracer creates a new Transport. -func NewBodyTracer(roundTripper http.RoundTripper) *BodyTracer { - return &BodyTracer{Transport: roundTripper} -} - -// RoundTrip executes a single HTTP transaction, returning -// a Response for the provided Request. -func (t *BodyTracer) RoundTrip(req *http.Request) (resp *http.Response, err error) { - resp, err = t.Transport.RoundTrip(req) - if err != nil { - return - } - // "The http Client and Transport guarantee that Body is always - // non-nil, even on responses without a body or responses with - // a zero-length body." (from the docs) - resp.Body = &bodyWrapper{ - ReadCloser: resp.Body, - root: modelx.ContextMeasurementRootOrDefault(req.Context()), - } - return -} - -// CloseIdleConnections closes the idle connections. -func (t *BodyTracer) CloseIdleConnections() { - // Adapted from net/http code - type closeIdler interface { - CloseIdleConnections() - } - if tr, ok := t.Transport.(closeIdler); ok { - tr.CloseIdleConnections() - } -} - -type bodyWrapper struct { - io.ReadCloser - root *modelx.MeasurementRoot -} - -func (bw *bodyWrapper) Read(b []byte) (n int, err error) { - n, err = bw.ReadCloser.Read(b) - bw.root.Handler.OnMeasurement(modelx.Measurement{ - HTTPResponseBodyPart: &modelx.HTTPResponseBodyPartEvent{ - // "Read reads up to len(p) bytes into p. It returns the number of - // bytes read (0 <= n <= len(p)) and any error encountered." - Data: b[:n], - Error: err, - DurationSinceBeginning: time.Since(bw.root.Beginning), - }, - }) - return -} - -func (bw *bodyWrapper) Close() (err error) { - err = bw.ReadCloser.Close() - bw.root.Handler.OnMeasurement(modelx.Measurement{ - HTTPResponseDone: &modelx.HTTPResponseDoneEvent{ - DurationSinceBeginning: time.Since(bw.root.Beginning), - }, - }) - return -} diff --git a/internal/engine/legacy/netx/oldhttptransport/bodytracer_test.go b/internal/engine/legacy/netx/oldhttptransport/bodytracer_test.go deleted file mode 100644 index 16cf011..0000000 --- a/internal/engine/legacy/netx/oldhttptransport/bodytracer_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package oldhttptransport - -import ( - "context" - "net/http" - "testing" - - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestBodyTracerSuccess(t *testing.T) { - client := &http.Client{ - Transport: NewBodyTracer(http.DefaultTransport), - } - resp, err := client.Get("https://www.google.com") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - _, err = netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - client.CloseIdleConnections() -} - -func TestBodyTracerFailure(t *testing.T) { - client := &http.Client{ - Transport: NewBodyTracer(http.DefaultTransport), - } - // This fails the request because we attempt to speak cleartext HTTP with - // a server that instead is expecting TLS. - resp, err := client.Get("http://www.google.com:443") - if err == nil { - t.Fatal("expected an error here") - } - if resp != nil { - t.Fatal("expected a nil response here") - } - client.CloseIdleConnections() -} diff --git a/internal/engine/legacy/netx/oldhttptransport/httptransport.go b/internal/engine/legacy/netx/oldhttptransport/httptransport.go deleted file mode 100644 index 1d934f7..0000000 --- a/internal/engine/legacy/netx/oldhttptransport/httptransport.go +++ /dev/null @@ -1,37 +0,0 @@ -// Package oldhttptransport contains HTTP transport extensions. Here we -// define a http.Transport that emits events. -package oldhttptransport - -import ( - "net/http" -) - -// Transport performs single HTTP transactions and emits -// measurement events as they happen. -type Transport struct { - roundTripper http.RoundTripper -} - -// New creates a new Transport. -func New(roundTripper http.RoundTripper) *Transport { - return &Transport{ - roundTripper: NewBodyTracer(NewTraceTripper(roundTripper)), - } -} - -// RoundTrip executes a single HTTP transaction, returning -// a Response for the provided Request. -func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) { - return t.roundTripper.RoundTrip(req) -} - -// CloseIdleConnections closes the idle connections. -func (t *Transport) CloseIdleConnections() { - // Adapted from net/http code - type closeIdler interface { - CloseIdleConnections() - } - if tr, ok := t.roundTripper.(closeIdler); ok { - tr.CloseIdleConnections() - } -} diff --git a/internal/engine/legacy/netx/oldhttptransport/httptransport_test.go b/internal/engine/legacy/netx/oldhttptransport/httptransport_test.go deleted file mode 100644 index 05c1bd3..0000000 --- a/internal/engine/legacy/netx/oldhttptransport/httptransport_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package oldhttptransport - -import ( - "context" - "net/http" - "testing" - - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestGood(t *testing.T) { - client := &http.Client{ - Transport: New(http.DefaultTransport), - } - resp, err := client.Get("https://www.google.com") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - _, err = netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - client.CloseIdleConnections() -} - -func TestFailure(t *testing.T) { - client := &http.Client{ - Transport: New(http.DefaultTransport), - } - // This fails the request because we attempt to speak cleartext HTTP with - // a server that instead is expecting TLS. - resp, err := client.Get("http://www.google.com:443") - if err == nil { - t.Fatal("expected an error here") - } - if resp != nil { - t.Fatal("expected a nil response here") - } - client.CloseIdleConnections() -} diff --git a/internal/engine/legacy/netx/oldhttptransport/tracetripper.go b/internal/engine/legacy/netx/oldhttptransport/tracetripper.go deleted file mode 100644 index 46777ff..0000000 --- a/internal/engine/legacy/netx/oldhttptransport/tracetripper.go +++ /dev/null @@ -1,254 +0,0 @@ -package oldhttptransport - -import ( - "bytes" - "context" - "crypto/tls" - "io" - "net/http" - "net/http/httptrace" - "sync" - "time" - - "github.com/ooni/probe-cli/v3/internal/atomicx" - errorsxlegacy "github.com/ooni/probe-cli/v3/internal/engine/legacy/errorsx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// TraceTripper performs single HTTP transactions. -type TraceTripper struct { - readAllErrs *atomicx.Int64 - readAllContext func(ctx context.Context, r io.Reader) ([]byte, error) - roundTripper http.RoundTripper -} - -// NewTraceTripper creates a new Transport. -func NewTraceTripper(roundTripper http.RoundTripper) *TraceTripper { - return &TraceTripper{ - readAllErrs: &atomicx.Int64{}, - readAllContext: netxlite.ReadAllContext, - roundTripper: roundTripper, - } -} - -type readCloseWrapper struct { - closer io.Closer - reader io.Reader -} - -func newReadCloseWrapper( - reader io.Reader, closer io.ReadCloser, -) *readCloseWrapper { - return &readCloseWrapper{ - closer: closer, - reader: reader, - } -} - -func (c *readCloseWrapper) Read(p []byte) (int, error) { - return c.reader.Read(p) -} - -func (c *readCloseWrapper) Close() error { - return c.closer.Close() -} - -func readSnap( - ctx context.Context, source *io.ReadCloser, limit int64, - readAllContext func(ctx context.Context, r io.Reader) ([]byte, error), -) (data []byte, err error) { - data, err = readAllContext(ctx, io.LimitReader(*source, limit)) - if err == nil { - *source = newReadCloseWrapper( - io.MultiReader(bytes.NewReader(data), *source), - *source, - ) - } - return -} - -// RoundTrip executes a single HTTP transaction, returning -// a Response for the provided Request. -func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { - root := modelx.ContextMeasurementRootOrDefault(req.Context()) - - root.Handler.OnMeasurement(modelx.Measurement{ - HTTPRoundTripStart: &modelx.HTTPRoundTripStartEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - Method: req.Method, - URL: req.URL.String(), - }, - }) - - var ( - err error - majorOp = netxlite.HTTPRoundTripOperation - majorOpMu sync.Mutex - requestBody []byte - requestHeaders = http.Header{} - requestHeadersMu sync.Mutex - snapSize = modelx.ComputeBodySnapSize(root.MaxBodySnapSize) - ) - - // Save a snapshot of the request body - if req.Body != nil { - requestBody, err = readSnap(req.Context(), &req.Body, snapSize, t.readAllContext) - if err != nil { - return nil, err - } - } - - // Prepare a tracer for delivering events - tracer := &httptrace.ClientTrace{ - TLSHandshakeStart: func() { - majorOpMu.Lock() - majorOp = netxlite.TLSHandshakeOperation - majorOpMu.Unlock() - // Event emitted by net/http when DialTLS is not - // configured in the http.Transport - root.Handler.OnMeasurement(modelx.Measurement{ - TLSHandshakeStart: &modelx.TLSHandshakeStartEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - }, - }) - }, - TLSHandshakeDone: func(state tls.ConnectionState, err error) { - // Wrapping the error even if we're not returning it because it may - // less confusing to users to see the wrapped name - err = errorsxlegacy.SafeErrWrapperBuilder{ - Error: err, - Operation: netxlite.TLSHandshakeOperation, - }.MaybeBuild() - durationSinceBeginning := time.Since(root.Beginning) - // Event emitted by net/http when DialTLS is not - // configured in the http.Transport - root.Handler.OnMeasurement(modelx.Measurement{ - TLSHandshakeDone: &modelx.TLSHandshakeDoneEvent{ - ConnectionState: modelx.NewTLSConnectionState(state), - Error: err, - DurationSinceBeginning: durationSinceBeginning, - }, - }) - }, - GotConn: func(info httptrace.GotConnInfo) { - majorOpMu.Lock() - majorOp = netxlite.HTTPRoundTripOperation - majorOpMu.Unlock() - root.Handler.OnMeasurement(modelx.Measurement{ - HTTPConnectionReady: &modelx.HTTPConnectionReadyEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - }, - }) - }, - WroteHeaderField: func(key string, values []string) { - requestHeadersMu.Lock() - // Important: do not set directly into the headers map using - // the [] operator because net/http expects to be able to - // perform normalization of header names! - for _, value := range values { - requestHeaders.Add(key, value) - } - requestHeadersMu.Unlock() - root.Handler.OnMeasurement(modelx.Measurement{ - HTTPRequestHeader: &modelx.HTTPRequestHeaderEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - Key: key, - Value: values, - }, - }) - }, - WroteHeaders: func() { - root.Handler.OnMeasurement(modelx.Measurement{ - HTTPRequestHeadersDone: &modelx.HTTPRequestHeadersDoneEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - Headers: requestHeaders, // [*] - Method: req.Method, // [*] - URL: req.URL, // [*] - }, - }) - }, - WroteRequest: func(info httptrace.WroteRequestInfo) { - // Wrapping the error even if we're not returning it because it may - // less confusing to users to see the wrapped name - err := errorsxlegacy.SafeErrWrapperBuilder{ - Error: info.Err, - Operation: netxlite.HTTPRoundTripOperation, - }.MaybeBuild() - root.Handler.OnMeasurement(modelx.Measurement{ - HTTPRequestDone: &modelx.HTTPRequestDoneEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - Error: err, - }, - }) - }, - GotFirstResponseByte: func() { - root.Handler.OnMeasurement(modelx.Measurement{ - HTTPResponseStart: &modelx.HTTPResponseStartEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - }, - }) - }, - } - - // If we don't have already a tracer this is a toplevel request, so just - // set the tracer. Otherwise, we're doing DoH. We cannot set anothert trace - // because they'd be merged. Instead, replace the existing trace content - // with the new trace and then remember to reset it. - origtracer := httptrace.ContextClientTrace(req.Context()) - if origtracer != nil { - bkp := *origtracer - *origtracer = *tracer - defer func() { - *origtracer = bkp - }() - } else { - req = req.WithContext(httptrace.WithClientTrace(req.Context(), tracer)) - } - - resp, err := t.roundTripper.RoundTrip(req) - err = errorsxlegacy.SafeErrWrapperBuilder{ - Error: err, - Operation: majorOp, - }.MaybeBuild() - // [*] Require less event joining work by providing info that - // makes this event alone actionable for OONI - event := &modelx.HTTPRoundTripDoneEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - Error: err, - RequestBodySnap: requestBody, - RequestHeaders: requestHeaders, // [*] - RequestMethod: req.Method, // [*] - RequestURL: req.URL.String(), // [*] - MaxBodySnapSize: snapSize, - } - if resp != nil { - event.ResponseHeaders = resp.Header - event.ResponseStatusCode = int64(resp.StatusCode) - event.ResponseProto = resp.Proto - // Save a snapshot of the response body - var data []byte - data, err = readSnap(req.Context(), &resp.Body, snapSize, t.readAllContext) - if err != nil { - t.readAllErrs.Add(1) - resp = nil // this is how net/http likes it - } else { - event.ResponseBodySnap = data - } - } - root.Handler.OnMeasurement(modelx.Measurement{ - HTTPRoundTripDone: event, - }) - return resp, err -} - -// CloseIdleConnections closes the idle connections. -func (t *TraceTripper) CloseIdleConnections() { - // Adapted from net/http code - type closeIdler interface { - CloseIdleConnections() - } - if tr, ok := t.roundTripper.(closeIdler); ok { - tr.CloseIdleConnections() - } -} diff --git a/internal/engine/legacy/netx/oldhttptransport/tracetripper_test.go b/internal/engine/legacy/netx/oldhttptransport/tracetripper_test.go deleted file mode 100644 index ac1c6ae..0000000 --- a/internal/engine/legacy/netx/oldhttptransport/tracetripper_test.go +++ /dev/null @@ -1,272 +0,0 @@ -package oldhttptransport - -import ( - "bytes" - "context" - "errors" - "io" - "net/http" - "net/http/httptrace" - "sync" - "testing" - "time" - - "github.com/miekg/dns" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestTraceTripperSuccess(t *testing.T) { - client := &http.Client{ - Transport: NewTraceTripper(http.DefaultTransport), - } - resp, err := client.Get("https://www.google.com") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - _, err = netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - client.CloseIdleConnections() -} - -type roundTripHandler struct { - roundTrips []*modelx.HTTPRoundTripDoneEvent - mu sync.Mutex -} - -func (h *roundTripHandler) OnMeasurement(m modelx.Measurement) { - if m.HTTPRoundTripDone != nil { - h.mu.Lock() - defer h.mu.Unlock() - h.roundTrips = append(h.roundTrips, m.HTTPRoundTripDone) - } -} - -func TestTraceTripperReadAllFailure(t *testing.T) { - transport := NewTraceTripper(http.DefaultTransport) - transport.readAllContext = func(ctx context.Context, r io.Reader) ([]byte, error) { - return nil, io.EOF - } - client := &http.Client{Transport: transport} - resp, err := client.Get("https://google.com") - if err == nil { - t.Fatal("expected an error here") - } - if !errors.Is(err, io.EOF) { - t.Fatal("not the error we expected") - } - if resp != nil { - t.Fatal("expected nil response here") - } - if transport.readAllErrs.Load() <= 0 { - t.Fatal("not the error we expected") - } - client.CloseIdleConnections() -} - -func TestTraceTripperFailure(t *testing.T) { - client := &http.Client{ - Transport: NewTraceTripper(http.DefaultTransport), - } - // This fails the request because we attempt to speak cleartext HTTP with - // a server that instead is expecting TLS. - resp, err := client.Get("http://www.google.com:443") - if err == nil { - t.Fatal("expected an error here") - } - if resp != nil { - t.Fatal("expected a nil response here") - } - client.CloseIdleConnections() -} - -func TestTraceTripperWithClientTrace(t *testing.T) { - client := &http.Client{ - Transport: NewTraceTripper(http.DefaultTransport), - } - req, err := http.NewRequest("GET", "https://www.kernel.org/", nil) - if err != nil { - t.Fatal(err) - } - req = req.WithContext( - httptrace.WithClientTrace(req.Context(), new(httptrace.ClientTrace)), - ) - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - if resp == nil { - t.Fatal("expected a good response here") - } - resp.Body.Close() - client.CloseIdleConnections() -} - -func TestTraceTripperWithCorrectSnaps(t *testing.T) { - // Prepare a DNS query for dns.google.com A, for which we - // know the answer in terms of well know IP addresses - query := new(dns.Msg) - query.Id = dns.Id() - query.RecursionDesired = true - query.Question = make([]dns.Question, 1) - query.Question[0] = dns.Question{ - Name: dns.Fqdn("dns.google.com"), - Qtype: dns.TypeA, - Qclass: dns.ClassINET, - } - queryData, err := query.Pack() - if err != nil { - t.Fatal(err) - } - - // Prepare a new transport with limited snapshot size and - // use such transport to configure an ordinary client - transport := NewTraceTripper(http.DefaultTransport) - const snapSize = 15 - client := &http.Client{Transport: transport} - - // Prepare a new request for Cloudflare DNS, register - // a handler, issue the request, fetch the response. - req, err := http.NewRequest( - "POST", "https://cloudflare-dns.com/dns-query", bytes.NewReader(queryData), - ) - if err != nil { - t.Fatal(err) - } - req.Header.Set("Content-Type", "application/dns-message") - handler := &roundTripHandler{} - ctx := modelx.WithMeasurementRoot( - context.Background(), &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: handler, - MaxBodySnapSize: snapSize, - }, - ) - req = req.WithContext(ctx) - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - if resp.StatusCode != 200 { - t.Fatal("HTTP request failed") - } - - // Read the whole response body, parse it as valid DNS - // reply and verify we obtained what we expected - replyData, err := netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - reply := new(dns.Msg) - err = reply.Unpack(replyData) - if err != nil { - t.Fatal(err) - } - if reply.Rcode != 0 { - t.Fatal("unexpected Rcode") - } - if len(reply.Answer) < 1 { - t.Fatal("no answers?!") - } - found8888, found8844, foundother := false, false, false - for _, answer := range reply.Answer { - if rra, ok := answer.(*dns.A); ok { - ip := rra.A.String() - if ip == "8.8.8.8" { - found8888 = true - } else if ip == "8.8.4.4" { - found8844 = true - } else { - foundother = true - } - } - } - if !found8888 || !found8844 || foundother { - t.Fatal("unexpected reply") - } - - // Finally, make sure we have captured the correct - // snapshots for the request and response bodies - if len(handler.roundTrips) != 1 { - t.Fatal("more round trips than expected") - } - roundTrip := handler.roundTrips[0] - if len(roundTrip.RequestBodySnap) != snapSize { - t.Fatal("unexpected request body snap length") - } - if len(roundTrip.ResponseBodySnap) != snapSize { - t.Fatal("unexpected response body snap length") - } - if !bytes.Equal(roundTrip.RequestBodySnap, queryData[:snapSize]) { - t.Fatal("the request body snap is wrong") - } - if !bytes.Equal(roundTrip.ResponseBodySnap, replyData[:snapSize]) { - t.Fatal("the response body snap is wrong") - } -} - -func TestTraceTripperWithReadAllFailingForBody(t *testing.T) { - // Prepare a DNS query for dns.google.com A, for which we - // know the answer in terms of well know IP addresses - query := new(dns.Msg) - query.Id = dns.Id() - query.RecursionDesired = true - query.Question = make([]dns.Question, 1) - query.Question[0] = dns.Question{ - Name: dns.Fqdn("dns.google.com"), - Qtype: dns.TypeA, - Qclass: dns.ClassINET, - } - queryData, err := query.Pack() - if err != nil { - t.Fatal(err) - } - - // Prepare a new transport with limited snapshot size and - // use such transport to configure an ordinary client - transport := NewTraceTripper(http.DefaultTransport) - errorMocked := errors.New("mocked error") - transport.readAllContext = func(ctx context.Context, r io.Reader) ([]byte, error) { - return nil, errorMocked - } - const snapSize = 15 - client := &http.Client{Transport: transport} - - // Prepare a new request for Cloudflare DNS, register - // a handler, issue the request, fetch the response. - req, err := http.NewRequest( - "POST", "https://cloudflare-dns.com/dns-query", bytes.NewReader(queryData), - ) - if err != nil { - t.Fatal(err) - } - req.Header.Set("Content-Type", "application/dns-message") - handler := &roundTripHandler{} - ctx := modelx.WithMeasurementRoot( - context.Background(), &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: handler, - MaxBodySnapSize: snapSize, - }, - ) - req = req.WithContext(ctx) - resp, err := client.Do(req) - if err == nil { - t.Fatal("expected an error here") - } - if !errors.Is(err, errorMocked) { - t.Fatal("not the error we expected") - } - if resp != nil { - t.Fatal("expected nil response here") - } - - // Finally, make sure we got something that makes sense - if len(handler.roundTrips) != 0 { - t.Fatal("more round trips than expected") - } -} diff --git a/internal/engine/legacy/netx/resolver.go b/internal/engine/legacy/netx/resolver.go deleted file mode 100644 index cea6fc4..0000000 --- a/internal/engine/legacy/netx/resolver.go +++ /dev/null @@ -1,181 +0,0 @@ -package netx - -import ( - "context" - "errors" - "net" - "net/http" - "strings" - "sync" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/errorsx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -var ( - dohClientHandle *http.Client - dohClientOnce sync.Once -) - -func newHTTPClientForDoH(beginning time.Time, handler modelx.Handler) *http.Client { - if handler == handlers.NoHandler { - // A bit of extra complexity for a good reason: if the user is not - // interested into setting a default handler, then it is fine to - // always return the same *http.Client for DoH. This means that we - // don't need to care about closing the connections used by this - // *http.Client, therefore we don't leak resources because we fail - // to close the idle connections. - dohClientOnce.Do(func() { - transport := newHTTPTransport( - time.Now(), - handlers.NoHandler, - newDialer(time.Now(), handler), - false, // DisableKeepAlives - http.ProxyFromEnvironment, - ) - dohClientHandle = &http.Client{Transport: transport} - }) - return dohClientHandle - } - // Otherwise, if the user wants to have a default handler, we - // return a transport that does not leak connections. - transport := newHTTPTransport( - beginning, - handler, - newDialer(beginning, handler), - true, // DisableKeepAlives - http.ProxyFromEnvironment, - ) - return &http.Client{Transport: transport} -} - -func withPort(address, port string) string { - // Handle the case where port was not specified. We have written in - // a bunch of places that we can just pass a domain in this case and - // so we need to gracefully ensure this is still possible. - _, _, err := net.SplitHostPort(address) - if err != nil && strings.Contains(err.Error(), "missing port in address") { - address = net.JoinHostPort(address, port) - } - return address -} - -type resolverWrapper struct { - beginning time.Time - handler modelx.Handler - resolver modelx.DNSResolver -} - -func newResolverWrapper( - beginning time.Time, handler modelx.Handler, - resolver modelx.DNSResolver, -) *resolverWrapper { - return &resolverWrapper{ - beginning: beginning, - handler: handler, - resolver: resolver, - } -} - -// LookupHost returns the IP addresses of a host -func (r *resolverWrapper) LookupHost(ctx context.Context, hostname string) ([]string, error) { - ctx = maybeWithMeasurementRoot(ctx, r.beginning, r.handler) - return r.resolver.LookupHost(ctx, hostname) -} - -func newResolver( - beginning time.Time, handler modelx.Handler, network, address string, -) (modelx.DNSResolver, error) { - // Implementation note: system need to be dealt with - // separately because it doesn't have any transport. - if network == "system" || network == "" { - return newResolverWrapper( - beginning, handler, newResolverSystem()), nil - } - if network == "doh" { - return newResolverWrapper(beginning, handler, newResolverHTTPS( - newHTTPClientForDoH(beginning, handler), address, - )), nil - } - if network == "dot" { - // We need a child dialer here to avoid an endless loop where the - // dialer will ask us to resolve, we'll tell the dialer to dial, it - // will ask us to resolve, ... - return newResolverWrapper(beginning, handler, newResolverTLS( - newDialer(beginning, handler).DialTLSContext, withPort(address, "853"), - )), nil - } - if network == "tcp" { - // Same rationale as above: avoid possible endless loop - return newResolverWrapper(beginning, handler, newResolverTCP( - newDialer(beginning, handler).DialContext, withPort(address, "53"), - )), nil - } - if network == "udp" { - // Same rationale as above: avoid possible endless loop - return newResolverWrapper(beginning, handler, newResolverUDP( - netxlite.NewDialerLegacyAdapter(newDialer(beginning, handler)), - withPort(address, "53"), - )), nil - } - return nil, errors.New("resolver.New: unsupported network value") -} - -// NewResolver creates a standalone Resolver -func NewResolver(network, address string) (modelx.DNSResolver, error) { - return newResolver(time.Now(), handlers.NoHandler, network, address) -} - -type chainWrapperResolver struct { - modelx.DNSResolver -} - -func (r chainWrapperResolver) Network() string { - return "chain" -} - -func (r chainWrapperResolver) Address() string { - return "" -} - -// ChainResolvers chains a primary and a secondary resolver such that -// we can fallback to the secondary if primary is broken. -func ChainResolvers(primary, secondary modelx.DNSResolver) modelx.DNSResolver { - return resolver.ChainResolver{ - Primary: chainWrapperResolver{DNSResolver: primary}, - Secondary: chainWrapperResolver{DNSResolver: secondary}, - } -} - -func resolverWrapResolver(r resolver.Resolver) resolver.EmitterResolver { - return resolver.EmitterResolver{Resolver: &errorsx.ErrorWrapperResolver{Resolver: r}} -} - -func resolverWrapTransport(txp resolver.RoundTripper) resolver.EmitterResolver { - return resolverWrapResolver(resolver.NewSerialResolver( - resolver.EmitterTransport{RoundTripper: txp})) -} - -func newResolverSystem() resolver.EmitterResolver { - return resolverWrapResolver(&netxlite.ResolverSystem{}) -} - -func newResolverUDP(dialer resolver.Dialer, address string) resolver.EmitterResolver { - return resolverWrapTransport(resolver.NewDNSOverUDP(dialer, address)) -} - -func newResolverTCP(dial resolver.DialContextFunc, address string) resolver.EmitterResolver { - return resolverWrapTransport(resolver.NewDNSOverTCP(dial, address)) -} - -func newResolverTLS(dial resolver.DialContextFunc, address string) resolver.EmitterResolver { - return resolverWrapTransport(resolver.NewDNSOverTLS(dial, address)) -} - -func newResolverHTTPS(client *http.Client, address string) resolver.EmitterResolver { - return resolverWrapTransport(resolver.NewDNSOverHTTPS(client, address)) -} diff --git a/internal/engine/legacy/netx/resolver_internal_test.go b/internal/engine/legacy/netx/resolver_internal_test.go deleted file mode 100644 index 68351cc..0000000 --- a/internal/engine/legacy/netx/resolver_internal_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package netx - -import ( - "net/http" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" -) - -func NewHTTPClientForDoH(beginning time.Time, handler modelx.Handler) *http.Client { - return newHTTPClientForDoH(beginning, handler) -} - -type ChainWrapperResolver = chainWrapperResolver diff --git a/internal/engine/legacy/netx/resolver_test.go b/internal/engine/legacy/netx/resolver_test.go deleted file mode 100644 index 724194c..0000000 --- a/internal/engine/legacy/netx/resolver_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package netx_test - -import ( - "context" - "io" - "os" - "testing" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" -) - -func testresolverquick(t *testing.T, network, address string) { - resolver, err := netx.NewResolver(network, address) - if err != nil { - t.Fatal(err) - } - if resolver == nil { - t.Fatal("expected non-nil resolver here") - } - addrs, err := resolver.LookupHost(context.Background(), "dns.google.com") - if err != nil { - t.Fatalf("legacy/netx/resolver_test.go: %+v with %s/%s", err, network, address) - } - if addrs == nil { - t.Fatal("expected non-nil addrs here") - } - var foundquad8 bool - for _, addr := range addrs { - // See https://github.com/ooni/probe-engine/pull/954/checks?check_run_id=1182269025 - if addr == "8.8.8.8" || addr == "2001:4860:4860::8888" { - foundquad8 = true - } - } - if !foundquad8 { - t.Fatalf("did not find 8.8.8.8 in output; output=%+v", addrs) - } -} - -func TestNewResolverUDPAddress(t *testing.T) { - testresolverquick(t, "udp", "8.8.8.8:53") -} - -func TestNewResolverUDPAddressNoPort(t *testing.T) { - testresolverquick(t, "udp", "8.8.8.8") -} - -func TestNewResolverUDPDomain(t *testing.T) { - testresolverquick(t, "udp", "dns.google.com:53") -} - -func TestNewResolverUDPDomainNoPort(t *testing.T) { - testresolverquick(t, "udp", "dns.google.com") -} - -func TestNewResolverSystem(t *testing.T) { - testresolverquick(t, "system", "") -} - -func TestNewResolverTCPAddress(t *testing.T) { - testresolverquick(t, "tcp", "8.8.8.8:53") -} - -func TestNewResolverTCPAddressNoPort(t *testing.T) { - testresolverquick(t, "tcp", "8.8.8.8") -} - -func TestNewResolverTCPDomain(t *testing.T) { - testresolverquick(t, "tcp", "dns.google.com:53") -} - -func TestNewResolverTCPDomainNoPort(t *testing.T) { - testresolverquick(t, "tcp", "dns.google.com") -} - -func TestNewResolverDoTAddress(t *testing.T) { - if os.Getenv("GITHUB_ACTIONS") == "true" { - t.Skip("this test is not reliable in GitHub actions") - } - testresolverquick(t, "dot", "9.9.9.9:853") -} - -func TestNewResolverDoTAddressNoPort(t *testing.T) { - if os.Getenv("GITHUB_ACTIONS") == "true" { - t.Skip("this test is not reliable in GitHub actions") - } - testresolverquick(t, "dot", "9.9.9.9") -} - -func TestNewResolverDoTDomain(t *testing.T) { - if os.Getenv("GITHUB_ACTIONS") == "true" { - t.Skip("this test is not reliable in GitHub actions") - } - testresolverquick(t, "dot", "dns.quad9.net:853") -} - -func TestNewResolverDoTDomainNoPort(t *testing.T) { - if os.Getenv("GITHUB_ACTIONS") == "true" { - t.Skip("this test is not reliable in GitHub actions") - } - testresolverquick(t, "dot", "dns.quad9.net") -} - -func TestNewResolverDoH(t *testing.T) { - testresolverquick(t, "doh", "https://cloudflare-dns.com/dns-query") -} - -func TestNewResolverInvalid(t *testing.T) { - resolver, err := netx.NewResolver( - "antani", "https://cloudflare-dns.com/dns-query", - ) - if err == nil { - t.Fatal("expected an error here") - } - if resolver != nil { - t.Fatal("expected a nil resolver here") - } -} - -type failingResolver struct{} - -func (failingResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) { - return nil, io.EOF -} - -func TestChainResolvers(t *testing.T) { - fallback, err := netx.NewResolver("udp", "1.1.1.1:53") - if err != nil { - t.Fatal(err) - } - dialer := netx.NewDialer() - resolver := netx.ChainResolvers(failingResolver{}, fallback) - dialer.SetResolver(resolver) - conn, err := dialer.Dial("tcp", "www.google.com:80") - if err != nil { - t.Fatal(err) // we don't expect error because good resolver is first - } - defer conn.Close() -} - -func TestNewHTTPClientForDoH(t *testing.T) { - first := netx.NewHTTPClientForDoH( - time.Now(), handlers.NoHandler, - ) - second := netx.NewHTTPClientForDoH( - time.Now(), handlers.NoHandler, - ) - if first != second { - t.Fatal("expected to see same client here") - } - third := netx.NewHTTPClientForDoH( - time.Now(), handlers.StdoutHandler, - ) - if first == third { - t.Fatal("expected to see different client here") - } -} - -func TestChainWrapperResolver(t *testing.T) { - r := netx.ChainWrapperResolver{} - if r.Address() != "" { - t.Fatal("invalid Address") - } - if r.Network() != "chain" { - t.Fatal("invalid Network") - } -} diff --git a/internal/engine/legacy/netx/testdata/cacert-invalid.pem b/internal/engine/legacy/netx/testdata/cacert-invalid.pem deleted file mode 100644 index d4edb11..0000000 --- a/internal/engine/legacy/netx/testdata/cacert-invalid.pem +++ /dev/null @@ -1,13 +0,0 @@ -# -# The following is a truncated CA bundle for integration testing. This -# will give us confidence that we fail if the file is wrong. -# - -emSign ECC Root CA - C3 -======================= ------BEGIN CERTIFICATE----- -MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQGEwJVUzETMBEG -A1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMxIDAeBgNVBAMTF2VtU2lnbiBF -Q0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAwMFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UE -BhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQD -ExdlbVNpZ24gRUNDIFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd diff --git a/internal/engine/legacy/netx/testdata/cacert.pem b/internal/engine/legacy/netx/testdata/cacert.pem deleted file mode 100644 index 3a96ced..0000000 --- a/internal/engine/legacy/netx/testdata/cacert.pem +++ /dev/null @@ -1,54 +0,0 @@ -# -# The following is a minimal, valid CA bundle. We do not include -# however the certificates required to validate www.google.com -# and we check in tests that we cannot connect to it and successfully -# complete a TLS handshake. This gives us confidence that we can -# actually override the CA bundle path. -# - -emSign ECC Root CA - C3 -======================= ------BEGIN CERTIFICATE----- -MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQGEwJVUzETMBEG -A1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMxIDAeBgNVBAMTF2VtU2lnbiBF -Q0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAwMFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UE -BhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQD -ExdlbVNpZ24gRUNDIFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd -6bciMK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4OjavtisIGJAnB9 -SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0OBBYEFPtaSNCAIEDyqOkA -B2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMDA2gA -MGUCMQC02C8Cif22TGK6Q04ThHK1rt0c3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwU -ZOR8loMRnLDRWmFLpg9J0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== ------END CERTIFICATE----- - -Hongkong Post Root CA 3 -======================= ------BEGIN CERTIFICATE----- -MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQELBQAwbzELMAkG -A1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJSG9uZyBLb25nMRYwFAYDVQQK -Ew1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25na29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2 -MDMwMjI5NDZaFw00MjA2MDMwMjI5NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtv -bmcxEjAQBgNVBAcTCUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMX -SG9uZ2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCz -iNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFOdem1p+/l6TWZ5Mwc50tf -jTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mIVoBc+L0sPOFMV4i707mV78vH9toxdCim -5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOe -sL4jpNrcyCse2m5FHomY2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj -0mRiikKYvLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+TtbNe/ -JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZbx39ri1UbSsUgYT2u -y1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+l2oBlKN8W4UdKjk60FSh0Tlxnf0h -+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YKTE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsG -xVd7GYYKecsAyVKvQv83j+GjHno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwID -AQABo2MwYTAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e -i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEwDQYJKoZIhvcN -AQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG7BJ8dNVI0lkUmcDrudHr9Egw -W62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCkMpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWld -y8joRTnU+kLBEUx3XZL7av9YROXrgZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov -+BS5gLNdTaqX4fnkGMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDc -eqFS3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJmOzj/2ZQw -9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+l6mc1X5VTMbeRRAc6uk7 -nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6cJfTzPV4e0hz5sy229zdcxsshTrD3mUcY -hcErulWuBurQB7Lcq9CClnXO0lD+mefPL5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB -60PZ2Pierc+xYw5F9KBaLJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fq -dBb9HxEGmpv0 ------END CERTIFICATE----- diff --git a/internal/engine/legacy/netxlogger/netxlogger.go b/internal/engine/legacy/netxlogger/netxlogger.go deleted file mode 100644 index 083fb6a..0000000 --- a/internal/engine/legacy/netxlogger/netxlogger.go +++ /dev/null @@ -1,148 +0,0 @@ -// Package netxlogger is a logger for netx events. -// -// This package is a fork of github.com/ooni/netx/x/logger where -// we applied ooni/probe-engine specific customisations. -package netxlogger - -import ( - "net/http" - "strings" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// Handler is a handler that logs events. -type Handler struct { - logger model.DebugLogger -} - -// NewHandler returns a new logging handler. -func NewHandler(logger model.DebugLogger) *Handler { - return &Handler{logger: logger} -} - -// OnMeasurement logs the specific measurement -func (h *Handler) OnMeasurement(m modelx.Measurement) { - // DNS - if m.ResolveStart != nil { - h.logger.Debugf( - "resolving: %s", - m.ResolveStart.Hostname, - ) - } - if m.ResolveDone != nil { - h.logger.Debugf( - "resolve done: %s, %s", - fmtError(m.ResolveDone.Error), - m.ResolveDone.Addresses, - ) - } - - // Syscalls - if m.Connect != nil { - h.logger.Debugf( - "connect done: %s, %s (rtt=%s)", - fmtError(m.Connect.Error), - m.Connect.RemoteAddress, - m.Connect.SyscallDuration, - ) - } - - // TLS - if m.TLSHandshakeStart != nil { - h.logger.Debugf( - "TLS handshake: (forceSNI='%s')", - m.TLSHandshakeStart.SNI, - ) - } - if m.TLSHandshakeDone != nil { - h.logger.Debugf( - "TLS done: %s, %s (alpn='%s')", - fmtError(m.TLSHandshakeDone.Error), - netxlite.TLSVersionString(m.TLSHandshakeDone.ConnectionState.Version), - m.TLSHandshakeDone.ConnectionState.NegotiatedProtocol, - ) - } - - // HTTP round trip - if m.HTTPRequestHeadersDone != nil { - proto := "HTTP/1.1" - for key := range m.HTTPRequestHeadersDone.Headers { - if strings.HasPrefix(key, ":") { - proto = "HTTP/2.0" - break - } - } - h.logger.Debugf( - "> %s %s %s", - m.HTTPRequestHeadersDone.Method, - m.HTTPRequestHeadersDone.URL.RequestURI(), - proto, - ) - if proto == "HTTP/2.0" { - h.logger.Debugf( - "> Host: %s", - m.HTTPRequestHeadersDone.URL.Host, - ) - } - for key, values := range m.HTTPRequestHeadersDone.Headers { - if strings.HasPrefix(key, ":") { - continue - } - for _, value := range values { - h.logger.Debugf( - "> %s: %s", - key, value, - ) - } - } - h.logger.Debug(">") - } - if m.HTTPRequestDone != nil { - h.logger.Debug("request sent; waiting for response") - } - if m.HTTPResponseStart != nil { - h.logger.Debug("start receiving response") - } - if m.HTTPRoundTripDone != nil && m.HTTPRoundTripDone.Error == nil { - h.logger.Debugf( - "< %s %d %s", - m.HTTPRoundTripDone.ResponseProto, - m.HTTPRoundTripDone.ResponseStatusCode, - http.StatusText(int(m.HTTPRoundTripDone.ResponseStatusCode)), - ) - for key, values := range m.HTTPRoundTripDone.ResponseHeaders { - for _, value := range values { - h.logger.Debugf( - "< %s: %s", - key, value, - ) - } - } - h.logger.Debug("<") - } - - // HTTP response body - if m.HTTPResponseBodyPart != nil { - h.logger.Debugf( - "body part: %s, %d", - fmtError(m.HTTPResponseBodyPart.Error), - len(m.HTTPResponseBodyPart.Data), - ) - } - if m.HTTPResponseDone != nil { - h.logger.Debug( - "end of response", - ) - } -} - -func fmtError(err error) (s string) { - s = "success" - if err != nil { - s = err.Error() - } - return -} diff --git a/internal/engine/legacy/netxlogger/netxlogger_test.go b/internal/engine/legacy/netxlogger/netxlogger_test.go deleted file mode 100644 index ca8d998..0000000 --- a/internal/engine/legacy/netxlogger/netxlogger_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package netxlogger - -import ( - "context" - "net/http" - "testing" - "time" - - "github.com/apex/log" - "github.com/apex/log/handlers/discard" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestGood(t *testing.T) { - log.SetHandler(discard.Default) - client := netx.NewHTTPClient() - client.ConfigureDNS("udp", "dns.google.com:53") - req, err := http.NewRequest("GET", "http://www.facebook.com", nil) - if err != nil { - t.Fatal(err) - } - req = req.WithContext(modelx.WithMeasurementRoot(req.Context(), &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: NewHandler(log.Log), - })) - resp, err := client.HTTPClient.Do(req) - if err != nil { - t.Fatal(err) - } - if resp == nil { - t.Fatal("expected non-nil resp here") - } - defer resp.Body.Close() - _, err = netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - client.HTTPClient.CloseIdleConnections() -} diff --git a/internal/engine/legacy/oonidatamodel/oonidatamodel.go b/internal/engine/legacy/oonidatamodel/oonidatamodel.go deleted file mode 100644 index 0ae7e0d..0000000 --- a/internal/engine/legacy/oonidatamodel/oonidatamodel.go +++ /dev/null @@ -1,481 +0,0 @@ -// Package oonidatamodel contains the OONI data model. -// -// The input of this package is data generated by netx and the -// output is a format consistent with OONI specs. -// -// Deprecated by the archival package. -package oonidatamodel - -import ( - "encoding/base64" - "encoding/json" - "errors" - "net" - "net/http" - "strconv" - "strings" - "unicode/utf8" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/oonitemplates" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// ExtSpec describes a data format extension -type ExtSpec struct { - Name string // extension name - V int64 // extension version -} - -// AddTo adds the current ExtSpec to the specified measurement -func (spec ExtSpec) AddTo(m *model.Measurement) { - if m.Extensions == nil { - m.Extensions = make(map[string]int64) - } - m.Extensions[spec.Name] = spec.V -} - -var ( - // ExtDNS is the version of df-002-dnst.md - ExtDNS = ExtSpec{Name: "dnst", V: 0} - - // ExtNetevents is the version of df-008-netevents.md - ExtNetevents = ExtSpec{Name: "netevents", V: 0} - - // ExtHTTP is the version of df-001-httpt.md - ExtHTTP = ExtSpec{Name: "httpt", V: 0} - - // ExtTCPConnect is the version of df-005-tcpconnect.md - ExtTCPConnect = ExtSpec{Name: "tcpconnect", V: 0} - - // ExtTLSHandshake is the version of df-006-tlshandshake.md - ExtTLSHandshake = ExtSpec{Name: "tlshandshake", V: 0} -) - -// TCPConnectStatus contains the TCP connect status. -type TCPConnectStatus struct { - Failure *string `json:"failure"` - Success bool `json:"success"` -} - -// TCPConnectEntry contains one of the entries that are part -// of the "tcp_connect" key of a OONI report. -type TCPConnectEntry struct { - IP string `json:"ip"` - Port int `json:"port"` - Status TCPConnectStatus `json:"status"` - T float64 `json:"t"` -} - -// TCPConnectList is a list of TCPConnectEntry -type TCPConnectList []TCPConnectEntry - -// NewTCPConnectList creates a new TCPConnectList -func NewTCPConnectList(results oonitemplates.Results) TCPConnectList { - var out TCPConnectList - for _, connect := range results.Connects { - // We assume Go is passing us legit data structs - ip, sport, _ := net.SplitHostPort(connect.RemoteAddress) - iport, _ := strconv.Atoi(sport) - out = append(out, TCPConnectEntry{ - IP: ip, - Port: iport, - Status: TCPConnectStatus{ - Failure: makeFailure(connect.Error), - Success: connect.Error == nil, - }, - T: connect.DurationSinceBeginning.Seconds(), - }) - } - return out -} - -func makeFailure(err error) (s *string) { - if err != nil { - serio := err.Error() - s = &serio - } - return -} - -// HTTPTor contains Tor information -type HTTPTor struct { - ExitIP *string `json:"exit_ip"` - ExitName *string `json:"exit_name"` - IsTor bool `json:"is_tor"` -} - -// MaybeBinaryValue is a possibly binary string. We use this helper class -// to define a custom JSON encoder that allows us to choose the proper -// representation depending on whether the Value field is valid UTF-8 or not. -type MaybeBinaryValue struct { - Value string -} - -// MarshalJSON marshals a string-like to JSON following the OONI spec that -// says that UTF-8 content is represened as string and non-UTF-8 content is -// instead represented using `{"format":"base64","data":"..."}`. -func (hb MaybeBinaryValue) MarshalJSON() ([]byte, error) { - if utf8.ValidString(hb.Value) { - return json.Marshal(hb.Value) - } - er := make(map[string]string) - er["format"] = "base64" - er["data"] = base64.StdEncoding.EncodeToString([]byte(hb.Value)) - return json.Marshal(er) -} - -// UnmarshalJSON is the opposite of MarshalJSON. -func (hb *MaybeBinaryValue) UnmarshalJSON(d []byte) error { - if err := json.Unmarshal(d, &hb.Value); err == nil { - return nil - } - er := make(map[string]string) - if err := json.Unmarshal(d, &er); err != nil { - return err - } - if v, ok := er["format"]; !ok || v != "base64" { - return errors.New("missing or invalid format field") - } - if _, ok := er["data"]; !ok { - return errors.New("missing data field") - } - b64, err := base64.StdEncoding.DecodeString(er["data"]) - if err != nil { - return err - } - hb.Value = string(b64) - return nil -} - -// HTTPBody is an HTTP body. As an implementation note, this type must be -// an alias for the MaybeBinaryValue type, otherwise the specific serialisation -// mechanism implemented by MaybeBinaryValue is not working. -type HTTPBody = MaybeBinaryValue - -// HTTPHeaders contains HTTP headers. This headers representation is -// deprecated in favour of HTTPHeadersList since data format 0.3.0. -type HTTPHeaders map[string]MaybeBinaryValue - -// HTTPHeader is a single HTTP header. -type HTTPHeader struct { - Key string - Value MaybeBinaryValue -} - -// MarshalJSON marshals a single HTTP header to a tuple where the first -// element is a string and the second element is maybe-binary data. -func (hh HTTPHeader) MarshalJSON() ([]byte, error) { - if utf8.ValidString(hh.Value.Value) { - return json.Marshal([]string{hh.Key, hh.Value.Value}) - } - value := make(map[string]string) - value["format"] = "base64" - value["data"] = base64.StdEncoding.EncodeToString([]byte(hh.Value.Value)) - return json.Marshal([]interface{}{hh.Key, value}) -} - -// UnmarshalJSON is the opposite of MarshalJSON. -func (hh *HTTPHeader) UnmarshalJSON(d []byte) error { - var pair []interface{} - if err := json.Unmarshal(d, &pair); err != nil { - return err - } - if len(pair) != 2 { - return errors.New("unexpected pair length") - } - key, ok := pair[0].(string) - if !ok { - return errors.New("the key is not a string") - } - value, ok := pair[1].(string) - if !ok { - mapvalue, ok := pair[1].(map[string]interface{}) - if !ok { - return errors.New("the value is neither a string nor a map[string]interface{}") - } - if _, ok := mapvalue["format"]; !ok { - return errors.New("missing format") - } - if v, ok := mapvalue["format"].(string); !ok || v != "base64" { - return errors.New("invalid format") - } - if _, ok := mapvalue["data"]; !ok { - return errors.New("missing data field") - } - v, ok := mapvalue["data"].(string) - if !ok { - return errors.New("the data field is not a string") - } - b64, err := base64.StdEncoding.DecodeString(v) - if err != nil { - return err - } - value = string(b64) - } - hh.Key, hh.Value = key, MaybeBinaryValue{Value: value} - return nil -} - -// HTTPHeadersList is a list of headers. -type HTTPHeadersList []HTTPHeader - -// HTTPRequest contains an HTTP request. -// -// Headers are a map in Web Connectivity data format but -// we have added support for a list since data format version -// equal to 0.2.1 (later renamed to 0.3.0). -type HTTPRequest struct { - Body HTTPBody `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - HeadersList HTTPHeadersList `json:"headers_list"` - Headers HTTPHeaders `json:"headers"` - Method string `json:"method"` - Tor HTTPTor `json:"tor"` - URL string `json:"url"` -} - -// HTTPResponse contains an HTTP response. -// -// Headers are a map in Web Connectivity data format but -// we have added support for a list since data format version -// equal to 0.2.1 (later renamed to 0.3.0). -type HTTPResponse struct { - Body HTTPBody `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - Code int64 `json:"code"` - HeadersList HTTPHeadersList `json:"headers_list"` - Headers HTTPHeaders `json:"headers"` -} - -// RequestEntry is one of the entries that are part of -// the "requests" key of a OONI report. -type RequestEntry struct { - Failure *string `json:"failure"` - Request HTTPRequest `json:"request"` - Response HTTPResponse `json:"response"` -} - -// RequestList is a list of RequestEntry -type RequestList []RequestEntry - -func addheaders( - source http.Header, - destList *HTTPHeadersList, - destMap *HTTPHeaders, -) { - for key, values := range source { - for index, value := range values { - value := MaybeBinaryValue{Value: value} - // With the map representation we can only represent a single - // value for every key. Hence the list representation. - if index == 0 { - (*destMap)[key] = value - } - *destList = append(*destList, HTTPHeader{ - Key: key, - Value: value, - }) - } - } -} - -// NewRequestList returns the list for "requests" -func NewRequestList(results oonitemplates.Results) RequestList { - var out RequestList - in := results.HTTPRequests - // OONI's data format wants more recent request first - for idx := len(in) - 1; idx >= 0; idx-- { - var entry RequestEntry - entry.Failure = makeFailure(in[idx].Error) - entry.Request.Headers = make(HTTPHeaders) - addheaders( - in[idx].RequestHeaders, &entry.Request.HeadersList, - &entry.Request.Headers, - ) - entry.Request.Method = in[idx].RequestMethod - entry.Request.URL = in[idx].RequestURL - entry.Request.Body.Value = string(in[idx].RequestBodySnap) - entry.Request.BodyIsTruncated = in[idx].MaxBodySnapSize > 0 && - int64(len(in[idx].RequestBodySnap)) >= in[idx].MaxBodySnapSize - entry.Response.Headers = make(HTTPHeaders) - addheaders( - in[idx].ResponseHeaders, &entry.Response.HeadersList, - &entry.Response.Headers, - ) - entry.Response.Code = in[idx].ResponseStatusCode - entry.Response.Body.Value = string(in[idx].ResponseBodySnap) - entry.Response.BodyIsTruncated = in[idx].MaxBodySnapSize > 0 && - int64(len(in[idx].ResponseBodySnap)) >= in[idx].MaxBodySnapSize - out = append(out, entry) - } - return out -} - -// DNSAnswerEntry is the answer to a DNS query -type DNSAnswerEntry struct { - AnswerType string `json:"answer_type"` - Hostname string `json:"hostname,omitempty"` - IPv4 string `json:"ipv4,omitempty"` - IPv6 string `json:"ipv6,omitempty"` - TTL *uint32 `json:"ttl"` -} - -// DNSQueryEntry is a DNS query with possibly an answer -type DNSQueryEntry struct { - Answers []DNSAnswerEntry `json:"answers"` - Engine string `json:"engine"` - Failure *string `json:"failure"` - Hostname string `json:"hostname"` - QueryType string `json:"query_type"` - ResolverHostname *string `json:"resolver_hostname"` - ResolverPort *string `json:"resolver_port"` - ResolverAddress string `json:"resolver_address"` - T float64 `json:"t"` -} - -type ( - // DNSQueriesList is a list of DNS queries - DNSQueriesList []DNSQueryEntry - dnsQueryType string -) - -// NewDNSQueriesList returns a list of DNS queries. -func NewDNSQueriesList(results oonitemplates.Results) DNSQueriesList { - // TODO(bassosimone): add support for CNAME lookups. - var out DNSQueriesList - for _, resolve := range results.Resolves { - for _, qtype := range []dnsQueryType{"A", "AAAA"} { - entry := qtype.makequeryentry(resolve) - for _, addr := range resolve.Addresses { - if qtype.ipoftype(addr) { - entry.Answers = append(entry.Answers, qtype.makeanswerentry(addr)) - } - } - out = append(out, entry) - } - } - return out -} - -func (qtype dnsQueryType) ipoftype(addr string) bool { - switch qtype { - case "A": - return strings.Contains(addr, ":") == false - case "AAAA": - return strings.Contains(addr, ":") == true - } - return false -} - -func (qtype dnsQueryType) makeanswerentry(addr string) DNSAnswerEntry { - answer := DNSAnswerEntry{AnswerType: string(qtype)} - switch qtype { - case "A": - answer.IPv4 = addr - case "AAAA": - answer.IPv6 = addr - } - return answer -} - -func (qtype dnsQueryType) makequeryentry(resolve *modelx.ResolveDoneEvent) DNSQueryEntry { - return DNSQueryEntry{ - Engine: resolve.TransportNetwork, - Failure: makeFailure(resolve.Error), - Hostname: resolve.Hostname, - QueryType: string(qtype), - ResolverAddress: resolve.TransportAddress, - T: resolve.DurationSinceBeginning.Seconds(), - } -} - -// NetworkEvent is a network event. -type NetworkEvent struct { - Address string `json:"address,omitempty"` - Failure *string `json:"failure"` - NumBytes int64 `json:"num_bytes,omitempty"` - Operation string `json:"operation"` - Proto string `json:"proto"` - T float64 `json:"t"` -} - -// NetworkEventsList is a list of network events. -type NetworkEventsList []*NetworkEvent - -var protocolName = map[bool]string{ - true: "tcp", - false: "udp", -} - -// NewNetworkEventsList returns a list of DNS queries. -func NewNetworkEventsList(results oonitemplates.Results) NetworkEventsList { - var out NetworkEventsList - for _, in := range results.NetworkEvents { - if in.Connect != nil { - out = append(out, &NetworkEvent{ - Address: in.Connect.RemoteAddress, - Failure: makeFailure(in.Connect.Error), - Operation: netxlite.ConnectOperation, - T: in.Connect.DurationSinceBeginning.Seconds(), - }) - // fallthrough - } - if in.Read != nil { - out = append(out, &NetworkEvent{ - Failure: makeFailure(in.Read.Error), - Operation: netxlite.ReadOperation, - NumBytes: in.Read.NumBytes, - T: in.Read.DurationSinceBeginning.Seconds(), - }) - // fallthrough - } - if in.Write != nil { - out = append(out, &NetworkEvent{ - Failure: makeFailure(in.Write.Error), - Operation: netxlite.WriteOperation, - NumBytes: in.Write.NumBytes, - T: in.Write.DurationSinceBeginning.Seconds(), - }) - // fallthrough - } - } - return out -} - -// TLSHandshake contains TLS handshake data -type TLSHandshake struct { - CipherSuite string `json:"cipher_suite"` - Failure *string `json:"failure"` - NegotiatedProtocol string `json:"negotiated_protocol"` - PeerCertificates []MaybeBinaryValue `json:"peer_certificates"` - T float64 `json:"t"` - TLSVersion string `json:"tls_version"` -} - -// TLSHandshakesList is a list of TLS handshakes -type TLSHandshakesList []TLSHandshake - -// NewTLSHandshakesList creates a new TLSHandshakesList -func NewTLSHandshakesList(results oonitemplates.Results) TLSHandshakesList { - var out TLSHandshakesList - for _, in := range results.TLSHandshakes { - out = append(out, TLSHandshake{ - CipherSuite: netxlite.TLSCipherSuiteString(in.ConnectionState.CipherSuite), - Failure: makeFailure(in.Error), - NegotiatedProtocol: in.ConnectionState.NegotiatedProtocol, - PeerCertificates: makePeerCerts(in.ConnectionState.PeerCertificates), - T: in.DurationSinceBeginning.Seconds(), - TLSVersion: netxlite.TLSVersionString(in.ConnectionState.Version), - }) - } - return out -} - -func makePeerCerts(in []modelx.X509Certificate) (out []MaybeBinaryValue) { - for _, e := range in { - out = append(out, MaybeBinaryValue{Value: string(e.Data)}) - } - return -} diff --git a/internal/engine/legacy/oonidatamodel/oonidatamodel_test.go b/internal/engine/legacy/oonidatamodel/oonidatamodel_test.go deleted file mode 100644 index 270c35f..0000000 --- a/internal/engine/legacy/oonidatamodel/oonidatamodel_test.go +++ /dev/null @@ -1,1085 +0,0 @@ -package oonidatamodel - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "errors" - "net/http" - "reflect" - "testing" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/oonitemplates" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestNewTCPConnectListEmpty(t *testing.T) { - out := NewTCPConnectList(oonitemplates.Results{}) - if len(out) != 0 { - t.Fatal("unexpected output length") - } -} - -func TestNewTCPConnectListSuccess(t *testing.T) { - out := NewTCPConnectList(oonitemplates.Results{ - Connects: []*modelx.ConnectEvent{ - { - RemoteAddress: "8.8.8.8:53", - }, - { - RemoteAddress: "8.8.4.4:853", - }, - }, - }) - if len(out) != 2 { - t.Fatal("unexpected output length") - } - if out[0].IP != "8.8.8.8" { - t.Fatal("unexpected out[0].IP") - } - if out[0].Port != 53 { - t.Fatal("unexpected out[0].Port") - } - if out[0].Status.Failure != nil { - t.Fatal("unexpected out[0].Failure") - } - if out[0].Status.Success != true { - t.Fatal("unexpected out[0].Success") - } - if out[1].IP != "8.8.4.4" { - t.Fatal("unexpected out[1].IP") - } - if out[1].Port != 853 { - t.Fatal("unexpected out[1].Port") - } - if out[1].Status.Failure != nil { - t.Fatal("unexpected out[0].Failure") - } - if out[1].Status.Success != true { - t.Fatal("unexpected out[0].Success") - } -} - -func TestNewTCPConnectListFailure(t *testing.T) { - out := NewTCPConnectList(oonitemplates.Results{ - Connects: []*modelx.ConnectEvent{ - { - RemoteAddress: "8.8.8.8:53", - Error: errors.New(netxlite.FailureConnectionReset), - }, - }, - }) - if len(out) != 1 { - t.Fatal("unexpected output length") - } - if out[0].IP != "8.8.8.8" { - t.Fatal("unexpected out[0].IP") - } - if out[0].Port != 53 { - t.Fatal("unexpected out[0].Port") - } - if *out[0].Status.Failure != netxlite.FailureConnectionReset { - t.Fatal("unexpected out[0].Failure") - } - if out[0].Status.Success != false { - t.Fatal("unexpected out[0].Success") - } -} - -func TestNewTCPConnectListInvalidInput(t *testing.T) { - out := NewTCPConnectList(oonitemplates.Results{ - Connects: []*modelx.ConnectEvent{ - { - RemoteAddress: "8.8.8.8", - Error: errors.New(netxlite.FailureConnectionReset), - }, - }, - }) - if len(out) != 1 { - t.Fatal("unexpected output length") - } - if out[0].IP != "" { - t.Fatal("unexpected out[0].IP") - } - if out[0].Port != 0 { - t.Fatal("unexpected out[0].Port") - } - if *out[0].Status.Failure != netxlite.FailureConnectionReset { - t.Fatal("unexpected out[0].Failure") - } - if out[0].Status.Success != false { - t.Fatal("unexpected out[0].Success") - } -} - -func TestNewRequestsListEmptyList(t *testing.T) { - out := NewRequestList(oonitemplates.Results{}) - if len(out) != 0 { - t.Fatal("unexpected output length") - } -} - -func TestNewRequestsListGood(t *testing.T) { - out := NewRequestList(oonitemplates.Results{ - HTTPRequests: []*modelx.HTTPRoundTripDoneEvent{ - // need two requests to test that order is inverted - { - RequestBodySnap: []byte("abcdefx"), - RequestHeaders: http.Header{ - "Content-Type": []string{ - "text/plain", - "foobar", - }, - "Content-Length": []string{ - "17", - }, - }, - RequestMethod: "GET", - RequestURL: "http://x.org/", - ResponseBodySnap: []byte("abcdef"), - ResponseHeaders: http.Header{ - "Content-Type": []string{ - "application/json", - "foobaz", - }, - "Server": []string{ - "antani", - }, - "Content-Length": []string{ - "14", - }, - }, - ResponseStatusCode: 451, - MaxBodySnapSize: 10, - }, - { - Error: errors.New("antani"), - }, - }, - }) - if len(out) != 2 { - t.Fatal("unexpected output length") - } - - if *out[0].Failure != "antani" { - t.Fatal("unexpected out[0].Failure") - } - if out[0].Request.Body.Value != "" { - t.Fatal("unexpected out[0].Request.Body.Value") - } - if len(out[0].Request.Headers) != 0 { - t.Fatal("unexpected out[0].Request.Headers") - } - if out[0].Request.Method != "" { - t.Fatal("unexpected out[0].Request.Method") - } - if out[0].Request.URL != "" { - t.Fatal("unexpected out[0].Request.URL") - } - if out[0].Request.BodyIsTruncated != false { - t.Fatal("unexpected out[0].Request.BodyIsTruncated") - } - if out[0].Response.Body.Value != "" { - t.Fatal("unexpected out[0].Response.Body.Value") - } - if out[0].Response.Code != 0 { - t.Fatal("unexpected out[0].Response.Code") - } - if len(out[0].Response.Headers) != 0 { - t.Fatal("unexpected out[0].Response.Headers") - } - if out[0].Response.BodyIsTruncated != false { - t.Fatal("unexpected out[0].Response.BodyIsTruncated") - } - - if out[1].Failure != nil { - t.Fatal("unexpected out[1].Failure") - } - if out[1].Request.Body.Value != "abcdefx" { - t.Fatal("unexpected out[1].Request.Body.Value") - } - if len(out[1].Request.Headers) != 2 { - t.Fatal("unexpected out[1].Request.Headers") - } - if out[1].Request.Headers["Content-Type"].Value != "text/plain" { - t.Fatal("unexpected out[1].Request.Headers Content-Type value") - } - if out[1].Request.Headers["Content-Length"].Value != "17" { - t.Fatal("unexpected out[1].Request.Headers Content-Length value") - } - var ( - requestHasTextPlain bool - requestHasFoobar bool - requestHasContentLength bool - requestHasOther int64 - ) - for _, header := range out[1].Request.HeadersList { - if header.Key == "Content-Type" { - if header.Value.Value == "text/plain" { - requestHasTextPlain = true - } else if header.Value.Value == "foobar" { - requestHasFoobar = true - } else { - requestHasOther++ - } - } else if header.Key == "Content-Length" { - if header.Value.Value == "17" { - requestHasContentLength = true - } else { - requestHasOther++ - } - } else { - requestHasOther++ - } - } - if !requestHasTextPlain { - t.Fatal("missing text/plain for request") - } - if !requestHasFoobar { - t.Fatal("missing foobar for request") - } - if !requestHasContentLength { - t.Fatal("missing content_length for request") - } - if requestHasOther != 0 { - t.Fatal("seen something unexpected") - } - if out[1].Request.Method != "GET" { - t.Fatal("unexpected out[1].Request.Method") - } - if out[1].Request.URL != "http://x.org/" { - t.Fatal("unexpected out[1].Request.URL") - } - if out[1].Request.BodyIsTruncated != false { - t.Fatal("unexpected out[1].Request.BodyIsTruncated") - } - - if out[1].Response.Body.Value != "abcdef" { - t.Fatal("unexpected out[1].Response.Body.Value") - } - if out[1].Response.Code != 451 { - t.Fatal("unexpected out[1].Response.Code") - } - if len(out[1].Response.Headers) != 3 { - t.Fatal("unexpected out[1].Response.Headers") - } - if out[1].Response.Headers["Content-Type"].Value != "application/json" { - t.Fatal("unexpected out[1].Response.Headers Content-Type value") - } - if out[1].Response.Headers["Server"].Value != "antani" { - t.Fatal("unexpected out[1].Response.Headers Server value") - } - if out[1].Response.Headers["Content-Length"].Value != "14" { - t.Fatal("unexpected out[1].Response.Headers Content-Length value") - } - var ( - responseHasApplicationJSON bool - responseHasFoobaz bool - responseHasServer bool - responseHasContentLength bool - responseHasOther int64 - ) - for _, header := range out[1].Response.HeadersList { - if header.Key == "Content-Type" { - if header.Value.Value == "application/json" { - responseHasApplicationJSON = true - } else if header.Value.Value == "foobaz" { - responseHasFoobaz = true - } else { - responseHasOther++ - } - } else if header.Key == "Content-Length" { - if header.Value.Value == "14" { - responseHasContentLength = true - } else { - responseHasOther++ - } - } else if header.Key == "Server" { - if header.Value.Value == "antani" { - responseHasServer = true - } else { - responseHasOther++ - } - } else { - responseHasOther++ - } - } - if !responseHasApplicationJSON { - t.Fatal("missing application/json for response") - } - if !responseHasFoobaz { - t.Fatal("missing foobaz for response") - } - if !responseHasContentLength { - t.Fatal("missing content_length for response") - } - if !responseHasServer { - t.Fatal("missing server for response") - } - if responseHasOther != 0 { - t.Fatal("seen something unexpected") - } - if out[1].Response.BodyIsTruncated != false { - t.Fatal("unexpected out[1].Response.BodyIsTruncated") - } -} - -func TestNewRequestsSnaps(t *testing.T) { - out := NewRequestList(oonitemplates.Results{ - HTTPRequests: []*modelx.HTTPRoundTripDoneEvent{ - { - RequestBodySnap: []byte("abcd"), - MaxBodySnapSize: 4, - ResponseBodySnap: []byte("defg"), - }, - }, - }) - if len(out) != 1 { - t.Fatal("unexpected output length") - } - if out[0].Request.BodyIsTruncated != true { - t.Fatal("wrong out[0].Request.BodyIsTruncated") - } - if out[0].Response.BodyIsTruncated != true { - t.Fatal("wrong out[0].Response.BodyIsTruncated") - } -} - -func TestMarshalUnmarshalHTTPBodyString(t *testing.T) { - mbv := HTTPBody{ - Value: "1234", - } - data, err := json.Marshal(mbv) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(data, []byte(`"1234"`)) { - t.Fatal("result is unexpected") - } - var newbody HTTPBody - if err := json.Unmarshal(data, &newbody); err != nil { - t.Fatal(err) - } - if newbody.Value != mbv.Value { - t.Fatal("string value mistmatch") - } -} - -var binaryInput = []uint8{ - 0x57, 0xe5, 0x79, 0xfb, 0xa6, 0xbb, 0x0d, 0xbc, 0xce, 0xbd, 0xa7, 0xa0, - 0xba, 0xa4, 0x78, 0x78, 0x12, 0x59, 0xee, 0x68, 0x39, 0xa4, 0x07, 0x98, - 0xc5, 0x3e, 0xbc, 0x55, 0xcb, 0xfe, 0x34, 0x3c, 0x7e, 0x1b, 0x5a, 0xb3, - 0x22, 0x9d, 0xc1, 0x2d, 0x6e, 0xca, 0x5b, 0xf1, 0x10, 0x25, 0x47, 0x1e, - 0x44, 0xe2, 0x2d, 0x60, 0x08, 0xea, 0xb0, 0x0a, 0xcc, 0x05, 0x48, 0xa0, - 0xf5, 0x78, 0x38, 0xf0, 0xdb, 0x3f, 0x9d, 0x9f, 0x25, 0x6f, 0x89, 0x00, - 0x96, 0x93, 0xaf, 0x43, 0xac, 0x4d, 0xc9, 0xac, 0x13, 0xdb, 0x22, 0xbe, - 0x7a, 0x7d, 0xd9, 0x24, 0xa2, 0x52, 0x69, 0xd8, 0x89, 0xc1, 0xd1, 0x57, - 0xaa, 0x04, 0x2b, 0xa2, 0xd8, 0xb1, 0x19, 0xf6, 0xd5, 0x11, 0x39, 0xbb, - 0x80, 0xcf, 0x86, 0xf9, 0x5f, 0x9d, 0x8c, 0xab, 0xf5, 0xc5, 0x74, 0x24, - 0x3a, 0xa2, 0xd4, 0x40, 0x4e, 0xd7, 0x10, 0x1f, -} - -func TestMarshalUnmarshalHTTPBodyBinary(t *testing.T) { - mbv := HTTPBody{ - Value: string(binaryInput), - } - data, err := json.Marshal(mbv) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(data, []byte(`{"data":"V+V5+6a7DbzOvaeguqR4eBJZ7mg5pAeYxT68Vcv+NDx+G1qzIp3BLW7KW/EQJUceROItYAjqsArMBUig9Xg48Ns/nZ8lb4kAlpOvQ6xNyawT2yK+en3ZJKJSadiJwdFXqgQrotixGfbVETm7gM+G+V+djKv1xXQkOqLUQE7XEB8=","format":"base64"}`)) { - t.Fatal("result is unexpected") - } - var newbody HTTPBody - if err := json.Unmarshal(data, &newbody); err != nil { - t.Fatal(err) - } - if newbody.Value != mbv.Value { - t.Fatal("string value mistmatch") - } -} - -func TestMaybeBinaryValueUnmarshalJSON(t *testing.T) { - t.Run("when the code is not a map or string", func(t *testing.T) { - var ( - mbv MaybeBinaryValue - input = []byte("[1, 2, 3, 4]") - ) - if err := json.Unmarshal(input, &mbv); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the format field is missing", func(t *testing.T) { - var ( - mbv MaybeBinaryValue - input = []byte("{}") - ) - if err := json.Unmarshal(input, &mbv); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the format field is invalid", func(t *testing.T) { - var ( - mbv MaybeBinaryValue - input = []byte(`{"format":"antani"}`) - ) - if err := json.Unmarshal(input, &mbv); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the data field is missing", func(t *testing.T) { - var ( - mbv MaybeBinaryValue - input = []byte(`{"format":"base64"}`) - ) - if err := json.Unmarshal(input, &mbv); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the data field is not base64", func(t *testing.T) { - var ( - mbv MaybeBinaryValue - input = []byte(`{"format":"base64","data":"antani"}`) - ) - if err := json.Unmarshal(input, &mbv); err == nil { - t.Fatal("expected an error here") - } - }) -} - -func TestMarshalUnmarshalHTTPHeaderString(t *testing.T) { - mbh := HTTPHeadersList{ - HTTPHeader{ - Key: "Content-Type", - Value: MaybeBinaryValue{ - Value: "application/json", - }, - }, - HTTPHeader{ - Key: "Content-Type", - Value: MaybeBinaryValue{ - Value: "antani", - }, - }, - HTTPHeader{ - Key: "Content-Length", - Value: MaybeBinaryValue{ - Value: "17", - }, - }, - } - data, err := json.Marshal(mbh) - if err != nil { - t.Fatal(err) - } - expected := []byte( - `[["Content-Type","application/json"],["Content-Type","antani"],["Content-Length","17"]]`, - ) - if !bytes.Equal(data, expected) { - t.Fatal("result is unexpected") - } - var newlist HTTPHeadersList - if err := json.Unmarshal(data, &newlist); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(mbh, newlist) { - t.Fatal("result mismatch") - } -} - -func TestMarshalUnmarshalHTTPHeaderBinary(t *testing.T) { - mbh := HTTPHeadersList{ - HTTPHeader{ - Key: "Content-Type", - Value: MaybeBinaryValue{ - Value: "application/json", - }, - }, - HTTPHeader{ - Key: "Content-Type", - Value: MaybeBinaryValue{ - Value: string(binaryInput), - }, - }, - HTTPHeader{ - Key: "Content-Length", - Value: MaybeBinaryValue{ - Value: "17", - }, - }, - } - data, err := json.Marshal(mbh) - if err != nil { - t.Fatal(err) - } - expected := []byte( - `[["Content-Type","application/json"],["Content-Type",{"data":"V+V5+6a7DbzOvaeguqR4eBJZ7mg5pAeYxT68Vcv+NDx+G1qzIp3BLW7KW/EQJUceROItYAjqsArMBUig9Xg48Ns/nZ8lb4kAlpOvQ6xNyawT2yK+en3ZJKJSadiJwdFXqgQrotixGfbVETm7gM+G+V+djKv1xXQkOqLUQE7XEB8=","format":"base64"}],["Content-Length","17"]]`, - ) - if !bytes.Equal(data, expected) { - t.Fatal("result is unexpected") - } - var newlist HTTPHeadersList - if err := json.Unmarshal(data, &newlist); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(mbh, newlist) { - t.Fatal("result mismatch") - } -} - -func TestHTTPHeaderUnmarshalJSON(t *testing.T) { - t.Run("when the code is not a list", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`{"foo":1}`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the pair length is not two", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte("[1,2,3]") - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the first element is not a string", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`[1, "antani"]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the second element is not map[string]interface{}", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`["antani", ["base64", "foo"]]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the format field is missing", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`["antani", {}]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the format field is not a string", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`["antani", {"format":1}]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the format field is invalid", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`["antani", {"format":"antani"}]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the data field is missing", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`["antani", {"format":"base64"}]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the data field is not a string", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`["antani", {"format":"base64","data":10}]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the data field is not base64", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`["antani", {"format":"base64","data":"antani"}]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) - t.Run("when the data field is not base64", func(t *testing.T) { - var ( - hh HTTPHeader - input = []byte(`["antani", {"format":"base64","data":"antani"}]`) - ) - if err := json.Unmarshal(input, &hh); err == nil { - t.Fatal("expected an error here") - } - }) -} - -func TestNewDNSQueriesListEmpty(t *testing.T) { - out := NewDNSQueriesList(oonitemplates.Results{}) - if len(out) != 0 { - t.Fatal("unexpected output length") - } -} - -func TestNewDNSQueriesListSuccess(t *testing.T) { - out := NewDNSQueriesList(oonitemplates.Results{ - Resolves: []*modelx.ResolveDoneEvent{ - { - Addresses: []string{ - "8.8.4.4", "2001:4860:4860::8888", - }, - Hostname: "dns.google", - TransportNetwork: "system", - }, - { - Error: errors.New(netxlite.FailureDNSNXDOMAINError), - Hostname: "dns.googlex", - TransportNetwork: "system", - }, - }, - }) - if len(out) != 4 { - t.Fatal("unexpected output length") - } - var ( - foundDNSGoogleA bool - foundDNSGoogleAAAA bool - foundErrorA bool - foundErrorAAAA bool - foundOther bool - ) - for _, e := range out { - switch e.Hostname { - case "dns.google": - switch e.QueryType { - case "A": - foundDNSGoogleA = true - if err := dnscheckgood(e); err != nil { - t.Fatal(err) - } - case "AAAA": - foundDNSGoogleAAAA = true - if err := dnscheckgood(e); err != nil { - t.Fatal(err) - } - default: - foundOther = true - } - case "dns.googlex": - switch e.QueryType { - case "A": - foundErrorA = true - if err := dnscheckbad(e); err != nil { - t.Fatal(err) - } - case "AAAA": - foundErrorAAAA = true - if err := dnscheckbad(e); err != nil { - t.Fatal(err) - } - default: - foundOther = true - } - default: - foundOther = true - } - } - if foundDNSGoogleA == false { - t.Fatal("missing A for dns.google") - } - if foundDNSGoogleAAAA == false { - t.Fatal("missing AAAA for dns.google") - } - if foundErrorA == false { - t.Fatal("missing A for invalid domain") - } - if foundErrorAAAA == false { - t.Fatal("missing AAAA for invalid domain") - } - if foundOther == true { - t.Fatal("seen something unexpected") - } -} - -func dnscheckgood(e DNSQueryEntry) error { - if len(e.Answers) != 1 { - return errors.New("unexpected number of answers") - } - if e.Engine != "system" { - return errors.New("invalid engine") - } - if e.Failure != nil { - return errors.New("invalid failure") - } - if e.Hostname != "dns.google" { - return errors.New("invalid hostname") - } - switch e.QueryType { - case "A", "AAAA": - default: - return errors.New("invalid query type") - } - if e.Answers[0].AnswerType != e.QueryType { - return errors.New("AnswerType mismatch") - } - switch e.QueryType { - case "A": - if e.Answers[0].IPv4 != "8.8.4.4" { - return errors.New("unexpected IPv4 entry") - } - case "AAAA": - if e.Answers[0].IPv6 != "2001:4860:4860::8888" { - return errors.New("unexpected IPv6 entry") - } - } - if e.ResolverHostname != nil { - return errors.New("invalid resolver hostname") - } - if e.ResolverPort != nil { - return errors.New("invalid resolver port") - } - if e.ResolverAddress != "" { - return errors.New("invalid resolver address") - } - return nil -} - -func dnscheckbad(e DNSQueryEntry) error { - if len(e.Answers) != 0 { - return errors.New("unexpected number of answers") - } - if e.Engine != "system" { - return errors.New("invalid engine") - } - if *e.Failure != netxlite.FailureDNSNXDOMAINError { - return errors.New("invalid failure") - } - if e.Hostname != "dns.googlex" { - return errors.New("invalid hostname") - } - switch e.QueryType { - case "A", "AAAA": - default: - return errors.New("invalid query type") - } - if e.ResolverHostname != nil { - return errors.New("invalid resolver hostname") - } - if e.ResolverPort != nil { - return errors.New("invalid resolver port") - } - if e.ResolverAddress != "" { - return errors.New("invalid resolver address") - } - return nil -} - -func TestDNSQueryTypeIPOfType(t *testing.T) { - qtype := dnsQueryType("ANTANI") - if qtype.ipoftype("8.8.8.8") == true { - t.Fatal("ipoftype misbehaving") - } -} - -func TestNewNetworkEventsListEmpty(t *testing.T) { - out := NewNetworkEventsList(oonitemplates.Results{}) - if len(out) != 0 { - t.Fatal("unexpected output length") - } -} - -func TestNewNetworkEventsListNoSuitableEvents(t *testing.T) { - out := NewNetworkEventsList(oonitemplates.Results{ - NetworkEvents: []*modelx.Measurement{ - {}, - {}, - {}, - }, - }) - if len(out) != 0 { - t.Fatal("unexpected output length") - } -} - -func TestNewNetworkEventsListGood(t *testing.T) { - out := NewNetworkEventsList(oonitemplates.Results{ - NetworkEvents: []*modelx.Measurement{ - { - Connect: &modelx.ConnectEvent{ - DurationSinceBeginning: 10 * time.Millisecond, - RemoteAddress: "1.1.1.1:443", - }, - }, - { - Read: &modelx.ReadEvent{ - DurationSinceBeginning: 20 * time.Millisecond, - NumBytes: 1789, - }, - }, - { - Write: &modelx.WriteEvent{ - DurationSinceBeginning: 30 * time.Millisecond, - NumBytes: 17714, - }, - }, - }, - }) - if len(out) != 3 { - t.Fatal("unexpected output length") - } - - if out[0].Address != "1.1.1.1:443" { - t.Fatal("wrong out[0].Address") - } - if out[0].Failure != nil { - t.Fatal("wrong out[0].Failure") - } - if out[0].NumBytes != 0 { - t.Fatal("wrong out[0].NumBytes") - } - if out[0].Operation != netxlite.ConnectOperation { - t.Fatal("wrong out[0].Operation") - } - if !floatEquals(out[0].T, 0.010) { - t.Fatal("wrong out[0].T") - } - - if out[1].Address != "" { - t.Fatal("wrong out[1].Address") - } - if out[1].Failure != nil { - t.Fatal("wrong out[1].Failure") - } - if out[1].NumBytes != 1789 { - t.Fatal("wrong out[1].NumBytes") - } - if out[1].Operation != netxlite.ReadOperation { - t.Fatal("wrong out[1].Operation") - } - if !floatEquals(out[1].T, 0.020) { - t.Fatal("wrong out[1].T") - } - - if out[2].Address != "" { - t.Fatal("wrong out[2].Address") - } - if out[2].Failure != nil { - t.Fatal("wrong out[2].Failure") - } - if out[2].NumBytes != 17714 { - t.Fatal("wrong out[2].NumBytes") - } - if out[2].Operation != netxlite.WriteOperation { - t.Fatal("wrong out[2].Operation") - } - if !floatEquals(out[2].T, 0.030) { - t.Fatal("wrong out[2].T") - } -} - -func TestNewNetworkEventsListGoodUDPAndErrors(t *testing.T) { - out := NewNetworkEventsList(oonitemplates.Results{ - NetworkEvents: []*modelx.Measurement{ - { - Connect: &modelx.ConnectEvent{ - DurationSinceBeginning: 10 * time.Millisecond, - Error: errors.New("mocked error"), - RemoteAddress: "1.1.1.1:443", - }, - }, - { - Read: &modelx.ReadEvent{ - DurationSinceBeginning: 20 * time.Millisecond, - Error: errors.New("mocked error"), - NumBytes: 1789, - }, - }, - { - Write: &modelx.WriteEvent{ - DurationSinceBeginning: 30 * time.Millisecond, - Error: errors.New("mocked error"), - NumBytes: 17714, - }, - }, - }, - }) - if len(out) != 3 { - t.Fatal("unexpected output length") - } - - if out[0].Address != "1.1.1.1:443" { - t.Fatal("wrong out[0].Address") - } - if *out[0].Failure != "mocked error" { - t.Fatal("wrong out[0].Failure") - } - if out[0].NumBytes != 0 { - t.Fatal("wrong out[0].NumBytes") - } - if out[0].Operation != netxlite.ConnectOperation { - t.Fatal("wrong out[0].Operation") - } - if !floatEquals(out[0].T, 0.010) { - t.Fatal("wrong out[0].T") - } - - if out[1].Address != "" { - t.Fatal("wrong out[1].Address") - } - if *out[1].Failure != "mocked error" { - t.Fatal("wrong out[1].Failure") - } - if out[1].NumBytes != 1789 { - t.Fatal("wrong out[1].NumBytes") - } - if out[1].Operation != netxlite.ReadOperation { - t.Fatal("wrong out[1].Operation") - } - if !floatEquals(out[1].T, 0.020) { - t.Fatal("wrong out[1].T") - } - - if out[2].Address != "" { - t.Fatal("wrong out[2].Address") - } - if *out[2].Failure != "mocked error" { - t.Fatal("wrong out[2].Failure") - } - if out[2].NumBytes != 17714 { - t.Fatal("wrong out[2].NumBytes") - } - if out[2].Operation != netxlite.WriteOperation { - t.Fatal("wrong out[2].Operation") - } - if !floatEquals(out[2].T, 0.030) { - t.Fatal("wrong out[2].T") - } -} - -func floatEquals(a, b float64) bool { - const c = 1e-03 - return (a-b) < c && (b-a) < c -} - -func TestNewTLSHandshakesListEmpty(t *testing.T) { - out := NewTLSHandshakesList(oonitemplates.Results{}) - if len(out) != 0 { - t.Fatal("unexpected output length") - } -} - -func TestNewTLSHandshakesListSuccess(t *testing.T) { - out := NewTLSHandshakesList(oonitemplates.Results{ - TLSHandshakes: []*modelx.TLSHandshakeDoneEvent{ - {}, - { - Error: errors.New("mocked error"), - }, - { - ConnectionState: modelx.TLSConnectionState{ - CipherSuite: tls.TLS_AES_128_GCM_SHA256, - NegotiatedProtocol: "h2", - PeerCertificates: []modelx.X509Certificate{ - { - Data: []byte("deadbeef"), - }, - { - Data: []byte("abad1dea"), - }, - }, - Version: tls.VersionTLS11, - }, - DurationSinceBeginning: 10 * time.Millisecond, - }, - }, - }) - if len(out) != 3 { - t.Fatal("unexpected output length") - } - - if out[0].CipherSuite != "" { - t.Fatal("invalid out[0].CipherSuite") - } - if out[0].Failure != nil { - t.Fatal("invalid out[0].Failure") - } - if out[0].NegotiatedProtocol != "" { - t.Fatal("invalid out[0].NegotiatedProtocol") - } - if len(out[0].PeerCertificates) != 0 { - t.Fatal("invalid out[0].PeerCertificates") - } - if !floatEquals(out[0].T, 0) { - t.Fatal("invalid out[0].T") - } - if out[0].TLSVersion != "" { - t.Fatal("invalid out[0].TLSVersion") - } - - if out[1].CipherSuite != "" { - t.Fatal("invalid out[1].CipherSuite") - } - if *out[1].Failure != "mocked error" { - t.Fatal("invalid out[1].Failure") - } - if out[1].NegotiatedProtocol != "" { - t.Fatal("invalid out[1].NegotiatedProtocol") - } - if len(out[1].PeerCertificates) != 0 { - t.Fatal("invalid out[1].PeerCertificates") - } - if !floatEquals(out[1].T, 0) { - t.Fatal("invalid out[1].T") - } - if out[1].TLSVersion != "" { - t.Fatal("invalid out[1].TLSVersion") - } - - if out[2].CipherSuite != "TLS_AES_128_GCM_SHA256" { - t.Fatal("invalid out[2].CipherSuite") - } - if out[2].Failure != nil { - t.Fatal("invalid out[2].Failure") - } - if out[2].NegotiatedProtocol != "h2" { - t.Fatal("invalid out[2].NegotiatedProtocol") - } - if len(out[2].PeerCertificates) != 2 { - t.Fatal("invalid out[2].PeerCertificates") - } - if !floatEquals(out[2].T, 0.010) { - t.Fatal("invalid out[2].T") - } - if out[2].TLSVersion != "TLSv1.1" { - t.Fatal("invalid out[2].TLSVersion") - } - - for idx, mbv := range out[2].PeerCertificates { - if idx == 0 && mbv.Value != "deadbeef" { - t.Fatal("invalid first certificate") - } - if idx == 1 && mbv.Value != "abad1dea" { - t.Fatal("invalid second certificate") - } - if idx < 0 || idx > 1 { - t.Fatal("invalid index") - } - } -} diff --git a/internal/engine/legacy/oonitemplates/oonitemplates.go b/internal/engine/legacy/oonitemplates/oonitemplates.go deleted file mode 100644 index 4a6651a..0000000 --- a/internal/engine/legacy/oonitemplates/oonitemplates.go +++ /dev/null @@ -1,568 +0,0 @@ -// Package oonitemplates contains templates for experiments. -// -// Every experiment should possibly be based on code inside of -// this package. In the future we should perhaps unify the code -// in here with the code in oonidatamodel. -// -// This has been forked from ooni/netx/x/porcelain because it was -// causing too much changes to keep this code in there. -package oonitemplates - -import ( - "context" - "io" - "io/ioutil" - "math/rand" - "net" - "net/http" - "net/url" - "sync" - "time" - - goptlib "git.torproject.org/pluggable-transports/goptlib.git" - "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "gitlab.com/yawning/obfs4.git/transports" - obfs4base "gitlab.com/yawning/obfs4.git/transports/base" -) - -type channelHandler struct { - ch chan<- modelx.Measurement - lateWrites *atomicx.Int64 -} - -func newChannelHandler(ch chan<- modelx.Measurement) *channelHandler { - return &channelHandler{ - ch: ch, - lateWrites: &atomicx.Int64{}, - } -} - -func (h *channelHandler) OnMeasurement(m modelx.Measurement) { - // Implementation note: when we're closing idle connections it - // may be that they're closed once we have stopped reading - // therefore (1) we MUST NOT close the channel to signal that - // we're done BECAUSE THIS IS A LIE and (2) we MUST instead - // arrange here for non-blocking sends. - select { - case h.ch <- m: - case <-time.After(100 * time.Millisecond): - h.lateWrites.Add(1) - } -} - -// Results contains the results of every operation that we care -// about and information on the number of bytes received and sent. -// When counting the number of bytes sent and received, we do not -// take into account domain name resolutions performed using the -// system resolver. We estimated that using heuristics with MK but -// we currently don't have a good solution. TODO(bassosimone): this -// can be improved by emitting estimates when we know that we are -// using the system resolver, so we can pick up estimates here. -type Results struct { - Connects []*modelx.ConnectEvent - HTTPRequests []*modelx.HTTPRoundTripDoneEvent - NetworkEvents []*modelx.Measurement - Resolves []*modelx.ResolveDoneEvent - TLSHandshakes []*modelx.TLSHandshakeDoneEvent -} - -func (r *Results) onMeasurement(m modelx.Measurement, lowLevel bool) { - if m.Connect != nil { - r.Connects = append(r.Connects, m.Connect) - if lowLevel { - r.NetworkEvents = append(r.NetworkEvents, &m) - } - } - if m.HTTPRoundTripDone != nil { - r.HTTPRequests = append(r.HTTPRequests, m.HTTPRoundTripDone) - } - if m.ResolveDone != nil { - r.Resolves = append(r.Resolves, m.ResolveDone) - } - if m.TLSHandshakeDone != nil { - r.TLSHandshakes = append(r.TLSHandshakes, m.TLSHandshakeDone) - } - if m.Read != nil { - if lowLevel { - r.NetworkEvents = append(r.NetworkEvents, &m) - } - } - if m.Write != nil { - if lowLevel { - r.NetworkEvents = append(r.NetworkEvents, &m) - } - } -} - -func (r *Results) collect( - output <-chan modelx.Measurement, - handler modelx.Handler, - main func(), - lowLevel bool, -) { - if handler == nil { - handler = handlers.NoHandler - } - done := make(chan interface{}) - go func() { - defer close(done) - main() - }() - for { - select { - case m := <-output: - handler.OnMeasurement(m) - r.onMeasurement(m, lowLevel) - case <-done: - return - } - } -} - -type dnsFallback struct { - network, address string -} - -func configureDNS(seed int64, network, address string) (modelx.DNSResolver, error) { - resolver, err := netx.NewResolver(network, address) - if err != nil { - return nil, err - } - fallbacks := []dnsFallback{ - { - network: "doh", - address: "https://cloudflare-dns.com/dns-query", - }, - { - network: "doh", - address: "https://dns.google/dns-query", - }, - { - network: "dot", - address: "8.8.8.8:853", - }, - { - network: "dot", - address: "8.8.4.4:853", - }, - { - network: "dot", - address: "1.1.1.1:853", - }, - { - network: "dot", - address: "9.9.9.9:853", - }, - } - random := rand.New(rand.NewSource(seed)) - random.Shuffle(len(fallbacks), func(i, j int) { - fallbacks[i], fallbacks[j] = fallbacks[j], fallbacks[i] - }) - var configured int - for i := 0; configured < 2 && i < len(fallbacks); i++ { - if fallbacks[i].network == network { - continue - } - var fallback modelx.DNSResolver - fallback, err = netx.NewResolver(fallbacks[i].network, fallbacks[i].address) - runtimex.PanicOnError(err, "porcelain: invalid fallbacks table") - resolver = netx.ChainResolvers(resolver, fallback) - configured++ - } - return resolver, nil -} - -// DNSLookupConfig contains DNSLookup settings. -type DNSLookupConfig struct { - Beginning time.Time - Handler modelx.Handler - Hostname string - ServerAddress string - ServerNetwork string -} - -// DNSLookupResults contains the results of a DNSLookup -type DNSLookupResults struct { - TestKeys Results - Addresses []string - Error error -} - -// DNSLookup performs a DNS lookup. -func DNSLookup( - ctx context.Context, config DNSLookupConfig, -) *DNSLookupResults { - var ( - mu sync.Mutex - results = new(DNSLookupResults) - ) - if config.Beginning.IsZero() { - config.Beginning = time.Now() - } - channel := make(chan modelx.Measurement) - root := &modelx.MeasurementRoot{ - Beginning: config.Beginning, - Handler: newChannelHandler(channel), - } - ctx = modelx.WithMeasurementRoot(ctx, root) - resolver, err := netx.NewResolver(config.ServerNetwork, config.ServerAddress) - if err != nil { - results.Error = err - return results - } - results.TestKeys.collect(channel, config.Handler, func() { - addrs, err := resolver.LookupHost(ctx, config.Hostname) - mu.Lock() - defer mu.Unlock() - results.Addresses, results.Error = addrs, err - }, false) - return results -} - -// HTTPDoConfig contains HTTPDo settings. -type HTTPDoConfig struct { - Accept string - AcceptLanguage string - Beginning time.Time - Body []byte - DNSServerAddress string - DNSServerNetwork string - Handler modelx.Handler - InsecureSkipVerify bool - Method string - ProxyFunc func(*http.Request) (*url.URL, error) - URL string - UserAgent string - - // MaxEventsBodySnapSize controls the snap size that - // we're using for bodies returned as modelx.Measurement. - // - // Same rules as modelx.MeasurementRoot.MaxBodySnapSize. - MaxEventsBodySnapSize int64 - - // MaxResponseBodySnapSize controls the snap size that - // we're using for the HTTPDoResults.BodySnap. - // - // Same rules as modelx.MeasurementRoot.MaxBodySnapSize. - MaxResponseBodySnapSize int64 -} - -// HTTPDoResults contains the results of a HTTPDo -type HTTPDoResults struct { - TestKeys Results - StatusCode int64 - Headers http.Header - BodySnap []byte - Error error -} - -// HTTPDo performs a HTTP request -func HTTPDo( - origCtx context.Context, config HTTPDoConfig, -) *HTTPDoResults { - var ( - mu sync.Mutex - results = new(HTTPDoResults) - ) - if config.Beginning.IsZero() { - config.Beginning = time.Now() - } - channel := make(chan modelx.Measurement) - // TODO(bassosimone): tell client to use specific CA bundle? - root := &modelx.MeasurementRoot{ - Beginning: config.Beginning, - Handler: newChannelHandler(channel), - MaxBodySnapSize: config.MaxEventsBodySnapSize, - } - ctx := modelx.WithMeasurementRoot(origCtx, root) - client := netx.NewHTTPClientWithProxyFunc(config.ProxyFunc) - resolver, err := configureDNS( - time.Now().UnixNano(), - config.DNSServerNetwork, - config.DNSServerAddress, - ) - if err != nil { - results.Error = err - return results - } - client.SetResolver(resolver) - if config.InsecureSkipVerify { - client.ForceSkipVerify() - } - // TODO(bassosimone): implement sending body - req, err := http.NewRequest(config.Method, config.URL, nil) - if err != nil { - results.Error = err - return results - } - if config.Accept != "" { - req.Header.Set("Accept", config.Accept) - } - if config.AcceptLanguage != "" { - req.Header.Set("Accept-Language", config.AcceptLanguage) - } - req.Header.Set("User-Agent", config.UserAgent) - req = req.WithContext(ctx) - results.TestKeys.collect(channel, config.Handler, func() { - defer client.HTTPClient.CloseIdleConnections() - resp, err := client.HTTPClient.Do(req) - if err != nil { - mu.Lock() - results.Error = err - mu.Unlock() - return - } - mu.Lock() - results.StatusCode = int64(resp.StatusCode) - results.Headers = resp.Header - mu.Unlock() - defer resp.Body.Close() - reader := io.LimitReader( - resp.Body, modelx.ComputeBodySnapSize( - config.MaxResponseBodySnapSize, - ), - ) - data, err := netxlite.ReadAllContext(ctx, reader) - mu.Lock() - results.BodySnap, results.Error = data, err - mu.Unlock() - }, false) - return results -} - -// TLSConnectConfig contains TLSConnect settings. -type TLSConnectConfig struct { - Address string - Beginning time.Time - DNSServerAddress string - DNSServerNetwork string - Handler modelx.Handler - InsecureSkipVerify bool - SNI string -} - -// TLSConnectResults contains the results of a TLSConnect -type TLSConnectResults struct { - TestKeys Results - Error error -} - -// TLSConnect performs a TLS connect. -func TLSConnect( - ctx context.Context, config TLSConnectConfig, -) *TLSConnectResults { - var ( - mu sync.Mutex - results = new(TLSConnectResults) - ) - if config.Beginning.IsZero() { - config.Beginning = time.Now() - } - channel := make(chan modelx.Measurement) - root := &modelx.MeasurementRoot{ - Beginning: config.Beginning, - Handler: newChannelHandler(channel), - } - ctx = modelx.WithMeasurementRoot(ctx, root) - dialer := netx.NewDialer() - // TODO(bassosimone): tell dialer to use specific CA bundle? - resolver, err := configureDNS( - time.Now().UnixNano(), - config.DNSServerNetwork, - config.DNSServerAddress, - ) - if err != nil { - results.Error = err - return results - } - dialer.SetResolver(resolver) - if config.InsecureSkipVerify { - dialer.ForceSkipVerify() - } - // TODO(bassosimone): can this call really fail? - dialer.ForceSpecificSNI(config.SNI) - results.TestKeys.collect(channel, config.Handler, func() { - conn, err := dialer.DialTLSContext(ctx, "tcp", config.Address) - if conn != nil { - defer conn.Close() - } - mu.Lock() - defer mu.Unlock() - results.Error = err - }, true) - return results -} - -// TCPConnectConfig contains TCPConnect settings. -type TCPConnectConfig struct { - Address string - Beginning time.Time - DNSServerAddress string - DNSServerNetwork string - Handler modelx.Handler -} - -// TCPConnectResults contains the results of a TCPConnect -type TCPConnectResults struct { - TestKeys Results - Error error -} - -// TCPConnect performs a TCP connect. -func TCPConnect( - ctx context.Context, config TCPConnectConfig, -) *TCPConnectResults { - var ( - mu sync.Mutex - results = new(TCPConnectResults) - ) - if config.Beginning.IsZero() { - config.Beginning = time.Now() - } - channel := make(chan modelx.Measurement) - root := &modelx.MeasurementRoot{ - Beginning: config.Beginning, - Handler: newChannelHandler(channel), - } - ctx = modelx.WithMeasurementRoot(ctx, root) - dialer := netx.NewDialer() - // TODO(bassosimone): tell dialer to use specific CA bundle? - resolver, err := configureDNS( - time.Now().UnixNano(), - config.DNSServerNetwork, - config.DNSServerAddress, - ) - if err != nil { - results.Error = err - return results - } - dialer.SetResolver(resolver) - results.TestKeys.collect(channel, config.Handler, func() { - conn, err := dialer.DialContext(ctx, "tcp", config.Address) - if conn != nil { - defer conn.Close() - } - mu.Lock() - defer mu.Unlock() - results.Error = err - }, false) - return results -} - -func init() { - runtimex.PanicOnError(transports.Init(), "transport.Init() failed") -} - -// OBFS4ConnectConfig contains OBFS4Connect settings. -type OBFS4ConnectConfig struct { - Address string - Beginning time.Time - DNSServerAddress string - DNSServerNetwork string - Handler modelx.Handler - Params goptlib.Args - StateBaseDir string - Timeout time.Duration - ioutilTempDir func(dir string, prefix string) (string, error) - transportsGet func(name string) obfs4base.Transport - setDeadline func(net.Conn, time.Time) error -} - -// OBFS4ConnectResults contains the results of a OBFS4Connect -type OBFS4ConnectResults struct { - TestKeys Results - Error error -} - -// OBFS4Connect performs a TCP connect. -func OBFS4Connect( - ctx context.Context, config OBFS4ConnectConfig, -) *OBFS4ConnectResults { - var ( - mu sync.Mutex - results = new(OBFS4ConnectResults) - ) - if config.Beginning.IsZero() { - config.Beginning = time.Now() - } - channel := make(chan modelx.Measurement) - root := &modelx.MeasurementRoot{ - Beginning: config.Beginning, - Handler: newChannelHandler(channel), - } - ctx = modelx.WithMeasurementRoot(ctx, root) - dialer := netx.NewDialer() - // TODO(bassosimone): tell dialer to use specific CA bundle? - resolver, err := configureDNS( - time.Now().UnixNano(), - config.DNSServerNetwork, - config.DNSServerAddress, - ) - if err != nil { - results.Error = err - return results - } - dialer.SetResolver(resolver) - transportsGet := config.transportsGet - if transportsGet == nil { - transportsGet = transports.Get - } - txp := transportsGet("obfs4") - ioutilTempDir := config.ioutilTempDir - if ioutilTempDir == nil { - ioutilTempDir = ioutil.TempDir - } - dirname, err := ioutilTempDir(config.StateBaseDir, "obfs4") - if err != nil { - results.Error = err - return results - } - factory, err := txp.ClientFactory(dirname) - if err != nil { - results.Error = err - return results - } - parsedargs, err := factory.ParseArgs(&config.Params) - if err != nil { - results.Error = err - return results - } - results.TestKeys.collect(channel, config.Handler, func() { - dialfunc := func(network, address string) (net.Conn, error) { - conn, err := dialer.DialContext(ctx, network, address) - if err != nil { - return nil, err - } - // I didn't immediately see an API for limiting in time the - // duration of the handshake, so let's set a deadline. - timeout := config.Timeout - if timeout == 0 { - timeout = 30 * time.Second - } - setDeadline := config.setDeadline - if setDeadline == nil { - setDeadline = func(conn net.Conn, t time.Time) error { - return conn.SetDeadline(t) - } - } - if err := setDeadline(conn, time.Now().Add(timeout)); err != nil { - conn.Close() - return nil, err - } - return conn, nil - } - conn, err := factory.Dial("tcp", config.Address, dialfunc, parsedargs) - if conn != nil { - defer conn.Close() - } - mu.Lock() - defer mu.Unlock() - results.Error = err - }, true) - return results -} diff --git a/internal/engine/legacy/oonitemplates/oonitemplates_test.go b/internal/engine/legacy/oonitemplates/oonitemplates_test.go deleted file mode 100644 index 7ee27d9..0000000 --- a/internal/engine/legacy/oonitemplates/oonitemplates_test.go +++ /dev/null @@ -1,389 +0,0 @@ -package oonitemplates - -import ( - "context" - "errors" - "net" - "strings" - "sync" - "testing" - "time" - - goptlib "git.torproject.org/pluggable-transports/goptlib.git" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "gitlab.com/yawning/obfs4.git/transports" - obfs4base "gitlab.com/yawning/obfs4.git/transports/base" -) - -func TestChannelHandlerWriteLateOnChannel(t *testing.T) { - handler := newChannelHandler(make(chan modelx.Measurement)) - var waitgroup sync.WaitGroup - waitgroup.Add(1) - go func() { - time.Sleep(1 * time.Second) - handler.OnMeasurement(modelx.Measurement{}) - waitgroup.Done() - }() - waitgroup.Wait() - if handler.lateWrites.Load() != 1 { - t.Fatal("unexpected lateWrites value") - } -} - -func TestDNSLookupGood(t *testing.T) { - ctx := context.Background() - results := DNSLookup(ctx, DNSLookupConfig{ - Hostname: "ooni.io", - }) - if results.Error != nil { - t.Fatal(results.Error) - } - if len(results.Addresses) < 1 { - t.Fatal("no addresses returned?!") - } -} - -func TestDNSLookupCancellation(t *testing.T) { - ctx, cancel := context.WithTimeout( - context.Background(), time.Microsecond, - ) - defer cancel() - results := DNSLookup(ctx, DNSLookupConfig{ - Hostname: "ooni.io", - }) - if results.Error == nil { - t.Fatal("expected an error here") - } - if results.Error.Error() != netxlite.FailureGenericTimeoutError { - t.Fatal("not the error we expected") - } - if len(results.Addresses) > 0 { - t.Fatal("addresses returned?!") - } -} - -func TestDNSLookupUnknownDNS(t *testing.T) { - ctx := context.Background() - results := DNSLookup(ctx, DNSLookupConfig{ - Hostname: "ooni.io", - ServerNetwork: "antani", - }) - if !strings.HasSuffix(results.Error.Error(), "unsupported network value") { - t.Fatal("expected a different error here") - } -} - -func TestHTTPDoGood(t *testing.T) { - ctx := context.Background() - results := HTTPDo(ctx, HTTPDoConfig{ - Accept: "*/*", - AcceptLanguage: "en", - URL: "http://ooni.io", - }) - if results.Error != nil { - t.Fatal(results.Error) - } - if results.StatusCode != 200 { - t.Fatal("request failed?!") - } - if len(results.Headers) < 1 { - t.Fatal("no headers?!") - } - if len(results.BodySnap) < 1 { - t.Fatal("no body?!") - } -} - -func TestHTTPDoUnknownDNS(t *testing.T) { - ctx := context.Background() - results := HTTPDo(ctx, HTTPDoConfig{ - URL: "http://ooni.io", - DNSServerNetwork: "antani", - }) - if !strings.HasSuffix(results.Error.Error(), "unsupported network value") { - t.Fatal("not the error that we expected") - } -} - -func TestHTTPDoForceSkipVerify(t *testing.T) { - ctx := context.Background() - results := HTTPDo(ctx, HTTPDoConfig{ - URL: "https://self-signed.badssl.com/", - InsecureSkipVerify: true, - }) - if results.Error != nil { - t.Fatal(results.Error) - } -} - -func TestHTTPDoRoundTripError(t *testing.T) { - ctx := context.Background() - results := HTTPDo(ctx, HTTPDoConfig{ - URL: "http://ooni.io:443", // 443 with http - }) - if results.Error == nil { - t.Fatal("expected an error here") - } -} - -func TestHTTPDoBadURL(t *testing.T) { - ctx := context.Background() - results := HTTPDo(ctx, HTTPDoConfig{ - URL: "\t", - }) - if !strings.HasSuffix(results.Error.Error(), "invalid control character in URL") { - t.Fatal("not the error we expected") - } -} - -func TestTLSConnectGood(t *testing.T) { - ctx := context.Background() - results := TLSConnect(ctx, TLSConnectConfig{ - Address: "ooni.io:443", - }) - if results.Error != nil { - t.Fatal(results.Error) - } -} - -func TestTLSConnectGoodWithDoT(t *testing.T) { - ctx := context.Background() - results := TLSConnect(ctx, TLSConnectConfig{ - Address: "ooni.io:443", - DNSServerNetwork: "dot", - DNSServerAddress: "9.9.9.9:853", - }) - if results.Error != nil { - t.Fatal(results.Error) - } -} - -func TestTLSConnectCancellation(t *testing.T) { - ctx, cancel := context.WithTimeout( - context.Background(), time.Microsecond, - ) - defer cancel() - results := TLSConnect(ctx, TLSConnectConfig{ - Address: "ooni.io:443", - }) - if results.Error == nil { - t.Fatal("expected an error here") - } - if results.Error.Error() != netxlite.FailureGenericTimeoutError { - t.Fatal("not the error we expected") - } -} - -func TestTLSConnectUnknownDNS(t *testing.T) { - ctx := context.Background() - results := TLSConnect(ctx, TLSConnectConfig{ - Address: "ooni.io:443", - DNSServerNetwork: "antani", - }) - if !strings.HasSuffix(results.Error.Error(), "unsupported network value") { - t.Fatal("not the error that we expected") - } -} - -func TestTLSConnectForceSkipVerify(t *testing.T) { - ctx := context.Background() - results := TLSConnect(ctx, TLSConnectConfig{ - Address: "self-signed.badssl.com:443", - InsecureSkipVerify: true, - }) - if results.Error != nil { - t.Fatal(results.Error) - } -} - -func TestBodySnapSizes(t *testing.T) { - const ( - maxEventsBodySnapSize = 1 << 7 - maxResponseBodySnapSize = 1 << 8 - ) - ctx := context.Background() - results := HTTPDo(ctx, HTTPDoConfig{ - URL: "https://ooni.org", - MaxEventsBodySnapSize: maxEventsBodySnapSize, - MaxResponseBodySnapSize: maxResponseBodySnapSize, - }) - if results.Error != nil { - t.Fatal(results.Error) - } - if results.StatusCode != 200 { - t.Fatal("request failed?!") - } - if len(results.Headers) < 1 { - t.Fatal("no headers?!") - } - if len(results.BodySnap) != maxResponseBodySnapSize { - t.Fatal("invalid response body snap size") - } - if results.TestKeys.HTTPRequests == nil { - t.Fatal("no HTTPRequests?!") - } - for _, req := range results.TestKeys.HTTPRequests { - if len(req.ResponseBodySnap) != maxEventsBodySnapSize { - t.Fatal("invalid length of ResponseBodySnap") - } - if req.MaxBodySnapSize != maxEventsBodySnapSize { - t.Fatal("unexpected value of MaxBodySnapSize") - } - } -} - -func TestTCPConnectGood(t *testing.T) { - ctx := context.Background() - results := TCPConnect(ctx, TCPConnectConfig{ - Address: "ooni.io:443", - }) - if results.Error != nil { - t.Fatal(results.Error) - } -} - -func TestTCPConnectGoodWithDoT(t *testing.T) { - ctx := context.Background() - results := TCPConnect(ctx, TCPConnectConfig{ - Address: "ooni.io:443", - DNSServerNetwork: "dot", - DNSServerAddress: "9.9.9.9:853", - }) - if results.Error != nil { - t.Fatal(results.Error) - } -} - -func TestTCPConnectUnknownDNS(t *testing.T) { - ctx := context.Background() - results := TCPConnect(ctx, TCPConnectConfig{ - Address: "ooni.io:443", - DNSServerNetwork: "antani", - }) - if !strings.HasSuffix(results.Error.Error(), "unsupported network value") { - t.Fatal("not the error that we expected") - } -} - -func obfs4config() OBFS4ConnectConfig { - // TODO(bassosimone): this is a public working bridge we have found - // with @hellais. We should ask @phw whether there is some obfs4 bridge - // dedicated to integration testing that we should use instead. - return OBFS4ConnectConfig{ - Address: "109.105.109.165:10527", - StateBaseDir: "../../testdata/", - Params: map[string][]string{ - "cert": { - "Bvg/itxeL4TWKLP6N1MaQzSOC6tcRIBv6q57DYAZc3b2AzuM+/TfB7mqTFEfXILCjEwzVA", - }, - "iat-mode": {"1"}, - }, - } -} - -func TestOBFS4ConnectGood(t *testing.T) { - ctx := context.Background() - results := OBFS4Connect(ctx, obfs4config()) - if results.Error != nil { - t.Fatal(results.Error) - } -} - -func TestOBFS4ConnectGoodWithDoT(t *testing.T) { - ctx := context.Background() - config := obfs4config() - config.DNSServerNetwork = "dot" - config.DNSServerAddress = "9.9.9.9:853" - results := OBFS4Connect(ctx, config) - if results.Error != nil { - t.Fatal(results.Error) - } -} - -func TestOBFS4ConnectUnknownDNS(t *testing.T) { - ctx := context.Background() - config := obfs4config() - config.DNSServerNetwork = "antani" - results := OBFS4Connect(ctx, config) - if !strings.HasSuffix(results.Error.Error(), "unsupported network value") { - t.Fatal("not the error that we expected") - } -} - -func TestOBFS4IoutilTempDirError(t *testing.T) { - ctx := context.Background() - config := obfs4config() - expected := errors.New("mocked error") - config.ioutilTempDir = func(dir, prefix string) (string, error) { - return "", expected - } - results := OBFS4Connect(ctx, config) - if !errors.Is(results.Error, expected) { - t.Fatal("not the error that we expected") - } -} - -func TestOBFS4ClientFactoryError(t *testing.T) { - ctx := context.Background() - config := obfs4config() - config.transportsGet = func(name string) obfs4base.Transport { - txp := transports.Get(name) - if name == "obfs4" && txp != nil { - txp = &faketransport{txp: txp} - } - return txp - } - results := OBFS4Connect(ctx, config) - if results.Error.Error() != "mocked ClientFactory error" { - t.Fatal("not the error we expected") - } -} - -func TestOBFS4ParseArgsError(t *testing.T) { - ctx := context.Background() - config := obfs4config() - config.Params = make(map[string][]string) // cause ParseArgs error - results := OBFS4Connect(ctx, config) - if results.Error.Error() != "missing argument 'node-id'" { - t.Fatal("not the error we expected") - } -} - -func TestOBFS4DialContextError(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() // should cause DialContex to fail - config := obfs4config() - results := OBFS4Connect(ctx, config) - if results.Error.Error() != "interrupted" { - t.Fatal("not the error we expected") - } -} - -func TestOBFS4SetDeadlineError(t *testing.T) { - ctx := context.Background() - config := obfs4config() - config.setDeadline = func(net.Conn, time.Time) error { - return errors.New("mocked error") - } - results := OBFS4Connect(ctx, config) - if !strings.HasSuffix(results.Error.Error(), "mocked error") { - t.Fatal("not the error we expected") - } -} - -type faketransport struct { - txp obfs4base.Transport -} - -func (txp *faketransport) Name() string { - return txp.txp.Name() -} - -func (txp *faketransport) ClientFactory(stateDir string) (obfs4base.ClientFactory, error) { - return nil, errors.New("mocked ClientFactory error") -} - -func (txp *faketransport) ServerFactory(stateDir string, args *goptlib.Args) (obfs4base.ServerFactory, error) { - return txp.txp.ServerFactory(stateDir, args) -} diff --git a/internal/engine/netx/resolver/emitter.go b/internal/engine/netx/resolver/emitter.go deleted file mode 100644 index 519f05d..0000000 --- a/internal/engine/netx/resolver/emitter.go +++ /dev/null @@ -1,79 +0,0 @@ -package resolver - -import ( - "context" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" -) - -// EmitterTransport is a RoundTripper that emits events when they occur. -type EmitterTransport struct { - RoundTripper -} - -// RoundTrip implements RoundTripper.RoundTrip -func (txp EmitterTransport) RoundTrip(ctx context.Context, querydata []byte) ([]byte, error) { - root := modelx.ContextMeasurementRootOrDefault(ctx) - root.Handler.OnMeasurement(modelx.Measurement{ - DNSQuery: &modelx.DNSQueryEvent{ - Data: querydata, - DurationSinceBeginning: time.Since(root.Beginning), - }, - }) - replydata, err := txp.RoundTripper.RoundTrip(ctx, querydata) - if err != nil { - return nil, err - } - root.Handler.OnMeasurement(modelx.Measurement{ - DNSReply: &modelx.DNSReplyEvent{ - Data: replydata, - DurationSinceBeginning: time.Since(root.Beginning), - }, - }) - return replydata, nil -} - -// EmitterResolver is a resolver that emits events -type EmitterResolver struct { - Resolver -} - -// LookupHost returns the IP addresses of a host -func (r EmitterResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) { - var ( - network string - address string - ) - type queryableResolver interface { - Transport() RoundTripper - } - if qr, ok := r.Resolver.(queryableResolver); ok { - txp := qr.Transport() - network, address = txp.Network(), txp.Address() - } - root := modelx.ContextMeasurementRootOrDefault(ctx) - root.Handler.OnMeasurement(modelx.Measurement{ - ResolveStart: &modelx.ResolveStartEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - Hostname: hostname, - TransportAddress: address, - TransportNetwork: network, - }, - }) - addrs, err := r.Resolver.LookupHost(ctx, hostname) - root.Handler.OnMeasurement(modelx.Measurement{ - ResolveDone: &modelx.ResolveDoneEvent{ - Addresses: addrs, - DurationSinceBeginning: time.Since(root.Beginning), - Error: err, - Hostname: hostname, - TransportAddress: address, - TransportNetwork: network, - }, - }) - return addrs, err -} - -var _ RoundTripper = EmitterTransport{} -var _ Resolver = EmitterResolver{} diff --git a/internal/engine/netx/resolver/emitter_test.go b/internal/engine/netx/resolver/emitter_test.go deleted file mode 100644 index 84e8d26..0000000 --- a/internal/engine/netx/resolver/emitter_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package resolver_test - -import ( - "bytes" - "context" - "errors" - "io" - "net/http" - "testing" - "time" - - "github.com/miekg/dns" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" - "github.com/ooni/probe-cli/v3/internal/model/mocks" -) - -func TestEmitterTransportSuccess(t *testing.T) { - ctx := context.Background() - handler := &handlers.SavingHandler{} - root := &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: handler, - } - ctx = modelx.WithMeasurementRoot(ctx, root) - txp := resolver.EmitterTransport{RoundTripper: resolver.FakeTransport{ - Data: resolver.GenReplySuccess(t, dns.TypeA, "8.8.8.8"), - }} - e := resolver.MiekgEncoder{} - querydata, err := e.Encode("www.google.com", dns.TypeAAAA, true) - if err != nil { - t.Fatal(err) - } - replydata, err := txp.RoundTrip(ctx, querydata) - if err != nil { - t.Fatal(err) - } - events := handler.Read() - if len(events) != 2 { - t.Fatal("unexpected number of events") - } - if events[0].DNSQuery == nil { - t.Fatal("missing DNSQuery field") - } - if !bytes.Equal(events[0].DNSQuery.Data, querydata) { - t.Fatal("invalid query data") - } - if events[0].DNSQuery.DurationSinceBeginning <= 0 { - t.Fatal("invalid duration since beginning") - } - if events[1].DNSReply == nil { - t.Fatal("missing DNSReply field") - } - if !bytes.Equal(events[1].DNSReply.Data, replydata) { - t.Fatal("missing reply data") - } - if events[1].DNSReply.DurationSinceBeginning <= 0 { - t.Fatal("invalid duration since beginning") - } -} - -func TestEmitterTransportFailure(t *testing.T) { - ctx := context.Background() - handler := &handlers.SavingHandler{} - root := &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: handler, - } - ctx = modelx.WithMeasurementRoot(ctx, root) - mocked := errors.New("mocked error") - txp := resolver.EmitterTransport{RoundTripper: resolver.FakeTransport{ - Err: mocked, - }} - e := resolver.MiekgEncoder{} - querydata, err := e.Encode("www.google.com", dns.TypeAAAA, true) - if err != nil { - t.Fatal(err) - } - replydata, err := txp.RoundTrip(ctx, querydata) - if !errors.Is(err, mocked) { - t.Fatal("not the error we expected") - } - if replydata != nil { - t.Fatal("expected nil replydata") - } - events := handler.Read() - if len(events) != 1 { - t.Fatal("unexpected number of events") - } - if events[0].DNSQuery == nil { - t.Fatal("missing DNSQuery field") - } - if !bytes.Equal(events[0].DNSQuery.Data, querydata) { - t.Fatal("invalid query data") - } - if events[0].DNSQuery.DurationSinceBeginning <= 0 { - t.Fatal("invalid duration since beginning") - } -} - -func TestEmitterResolverFailure(t *testing.T) { - ctx := context.Background() - handler := &handlers.SavingHandler{} - root := &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: handler, - } - ctx = modelx.WithMeasurementRoot(ctx, root) - r := resolver.EmitterResolver{Resolver: resolver.NewSerialResolver( - &resolver.DNSOverHTTPS{ - Client: &mocks.HTTPClient{ - MockDo: func(req *http.Request) (*http.Response, error) { - return nil, io.EOF - }, - }, - URL: "https://dns.google.com/", - }, - )} - replies, err := r.LookupHost(ctx, "www.google.com") - if !errors.Is(err, io.EOF) { - t.Fatal("not the error we expected") - } - if replies != nil { - t.Fatal("expected nil replies") - } - events := handler.Read() - if len(events) != 2 { - t.Fatal("unexpected number of events") - } - if events[0].ResolveStart == nil { - t.Fatal("missing ResolveStart field") - } - if events[0].ResolveStart.DurationSinceBeginning <= 0 { - t.Fatal("invalid duration since beginning") - } - if events[0].ResolveStart.Hostname != "www.google.com" { - t.Fatal("invalid Hostname") - } - if events[0].ResolveStart.TransportAddress != "https://dns.google.com/" { - t.Fatal("invalid TransportAddress") - } - if events[0].ResolveStart.TransportNetwork != "doh" { - t.Fatal("invalid TransportNetwork") - } - if events[1].ResolveDone == nil { - t.Fatal("missing ResolveDone field") - } - if events[1].ResolveDone.DurationSinceBeginning <= 0 { - t.Fatal("invalid duration since beginning") - } - if events[1].ResolveDone.Error != io.EOF { - t.Fatal("invalid Error") - } - if events[1].ResolveDone.Hostname != "www.google.com" { - t.Fatal("invalid Hostname") - } - if events[1].ResolveDone.TransportAddress != "https://dns.google.com/" { - t.Fatal("invalid TransportAddress") - } - if events[1].ResolveDone.TransportNetwork != "doh" { - t.Fatal("invalid TransportNetwork") - } -} - -func TestEmitterResolverSuccess(t *testing.T) { - ctx := context.Background() - handler := &handlers.SavingHandler{} - root := &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: handler, - } - ctx = modelx.WithMeasurementRoot(ctx, root) - r := resolver.EmitterResolver{Resolver: resolver.NewFakeResolverWithResult( - []string{"8.8.8.8"}, - )} - replies, err := r.LookupHost(ctx, "dns.google.com") - if err != nil { - t.Fatal(err) - } - if len(replies) != 1 { - t.Fatal("expected a single replies") - } - events := handler.Read() - if len(events) != 2 { - t.Fatal("unexpected number of events") - } - if events[1].ResolveDone == nil { - t.Fatal("missing ResolveDone field") - } - if events[1].ResolveDone.Addresses[0] != "8.8.8.8" { - t.Fatal("invalid Addresses") - } -} diff --git a/internal/engine/netx/tlsdialer/tls.go b/internal/engine/netx/tlsdialer/tls.go index a1de437..fcdf676 100644 --- a/internal/engine/netx/tlsdialer/tls.go +++ b/internal/engine/netx/tlsdialer/tls.go @@ -5,9 +5,6 @@ import ( "context" "crypto/tls" "net" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" ) // UnderlyingDialer is the underlying dialer type. @@ -20,30 +17,3 @@ type TLSHandshaker interface { Handshake(ctx context.Context, conn net.Conn, config *tls.Config) ( net.Conn, tls.ConnectionState, error) } - -// EmitterTLSHandshaker emits events using the MeasurementRoot -type EmitterTLSHandshaker struct { - TLSHandshaker -} - -// Handshake implements Handshaker.Handshake -func (h EmitterTLSHandshaker) Handshake( - ctx context.Context, conn net.Conn, config *tls.Config, -) (net.Conn, tls.ConnectionState, error) { - root := modelx.ContextMeasurementRootOrDefault(ctx) - root.Handler.OnMeasurement(modelx.Measurement{ - TLSHandshakeStart: &modelx.TLSHandshakeStartEvent{ - DurationSinceBeginning: time.Since(root.Beginning), - SNI: config.ServerName, - }, - }) - tlsconn, state, err := h.TLSHandshaker.Handshake(ctx, conn, config) - root.Handler.OnMeasurement(modelx.Measurement{ - TLSHandshakeDone: &modelx.TLSHandshakeDoneEvent{ - ConnectionState: modelx.NewTLSConnectionState(state), - Error: err, - DurationSinceBeginning: time.Since(root.Beginning), - }, - }) - return tlsconn, state, err -} diff --git a/internal/engine/netx/tlsdialer/tls_test.go b/internal/engine/netx/tlsdialer/tls_test.go index 40257f9..9efeb98 100644 --- a/internal/engine/netx/tlsdialer/tls_test.go +++ b/internal/engine/netx/tlsdialer/tls_test.go @@ -3,13 +3,10 @@ package tlsdialer_test import ( "context" "crypto/tls" - "errors" "io" "testing" "time" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" - "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" "github.com/ooni/probe-cli/v3/internal/engine/netx/tlsdialer" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -36,40 +33,3 @@ func (c *SetDeadlineConn) SetDeadline(t time.Time) error { c.deadlines = append(c.deadlines, t) return nil } - -func TestEmitterTLSHandshakerFailure(t *testing.T) { - saver := &handlers.SavingHandler{} - ctx := modelx.WithMeasurementRoot(context.Background(), &modelx.MeasurementRoot{ - Beginning: time.Now(), - Handler: saver, - }) - h := tlsdialer.EmitterTLSHandshaker{TLSHandshaker: tlsdialer.EOFTLSHandshaker{}} - conn, _, err := h.Handshake(ctx, tlsdialer.EOFConn{}, &tls.Config{ - ServerName: "www.kernel.org", - }) - if !errors.Is(err, io.EOF) { - t.Fatal("not the error that we expected") - } - if conn != nil { - t.Fatal("expected nil con here") - } - events := saver.Read() - if len(events) != 2 { - t.Fatal("Wrong number of events") - } - if events[0].TLSHandshakeStart == nil { - t.Fatal("missing TLSHandshakeStart event") - } - if events[0].TLSHandshakeStart.DurationSinceBeginning == 0 { - t.Fatal("expected nonzero DurationSinceBeginning") - } - if events[0].TLSHandshakeStart.SNI != "www.kernel.org" { - t.Fatal("expected nonzero SNI") - } - if events[1].TLSHandshakeDone == nil { - t.Fatal("missing TLSHandshakeDone event") - } - if events[1].TLSHandshakeDone.DurationSinceBeginning == 0 { - t.Fatal("expected nonzero DurationSinceBeginning") - } -}