diff --git a/internal/cmd/oohelperd/internal/http.go b/internal/cmd/oohelperd/internal/http.go index 3f25bf8..3c2f71a 100644 --- a/internal/cmd/oohelperd/internal/http.go +++ b/internal/cmd/oohelperd/internal/http.go @@ -37,9 +37,7 @@ func HTTPDo(ctx context.Context, config *HTTPConfig) { // we're implementing (for now?) a more liberal approach. for k, vs := range config.Headers { switch strings.ToLower(k) { - case "user-agent": - case "accept": - case "accept-language": + case "user-agent", "accept", "accept-language": for _, v := range vs { req.Header.Add(k, v) } diff --git a/internal/cmd/oohelperd/internal/internal.go b/internal/cmd/oohelperd/internal/internal.go index 0bf5b44..071f61a 100644 --- a/internal/cmd/oohelperd/internal/internal.go +++ b/internal/cmd/oohelperd/internal/internal.go @@ -8,6 +8,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/iox" + "github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/version" ) @@ -27,10 +28,6 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.WriteHeader(400) return } - if req.Header.Get("content-type") != "application/json" { - w.WriteHeader(400) - return - } reader := &io.LimitedReader{R: req.Body, N: h.MaxAcceptableBody} data, err := iox.ReadAllContext(req.Context(), reader) if err != nil { @@ -50,7 +47,8 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // We assume that the following call cannot fail because it's a // clearly serializable data structure. - data, _ = json.Marshal(cresp) + data, err = json.Marshal(cresp) + runtimex.PanicOnError(err, "json.Marshal failed") w.Header().Add("Content-Type", "application/json") w.Write(data) } diff --git a/internal/cmd/oohelperd/internal/nwcth/explore.go b/internal/cmd/oohelperd/internal/nwcth/explore.go new file mode 100644 index 0000000..8c12c93 --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/explore.go @@ -0,0 +1,166 @@ +package nwcth + +import ( + "crypto/tls" + "net/http" + "net/http/cookiejar" + "net/url" + "sort" + "strings" + + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// Explore is the second step of the test helper algorithm. Its objective +// is to enumerate all the URLs we can discover by redirection from +// the original URL in the test list. Because the test list contains by +// definition noisy data, we need this preprocessing step to learn all +// the URLs that are actually implied by the original URL. + +// Explorer is the interface responsible for running Explore. +type Explorer interface { + Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) +} + +// DefaultExplorer is the default Explorer. +type DefaultExplorer struct { + resolver netxlite.Resolver +} + +// Explore returns a list of round trips sorted so that the first +// round trip is the first element in the list, and so on. +// Explore uses the URL and the optional headers provided by the CtrlRequest. +func (e *DefaultExplorer) Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) { + resp, err := e.get(URL, headers) + if err != nil { + return nil, err + } + rts := e.rearrange(resp, nil) + h3URL, err := getH3URL(resp) + if err != nil { + // If we cannot find the HTTP/3 URL for subsequent measurements, we just continue + // the measurement using the URLs we have found so far. + return rts, nil + } + resp, err = e.getH3(h3URL, headers) + if err != nil { + // If we cannot follow the HTTP/3 chain, we just continue + // the measurement using the URLs we have found so far. + return rts, nil + } + rts = append(rts, e.rearrange(resp, h3URL)...) + return rts, nil +} + +// rearrange takes in input the final response of an HTTP transaction and an optional h3URL +// (which is needed to derive the type of h3 protocol, i.e. h3 or h3-29), +// and produces in output a list of round trips sorted +// such that the first round trip is the first element in the out array. +func (e *DefaultExplorer) rearrange(resp *http.Response, h3URL *h3URL) (out []*RoundTrip) { + index := 0 + for resp != nil && resp.Request != nil { + proto := resp.Request.URL.Scheme + if h3URL != nil { + proto = h3URL.proto + } + out = append(out, &RoundTrip{ + Proto: proto, + SortIndex: index, + Request: resp.Request, + Response: resp, + }) + index += 1 + resp = resp.Request.Response + } + sh := &sortHelper{out} + sort.Sort(sh) + return +} + +// sortHelper is the helper structure to sort round trips. +type sortHelper struct { + v []*RoundTrip +} + +// Len implements sort.Interface.Len. +func (sh *sortHelper) Len() int { + return len(sh.v) +} + +// Less implements sort.Interface.Less. +func (sh *sortHelper) Less(i, j int) bool { + return sh.v[i].SortIndex >= sh.v[j].SortIndex +} + +// Swap implements sort.Interface.Swap. +func (sh *sortHelper) Swap(i, j int) { + sh.v[i], sh.v[j] = sh.v[j], sh.v[i] +} + +// get gets the given URL and returns the final response after +// redirection, and an error. If the error is nil, the final response is valid. +func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http.Response, error) { + tlsConf := &tls.Config{ + NextProtos: []string{"h2", "http/1.1"}, + } + transport := netxlite.NewHTTPTransport(NewDialerResolver(e.resolver), tlsConf, &netxlite.TLSHandshakerConfigurable{}) + // TODO(bassosimone): here we should use runtimex.PanicOnError + jarjar, _ := cookiejar.New(nil) + clnt := &http.Client{ + Transport: transport, + Jar: jarjar, + } + // TODO(bassosimone): document why e.newRequest cannot fail. + req, err := e.newRequest(URL, headers) + runtimex.PanicOnError(err, "newRequest failed") + resp, err := clnt.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + // Note that we ignore the response body. + return resp, nil +} + +// getH3 uses HTTP/3 to get the given URL and returns the final +// response after redirection, and an error. If the error is nil, the final response is valid. +func (e *DefaultExplorer) getH3(h3URL *h3URL, headers map[string][]string) (*http.Response, error) { + dialer := NewQUICDialerResolver(e.resolver) + tlsConf := &tls.Config{ + NextProtos: []string{h3URL.proto}, + } + transport := netxlite.NewHTTP3Transport(dialer, tlsConf) + // TODO(bassosimone): here we should use runtimex.PanicOnError + jarjar, _ := cookiejar.New(nil) + clnt := &http.Client{ + Transport: transport, + Jar: jarjar, + } + // TODO(bassosimone): document why e.newRequest cannot fail. + req, err := e.newRequest(h3URL.URL, headers) + runtimex.PanicOnError(err, "newRequest failed") + resp, err := clnt.Do(req) + if err != nil { + return nil, err + } + // Note that we ignore the response body. + defer resp.Body.Close() + return resp, nil +} + +func (e *DefaultExplorer) newRequest(URL *url.URL, headers map[string][]string) (*http.Request, error) { + req, err := http.NewRequest("GET", URL.String(), nil) + if err != nil { + return nil, err + } + for k, vs := range headers { + switch strings.ToLower(k) { + case "user-agent", "accept", "accept-language": + for _, v := range vs { + req.Header.Add(k, v) + } + } + } + return req, nil +} diff --git a/internal/cmd/oohelperd/internal/nwcth/explore_test.go b/internal/cmd/oohelperd/internal/nwcth/explore_test.go new file mode 100644 index 0000000..2bf4d5d --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/explore_test.go @@ -0,0 +1,156 @@ +package nwcth + +import ( + "net/http" + "net/url" + "testing" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +var explorer = &DefaultExplorer{resolver: newResolver()} + +func TestExploreSuccess(t *testing.T) { + u, err := url.Parse("https://example.com") + runtimex.PanicOnError(err, "url.Parse failed") + rts, err := explorer.Explore(u, nil) + if err != nil { + t.Fatal("unexpected error") + } + if len(rts) != 1 { + t.Fatal("unexpected number of roundtrips") + } +} + +func TestExploreFailure(t *testing.T) { + u, err := url.Parse("https://example.example") + runtimex.PanicOnError(err, "url.Parse failed") + rts, err := explorer.Explore(u, nil) + if err == nil { + t.Fatal("expected an error here") + } + if rts != nil { + t.Fatal("rts should be nil") + } +} + +func TestExploreSuccessWithH3(t *testing.T) { + // TODO(bassosimone): figure out why this happens. + t.Skip("this test does not work in GHA") + u, err := url.Parse("https://www.google.com") + runtimex.PanicOnError(err, "url.Parse failed") + rts, err := explorer.Explore(u, nil) + if err != nil { + t.Fatal("unexpected error") + } + if len(rts) != 2 { + t.Fatal("unexpected number of roundtrips") + } + if rts[0].Proto != "https" { + t.Fatal("unexpected protocol") + } + if rts[1].Proto != "h3" { + t.Fatal("unexpected protocol") + } +} + +func TestGetSuccess(t *testing.T) { + u, err := url.Parse("https://example.com") + resp, err := explorer.get(u, nil) + if err != nil { + t.Fatal("unexpected error") + } + if resp == nil { + t.Fatal("unexpected nil response") + } + buf := make([]byte, 100) + if n, _ := resp.Body.Read(buf); n != 0 { + t.Fatal("expected response body tom be closed") + } + +} + +func TestGetFailure(t *testing.T) { + u, err := url.Parse("https://example.example") + resp, err := explorer.get(u, nil) + if err == nil { + t.Fatal("expected an error here") + } + if resp != nil { + t.Fatal("response should be nil") + } +} + +func TestGetH3Success(t *testing.T) { + u, err := url.Parse("https://www.google.com") + h3u := &h3URL{URL: u, proto: "h3"} + resp, err := explorer.getH3(h3u, nil) + if err != nil { + t.Fatal("unexpected error") + } + if resp == nil { + t.Fatal("unexpected nil response") + } + buf := make([]byte, 100) + if n, _ := resp.Body.Read(buf); n != 0 { + t.Fatal("expected response body tom be closed") + } + +} + +func TestGetH3Failure(t *testing.T) { + u, err := url.Parse("https://www.google.google") + h3u := &h3URL{URL: u, proto: "h3"} + resp, err := explorer.getH3(h3u, nil) + if err == nil { + t.Fatal("expected an error here") + } + if resp != nil { + t.Fatal("response should be nil") + } +} + +func TestRearrange(t *testing.T) { + u, err := url.Parse("https://example.com") + runtimex.PanicOnError(err, "url.Parse failed") + resp := &http.Response{ + // the ProtoMajor field identifies the request/response structs and indicates the correct order + ProtoMajor: 2, + Request: &http.Request{ + ProtoMajor: 2, + URL: u, + Response: &http.Response{ + ProtoMajor: 1, + Request: &http.Request{ + ProtoMajor: 1, + URL: u, + Response: &http.Response{ + ProtoMajor: 0, + Request: &http.Request{ + ProtoMajor: 0, + URL: u, + }, + }, + }, + }, + }, + } + h3URL := &h3URL{URL: u, proto: "expected"} + rts := explorer.rearrange(resp, h3URL) + expectedIndex := 0 + for _, rt := range rts { + if rt.Request == nil || rt.Response == nil { + t.Fatal("unexpected nil value") + } + if rt.Request.ProtoMajor != expectedIndex { + t.Fatal("unexpected order") + } + if rt.Response.ProtoMajor != expectedIndex { + t.Fatal("unexpected order") + } + if rt.Proto != h3URL.proto { + t.Fatal("unexpected protocol") + } + expectedIndex += 1 + } +} diff --git a/internal/cmd/oohelperd/internal/nwcth/factory.go b/internal/cmd/oohelperd/internal/nwcth/factory.go new file mode 100644 index 0000000..dd0a7f7 --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/factory.go @@ -0,0 +1,96 @@ +package nwcth + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + "sync" + + "github.com/apex/log" + "github.com/lucas-clemente/quic-go" + "github.com/lucas-clemente/quic-go/http3" + "github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer" + "github.com/ooni/probe-cli/v3/internal/errorsx" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +var ErrNoConnReuse = errors.New("cannot reuse connection") + +// NewDialerResolver contructs a new dialer for TCP connections, +// with default, errorwrapping and resolve functionalities +func NewDialerResolver(resolver netxlite.Resolver) netxlite.Dialer { + var d netxlite.Dialer = netxlite.DefaultDialer + d = &errorsx.ErrorWrapperDialer{Dialer: d} + d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d} + return d +} + +// NewQUICDialerResolver creates a new QUICDialerResolver +// with default, errorwrapping and resolve functionalities +func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDialer { + var ql quicdialer.QUICListener = &netxlite.QUICListenerStdlib{} + ql = &errorsx.ErrorWrapperQUICListener{QUICListener: ql} + var dialer netxlite.QUICContextDialer = &netxlite.QUICDialerQUICGo{ + QUICListener: ql, + } + dialer = &errorsx.ErrorWrapperQUICDialer{Dialer: dialer} + dialer = &netxlite.QUICDialerResolver{Resolver: resolver, Dialer: dialer} + return dialer +} + +// NewSingleH3Transport creates an http3.RoundTripper +func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper { + transport := &http3.RoundTripper{ + DisableCompression: true, + TLSClientConfig: tlscfg, + QuicConfig: qcfg, + Dial: (&SingleDialerH3{qsess: &qsess}).Dial, + } + return transport +} + +// NewSingleTransport determines the appropriate HTTP Transport from the ALPN +func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) { + singledialer := &SingleDialer{conn: &conn} + transport = http.DefaultTransport.(*http.Transport).Clone() + transport.(*http.Transport).DialContext = singledialer.DialContext + transport.(*http.Transport).DialTLSContext = singledialer.DialContext + transport.(*http.Transport).DisableCompression = true + transport.(*http.Transport).MaxConnsPerHost = 1 + transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)} + return transport +} + +type SingleDialer struct { + sync.Mutex + conn *net.Conn +} + +func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { + s.Lock() + defer s.Unlock() + if s.conn == nil { + return nil, ErrNoConnReuse + } + c := s.conn + s.conn = nil + return *c, nil +} + +type SingleDialerH3 struct { + sync.Mutex + qsess *quic.EarlySession +} + +func (s *SingleDialerH3) Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) { + s.Lock() + defer s.Unlock() + if s.qsess == nil { + return nil, ErrNoConnReuse + } + qs := s.qsess + s.qsess = nil + return *qs, nil +} diff --git a/internal/cmd/oohelperd/internal/nwcth/generate.go b/internal/cmd/oohelperd/internal/nwcth/generate.go new file mode 100644 index 0000000..3743ccd --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/generate.go @@ -0,0 +1,287 @@ +package nwcth + +import ( + "context" + "crypto/tls" + "net" + "net/http" + + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Generate is the third step of the algorithm. Given the +// observed round trips, we generate measurement targets and +// execute those measurements so the probe has a benchmark. + +// Generator is the interface responsible for running Generate. +type Generator interface { + Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) +} + +// DefaultGenerator is the default Generator. +type DefaultGenerator struct { + dialer netxlite.Dialer + quicDialer netxlite.QUICContextDialer + resolver netxlite.Resolver + transport http.RoundTripper +} + +// the testhelper uses the same network operations as websteps +var ( + DNSDo = websteps.DNSDo + TCPDo = websteps.TCPDo + QUICDo = websteps.QUICDo + TLSDo = websteps.TLSDo + HTTPDo = websteps.HTTPDo +) + +// Generate takes in input a list of round trips and outputs +// a list of connectivity measurements for each of them. +func (g *DefaultGenerator) Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) { + var out []*URLMeasurement + for _, rt := range rts { + currentURL := g.GenerateURL(ctx, rt, clientResolutions) + out = append(out, currentURL) + } + return out, nil +} + +// GenerateURL returns a URLMeasurement. +func (g *DefaultGenerator) GenerateURL(ctx context.Context, rt *RoundTrip, clientResolutions []string) *URLMeasurement { + addrs, err := DNSDo(ctx, websteps.DNSConfig{ + Domain: rt.Request.URL.Hostname(), + Resolver: g.resolver, + }) + currentURL := &URLMeasurement{ + DNS: &DNSMeasurement{ + Domain: rt.Request.URL.Hostname(), + Addrs: addrs, + Failure: newfailure(err), + }, + RoundTrip: rt, + URL: rt.Request.URL.String(), + } + addrs = g.mergeAddresses(addrs, clientResolutions) + if len(addrs) == 0 { + return currentURL + } + for _, addr := range addrs { + var port string + explicitPort := rt.Request.URL.Port() + scheme := rt.Request.URL.Scheme + switch { + case explicitPort != "": + port = explicitPort + case scheme == "http": + port = "80" + case scheme == "https": + port = "443" + default: + panic("should not happen") + } + endpoint := net.JoinHostPort(addr, port) + var currentEndpoint *EndpointMeasurement + _, h3 := websteps.SupportedQUICVersions[rt.Proto] + switch { + case h3: + currentEndpoint = g.GenerateH3Endpoint(ctx, rt, endpoint) + case rt.Proto == "http": + currentEndpoint = g.GenerateHTTPEndpoint(ctx, rt, endpoint) + case rt.Proto == "https": + currentEndpoint = g.GenerateHTTPSEndpoint(ctx, rt, endpoint) + default: + // TODO(kelmenhorst): do we have to register this error somewhere in the result struct? + continue + } + currentURL.Endpoints = append(currentURL.Endpoints, currentEndpoint) + } + return currentURL +} + +// GenerateHTTPEndpoint performs an HTTP Request by +// a) establishing a TCP connection to the target (TCPDo), +// b) performing an HTTP GET request to the endpoint (HTTPDo). +// It returns an EndpointMeasurement. +func (g *DefaultGenerator) GenerateHTTPEndpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement { + currentEndpoint := &EndpointMeasurement{ + Endpoint: endpoint, + Protocol: "http", + } + tcpConn, err := TCPDo(ctx, websteps.TCPConfig{ + Dialer: g.dialer, + Endpoint: endpoint, + Resolver: g.resolver, + }) + currentEndpoint.TCPConnectMeasurement = &TCPConnectMeasurement{ + Failure: newfailure(err), + } + if err != nil { + return currentEndpoint + } + defer tcpConn.Close() + + // prepare HTTPRoundTripMeasurement of this endpoint + currentEndpoint.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{ + Request: &HTTPRequestMeasurement{ + Headers: rt.Request.Header, + Method: "GET", + URL: rt.Request.URL.String(), + }, + } + transport := NewSingleTransport(tcpConn) + if g.transport != nil { + transport = g.transport + } + resp, body, err := HTTPDo(rt.Request, transport) + if err != nil { + // failed Response + currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + Failure: newfailure(err), + } + return currentEndpoint + } + // successful Response + currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + BodyLength: int64(len(body)), + Failure: nil, + Headers: resp.Header, + StatusCode: int64(resp.StatusCode), + } + return currentEndpoint +} + +// GenerateHTTPSEndpoint performs an HTTPS Request by +// a) establishing a TCP connection to the target (TCPDo), +// b) establishing a TLS connection to the target (TLSDo), +// c) performing an HTTP GET request to the endpoint (HTTPDo). +// It returns an EndpointMeasurement. +func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement { + currentEndpoint := &EndpointMeasurement{ + Endpoint: endpoint, + Protocol: "https", + } + var tcpConn, tlsConn net.Conn + tcpConn, err := TCPDo(ctx, websteps.TCPConfig{ + Dialer: g.dialer, + Endpoint: endpoint, + Resolver: g.resolver, + }) + currentEndpoint.TCPConnectMeasurement = &TCPConnectMeasurement{ + Failure: newfailure(err), + } + if err != nil { + return currentEndpoint + } + defer tcpConn.Close() + + tlsConn, err = TLSDo(tcpConn, rt.Request.URL.Hostname()) + currentEndpoint.TLSHandshakeMeasurement = &TLSHandshakeMeasurement{ + Failure: newfailure(err), + } + if err != nil { + return currentEndpoint + } + defer tlsConn.Close() + + // prepare HTTPRoundTripMeasurement of this endpoint + currentEndpoint.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{ + Request: &HTTPRequestMeasurement{ + Headers: rt.Request.Header, + Method: "GET", + URL: rt.Request.URL.String(), + }, + } + transport := NewSingleTransport(tlsConn) + if g.transport != nil { + transport = g.transport + } + resp, body, err := HTTPDo(rt.Request, transport) + if err != nil { + // failed Response + currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + Failure: newfailure(err), + } + return currentEndpoint + } + // successful Response + currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + BodyLength: int64(len(body)), + Failure: nil, + Headers: resp.Header, + StatusCode: int64(resp.StatusCode), + } + return currentEndpoint +} + +// GenerateH3Endpoint performs an HTTP/3 Request by +// a) establishing a QUIC connection to the target (QUICDo), +// b) performing an HTTP GET request to the endpoint (HTTPDo). +// It returns an EndpointMeasurement. +func (g *DefaultGenerator) GenerateH3Endpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement { + currentEndpoint := &EndpointMeasurement{ + Endpoint: endpoint, + Protocol: rt.Proto, + } + tlsConf := &tls.Config{ + ServerName: rt.Request.URL.Hostname(), + NextProtos: []string{rt.Proto}, + } + sess, err := QUICDo(ctx, websteps.QUICConfig{ + Endpoint: endpoint, + QUICDialer: g.quicDialer, + TLSConf: tlsConf, + Resolver: g.resolver, + }) + currentEndpoint.QUICHandshakeMeasurement = &TLSHandshakeMeasurement{ + Failure: newfailure(err), + } + if err != nil { + return currentEndpoint + } + // prepare HTTPRoundTripMeasurement of this endpoint + currentEndpoint.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{ + Request: &HTTPRequestMeasurement{ + Headers: rt.Request.Header, + Method: "GET", + URL: rt.Request.URL.String(), + }, + } + var transport http.RoundTripper = NewSingleH3Transport(sess, tlsConf, &quic.Config{}) + if g.transport != nil { + transport = g.transport + } + resp, body, err := HTTPDo(rt.Request, transport) + if err != nil { + // failed Response + currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + Failure: newfailure(err), + } + return currentEndpoint + } + // successful Response + currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + BodyLength: int64(len(body)), + Failure: nil, + Headers: resp.Header, + StatusCode: int64(resp.StatusCode), + } + return currentEndpoint +} + +// mergeAddresses creates a (duplicate-free) union set of the IP addresses provided by the client, +// and the addresses resulting from the testhelper's DNS step +func (g *DefaultGenerator) mergeAddresses(addrs []string, clientAddrs []string) (out []string) { + unique := make(map[string]bool, len(addrs)+len(clientAddrs)) + for _, a := range addrs { + unique[a] = true + } + for _, a := range clientAddrs { + unique[a] = true + } + for key := range unique { + out = append(out, key) + } + return out +} diff --git a/internal/cmd/oohelperd/internal/nwcth/generate_test.go b/internal/cmd/oohelperd/internal/nwcth/generate_test.go new file mode 100644 index 0000000..d38d42c --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/generate_test.go @@ -0,0 +1,461 @@ +package nwcth + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/errorsx" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +var generator = &DefaultGenerator{resolver: newResolver()} + +type fakeTransport struct { + err error +} + +func (txp fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return nil, txp.err +} +func (txp fakeTransport) CloseIdleConnections() {} + +type fakeQUICDialer struct { + err error +} + +func (d fakeQUICDialer) DialContext(ctx context.Context, network, address string, + tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlySession, error) { + return nil, d.err +} + +type fakeDialer struct { + err error +} + +func (d fakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + return nil, d.err +} + +func TestGenerateDNSFailure(t *testing.T) { + u, err := url.Parse("https://www.google.google") + runtimex.PanicOnError(err, "url.Parse failed") + rts := []*RoundTrip{ + { + Proto: "https", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + }, + } + urlMeasurements, err := generator.Generate(context.Background(), rts, []string{}) + if err != nil { + t.Fatal("unexpected error") + } + if len(urlMeasurements) != 1 { + t.Fatal("unexpected urlMeasurements length") + } + if urlMeasurements[0].DNS == nil { + t.Fatal("DNS should not be nil") + } + if urlMeasurements[0].DNS.Failure == nil || *urlMeasurements[0].DNS.Failure != errorsx.FailureDNSNXDOMAINError { + t.Fatal("unexpected DNS failure type") + } +} + +func TestGenerate(t *testing.T) { + u, err := url.Parse("http://www.google.com") + u2, err := url.Parse("https://www.google.com") + runtimex.PanicOnError(err, "url.Parse failed") + rts := []*RoundTrip{ + { + Proto: "http", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + }, + { + Proto: "https", + Request: &http.Request{ + URL: u2, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + }, + { + Proto: "h3", + Request: &http.Request{ + URL: u2, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + }, + } + urlMeasurements, err := generator.Generate(context.Background(), rts, []string{}) + if err != nil { + t.Fatal("unexpected err") + } + if urlMeasurements == nil { + t.Fatal("unexpected nil urlMeasurement") + } + if len(urlMeasurements) < 3 { + t.Fatal("unexpected number of urlMeasurements", len(urlMeasurements)) + } +} + +func TestGenerateUnexpectedProtocol(t *testing.T) { + u, err := url.Parse("https://www.google.com") + runtimex.PanicOnError(err, "url.Parse failed") + rts := []*RoundTrip{ + { + Proto: "h3-27", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + }, + } + urlMeasurements, err := generator.Generate(context.Background(), rts, []string{}) + if err != nil { + t.Fatal("unexpected err") + } + if urlMeasurements == nil { + t.Fatal("unexpected nil urlMeasurement") + } + if len(urlMeasurements) != 1 { + t.Fatal("unexpected number of urlMeasurements") + } + measurement := urlMeasurements[0] + if measurement.URL != u.String() { + t.Fatal("unexpected URL") + } + if measurement.DNS == nil { + t.Fatal("DNS should not be nil") + } + if measurement.RoundTrip == nil { + t.Fatal("RoundTrip should not be nil") + } + if measurement.Endpoints != nil { + t.Fatal("Endpoints should be nil") + } +} + +func TestGenerateURLWithClientResolutions(t *testing.T) { + u, err := url.Parse("https://www.google.com") + runtimex.PanicOnError(err, "url.Parse failed") + rt := &RoundTrip{ + Proto: "h3", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + } + clientResolution := "142.250.186.36" + urlMeasurement := generator.GenerateURL(context.Background(), rt, []string{clientResolution}) + if err != nil { + t.Fatal("unexpected err") + } + if urlMeasurement == nil { + t.Fatal("unexpected nil urlMeasurement") + } + if urlMeasurement.DNS == nil { + t.Fatal("DNS should not be nil") + } + if len(urlMeasurement.Endpoints) < 2 { + t.Fatal("unexpected number of endpoints") + } + clientAddrsFound := false + for _, e := range urlMeasurement.Endpoints { + if e.Endpoint == clientResolution+":443" { + clientAddrsFound = true + } + } + if !clientAddrsFound { + t.Fatal("did not use provided client resolution") + } +} + +func TestGenerateHTTP(t *testing.T) { + u, err := url.Parse("http://example.com") + runtimex.PanicOnError(err, "url.Parse failed") + rt := &RoundTrip{ + Proto: "http", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + } + endpointMeasurement := generator.GenerateHTTPEndpoint(context.Background(), rt, "93.184.216.34:80") + if err != nil { + t.Fatal("unexpected err") + } + if endpointMeasurement == nil { + t.Fatal("unexpected nil urlMeasurement") + } + if endpointMeasurement.TCPConnectMeasurement == nil { + t.Fatal("TCPConnectMeasurement should not be nil") + } + if endpointMeasurement.HTTPRoundTripMeasurement == nil { + t.Fatal("HTTPRoundTripMeasurement should not be nil") + } +} + +func TestGenerateHTTPS(t *testing.T) { + u, err := url.Parse("https://example.com") + runtimex.PanicOnError(err, "url.Parse failed") + rt := &RoundTrip{ + Proto: "https", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + } + endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "93.184.216.34:443") + if err != nil { + t.Fatal("unexpected err") + } + if endpointMeasurement == nil { + t.Fatal("unexpected nil urlMeasurement") + } + if endpointMeasurement.TCPConnectMeasurement == nil { + t.Fatal("TCPConnectMeasurement should not be nil") + } + if endpointMeasurement.TLSHandshakeMeasurement == nil { + t.Fatal("TCPConnectMeasurement should not be nil") + } + if endpointMeasurement.HTTPRoundTripMeasurement == nil { + t.Fatal("HTTPRoundTripMeasurement should not be nil") + } +} + +func TestGenerateHTTPSTLSFailure(t *testing.T) { + u, err := url.Parse("https://wrong.host.badssl.com/") + runtimex.PanicOnError(err, "url.Parse failed") + rt := &RoundTrip{ + Proto: "https", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + } + endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "104.154.89.105:443") + if err != nil { + t.Fatal("unexpected err") + } + if endpointMeasurement == nil { + t.Fatal("unexpected nil urlMeasurement") + } + if endpointMeasurement.TCPConnectMeasurement == nil { + t.Fatal("TCPConnectMeasurement should not be nil") + } + if endpointMeasurement.TLSHandshakeMeasurement == nil { + t.Fatal("TCPConnectMeasurement should not be nil") + } + if endpointMeasurement.HTTPRoundTripMeasurement != nil { + t.Fatal("HTTPRoundTripMeasurement should be nil") + } +} + +func TestGenerateH3(t *testing.T) { + u, err := url.Parse("https://www.google.com") + runtimex.PanicOnError(err, "url.Parse failed") + rt := &RoundTrip{ + Proto: "h3", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + } + endpointMeasurement := generator.GenerateH3Endpoint(context.Background(), rt, "173.194.76.103:443") + if err != nil { + t.Fatal("unexpected err") + } + if endpointMeasurement == nil { + t.Fatal("unexpected nil urlMeasurement") + } + if endpointMeasurement.QUICHandshakeMeasurement == nil { + t.Fatal("TCPConnectMeasurement should not be nil") + } + if endpointMeasurement.HTTPRoundTripMeasurement == nil { + t.Fatal("HTTPRoundTripMeasurement should not be nil") + } +} + +func TestGenerateTCPDoFails(t *testing.T) { + expected := errors.New("expected") + generator := &DefaultGenerator{ + dialer: fakeDialer{err: expected}, + resolver: newResolver(), + } + u, err := url.Parse("https://www.google.com") + runtimex.PanicOnError(err, "url.Parse failed") + rt := &RoundTrip{ + Proto: "https", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + } + endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "173.194.76.103:443") + if err != nil { + t.Fatal("unexpected err") + } + if endpointMeasurement.TCPConnectMeasurement == nil { + t.Fatal("QUIC handshake should not be nil") + } + if endpointMeasurement.TCPConnectMeasurement.Failure == nil { + t.Fatal("expected an error here") + } + if *endpointMeasurement.TCPConnectMeasurement.Failure != *newfailure(expected) { + t.Fatal("unexpected error type") + } +} + +func TestGenerateQUICDoFails(t *testing.T) { + expected := errors.New("expected") + generator := &DefaultGenerator{ + quicDialer: fakeQUICDialer{err: expected}, + resolver: newResolver(), + } + u, err := url.Parse("https://www.google.com") + runtimex.PanicOnError(err, "url.Parse failed") + rt := &RoundTrip{ + Proto: "h3", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + } + endpointMeasurement := generator.GenerateH3Endpoint(context.Background(), rt, "173.194.76.103:443") + if err != nil { + t.Fatal("unexpected err") + } + if endpointMeasurement.QUICHandshakeMeasurement == nil { + t.Fatal("QUIC handshake should not be nil") + } + if endpointMeasurement.QUICHandshakeMeasurement.Failure == nil { + t.Fatal("expected an error here") + } + if *endpointMeasurement.QUICHandshakeMeasurement.Failure != *newfailure(expected) { + t.Fatal("unexpected error type") + } +} + +func TestGenerateHTTPDoFails(t *testing.T) { + expected := errors.New("expected") + generator := &DefaultGenerator{ + transport: fakeTransport{err: expected}, + resolver: newResolver(), + } + u, err := url.Parse("http://www.google.com") + u2, err := url.Parse("https://www.google.com") + runtimex.PanicOnError(err, "url.Parse failed") + rts := []*RoundTrip{ + { + Proto: "http", + Request: &http.Request{ + URL: u, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + }, + { + Proto: "https", + Request: &http.Request{ + URL: u2, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + }, + { + Proto: "h3", + Request: &http.Request{ + URL: u2, + }, + Response: &http.Response{ + StatusCode: 200, + }, + SortIndex: 0, + }, + } + urlMeasurements, err := generator.Generate(context.Background(), rts, []string{}) + if err != nil { + t.Fatal("unexpected err") + } + if len(urlMeasurements) != 3 { + t.Fatal("unexpected number of urlMeasurements") + } + for _, u := range urlMeasurements { + if u.DNS == nil { + t.Fatal("unexpected DNS failure") + } + if len(u.Endpoints) < 1 { + t.Fatal("unexpected number of endpoints", len(u.Endpoints)) + } + // this can occur when the network is unreachable, but it is irrelevant for checking HTTP behavior + if u.Endpoints[0].TCPConnectMeasurement != nil && u.Endpoints[0].TCPConnectMeasurement.Failure != nil { + continue + } + if u.Endpoints[0].QUICHandshakeMeasurement != nil && u.Endpoints[0].QUICHandshakeMeasurement.Failure != nil { + continue + } + if u.Endpoints[0].HTTPRoundTripMeasurement == nil { + t.Fatal("roundtrip should not be nil", u.Endpoints[0].TCPConnectMeasurement.Failure, "jaaaa") + } + if u.Endpoints[0].HTTPRoundTripMeasurement.Response == nil { + t.Fatal("roundtrip response should not be nil") + } + if u.Endpoints[0].HTTPRoundTripMeasurement.Response.Failure == nil { + t.Fatal("expected an HTTP error") + } + if !strings.HasSuffix(*u.Endpoints[0].HTTPRoundTripMeasurement.Response.Failure, expected.Error()) { + t.Fatal("unexpected failure type") + } + } +} diff --git a/internal/cmd/oohelperd/internal/nwcth/h3.go b/internal/cmd/oohelperd/internal/nwcth/h3.go new file mode 100644 index 0000000..c1bd2c7 --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/h3.go @@ -0,0 +1,72 @@ +package nwcth + +import ( + "errors" + "net" + "net/http" + "net/url" + "strings" + + "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +type h3URL struct { + URL *url.URL + proto string +} + +type altSvcH3 struct { + authority string + proto string +} + +var ErrNoH3Location = errors.New("no h3 location found") + +// getH3URL returns the URL for HTTP/3 requests, if the target supports HTTP/3. +// Returns nil, if no HTTP/3 support is advertised. +func getH3URL(resp *http.Response) (*h3URL, error) { + URL := resp.Request.URL + if URL == nil { + return nil, ErrInvalidURL + } + h3Svc, err := parseAltSvc(resp, URL) + if err != nil { + return nil, err + } + quicURL, err := url.Parse(URL.String()) + runtimex.PanicOnError(err, "url.Parse failed") + quicURL.Host = h3Svc.authority + return &h3URL{URL: quicURL, proto: h3Svc.proto}, nil +} + +// parseAltSvc parses the Alt-Svc HTTP header for entries advertising the use of H3 +func parseAltSvc(resp *http.Response, URL *url.URL) (*altSvcH3, error) { + // TODO(bassosimone,kelmenhorst): see if we can make this algorithm more robust. + if URL.Scheme != "https" { + return nil, ErrUnsupportedScheme + } + alt_svc := resp.Header.Get("Alt-Svc") + // syntax: Alt-Svc: =; ma=; persist=1 + entries := strings.Split(alt_svc, ",") + for _, e := range entries { + keyvalpairs := strings.Split(e, ";") + for _, p := range keyvalpairs { + p = strings.Replace(p, "\"", "", -1) + kv := strings.Split(p, "=") + if len(kv) != 2 { + continue + } + if _, ok := websteps.SupportedQUICVersions[kv[0]]; ok { + host, port, err := net.SplitHostPort(kv[1]) + runtimex.PanicOnError(err, "net.SplitHostPort failed") + if host == "" { + host = URL.Hostname() + } + authority := net.JoinHostPort(host, port) + return &altSvcH3{authority: authority, proto: kv[0]}, nil + } + } + } + return nil, ErrNoH3Location +} diff --git a/internal/cmd/oohelperd/internal/nwcth/h3_test.go b/internal/cmd/oohelperd/internal/nwcth/h3_test.go new file mode 100644 index 0000000..c028ecb --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/h3_test.go @@ -0,0 +1,39 @@ +package nwcth + +import ( + "net/http" + "net/url" + "testing" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func TestGetH3URLInvalidURL(t *testing.T) { + resp := &http.Response{ + Request: &http.Request{}, + } + h3URL, err := getH3URL(resp) + if err == nil { + t.Fatal("expected an error here") + } + if h3URL != nil { + t.Fatal("h3URL should be nil") + } + +} + +func TestParseUnsupportedScheme(t *testing.T) { + URL, err := url.Parse("h3://google.com") + runtimex.PanicOnError(err, "url.Parse failed") + parsed, err := parseAltSvc(nil, URL) + if err == nil { + t.Fatal("expected an error here") + } + if err != ErrUnsupportedScheme { + t.Fatal("unexpected error type") + } + if parsed != nil { + t.Fatal("h3URL should be nil") + } + +} diff --git a/internal/cmd/oohelperd/internal/nwcth/initialchecks.go b/internal/cmd/oohelperd/internal/nwcth/initialchecks.go new file mode 100644 index 0000000..7a312f1 --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/initialchecks.go @@ -0,0 +1,61 @@ +package nwcth + +import ( + "context" + "errors" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// InitialChecks is the first step of the test helper algorithm. We +// make sure we can parse the URL, we handle the scheme, and the domain +// name inside the URL's authority is valid. + +// Errors returned by Preresolve. +var ( + // ErrInvalidURL indicates that the URL is invalid. + ErrInvalidURL = errors.New("the URL is invalid") + + // ErrUnsupportedScheme indicates that we don't support the scheme. + ErrUnsupportedScheme = errors.New("unsupported scheme") + + // ErrNoSuchHost indicates that the DNS resolution failed. + ErrNoSuchHost = errors.New("no such host") +) + +// InitChecker is the interface responsible for running InitialChecks. +type InitChecker interface { + InitialChecks(URL string) (*url.URL, error) +} + +// DefaultInitChecker is the default InitChecker. +type DefaultInitChecker struct { + resolver netxlite.Resolver +} + +// InitialChecks checks whether the URL is valid and whether the +// domain inside the URL is an existing one. If these preliminary +// checks fail, there's no point in continuing. +// If they succeed, InitialChecks returns the URL, if not an error. +func (i *DefaultInitChecker) InitialChecks(URL string) (*url.URL, error) { + parsed, err := url.Parse(URL) + if err != nil { + return nil, ErrInvalidURL + } + switch parsed.Scheme { + case "http", "https": + default: + return nil, ErrUnsupportedScheme + } + // Assumptions: + // + // 1. the resolver will cache the resolution for later + // + // 2. an IP address does not cause an error because we are using + // a resolve that behaves like getaddrinfo + if _, err := i.resolver.LookupHost(context.Background(), parsed.Hostname()); err != nil { + return nil, ErrNoSuchHost + } + return parsed, nil +} diff --git a/internal/cmd/oohelperd/internal/nwcth/initialchecks_test.go b/internal/cmd/oohelperd/internal/nwcth/initialchecks_test.go new file mode 100644 index 0000000..fe2545d --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/initialchecks_test.go @@ -0,0 +1,31 @@ +package nwcth + +import ( + "testing" +) + +var checker = &DefaultInitChecker{resolver: newResolver()} + +func TestMeasureWithInvalidURL(t *testing.T) { + _, err := checker.InitialChecks("http://[::1]aaaa") + + if err == nil || err != ErrInvalidURL { + t.Fatal("expected an error here") + } +} + +func TestMeasureWithUnsupportedScheme(t *testing.T) { + _, err := checker.InitialChecks("abc://example.com") + + if err == nil || err != ErrUnsupportedScheme { + t.Fatal("expected an error here") + } +} + +func TestMeasureWithInvalidHost(t *testing.T) { + _, err := checker.InitialChecks("http://www.ooni.ooni") + + if err == nil || err != ErrNoSuchHost { + t.Fatal("expected an error here") + } +} diff --git a/internal/cmd/oohelperd/internal/nwcth/measure.go b/internal/cmd/oohelperd/internal/nwcth/measure.go new file mode 100644 index 0000000..73e2316 --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/measure.go @@ -0,0 +1,96 @@ +package nwcth + +import ( + "context" + "errors" + "net/url" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" + "github.com/ooni/probe-cli/v3/internal/engine/netx" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +type ( + CtrlRequest = websteps.CtrlRequest + ControlResponse = websteps.ControlResponse +) + +var ErrInternalServer = errors.New("Internal server failure") + +// Config contains the building blocks of the testhelper algorithm +type Config struct { + checker InitChecker + explorer Explorer + generator Generator + resolver netxlite.Resolver +} + +// Measure performs the three consecutive steps of the testhelper algorithm: +// 1. InitialChecks +// 2. Explore +// 3. Generate +func Measure(ctx context.Context, creq *CtrlRequest, config *Config) (*ControlResponse, error) { + var ( + URL *url.URL + err error + ) + resolver := config.resolver + if resolver == nil { + // use a central resolver + resolver = newResolver() + } + checker := config.checker + if checker == nil { + checker = &DefaultInitChecker{resolver: resolver} + } + URL, err = checker.InitialChecks(creq.HTTPRequest) + if err != nil { + // return a valid response in case of NXDOMAIN so the probe can compare the failure + if err == ErrNoSuchHost { + return newDNSFailedResponse(err, creq.HTTPRequest), nil + } + return nil, err + } + explorer := config.explorer + if explorer == nil { + explorer = &DefaultExplorer{resolver: resolver} + } + rts, err := explorer.Explore(URL, creq.HTTPRequestHeaders) + if err != nil { + return nil, ErrInternalServer + } + generator := config.generator + if generator == nil { + generator = &DefaultGenerator{resolver: resolver} + } + meas, err := generator.Generate(ctx, rts, creq.Addrs) + if err != nil { + return nil, err + } + return &ControlResponse{URLMeasurements: meas}, nil +} + +// newDNSFailedResponse creates a new response with one URLMeasurement entry +// indicating that the DNS step failed +func newDNSFailedResponse(err error, URL string) *ControlResponse { + resp := &ControlResponse{} + m := &URLMeasurement{ + URL: URL, + DNS: &DNSMeasurement{ + Failure: newfailure(err), + }, + } + resp.URLMeasurements = append(resp.URLMeasurements, m) + return resp +} + +// newResolver creates a new DNS resolver instance +func newResolver() netxlite.Resolver { + childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google") + runtimex.PanicOnError(err, "NewDNSClient failed") + var r netxlite.Resolver = childResolver + r = &netxlite.IDNAResolver{Resolver: r} + return r +} diff --git a/internal/cmd/oohelperd/internal/nwcth/measure_test.go b/internal/cmd/oohelperd/internal/nwcth/measure_test.go new file mode 100644 index 0000000..ce70f88 --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/measure_test.go @@ -0,0 +1,117 @@ +package nwcth + +import ( + "context" + "errors" + "net/url" + "testing" + + "github.com/ooni/probe-cli/v3/internal/errorsx" +) + +func TestMeasureSuccess(t *testing.T) { + req := &CtrlRequest{ + HTTPRequest: "https://example.com", + } + resp, err := Measure(context.Background(), req, &Config{}) + if err != nil { + t.Fatal("unexpected error") + } + if resp == nil { + t.Fatal("unexpected nil response") + } +} + +type MockChecker struct { + err error +} + +func (c *MockChecker) InitialChecks(URL string) (*url.URL, error) { + return nil, c.err +} + +type MockExplorer struct{} + +func (c *MockExplorer) Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) { + return nil, ErrExpectedExplore +} + +type MockGenerator struct{} + +func (c *MockGenerator) Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) { + return nil, ErrExpectedGenerate +} + +var ErrExpectedCheck error = errors.New("expected error checker") +var ErrExpectedExplore error = errors.New("expected error explorer") +var ErrExpectedGenerate error = errors.New("expected error generator") + +func TestMeasureInitialChecksFail(t *testing.T) { + req := &CtrlRequest{ + HTTPRequest: "https://example.com", + } + resp, err := Measure(context.Background(), req, &Config{checker: &MockChecker{err: ErrExpectedCheck}}) + if err == nil { + t.Fatal("expected an error here") + } + if err != ErrExpectedCheck { + t.Fatal("unexpected error type") + } + if resp != nil { + t.Fatal("resp should be nil") + } +} + +func TestMeasureInitialChecksFailWithNXDOMAIN(t *testing.T) { + req := &CtrlRequest{ + HTTPRequest: "https://example.com", + } + resp, err := Measure(context.Background(), req, &Config{checker: &MockChecker{err: ErrNoSuchHost}}) + if err != nil { + t.Fatal("unexpected error") + } + if resp == nil { + t.Fatal("resp should not be nil") + } + if len(resp.URLMeasurements) != 1 { + t.Fatal("unexpected number of measurements") + } + if resp.URLMeasurements[0].DNS == nil { + t.Fatal("DNS entry should not be nil") + } + if *resp.URLMeasurements[0].DNS.Failure != errorsx.FailureDNSNXDOMAINError { + t.Fatal("unexpected failure") + } +} + +func TestMeasureExploreFails(t *testing.T) { + req := &CtrlRequest{ + HTTPRequest: "https://example.com", + } + resp, err := Measure(context.Background(), req, &Config{explorer: &MockExplorer{}}) + if err == nil { + t.Fatal("expected an error here") + } + if err != ErrInternalServer { + t.Fatal("unexpected error type") + } + if resp != nil { + t.Fatal("resp should be nil") + } +} + +func TestMeasureGenerateFails(t *testing.T) { + req := &CtrlRequest{ + HTTPRequest: "https://example.com", + } + resp, err := Measure(context.Background(), req, &Config{generator: &MockGenerator{}}) + if err == nil { + t.Fatal("expected an error here") + } + if err != ErrExpectedGenerate { + t.Fatal("unexpected error type") + } + if resp != nil { + t.Fatal("resp should be nil") + } +} diff --git a/internal/cmd/oohelperd/internal/nwcth/model.go b/internal/cmd/oohelperd/internal/nwcth/model.go new file mode 100644 index 0000000..325c1ac --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/model.go @@ -0,0 +1,17 @@ +package nwcth + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" +) + +type ( + URLMeasurement = websteps.URLMeasurement + DNSMeasurement = websteps.DNSMeasurement + EndpointMeasurement = websteps.EndpointMeasurement + TCPConnectMeasurement = websteps.TCPConnectMeasurement + HTTPRoundTripMeasurement = websteps.HTTPRoundTripMeasurement + TLSHandshakeMeasurement = websteps.TLSHandshakeMeasurement + HTTPRequestMeasurement = websteps.HTTPRequestMeasurement + HTTPResponseMeasurement = websteps.HTTPResponseMeasurement + RoundTrip = websteps.RoundTrip +) diff --git a/internal/cmd/oohelperd/internal/nwcth/nwcth.go b/internal/cmd/oohelperd/internal/nwcth/nwcth.go new file mode 100644 index 0000000..427e736 --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/nwcth.go @@ -0,0 +1,67 @@ +// Package nwcth implements the new web connectivity test helper. +// +// See https://github.com/ooni/spec/blob/master/backends/th-007-nwcth.md +package nwcth + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/iox" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/version" +) + +// newfailure is a convenience shortcut to save typing +var newfailure = archival.NewFailure + +// maxAcceptableBody is _at the same time_ the maximum acceptable body for incoming +// API requests and the maximum acceptable body when fetching arbitrary URLs. See +// https://github.com/ooni/probe/issues/1727 for statistics regarding the test lists +// including the empirical CDF of the body size for test lists URLs. +const maxAcceptableBody = 1 << 24 + +// Handler implements the Web Connectivity test helper HTTP API. +type Handler struct { + Config *Config +} + +// ServeHTTP implements http.Handler.ServeHTTP. +func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + w.Header().Add("Server", fmt.Sprintf( + "oohelperd/%s ooniprobe-engine/%s", version.Version, version.Version, + )) + if req.Method != "POST" { + w.WriteHeader(400) + return + } + reader := &io.LimitedReader{R: req.Body, N: maxAcceptableBody} + data, err := iox.ReadAllContext(req.Context(), reader) + if err != nil { + w.WriteHeader(400) + return + } + var creq CtrlRequest + if err := json.Unmarshal(data, &creq); err != nil { + w.WriteHeader(400) + return + } + cresp, err := Measure(req.Context(), &creq, h.Config) + if err != nil { + if err == ErrInternalServer { + w.WriteHeader(500) + return + } + w.WriteHeader(400) + return + } + // We assume that the following call cannot fail because it's a + // clearly serializable data structure. + data, err = json.Marshal(cresp) + runtimex.PanicOnError(err, "json.Marshal failed") + w.Header().Add("Content-Type", "application/json") + w.Write(data) +} diff --git a/internal/cmd/oohelperd/internal/nwcth/nwcth_test.go b/internal/cmd/oohelperd/internal/nwcth/nwcth_test.go new file mode 100644 index 0000000..5202c1c --- /dev/null +++ b/internal/cmd/oohelperd/internal/nwcth/nwcth_test.go @@ -0,0 +1,281 @@ +package nwcth + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/ooni/probe-cli/v3/internal/iox" +) + +const requestnoredirect = `{ + "url": "https://ooni.org", + "headers": { + "Accept": [ + "*/*" + ], + "Accept-Language": [ + "en-US;q=0.8,en;q=0.5" + ], + "User-Agent": [ + "Mozilla/5.0" + ] + }, + "addrs": [ + "104.198.14.52:443" + ] +}` + +const requestredirect = `{ + "url": "https://www.ooni.org", + "headers": { + "Accept": [ + "*/*" + ], + "Accept-Language": [ + "en-US;q=0.8,en;q=0.5" + ], + "User-Agent": [ + "Mozilla/5.0" + ] + }, + "addrs": [ + "18.192.76.182:443" + ] +}` + +const requestIPaddressinput = `{ + "url": "https://172.217.168.4", + "headers": { + "Accept": [ + "*/*" + ], + "Accept-Language": [ + "en-US;q=0.8,en;q=0.5" + ], + "User-Agent": [ + "Mozilla/5.0" + ] + }, + "addrs": [ + "172.217.168.4:443" + ] +}` + +const requestwithquic = `{ + "url": "https://www.google.com", + "headers": { + "Accept": [ + "*/*" + ], + "Accept-Language": [ + "en-US;q=0.8,en;q=0.5" + ], + "User-Agent": [ + "Mozilla/5.0" + ] + }, + "addrs": [ + "142.250.74.196:443" + ] +}` + +const requestWithoutDomainName = `{ + "url": "https://8.8.8.8", + "headers": { + "Accept": [ + "*/*" + ], + "Accept-Language": [ + "en-US;q=0.8,en;q=0.5" + ], + "User-Agent": [ + "Mozilla/5.0" + ] + }, + "addrs": [ + "8.8.8.8:443" + ] +}` + +func TestWorkingAsIntended(t *testing.T) { + handler := Handler{Config: &Config{}} + srv := httptest.NewServer(handler) + defer srv.Close() + type expectationSpec struct { + name string + reqMethod string + reqContentType string + reqBody string + respStatusCode int + respContentType string + parseBody bool + } + expectations := []expectationSpec{{ + name: "check for invalid method", + reqMethod: "GET", + respStatusCode: 400, + }, { + name: "check for invalid request body", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: "{", + respStatusCode: 400, + }, { + name: "with measurement failure", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: `{"url": "http://[::1]aaaa"}`, + respStatusCode: 400, + }, { + name: "request without redirect or H3 follow-up request", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: requestnoredirect, + respStatusCode: 200, + respContentType: "application/json", + parseBody: true, + }, { + name: "request triggering one redirect, without H3 follow-up request", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: requestredirect, + respStatusCode: 200, + respContentType: "application/json", + parseBody: true, + }, { + name: "request with an IP address as input", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: requestIPaddressinput, + respStatusCode: 200, + respContentType: "application/json", + parseBody: true, + }, { + name: "request triggering H3 follow-up request, without redirect", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: requestwithquic, + respStatusCode: 200, + respContentType: "application/json", + parseBody: true, + }, { + name: "when there's no domain name in the request", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: requestWithoutDomainName, + respStatusCode: 200, + respContentType: "application/json", + parseBody: true, + }} + for _, expect := range expectations { + t.Run(expect.name, func(t *testing.T) { + body := strings.NewReader(expect.reqBody) + req, err := http.NewRequest(expect.reqMethod, srv.URL, body) + if err != nil { + t.Fatalf("%s: %+v", expect.name, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s: %+v", expect.name, err) + } + defer resp.Body.Close() + if resp.StatusCode != expect.respStatusCode { + t.Fatalf("unexpected status code: %+v", resp.StatusCode) + } + data, err := iox.ReadAllContext(context.Background(), resp.Body) + if err != nil { + t.Fatal(err) + } + if !expect.parseBody { + return + } + var v interface{} + if err := json.Unmarshal(data, &v); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestHandlerWithInternalServerError(t *testing.T) { + handler := Handler{Config: &Config{explorer: &MockExplorer{}}} + srv := httptest.NewServer(handler) + defer srv.Close() + body := strings.NewReader(`{"url": "https://example.com"}`) + req, err := http.NewRequest("POST", srv.URL, body) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 500 { + t.Fatalf("unexpected status code: %+v", resp.StatusCode) + } + _, err = iox.ReadAllContext(context.Background(), resp.Body) + if err != nil { + t.Fatal(err) + } +} + +func TestHandlerWithRequestBodyReadingError(t *testing.T) { + expected := errors.New("mocked error") + handler := Handler{Config: &Config{}} + rw := NewFakeResponseWriter() + req := &http.Request{ + Method: "POST", + Header: map[string][]string{ + "Content-Type": {"application/json"}, + "Content-Length": {"2048"}, + }, + Body: &FakeBody{Err: expected}, + } + handler.ServeHTTP(rw, req) + if rw.StatusCode != 400 { + t.Fatal("unexpected status code") + } +} + +type FakeBody struct { + Err error +} + +func (fb FakeBody) Read(p []byte) (int, error) { + time.Sleep(10 * time.Microsecond) + return 0, fb.Err +} + +func (fb FakeBody) Close() error { + return nil +} + +type FakeResponseWriter struct { + Body [][]byte + HeaderMap http.Header + StatusCode int +} + +func NewFakeResponseWriter() *FakeResponseWriter { + return &FakeResponseWriter{HeaderMap: make(http.Header)} +} + +func (frw *FakeResponseWriter) Header() http.Header { + return frw.HeaderMap +} + +func (frw *FakeResponseWriter) Write(b []byte) (int, error) { + frw.Body = append(frw.Body, b) + return len(b), nil +} + +func (frw *FakeResponseWriter) WriteHeader(statusCode int) { + frw.StatusCode = statusCode +} diff --git a/internal/cmd/oohelperd/oohelperd.go b/internal/cmd/oohelperd/oohelperd.go index 6b95ce0..7aef143 100644 --- a/internal/cmd/oohelperd/oohelperd.go +++ b/internal/cmd/oohelperd/oohelperd.go @@ -10,6 +10,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal" + "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/nwcth" "github.com/ooni/probe-cli/v3/internal/engine/netx" ) @@ -52,6 +53,7 @@ func main() { func testableMain() { mux := http.NewServeMux() + mux.Handle("/api/unstable/nwcth", nwcth.Handler{Config: &nwcth.Config{}}) mux.Handle("/", internal.Handler{ Client: httpx, Dialer: dialer, diff --git a/internal/engine/allexperiments.go b/internal/engine/allexperiments.go index 1accbcc..d587d6e 100644 --- a/internal/engine/allexperiments.go +++ b/internal/engine/allexperiments.go @@ -23,6 +23,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" ) @@ -326,6 +327,18 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{ } }, + "websteps": func(session *Session) *ExperimentBuilder { + return &ExperimentBuilder{ + build: func(config interface{}) *Experiment { + return NewExperiment(session, websteps.NewExperimentMeasurer( + *config.(*websteps.Config), + )) + }, + config: &websteps.Config{}, + inputPolicy: InputOrQueryBackend, + } + }, + "whatsapp": func(session *Session) *ExperimentBuilder { return &ExperimentBuilder{ build: func(config interface{}) *Experiment { diff --git a/internal/engine/experiment/websteps/control.go b/internal/engine/experiment/websteps/control.go new file mode 100644 index 0000000..ab9386a --- /dev/null +++ b/internal/engine/experiment/websteps/control.go @@ -0,0 +1,38 @@ +package websteps + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/engine/httpx" + "github.com/ooni/probe-cli/v3/internal/engine/model" + "github.com/ooni/probe-cli/v3/internal/errorsx" +) + +// CtrlRequest is the request sent by the probe +type CtrlRequest struct { + HTTPRequest string `json:"url"` + HTTPRequestHeaders map[string][]string `json:"headers"` + Addrs []string `json:"addrs"` +} + +// ControlResponse is the response from the control service. +type ControlResponse struct { + URLMeasurements []*URLMeasurement `json:"urls"` +} + +// Control performs the control request and returns the response. +func Control( + ctx context.Context, sess model.ExperimentSession, + thAddr string, creq CtrlRequest) (out ControlResponse, err error) { + clnt := httpx.Client{ + BaseURL: thAddr, + HTTPClient: sess.DefaultHTTPClient(), + Logger: sess.Logger(), + } + // make sure error is wrapped + err = errorsx.SafeErrWrapperBuilder{ + Error: clnt.PostJSON(ctx, "/", creq, &out), + Operation: errorsx.TopLevelOperation, + }.MaybeBuild() + return +} diff --git a/internal/engine/experiment/websteps/dns.go b/internal/engine/experiment/websteps/dns.go new file mode 100644 index 0000000..e521059 --- /dev/null +++ b/internal/engine/experiment/websteps/dns.go @@ -0,0 +1,27 @@ +package websteps + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/engine/netx" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +type DNSConfig struct { + Domain string + Resolver netxlite.Resolver +} + +// DNSDo performs the DNS check. +func DNSDo(ctx context.Context, config DNSConfig) ([]string, error) { + resolver := config.Resolver + if resolver == nil { + childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google") + runtimex.PanicOnError(err, "NewDNSClient failed") + var resolver netxlite.Resolver = childResolver + resolver = &netxlite.IDNAResolver{Resolver: resolver} + } + return config.Resolver.LookupHost(ctx, config.Domain) +} diff --git a/internal/engine/experiment/websteps/factory.go b/internal/engine/experiment/websteps/factory.go new file mode 100644 index 0000000..50693cd --- /dev/null +++ b/internal/engine/experiment/websteps/factory.go @@ -0,0 +1,110 @@ +package websteps + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + "net/url" + "sync" + + "github.com/apex/log" + "github.com/lucas-clemente/quic-go" + "github.com/lucas-clemente/quic-go/http3" + "github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer" + "github.com/ooni/probe-cli/v3/internal/errorsx" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +var ErrNoConnReuse = errors.New("cannot reuse connection") + +func NewRequest(ctx context.Context, URL *url.URL, headers http.Header) *http.Request { + req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil) + runtimex.PanicOnError(err, "NewRequestWithContect failed") + for k, vs := range headers { + for _, v := range vs { + req.Header.Add(k, v) + } + } + return req +} + +// NewDialerResolver contructs a new dialer for TCP connections, +// with default, errorwrapping and resolve functionalities +func NewDialerResolver(resolver netxlite.Resolver) netxlite.Dialer { + var d netxlite.Dialer = netxlite.DefaultDialer + d = &errorsx.ErrorWrapperDialer{Dialer: d} + d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d} + return d +} + +// NewQUICDialerResolver creates a new QUICDialerResolver +// with default, errorwrapping and resolve functionalities +func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDialer { + var ql quicdialer.QUICListener = &netxlite.QUICListenerStdlib{} + ql = &errorsx.ErrorWrapperQUICListener{QUICListener: ql} + var dialer netxlite.QUICContextDialer = &netxlite.QUICDialerQUICGo{ + QUICListener: ql, + } + dialer = &errorsx.ErrorWrapperQUICDialer{Dialer: dialer} + dialer = &netxlite.QUICDialerResolver{Resolver: resolver, Dialer: dialer} + return dialer +} + +// NewSingleH3Transport creates an http3.RoundTripper +func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper { + transport := &http3.RoundTripper{ + DisableCompression: true, + TLSClientConfig: tlscfg, + QuicConfig: qcfg, + Dial: (&SingleDialerH3{qsess: &qsess}).Dial, + } + return transport +} + +// NewSingleTransport determines the appropriate HTTP Transport from the ALPN +func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) { + singledialer := &SingleDialer{conn: &conn} + transport = http.DefaultTransport.(*http.Transport).Clone() + transport.(*http.Transport).DialContext = singledialer.DialContext + transport.(*http.Transport).DialTLSContext = singledialer.DialContext + transport.(*http.Transport).DisableCompression = true + transport.(*http.Transport).MaxConnsPerHost = 1 + + transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)} + return transport +} + +type SingleDialer struct { + sync.Mutex + conn *net.Conn +} + +func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { + s.Lock() + defer s.Unlock() + if s.conn == nil { + return nil, ErrNoConnReuse + } + c := s.conn + s.conn = nil + return *c, nil +} + +type SingleDialerH3 struct { + sync.Mutex + qsess *quic.EarlySession +} + +func (s *SingleDialerH3) Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) { + s.Lock() + defer s.Unlock() + if s.qsess == nil { + return nil, ErrNoConnReuse + } + qs := s.qsess + s.qsess = nil + return *qs, nil +} diff --git a/internal/engine/experiment/websteps/http.go b/internal/engine/experiment/websteps/http.go new file mode 100644 index 0000000..ec6a965 --- /dev/null +++ b/internal/engine/experiment/websteps/http.go @@ -0,0 +1,33 @@ +package websteps + +import ( + "io" + "net/http" +) + +// HTTPDo performs the HTTP check. +// Input: +// req *http.Request +// The same request than the one used by the Explore step. +// This means that req contains the headers set by the original CtrlRequest, as well as, +// in case of a redirect chain, additional headers that were added due to redirects +// transport http.RoundTripper: +// The transport to use, either http.Transport, or http3.RoundTripper. +func HTTPDo(req *http.Request, transport http.RoundTripper) (*http.Response, []byte, error) { + clnt := http.Client{ + CheckRedirect: func(r *http.Request, reqs []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: transport, + } + resp, err := clnt.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return resp, nil, nil + } + return resp, body, nil +} diff --git a/internal/engine/experiment/websteps/model.go b/internal/engine/experiment/websteps/model.go new file mode 100644 index 0000000..0fb7dc3 --- /dev/null +++ b/internal/engine/experiment/websteps/model.go @@ -0,0 +1,102 @@ +package websteps + +import "net/http" + +// RoundTrip describes a specific round trip. +type RoundTrip struct { + // proto is the protocol used, it can be "h2", "http/1.1", "h3", "h3-29" + Proto string + + // Request is the original HTTP request. The headers + // also include cookies. + Request *http.Request + + // Response is the HTTP response. + Response *http.Response + + // sortIndex is an internal field using for sorting. + SortIndex int +} + +// URLMeasurement is a measurement of a given URL that +// includes connectivity measurement for each endpoint +// implied by the given URL. +type URLMeasurement struct { + // URL is the URL we're using + URL string `json:"url"` + + // DNS contains the domain names resolved by the helper. + DNS *DNSMeasurement `json:"dns"` + + // RoundTrip is the related round trip. + RoundTrip *RoundTrip `json:"-"` + + // Endpoints contains endpoint measurements. + Endpoints []*EndpointMeasurement `json:"endpoints"` +} + +// DNSMeasurement is a DNS measurement. +type DNSMeasurement struct { + // Domain is the domain we wanted to resolve. + Domain string `json:"domain"` + + // Addrs contains the resolved addresses. + Addrs []string `json:"addrs"` + + // Failure is the error that occurred. + Failure *string `json:"failure"` +} + +// HTTPSEndpointMeasurement is the measurement of requesting a specific endpoint via HTTPS. +type EndpointMeasurement struct { + // Endpoint is the endpoint we're measuring. + Endpoint string `json:"endpoint"` + + // Protocol is the used protocol. It can be "http", "https", "h3", "h3-29" or other supported QUIC protocols + Protocol string `json:"protocol"` + + // TCPConnectMeasurement is the related TCP connect measurement, if applicable (nil for h3 requests) + TCPConnectMeasurement *TCPConnectMeasurement `json:"tcp_connect"` + + // QUICHandshakeMeasurement is the related QUIC(TLS 1.3) handshake measurement, if applicable (nil for http, https requests) + QUICHandshakeMeasurement *TLSHandshakeMeasurement `json:"quic_handshake"` + + // TLSHandshakeMeasurement is the related TLS handshake measurement, if applicable (nil for http, h3 requests) + TLSHandshakeMeasurement *TLSHandshakeMeasurement `json:"tls_handshake"` + + // HTTPRoundTripMeasurement is the related HTTP GET measurement. + HTTPRoundTripMeasurement *HTTPRoundTripMeasurement `json:"http_round_trip"` +} + +// TCPConnectMeasurement is a TCP connect measurement. +type TCPConnectMeasurement struct { + // Failure is the error that occurred. + Failure *string `json:"failure"` +} + +// TLSHandshakeMeasurement is a TLS handshake measurement. +type TLSHandshakeMeasurement struct { + // Failure is the error that occurred. + Failure *string `json:"failure"` +} + +// HTTPRoundTripMeasurement contains a measured HTTP request and the corresponding response. +type HTTPRoundTripMeasurement struct { + Request *HTTPRequestMeasurement `json:"request"` + Response *HTTPResponseMeasurement `json:"response"` +} + +// HTTPRequestMeasurement contains the headers of the measured HTTP Get request. +type HTTPRequestMeasurement struct { + Headers http.Header `json:"headers"` + Method string `json:"method"` + URL string `json:"url"` +} + +// HTTPResponseMeasurement contains the response of the measured HTTP Get request. +type HTTPResponseMeasurement struct { + BodyLength int64 `json:"body_length"` + Failure *string `json:"failure"` + Headers http.Header `json:"headers"` + StatusCode int64 `json:"status_code"` +} diff --git a/internal/engine/experiment/websteps/quic.go b/internal/engine/experiment/websteps/quic.go new file mode 100644 index 0000000..8667252 --- /dev/null +++ b/internal/engine/experiment/websteps/quic.go @@ -0,0 +1,29 @@ +package websteps + +import ( + "context" + "crypto/tls" + + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +type QUICConfig struct { + Endpoint string + QUICDialer netxlite.QUICContextDialer + Resolver netxlite.Resolver + TLSConf *tls.Config +} + +// QUICDo performs the QUIC check. +func QUICDo(ctx context.Context, config QUICConfig) (quic.EarlySession, error) { + if config.QUICDialer != nil { + return config.QUICDialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{}) + } + resolver := config.Resolver + if resolver == nil { + resolver = &netxlite.ResolverSystem{} + } + dialer := NewQUICDialerResolver(resolver) + return dialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{}) +} diff --git a/internal/engine/experiment/websteps/tcp.go b/internal/engine/experiment/websteps/tcp.go new file mode 100644 index 0000000..3e7481d --- /dev/null +++ b/internal/engine/experiment/websteps/tcp.go @@ -0,0 +1,27 @@ +package websteps + +import ( + "context" + "net" + + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +type TCPConfig struct { + Dialer netxlite.Dialer + Endpoint string + Resolver netxlite.Resolver +} + +// TCPDo performs the TCP check. +func TCPDo(ctx context.Context, config TCPConfig) (net.Conn, error) { + if config.Dialer != nil { + return config.Dialer.DialContext(ctx, "tcp", config.Endpoint) + } + resolver := config.Resolver + if resolver == nil { + resolver = &netxlite.ResolverSystem{} + } + dialer := NewDialerResolver(resolver) + return dialer.DialContext(ctx, "tcp", config.Endpoint) +} diff --git a/internal/engine/experiment/websteps/tls.go b/internal/engine/experiment/websteps/tls.go new file mode 100644 index 0000000..1ed1176 --- /dev/null +++ b/internal/engine/experiment/websteps/tls.go @@ -0,0 +1,16 @@ +package websteps + +import ( + "crypto/tls" + "net" +) + +// TLSDo performs the TLS check. +func TLSDo(conn net.Conn, hostname string) (*tls.Conn, error) { + tlsConn := tls.Client(conn, &tls.Config{ + ServerName: hostname, + NextProtos: []string{"h2", "http/1.1"}, + }) + err := tlsConn.Handshake() + return tlsConn, err +} diff --git a/internal/engine/experiment/websteps/websteps.go b/internal/engine/experiment/websteps/websteps.go new file mode 100644 index 0000000..093223e --- /dev/null +++ b/internal/engine/experiment/websteps/websteps.go @@ -0,0 +1,355 @@ +package websteps + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + "net/url" + "time" + + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/engine/httpheader" + "github.com/ooni/probe-cli/v3/internal/engine/model" + "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +const ( + testName = "websteps" + testVersion = "0.0.1" +) + +// Config contains the experiment config. +type Config struct{} + +// TestKeys contains webconnectivity test keys. +type TestKeys struct { + Agent string `json:"agent"` + ClientResolver string `json:"client_resolver"` + URLMeasurements []*URLMeasurement +} + +// Measurer performs the measurement. +type Measurer struct { + Config Config +} + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return Measurer{Config: config} +} + +// ExperimentName implements ExperimentMeasurer.ExperExperimentName. +func (m Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion. +func (m Measurer) ExperimentVersion() string { + return testVersion +} + +// SupportedQUICVersions are the H3 over QUIC versions we currently support +var SupportedQUICVersions = map[string]bool{ + "h3": true, + "h3-29": true, +} + +var ( + // ErrNoAvailableTestHelpers is emitted when there are no available test helpers. + ErrNoAvailableTestHelpers = errors.New("no available helpers") + + // ErrNoInput indicates that no input was provided + ErrNoInput = errors.New("no input provided") + + // ErrInputIsNotAnURL indicates that the input is not an URL. + ErrInputIsNotAnURL = errors.New("input is not an URL") + + // ErrUnsupportedInput indicates that the input URL scheme is unsupported. + ErrUnsupportedInput = errors.New("unsupported input scheme") +) + +// Run implements ExperimentMeasurer.Run. +func (m Measurer) Run( + ctx context.Context, + sess model.ExperimentSession, + measurement *model.Measurement, + callbacks model.ExperimentCallbacks, +) error { + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + tk := new(TestKeys) + measurement.TestKeys = tk + tk.Agent = "redirect" + tk.ClientResolver = sess.ResolverIP() + + // 1. Parse and verify URL + URL, err := url.Parse(string(measurement.Input)) + if err != nil { + return ErrInputIsNotAnURL + } + if URL.Scheme != "http" && URL.Scheme != "https" { + return ErrUnsupportedInput + } + // 2. Perform the initial DNS lookup step + addrs, err := DNSDo(ctx, DNSConfig{Domain: URL.Hostname()}) + endpoints := makeEndpoints(addrs, URL) + // 3. Find the testhelper + testhelpers, _ := sess.GetTestHelpersByName("web-connectivity") + var testhelper *model.Service + for _, th := range testhelpers { + if th.Type == "https" { + testhelper = &th + break + } + } + if testhelper == nil { + return ErrNoAvailableTestHelpers + } + measurement.TestHelpers = map[string]interface{}{ + "backend": testhelper, + } + // 4. Query the testhelper + resp, err := Control(ctx, sess, testhelper.Address, CtrlRequest{ + HTTPRequest: URL.String(), + HTTPRequestHeaders: map[string][]string{ + "Accept": {httpheader.Accept()}, + "Accept-Language": {httpheader.AcceptLanguage()}, + "User-Agent": {httpheader.UserAgent()}, + }, + Addrs: endpoints, + }) + if err != nil || resp.URLMeasurements == nil { + return errors.New("no control response") + } + // 5. Go over the Control URL measurements and reproduce them without following redirects, one by one. + for _, controlURLMeasurement := range resp.URLMeasurements { + urlMeasurement := &URLMeasurement{ + URL: controlURLMeasurement.URL, + Endpoints: []*EndpointMeasurement{}, + } + URL, err = url.Parse(controlURLMeasurement.URL) + runtimex.PanicOnError(err, "url.Parse failed") + // DNS step + addrs, err = DNSDo(ctx, DNSConfig{Domain: URL.Hostname()}) + urlMeasurement.DNS = &DNSMeasurement{ + Domain: URL.Hostname(), + Addrs: addrs, + Failure: archival.NewFailure(err), + } + if controlURLMeasurement.Endpoints == nil { + tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement) + continue + } + // the testhelper tells us which endpoints to measure + for _, controlEndpoint := range controlURLMeasurement.Endpoints { + rt := controlEndpoint.HTTPRoundTripMeasurement + if rt == nil || rt.Request == nil { + continue + } + var endpointMeasurement *EndpointMeasurement + proto := controlEndpoint.Protocol + _, h3 := SupportedQUICVersions[proto] + switch { + case h3: + endpointMeasurement = m.measureEndpointH3(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers, proto) + case proto == "http": + endpointMeasurement = m.measureEndpointHTTP(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers) + case proto == "https": + endpointMeasurement = m.measureEndpointHTTPS(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers) + default: + panic("should not happen") + } + urlMeasurement.Endpoints = append(urlMeasurement.Endpoints, endpointMeasurement) + } + tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement) + } + return nil +} + +func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement { + endpointMeasurement := &EndpointMeasurement{ + Endpoint: endpoint, + Protocol: "http", + } + // TCP connect step + conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint}) + endpointMeasurement.TCPConnectMeasurement = &TCPConnectMeasurement{ + Failure: archival.NewFailure(err), + } + if err != nil { + return endpointMeasurement + } + defer conn.Close() + + // HTTP roundtrip step + request := NewRequest(ctx, URL, headers) + endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{ + Request: &HTTPRequestMeasurement{ + Headers: request.Header, + Method: "GET", + URL: URL.String(), + }, + } + transport := NewSingleTransport(conn) + resp, body, err := HTTPDo(request, transport) + if err != nil { + // failed Response + endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + Failure: archival.NewFailure(err), + } + return endpointMeasurement + } + // successful Response + endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + BodyLength: int64(len(body)), + Failure: nil, + Headers: resp.Header, + StatusCode: int64(resp.StatusCode), + } + return endpointMeasurement +} + +func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement { + endpointMeasurement := &EndpointMeasurement{ + Endpoint: endpoint, + Protocol: "https", + } + // TCP connect step + conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint}) + endpointMeasurement.TCPConnectMeasurement = &TCPConnectMeasurement{ + Failure: archival.NewFailure(err), + } + if err != nil { + return endpointMeasurement + } + defer conn.Close() + + // TLS handshake step + tlsconn, err := TLSDo(conn, URL.Hostname()) + endpointMeasurement.TLSHandshakeMeasurement = &TLSHandshakeMeasurement{ + Failure: archival.NewFailure(err), + } + if err != nil { + return endpointMeasurement + } + defer tlsconn.Close() + + // HTTP roundtrip step + request := NewRequest(ctx, URL, headers) + endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{ + Request: &HTTPRequestMeasurement{ + Headers: request.Header, + Method: "GET", + URL: URL.String(), + }, + } + transport := NewSingleTransport(tlsconn) + resp, body, err := HTTPDo(request, transport) + if err != nil { + // failed Response + endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + Failure: archival.NewFailure(err), + } + return endpointMeasurement + } + // successful Response + endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + BodyLength: int64(len(body)), + Failure: nil, + Headers: resp.Header, + StatusCode: int64(resp.StatusCode), + } + return endpointMeasurement +} + +func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint string, headers http.Header, proto string) *EndpointMeasurement { + endpointMeasurement := &EndpointMeasurement{ + Endpoint: endpoint, + Protocol: proto, + } + tlsConf := &tls.Config{ + ServerName: URL.Hostname(), + NextProtos: []string{proto}, + } + // QUIC handshake step + sess, err := QUICDo(ctx, QUICConfig{ + Endpoint: endpoint, + TLSConf: tlsConf, + }) + endpointMeasurement.QUICHandshakeMeasurement = &TLSHandshakeMeasurement{ + Failure: archival.NewFailure(err), + } + if err != nil { + return endpointMeasurement + } + // HTTP roundtrip step + request := NewRequest(ctx, URL, headers) + endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{ + Request: &HTTPRequestMeasurement{ + Headers: request.Header, + Method: "GET", + URL: URL.String(), + }, + } + transport := NewSingleH3Transport(sess, tlsConf, &quic.Config{}) + resp, body, err := HTTPDo(request, transport) + if err != nil { + // failed Response + endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + Failure: archival.NewFailure(err), + } + return endpointMeasurement + } + // successful Response + endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{ + BodyLength: int64(len(body)), + Failure: nil, + Headers: resp.Header, + StatusCode: int64(resp.StatusCode), + } + return endpointMeasurement + +} + +// SummaryKeys contains summary keys for this experiment. +// +// Note that this structure is part of the ABI contract with probe-cli +// therefore we should be careful when changing it. +type SummaryKeys struct { + Accessible bool `json:"accessible"` + Blocking string `json:"blocking"` + IsAnomaly bool `json:"-"` +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + sk := SummaryKeys{} + return sk, nil +} + +func makeEndpoints(addrs []string, URL *url.URL) []string { + endpoints := []string{} + if addrs == nil { + return endpoints + } + for _, addr := range addrs { + var port string + explicitPort := URL.Port() + scheme := URL.Scheme + switch { + case explicitPort != "": + port = explicitPort + case scheme == "http": + port = "80" + case scheme == "https": + port = "443" + default: + panic("should not happen") + } + endpoints = append(endpoints, net.JoinHostPort(addr, port)) + } + return endpoints +} diff --git a/internal/netxlite/resolver.go b/internal/netxlite/resolver.go index 37db6e4..393be5e 100644 --- a/internal/netxlite/resolver.go +++ b/internal/netxlite/resolver.go @@ -4,6 +4,8 @@ import ( "context" "net" "time" + + "golang.org/x/net/idna" ) // Resolver performs domain name resolutions. @@ -80,3 +82,34 @@ func (r *ResolverLogger) Address() string { } return "" } + +// IDNAResolver is to support resolving Internationalized Domain Names. +// See RFC3492 for more information. +type IDNAResolver struct { + Resolver +} + +// LookupHost implements Resolver.LookupHost +func (r *IDNAResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) { + host, err := idna.ToASCII(hostname) + if err != nil { + return nil, err + } + return r.Resolver.LookupHost(ctx, host) +} + +// Network implements Resolver.Network. +func (r *IDNAResolver) Network() string { + if rn, ok := r.Resolver.(resolverNetworker); ok { + return rn.Network() + } + return "idna" +} + +// Address implements Resolver.Address. +func (r *IDNAResolver) Address() string { + if ra, ok := r.Resolver.(resolverAddresser); ok { + return ra.Address() + } + return "" +}