From 1776ea1288e5233f679d31bcb8ba5d8c2a74375d Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 13 May 2022 15:06:03 +0200 Subject: [PATCH] cleanup: remove websteps summer 2021 implementation (#722) See https://github.com/ooni/probe/issues/2094 --- .../oohelperd/internal/websteps/explore.go | 175 ------- .../internal/websteps/explore_test.go | 160 ------ .../oohelperd/internal/websteps/generate.go | 287 ----------- .../internal/websteps/generate_test.go | 470 ------------------ .../cmd/oohelperd/internal/websteps/h3.go | 72 --- .../oohelperd/internal/websteps/h3_test.go | 39 -- .../internal/websteps/initialchecks.go | 61 --- .../internal/websteps/initialchecks_test.go | 31 -- .../oohelperd/internal/websteps/measure.go | 100 ---- .../internal/websteps/measure_test.go | 117 ----- .../cmd/oohelperd/internal/websteps/model.go | 17 - .../oohelperd/internal/websteps/websteps.go | 70 --- .../internal/websteps/websteps_test.go | 281 ----------- internal/cmd/oohelperd/oohelperd.go | 2 - .../engine/experiment/websteps/control.go | 26 - internal/engine/experiment/websteps/dns.go | 30 -- .../engine/experiment/websteps/factory.go | 132 ----- internal/engine/experiment/websteps/http.go | 34 -- internal/engine/experiment/websteps/model.go | 155 ------ internal/engine/experiment/websteps/quic.go | 30 -- internal/engine/experiment/websteps/tcp.go | 28 -- internal/engine/experiment/websteps/tls.go | 23 - .../engine/experiment/websteps/websteps.go | 369 -------------- 23 files changed, 2709 deletions(-) delete mode 100644 internal/cmd/oohelperd/internal/websteps/explore.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/explore_test.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/generate.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/generate_test.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/h3.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/h3_test.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/initialchecks.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/initialchecks_test.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/measure.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/measure_test.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/model.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/websteps.go delete mode 100644 internal/cmd/oohelperd/internal/websteps/websteps_test.go delete mode 100644 internal/engine/experiment/websteps/control.go delete mode 100644 internal/engine/experiment/websteps/dns.go delete mode 100644 internal/engine/experiment/websteps/factory.go delete mode 100644 internal/engine/experiment/websteps/http.go delete mode 100644 internal/engine/experiment/websteps/model.go delete mode 100644 internal/engine/experiment/websteps/quic.go delete mode 100644 internal/engine/experiment/websteps/tcp.go delete mode 100644 internal/engine/experiment/websteps/tls.go delete mode 100644 internal/engine/experiment/websteps/websteps.go diff --git a/internal/cmd/oohelperd/internal/websteps/explore.go b/internal/cmd/oohelperd/internal/websteps/explore.go deleted file mode 100644 index 9fc1c17..0000000 --- a/internal/cmd/oohelperd/internal/websteps/explore.go +++ /dev/null @@ -1,175 +0,0 @@ -package websteps - -import ( - "crypto/tls" - "net/http" - "net/http/cookiejar" - "net/url" - "sort" - "strings" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "github.com/ooni/probe-cli/v3/internal/runtimex" - utls "gitlab.com/yawning/utls.git" -) - -// Explore is the second step of the test helper algorithm. Its objective -// is to enumerate all the URLs we can discover by redirection from -// the original URL in the test list. Because the test list contains by -// definition noisy data, we need this preprocessing step to learn all -// the URLs that are actually implied by the original URL. - -// Explorer is the interface responsible for running Explore. -type Explorer interface { - Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) -} - -// DefaultExplorer is the default Explorer. -type DefaultExplorer struct { - resolver model.Resolver -} - -// Explore returns a list of round trips sorted so that the first -// round trip is the first element in the list, and so on. -// Explore uses the URL and the optional headers provided by the CtrlRequest. -func (e *DefaultExplorer) Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) { - resp, err := e.get(URL, headers) - if err != nil { - return nil, err - } - rts := e.rearrange(resp, nil) - h3URL, err := getH3URL(resp) - if err != nil { - // If we cannot find the HTTP/3 URL for subsequent measurements, we just continue - // the measurement using the URLs we have found so far. - return rts, nil - } - resp, err = e.getH3(h3URL, headers) - if err != nil { - // If we cannot follow the HTTP/3 chain, we just continue - // the measurement using the URLs we have found so far. - return rts, nil - } - rts = append(rts, e.rearrange(resp, h3URL)...) - return rts, nil -} - -// rearrange takes in input the final response of an HTTP transaction and an optional h3URL -// (which is needed to derive the type of h3 protocol, typically h3), -// and produces in output a list of round trips sorted -// such that the first round trip is the first element in the out array. -func (e *DefaultExplorer) rearrange(resp *http.Response, h3URL *h3URL) (out []*RoundTrip) { - index := 0 - for resp != nil && resp.Request != nil { - proto := resp.Request.URL.Scheme - if h3URL != nil { - proto = h3URL.proto - } - out = append(out, &RoundTrip{ - Proto: proto, - SortIndex: index, - Request: resp.Request, - Response: resp, - }) - index += 1 - resp = resp.Request.Response - } - sh := &sortHelper{out} - sort.Sort(sh) - return -} - -// sortHelper is the helper structure to sort round trips. -type sortHelper struct { - v []*RoundTrip -} - -// Len implements sort.Interface.Len. -func (sh *sortHelper) Len() int { - return len(sh.v) -} - -// Less implements sort.Interface.Less. -func (sh *sortHelper) Less(i, j int) bool { - return sh.v[i].SortIndex >= sh.v[j].SortIndex -} - -// Swap implements sort.Interface.Swap. -func (sh *sortHelper) Swap(i, j int) { - sh.v[i], sh.v[j] = sh.v[j], sh.v[i] -} - -// get gets the given URL and returns the final response after -// redirection, and an error. If the error is nil, the final response is valid. -func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http.Response, error) { - tlsConf := &tls.Config{ - NextProtos: []string{"h2", "http/1.1"}, - } - handshaker := &netxlite.TLSHandshakerConfigurable{ - NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto), - } - transport := websteps.NewTransportWithDialer(websteps.NewDialerResolver(e.resolver), tlsConf, handshaker) - // TODO(bassosimone): here we should use runtimex.PanicOnError - jarjar, _ := cookiejar.New(nil) - clnt := &http.Client{ - Transport: transport, - Jar: jarjar, - } - // TODO(bassosimone): document why e.newRequest cannot fail. - req, err := e.newRequest(URL, headers) - runtimex.PanicOnError(err, "newRequest failed") - resp, err := clnt.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - // Note that we ignore the response body. - return resp, nil -} - -// getH3 uses HTTP/3 to get the given URL and returns the final -// response after redirection, and an error. If the error is nil, the final response is valid. -func (e *DefaultExplorer) getH3(h3URL *h3URL, headers map[string][]string) (*http.Response, error) { - dialer := websteps.NewQUICDialerResolver(e.resolver) - tlsConf := &tls.Config{ - NextProtos: []string{h3URL.proto}, - } - // Rationale for using log.Log here: we're already using log.Log - // in this package, so it seems fair to use it also here - transport := netxlite.NewHTTP3Transport(log.Log, dialer, tlsConf) - // TODO(bassosimone): here we should use runtimex.PanicOnError - jarjar, _ := cookiejar.New(nil) - clnt := &http.Client{ - Transport: transport, - Jar: jarjar, - } - // TODO(bassosimone): document why e.newRequest cannot fail. - req, err := e.newRequest(h3URL.URL, headers) - runtimex.PanicOnError(err, "newRequest failed") - resp, err := clnt.Do(req) - if err != nil { - return nil, err - } - // Note that we ignore the response body. - defer resp.Body.Close() - return resp, nil -} - -func (e *DefaultExplorer) newRequest(URL *url.URL, headers map[string][]string) (*http.Request, error) { - req, err := http.NewRequest("GET", URL.String(), nil) - if err != nil { - return nil, err - } - for k, vs := range headers { - switch strings.ToLower(k) { - case "user-agent", "accept", "accept-language": - for _, v := range vs { - req.Header.Add(k, v) - } - } - } - return req, nil -} diff --git a/internal/cmd/oohelperd/internal/websteps/explore_test.go b/internal/cmd/oohelperd/internal/websteps/explore_test.go deleted file mode 100644 index c10c09a..0000000 --- a/internal/cmd/oohelperd/internal/websteps/explore_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package websteps - -import ( - "net/http" - "net/url" - "testing" - - "github.com/ooni/probe-cli/v3/internal/netxlite/quictesting" - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -var explorer = &DefaultExplorer{resolver: newResolver()} - -func TestExploreSuccess(t *testing.T) { - u, err := url.Parse("https://example.com") - runtimex.PanicOnError(err, "url.Parse failed") - rts, err := explorer.Explore(u, nil) - if err != nil { - t.Fatal("unexpected error") - } - if len(rts) != 1 { - t.Fatal("unexpected number of roundtrips") - } -} - -func TestExploreFailure(t *testing.T) { - u, err := url.Parse("https://example.example") - runtimex.PanicOnError(err, "url.Parse failed") - rts, err := explorer.Explore(u, nil) - if err == nil { - t.Fatal("expected an error here") - } - if rts != nil { - t.Fatal("rts should be nil") - } -} - -func TestExploreSuccessWithH3(t *testing.T) { - // TODO(bassosimone): figure out why this happens. - t.Skip("this test does not work in GHA") - u, err := url.Parse("https://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed") - rts, err := explorer.Explore(u, nil) - if err != nil { - t.Fatal("unexpected error") - } - if len(rts) != 2 { - t.Fatal("unexpected number of roundtrips") - } - if rts[0].Proto != "https" { - t.Fatal("unexpected protocol") - } - if rts[1].Proto != "h3" { - t.Fatal("unexpected protocol") - } -} - -func TestGetSuccess(t *testing.T) { - u, err := url.Parse("https://example.com") - runtimex.PanicOnError(err, "url.Parse failed for clearly good URL") - resp, err := explorer.get(u, nil) - if err != nil { - t.Fatal("unexpected error") - } - if resp == nil { - t.Fatal("unexpected nil response") - } - buf := make([]byte, 100) - if n, _ := resp.Body.Read(buf); n != 0 { - t.Fatal("expected response body tom be closed") - } - -} - -func TestGetFailure(t *testing.T) { - u, err := url.Parse("https://example.example") - runtimex.PanicOnError(err, "url.Parse failed for clearly good URL") - resp, err := explorer.get(u, nil) - if err == nil { - t.Fatal("expected an error here") - } - if resp != nil { - t.Fatal("response should be nil") - } -} - -func TestGetH3Success(t *testing.T) { - u := &url.URL{Scheme: "https", Host: quictesting.Domain, Path: "/"} - h3u := &h3URL{URL: u, proto: "h3"} - resp, err := explorer.getH3(h3u, nil) - if err != nil { - t.Fatal("unexpected error", err) - } - if resp == nil { - t.Fatal("unexpected nil response") - } - buf := make([]byte, 100) - if n, _ := resp.Body.Read(buf); n != 0 { - t.Fatal("expected response body tom be closed") - } - -} - -func TestGetH3Failure(t *testing.T) { - u, err := url.Parse("https://www.google.google") - runtimex.PanicOnError(err, "url.Parse failed for clearly good URL") - h3u := &h3URL{URL: u, proto: "h3"} - resp, err := explorer.getH3(h3u, nil) - if err == nil { - t.Fatal("expected an error here") - } - if resp != nil { - t.Fatal("response should be nil") - } -} - -func TestRearrange(t *testing.T) { - u, err := url.Parse("https://example.com") - runtimex.PanicOnError(err, "url.Parse failed") - resp := &http.Response{ - // the ProtoMajor field identifies the request/response structs and indicates the correct order - ProtoMajor: 2, - Request: &http.Request{ - ProtoMajor: 2, - URL: u, - Response: &http.Response{ - ProtoMajor: 1, - Request: &http.Request{ - ProtoMajor: 1, - URL: u, - Response: &http.Response{ - ProtoMajor: 0, - Request: &http.Request{ - ProtoMajor: 0, - URL: u, - }, - }, - }, - }, - }, - } - h3URL := &h3URL{URL: u, proto: "expected"} - rts := explorer.rearrange(resp, h3URL) - expectedIndex := 0 - for _, rt := range rts { - if rt.Request == nil || rt.Response == nil { - t.Fatal("unexpected nil value") - } - if rt.Request.ProtoMajor != expectedIndex { - t.Fatal("unexpected order") - } - if rt.Response.ProtoMajor != expectedIndex { - t.Fatal("unexpected order") - } - if rt.Proto != h3URL.proto { - t.Fatal("unexpected protocol") - } - expectedIndex += 1 - } -} diff --git a/internal/cmd/oohelperd/internal/websteps/generate.go b/internal/cmd/oohelperd/internal/websteps/generate.go deleted file mode 100644 index 246acc1..0000000 --- a/internal/cmd/oohelperd/internal/websteps/generate.go +++ /dev/null @@ -1,287 +0,0 @@ -package websteps - -import ( - "context" - "crypto/tls" - "net" - "net/http" - - "github.com/lucas-clemente/quic-go" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" - "github.com/ooni/probe-cli/v3/internal/model" -) - -// Generate is the third step of the algorithm. Given the -// observed round trips, we generate measurement targets and -// execute those measurements so the probe has a benchmark. - -// Generator is the interface responsible for running Generate. -type Generator interface { - Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) -} - -// DefaultGenerator is the default Generator. -type DefaultGenerator struct { - dialer model.Dialer - quicDialer model.QUICDialer - resolver model.Resolver - transport http.RoundTripper -} - -// the testhelper uses the same network operations as websteps -var ( - DNSDo = websteps.DNSDo - TCPDo = websteps.TCPDo - QUICDo = websteps.QUICDo - TLSDo = websteps.TLSDo - HTTPDo = websteps.HTTPDo -) - -// Generate takes in input a list of round trips and outputs -// a list of connectivity measurements for each of them. -func (g *DefaultGenerator) Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) { - var out []*URLMeasurement - for _, rt := range rts { - currentURL := g.GenerateURL(ctx, rt, clientResolutions) - out = append(out, currentURL) - } - return out, nil -} - -// GenerateURL returns a URLMeasurement. -func (g *DefaultGenerator) GenerateURL(ctx context.Context, rt *RoundTrip, clientResolutions []string) *URLMeasurement { - addrs, err := DNSDo(ctx, websteps.DNSConfig{ - Domain: rt.Request.URL.Hostname(), - Resolver: g.resolver, - }) - currentURL := &URLMeasurement{ - DNS: &DNSMeasurement{ - Domain: rt.Request.URL.Hostname(), - Addrs: addrs, - Failure: newfailure(err), - }, - RoundTrip: rt, - URL: rt.Request.URL.String(), - } - addrs = g.mergeAddresses(addrs, clientResolutions) - if len(addrs) == 0 { - return currentURL - } - for _, addr := range addrs { - var port string - explicitPort := rt.Request.URL.Port() - scheme := rt.Request.URL.Scheme - switch { - case explicitPort != "": - port = explicitPort - case scheme == "http": - port = "80" - case scheme == "https": - port = "443" - default: - panic("should not happen") - } - endpoint := net.JoinHostPort(addr, port) - var currentEndpoint *EndpointMeasurement - _, h3 := websteps.SupportedQUICVersions[rt.Proto] - switch { - case h3: - currentEndpoint = g.GenerateH3Endpoint(ctx, rt, endpoint) - case rt.Proto == "http": - currentEndpoint = g.GenerateHTTPEndpoint(ctx, rt, endpoint) - case rt.Proto == "https": - currentEndpoint = g.GenerateHTTPSEndpoint(ctx, rt, endpoint) - default: - // TODO(kelmenhorst): do we have to register this error somewhere in the result struct? - continue - } - currentURL.Endpoints = append(currentURL.Endpoints, currentEndpoint) - } - return currentURL -} - -// GenerateHTTPEndpoint performs an HTTP Request by -// a) establishing a TCP connection to the target (TCPDo), -// b) performing an HTTP GET request to the endpoint (HTTPDo). -// It returns an EndpointMeasurement. -func (g *DefaultGenerator) GenerateHTTPEndpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement { - currentEndpoint := &EndpointMeasurement{ - Endpoint: endpoint, - Protocol: "http", - } - tcpConn, err := TCPDo(ctx, websteps.TCPConfig{ - Dialer: g.dialer, - Endpoint: endpoint, - Resolver: g.resolver, - }) - currentEndpoint.TCPConnect = &TCPConnectMeasurement{ - Failure: newfailure(err), - } - if err != nil { - return currentEndpoint - } - defer tcpConn.Close() - - // prepare HTTPRoundTripMeasurement of this endpoint - currentEndpoint.HTTPRoundTrip = &HTTPRoundTripMeasurement{ - Request: &HTTPRequestMeasurement{ - Headers: rt.Request.Header, - Method: "GET", - URL: rt.Request.URL.String(), - }, - } - transport := websteps.NewSingleTransport(tcpConn) - if g.transport != nil { - transport = g.transport - } - resp, body, err := HTTPDo(rt.Request, transport) - if err != nil { - // failed Response - currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - Failure: newfailure(err), - } - return currentEndpoint - } - // successful Response - currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - BodyLength: int64(len(body)), - Failure: nil, - Headers: resp.Header, - StatusCode: int64(resp.StatusCode), - } - return currentEndpoint -} - -// GenerateHTTPSEndpoint performs an HTTPS Request by -// a) establishing a TCP connection to the target (TCPDo), -// b) establishing a TLS connection to the target (TLSDo), -// c) performing an HTTP GET request to the endpoint (HTTPDo). -// It returns an EndpointMeasurement. -func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement { - currentEndpoint := &EndpointMeasurement{ - Endpoint: endpoint, - Protocol: "https", - } - var tcpConn, tlsConn net.Conn - tcpConn, err := TCPDo(ctx, websteps.TCPConfig{ - Dialer: g.dialer, - Endpoint: endpoint, - Resolver: g.resolver, - }) - currentEndpoint.TCPConnect = &TCPConnectMeasurement{ - Failure: newfailure(err), - } - if err != nil { - return currentEndpoint - } - defer tcpConn.Close() - - tlsConn, err = TLSDo(ctx, tcpConn, rt.Request.URL.Hostname()) - currentEndpoint.TLSHandshake = &TLSHandshakeMeasurement{ - Failure: newfailure(err), - } - if err != nil { - return currentEndpoint - } - defer tlsConn.Close() - - // prepare HTTPRoundTripMeasurement of this endpoint - currentEndpoint.HTTPRoundTrip = &HTTPRoundTripMeasurement{ - Request: &HTTPRequestMeasurement{ - Headers: rt.Request.Header, - Method: "GET", - URL: rt.Request.URL.String(), - }, - } - transport := websteps.NewSingleTransport(tlsConn) - if g.transport != nil { - transport = g.transport - } - resp, body, err := HTTPDo(rt.Request, transport) - if err != nil { - // failed Response - currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - Failure: newfailure(err), - } - return currentEndpoint - } - // successful Response - currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - BodyLength: int64(len(body)), - Failure: nil, - Headers: resp.Header, - StatusCode: int64(resp.StatusCode), - } - return currentEndpoint -} - -// GenerateH3Endpoint performs an HTTP/3 Request by -// a) establishing a QUIC connection to the target (QUICDo), -// b) performing an HTTP GET request to the endpoint (HTTPDo). -// It returns an EndpointMeasurement. -func (g *DefaultGenerator) GenerateH3Endpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement { - currentEndpoint := &EndpointMeasurement{ - Endpoint: endpoint, - Protocol: rt.Proto, - } - tlsConf := &tls.Config{ - ServerName: rt.Request.URL.Hostname(), - NextProtos: []string{rt.Proto}, - } - sess, err := QUICDo(ctx, websteps.QUICConfig{ - Endpoint: endpoint, - QUICDialer: g.quicDialer, - TLSConf: tlsConf, - Resolver: g.resolver, - }) - currentEndpoint.QUICHandshake = &TLSHandshakeMeasurement{ - Failure: newfailure(err), - } - if err != nil { - return currentEndpoint - } - // prepare HTTPRoundTripMeasurement of this endpoint - currentEndpoint.HTTPRoundTrip = &HTTPRoundTripMeasurement{ - Request: &HTTPRequestMeasurement{ - Headers: rt.Request.Header, - Method: "GET", - URL: rt.Request.URL.String(), - }, - } - var transport http.RoundTripper = websteps.NewSingleH3Transport(sess, tlsConf, &quic.Config{}) - if g.transport != nil { - transport = g.transport - } - resp, body, err := HTTPDo(rt.Request, transport) - if err != nil { - // failed Response - currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - Failure: newfailure(err), - } - return currentEndpoint - } - // successful Response - currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - BodyLength: int64(len(body)), - Failure: nil, - Headers: resp.Header, - StatusCode: int64(resp.StatusCode), - } - return currentEndpoint -} - -// mergeAddresses creates a (duplicate-free) union set of the IP addresses provided by the client, -// and the addresses resulting from the testhelper's DNS step -func (g *DefaultGenerator) mergeAddresses(addrs []string, clientAddrs []string) (out []string) { - unique := make(map[string]bool, len(addrs)+len(clientAddrs)) - for _, a := range addrs { - unique[a] = true - } - for _, a := range clientAddrs { - unique[a] = true - } - for key := range unique { - out = append(out, key) - } - return out -} diff --git a/internal/cmd/oohelperd/internal/websteps/generate_test.go b/internal/cmd/oohelperd/internal/websteps/generate_test.go deleted file mode 100644 index ee504e2..0000000 --- a/internal/cmd/oohelperd/internal/websteps/generate_test.go +++ /dev/null @@ -1,470 +0,0 @@ -package websteps - -import ( - "context" - "crypto/tls" - "errors" - "net" - "net/http" - "net/url" - "strings" - "testing" - - "github.com/lucas-clemente/quic-go" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "github.com/ooni/probe-cli/v3/internal/netxlite/quictesting" - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -var generator = &DefaultGenerator{resolver: newResolver()} - -type fakeTransport struct { - err error -} - -func (txp fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { - return nil, txp.err -} -func (txp fakeTransport) CloseIdleConnections() {} - -type fakeQUICDialer struct { - err error -} - -func (d fakeQUICDialer) DialContext(ctx context.Context, network, address string, - tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { - return nil, d.err -} - -func (d fakeQUICDialer) CloseIdleConnections() {} - -type fakeDialer struct { - err error -} - -func (d fakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - return nil, d.err -} - -func (d fakeDialer) CloseIdleConnections() {} - -func TestGenerateDNSFailure(t *testing.T) { - u, err := url.Parse("https://www.google.google") - runtimex.PanicOnError(err, "url.Parse failed") - rts := []*RoundTrip{ - { - Proto: "https", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - }, - } - urlMeasurements, err := generator.Generate(context.Background(), rts, []string{}) - if err != nil { - t.Fatal("unexpected error") - } - if len(urlMeasurements) != 1 { - t.Fatal("unexpected urlMeasurements length") - } - if urlMeasurements[0].DNS == nil { - t.Fatal("DNS should not be nil") - } - if urlMeasurements[0].DNS.Failure == nil || *urlMeasurements[0].DNS.Failure != netxlite.FailureDNSNXDOMAINError { - t.Fatal("unexpected DNS failure type") - } -} - -func TestGenerate(t *testing.T) { - u, err := url.Parse("http://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed for clearly good URL") - u2, err := url.Parse("https://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed for clearly good URL") - rts := []*RoundTrip{ - { - Proto: "http", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - }, - { - Proto: "https", - Request: &http.Request{ - URL: u2, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - }, - { - Proto: "h3", - Request: &http.Request{ - URL: u2, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - }, - } - urlMeasurements, err := generator.Generate(context.Background(), rts, []string{}) - if err != nil { - t.Fatal("unexpected err") - } - if urlMeasurements == nil { - t.Fatal("unexpected nil urlMeasurement") - } - if len(urlMeasurements) < 3 { - t.Fatal("unexpected number of urlMeasurements", len(urlMeasurements)) - } -} - -func TestGenerateUnexpectedProtocol(t *testing.T) { - u, err := url.Parse("https://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed") - rts := []*RoundTrip{ - { - Proto: "h3-27", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - }, - } - urlMeasurements, err := generator.Generate(context.Background(), rts, []string{}) - if err != nil { - t.Fatal("unexpected err") - } - if urlMeasurements == nil { - t.Fatal("unexpected nil urlMeasurement") - } - if len(urlMeasurements) != 1 { - t.Fatal("unexpected number of urlMeasurements") - } - measurement := urlMeasurements[0] - if measurement.URL != u.String() { - t.Fatal("unexpected URL") - } - if measurement.DNS == nil { - t.Fatal("DNS should not be nil") - } - if measurement.RoundTrip == nil { - t.Fatal("RoundTrip should not be nil") - } - if measurement.Endpoints != nil { - t.Fatal("Endpoints should be nil") - } -} - -func TestGenerateURLWithClientResolutions(t *testing.T) { - u, err := url.Parse("https://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed") - rt := &RoundTrip{ - Proto: "h3", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - } - clientResolution := "142.250.186.36" - urlMeasurement := generator.GenerateURL(context.Background(), rt, []string{clientResolution}) - if err != nil { - t.Fatal("unexpected err") - } - if urlMeasurement == nil { - t.Fatal("unexpected nil urlMeasurement") - } - if urlMeasurement.DNS == nil { - t.Fatal("DNS should not be nil") - } - if len(urlMeasurement.Endpoints) < 2 { - t.Fatal("unexpected number of endpoints") - } - clientAddrsFound := false - for _, e := range urlMeasurement.Endpoints { - if e.Endpoint == clientResolution+":443" { - clientAddrsFound = true - } - } - if !clientAddrsFound { - t.Fatal("did not use provided client resolution") - } -} - -func TestGenerateHTTP(t *testing.T) { - u, err := url.Parse("http://example.com") - runtimex.PanicOnError(err, "url.Parse failed") - rt := &RoundTrip{ - Proto: "http", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - } - endpointMeasurement := generator.GenerateHTTPEndpoint(context.Background(), rt, "93.184.216.34:80") - if err != nil { - t.Fatal("unexpected err") - } - if endpointMeasurement == nil { - t.Fatal("unexpected nil urlMeasurement") - } - if endpointMeasurement.TCPConnect == nil { - t.Fatal("TCPConnectMeasurement should not be nil") - } - if endpointMeasurement.HTTPRoundTrip == nil { - t.Fatal("HTTPRoundTripMeasurement should not be nil") - } -} - -func TestGenerateHTTPS(t *testing.T) { - u, err := url.Parse("https://example.com") - runtimex.PanicOnError(err, "url.Parse failed") - rt := &RoundTrip{ - Proto: "https", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - } - endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "93.184.216.34:443") - if err != nil { - t.Fatal("unexpected err") - } - if endpointMeasurement == nil { - t.Fatal("unexpected nil urlMeasurement") - } - if endpointMeasurement.TCPConnect == nil { - t.Fatal("TCPConnectMeasurement should not be nil") - } - if endpointMeasurement.TLSHandshake == nil { - t.Fatal("TLSHandshakeMeasurement should not be nil") - } - if endpointMeasurement.TLSHandshake.Failure != nil { - t.Fatal("unexpected failure at TLSHandshakeMeasurement") - } - if endpointMeasurement.HTTPRoundTrip == nil { - t.Fatal("HTTPRoundTripMeasurement should not be nil") - } -} - -func TestGenerateHTTPSTLSFailure(t *testing.T) { - u, err := url.Parse("https://wrong.host.badssl.com/") - runtimex.PanicOnError(err, "url.Parse failed") - rt := &RoundTrip{ - Proto: "https", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - } - endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "104.154.89.105:443") - if err != nil { - t.Fatal("unexpected err") - } - if endpointMeasurement == nil { - t.Fatal("unexpected nil urlMeasurement") - } - if endpointMeasurement.TCPConnect == nil { - t.Fatal("TCPConnectMeasurement should not be nil") - } - if endpointMeasurement.TLSHandshake == nil { - t.Fatal("TLSHandshakeMeasurement should not be nil") - } - if endpointMeasurement.TLSHandshake.Failure == nil { - t.Fatal("expected failure at TLSHandshakeMeasurement") - } - if endpointMeasurement.HTTPRoundTrip != nil { - t.Fatal("HTTPRoundTripMeasurement should be nil") - } -} - -func TestGenerateH3(t *testing.T) { - u := &url.URL{Scheme: "https", Host: quictesting.Domain, Path: "/"} - rt := &RoundTrip{ - Proto: "h3", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - } - endpointMeasurement := generator.GenerateH3Endpoint(context.Background(), rt, quictesting.Endpoint("443")) - if endpointMeasurement == nil { - t.Fatal("unexpected nil urlMeasurement") - } - if endpointMeasurement.QUICHandshake == nil { - t.Fatal("TCPConnectMeasurement should not be nil") - } - if endpointMeasurement.HTTPRoundTrip == nil { - t.Fatal("HTTPRoundTripMeasurement should not be nil") - } -} - -func TestGenerateTCPDoFails(t *testing.T) { - expected := errors.New("expected") - generator := &DefaultGenerator{ - dialer: fakeDialer{err: expected}, - resolver: newResolver(), - } - u, err := url.Parse("https://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed") - rt := &RoundTrip{ - Proto: "https", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - } - endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "173.194.76.103:443") - if err != nil { - t.Fatal("unexpected err") - } - if endpointMeasurement.TCPConnect == nil { - t.Fatal("QUIC handshake should not be nil") - } - if endpointMeasurement.TCPConnect.Failure == nil { - t.Fatal("expected an error here") - } - if *endpointMeasurement.TCPConnect.Failure != *newfailure(expected) { - t.Fatal("unexpected error type") - } -} - -func TestGenerateQUICDoFails(t *testing.T) { - expected := errors.New("expected") - generator := &DefaultGenerator{ - quicDialer: fakeQUICDialer{err: expected}, - resolver: newResolver(), - } - u, err := url.Parse("https://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed") - rt := &RoundTrip{ - Proto: "h3", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - } - endpointMeasurement := generator.GenerateH3Endpoint(context.Background(), rt, "173.194.76.103:443") - if err != nil { - t.Fatal("unexpected err") - } - if endpointMeasurement.QUICHandshake == nil { - t.Fatal("QUIC handshake should not be nil") - } - if endpointMeasurement.QUICHandshake.Failure == nil { - t.Fatal("expected an error here") - } - if *endpointMeasurement.QUICHandshake.Failure != *newfailure(expected) { - t.Fatal("unexpected error type") - } -} - -func TestGenerateHTTPDoFails(t *testing.T) { - expected := errors.New("expected") - generator := &DefaultGenerator{ - transport: fakeTransport{err: expected}, - resolver: newResolver(), - } - u, err := url.Parse("http://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed for clearly good URL") - u2, err := url.Parse("https://www.google.com") - runtimex.PanicOnError(err, "url.Parse failed for clearly good URL") - rts := []*RoundTrip{ - { - Proto: "http", - Request: &http.Request{ - URL: u, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - }, - { - Proto: "https", - Request: &http.Request{ - URL: u2, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - }, - { - Proto: "h3", - Request: &http.Request{ - URL: u2, - }, - Response: &http.Response{ - StatusCode: 200, - }, - SortIndex: 0, - }, - } - urlMeasurements, err := generator.Generate(context.Background(), rts, []string{}) - if err != nil { - t.Fatal("unexpected err") - } - if len(urlMeasurements) != 3 { - t.Fatal("unexpected number of urlMeasurements") - } - for _, u := range urlMeasurements { - if u.DNS == nil { - t.Fatal("unexpected DNS failure") - } - if len(u.Endpoints) < 1 { - t.Fatal("unexpected number of endpoints", len(u.Endpoints)) - } - // this can occur when the network is unreachable, but it is irrelevant for checking HTTP behavior - if u.Endpoints[0].TCPConnect != nil && u.Endpoints[0].TCPConnect.Failure != nil { - continue - } - if u.Endpoints[0].QUICHandshake != nil && u.Endpoints[0].QUICHandshake.Failure != nil { - continue - } - if u.Endpoints[0].HTTPRoundTrip == nil { - t.Fatal("roundtrip should not be nil", u.Endpoints[0].TCPConnect.Failure, "jaaaa") - } - if u.Endpoints[0].HTTPRoundTrip.Response == nil { - t.Fatal("roundtrip response should not be nil") - } - if u.Endpoints[0].HTTPRoundTrip.Response.Failure == nil { - t.Fatal("expected an HTTP error") - } - if !strings.HasSuffix(*u.Endpoints[0].HTTPRoundTrip.Response.Failure, expected.Error()) { - t.Fatal("unexpected failure type") - } - } -} diff --git a/internal/cmd/oohelperd/internal/websteps/h3.go b/internal/cmd/oohelperd/internal/websteps/h3.go deleted file mode 100644 index 17e315e..0000000 --- a/internal/cmd/oohelperd/internal/websteps/h3.go +++ /dev/null @@ -1,72 +0,0 @@ -package websteps - -import ( - "errors" - "net" - "net/http" - "net/url" - "strings" - - "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -type h3URL struct { - URL *url.URL - proto string -} - -type altSvcH3 struct { - authority string - proto string -} - -var ErrNoH3Location = errors.New("no h3 location found") - -// getH3URL returns the URL for HTTP/3 requests, if the target supports HTTP/3. -// Returns nil, if no HTTP/3 support is advertised. -func getH3URL(resp *http.Response) (*h3URL, error) { - URL := resp.Request.URL - if URL == nil { - return nil, ErrInvalidURL - } - h3Svc, err := parseAltSvc(resp, URL) - if err != nil { - return nil, err - } - quicURL, err := url.Parse(URL.String()) - runtimex.PanicOnError(err, "url.Parse failed") - quicURL.Host = h3Svc.authority - return &h3URL{URL: quicURL, proto: h3Svc.proto}, nil -} - -// parseAltSvc parses the Alt-Svc HTTP header for entries advertising the use of H3 -func parseAltSvc(resp *http.Response, URL *url.URL) (*altSvcH3, error) { - // TODO(bassosimone,kelmenhorst): see if we can make this algorithm more robust. - if URL.Scheme != "https" { - return nil, ErrUnsupportedScheme - } - alt_svc := resp.Header.Get("Alt-Svc") - // syntax: Alt-Svc: =; ma=; persist=1 - entries := strings.Split(alt_svc, ",") - for _, e := range entries { - keyvalpairs := strings.Split(e, ";") - for _, p := range keyvalpairs { - p = strings.Replace(p, "\"", "", -1) - kv := strings.Split(p, "=") - if len(kv) != 2 { - continue - } - if _, ok := websteps.SupportedQUICVersions[kv[0]]; ok { - host, port, err := net.SplitHostPort(kv[1]) - runtimex.PanicOnError(err, "net.SplitHostPort failed") - if host == "" { - host = URL.Hostname() - } - authority := net.JoinHostPort(host, port) - return &altSvcH3{authority: authority, proto: kv[0]}, nil - } - } - } - return nil, ErrNoH3Location -} diff --git a/internal/cmd/oohelperd/internal/websteps/h3_test.go b/internal/cmd/oohelperd/internal/websteps/h3_test.go deleted file mode 100644 index 407d2cb..0000000 --- a/internal/cmd/oohelperd/internal/websteps/h3_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package websteps - -import ( - "net/http" - "net/url" - "testing" - - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -func TestGetH3URLInvalidURL(t *testing.T) { - resp := &http.Response{ - Request: &http.Request{}, - } - h3URL, err := getH3URL(resp) - if err == nil { - t.Fatal("expected an error here") - } - if h3URL != nil { - t.Fatal("h3URL should be nil") - } - -} - -func TestParseUnsupportedScheme(t *testing.T) { - URL, err := url.Parse("h3://google.com") - runtimex.PanicOnError(err, "url.Parse failed") - parsed, err := parseAltSvc(nil, URL) - if err == nil { - t.Fatal("expected an error here") - } - if err != ErrUnsupportedScheme { - t.Fatal("unexpected error type") - } - if parsed != nil { - t.Fatal("h3URL should be nil") - } - -} diff --git a/internal/cmd/oohelperd/internal/websteps/initialchecks.go b/internal/cmd/oohelperd/internal/websteps/initialchecks.go deleted file mode 100644 index a0d058b..0000000 --- a/internal/cmd/oohelperd/internal/websteps/initialchecks.go +++ /dev/null @@ -1,61 +0,0 @@ -package websteps - -import ( - "context" - "errors" - "net/url" - - "github.com/ooni/probe-cli/v3/internal/model" -) - -// InitialChecks is the first step of the test helper algorithm. We -// make sure we can parse the URL, we handle the scheme, and the domain -// name inside the URL's authority is valid. - -// Errors returned by Preresolve. -var ( - // ErrInvalidURL indicates that the URL is invalid. - ErrInvalidURL = errors.New("the URL is invalid") - - // ErrUnsupportedScheme indicates that we don't support the scheme. - ErrUnsupportedScheme = errors.New("unsupported scheme") - - // ErrNoSuchHost indicates that the DNS resolution failed. - ErrNoSuchHost = errors.New("no such host") -) - -// InitChecker is the interface responsible for running InitialChecks. -type InitChecker interface { - InitialChecks(URL string) (*url.URL, error) -} - -// DefaultInitChecker is the default InitChecker. -type DefaultInitChecker struct { - resolver model.Resolver -} - -// InitialChecks checks whether the URL is valid and whether the -// domain inside the URL is an existing one. If these preliminary -// checks fail, there's no point in continuing. -// If they succeed, InitialChecks returns the URL, if not an error. -func (i *DefaultInitChecker) InitialChecks(URL string) (*url.URL, error) { - parsed, err := url.Parse(URL) - if err != nil { - return nil, ErrInvalidURL - } - switch parsed.Scheme { - case "http", "https": - default: - return nil, ErrUnsupportedScheme - } - // Assumptions: - // - // 1. the resolver will cache the resolution for later - // - // 2. an IP address does not cause an error because we are using - // a resolve that behaves like getaddrinfo - if _, err := i.resolver.LookupHost(context.Background(), parsed.Hostname()); err != nil { - return nil, ErrNoSuchHost - } - return parsed, nil -} diff --git a/internal/cmd/oohelperd/internal/websteps/initialchecks_test.go b/internal/cmd/oohelperd/internal/websteps/initialchecks_test.go deleted file mode 100644 index d0a5da9..0000000 --- a/internal/cmd/oohelperd/internal/websteps/initialchecks_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package websteps - -import ( - "testing" -) - -var checker = &DefaultInitChecker{resolver: newResolver()} - -func TestMeasureWithInvalidURL(t *testing.T) { - _, err := checker.InitialChecks("http://[::1]aaaa") - - if err == nil || err != ErrInvalidURL { - t.Fatal("expected an error here") - } -} - -func TestMeasureWithUnsupportedScheme(t *testing.T) { - _, err := checker.InitialChecks("abc://example.com") - - if err == nil || err != ErrUnsupportedScheme { - t.Fatal("expected an error here") - } -} - -func TestMeasureWithInvalidHost(t *testing.T) { - _, err := checker.InitialChecks("http://www.ooni.ooni") - - if err == nil || err != ErrNoSuchHost { - t.Fatal("expected an error here") - } -} diff --git a/internal/cmd/oohelperd/internal/websteps/measure.go b/internal/cmd/oohelperd/internal/websteps/measure.go deleted file mode 100644 index da0bb3c..0000000 --- a/internal/cmd/oohelperd/internal/websteps/measure.go +++ /dev/null @@ -1,100 +0,0 @@ -package websteps - -import ( - "context" - "errors" - "net/url" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" - "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -type ( - CtrlRequest = websteps.CtrlRequest - ControlResponse = websteps.CtrlResponse -) - -var ErrInternalServer = errors.New("internal server error") - -// Config contains the building blocks of the testhelper algorithm -type Config struct { - checker InitChecker - explorer Explorer - generator Generator - resolver model.Resolver -} - -// Measure performs the three consecutive steps of the testhelper algorithm: -// -// 1. InitialChecks -// -// 2. Explore -// -// 3. Generate -func Measure(ctx context.Context, creq *CtrlRequest, config *Config) (*ControlResponse, error) { - var ( - URL *url.URL - err error - ) - resolver := config.resolver - if resolver == nil { - // use a central resolver - resolver = newResolver() - } - checker := config.checker - if checker == nil { - checker = &DefaultInitChecker{resolver: resolver} - } - URL, err = checker.InitialChecks(creq.URL) - if err != nil { - // return a valid response in case of NXDOMAIN so the probe can compare the failure - if err == ErrNoSuchHost { - return newDNSFailedResponse(err, creq.URL), nil - } - return nil, err - } - explorer := config.explorer - if explorer == nil { - explorer = &DefaultExplorer{resolver: resolver} - } - rts, err := explorer.Explore(URL, creq.Headers) - if err != nil { - return nil, ErrInternalServer - } - generator := config.generator - if generator == nil { - generator = &DefaultGenerator{resolver: resolver} - } - meas, err := generator.Generate(ctx, rts, creq.Addrs) - if err != nil { - return nil, err - } - return &ControlResponse{URLs: meas}, nil -} - -// newDNSFailedResponse creates a new response with one URLMeasurement entry -// indicating that the DNS step failed -func newDNSFailedResponse(err error, URL string) *ControlResponse { - resp := &ControlResponse{} - m := &URLMeasurement{ - URL: URL, - DNS: &DNSMeasurement{ - Failure: newfailure(err), - }, - } - resp.URLs = append(resp.URLs, m) - return resp -} - -// newResolver creates a new DNS resolver instance -func newResolver() model.Resolver { - childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google") - runtimex.PanicOnError(err, "NewDNSClient failed") - var r model.Resolver = childResolver - r = &netxlite.ResolverIDNA{Resolver: r} - return r -} diff --git a/internal/cmd/oohelperd/internal/websteps/measure_test.go b/internal/cmd/oohelperd/internal/websteps/measure_test.go deleted file mode 100644 index a0ace19..0000000 --- a/internal/cmd/oohelperd/internal/websteps/measure_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package websteps - -import ( - "context" - "errors" - "net/url" - "testing" - - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestMeasureSuccess(t *testing.T) { - req := &CtrlRequest{ - URL: "https://example.com", - } - resp, err := Measure(context.Background(), req, &Config{}) - if err != nil { - t.Fatal("unexpected error") - } - if resp == nil { - t.Fatal("unexpected nil response") - } -} - -type MockChecker struct { - err error -} - -func (c *MockChecker) InitialChecks(URL string) (*url.URL, error) { - return nil, c.err -} - -type MockExplorer struct{} - -func (c *MockExplorer) Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) { - return nil, ErrExpectedExplore -} - -type MockGenerator struct{} - -func (c *MockGenerator) Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) { - return nil, ErrExpectedGenerate -} - -var ErrExpectedCheck error = errors.New("expected error checker") -var ErrExpectedExplore error = errors.New("expected error explorer") -var ErrExpectedGenerate error = errors.New("expected error generator") - -func TestMeasureInitialChecksFail(t *testing.T) { - req := &CtrlRequest{ - URL: "https://example.com", - } - resp, err := Measure(context.Background(), req, &Config{checker: &MockChecker{err: ErrExpectedCheck}}) - if err == nil { - t.Fatal("expected an error here") - } - if err != ErrExpectedCheck { - t.Fatal("unexpected error type") - } - if resp != nil { - t.Fatal("resp should be nil") - } -} - -func TestMeasureInitialChecksFailWithNXDOMAIN(t *testing.T) { - req := &CtrlRequest{ - URL: "https://example.com", - } - resp, err := Measure(context.Background(), req, &Config{checker: &MockChecker{err: ErrNoSuchHost}}) - if err != nil { - t.Fatal("unexpected error") - } - if resp == nil { - t.Fatal("resp should not be nil") - } - if len(resp.URLs) != 1 { - t.Fatal("unexpected number of measurements") - } - if resp.URLs[0].DNS == nil { - t.Fatal("DNS entry should not be nil") - } - if *resp.URLs[0].DNS.Failure != netxlite.FailureDNSNXDOMAINError { - t.Fatal("unexpected failure") - } -} - -func TestMeasureExploreFails(t *testing.T) { - req := &CtrlRequest{ - URL: "https://example.com", - } - resp, err := Measure(context.Background(), req, &Config{explorer: &MockExplorer{}}) - if err == nil { - t.Fatal("expected an error here") - } - if err != ErrInternalServer { - t.Fatal("unexpected error type") - } - if resp != nil { - t.Fatal("resp should be nil") - } -} - -func TestMeasureGenerateFails(t *testing.T) { - req := &CtrlRequest{ - URL: "https://example.com", - } - resp, err := Measure(context.Background(), req, &Config{generator: &MockGenerator{}}) - if err == nil { - t.Fatal("expected an error here") - } - if err != ErrExpectedGenerate { - t.Fatal("unexpected error type") - } - if resp != nil { - t.Fatal("resp should be nil") - } -} diff --git a/internal/cmd/oohelperd/internal/websteps/model.go b/internal/cmd/oohelperd/internal/websteps/model.go deleted file mode 100644 index fcaae26..0000000 --- a/internal/cmd/oohelperd/internal/websteps/model.go +++ /dev/null @@ -1,17 +0,0 @@ -package websteps - -import ( - "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" -) - -type ( - URLMeasurement = websteps.URLMeasurement - DNSMeasurement = websteps.DNSMeasurement - EndpointMeasurement = websteps.EndpointMeasurement - TCPConnectMeasurement = websteps.TCPConnectMeasurement - HTTPRoundTripMeasurement = websteps.HTTPRoundTripMeasurement - TLSHandshakeMeasurement = websteps.TLSHandshakeMeasurement - HTTPRequestMeasurement = websteps.HTTPRequestMeasurement - HTTPResponseMeasurement = websteps.HTTPResponseMeasurement - RoundTrip = websteps.RoundTripInfo -) diff --git a/internal/cmd/oohelperd/internal/websteps/websteps.go b/internal/cmd/oohelperd/internal/websteps/websteps.go deleted file mode 100644 index 907838b..0000000 --- a/internal/cmd/oohelperd/internal/websteps/websteps.go +++ /dev/null @@ -1,70 +0,0 @@ -// Package websteps implements the websteps test helper. -// -// See the https://github.com/ooni/spec/blob/master/backends/th-007-websteps.md -// related specification document. -// -// This implementation uses version 202108.17.1114 of the spec. -package websteps - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/ooni/probe-cli/v3/internal/version" -) - -// newfailure is a convenience shortcut to save typing -var newfailure = archival.NewFailure - -// maxAcceptableBody is _at the same time_ the maximum acceptable body for incoming -// API requests and the maximum acceptable body when fetching arbitrary URLs. See -// https://github.com/ooni/probe/issues/1727 for statistics regarding the test lists -// including the empirical CDF of the body size for test lists URLs. -const maxAcceptableBody = 1 << 24 - -// Handler implements the websteps test helper HTTP API. -type Handler struct { - Config *Config -} - -// ServeHTTP implements http.Handler.ServeHTTP. -func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - w.Header().Add("Server", fmt.Sprintf( - "oohelperd/%s ooniprobe-engine/%s", version.Version, version.Version, - )) - if req.Method != "POST" { - w.WriteHeader(400) - return - } - reader := &io.LimitedReader{R: req.Body, N: maxAcceptableBody} - data, err := netxlite.ReadAllContext(req.Context(), reader) - if err != nil { - w.WriteHeader(400) - return - } - var creq CtrlRequest - if err := json.Unmarshal(data, &creq); err != nil { - w.WriteHeader(400) - return - } - cresp, err := Measure(req.Context(), &creq, h.Config) - if err != nil { - if err == ErrInternalServer { - w.WriteHeader(500) - return - } - w.WriteHeader(400) - return - } - // We assume that the following call cannot fail because it's a - // clearly serializable data structure. - data, err = json.Marshal(cresp) - runtimex.PanicOnError(err, "json.Marshal failed") - w.Header().Add("Content-Type", "application/json") - w.Write(data) -} diff --git a/internal/cmd/oohelperd/internal/websteps/websteps_test.go b/internal/cmd/oohelperd/internal/websteps/websteps_test.go deleted file mode 100644 index 567097e..0000000 --- a/internal/cmd/oohelperd/internal/websteps/websteps_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package websteps - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -const requestnoredirect = `{ - "url": "https://ooni.org", - "headers": { - "Accept": [ - "*/*" - ], - "Accept-Language": [ - "en-US;q=0.8,en;q=0.5" - ], - "User-Agent": [ - "Mozilla/5.0" - ] - }, - "addrs": [ - "104.198.14.52:443" - ] -}` - -const requestredirect = `{ - "url": "https://www.ooni.org", - "headers": { - "Accept": [ - "*/*" - ], - "Accept-Language": [ - "en-US;q=0.8,en;q=0.5" - ], - "User-Agent": [ - "Mozilla/5.0" - ] - }, - "addrs": [ - "18.192.76.182:443" - ] -}` - -const requestIPaddressinput = `{ - "url": "https://172.217.168.4", - "headers": { - "Accept": [ - "*/*" - ], - "Accept-Language": [ - "en-US;q=0.8,en;q=0.5" - ], - "User-Agent": [ - "Mozilla/5.0" - ] - }, - "addrs": [ - "172.217.168.4:443" - ] -}` - -const requestwithquic = `{ - "url": "https://www.google.com", - "headers": { - "Accept": [ - "*/*" - ], - "Accept-Language": [ - "en-US;q=0.8,en;q=0.5" - ], - "User-Agent": [ - "Mozilla/5.0" - ] - }, - "addrs": [ - "142.250.74.196:443" - ] -}` - -const requestWithoutDomainName = `{ - "url": "https://8.8.8.8", - "headers": { - "Accept": [ - "*/*" - ], - "Accept-Language": [ - "en-US;q=0.8,en;q=0.5" - ], - "User-Agent": [ - "Mozilla/5.0" - ] - }, - "addrs": [ - "8.8.8.8:443" - ] -}` - -func TestWorkingAsIntended(t *testing.T) { - handler := Handler{Config: &Config{}} - srv := httptest.NewServer(&handler) - defer srv.Close() - type expectationSpec struct { - name string - reqMethod string - reqContentType string - reqBody string - respStatusCode int - respContentType string - parseBody bool - } - expectations := []expectationSpec{{ - name: "check for invalid method", - reqMethod: "GET", - respStatusCode: 400, - }, { - name: "check for invalid request body", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: "{", - respStatusCode: 400, - }, { - name: "with measurement failure", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: `{"url": "http://[::1]aaaa"}`, - respStatusCode: 400, - }, { - name: "request without redirect or H3 follow-up request", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: requestnoredirect, - respStatusCode: 200, - respContentType: "application/json", - parseBody: true, - }, { - name: "request triggering one redirect, without H3 follow-up request", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: requestredirect, - respStatusCode: 200, - respContentType: "application/json", - parseBody: true, - }, { - name: "request with an IP address as input", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: requestIPaddressinput, - respStatusCode: 200, - respContentType: "application/json", - parseBody: true, - }, { - name: "request triggering H3 follow-up request, without redirect", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: requestwithquic, - respStatusCode: 200, - respContentType: "application/json", - parseBody: true, - }, { - name: "when there's no domain name in the request", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: requestWithoutDomainName, - respStatusCode: 200, - respContentType: "application/json", - parseBody: true, - }} - for _, expect := range expectations { - t.Run(expect.name, func(t *testing.T) { - body := strings.NewReader(expect.reqBody) - req, err := http.NewRequest(expect.reqMethod, srv.URL, body) - if err != nil { - t.Fatalf("%s: %+v", expect.name, err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("%s: %+v", expect.name, err) - } - defer resp.Body.Close() - if resp.StatusCode != expect.respStatusCode { - t.Fatalf("unexpected status code: %+v", resp.StatusCode) - } - data, err := netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - if !expect.parseBody { - return - } - var v interface{} - if err := json.Unmarshal(data, &v); err != nil { - t.Fatal(err) - } - }) - } -} - -func TestHandlerWithInternalServerError(t *testing.T) { - handler := Handler{Config: &Config{explorer: &MockExplorer{}}} - srv := httptest.NewServer(&handler) - defer srv.Close() - body := strings.NewReader(`{"url": "https://example.com"}`) - req, err := http.NewRequest("POST", srv.URL, body) - if err != nil { - t.Fatal(err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 500 { - t.Fatalf("unexpected status code: %+v", resp.StatusCode) - } - _, err = netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } -} - -func TestHandlerWithRequestBodyReadingError(t *testing.T) { - expected := errors.New("mocked error") - handler := Handler{Config: &Config{}} - rw := NewFakeResponseWriter() - req := &http.Request{ - Method: "POST", - Header: map[string][]string{ - "Content-Type": {"application/json"}, - "Content-Length": {"2048"}, - }, - Body: &FakeBody{Err: expected}, - } - handler.ServeHTTP(rw, req) - if rw.StatusCode != 400 { - t.Fatal("unexpected status code") - } -} - -type FakeBody struct { - Err error -} - -func (fb FakeBody) Read(p []byte) (int, error) { - time.Sleep(10 * time.Microsecond) - return 0, fb.Err -} - -func (fb FakeBody) Close() error { - return nil -} - -type FakeResponseWriter struct { - Body [][]byte - HeaderMap http.Header - StatusCode int -} - -func NewFakeResponseWriter() *FakeResponseWriter { - return &FakeResponseWriter{HeaderMap: make(http.Header)} -} - -func (frw *FakeResponseWriter) Header() http.Header { - return frw.HeaderMap -} - -func (frw *FakeResponseWriter) Write(b []byte) (int, error) { - frw.Body = append(frw.Body, b) - return len(b), nil -} - -func (frw *FakeResponseWriter) WriteHeader(statusCode int) { - frw.StatusCode = statusCode -} diff --git a/internal/cmd/oohelperd/oohelperd.go b/internal/cmd/oohelperd/oohelperd.go index 8f89b96..631ce4c 100644 --- a/internal/cmd/oohelperd/oohelperd.go +++ b/internal/cmd/oohelperd/oohelperd.go @@ -10,7 +10,6 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/websteps" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx" "github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/model" @@ -59,7 +58,6 @@ func main() { func testableMain() { mux := http.NewServeMux() - mux.Handle("/api/unstable/websteps", &websteps.Handler{Config: &websteps.Config{}}) mux.Handle("/api/v1/websteps", &webstepsx.THHandler{}) mux.Handle("/", webconnectivity.Handler{ Client: httpx, diff --git a/internal/engine/experiment/websteps/control.go b/internal/engine/experiment/websteps/control.go deleted file mode 100644 index c1c3f0f..0000000 --- a/internal/engine/experiment/websteps/control.go +++ /dev/null @@ -1,26 +0,0 @@ -package websteps - -import ( - "context" - - "github.com/ooni/probe-cli/v3/internal/httpx" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// Control performs the control request and returns the response. -func Control( - ctx context.Context, sess model.ExperimentSession, - thAddr string, resourcePath string, creq CtrlRequest) (out CtrlResponse, err error) { - clnt := &httpx.APIClientTemplate{ - BaseURL: thAddr, - HTTPClient: sess.DefaultHTTPClient(), - Logger: sess.Logger(), - } - // make sure error is wrapped - err = clnt.WithBodyLogging().Build().PostJSON(ctx, resourcePath, creq, &out) - if err != nil { - err = netxlite.NewTopLevelGenericErrWrapper(err) - } - return -} diff --git a/internal/engine/experiment/websteps/dns.go b/internal/engine/experiment/websteps/dns.go deleted file mode 100644 index 650e508..0000000 --- a/internal/engine/experiment/websteps/dns.go +++ /dev/null @@ -1,30 +0,0 @@ -package websteps - -import ( - "context" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -type DNSConfig struct { - Domain string - Resolver model.Resolver -} - -// DNSDo performs the DNS check. -func DNSDo(ctx context.Context, config DNSConfig) ([]string, error) { - resolver := config.Resolver - if resolver == nil { - childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google") - runtimex.PanicOnError(err, "NewDNSClient failed") - resolver = childResolver - resolver = &netxlite.ResolverIDNA{ - Resolver: resolver, - } - } - return resolver.LookupHost(ctx, config.Domain) -} diff --git a/internal/engine/experiment/websteps/factory.go b/internal/engine/experiment/websteps/factory.go deleted file mode 100644 index 84e33a9..0000000 --- a/internal/engine/experiment/websteps/factory.go +++ /dev/null @@ -1,132 +0,0 @@ -package websteps - -import ( - "context" - "crypto/tls" - "errors" - "net" - "net/http" - "net/url" - "sync" - - "github.com/lucas-clemente/quic-go" - "github.com/lucas-clemente/quic-go/http3" - oohttp "github.com/ooni/oohttp" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -var ErrNoConnReuse = errors.New("cannot reuse connection") - -func NewRequest(ctx context.Context, URL *url.URL, headers http.Header) *http.Request { - req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil) - runtimex.PanicOnError(err, "NewRequestWithContect failed") - for k, vs := range headers { - for _, v := range vs { - req.Header.Add(k, v) - } - } - return req -} - -// NewDialerResolver contructs a new dialer for TCP connections, -// with default, errorwrapping and resolve functionalities -func NewDialerResolver(resolver model.Resolver) model.Dialer { - var d model.Dialer = netxlite.DefaultDialer - d = &netxlite.ErrorWrapperDialer{Dialer: d} - d = &netxlite.DialerResolver{ - Resolver: resolver, - Dialer: d, - } - return d -} - -// NewQUICDialerResolver creates a new QUICDialerResolver -// with default, errorwrapping and resolve functionalities -func NewQUICDialerResolver(resolver model.Resolver) model.QUICDialer { - var ql model.QUICListener = &netxlite.QUICListenerStdlib{} - ql = &netxlite.ErrorWrapperQUICListener{QUICListener: ql} - var dialer model.QUICDialer = &netxlite.QUICDialerQUICGo{ - QUICListener: ql, - } - dialer = &netxlite.ErrorWrapperQUICDialer{QUICDialer: dialer} - dialer = &netxlite.QUICDialerResolver{ - Resolver: resolver, - Dialer: dialer, - } - return dialer -} - -// NewSingleH3Transport creates an http3.RoundTripper. -func NewSingleH3Transport(qconn quic.EarlyConnection, tlscfg *tls.Config, qcfg *quic.Config) http.RoundTripper { - transport := &http3.RoundTripper{ - DisableCompression: true, - TLSClientConfig: tlscfg, - QuicConfig: qcfg, - Dial: (&SingleDialerH3{qconn: &qconn}).Dial, - } - return transport -} - -// NewSingleTransport creates a new HTTP transport with a single-use dialer. -func NewSingleTransport(conn net.Conn) http.RoundTripper { - singledialer := &SingleDialer{conn: &conn} - transport := newBaseTransport() - transport.DialContext = singledialer.DialContext - transport.DialTLSContext = singledialer.DialContext - return transport -} - -// NewSingleTransport creates a new HTTP transport with a custom dialer and handshaker. -func NewTransportWithDialer(dialer model.Dialer, tlsConfig *tls.Config, handshaker model.TLSHandshaker) http.RoundTripper { - transport := newBaseTransport() - transport.DialContext = dialer.DialContext - transport.DialTLSContext = (&netxlite.TLSDialerLegacy{ - Config: tlsConfig, - Dialer: dialer, - TLSHandshaker: handshaker, - }).DialTLSContext - return transport -} - -// newBaseTransport creates a new HTTP transport with the default dialer. -func newBaseTransport() (transport *oohttp.StdlibTransport) { - base := oohttp.DefaultTransport.(*oohttp.Transport).Clone() - base.DisableCompression = true - base.MaxConnsPerHost = 1 - transport = &oohttp.StdlibTransport{Transport: base} - return transport -} - -type SingleDialer struct { - sync.Mutex - conn *net.Conn -} - -func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { - s.Lock() - defer s.Unlock() - if s.conn == nil { - return nil, ErrNoConnReuse - } - c := s.conn - s.conn = nil - return *c, nil -} - -type SingleDialerH3 struct { - sync.Mutex - qconn *quic.EarlyConnection -} - -func (s *SingleDialerH3) Dial(ctx context.Context, network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) { - s.Lock() - defer s.Unlock() - if s.qconn == nil { - return nil, ErrNoConnReuse - } - qs := s.qconn - s.qconn = nil - return *qs, nil -} diff --git a/internal/engine/experiment/websteps/http.go b/internal/engine/experiment/websteps/http.go deleted file mode 100644 index 3b1ad34..0000000 --- a/internal/engine/experiment/websteps/http.go +++ /dev/null @@ -1,34 +0,0 @@ -package websteps - -import ( - "net/http" - - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// HTTPDo performs the HTTP check. -// Input: -// req *http.Request -// The same request than the one used by the Explore step. -// This means that req contains the headers set by the original CtrlRequest, as well as, -// in case of a redirect chain, additional headers that were added due to redirects -// transport http.RoundTripper: -// The transport to use, either http.Transport, or http3.RoundTripper. -func HTTPDo(req *http.Request, transport http.RoundTripper) (*http.Response, []byte, error) { - clnt := http.Client{ - CheckRedirect: func(r *http.Request, reqs []*http.Request) error { - return http.ErrUseLastResponse - }, - Transport: transport, - } - resp, err := clnt.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - body, err := netxlite.ReadAllContext(req.Context(), resp.Body) - if err != nil { - return resp, nil, nil - } - return resp, body, nil -} diff --git a/internal/engine/experiment/websteps/model.go b/internal/engine/experiment/websteps/model.go deleted file mode 100644 index 926f87f..0000000 --- a/internal/engine/experiment/websteps/model.go +++ /dev/null @@ -1,155 +0,0 @@ -package websteps - -import "net/http" - -// Websteps test helper spec messages: - -// CtrlRequest is the request sent by the probe to the test helper. -type CtrlRequest struct { - // URL is the mandatory URL to measure. - URL string `json:"url"` - - // Headers contains optional headers. - Headers map[string][]string `json:"headers"` - - // Addrs contains the optional IP addresses resolved by the - // probe for the domain inside URL. - Addrs []string `json:"addrs"` -} - -// CtrlResponse is the response from the test helper. -type CtrlResponse struct { - // URLs contains the URLs we should measure. These URLs - // derive from CtrlRequest.URL. - URLs []*URLMeasurement `json:"urls"` -} - -// URLMeasurement contains all the URLs measured by the test helper. -type URLMeasurement struct { - // URL is the URL to which this measurement refers. - URL string `json:"url"` - - // DNS contains the domain names resolved by the test helper. - DNS *DNSMeasurement `json:"dns"` - - // Endpoints contains endpoint measurements. - Endpoints []*EndpointMeasurement `json:"endpoints"` - - // RoundTrip is the related round trip. This field MUST NOT be - // exported as JSON, since it's only used internally by the test - // helper and it's completely ignored by the probe. - RoundTrip *RoundTripInfo `json:"-"` -} - -// RoundTripInfo contains info on a specific round trip. This data -// structure is not part of the test helper protocol. We use it -// _inside_ the test helper to describe the discovery phase where -// we gather all the URLs that can derive from a given URL. -type RoundTripInfo struct { - // Proto is the protocol used, it can be "h2", "http/1.1", "h3". - Proto string - - // Request is the original HTTP request. Headers also include cookies. - Request *http.Request - - // Response is the HTTP response. - Response *http.Response - - // SortIndex is the index using for sorting round trips. - SortIndex int -} - -// DNSMeasurement is a DNS measurement. -type DNSMeasurement struct { - // Domain is the domain we wanted to resolve. - Domain string `json:"domain"` - - // Failure is the error that occurred. - Failure *string `json:"failure"` - - // Addrs contains the resolved addresses. - Addrs []string `json:"addrs"` -} - -// EndpointMeasurement is an HTTP measurement where we are using -// a specific TCP/TLS/QUIC endpoint to get the URL. -// -// The specification describes this data structure as the sum of -// three distinct types: HTTPEndpointMeasurement for "http", -// HTTPSEndpointMeasurement for "https", and H3EndpointMeasurement -// for "h3". We don't have sum types here, therefore we use the -// Protocol field to indicate which fields are meaningful. -type EndpointMeasurement struct { - // Endpoint is the endpoint we're measuring. - Endpoint string `json:"endpoint"` - - // Protocol is one of "http", "https", and "h3". - Protocol string `json:"protocol"` - - // TCPConnect is the TCP connect measurement. This field - // is only meaningful when protocol is "http" or "https." - TCPConnect *TCPConnectMeasurement `json:"tcp_connect"` - - // QUICHandshake is the QUIC handshake measurement. This field - // is only meaningful when the protocol is "h3". - QUICHandshake *QUICHandshakeMeasurement `json:"quic_handshake"` - - // TLSHandshake is the TLS handshake measurement. This field - // is only meaningful when the protocol is "https". - TLSHandshake *TLSHandshakeMeasurement `json:"tls_handshake"` - - // HTTPRoundTrip is the related HTTP GET measurement. - HTTPRoundTrip *HTTPRoundTripMeasurement `json:"http_round_trip"` -} - -// TCPConnectMeasurement is a TCP connect measurement. -type TCPConnectMeasurement struct { - // Failure is the error that occurred. - Failure *string `json:"failure"` -} - -// TLSHandshakeMeasurement is a TLS handshake measurement. -type TLSHandshakeMeasurement struct { - // Failure is the error that occurred. - Failure *string `json:"failure"` -} - -// QUICHandshakeMeasurement is a QUIC handshake measurement. -type QUICHandshakeMeasurement = TLSHandshakeMeasurement - -// HTTPRoundTripMeasurement contains a measured HTTP request and -// the corresponding response. -type HTTPRoundTripMeasurement struct { - // Request contains request data. - Request *HTTPRequestMeasurement `json:"request"` - - // Response contains response data. - Response *HTTPResponseMeasurement `json:"response"` -} - -// HTTPRequestMeasurement contains request data. -type HTTPRequestMeasurement struct { - // Method is the request method. - Method string `json:"method"` - - // URL is the request URL. - URL string `json:"url"` - - // Headers contains request headers. - Headers http.Header `json:"headers"` -} - -// HTTPResponseMeasurement contains response data. -type HTTPResponseMeasurement struct { - // BodyLength contains the body length in bytes. - BodyLength int64 `json:"body_length"` - - // Failure is the error that occurred. - Failure *string `json:"failure"` - - // Headers contains response headers. - Headers http.Header `json:"headers"` - - // StatusCode is the response status code. - StatusCode int64 `json:"status_code"` -} diff --git a/internal/engine/experiment/websteps/quic.go b/internal/engine/experiment/websteps/quic.go deleted file mode 100644 index ef62dab..0000000 --- a/internal/engine/experiment/websteps/quic.go +++ /dev/null @@ -1,30 +0,0 @@ -package websteps - -import ( - "context" - "crypto/tls" - - "github.com/lucas-clemente/quic-go" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -type QUICConfig struct { - Endpoint string - QUICDialer model.QUICDialer - Resolver model.Resolver - TLSConf *tls.Config -} - -// QUICDo performs the QUIC check. -func QUICDo(ctx context.Context, config QUICConfig) (quic.EarlyConnection, error) { - if config.QUICDialer != nil { - return config.QUICDialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{}) - } - resolver := config.Resolver - if resolver == nil { - resolver = &netxlite.ResolverSystem{} - } - dialer := NewQUICDialerResolver(resolver) - return dialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{}) -} diff --git a/internal/engine/experiment/websteps/tcp.go b/internal/engine/experiment/websteps/tcp.go deleted file mode 100644 index a0bf7e5..0000000 --- a/internal/engine/experiment/websteps/tcp.go +++ /dev/null @@ -1,28 +0,0 @@ -package websteps - -import ( - "context" - "net" - - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -type TCPConfig struct { - Dialer model.Dialer - Endpoint string - Resolver model.Resolver -} - -// TCPDo performs the TCP check. -func TCPDo(ctx context.Context, config TCPConfig) (net.Conn, error) { - if config.Dialer != nil { - return config.Dialer.DialContext(ctx, "tcp", config.Endpoint) - } - resolver := config.Resolver - if resolver == nil { - resolver = &netxlite.ResolverSystem{} - } - dialer := NewDialerResolver(resolver) - return dialer.DialContext(ctx, "tcp", config.Endpoint) -} diff --git a/internal/engine/experiment/websteps/tls.go b/internal/engine/experiment/websteps/tls.go deleted file mode 100644 index 911c725..0000000 --- a/internal/engine/experiment/websteps/tls.go +++ /dev/null @@ -1,23 +0,0 @@ -package websteps - -import ( - "context" - "crypto/tls" - "net" - - "github.com/ooni/probe-cli/v3/internal/netxlite" - utls "gitlab.com/yawning/utls.git" -) - -// TLSDo performs the TLS check. -func TLSDo(ctx context.Context, conn net.Conn, hostname string) (net.Conn, error) { - tlsConf := &tls.Config{ - ServerName: hostname, - NextProtos: []string{"h2", "http/1.1"}, - } - h := &netxlite.TLSHandshakerConfigurable{ - NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto), - } - tlsConn, _, err := h.Handshake(ctx, conn, tlsConf) - return tlsConn, err -} diff --git a/internal/engine/experiment/websteps/websteps.go b/internal/engine/experiment/websteps/websteps.go deleted file mode 100644 index 49bd988..0000000 --- a/internal/engine/experiment/websteps/websteps.go +++ /dev/null @@ -1,369 +0,0 @@ -// Package websteps implements the websteps experiment. -// -// Specifications: -// -// - test helper: https://github.com/ooni/spec/blob/master/backends/th-007-websteps.md -// -// - experiment: N/A. -// -// We are currently implementing: -// -// - version 202108.17.1114 of the test helper spec. -// -// - version N/A of the experiment spec. -package websteps - -import ( - "context" - "crypto/tls" - "errors" - "net" - "net/http" - "net/url" - "time" - - "github.com/lucas-clemente/quic-go" - "github.com/ooni/probe-cli/v3/internal/engine/httpheader" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -const ( - testName = "websteps" - testVersion = "0.0.1" -) - -// Config contains the experiment config. -type Config struct{} - -// TestKeys contains webconnectivity test keys. -type TestKeys struct { - Agent string `json:"agent"` - ClientResolver string `json:"client_resolver"` - URLMeasurements []*URLMeasurement -} - -// Measurer performs the measurement. -type Measurer struct { - Config Config -} - -// NewExperimentMeasurer creates a new ExperimentMeasurer. -func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { - return Measurer{Config: config} -} - -// ExperimentName implements ExperimentMeasurer.ExperExperimentName. -func (m Measurer) ExperimentName() string { - return testName -} - -// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion. -func (m Measurer) ExperimentVersion() string { - return testVersion -} - -// SupportedQUICVersions are the H3 over QUIC versions we currently support -var SupportedQUICVersions = map[string]bool{ - "h3": true, -} - -var ( - // ErrNoAvailableTestHelpers is emitted when there are no available test helpers. - ErrNoAvailableTestHelpers = errors.New("no available helpers") - - // ErrNoInput indicates that no input was provided - ErrNoInput = errors.New("no input provided") - - // ErrInputIsNotAnURL indicates that the input is not an URL. - ErrInputIsNotAnURL = errors.New("input is not an URL") - - // ErrUnsupportedInput indicates that the input URL scheme is unsupported. - ErrUnsupportedInput = errors.New("unsupported input scheme") -) - -// Run implements ExperimentMeasurer.Run. -func (m Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { - ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - tk := new(TestKeys) - measurement.TestKeys = tk - tk.Agent = "redirect" - tk.ClientResolver = sess.ResolverIP() - - // 1. Parse and verify URL - URL, err := url.Parse(string(measurement.Input)) - if err != nil { - return ErrInputIsNotAnURL - } - if URL.Scheme != "http" && URL.Scheme != "https" { - return ErrUnsupportedInput - } - // 2. Perform the initial DNS lookup step - addrs, err := DNSDo(ctx, DNSConfig{Domain: URL.Hostname()}) - endpoints := makeEndpoints(addrs, URL) - // 3. Find the testhelper - // TODO(kelmenhorst,bassosimone): this is not used at the moment, but the hardcoded local address - testhelpers, _ := sess.GetTestHelpersByName("web-connectivity") - var testhelper *model.OOAPIService - for _, th := range testhelpers { - if th.Type == "https" { - testhelper = &th - break - } - } - if testhelper == nil { - return ErrNoAvailableTestHelpers - } - measurement.TestHelpers = map[string]interface{}{ - "backend": testhelper, - } - // 4. Query the testhelper - // TODO(kelmenhorst,bassosimone): remove hardcoded version here, this is only for testing purposes - resp, err := Control(ctx, sess, "http://localhost:8080", "/api/unstable/websteps", CtrlRequest{ - URL: URL.String(), - Headers: map[string][]string{ - "Accept": {httpheader.Accept()}, - "Accept-Language": {httpheader.AcceptLanguage()}, - "User-Agent": {httpheader.UserAgent()}, - }, - Addrs: endpoints, - }) - if err != nil || resp.URLs == nil { - return errors.New("no control response") - } - // 5. Go over the Control URL measurements and reproduce them without following redirects, one by one. - for _, controlURLMeasurement := range resp.URLs { - urlMeasurement := &URLMeasurement{ - URL: controlURLMeasurement.URL, - Endpoints: []*EndpointMeasurement{}, - } - URL, err = url.Parse(controlURLMeasurement.URL) - runtimex.PanicOnError(err, "url.Parse failed") - // DNS step - addrs, err = DNSDo(ctx, DNSConfig{Domain: URL.Hostname()}) - urlMeasurement.DNS = &DNSMeasurement{ - Domain: URL.Hostname(), - Addrs: addrs, - Failure: archival.NewFailure(err), - } - if controlURLMeasurement.Endpoints == nil { - tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement) - continue - } - // the testhelper tells us which endpoints to measure - for _, controlEndpoint := range controlURLMeasurement.Endpoints { - rt := controlEndpoint.HTTPRoundTrip - if rt == nil || rt.Request == nil { - continue - } - var endpointMeasurement *EndpointMeasurement - proto := controlEndpoint.Protocol - _, h3 := SupportedQUICVersions[proto] - switch { - case h3: - endpointMeasurement = m.measureEndpointH3(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers, proto) - case proto == "http": - endpointMeasurement = m.measureEndpointHTTP(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers) - case proto == "https": - endpointMeasurement = m.measureEndpointHTTPS(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers) - default: - panic("should not happen") - } - urlMeasurement.Endpoints = append(urlMeasurement.Endpoints, endpointMeasurement) - } - tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement) - } - return nil -} - -func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement { - endpointMeasurement := &EndpointMeasurement{ - Endpoint: endpoint, - Protocol: "http", - } - // TCP connect step - conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint}) - endpointMeasurement.TCPConnect = &TCPConnectMeasurement{ - Failure: archival.NewFailure(err), - } - if err != nil { - return endpointMeasurement - } - defer conn.Close() - - // HTTP roundtrip step - request := NewRequest(ctx, URL, headers) - endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{ - Request: &HTTPRequestMeasurement{ - Headers: request.Header, - Method: "GET", - URL: URL.String(), - }, - } - transport := NewSingleTransport(conn) - resp, body, err := HTTPDo(request, transport) - if err != nil { - // failed Response - endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - Failure: archival.NewFailure(err), - } - return endpointMeasurement - } - // successful Response - endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - BodyLength: int64(len(body)), - Failure: nil, - Headers: resp.Header, - StatusCode: int64(resp.StatusCode), - } - return endpointMeasurement -} - -func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement { - endpointMeasurement := &EndpointMeasurement{ - Endpoint: endpoint, - Protocol: "https", - } - // TCP connect step - conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint}) - endpointMeasurement.TCPConnect = &TCPConnectMeasurement{ - Failure: archival.NewFailure(err), - } - if err != nil { - return endpointMeasurement - } - defer conn.Close() - - // TLS handshake step - tlsconn, err := TLSDo(ctx, conn, URL.Hostname()) - endpointMeasurement.TLSHandshake = &TLSHandshakeMeasurement{ - Failure: archival.NewFailure(err), - } - if err != nil { - return endpointMeasurement - } - defer tlsconn.Close() - - // HTTP roundtrip step - request := NewRequest(ctx, URL, headers) - endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{ - Request: &HTTPRequestMeasurement{ - Headers: request.Header, - Method: "GET", - URL: URL.String(), - }, - } - transport := NewSingleTransport(tlsconn) - resp, body, err := HTTPDo(request, transport) - if err != nil { - // failed Response - endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - Failure: archival.NewFailure(err), - } - return endpointMeasurement - } - // successful Response - endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - BodyLength: int64(len(body)), - Failure: nil, - Headers: resp.Header, - StatusCode: int64(resp.StatusCode), - } - return endpointMeasurement -} - -func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint string, headers http.Header, proto string) *EndpointMeasurement { - endpointMeasurement := &EndpointMeasurement{ - Endpoint: endpoint, - Protocol: proto, - } - tlsConf := &tls.Config{ - ServerName: URL.Hostname(), - NextProtos: []string{proto}, - } - // QUIC handshake step - sess, err := QUICDo(ctx, QUICConfig{ - Endpoint: endpoint, - TLSConf: tlsConf, - }) - endpointMeasurement.QUICHandshake = &TLSHandshakeMeasurement{ - Failure: archival.NewFailure(err), - } - if err != nil { - return endpointMeasurement - } - // HTTP roundtrip step - request := NewRequest(ctx, URL, headers) - endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{ - Request: &HTTPRequestMeasurement{ - Headers: request.Header, - Method: "GET", - URL: URL.String(), - }, - } - transport := NewSingleH3Transport(sess, tlsConf, &quic.Config{}) - resp, body, err := HTTPDo(request, transport) - if err != nil { - // failed Response - endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - Failure: archival.NewFailure(err), - } - return endpointMeasurement - } - // successful Response - endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{ - BodyLength: int64(len(body)), - Failure: nil, - Headers: resp.Header, - StatusCode: int64(resp.StatusCode), - } - return endpointMeasurement - -} - -// SummaryKeys contains summary keys for this experiment. -// -// Note that this structure is part of the ABI contract with ooniprobe -// therefore we should be careful when changing it. -type SummaryKeys struct { - Accessible bool `json:"accessible"` - Blocking string `json:"blocking"` - IsAnomaly bool `json:"-"` -} - -// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. -func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { - sk := SummaryKeys{} - return sk, nil -} - -func makeEndpoints(addrs []string, URL *url.URL) []string { - endpoints := []string{} - if addrs == nil { - return endpoints - } - for _, addr := range addrs { - var port string - explicitPort := URL.Port() - scheme := URL.Scheme - switch { - case explicitPort != "": - port = explicitPort - case scheme == "http": - port = "80" - case scheme == "https": - port = "443" - default: - panic("should not happen") - } - endpoints = append(endpoints, net.JoinHostPort(addr, port)) - } - return endpoints -}