c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cleanup: remove websteps summer 2021 implementation (#722)

Browse Source 
See https://github.com/ooni/probe/issues/2094
This commit is contained in:
Simone Basso 2022-05-13 15:06:03 +02:00 committed by GitHub
parent e93756be20
commit 1776ea1288
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 0 additions and 2709 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
internal/cmd/oohelperd/internal/websteps/explore.go
View File

@ -1,175 +0,0 @@
package websteps
import (
"crypto/tls"
"net/http"
"net/http/cookiejar"
"net/url"
"sort"
"strings"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
utls "gitlab.com/yawning/utls.git"
)
// Explore is the second step of the test helper algorithm. Its objective
// is to enumerate all the URLs we can discover by redirection from
// the original URL in the test list. Because the test list contains by
// definition noisy data, we need this preprocessing step to learn all
// the URLs that are actually implied by the original URL.
// Explorer is the interface responsible for running Explore.
type Explorer interface {
Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error)
}
// DefaultExplorer is the default Explorer.
type DefaultExplorer struct {
resolver model.Resolver
}
// Explore returns a list of round trips sorted so that the first
// round trip is the first element in the list, and so on.
// Explore uses the URL and the optional headers provided by the CtrlRequest.
func (e *DefaultExplorer) Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) {
resp, err := e.get(URL, headers)
if err != nil {
return nil, err
}
rts := e.rearrange(resp, nil)
h3URL, err := getH3URL(resp)
if err != nil {
// If we cannot find the HTTP/3 URL for subsequent measurements, we just continue
// the measurement using the URLs we have found so far.
return rts, nil
}
resp, err = e.getH3(h3URL, headers)
if err != nil {
// If we cannot follow the HTTP/3 chain, we just continue
// the measurement using the URLs we have found so far.
return rts, nil
}
rts = append(rts, e.rearrange(resp, h3URL)...)
return rts, nil
}
// rearrange takes in input the final response of an HTTP transaction and an optional h3URL
// (which is needed to derive the type of h3 protocol, typically h3),
// and produces in output a list of round trips sorted
// such that the first round trip is the first element in the out array.
func (e *DefaultExplorer) rearrange(resp *http.Response, h3URL *h3URL) (out []*RoundTrip) {
index := 0
for resp != nil && resp.Request != nil {
proto := resp.Request.URL.Scheme
if h3URL != nil {
proto = h3URL.proto
}
out = append(out, &RoundTrip{
Proto: proto,
SortIndex: index,
Request: resp.Request,
Response: resp,
})
index += 1
resp = resp.Request.Response
}
sh := &sortHelper{out}
sort.Sort(sh)
return
}
// sortHelper is the helper structure to sort round trips.
type sortHelper struct {
v []*RoundTrip
}
// Len implements sort.Interface.Len.
func (sh *sortHelper) Len() int {
return len(sh.v)
}
// Less implements sort.Interface.Less.
func (sh *sortHelper) Less(i, j int) bool {
return sh.v[i].SortIndex >= sh.v[j].SortIndex
}
// Swap implements sort.Interface.Swap.
func (sh *sortHelper) Swap(i, j int) {
sh.v[i], sh.v[j] = sh.v[j], sh.v[i]
}
// get gets the given URL and returns the final response after
// redirection, and an error. If the error is nil, the final response is valid.
func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http.Response, error) {
tlsConf := &tls.Config{
NextProtos: []string{"h2", "http/1.1"},
}
handshaker := &netxlite.TLSHandshakerConfigurable{
NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto),
}
transport := websteps.NewTransportWithDialer(websteps.NewDialerResolver(e.resolver), tlsConf, handshaker)
// TODO(bassosimone): here we should use runtimex.PanicOnError
jarjar, _ := cookiejar.New(nil)
clnt := &http.Client{
Transport: transport,
Jar: jarjar,
}
// TODO(bassosimone): document why e.newRequest cannot fail.
req, err := e.newRequest(URL, headers)
runtimex.PanicOnError(err, "newRequest failed")
resp, err := clnt.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// Note that we ignore the response body.
return resp, nil
}
// getH3 uses HTTP/3 to get the given URL and returns the final
// response after redirection, and an error. If the error is nil, the final response is valid.
func (e *DefaultExplorer) getH3(h3URL *h3URL, headers map[string][]string) (*http.Response, error) {
dialer := websteps.NewQUICDialerResolver(e.resolver)
tlsConf := &tls.Config{
NextProtos: []string{h3URL.proto},
}
// Rationale for using log.Log here: we're already using log.Log
// in this package, so it seems fair to use it also here
transport := netxlite.NewHTTP3Transport(log.Log, dialer, tlsConf)
// TODO(bassosimone): here we should use runtimex.PanicOnError
jarjar, _ := cookiejar.New(nil)
clnt := &http.Client{
Transport: transport,
Jar: jarjar,
}
// TODO(bassosimone): document why e.newRequest cannot fail.
req, err := e.newRequest(h3URL.URL, headers)
runtimex.PanicOnError(err, "newRequest failed")
resp, err := clnt.Do(req)
if err != nil {
return nil, err
}
// Note that we ignore the response body.
defer resp.Body.Close()
return resp, nil
}
func (e *DefaultExplorer) newRequest(URL *url.URL, headers map[string][]string) (*http.Request, error) {
req, err := http.NewRequest("GET", URL.String(), nil)
if err != nil {
return nil, err
}
for k, vs := range headers {
switch strings.ToLower(k) {
case "user-agent", "accept", "accept-language":
for _, v := range vs {
req.Header.Add(k, v)
}
}
}
return req, nil
}

160
internal/cmd/oohelperd/internal/websteps/explore_test.go
View File

@ -1,160 +0,0 @@
package websteps
import (
"net/http"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/netxlite/quictesting"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
var explorer = &DefaultExplorer{resolver: newResolver()}
func TestExploreSuccess(t *testing.T) {
u, err := url.Parse("https://example.com")
runtimex.PanicOnError(err, "url.Parse failed")
rts, err := explorer.Explore(u, nil)
if err != nil {
t.Fatal("unexpected error")
}
if len(rts) != 1 {
t.Fatal("unexpected number of roundtrips")
}
}
func TestExploreFailure(t *testing.T) {
u, err := url.Parse("https://example.example")
runtimex.PanicOnError(err, "url.Parse failed")
rts, err := explorer.Explore(u, nil)
if err == nil {
t.Fatal("expected an error here")
}
if rts != nil {
t.Fatal("rts should be nil")
}
}
func TestExploreSuccessWithH3(t *testing.T) {
// TODO(bassosimone): figure out why this happens.
t.Skip("this test does not work in GHA")
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rts, err := explorer.Explore(u, nil)
if err != nil {
t.Fatal("unexpected error")
}
if len(rts) != 2 {
t.Fatal("unexpected number of roundtrips")
}
if rts[0].Proto != "https" {
t.Fatal("unexpected protocol")
}
if rts[1].Proto != "h3" {
t.Fatal("unexpected protocol")
}
}
func TestGetSuccess(t *testing.T) {
u, err := url.Parse("https://example.com")
runtimex.PanicOnError(err, "url.Parse failed for clearly good URL")
resp, err := explorer.get(u, nil)
if err != nil {
t.Fatal("unexpected error")
}
if resp == nil {
t.Fatal("unexpected nil response")
}
buf := make([]byte, 100)
if n, _ := resp.Body.Read(buf); n != 0 {
t.Fatal("expected response body tom be closed")
}
}
func TestGetFailure(t *testing.T) {
u, err := url.Parse("https://example.example")
runtimex.PanicOnError(err, "url.Parse failed for clearly good URL")
resp, err := explorer.get(u, nil)
if err == nil {
t.Fatal("expected an error here")
}
if resp != nil {
t.Fatal("response should be nil")
}
}
func TestGetH3Success(t *testing.T) {
u := &url.URL{Scheme: "https", Host: quictesting.Domain, Path: "/"}
h3u := &h3URL{URL: u, proto: "h3"}
resp, err := explorer.getH3(h3u, nil)
if err != nil {
t.Fatal("unexpected error", err)
}
if resp == nil {
t.Fatal("unexpected nil response")
}
buf := make([]byte, 100)
if n, _ := resp.Body.Read(buf); n != 0 {
t.Fatal("expected response body tom be closed")
}
}
func TestGetH3Failure(t *testing.T) {
u, err := url.Parse("https://www.google.google")
runtimex.PanicOnError(err, "url.Parse failed for clearly good URL")
h3u := &h3URL{URL: u, proto: "h3"}
resp, err := explorer.getH3(h3u, nil)
if err == nil {
t.Fatal("expected an error here")
}
if resp != nil {
t.Fatal("response should be nil")
}
}
func TestRearrange(t *testing.T) {
u, err := url.Parse("https://example.com")
runtimex.PanicOnError(err, "url.Parse failed")
resp := &http.Response{
// the ProtoMajor field identifies the request/response structs and indicates the correct order
ProtoMajor: 2,
Request: &http.Request{
ProtoMajor: 2,
URL: u,
Response: &http.Response{
ProtoMajor: 1,
Request: &http.Request{
ProtoMajor: 1,
URL: u,
Response: &http.Response{
ProtoMajor: 0,
Request: &http.Request{
ProtoMajor: 0,
URL: u,
},
},
},
},
},
}
h3URL := &h3URL{URL: u, proto: "expected"}
rts := explorer.rearrange(resp, h3URL)
expectedIndex := 0
for _, rt := range rts {
if rt.Request == nil || rt.Response == nil {
t.Fatal("unexpected nil value")
}
if rt.Request.ProtoMajor != expectedIndex {
t.Fatal("unexpected order")
}
if rt.Response.ProtoMajor != expectedIndex {
t.Fatal("unexpected order")
}
if rt.Proto != h3URL.proto {
t.Fatal("unexpected protocol")
}
expectedIndex += 1
}
}

287
internal/cmd/oohelperd/internal/websteps/generate.go
View File

@ -1,287 +0,0 @@
package websteps
import (
"context"
"crypto/tls"
"net"
"net/http"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/model"
)
// Generate is the third step of the algorithm. Given the
// observed round trips, we generate measurement targets and
// execute those measurements so the probe has a benchmark.
// Generator is the interface responsible for running Generate.
type Generator interface {
Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error)
}
// DefaultGenerator is the default Generator.
type DefaultGenerator struct {
dialer model.Dialer
quicDialer model.QUICDialer
resolver model.Resolver
transport http.RoundTripper
}
// the testhelper uses the same network operations as websteps
var (
DNSDo = websteps.DNSDo
TCPDo = websteps.TCPDo
QUICDo = websteps.QUICDo
TLSDo = websteps.TLSDo
HTTPDo = websteps.HTTPDo
)
// Generate takes in input a list of round trips and outputs
// a list of connectivity measurements for each of them.
func (g *DefaultGenerator) Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) {
var out []*URLMeasurement
for _, rt := range rts {
currentURL := g.GenerateURL(ctx, rt, clientResolutions)
out = append(out, currentURL)
}
return out, nil
}
// GenerateURL returns a URLMeasurement.
func (g *DefaultGenerator) GenerateURL(ctx context.Context, rt *RoundTrip, clientResolutions []string) *URLMeasurement {
addrs, err := DNSDo(ctx, websteps.DNSConfig{
Domain: rt.Request.URL.Hostname(),
Resolver: g.resolver,
})
currentURL := &URLMeasurement{
DNS: &DNSMeasurement{
Domain: rt.Request.URL.Hostname(),
Addrs: addrs,
Failure: newfailure(err),
},
RoundTrip: rt,
URL: rt.Request.URL.String(),
}
addrs = g.mergeAddresses(addrs, clientResolutions)
if len(addrs) == 0 {
return currentURL
}
for _, addr := range addrs {
var port string
explicitPort := rt.Request.URL.Port()
scheme := rt.Request.URL.Scheme
switch {
case explicitPort != "":
port = explicitPort
case scheme == "http":
port = "80"
case scheme == "https":
port = "443"
default:
panic("should not happen")
}
endpoint := net.JoinHostPort(addr, port)
var currentEndpoint *EndpointMeasurement
_, h3 := websteps.SupportedQUICVersions[rt.Proto]
switch {
case h3:
currentEndpoint = g.GenerateH3Endpoint(ctx, rt, endpoint)
case rt.Proto == "http":
currentEndpoint = g.GenerateHTTPEndpoint(ctx, rt, endpoint)
case rt.Proto == "https":
currentEndpoint = g.GenerateHTTPSEndpoint(ctx, rt, endpoint)
default:
// TODO(kelmenhorst): do we have to register this error somewhere in the result struct?
continue
}
currentURL.Endpoints = append(currentURL.Endpoints, currentEndpoint)
}
return currentURL
}
// GenerateHTTPEndpoint performs an HTTP Request by
// a) establishing a TCP connection to the target (TCPDo),
// b) performing an HTTP GET request to the endpoint (HTTPDo).
// It returns an EndpointMeasurement.
func (g *DefaultGenerator) GenerateHTTPEndpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement {
currentEndpoint := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: "http",
}
tcpConn, err := TCPDo(ctx, websteps.TCPConfig{
Dialer: g.dialer,
Endpoint: endpoint,
Resolver: g.resolver,
})
currentEndpoint.TCPConnect = &TCPConnectMeasurement{
Failure: newfailure(err),
}
if err != nil {
return currentEndpoint
}
defer tcpConn.Close()
// prepare HTTPRoundTripMeasurement of this endpoint
currentEndpoint.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: rt.Request.Header,
Method: "GET",
URL: rt.Request.URL.String(),
},
}
transport := websteps.NewSingleTransport(tcpConn)
if g.transport != nil {
transport = g.transport
}
resp, body, err := HTTPDo(rt.Request, transport)
if err != nil {
// failed Response
currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: newfailure(err),
}
return currentEndpoint
}
// successful Response
currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return currentEndpoint
}
// GenerateHTTPSEndpoint performs an HTTPS Request by
// a) establishing a TCP connection to the target (TCPDo),
// b) establishing a TLS connection to the target (TLSDo),
// c) performing an HTTP GET request to the endpoint (HTTPDo).
// It returns an EndpointMeasurement.
func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement {
currentEndpoint := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: "https",
}
var tcpConn, tlsConn net.Conn
tcpConn, err := TCPDo(ctx, websteps.TCPConfig{
Dialer: g.dialer,
Endpoint: endpoint,
Resolver: g.resolver,
})
currentEndpoint.TCPConnect = &TCPConnectMeasurement{
Failure: newfailure(err),
}
if err != nil {
return currentEndpoint
}
defer tcpConn.Close()
tlsConn, err = TLSDo(ctx, tcpConn, rt.Request.URL.Hostname())
currentEndpoint.TLSHandshake = &TLSHandshakeMeasurement{
Failure: newfailure(err),
}
if err != nil {
return currentEndpoint
}
defer tlsConn.Close()
// prepare HTTPRoundTripMeasurement of this endpoint
currentEndpoint.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: rt.Request.Header,
Method: "GET",
URL: rt.Request.URL.String(),
},
}
transport := websteps.NewSingleTransport(tlsConn)
if g.transport != nil {
transport = g.transport
}
resp, body, err := HTTPDo(rt.Request, transport)
if err != nil {
// failed Response
currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: newfailure(err),
}
return currentEndpoint
}
// successful Response
currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return currentEndpoint
}
// GenerateH3Endpoint performs an HTTP/3 Request by
// a) establishing a QUIC connection to the target (QUICDo),
// b) performing an HTTP GET request to the endpoint (HTTPDo).
// It returns an EndpointMeasurement.
func (g *DefaultGenerator) GenerateH3Endpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement {
currentEndpoint := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: rt.Proto,
}
tlsConf := &tls.Config{
ServerName: rt.Request.URL.Hostname(),
NextProtos: []string{rt.Proto},
}
sess, err := QUICDo(ctx, websteps.QUICConfig{
Endpoint: endpoint,
QUICDialer: g.quicDialer,
TLSConf: tlsConf,
Resolver: g.resolver,
})
currentEndpoint.QUICHandshake = &TLSHandshakeMeasurement{
Failure: newfailure(err),
}
if err != nil {
return currentEndpoint
}
// prepare HTTPRoundTripMeasurement of this endpoint
currentEndpoint.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: rt.Request.Header,
Method: "GET",
URL: rt.Request.URL.String(),
},
}
var transport http.RoundTripper = websteps.NewSingleH3Transport(sess, tlsConf, &quic.Config{})
if g.transport != nil {
transport = g.transport
}
resp, body, err := HTTPDo(rt.Request, transport)
if err != nil {
// failed Response
currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: newfailure(err),
}
return currentEndpoint
}
// successful Response
currentEndpoint.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return currentEndpoint
}
// mergeAddresses creates a (duplicate-free) union set of the IP addresses provided by the client,
// and the addresses resulting from the testhelper's DNS step
func (g *DefaultGenerator) mergeAddresses(addrs []string, clientAddrs []string) (out []string) {
unique := make(map[string]bool, len(addrs)+len(clientAddrs))
for _, a := range addrs {
unique[a] = true
}
for _, a := range clientAddrs {
unique[a] = true
}
for key := range unique {
out = append(out, key)
}
return out
}

470
internal/cmd/oohelperd/internal/websteps/generate_test.go
View File

@ -1,470 +0,0 @@
package websteps
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/url"
"strings"
"testing"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/netxlite/quictesting"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
var generator = &DefaultGenerator{resolver: newResolver()}
type fakeTransport struct {
err error
}
func (txp fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return nil, txp.err
}
func (txp fakeTransport) CloseIdleConnections() {}
type fakeQUICDialer struct {
err error
}
func (d fakeQUICDialer) DialContext(ctx context.Context, network, address string,
tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) {
return nil, d.err
}
func (d fakeQUICDialer) CloseIdleConnections() {}
type fakeDialer struct {
err error
}
func (d fakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
return nil, d.err
}
func (d fakeDialer) CloseIdleConnections() {}
func TestGenerateDNSFailure(t *testing.T) {
u, err := url.Parse("https://www.google.google")
runtimex.PanicOnError(err, "url.Parse failed")
rts := []*RoundTrip{
{
Proto: "https",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
}
urlMeasurements, err := generator.Generate(context.Background(), rts, []string{})
if err != nil {
t.Fatal("unexpected error")
}
if len(urlMeasurements) != 1 {
t.Fatal("unexpected urlMeasurements length")
}
if urlMeasurements[0].DNS == nil {
t.Fatal("DNS should not be nil")
}
if urlMeasurements[0].DNS.Failure == nil || *urlMeasurements[0].DNS.Failure != netxlite.FailureDNSNXDOMAINError {
t.Fatal("unexpected DNS failure type")
}
}
func TestGenerate(t *testing.T) {
u, err := url.Parse("http://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed for clearly good URL")
u2, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed for clearly good URL")
rts := []*RoundTrip{
{
Proto: "http",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
{
Proto: "https",
Request: &http.Request{
URL: u2,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
{
Proto: "h3",
Request: &http.Request{
URL: u2,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
}
urlMeasurements, err := generator.Generate(context.Background(), rts, []string{})
if err != nil {
t.Fatal("unexpected err")
}
if urlMeasurements == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if len(urlMeasurements) < 3 {
t.Fatal("unexpected number of urlMeasurements", len(urlMeasurements))
}
}
func TestGenerateUnexpectedProtocol(t *testing.T) {
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rts := []*RoundTrip{
{
Proto: "h3-27",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
}
urlMeasurements, err := generator.Generate(context.Background(), rts, []string{})
if err != nil {
t.Fatal("unexpected err")
}
if urlMeasurements == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if len(urlMeasurements) != 1 {
t.Fatal("unexpected number of urlMeasurements")
}
measurement := urlMeasurements[0]
if measurement.URL != u.String() {
t.Fatal("unexpected URL")
}
if measurement.DNS == nil {
t.Fatal("DNS should not be nil")
}
if measurement.RoundTrip == nil {
t.Fatal("RoundTrip should not be nil")
}
if measurement.Endpoints != nil {
t.Fatal("Endpoints should be nil")
}
}
func TestGenerateURLWithClientResolutions(t *testing.T) {
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "h3",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
clientResolution := "142.250.186.36"
urlMeasurement := generator.GenerateURL(context.Background(), rt, []string{clientResolution})
if err != nil {
t.Fatal("unexpected err")
}
if urlMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if urlMeasurement.DNS == nil {
t.Fatal("DNS should not be nil")
}
if len(urlMeasurement.Endpoints) < 2 {
t.Fatal("unexpected number of endpoints")
}
clientAddrsFound := false
for _, e := range urlMeasurement.Endpoints {
if e.Endpoint == clientResolution+":443" {
clientAddrsFound = true
}
}
if !clientAddrsFound {
t.Fatal("did not use provided client resolution")
}
}
func TestGenerateHTTP(t *testing.T) {
u, err := url.Parse("http://example.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "http",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateHTTPEndpoint(context.Background(), rt, "93.184.216.34:80")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if endpointMeasurement.TCPConnect == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.HTTPRoundTrip == nil {
t.Fatal("HTTPRoundTripMeasurement should not be nil")
}
}
func TestGenerateHTTPS(t *testing.T) {
u, err := url.Parse("https://example.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "https",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "93.184.216.34:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if endpointMeasurement.TCPConnect == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshake == nil {
t.Fatal("TLSHandshakeMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshake.Failure != nil {
t.Fatal("unexpected failure at TLSHandshakeMeasurement")
}
if endpointMeasurement.HTTPRoundTrip == nil {
t.Fatal("HTTPRoundTripMeasurement should not be nil")
}
}
func TestGenerateHTTPSTLSFailure(t *testing.T) {
u, err := url.Parse("https://wrong.host.badssl.com/")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "https",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "104.154.89.105:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if endpointMeasurement.TCPConnect == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshake == nil {
t.Fatal("TLSHandshakeMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshake.Failure == nil {
t.Fatal("expected failure at TLSHandshakeMeasurement")
}
if endpointMeasurement.HTTPRoundTrip != nil {
t.Fatal("HTTPRoundTripMeasurement should be nil")
}
}
func TestGenerateH3(t *testing.T) {
u := &url.URL{Scheme: "https", Host: quictesting.Domain, Path: "/"}
rt := &RoundTrip{
Proto: "h3",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateH3Endpoint(context.Background(), rt, quictesting.Endpoint("443"))
if endpointMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if endpointMeasurement.QUICHandshake == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.HTTPRoundTrip == nil {
t.Fatal("HTTPRoundTripMeasurement should not be nil")
}
}
func TestGenerateTCPDoFails(t *testing.T) {
expected := errors.New("expected")
generator := &DefaultGenerator{
dialer: fakeDialer{err: expected},
resolver: newResolver(),
}
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "https",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "173.194.76.103:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement.TCPConnect == nil {
t.Fatal("QUIC handshake should not be nil")
}
if endpointMeasurement.TCPConnect.Failure == nil {
t.Fatal("expected an error here")
}
if *endpointMeasurement.TCPConnect.Failure != *newfailure(expected) {
t.Fatal("unexpected error type")
}
}
func TestGenerateQUICDoFails(t *testing.T) {
expected := errors.New("expected")
generator := &DefaultGenerator{
quicDialer: fakeQUICDialer{err: expected},
resolver: newResolver(),
}
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "h3",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateH3Endpoint(context.Background(), rt, "173.194.76.103:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement.QUICHandshake == nil {
t.Fatal("QUIC handshake should not be nil")
}
if endpointMeasurement.QUICHandshake.Failure == nil {
t.Fatal("expected an error here")
}
if *endpointMeasurement.QUICHandshake.Failure != *newfailure(expected) {
t.Fatal("unexpected error type")
}
}
func TestGenerateHTTPDoFails(t *testing.T) {
expected := errors.New("expected")
generator := &DefaultGenerator{
transport: fakeTransport{err: expected},
resolver: newResolver(),
}
u, err := url.Parse("http://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed for clearly good URL")
u2, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed for clearly good URL")
rts := []*RoundTrip{
{
Proto: "http",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
{
Proto: "https",
Request: &http.Request{
URL: u2,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
{
Proto: "h3",
Request: &http.Request{
URL: u2,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
}
urlMeasurements, err := generator.Generate(context.Background(), rts, []string{})
if err != nil {
t.Fatal("unexpected err")
}
if len(urlMeasurements) != 3 {
t.Fatal("unexpected number of urlMeasurements")
}
for _, u := range urlMeasurements {
if u.DNS == nil {
t.Fatal("unexpected DNS failure")
}
if len(u.Endpoints) < 1 {
t.Fatal("unexpected number of endpoints", len(u.Endpoints))
}
// this can occur when the network is unreachable, but it is irrelevant for checking HTTP behavior
if u.Endpoints[0].TCPConnect != nil && u.Endpoints[0].TCPConnect.Failure != nil {
continue
}
if u.Endpoints[0].QUICHandshake != nil && u.Endpoints[0].QUICHandshake.Failure != nil {
continue
}
if u.Endpoints[0].HTTPRoundTrip == nil {
t.Fatal("roundtrip should not be nil", u.Endpoints[0].TCPConnect.Failure, "jaaaa")
}
if u.Endpoints[0].HTTPRoundTrip.Response == nil {
t.Fatal("roundtrip response should not be nil")
}
if u.Endpoints[0].HTTPRoundTrip.Response.Failure == nil {
t.Fatal("expected an HTTP error")
}
if !strings.HasSuffix(*u.Endpoints[0].HTTPRoundTrip.Response.Failure, expected.Error()) {
t.Fatal("unexpected failure type")
}
}
}

72
internal/cmd/oohelperd/internal/websteps/h3.go
View File

@ -1,72 +0,0 @@
package websteps
import (
"errors"
"net"
"net/http"
"net/url"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
type h3URL struct {
URL *url.URL
proto string
}
type altSvcH3 struct {
authority string
proto string
}
var ErrNoH3Location = errors.New("no h3 location found")
// getH3URL returns the URL for HTTP/3 requests, if the target supports HTTP/3.
// Returns nil, if no HTTP/3 support is advertised.
func getH3URL(resp *http.Response) (*h3URL, error) {
URL := resp.Request.URL
if URL == nil {
return nil, ErrInvalidURL
}
h3Svc, err := parseAltSvc(resp, URL)
if err != nil {
return nil, err
}
quicURL, err := url.Parse(URL.String())
runtimex.PanicOnError(err, "url.Parse failed")
quicURL.Host = h3Svc.authority
return &h3URL{URL: quicURL, proto: h3Svc.proto}, nil
}
// parseAltSvc parses the Alt-Svc HTTP header for entries advertising the use of H3
func parseAltSvc(resp *http.Response, URL *url.URL) (*altSvcH3, error) {
// TODO(bassosimone,kelmenhorst): see if we can make this algorithm more robust.
if URL.Scheme != "https" {
return nil, ErrUnsupportedScheme
}
alt_svc := resp.Header.Get("Alt-Svc")
// syntax: Alt-Svc: <protocol-id>=<alt-authority>; ma=<max-age>; persist=1
entries := strings.Split(alt_svc, ",")
for _, e := range entries {
keyvalpairs := strings.Split(e, ";")
for _, p := range keyvalpairs {
p = strings.Replace(p, "\"", "", -1)
kv := strings.Split(p, "=")
if len(kv) != 2 {
continue
}
if _, ok := websteps.SupportedQUICVersions[kv[0]]; ok {
host, port, err := net.SplitHostPort(kv[1])
runtimex.PanicOnError(err, "net.SplitHostPort failed")
if host == "" {
host = URL.Hostname()
}
authority := net.JoinHostPort(host, port)
return &altSvcH3{authority: authority, proto: kv[0]}, nil
}
}
}
return nil, ErrNoH3Location
}

39
internal/cmd/oohelperd/internal/websteps/h3_test.go
View File

@ -1,39 +0,0 @@
package websteps
import (
"net/http"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func TestGetH3URLInvalidURL(t *testing.T) {
resp := &http.Response{
Request: &http.Request{},
}
h3URL, err := getH3URL(resp)
if err == nil {
t.Fatal("expected an error here")
}
if h3URL != nil {
t.Fatal("h3URL should be nil")
}
}
func TestParseUnsupportedScheme(t *testing.T) {
URL, err := url.Parse("h3://google.com")
runtimex.PanicOnError(err, "url.Parse failed")
parsed, err := parseAltSvc(nil, URL)
if err == nil {
t.Fatal("expected an error here")
}
if err != ErrUnsupportedScheme {
t.Fatal("unexpected error type")
}
if parsed != nil {
t.Fatal("h3URL should be nil")
}
}

61
internal/cmd/oohelperd/internal/websteps/initialchecks.go
View File

@ -1,61 +0,0 @@
package websteps
import (
"context"
"errors"
"net/url"
"github.com/ooni/probe-cli/v3/internal/model"
)
// InitialChecks is the first step of the test helper algorithm. We
// make sure we can parse the URL, we handle the scheme, and the domain
// name inside the URL's authority is valid.
// Errors returned by Preresolve.
var (
// ErrInvalidURL indicates that the URL is invalid.
ErrInvalidURL = errors.New("the URL is invalid")
// ErrUnsupportedScheme indicates that we don't support the scheme.
ErrUnsupportedScheme = errors.New("unsupported scheme")
// ErrNoSuchHost indicates that the DNS resolution failed.
ErrNoSuchHost = errors.New("no such host")
)
// InitChecker is the interface responsible for running InitialChecks.
type InitChecker interface {
InitialChecks(URL string) (*url.URL, error)
}
// DefaultInitChecker is the default InitChecker.
type DefaultInitChecker struct {
resolver model.Resolver
}
// InitialChecks checks whether the URL is valid and whether the
// domain inside the URL is an existing one. If these preliminary
// checks fail, there's no point in continuing.
// If they succeed, InitialChecks returns the URL, if not an error.
func (i *DefaultInitChecker) InitialChecks(URL string) (*url.URL, error) {
parsed, err := url.Parse(URL)
if err != nil {
return nil, ErrInvalidURL
}
switch parsed.Scheme {
case "http", "https":
default:
return nil, ErrUnsupportedScheme
}
// Assumptions:
//
// 1. the resolver will cache the resolution for later
//
// 2. an IP address does not cause an error because we are using
// a resolve that behaves like getaddrinfo
if _, err := i.resolver.LookupHost(context.Background(), parsed.Hostname()); err != nil {
return nil, ErrNoSuchHost
}
return parsed, nil
}

31
internal/cmd/oohelperd/internal/websteps/initialchecks_test.go
View File

@ -1,31 +0,0 @@
package websteps
import (
"testing"
)
var checker = &DefaultInitChecker{resolver: newResolver()}
func TestMeasureWithInvalidURL(t *testing.T) {
_, err := checker.InitialChecks("http://[::1]aaaa")
if err == nil || err != ErrInvalidURL {
t.Fatal("expected an error here")
}
}
func TestMeasureWithUnsupportedScheme(t *testing.T) {
_, err := checker.InitialChecks("abc://example.com")
if err == nil || err != ErrUnsupportedScheme {
t.Fatal("expected an error here")
}
}
func TestMeasureWithInvalidHost(t *testing.T) {
_, err := checker.InitialChecks("http://www.ooni.ooni")
if err == nil || err != ErrNoSuchHost {
t.Fatal("expected an error here")
}
}

100
internal/cmd/oohelperd/internal/websteps/measure.go
View File

@ -1,100 +0,0 @@
package websteps
import (
"context"
"errors"
"net/url"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
type (
CtrlRequest = websteps.CtrlRequest
ControlResponse = websteps.CtrlResponse
)
var ErrInternalServer = errors.New("internal server error")
// Config contains the building blocks of the testhelper algorithm
type Config struct {
checker InitChecker
explorer Explorer
generator Generator
resolver model.Resolver
}
// Measure performs the three consecutive steps of the testhelper algorithm:
//
// 1. InitialChecks
//
// 2. Explore
//
// 3. Generate
func Measure(ctx context.Context, creq *CtrlRequest, config *Config) (*ControlResponse, error) {
var (
URL *url.URL
err error
)
resolver := config.resolver
if resolver == nil {
// use a central resolver
resolver = newResolver()
}
checker := config.checker
if checker == nil {
checker = &DefaultInitChecker{resolver: resolver}
}
URL, err = checker.InitialChecks(creq.URL)
if err != nil {
// return a valid response in case of NXDOMAIN so the probe can compare the failure
if err == ErrNoSuchHost {
return newDNSFailedResponse(err, creq.URL), nil
}
return nil, err
}
explorer := config.explorer
if explorer == nil {
explorer = &DefaultExplorer{resolver: resolver}
}
rts, err := explorer.Explore(URL, creq.Headers)
if err != nil {
return nil, ErrInternalServer
}
generator := config.generator
if generator == nil {
generator = &DefaultGenerator{resolver: resolver}
}
meas, err := generator.Generate(ctx, rts, creq.Addrs)
if err != nil {
return nil, err
}
return &ControlResponse{URLs: meas}, nil
}
// newDNSFailedResponse creates a new response with one URLMeasurement entry
// indicating that the DNS step failed
func newDNSFailedResponse(err error, URL string) *ControlResponse {
resp := &ControlResponse{}
m := &URLMeasurement{
URL: URL,
DNS: &DNSMeasurement{
Failure: newfailure(err),
},
}
resp.URLs = append(resp.URLs, m)
return resp
}
// newResolver creates a new DNS resolver instance
func newResolver() model.Resolver {
childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google")
runtimex.PanicOnError(err, "NewDNSClient failed")
var r model.Resolver = childResolver
r = &netxlite.ResolverIDNA{Resolver: r}
return r
}

117
internal/cmd/oohelperd/internal/websteps/measure_test.go
View File

@ -1,117 +0,0 @@
package websteps
import (
"context"
"errors"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestMeasureSuccess(t *testing.T) {
req := &CtrlRequest{
URL: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{})
if err != nil {
t.Fatal("unexpected error")
}
if resp == nil {
t.Fatal("unexpected nil response")
}
}
type MockChecker struct {
err error
}
func (c *MockChecker) InitialChecks(URL string) (*url.URL, error) {
return nil, c.err
}
type MockExplorer struct{}
func (c *MockExplorer) Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) {
return nil, ErrExpectedExplore
}
type MockGenerator struct{}
func (c *MockGenerator) Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) {
return nil, ErrExpectedGenerate
}
var ErrExpectedCheck error = errors.New("expected error checker")
var ErrExpectedExplore error = errors.New("expected error explorer")
var ErrExpectedGenerate error = errors.New("expected error generator")
func TestMeasureInitialChecksFail(t *testing.T) {
req := &CtrlRequest{
URL: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{checker: &MockChecker{err: ErrExpectedCheck}})
if err == nil {
t.Fatal("expected an error here")
}
if err != ErrExpectedCheck {
t.Fatal("unexpected error type")
}
if resp != nil {
t.Fatal("resp should be nil")
}
}
func TestMeasureInitialChecksFailWithNXDOMAIN(t *testing.T) {
req := &CtrlRequest{
URL: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{checker: &MockChecker{err: ErrNoSuchHost}})
if err != nil {
t.Fatal("unexpected error")
}
if resp == nil {
t.Fatal("resp should not be nil")
}
if len(resp.URLs) != 1 {
t.Fatal("unexpected number of measurements")
}
if resp.URLs[0].DNS == nil {
t.Fatal("DNS entry should not be nil")
}
if *resp.URLs[0].DNS.Failure != netxlite.FailureDNSNXDOMAINError {
t.Fatal("unexpected failure")
}
}
func TestMeasureExploreFails(t *testing.T) {
req := &CtrlRequest{
URL: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{explorer: &MockExplorer{}})
if err == nil {
t.Fatal("expected an error here")
}
if err != ErrInternalServer {
t.Fatal("unexpected error type")
}
if resp != nil {
t.Fatal("resp should be nil")
}
}
func TestMeasureGenerateFails(t *testing.T) {
req := &CtrlRequest{
URL: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{generator: &MockGenerator{}})
if err == nil {
t.Fatal("expected an error here")
}
if err != ErrExpectedGenerate {
t.Fatal("unexpected error type")
}
if resp != nil {
t.Fatal("resp should be nil")
}
}

17
internal/cmd/oohelperd/internal/websteps/model.go
View File

@ -1,17 +0,0 @@
package websteps
import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
)
type (
URLMeasurement = websteps.URLMeasurement
DNSMeasurement = websteps.DNSMeasurement
EndpointMeasurement = websteps.EndpointMeasurement
TCPConnectMeasurement = websteps.TCPConnectMeasurement
HTTPRoundTripMeasurement = websteps.HTTPRoundTripMeasurement
TLSHandshakeMeasurement = websteps.TLSHandshakeMeasurement
HTTPRequestMeasurement = websteps.HTTPRequestMeasurement
HTTPResponseMeasurement = websteps.HTTPResponseMeasurement
RoundTrip = websteps.RoundTripInfo
)

70
internal/cmd/oohelperd/internal/websteps/websteps.go
View File

@ -1,70 +0,0 @@
// Package websteps implements the websteps test helper.
//
// See the https://github.com/ooni/spec/blob/master/backends/th-007-websteps.md
// related specification document.
//
// This implementation uses version 202108.17.1114 of the spec.
package websteps
import (
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version"
)
// newfailure is a convenience shortcut to save typing
var newfailure = archival.NewFailure
// maxAcceptableBody is _at the same time_ the maximum acceptable body for incoming
// API requests and the maximum acceptable body when fetching arbitrary URLs. See
// https://github.com/ooni/probe/issues/1727 for statistics regarding the test lists
// including the empirical CDF of the body size for test lists URLs.
const maxAcceptableBody = 1 << 24
// Handler implements the websteps test helper HTTP API.
type Handler struct {
Config *Config
}
// ServeHTTP implements http.Handler.ServeHTTP.
func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Add("Server", fmt.Sprintf(
"oohelperd/%s ooniprobe-engine/%s", version.Version, version.Version,
))
if req.Method != "POST" {
w.WriteHeader(400)
return
}
reader := &io.LimitedReader{R: req.Body, N: maxAcceptableBody}
data, err := netxlite.ReadAllContext(req.Context(), reader)
if err != nil {
w.WriteHeader(400)
return
}
var creq CtrlRequest
if err := json.Unmarshal(data, &creq); err != nil {
w.WriteHeader(400)
return
}
cresp, err := Measure(req.Context(), &creq, h.Config)
if err != nil {
if err == ErrInternalServer {
w.WriteHeader(500)
return
}
w.WriteHeader(400)
return
}
// We assume that the following call cannot fail because it's a
// clearly serializable data structure.
data, err = json.Marshal(cresp)
runtimex.PanicOnError(err, "json.Marshal failed")
w.Header().Add("Content-Type", "application/json")
w.Write(data)
}

281
internal/cmd/oohelperd/internal/websteps/websteps_test.go
View File

@ -1,281 +0,0 @@
package websteps
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
const requestnoredirect = `{
"url": "https://ooni.org",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"104.198.14.52:443"
]
}`
const requestredirect = `{
"url": "https://www.ooni.org",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"18.192.76.182:443"
]
}`
const requestIPaddressinput = `{
"url": "https://172.217.168.4",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"172.217.168.4:443"
]
}`
const requestwithquic = `{
"url": "https://www.google.com",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"142.250.74.196:443"
]
}`
const requestWithoutDomainName = `{
"url": "https://8.8.8.8",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"8.8.8.8:443"
]
}`
func TestWorkingAsIntended(t *testing.T) {
handler := Handler{Config: &Config{}}
srv := httptest.NewServer(&handler)
defer srv.Close()
type expectationSpec struct {
name string
reqMethod string
reqContentType string
reqBody string
respStatusCode int
respContentType string
parseBody bool
}
expectations := []expectationSpec{{
name: "check for invalid method",
reqMethod: "GET",
respStatusCode: 400,
}, {
name: "check for invalid request body",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: "{",
respStatusCode: 400,
}, {
name: "with measurement failure",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: `{"url": "http://[::1]aaaa"}`,
respStatusCode: 400,
}, {
name: "request without redirect or H3 follow-up request",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestnoredirect,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}, {
name: "request triggering one redirect, without H3 follow-up request",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestredirect,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}, {
name: "request with an IP address as input",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestIPaddressinput,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}, {
name: "request triggering H3 follow-up request, without redirect",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestwithquic,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}, {
name: "when there's no domain name in the request",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestWithoutDomainName,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}}
for _, expect := range expectations {
t.Run(expect.name, func(t *testing.T) {
body := strings.NewReader(expect.reqBody)
req, err := http.NewRequest(expect.reqMethod, srv.URL, body)
if err != nil {
t.Fatalf("%s: %+v", expect.name, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("%s: %+v", expect.name, err)
}
defer resp.Body.Close()
if resp.StatusCode != expect.respStatusCode {
t.Fatalf("unexpected status code: %+v", resp.StatusCode)
}
data, err := netxlite.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
if !expect.parseBody {
return
}
var v interface{}
if err := json.Unmarshal(data, &v); err != nil {
t.Fatal(err)
}
})
}
}
func TestHandlerWithInternalServerError(t *testing.T) {
handler := Handler{Config: &Config{explorer: &MockExplorer{}}}
srv := httptest.NewServer(&handler)
defer srv.Close()
body := strings.NewReader(`{"url": "https://example.com"}`)
req, err := http.NewRequest("POST", srv.URL, body)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 500 {
t.Fatalf("unexpected status code: %+v", resp.StatusCode)
}
_, err = netxlite.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
}
func TestHandlerWithRequestBodyReadingError(t *testing.T) {
expected := errors.New("mocked error")
handler := Handler{Config: &Config{}}
rw := NewFakeResponseWriter()
req := &http.Request{
Method: "POST",
Header: map[string][]string{
"Content-Type": {"application/json"},
"Content-Length": {"2048"},
},
Body: &FakeBody{Err: expected},
}
handler.ServeHTTP(rw, req)
if rw.StatusCode != 400 {
t.Fatal("unexpected status code")
}
}
type FakeBody struct {
Err error
}
func (fb FakeBody) Read(p []byte) (int, error) {
time.Sleep(10 * time.Microsecond)
return 0, fb.Err
}
func (fb FakeBody) Close() error {
return nil
}
type FakeResponseWriter struct {
Body [][]byte
HeaderMap http.Header
StatusCode int
}
func NewFakeResponseWriter() *FakeResponseWriter {
return &FakeResponseWriter{HeaderMap: make(http.Header)}
}
func (frw *FakeResponseWriter) Header() http.Header {
return frw.HeaderMap
}
func (frw *FakeResponseWriter) Write(b []byte) (int, error) {
frw.Body = append(frw.Body, b)
return len(b), nil
}
func (frw *FakeResponseWriter) WriteHeader(statusCode int) {
frw.StatusCode = statusCode
}

2
internal/cmd/oohelperd/oohelperd.go
View File

@ -10,7 +10,6 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/websteps"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/model"
@ -59,7 +58,6 @@ func main() {
func testableMain() {
mux := http.NewServeMux()
mux.Handle("/api/unstable/websteps", &websteps.Handler{Config: &websteps.Config{}})
mux.Handle("/api/v1/websteps", &webstepsx.THHandler{})
mux.Handle("/", webconnectivity.Handler{
Client: httpx,

26
internal/engine/experiment/websteps/control.go
View File

@ -1,26 +0,0 @@
package websteps
import (
"context"
"github.com/ooni/probe-cli/v3/internal/httpx"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// Control performs the control request and returns the response.
func Control(
ctx context.Context, sess model.ExperimentSession,
thAddr string, resourcePath string, creq CtrlRequest) (out CtrlResponse, err error) {
clnt := &httpx.APIClientTemplate{
BaseURL: thAddr,
HTTPClient: sess.DefaultHTTPClient(),
Logger: sess.Logger(),
}
// make sure error is wrapped
err = clnt.WithBodyLogging().Build().PostJSON(ctx, resourcePath, creq, &out)
if err != nil {
err = netxlite.NewTopLevelGenericErrWrapper(err)
}
return
}

30
internal/engine/experiment/websteps/dns.go
View File

@ -1,30 +0,0 @@
package websteps
import (
"context"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
type DNSConfig struct {
Domain string
Resolver model.Resolver
}
// DNSDo performs the DNS check.
func DNSDo(ctx context.Context, config DNSConfig) ([]string, error) {
resolver := config.Resolver
if resolver == nil {
childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google")
runtimex.PanicOnError(err, "NewDNSClient failed")
resolver = childResolver
resolver = &netxlite.ResolverIDNA{
Resolver: resolver,
}
}
return resolver.LookupHost(ctx, config.Domain)
}

132
internal/engine/experiment/websteps/factory.go
View File

@ -1,132 +0,0 @@
package websteps
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/url"
"sync"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
oohttp "github.com/ooni/oohttp"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
var ErrNoConnReuse = errors.New("cannot reuse connection")
func NewRequest(ctx context.Context, URL *url.URL, headers http.Header) *http.Request {
req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil)
runtimex.PanicOnError(err, "NewRequestWithContect failed")
for k, vs := range headers {
for _, v := range vs {
req.Header.Add(k, v)
}
}
return req
}
// NewDialerResolver contructs a new dialer for TCP connections,
// with default, errorwrapping and resolve functionalities
func NewDialerResolver(resolver model.Resolver) model.Dialer {
var d model.Dialer = netxlite.DefaultDialer
d = &netxlite.ErrorWrapperDialer{Dialer: d}
d = &netxlite.DialerResolver{
Resolver: resolver,
Dialer: d,
}
return d
}
// NewQUICDialerResolver creates a new QUICDialerResolver
// with default, errorwrapping and resolve functionalities
func NewQUICDialerResolver(resolver model.Resolver) model.QUICDialer {
var ql model.QUICListener = &netxlite.QUICListenerStdlib{}
ql = &netxlite.ErrorWrapperQUICListener{QUICListener: ql}
var dialer model.QUICDialer = &netxlite.QUICDialerQUICGo{
QUICListener: ql,
}
dialer = &netxlite.ErrorWrapperQUICDialer{QUICDialer: dialer}
dialer = &netxlite.QUICDialerResolver{
Resolver: resolver,
Dialer: dialer,
}
return dialer
}
// NewSingleH3Transport creates an http3.RoundTripper.
func NewSingleH3Transport(qconn quic.EarlyConnection, tlscfg *tls.Config, qcfg *quic.Config) http.RoundTripper {
transport := &http3.RoundTripper{
DisableCompression: true,
TLSClientConfig: tlscfg,
QuicConfig: qcfg,
Dial: (&SingleDialerH3{qconn: &qconn}).Dial,
}
return transport
}
// NewSingleTransport creates a new HTTP transport with a single-use dialer.
func NewSingleTransport(conn net.Conn) http.RoundTripper {
singledialer := &SingleDialer{conn: &conn}
transport := newBaseTransport()
transport.DialContext = singledialer.DialContext
transport.DialTLSContext = singledialer.DialContext
return transport
}
// NewSingleTransport creates a new HTTP transport with a custom dialer and handshaker.
func NewTransportWithDialer(dialer model.Dialer, tlsConfig *tls.Config, handshaker model.TLSHandshaker) http.RoundTripper {
transport := newBaseTransport()
transport.DialContext = dialer.DialContext
transport.DialTLSContext = (&netxlite.TLSDialerLegacy{
Config: tlsConfig,
Dialer: dialer,
TLSHandshaker: handshaker,
}).DialTLSContext
return transport
}
// newBaseTransport creates a new HTTP transport with the default dialer.
func newBaseTransport() (transport *oohttp.StdlibTransport) {
base := oohttp.DefaultTransport.(*oohttp.Transport).Clone()
base.DisableCompression = true
base.MaxConnsPerHost = 1
transport = &oohttp.StdlibTransport{Transport: base}
return transport
}
type SingleDialer struct {
sync.Mutex
conn *net.Conn
}
func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) {
s.Lock()
defer s.Unlock()
if s.conn == nil {
return nil, ErrNoConnReuse
}
c := s.conn
s.conn = nil
return *c, nil
}
type SingleDialerH3 struct {
sync.Mutex
qconn *quic.EarlyConnection
}
func (s *SingleDialerH3) Dial(ctx context.Context, network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) {
s.Lock()
defer s.Unlock()
if s.qconn == nil {
return nil, ErrNoConnReuse
}
qs := s.qconn
s.qconn = nil
return *qs, nil
}

34
internal/engine/experiment/websteps/http.go
View File

@ -1,34 +0,0 @@
package websteps
import (
"net/http"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// HTTPDo performs the HTTP check.
// Input:
// req *http.Request
// The same request than the one used by the Explore step.
// This means that req contains the headers set by the original CtrlRequest, as well as,
// in case of a redirect chain, additional headers that were added due to redirects
// transport http.RoundTripper:
// The transport to use, either http.Transport, or http3.RoundTripper.
func HTTPDo(req *http.Request, transport http.RoundTripper) (*http.Response, []byte, error) {
clnt := http.Client{
CheckRedirect: func(r *http.Request, reqs []*http.Request) error {
return http.ErrUseLastResponse
},
Transport: transport,
}
resp, err := clnt.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
body, err := netxlite.ReadAllContext(req.Context(), resp.Body)
if err != nil {
return resp, nil, nil
}
return resp, body, nil
}

155
internal/engine/experiment/websteps/model.go
View File

@ -1,155 +0,0 @@
package websteps
import "net/http"
// Websteps test helper spec messages:
// CtrlRequest is the request sent by the probe to the test helper.
type CtrlRequest struct {
// URL is the mandatory URL to measure.
URL string `json:"url"`
// Headers contains optional headers.
Headers map[string][]string `json:"headers"`
// Addrs contains the optional IP addresses resolved by the
// probe for the domain inside URL.
Addrs []string `json:"addrs"`
}
// CtrlResponse is the response from the test helper.
type CtrlResponse struct {
// URLs contains the URLs we should measure. These URLs
// derive from CtrlRequest.URL.
URLs []*URLMeasurement `json:"urls"`
}
// URLMeasurement contains all the URLs measured by the test helper.
type URLMeasurement struct {
// URL is the URL to which this measurement refers.
URL string `json:"url"`
// DNS contains the domain names resolved by the test helper.
DNS *DNSMeasurement `json:"dns"`
// Endpoints contains endpoint measurements.
Endpoints []*EndpointMeasurement `json:"endpoints"`
// RoundTrip is the related round trip. This field MUST NOT be
// exported as JSON, since it's only used internally by the test
// helper and it's completely ignored by the probe.
RoundTrip *RoundTripInfo `json:"-"`
}
// RoundTripInfo contains info on a specific round trip. This data
// structure is not part of the test helper protocol. We use it
// _inside_ the test helper to describe the discovery phase where
// we gather all the URLs that can derive from a given URL.
type RoundTripInfo struct {
// Proto is the protocol used, it can be "h2", "http/1.1", "h3".
Proto string
// Request is the original HTTP request. Headers also include cookies.
Request *http.Request
// Response is the HTTP response.
Response *http.Response
// SortIndex is the index using for sorting round trips.
SortIndex int
}
// DNSMeasurement is a DNS measurement.
type DNSMeasurement struct {
// Domain is the domain we wanted to resolve.
Domain string `json:"domain"`
// Failure is the error that occurred.
Failure *string `json:"failure"`
// Addrs contains the resolved addresses.
Addrs []string `json:"addrs"`
}
// EndpointMeasurement is an HTTP measurement where we are using
// a specific TCP/TLS/QUIC endpoint to get the URL.
//
// The specification describes this data structure as the sum of
// three distinct types: HTTPEndpointMeasurement for "http",
// HTTPSEndpointMeasurement for "https", and H3EndpointMeasurement
// for "h3". We don't have sum types here, therefore we use the
// Protocol field to indicate which fields are meaningful.
type EndpointMeasurement struct {
// Endpoint is the endpoint we're measuring.
Endpoint string `json:"endpoint"`
// Protocol is one of "http", "https", and "h3".
Protocol string `json:"protocol"`
// TCPConnect is the TCP connect measurement. This field
// is only meaningful when protocol is "http" or "https."
TCPConnect *TCPConnectMeasurement `json:"tcp_connect"`
// QUICHandshake is the QUIC handshake measurement. This field
// is only meaningful when the protocol is "h3".
QUICHandshake *QUICHandshakeMeasurement `json:"quic_handshake"`
// TLSHandshake is the TLS handshake measurement. This field
// is only meaningful when the protocol is "https".
TLSHandshake *TLSHandshakeMeasurement `json:"tls_handshake"`
// HTTPRoundTrip is the related HTTP GET measurement.
HTTPRoundTrip *HTTPRoundTripMeasurement `json:"http_round_trip"`
}
// TCPConnectMeasurement is a TCP connect measurement.
type TCPConnectMeasurement struct {
// Failure is the error that occurred.
Failure *string `json:"failure"`
}
// TLSHandshakeMeasurement is a TLS handshake measurement.
type TLSHandshakeMeasurement struct {
// Failure is the error that occurred.
Failure *string `json:"failure"`
}
// QUICHandshakeMeasurement is a QUIC handshake measurement.
type QUICHandshakeMeasurement = TLSHandshakeMeasurement
// HTTPRoundTripMeasurement contains a measured HTTP request and
// the corresponding response.
type HTTPRoundTripMeasurement struct {
// Request contains request data.
Request *HTTPRequestMeasurement `json:"request"`
// Response contains response data.
Response *HTTPResponseMeasurement `json:"response"`
}
// HTTPRequestMeasurement contains request data.
type HTTPRequestMeasurement struct {
// Method is the request method.
Method string `json:"method"`
// URL is the request URL.
URL string `json:"url"`
// Headers contains request headers.
Headers http.Header `json:"headers"`
}
// HTTPResponseMeasurement contains response data.
type HTTPResponseMeasurement struct {
// BodyLength contains the body length in bytes.
BodyLength int64 `json:"body_length"`
// Failure is the error that occurred.
Failure *string `json:"failure"`
// Headers contains response headers.
Headers http.Header `json:"headers"`
// StatusCode is the response status code.
StatusCode int64 `json:"status_code"`
}

30
internal/engine/experiment/websteps/quic.go
View File

@ -1,30 +0,0 @@
package websteps
import (
"context"
"crypto/tls"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
type QUICConfig struct {
Endpoint string
QUICDialer model.QUICDialer
Resolver model.Resolver
TLSConf *tls.Config
}
// QUICDo performs the QUIC check.
func QUICDo(ctx context.Context, config QUICConfig) (quic.EarlyConnection, error) {
if config.QUICDialer != nil {
return config.QUICDialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{})
}
resolver := config.Resolver
if resolver == nil {
resolver = &netxlite.ResolverSystem{}
}
dialer := NewQUICDialerResolver(resolver)
return dialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{})
}

28
internal/engine/experiment/websteps/tcp.go
View File

@ -1,28 +0,0 @@
package websteps
import (
"context"
"net"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
type TCPConfig struct {
Dialer model.Dialer
Endpoint string
Resolver model.Resolver
}
// TCPDo performs the TCP check.
func TCPDo(ctx context.Context, config TCPConfig) (net.Conn, error) {
if config.Dialer != nil {
return config.Dialer.DialContext(ctx, "tcp", config.Endpoint)
}
resolver := config.Resolver
if resolver == nil {
resolver = &netxlite.ResolverSystem{}
}
dialer := NewDialerResolver(resolver)
return dialer.DialContext(ctx, "tcp", config.Endpoint)
}

23
internal/engine/experiment/websteps/tls.go
View File

@ -1,23 +0,0 @@
package websteps
import (
"context"
"crypto/tls"
"net"
"github.com/ooni/probe-cli/v3/internal/netxlite"
utls "gitlab.com/yawning/utls.git"
)
// TLSDo performs the TLS check.
func TLSDo(ctx context.Context, conn net.Conn, hostname string) (net.Conn, error) {
tlsConf := &tls.Config{
ServerName: hostname,
NextProtos: []string{"h2", "http/1.1"},
}
h := &netxlite.TLSHandshakerConfigurable{
NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto),
}
tlsConn, _, err := h.Handshake(ctx, conn, tlsConf)
return tlsConn, err
}

369
internal/engine/experiment/websteps/websteps.go
View File

@ -1,369 +0,0 @@
// Package websteps implements the websteps experiment.
//
// Specifications:
//
// - test helper: https://github.com/ooni/spec/blob/master/backends/th-007-websteps.md
//
// - experiment: N/A.
//
// We are currently implementing:
//
// - version 202108.17.1114 of the test helper spec.
//
// - version N/A of the experiment spec.
package websteps
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/url"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
const (
testName = "websteps"
testVersion = "0.0.1"
)
// Config contains the experiment config.
type Config struct{}
// TestKeys contains webconnectivity test keys.
type TestKeys struct {
Agent string `json:"agent"`
ClientResolver string `json:"client_resolver"`
URLMeasurements []*URLMeasurement
}
// Measurer performs the measurement.
type Measurer struct {
Config Config
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return Measurer{Config: config}
}
// ExperimentName implements ExperimentMeasurer.ExperExperimentName.
func (m Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion.
func (m Measurer) ExperimentVersion() string {
return testVersion
}
// SupportedQUICVersions are the H3 over QUIC versions we currently support
var SupportedQUICVersions = map[string]bool{
"h3": true,
}
var (
// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
ErrNoAvailableTestHelpers = errors.New("no available helpers")
// ErrNoInput indicates that no input was provided
ErrNoInput = errors.New("no input provided")
// ErrInputIsNotAnURL indicates that the input is not an URL.
ErrInputIsNotAnURL = errors.New("input is not an URL")
// ErrUnsupportedInput indicates that the input URL scheme is unsupported.
ErrUnsupportedInput = errors.New("unsupported input scheme")
)
// Run implements ExperimentMeasurer.Run.
func (m Measurer) Run(
ctx context.Context,
sess model.ExperimentSession,
measurement *model.Measurement,
callbacks model.ExperimentCallbacks,
) error {
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
tk := new(TestKeys)
measurement.TestKeys = tk
tk.Agent = "redirect"
tk.ClientResolver = sess.ResolverIP()
// 1. Parse and verify URL
URL, err := url.Parse(string(measurement.Input))
if err != nil {
return ErrInputIsNotAnURL
}
if URL.Scheme != "http" && URL.Scheme != "https" {
return ErrUnsupportedInput
}
// 2. Perform the initial DNS lookup step
addrs, err := DNSDo(ctx, DNSConfig{Domain: URL.Hostname()})
endpoints := makeEndpoints(addrs, URL)
// 3. Find the testhelper
// TODO(kelmenhorst,bassosimone): this is not used at the moment, but the hardcoded local address
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
var testhelper *model.OOAPIService
for _, th := range testhelpers {
if th.Type == "https" {
testhelper = &th
break
}
}
if testhelper == nil {
return ErrNoAvailableTestHelpers
}
measurement.TestHelpers = map[string]interface{}{
"backend": testhelper,
}
// 4. Query the testhelper
// TODO(kelmenhorst,bassosimone): remove hardcoded version here, this is only for testing purposes
resp, err := Control(ctx, sess, "http://localhost:8080", "/api/unstable/websteps", CtrlRequest{
URL: URL.String(),
Headers: map[string][]string{
"Accept": {httpheader.Accept()},
"Accept-Language": {httpheader.AcceptLanguage()},
"User-Agent": {httpheader.UserAgent()},
},
Addrs: endpoints,
})
if err != nil || resp.URLs == nil {
return errors.New("no control response")
}
// 5. Go over the Control URL measurements and reproduce them without following redirects, one by one.
for _, controlURLMeasurement := range resp.URLs {
urlMeasurement := &URLMeasurement{
URL: controlURLMeasurement.URL,
Endpoints: []*EndpointMeasurement{},
}
URL, err = url.Parse(controlURLMeasurement.URL)
runtimex.PanicOnError(err, "url.Parse failed")
// DNS step
addrs, err = DNSDo(ctx, DNSConfig{Domain: URL.Hostname()})
urlMeasurement.DNS = &DNSMeasurement{
Domain: URL.Hostname(),
Addrs: addrs,
Failure: archival.NewFailure(err),
}
if controlURLMeasurement.Endpoints == nil {
tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement)
continue
}
// the testhelper tells us which endpoints to measure
for _, controlEndpoint := range controlURLMeasurement.Endpoints {
rt := controlEndpoint.HTTPRoundTrip
if rt == nil || rt.Request == nil {
continue
}
var endpointMeasurement *EndpointMeasurement
proto := controlEndpoint.Protocol
_, h3 := SupportedQUICVersions[proto]
switch {
case h3:
endpointMeasurement = m.measureEndpointH3(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers, proto)
case proto == "http":
endpointMeasurement = m.measureEndpointHTTP(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers)
case proto == "https":
endpointMeasurement = m.measureEndpointHTTPS(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers)
default:
panic("should not happen")
}
urlMeasurement.Endpoints = append(urlMeasurement.Endpoints, endpointMeasurement)
}
tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement)
}
return nil
}
func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement {
endpointMeasurement := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: "http",
}
// TCP connect step
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
endpointMeasurement.TCPConnect = &TCPConnectMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
return endpointMeasurement
}
defer conn.Close()
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
URL: URL.String(),
},
}
transport := NewSingleTransport(conn)
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return endpointMeasurement
}
func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement {
endpointMeasurement := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: "https",
}
// TCP connect step
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
endpointMeasurement.TCPConnect = &TCPConnectMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
return endpointMeasurement
}
defer conn.Close()
// TLS handshake step
tlsconn, err := TLSDo(ctx, conn, URL.Hostname())
endpointMeasurement.TLSHandshake = &TLSHandshakeMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
return endpointMeasurement
}
defer tlsconn.Close()
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
URL: URL.String(),
},
}
transport := NewSingleTransport(tlsconn)
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return endpointMeasurement
}
func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint string, headers http.Header, proto string) *EndpointMeasurement {
endpointMeasurement := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: proto,
}
tlsConf := &tls.Config{
ServerName: URL.Hostname(),
NextProtos: []string{proto},
}
// QUIC handshake step
sess, err := QUICDo(ctx, QUICConfig{
Endpoint: endpoint,
TLSConf: tlsConf,
})
endpointMeasurement.QUICHandshake = &TLSHandshakeMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
return endpointMeasurement
}
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
URL: URL.String(),
},
}
transport := NewSingleH3Transport(sess, tlsConf, &quic.Config{})
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return endpointMeasurement
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with ooniprobe
// therefore we should be careful when changing it.
type SummaryKeys struct {
Accessible bool `json:"accessible"`
Blocking string `json:"blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{}
return sk, nil
}
func makeEndpoints(addrs []string, URL *url.URL) []string {
endpoints := []string{}
if addrs == nil {
return endpoints
}
for _, addr := range addrs {
var port string
explicitPort := URL.Port()
scheme := URL.Scheme
switch {
case explicitPort != "":
port = explicitPort
case scheme == "http":
port = "80"
case scheme == "https":
port = "443"
default:
panic("should not happen")
}
endpoints = append(endpoints, net.JoinHostPort(addr, port))
}
return endpoints
}