c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cli: new testhelper and the websteps experiment prototype (#432)

Browse Source 
This is the extension of https://github.com/ooni/probe-cli/pull/431, and my final deliverable for GSoC 2021.

The diff introduces:

1) The new `testhelper` which supports testing multiple IP endpoints per domain and introduces HTTP/3 control measurements. The specification of the `testhelper` can be found at https://github.com/ooni/spec/pull/219. The `testhelper` algorithm consists of three main steps:

   * `InitialChecks` verifies that the input URL can be parsed, has an expected scheme, and contains a valid domain name.

   * `Explore` enumerates all the URLs that it discovers by redirection from the original URL, or by detecting h3 support at the target host.

   * `Generate` performs a step-by-step measurement of each discovered URL.

2) A prototype of the corresponding new experiment `websteps` which uses the control measurement of the `testhelper` to know which URLs to measure, and what to expect. The prototype does not yet have:

   * unit and integration tests,

   * an analysis tool to compare the control and the probe measurement.

This PR is my final deliverable as it is the outcome of the trials, considerations and efforts of my GSoC weeks at OONI. 
It fully integrates HTTP/3 (QUIC) support which has been only used in the `urlgetter` experiment until now.

Related issues: https://github.com/ooni/probe/issues/1729 and https://github.com/ooni/probe/issues/1733.
This commit is contained in:
kelmenhorst 2021-08-17 10:29:06 +02:00 committed by GitHub
parent 14c1640f7f
commit c31591f298
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 2736 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/cmd/oohelperd/internal/http.go
View File

@ -37,9 +37,7 @@ func HTTPDo(ctx context.Context, config *HTTPConfig) {
// we're implementing (for now?) a more liberal approach.
for k, vs := range config.Headers {
switch strings.ToLower(k) {
case "user-agent":
case "accept":
case "accept-language":
case "user-agent", "accept", "accept-language":
for _, v := range vs {
req.Header.Add(k, v)
}

8
internal/cmd/oohelperd/internal/internal.go
View File

@ -8,6 +8,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version"
)
@ -27,10 +28,6 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(400)
return
}
if req.Header.Get("content-type") != "application/json" {
w.WriteHeader(400)
return
}
reader := &io.LimitedReader{R: req.Body, N: h.MaxAcceptableBody}
data, err := iox.ReadAllContext(req.Context(), reader)
if err != nil {
@ -50,7 +47,8 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
// We assume that the following call cannot fail because it's a
// clearly serializable data structure.
data, _ = json.Marshal(cresp)
data, err = json.Marshal(cresp)
runtimex.PanicOnError(err, "json.Marshal failed")
w.Header().Add("Content-Type", "application/json")
w.Write(data)
}

166
internal/cmd/oohelperd/internal/nwcth/explore.go Normal file
View File

@ -0,0 +1,166 @@
package nwcth
import (
"crypto/tls"
"net/http"
"net/http/cookiejar"
"net/url"
"sort"
"strings"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
// Explore is the second step of the test helper algorithm. Its objective
// is to enumerate all the URLs we can discover by redirection from
// the original URL in the test list. Because the test list contains by
// definition noisy data, we need this preprocessing step to learn all
// the URLs that are actually implied by the original URL.
// Explorer is the interface responsible for running Explore.
type Explorer interface {
Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error)
}
// DefaultExplorer is the default Explorer.
type DefaultExplorer struct {
resolver netxlite.Resolver
}
// Explore returns a list of round trips sorted so that the first
// round trip is the first element in the list, and so on.
// Explore uses the URL and the optional headers provided by the CtrlRequest.
func (e *DefaultExplorer) Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) {
resp, err := e.get(URL, headers)
if err != nil {
return nil, err
}
rts := e.rearrange(resp, nil)
h3URL, err := getH3URL(resp)
if err != nil {
// If we cannot find the HTTP/3 URL for subsequent measurements, we just continue
// the measurement using the URLs we have found so far.
return rts, nil
}
resp, err = e.getH3(h3URL, headers)
if err != nil {
// If we cannot follow the HTTP/3 chain, we just continue
// the measurement using the URLs we have found so far.
return rts, nil
}
rts = append(rts, e.rearrange(resp, h3URL)...)
return rts, nil
}
// rearrange takes in input the final response of an HTTP transaction and an optional h3URL
// (which is needed to derive the type of h3 protocol, i.e. h3 or h3-29),
// and produces in output a list of round trips sorted
// such that the first round trip is the first element in the out array.
func (e *DefaultExplorer) rearrange(resp *http.Response, h3URL *h3URL) (out []*RoundTrip) {
index := 0
for resp != nil && resp.Request != nil {
proto := resp.Request.URL.Scheme
if h3URL != nil {
proto = h3URL.proto
}
out = append(out, &RoundTrip{
Proto: proto,
SortIndex: index,
Request: resp.Request,
Response: resp,
})
index += 1
resp = resp.Request.Response
}
sh := &sortHelper{out}
sort.Sort(sh)
return
}
// sortHelper is the helper structure to sort round trips.
type sortHelper struct {
v []*RoundTrip
}
// Len implements sort.Interface.Len.
func (sh *sortHelper) Len() int {
return len(sh.v)
}
// Less implements sort.Interface.Less.
func (sh *sortHelper) Less(i, j int) bool {
return sh.v[i].SortIndex >= sh.v[j].SortIndex
}
// Swap implements sort.Interface.Swap.
func (sh *sortHelper) Swap(i, j int) {
sh.v[i], sh.v[j] = sh.v[j], sh.v[i]
}
// get gets the given URL and returns the final response after
// redirection, and an error. If the error is nil, the final response is valid.
func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http.Response, error) {
tlsConf := &tls.Config{
NextProtos: []string{"h2", "http/1.1"},
}
transport := netxlite.NewHTTPTransport(NewDialerResolver(e.resolver), tlsConf, &netxlite.TLSHandshakerConfigurable{})
// TODO(bassosimone): here we should use runtimex.PanicOnError
jarjar, _ := cookiejar.New(nil)
clnt := &http.Client{
Transport: transport,
Jar: jarjar,
}
// TODO(bassosimone): document why e.newRequest cannot fail.
req, err := e.newRequest(URL, headers)
runtimex.PanicOnError(err, "newRequest failed")
resp, err := clnt.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// Note that we ignore the response body.
return resp, nil
}
// getH3 uses HTTP/3 to get the given URL and returns the final
// response after redirection, and an error. If the error is nil, the final response is valid.
func (e *DefaultExplorer) getH3(h3URL *h3URL, headers map[string][]string) (*http.Response, error) {
dialer := NewQUICDialerResolver(e.resolver)
tlsConf := &tls.Config{
NextProtos: []string{h3URL.proto},
}
transport := netxlite.NewHTTP3Transport(dialer, tlsConf)
// TODO(bassosimone): here we should use runtimex.PanicOnError
jarjar, _ := cookiejar.New(nil)
clnt := &http.Client{
Transport: transport,
Jar: jarjar,
}
// TODO(bassosimone): document why e.newRequest cannot fail.
req, err := e.newRequest(h3URL.URL, headers)
runtimex.PanicOnError(err, "newRequest failed")
resp, err := clnt.Do(req)
if err != nil {
return nil, err
}
// Note that we ignore the response body.
defer resp.Body.Close()
return resp, nil
}
func (e *DefaultExplorer) newRequest(URL *url.URL, headers map[string][]string) (*http.Request, error) {
req, err := http.NewRequest("GET", URL.String(), nil)
if err != nil {
return nil, err
}
for k, vs := range headers {
switch strings.ToLower(k) {
case "user-agent", "accept", "accept-language":
for _, v := range vs {
req.Header.Add(k, v)
}
}
}
return req, nil
}

156
internal/cmd/oohelperd/internal/nwcth/explore_test.go Normal file
View File

@ -0,0 +1,156 @@
package nwcth
import (
"net/http"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
var explorer = &DefaultExplorer{resolver: newResolver()}
func TestExploreSuccess(t *testing.T) {
u, err := url.Parse("https://example.com")
runtimex.PanicOnError(err, "url.Parse failed")
rts, err := explorer.Explore(u, nil)
if err != nil {
t.Fatal("unexpected error")
}
if len(rts) != 1 {
t.Fatal("unexpected number of roundtrips")
}
}
func TestExploreFailure(t *testing.T) {
u, err := url.Parse("https://example.example")
runtimex.PanicOnError(err, "url.Parse failed")
rts, err := explorer.Explore(u, nil)
if err == nil {
t.Fatal("expected an error here")
}
if rts != nil {
t.Fatal("rts should be nil")
}
}
func TestExploreSuccessWithH3(t *testing.T) {
// TODO(bassosimone): figure out why this happens.
t.Skip("this test does not work in GHA")
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rts, err := explorer.Explore(u, nil)
if err != nil {
t.Fatal("unexpected error")
}
if len(rts) != 2 {
t.Fatal("unexpected number of roundtrips")
}
if rts[0].Proto != "https" {
t.Fatal("unexpected protocol")
}
if rts[1].Proto != "h3" {
t.Fatal("unexpected protocol")
}
}
func TestGetSuccess(t *testing.T) {
u, err := url.Parse("https://example.com")
resp, err := explorer.get(u, nil)
if err != nil {
t.Fatal("unexpected error")
}
if resp == nil {
t.Fatal("unexpected nil response")
}
buf := make([]byte, 100)
if n, _ := resp.Body.Read(buf); n != 0 {
t.Fatal("expected response body tom be closed")
}
}
func TestGetFailure(t *testing.T) {
u, err := url.Parse("https://example.example")
resp, err := explorer.get(u, nil)
if err == nil {
t.Fatal("expected an error here")
}
if resp != nil {
t.Fatal("response should be nil")
}
}
func TestGetH3Success(t *testing.T) {
u, err := url.Parse("https://www.google.com")
h3u := &h3URL{URL: u, proto: "h3"}
resp, err := explorer.getH3(h3u, nil)
if err != nil {
t.Fatal("unexpected error")
}
if resp == nil {
t.Fatal("unexpected nil response")
}
buf := make([]byte, 100)
if n, _ := resp.Body.Read(buf); n != 0 {
t.Fatal("expected response body tom be closed")
}
}
func TestGetH3Failure(t *testing.T) {
u, err := url.Parse("https://www.google.google")
h3u := &h3URL{URL: u, proto: "h3"}
resp, err := explorer.getH3(h3u, nil)
if err == nil {
t.Fatal("expected an error here")
}
if resp != nil {
t.Fatal("response should be nil")
}
}
func TestRearrange(t *testing.T) {
u, err := url.Parse("https://example.com")
runtimex.PanicOnError(err, "url.Parse failed")
resp := &http.Response{
// the ProtoMajor field identifies the request/response structs and indicates the correct order
ProtoMajor: 2,
Request: &http.Request{
ProtoMajor: 2,
URL: u,
Response: &http.Response{
ProtoMajor: 1,
Request: &http.Request{
ProtoMajor: 1,
URL: u,
Response: &http.Response{
ProtoMajor: 0,
Request: &http.Request{
ProtoMajor: 0,
URL: u,
},
},
},
},
},
}
h3URL := &h3URL{URL: u, proto: "expected"}
rts := explorer.rearrange(resp, h3URL)
expectedIndex := 0
for _, rt := range rts {
if rt.Request == nil || rt.Response == nil {
t.Fatal("unexpected nil value")
}
if rt.Request.ProtoMajor != expectedIndex {
t.Fatal("unexpected order")
}
if rt.Response.ProtoMajor != expectedIndex {
t.Fatal("unexpected order")
}
if rt.Proto != h3URL.proto {
t.Fatal("unexpected protocol")
}
expectedIndex += 1
}
}

96
internal/cmd/oohelperd/internal/nwcth/factory.go Normal file
View File

@ -0,0 +1,96 @@
package nwcth
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"sync"
"github.com/apex/log"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
"github.com/ooni/probe-cli/v3/internal/errorsx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
var ErrNoConnReuse = errors.New("cannot reuse connection")
// NewDialerResolver contructs a new dialer for TCP connections,
// with default, errorwrapping and resolve functionalities
func NewDialerResolver(resolver netxlite.Resolver) netxlite.Dialer {
var d netxlite.Dialer = netxlite.DefaultDialer
d = &errorsx.ErrorWrapperDialer{Dialer: d}
d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d}
return d
}
// NewQUICDialerResolver creates a new QUICDialerResolver
// with default, errorwrapping and resolve functionalities
func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDialer {
var ql quicdialer.QUICListener = &netxlite.QUICListenerStdlib{}
ql = &errorsx.ErrorWrapperQUICListener{QUICListener: ql}
var dialer netxlite.QUICContextDialer = &netxlite.QUICDialerQUICGo{
QUICListener: ql,
}
dialer = &errorsx.ErrorWrapperQUICDialer{Dialer: dialer}
dialer = &netxlite.QUICDialerResolver{Resolver: resolver, Dialer: dialer}
return dialer
}
// NewSingleH3Transport creates an http3.RoundTripper
func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper {
transport := &http3.RoundTripper{
DisableCompression: true,
TLSClientConfig: tlscfg,
QuicConfig: qcfg,
Dial: (&SingleDialerH3{qsess: &qsess}).Dial,
}
return transport
}
// NewSingleTransport determines the appropriate HTTP Transport from the ALPN
func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) {
singledialer := &SingleDialer{conn: &conn}
transport = http.DefaultTransport.(*http.Transport).Clone()
transport.(*http.Transport).DialContext = singledialer.DialContext
transport.(*http.Transport).DialTLSContext = singledialer.DialContext
transport.(*http.Transport).DisableCompression = true
transport.(*http.Transport).MaxConnsPerHost = 1
transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)}
return transport
}
type SingleDialer struct {
sync.Mutex
conn *net.Conn
}
func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) {
s.Lock()
defer s.Unlock()
if s.conn == nil {
return nil, ErrNoConnReuse
}
c := s.conn
s.conn = nil
return *c, nil
}
type SingleDialerH3 struct {
sync.Mutex
qsess *quic.EarlySession
}
func (s *SingleDialerH3) Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
s.Lock()
defer s.Unlock()
if s.qsess == nil {
return nil, ErrNoConnReuse
}
qs := s.qsess
s.qsess = nil
return *qs, nil
}

287
internal/cmd/oohelperd/internal/nwcth/generate.go Normal file
View File

@ -0,0 +1,287 @@
package nwcth
import (
"context"
"crypto/tls"
"net"
"net/http"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// Generate is the third step of the algorithm. Given the
// observed round trips, we generate measurement targets and
// execute those measurements so the probe has a benchmark.
// Generator is the interface responsible for running Generate.
type Generator interface {
Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error)
}
// DefaultGenerator is the default Generator.
type DefaultGenerator struct {
dialer netxlite.Dialer
quicDialer netxlite.QUICContextDialer
resolver netxlite.Resolver
transport http.RoundTripper
}
// the testhelper uses the same network operations as websteps
var (
DNSDo = websteps.DNSDo
TCPDo = websteps.TCPDo
QUICDo = websteps.QUICDo
TLSDo = websteps.TLSDo
HTTPDo = websteps.HTTPDo
)
// Generate takes in input a list of round trips and outputs
// a list of connectivity measurements for each of them.
func (g *DefaultGenerator) Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) {
var out []*URLMeasurement
for _, rt := range rts {
currentURL := g.GenerateURL(ctx, rt, clientResolutions)
out = append(out, currentURL)
}
return out, nil
}
// GenerateURL returns a URLMeasurement.
func (g *DefaultGenerator) GenerateURL(ctx context.Context, rt *RoundTrip, clientResolutions []string) *URLMeasurement {
addrs, err := DNSDo(ctx, websteps.DNSConfig{
Domain: rt.Request.URL.Hostname(),
Resolver: g.resolver,
})
currentURL := &URLMeasurement{
DNS: &DNSMeasurement{
Domain: rt.Request.URL.Hostname(),
Addrs: addrs,
Failure: newfailure(err),
},
RoundTrip: rt,
URL: rt.Request.URL.String(),
}
addrs = g.mergeAddresses(addrs, clientResolutions)
if len(addrs) == 0 {
return currentURL
}
for _, addr := range addrs {
var port string
explicitPort := rt.Request.URL.Port()
scheme := rt.Request.URL.Scheme
switch {
case explicitPort != "":
port = explicitPort
case scheme == "http":
port = "80"
case scheme == "https":
port = "443"
default:
panic("should not happen")
}
endpoint := net.JoinHostPort(addr, port)
var currentEndpoint *EndpointMeasurement
_, h3 := websteps.SupportedQUICVersions[rt.Proto]
switch {
case h3:
currentEndpoint = g.GenerateH3Endpoint(ctx, rt, endpoint)
case rt.Proto == "http":
currentEndpoint = g.GenerateHTTPEndpoint(ctx, rt, endpoint)
case rt.Proto == "https":
currentEndpoint = g.GenerateHTTPSEndpoint(ctx, rt, endpoint)
default:
// TODO(kelmenhorst): do we have to register this error somewhere in the result struct?
continue
}
currentURL.Endpoints = append(currentURL.Endpoints, currentEndpoint)
}
return currentURL
}
// GenerateHTTPEndpoint performs an HTTP Request by
// a) establishing a TCP connection to the target (TCPDo),
// b) performing an HTTP GET request to the endpoint (HTTPDo).
// It returns an EndpointMeasurement.
func (g *DefaultGenerator) GenerateHTTPEndpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement {
currentEndpoint := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: "http",
}
tcpConn, err := TCPDo(ctx, websteps.TCPConfig{
Dialer: g.dialer,
Endpoint: endpoint,
Resolver: g.resolver,
})
currentEndpoint.TCPConnectMeasurement = &TCPConnectMeasurement{
Failure: newfailure(err),
}
if err != nil {
return currentEndpoint
}
defer tcpConn.Close()
// prepare HTTPRoundTripMeasurement of this endpoint
currentEndpoint.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: rt.Request.Header,
Method: "GET",
URL: rt.Request.URL.String(),
},
}
transport := NewSingleTransport(tcpConn)
if g.transport != nil {
transport = g.transport
}
resp, body, err := HTTPDo(rt.Request, transport)
if err != nil {
// failed Response
currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
Failure: newfailure(err),
}
return currentEndpoint
}
// successful Response
currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return currentEndpoint
}
// GenerateHTTPSEndpoint performs an HTTPS Request by
// a) establishing a TCP connection to the target (TCPDo),
// b) establishing a TLS connection to the target (TLSDo),
// c) performing an HTTP GET request to the endpoint (HTTPDo).
// It returns an EndpointMeasurement.
func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement {
currentEndpoint := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: "https",
}
var tcpConn, tlsConn net.Conn
tcpConn, err := TCPDo(ctx, websteps.TCPConfig{
Dialer: g.dialer,
Endpoint: endpoint,
Resolver: g.resolver,
})
currentEndpoint.TCPConnectMeasurement = &TCPConnectMeasurement{
Failure: newfailure(err),
}
if err != nil {
return currentEndpoint
}
defer tcpConn.Close()
tlsConn, err = TLSDo(tcpConn, rt.Request.URL.Hostname())
currentEndpoint.TLSHandshakeMeasurement = &TLSHandshakeMeasurement{
Failure: newfailure(err),
}
if err != nil {
return currentEndpoint
}
defer tlsConn.Close()
// prepare HTTPRoundTripMeasurement of this endpoint
currentEndpoint.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: rt.Request.Header,
Method: "GET",
URL: rt.Request.URL.String(),
},
}
transport := NewSingleTransport(tlsConn)
if g.transport != nil {
transport = g.transport
}
resp, body, err := HTTPDo(rt.Request, transport)
if err != nil {
// failed Response
currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
Failure: newfailure(err),
}
return currentEndpoint
}
// successful Response
currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return currentEndpoint
}
// GenerateH3Endpoint performs an HTTP/3 Request by
// a) establishing a QUIC connection to the target (QUICDo),
// b) performing an HTTP GET request to the endpoint (HTTPDo).
// It returns an EndpointMeasurement.
func (g *DefaultGenerator) GenerateH3Endpoint(ctx context.Context, rt *RoundTrip, endpoint string) *EndpointMeasurement {
currentEndpoint := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: rt.Proto,
}
tlsConf := &tls.Config{
ServerName: rt.Request.URL.Hostname(),
NextProtos: []string{rt.Proto},
}
sess, err := QUICDo(ctx, websteps.QUICConfig{
Endpoint: endpoint,
QUICDialer: g.quicDialer,
TLSConf: tlsConf,
Resolver: g.resolver,
})
currentEndpoint.QUICHandshakeMeasurement = &TLSHandshakeMeasurement{
Failure: newfailure(err),
}
if err != nil {
return currentEndpoint
}
// prepare HTTPRoundTripMeasurement of this endpoint
currentEndpoint.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: rt.Request.Header,
Method: "GET",
URL: rt.Request.URL.String(),
},
}
var transport http.RoundTripper = NewSingleH3Transport(sess, tlsConf, &quic.Config{})
if g.transport != nil {
transport = g.transport
}
resp, body, err := HTTPDo(rt.Request, transport)
if err != nil {
// failed Response
currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
Failure: newfailure(err),
}
return currentEndpoint
}
// successful Response
currentEndpoint.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return currentEndpoint
}
// mergeAddresses creates a (duplicate-free) union set of the IP addresses provided by the client,
// and the addresses resulting from the testhelper's DNS step
func (g *DefaultGenerator) mergeAddresses(addrs []string, clientAddrs []string) (out []string) {
unique := make(map[string]bool, len(addrs)+len(clientAddrs))
for _, a := range addrs {
unique[a] = true
}
for _, a := range clientAddrs {
unique[a] = true
}
for key := range unique {
out = append(out, key)
}
return out
}

461
internal/cmd/oohelperd/internal/nwcth/generate_test.go Normal file
View File

@ -0,0 +1,461 @@
package nwcth
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/url"
"strings"
"testing"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/errorsx"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
var generator = &DefaultGenerator{resolver: newResolver()}
type fakeTransport struct {
err error
}
func (txp fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return nil, txp.err
}
func (txp fakeTransport) CloseIdleConnections() {}
type fakeQUICDialer struct {
err error
}
func (d fakeQUICDialer) DialContext(ctx context.Context, network, address string,
tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlySession, error) {
return nil, d.err
}
type fakeDialer struct {
err error
}
func (d fakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
return nil, d.err
}
func TestGenerateDNSFailure(t *testing.T) {
u, err := url.Parse("https://www.google.google")
runtimex.PanicOnError(err, "url.Parse failed")
rts := []*RoundTrip{
{
Proto: "https",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
}
urlMeasurements, err := generator.Generate(context.Background(), rts, []string{})
if err != nil {
t.Fatal("unexpected error")
}
if len(urlMeasurements) != 1 {
t.Fatal("unexpected urlMeasurements length")
}
if urlMeasurements[0].DNS == nil {
t.Fatal("DNS should not be nil")
}
if urlMeasurements[0].DNS.Failure == nil || *urlMeasurements[0].DNS.Failure != errorsx.FailureDNSNXDOMAINError {
t.Fatal("unexpected DNS failure type")
}
}
func TestGenerate(t *testing.T) {
u, err := url.Parse("http://www.google.com")
u2, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rts := []*RoundTrip{
{
Proto: "http",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
{
Proto: "https",
Request: &http.Request{
URL: u2,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
{
Proto: "h3",
Request: &http.Request{
URL: u2,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
}
urlMeasurements, err := generator.Generate(context.Background(), rts, []string{})
if err != nil {
t.Fatal("unexpected err")
}
if urlMeasurements == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if len(urlMeasurements) < 3 {
t.Fatal("unexpected number of urlMeasurements", len(urlMeasurements))
}
}
func TestGenerateUnexpectedProtocol(t *testing.T) {
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rts := []*RoundTrip{
{
Proto: "h3-27",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
}
urlMeasurements, err := generator.Generate(context.Background(), rts, []string{})
if err != nil {
t.Fatal("unexpected err")
}
if urlMeasurements == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if len(urlMeasurements) != 1 {
t.Fatal("unexpected number of urlMeasurements")
}
measurement := urlMeasurements[0]
if measurement.URL != u.String() {
t.Fatal("unexpected URL")
}
if measurement.DNS == nil {
t.Fatal("DNS should not be nil")
}
if measurement.RoundTrip == nil {
t.Fatal("RoundTrip should not be nil")
}
if measurement.Endpoints != nil {
t.Fatal("Endpoints should be nil")
}
}
func TestGenerateURLWithClientResolutions(t *testing.T) {
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "h3",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
clientResolution := "142.250.186.36"
urlMeasurement := generator.GenerateURL(context.Background(), rt, []string{clientResolution})
if err != nil {
t.Fatal("unexpected err")
}
if urlMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if urlMeasurement.DNS == nil {
t.Fatal("DNS should not be nil")
}
if len(urlMeasurement.Endpoints) < 2 {
t.Fatal("unexpected number of endpoints")
}
clientAddrsFound := false
for _, e := range urlMeasurement.Endpoints {
if e.Endpoint == clientResolution+":443" {
clientAddrsFound = true
}
}
if !clientAddrsFound {
t.Fatal("did not use provided client resolution")
}
}
func TestGenerateHTTP(t *testing.T) {
u, err := url.Parse("http://example.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "http",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateHTTPEndpoint(context.Background(), rt, "93.184.216.34:80")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if endpointMeasurement.TCPConnectMeasurement == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.HTTPRoundTripMeasurement == nil {
t.Fatal("HTTPRoundTripMeasurement should not be nil")
}
}
func TestGenerateHTTPS(t *testing.T) {
u, err := url.Parse("https://example.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "https",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "93.184.216.34:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if endpointMeasurement.TCPConnectMeasurement == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshakeMeasurement == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.HTTPRoundTripMeasurement == nil {
t.Fatal("HTTPRoundTripMeasurement should not be nil")
}
}
func TestGenerateHTTPSTLSFailure(t *testing.T) {
u, err := url.Parse("https://wrong.host.badssl.com/")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "https",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "104.154.89.105:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if endpointMeasurement.TCPConnectMeasurement == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshakeMeasurement == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.HTTPRoundTripMeasurement != nil {
t.Fatal("HTTPRoundTripMeasurement should be nil")
}
}
func TestGenerateH3(t *testing.T) {
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "h3",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateH3Endpoint(context.Background(), rt, "173.194.76.103:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement == nil {
t.Fatal("unexpected nil urlMeasurement")
}
if endpointMeasurement.QUICHandshakeMeasurement == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.HTTPRoundTripMeasurement == nil {
t.Fatal("HTTPRoundTripMeasurement should not be nil")
}
}
func TestGenerateTCPDoFails(t *testing.T) {
expected := errors.New("expected")
generator := &DefaultGenerator{
dialer: fakeDialer{err: expected},
resolver: newResolver(),
}
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "https",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateHTTPSEndpoint(context.Background(), rt, "173.194.76.103:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement.TCPConnectMeasurement == nil {
t.Fatal("QUIC handshake should not be nil")
}
if endpointMeasurement.TCPConnectMeasurement.Failure == nil {
t.Fatal("expected an error here")
}
if *endpointMeasurement.TCPConnectMeasurement.Failure != *newfailure(expected) {
t.Fatal("unexpected error type")
}
}
func TestGenerateQUICDoFails(t *testing.T) {
expected := errors.New("expected")
generator := &DefaultGenerator{
quicDialer: fakeQUICDialer{err: expected},
resolver: newResolver(),
}
u, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rt := &RoundTrip{
Proto: "h3",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
}
endpointMeasurement := generator.GenerateH3Endpoint(context.Background(), rt, "173.194.76.103:443")
if err != nil {
t.Fatal("unexpected err")
}
if endpointMeasurement.QUICHandshakeMeasurement == nil {
t.Fatal("QUIC handshake should not be nil")
}
if endpointMeasurement.QUICHandshakeMeasurement.Failure == nil {
t.Fatal("expected an error here")
}
if *endpointMeasurement.QUICHandshakeMeasurement.Failure != *newfailure(expected) {
t.Fatal("unexpected error type")
}
}
func TestGenerateHTTPDoFails(t *testing.T) {
expected := errors.New("expected")
generator := &DefaultGenerator{
transport: fakeTransport{err: expected},
resolver: newResolver(),
}
u, err := url.Parse("http://www.google.com")
u2, err := url.Parse("https://www.google.com")
runtimex.PanicOnError(err, "url.Parse failed")
rts := []*RoundTrip{
{
Proto: "http",
Request: &http.Request{
URL: u,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
{
Proto: "https",
Request: &http.Request{
URL: u2,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
{
Proto: "h3",
Request: &http.Request{
URL: u2,
},
Response: &http.Response{
StatusCode: 200,
},
SortIndex: 0,
},
}
urlMeasurements, err := generator.Generate(context.Background(), rts, []string{})
if err != nil {
t.Fatal("unexpected err")
}
if len(urlMeasurements) != 3 {
t.Fatal("unexpected number of urlMeasurements")
}
for _, u := range urlMeasurements {
if u.DNS == nil {
t.Fatal("unexpected DNS failure")
}
if len(u.Endpoints) < 1 {
t.Fatal("unexpected number of endpoints", len(u.Endpoints))
}
// this can occur when the network is unreachable, but it is irrelevant for checking HTTP behavior
if u.Endpoints[0].TCPConnectMeasurement != nil && u.Endpoints[0].TCPConnectMeasurement.Failure != nil {
continue
}
if u.Endpoints[0].QUICHandshakeMeasurement != nil && u.Endpoints[0].QUICHandshakeMeasurement.Failure != nil {
continue
}
if u.Endpoints[0].HTTPRoundTripMeasurement == nil {
t.Fatal("roundtrip should not be nil", u.Endpoints[0].TCPConnectMeasurement.Failure, "jaaaa")
}
if u.Endpoints[0].HTTPRoundTripMeasurement.Response == nil {
t.Fatal("roundtrip response should not be nil")
}
if u.Endpoints[0].HTTPRoundTripMeasurement.Response.Failure == nil {
t.Fatal("expected an HTTP error")
}
if !strings.HasSuffix(*u.Endpoints[0].HTTPRoundTripMeasurement.Response.Failure, expected.Error()) {
t.Fatal("unexpected failure type")
}
}
}

72
internal/cmd/oohelperd/internal/nwcth/h3.go Normal file
View File

@ -0,0 +1,72 @@
package nwcth
import (
"errors"
"net"
"net/http"
"net/url"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
type h3URL struct {
URL *url.URL
proto string
}
type altSvcH3 struct {
authority string
proto string
}
var ErrNoH3Location = errors.New("no h3 location found")
// getH3URL returns the URL for HTTP/3 requests, if the target supports HTTP/3.
// Returns nil, if no HTTP/3 support is advertised.
func getH3URL(resp *http.Response) (*h3URL, error) {
URL := resp.Request.URL
if URL == nil {
return nil, ErrInvalidURL
}
h3Svc, err := parseAltSvc(resp, URL)
if err != nil {
return nil, err
}
quicURL, err := url.Parse(URL.String())
runtimex.PanicOnError(err, "url.Parse failed")
quicURL.Host = h3Svc.authority
return &h3URL{URL: quicURL, proto: h3Svc.proto}, nil
}
// parseAltSvc parses the Alt-Svc HTTP header for entries advertising the use of H3
func parseAltSvc(resp *http.Response, URL *url.URL) (*altSvcH3, error) {
// TODO(bassosimone,kelmenhorst): see if we can make this algorithm more robust.
if URL.Scheme != "https" {
return nil, ErrUnsupportedScheme
}
alt_svc := resp.Header.Get("Alt-Svc")
// syntax: Alt-Svc: <protocol-id>=<alt-authority>; ma=<max-age>; persist=1
entries := strings.Split(alt_svc, ",")
for _, e := range entries {
keyvalpairs := strings.Split(e, ";")
for _, p := range keyvalpairs {
p = strings.Replace(p, "\"", "", -1)
kv := strings.Split(p, "=")
if len(kv) != 2 {
continue
}
if _, ok := websteps.SupportedQUICVersions[kv[0]]; ok {
host, port, err := net.SplitHostPort(kv[1])
runtimex.PanicOnError(err, "net.SplitHostPort failed")
if host == "" {
host = URL.Hostname()
}
authority := net.JoinHostPort(host, port)
return &altSvcH3{authority: authority, proto: kv[0]}, nil
}
}
}
return nil, ErrNoH3Location
}

39
internal/cmd/oohelperd/internal/nwcth/h3_test.go Normal file
View File

@ -0,0 +1,39 @@
package nwcth
import (
"net/http"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func TestGetH3URLInvalidURL(t *testing.T) {
resp := &http.Response{
Request: &http.Request{},
}
h3URL, err := getH3URL(resp)
if err == nil {
t.Fatal("expected an error here")
}
if h3URL != nil {
t.Fatal("h3URL should be nil")
}
}
func TestParseUnsupportedScheme(t *testing.T) {
URL, err := url.Parse("h3://google.com")
runtimex.PanicOnError(err, "url.Parse failed")
parsed, err := parseAltSvc(nil, URL)
if err == nil {
t.Fatal("expected an error here")
}
if err != ErrUnsupportedScheme {
t.Fatal("unexpected error type")
}
if parsed != nil {
t.Fatal("h3URL should be nil")
}
}

61
internal/cmd/oohelperd/internal/nwcth/initialchecks.go Normal file
View File

@ -0,0 +1,61 @@
package nwcth
import (
"context"
"errors"
"net/url"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// InitialChecks is the first step of the test helper algorithm. We
// make sure we can parse the URL, we handle the scheme, and the domain
// name inside the URL's authority is valid.
// Errors returned by Preresolve.
var (
// ErrInvalidURL indicates that the URL is invalid.
ErrInvalidURL = errors.New("the URL is invalid")
// ErrUnsupportedScheme indicates that we don't support the scheme.
ErrUnsupportedScheme = errors.New("unsupported scheme")
// ErrNoSuchHost indicates that the DNS resolution failed.
ErrNoSuchHost = errors.New("no such host")
)
// InitChecker is the interface responsible for running InitialChecks.
type InitChecker interface {
InitialChecks(URL string) (*url.URL, error)
}
// DefaultInitChecker is the default InitChecker.
type DefaultInitChecker struct {
resolver netxlite.Resolver
}
// InitialChecks checks whether the URL is valid and whether the
// domain inside the URL is an existing one. If these preliminary
// checks fail, there's no point in continuing.
// If they succeed, InitialChecks returns the URL, if not an error.
func (i *DefaultInitChecker) InitialChecks(URL string) (*url.URL, error) {
parsed, err := url.Parse(URL)
if err != nil {
return nil, ErrInvalidURL
}
switch parsed.Scheme {
case "http", "https":
default:
return nil, ErrUnsupportedScheme
}
// Assumptions:
//
// 1. the resolver will cache the resolution for later
//
// 2. an IP address does not cause an error because we are using
// a resolve that behaves like getaddrinfo
if _, err := i.resolver.LookupHost(context.Background(), parsed.Hostname()); err != nil {
return nil, ErrNoSuchHost
}
return parsed, nil
}

31
internal/cmd/oohelperd/internal/nwcth/initialchecks_test.go Normal file
View File

@ -0,0 +1,31 @@
package nwcth
import (
"testing"
)
var checker = &DefaultInitChecker{resolver: newResolver()}
func TestMeasureWithInvalidURL(t *testing.T) {
_, err := checker.InitialChecks("http://[::1]aaaa")
if err == nil || err != ErrInvalidURL {
t.Fatal("expected an error here")
}
}
func TestMeasureWithUnsupportedScheme(t *testing.T) {
_, err := checker.InitialChecks("abc://example.com")
if err == nil || err != ErrUnsupportedScheme {
t.Fatal("expected an error here")
}
}
func TestMeasureWithInvalidHost(t *testing.T) {
_, err := checker.InitialChecks("http://www.ooni.ooni")
if err == nil || err != ErrNoSuchHost {
t.Fatal("expected an error here")
}
}

96
internal/cmd/oohelperd/internal/nwcth/measure.go Normal file
View File

@ -0,0 +1,96 @@
package nwcth
import (
"context"
"errors"
"net/url"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
type (
CtrlRequest = websteps.CtrlRequest
ControlResponse = websteps.ControlResponse
)
var ErrInternalServer = errors.New("Internal server failure")
// Config contains the building blocks of the testhelper algorithm
type Config struct {
checker InitChecker
explorer Explorer
generator Generator
resolver netxlite.Resolver
}
// Measure performs the three consecutive steps of the testhelper algorithm:
// 1. InitialChecks
// 2. Explore
// 3. Generate
func Measure(ctx context.Context, creq *CtrlRequest, config *Config) (*ControlResponse, error) {
var (
URL *url.URL
err error
)
resolver := config.resolver
if resolver == nil {
// use a central resolver
resolver = newResolver()
}
checker := config.checker
if checker == nil {
checker = &DefaultInitChecker{resolver: resolver}
}
URL, err = checker.InitialChecks(creq.HTTPRequest)
if err != nil {
// return a valid response in case of NXDOMAIN so the probe can compare the failure
if err == ErrNoSuchHost {
return newDNSFailedResponse(err, creq.HTTPRequest), nil
}
return nil, err
}
explorer := config.explorer
if explorer == nil {
explorer = &DefaultExplorer{resolver: resolver}
}
rts, err := explorer.Explore(URL, creq.HTTPRequestHeaders)
if err != nil {
return nil, ErrInternalServer
}
generator := config.generator
if generator == nil {
generator = &DefaultGenerator{resolver: resolver}
}
meas, err := generator.Generate(ctx, rts, creq.Addrs)
if err != nil {
return nil, err
}
return &ControlResponse{URLMeasurements: meas}, nil
}
// newDNSFailedResponse creates a new response with one URLMeasurement entry
// indicating that the DNS step failed
func newDNSFailedResponse(err error, URL string) *ControlResponse {
resp := &ControlResponse{}
m := &URLMeasurement{
URL: URL,
DNS: &DNSMeasurement{
Failure: newfailure(err),
},
}
resp.URLMeasurements = append(resp.URLMeasurements, m)
return resp
}
// newResolver creates a new DNS resolver instance
func newResolver() netxlite.Resolver {
childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google")
runtimex.PanicOnError(err, "NewDNSClient failed")
var r netxlite.Resolver = childResolver
r = &netxlite.IDNAResolver{Resolver: r}
return r
}

117
internal/cmd/oohelperd/internal/nwcth/measure_test.go Normal file
View File

@ -0,0 +1,117 @@
package nwcth
import (
"context"
"errors"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/errorsx"
)
func TestMeasureSuccess(t *testing.T) {
req := &CtrlRequest{
HTTPRequest: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{})
if err != nil {
t.Fatal("unexpected error")
}
if resp == nil {
t.Fatal("unexpected nil response")
}
}
type MockChecker struct {
err error
}
func (c *MockChecker) InitialChecks(URL string) (*url.URL, error) {
return nil, c.err
}
type MockExplorer struct{}
func (c *MockExplorer) Explore(URL *url.URL, headers map[string][]string) ([]*RoundTrip, error) {
return nil, ErrExpectedExplore
}
type MockGenerator struct{}
func (c *MockGenerator) Generate(ctx context.Context, rts []*RoundTrip, clientResolutions []string) ([]*URLMeasurement, error) {
return nil, ErrExpectedGenerate
}
var ErrExpectedCheck error = errors.New("expected error checker")
var ErrExpectedExplore error = errors.New("expected error explorer")
var ErrExpectedGenerate error = errors.New("expected error generator")
func TestMeasureInitialChecksFail(t *testing.T) {
req := &CtrlRequest{
HTTPRequest: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{checker: &MockChecker{err: ErrExpectedCheck}})
if err == nil {
t.Fatal("expected an error here")
}
if err != ErrExpectedCheck {
t.Fatal("unexpected error type")
}
if resp != nil {
t.Fatal("resp should be nil")
}
}
func TestMeasureInitialChecksFailWithNXDOMAIN(t *testing.T) {
req := &CtrlRequest{
HTTPRequest: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{checker: &MockChecker{err: ErrNoSuchHost}})
if err != nil {
t.Fatal("unexpected error")
}
if resp == nil {
t.Fatal("resp should not be nil")
}
if len(resp.URLMeasurements) != 1 {
t.Fatal("unexpected number of measurements")
}
if resp.URLMeasurements[0].DNS == nil {
t.Fatal("DNS entry should not be nil")
}
if *resp.URLMeasurements[0].DNS.Failure != errorsx.FailureDNSNXDOMAINError {
t.Fatal("unexpected failure")
}
}
func TestMeasureExploreFails(t *testing.T) {
req := &CtrlRequest{
HTTPRequest: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{explorer: &MockExplorer{}})
if err == nil {
t.Fatal("expected an error here")
}
if err != ErrInternalServer {
t.Fatal("unexpected error type")
}
if resp != nil {
t.Fatal("resp should be nil")
}
}
func TestMeasureGenerateFails(t *testing.T) {
req := &CtrlRequest{
HTTPRequest: "https://example.com",
}
resp, err := Measure(context.Background(), req, &Config{generator: &MockGenerator{}})
if err == nil {
t.Fatal("expected an error here")
}
if err != ErrExpectedGenerate {
t.Fatal("unexpected error type")
}
if resp != nil {
t.Fatal("resp should be nil")
}
}

17
internal/cmd/oohelperd/internal/nwcth/model.go Normal file
View File

@ -0,0 +1,17 @@
package nwcth
import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
)
type (
URLMeasurement = websteps.URLMeasurement
DNSMeasurement = websteps.DNSMeasurement
EndpointMeasurement = websteps.EndpointMeasurement
TCPConnectMeasurement = websteps.TCPConnectMeasurement
HTTPRoundTripMeasurement = websteps.HTTPRoundTripMeasurement
TLSHandshakeMeasurement = websteps.TLSHandshakeMeasurement
HTTPRequestMeasurement = websteps.HTTPRequestMeasurement
HTTPResponseMeasurement = websteps.HTTPResponseMeasurement
RoundTrip = websteps.RoundTrip
)

67
internal/cmd/oohelperd/internal/nwcth/nwcth.go Normal file
View File

@ -0,0 +1,67 @@
// Package nwcth implements the new web connectivity test helper.
//
// See https://github.com/ooni/spec/blob/master/backends/th-007-nwcth.md
package nwcth
import (
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version"
)
// newfailure is a convenience shortcut to save typing
var newfailure = archival.NewFailure
// maxAcceptableBody is _at the same time_ the maximum acceptable body for incoming
// API requests and the maximum acceptable body when fetching arbitrary URLs. See
// https://github.com/ooni/probe/issues/1727 for statistics regarding the test lists
// including the empirical CDF of the body size for test lists URLs.
const maxAcceptableBody = 1 << 24
// Handler implements the Web Connectivity test helper HTTP API.
type Handler struct {
Config *Config
}
// ServeHTTP implements http.Handler.ServeHTTP.
func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Add("Server", fmt.Sprintf(
"oohelperd/%s ooniprobe-engine/%s", version.Version, version.Version,
))
if req.Method != "POST" {
w.WriteHeader(400)
return
}
reader := &io.LimitedReader{R: req.Body, N: maxAcceptableBody}
data, err := iox.ReadAllContext(req.Context(), reader)
if err != nil {
w.WriteHeader(400)
return
}
var creq CtrlRequest
if err := json.Unmarshal(data, &creq); err != nil {
w.WriteHeader(400)
return
}
cresp, err := Measure(req.Context(), &creq, h.Config)
if err != nil {
if err == ErrInternalServer {
w.WriteHeader(500)
return
}
w.WriteHeader(400)
return
}
// We assume that the following call cannot fail because it's a
// clearly serializable data structure.
data, err = json.Marshal(cresp)
runtimex.PanicOnError(err, "json.Marshal failed")
w.Header().Add("Content-Type", "application/json")
w.Write(data)
}

281
internal/cmd/oohelperd/internal/nwcth/nwcth_test.go Normal file
View File

@ -0,0 +1,281 @@
package nwcth
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
const requestnoredirect = `{
"url": "https://ooni.org",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"104.198.14.52:443"
]
}`
const requestredirect = `{
"url": "https://www.ooni.org",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"18.192.76.182:443"
]
}`
const requestIPaddressinput = `{
"url": "https://172.217.168.4",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"172.217.168.4:443"
]
}`
const requestwithquic = `{
"url": "https://www.google.com",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"142.250.74.196:443"
]
}`
const requestWithoutDomainName = `{
"url": "https://8.8.8.8",
"headers": {
"Accept": [
"*/*"
],
"Accept-Language": [
"en-US;q=0.8,en;q=0.5"
],
"User-Agent": [
"Mozilla/5.0"
]
},
"addrs": [
"8.8.8.8:443"
]
}`
func TestWorkingAsIntended(t *testing.T) {
handler := Handler{Config: &Config{}}
srv := httptest.NewServer(handler)
defer srv.Close()
type expectationSpec struct {
name string
reqMethod string
reqContentType string
reqBody string
respStatusCode int
respContentType string
parseBody bool
}
expectations := []expectationSpec{{
name: "check for invalid method",
reqMethod: "GET",
respStatusCode: 400,
}, {
name: "check for invalid request body",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: "{",
respStatusCode: 400,
}, {
name: "with measurement failure",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: `{"url": "http://[::1]aaaa"}`,
respStatusCode: 400,
}, {
name: "request without redirect or H3 follow-up request",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestnoredirect,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}, {
name: "request triggering one redirect, without H3 follow-up request",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestredirect,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}, {
name: "request with an IP address as input",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestIPaddressinput,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}, {
name: "request triggering H3 follow-up request, without redirect",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestwithquic,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}, {
name: "when there's no domain name in the request",
reqMethod: "POST",
reqContentType: "application/json",
reqBody: requestWithoutDomainName,
respStatusCode: 200,
respContentType: "application/json",
parseBody: true,
}}
for _, expect := range expectations {
t.Run(expect.name, func(t *testing.T) {
body := strings.NewReader(expect.reqBody)
req, err := http.NewRequest(expect.reqMethod, srv.URL, body)
if err != nil {
t.Fatalf("%s: %+v", expect.name, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("%s: %+v", expect.name, err)
}
defer resp.Body.Close()
if resp.StatusCode != expect.respStatusCode {
t.Fatalf("unexpected status code: %+v", resp.StatusCode)
}
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
if !expect.parseBody {
return
}
var v interface{}
if err := json.Unmarshal(data, &v); err != nil {
t.Fatal(err)
}
})
}
}
func TestHandlerWithInternalServerError(t *testing.T) {
handler := Handler{Config: &Config{explorer: &MockExplorer{}}}
srv := httptest.NewServer(handler)
defer srv.Close()
body := strings.NewReader(`{"url": "https://example.com"}`)
req, err := http.NewRequest("POST", srv.URL, body)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 500 {
t.Fatalf("unexpected status code: %+v", resp.StatusCode)
}
_, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
}
func TestHandlerWithRequestBodyReadingError(t *testing.T) {
expected := errors.New("mocked error")
handler := Handler{Config: &Config{}}
rw := NewFakeResponseWriter()
req := &http.Request{
Method: "POST",
Header: map[string][]string{
"Content-Type": {"application/json"},
"Content-Length": {"2048"},
},
Body: &FakeBody{Err: expected},
}
handler.ServeHTTP(rw, req)
if rw.StatusCode != 400 {
t.Fatal("unexpected status code")
}
}
type FakeBody struct {
Err error
}
func (fb FakeBody) Read(p []byte) (int, error) {
time.Sleep(10 * time.Microsecond)
return 0, fb.Err
}
func (fb FakeBody) Close() error {
return nil
}
type FakeResponseWriter struct {
Body [][]byte
HeaderMap http.Header
StatusCode int
}
func NewFakeResponseWriter() *FakeResponseWriter {
return &FakeResponseWriter{HeaderMap: make(http.Header)}
}
func (frw *FakeResponseWriter) Header() http.Header {
return frw.HeaderMap
}
func (frw *FakeResponseWriter) Write(b []byte) (int, error) {
frw.Body = append(frw.Body, b)
return len(b), nil
}
func (frw *FakeResponseWriter) WriteHeader(statusCode int) {
frw.StatusCode = statusCode
}

2
internal/cmd/oohelperd/oohelperd.go
View File

@ -10,6 +10,7 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/nwcth"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
)
@ -52,6 +53,7 @@ func main() {
func testableMain() {
mux := http.NewServeMux()
mux.Handle("/api/unstable/nwcth", nwcth.Handler{Config: &nwcth.Config{}})
mux.Handle("/", internal.Handler{
Client: httpx,
Dialer: dialer,

13
internal/engine/allexperiments.go
View File

@ -23,6 +23,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp"
)
@ -326,6 +327,18 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
}
},
"websteps": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, websteps.NewExperimentMeasurer(
*config.(*websteps.Config),
))
},
config: &websteps.Config{},
inputPolicy: InputOrQueryBackend,
}
},
"whatsapp": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {

38
internal/engine/experiment/websteps/control.go Normal file
View File

@ -0,0 +1,38 @@
package websteps
import (
"context"
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/errorsx"
)
// CtrlRequest is the request sent by the probe
type CtrlRequest struct {
HTTPRequest string `json:"url"`
HTTPRequestHeaders map[string][]string `json:"headers"`
Addrs []string `json:"addrs"`
}
// ControlResponse is the response from the control service.
type ControlResponse struct {
URLMeasurements []*URLMeasurement `json:"urls"`
}
// Control performs the control request and returns the response.
func Control(
ctx context.Context, sess model.ExperimentSession,
thAddr string, creq CtrlRequest) (out ControlResponse, err error) {
clnt := httpx.Client{
BaseURL: thAddr,
HTTPClient: sess.DefaultHTTPClient(),
Logger: sess.Logger(),
}
// make sure error is wrapped
err = errorsx.SafeErrWrapperBuilder{
Error: clnt.PostJSON(ctx, "/", creq, &out),
Operation: errorsx.TopLevelOperation,
}.MaybeBuild()
return
}

27
internal/engine/experiment/websteps/dns.go Normal file
View File

@ -0,0 +1,27 @@
package websteps
import (
"context"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
type DNSConfig struct {
Domain string
Resolver netxlite.Resolver
}
// DNSDo performs the DNS check.
func DNSDo(ctx context.Context, config DNSConfig) ([]string, error) {
resolver := config.Resolver
if resolver == nil {
childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google")
runtimex.PanicOnError(err, "NewDNSClient failed")
var resolver netxlite.Resolver = childResolver
resolver = &netxlite.IDNAResolver{Resolver: resolver}
}
return config.Resolver.LookupHost(ctx, config.Domain)
}

110
internal/engine/experiment/websteps/factory.go Normal file
View File

@ -0,0 +1,110 @@
package websteps
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/url"
"sync"
"github.com/apex/log"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
"github.com/ooni/probe-cli/v3/internal/errorsx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
var ErrNoConnReuse = errors.New("cannot reuse connection")
func NewRequest(ctx context.Context, URL *url.URL, headers http.Header) *http.Request {
req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil)
runtimex.PanicOnError(err, "NewRequestWithContect failed")
for k, vs := range headers {
for _, v := range vs {
req.Header.Add(k, v)
}
}
return req
}
// NewDialerResolver contructs a new dialer for TCP connections,
// with default, errorwrapping and resolve functionalities
func NewDialerResolver(resolver netxlite.Resolver) netxlite.Dialer {
var d netxlite.Dialer = netxlite.DefaultDialer
d = &errorsx.ErrorWrapperDialer{Dialer: d}
d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d}
return d
}
// NewQUICDialerResolver creates a new QUICDialerResolver
// with default, errorwrapping and resolve functionalities
func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDialer {
var ql quicdialer.QUICListener = &netxlite.QUICListenerStdlib{}
ql = &errorsx.ErrorWrapperQUICListener{QUICListener: ql}
var dialer netxlite.QUICContextDialer = &netxlite.QUICDialerQUICGo{
QUICListener: ql,
}
dialer = &errorsx.ErrorWrapperQUICDialer{Dialer: dialer}
dialer = &netxlite.QUICDialerResolver{Resolver: resolver, Dialer: dialer}
return dialer
}
// NewSingleH3Transport creates an http3.RoundTripper
func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper {
transport := &http3.RoundTripper{
DisableCompression: true,
TLSClientConfig: tlscfg,
QuicConfig: qcfg,
Dial: (&SingleDialerH3{qsess: &qsess}).Dial,
}
return transport
}
// NewSingleTransport determines the appropriate HTTP Transport from the ALPN
func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) {
singledialer := &SingleDialer{conn: &conn}
transport = http.DefaultTransport.(*http.Transport).Clone()
transport.(*http.Transport).DialContext = singledialer.DialContext
transport.(*http.Transport).DialTLSContext = singledialer.DialContext
transport.(*http.Transport).DisableCompression = true
transport.(*http.Transport).MaxConnsPerHost = 1
transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)}
return transport
}
type SingleDialer struct {
sync.Mutex
conn *net.Conn
}
func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) {
s.Lock()
defer s.Unlock()
if s.conn == nil {
return nil, ErrNoConnReuse
}
c := s.conn
s.conn = nil
return *c, nil
}
type SingleDialerH3 struct {
sync.Mutex
qsess *quic.EarlySession
}
func (s *SingleDialerH3) Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
s.Lock()
defer s.Unlock()
if s.qsess == nil {
return nil, ErrNoConnReuse
}
qs := s.qsess
s.qsess = nil
return *qs, nil
}

33
internal/engine/experiment/websteps/http.go Normal file
View File

@ -0,0 +1,33 @@
package websteps
import (
"io"
"net/http"
)
// HTTPDo performs the HTTP check.
// Input:
// req *http.Request
// The same request than the one used by the Explore step.
// This means that req contains the headers set by the original CtrlRequest, as well as,
// in case of a redirect chain, additional headers that were added due to redirects
// transport http.RoundTripper:
// The transport to use, either http.Transport, or http3.RoundTripper.
func HTTPDo(req *http.Request, transport http.RoundTripper) (*http.Response, []byte, error) {
clnt := http.Client{
CheckRedirect: func(r *http.Request, reqs []*http.Request) error {
return http.ErrUseLastResponse
},
Transport: transport,
}
resp, err := clnt.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return resp, nil, nil
}
return resp, body, nil
}

102
internal/engine/experiment/websteps/model.go Normal file
View File

@ -0,0 +1,102 @@
package websteps
import "net/http"
// RoundTrip describes a specific round trip.
type RoundTrip struct {
// proto is the protocol used, it can be "h2", "http/1.1", "h3", "h3-29"
Proto string
// Request is the original HTTP request. The headers
// also include cookies.
Request *http.Request
// Response is the HTTP response.
Response *http.Response
// sortIndex is an internal field using for sorting.
SortIndex int
}
// URLMeasurement is a measurement of a given URL that
// includes connectivity measurement for each endpoint
// implied by the given URL.
type URLMeasurement struct {
// URL is the URL we're using
URL string `json:"url"`
// DNS contains the domain names resolved by the helper.
DNS *DNSMeasurement `json:"dns"`
// RoundTrip is the related round trip.
RoundTrip *RoundTrip `json:"-"`
// Endpoints contains endpoint measurements.
Endpoints []*EndpointMeasurement `json:"endpoints"`
}
// DNSMeasurement is a DNS measurement.
type DNSMeasurement struct {
// Domain is the domain we wanted to resolve.
Domain string `json:"domain"`
// Addrs contains the resolved addresses.
Addrs []string `json:"addrs"`
// Failure is the error that occurred.
Failure *string `json:"failure"`
}
// HTTPSEndpointMeasurement is the measurement of requesting a specific endpoint via HTTPS.
type EndpointMeasurement struct {
// Endpoint is the endpoint we're measuring.
Endpoint string `json:"endpoint"`
// Protocol is the used protocol. It can be "http", "https", "h3", "h3-29" or other supported QUIC protocols
Protocol string `json:"protocol"`
// TCPConnectMeasurement is the related TCP connect measurement, if applicable (nil for h3 requests)
TCPConnectMeasurement *TCPConnectMeasurement `json:"tcp_connect"`
// QUICHandshakeMeasurement is the related QUIC(TLS 1.3) handshake measurement, if applicable (nil for http, https requests)
QUICHandshakeMeasurement *TLSHandshakeMeasurement `json:"quic_handshake"`
// TLSHandshakeMeasurement is the related TLS handshake measurement, if applicable (nil for http, h3 requests)
TLSHandshakeMeasurement *TLSHandshakeMeasurement `json:"tls_handshake"`
// HTTPRoundTripMeasurement is the related HTTP GET measurement.
HTTPRoundTripMeasurement *HTTPRoundTripMeasurement `json:"http_round_trip"`
}
// TCPConnectMeasurement is a TCP connect measurement.
type TCPConnectMeasurement struct {
// Failure is the error that occurred.
Failure *string `json:"failure"`
}
// TLSHandshakeMeasurement is a TLS handshake measurement.
type TLSHandshakeMeasurement struct {
// Failure is the error that occurred.
Failure *string `json:"failure"`
}
// HTTPRoundTripMeasurement contains a measured HTTP request and the corresponding response.
type HTTPRoundTripMeasurement struct {
Request *HTTPRequestMeasurement `json:"request"`
Response *HTTPResponseMeasurement `json:"response"`
}
// HTTPRequestMeasurement contains the headers of the measured HTTP Get request.
type HTTPRequestMeasurement struct {
Headers http.Header `json:"headers"`
Method string `json:"method"`
URL string `json:"url"`
}
// HTTPResponseMeasurement contains the response of the measured HTTP Get request.
type HTTPResponseMeasurement struct {
BodyLength int64 `json:"body_length"`
Failure *string `json:"failure"`
Headers http.Header `json:"headers"`
StatusCode int64 `json:"status_code"`
}

29
internal/engine/experiment/websteps/quic.go Normal file
View File

@ -0,0 +1,29 @@
package websteps
import (
"context"
"crypto/tls"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
type QUICConfig struct {
Endpoint string
QUICDialer netxlite.QUICContextDialer
Resolver netxlite.Resolver
TLSConf *tls.Config
}
// QUICDo performs the QUIC check.
func QUICDo(ctx context.Context, config QUICConfig) (quic.EarlySession, error) {
if config.QUICDialer != nil {
return config.QUICDialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{})
}
resolver := config.Resolver
if resolver == nil {
resolver = &netxlite.ResolverSystem{}
}
dialer := NewQUICDialerResolver(resolver)
return dialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{})
}

27
internal/engine/experiment/websteps/tcp.go Normal file
View File

@ -0,0 +1,27 @@
package websteps
import (
"context"
"net"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
type TCPConfig struct {
Dialer netxlite.Dialer
Endpoint string
Resolver netxlite.Resolver
}
// TCPDo performs the TCP check.
func TCPDo(ctx context.Context, config TCPConfig) (net.Conn, error) {
if config.Dialer != nil {
return config.Dialer.DialContext(ctx, "tcp", config.Endpoint)
}
resolver := config.Resolver
if resolver == nil {
resolver = &netxlite.ResolverSystem{}
}
dialer := NewDialerResolver(resolver)
return dialer.DialContext(ctx, "tcp", config.Endpoint)
}

16
internal/engine/experiment/websteps/tls.go Normal file
View File

@ -0,0 +1,16 @@
package websteps
import (
"crypto/tls"
"net"
)
// TLSDo performs the TLS check.
func TLSDo(conn net.Conn, hostname string) (*tls.Conn, error) {
tlsConn := tls.Client(conn, &tls.Config{
ServerName: hostname,
NextProtos: []string{"h2", "http/1.1"},
})
err := tlsConn.Handshake()
return tlsConn, err
}

355
internal/engine/experiment/websteps/websteps.go Normal file
View File

@ -0,0 +1,355 @@
package websteps
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/url"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
const (
testName = "websteps"
testVersion = "0.0.1"
)
// Config contains the experiment config.
type Config struct{}
// TestKeys contains webconnectivity test keys.
type TestKeys struct {
Agent string `json:"agent"`
ClientResolver string `json:"client_resolver"`
URLMeasurements []*URLMeasurement
}
// Measurer performs the measurement.
type Measurer struct {
Config Config
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return Measurer{Config: config}
}
// ExperimentName implements ExperimentMeasurer.ExperExperimentName.
func (m Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion.
func (m Measurer) ExperimentVersion() string {
return testVersion
}
// SupportedQUICVersions are the H3 over QUIC versions we currently support
var SupportedQUICVersions = map[string]bool{
"h3": true,
"h3-29": true,
}
var (
// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
ErrNoAvailableTestHelpers = errors.New("no available helpers")
// ErrNoInput indicates that no input was provided
ErrNoInput = errors.New("no input provided")
// ErrInputIsNotAnURL indicates that the input is not an URL.
ErrInputIsNotAnURL = errors.New("input is not an URL")
// ErrUnsupportedInput indicates that the input URL scheme is unsupported.
ErrUnsupportedInput = errors.New("unsupported input scheme")
)
// Run implements ExperimentMeasurer.Run.
func (m Measurer) Run(
ctx context.Context,
sess model.ExperimentSession,
measurement *model.Measurement,
callbacks model.ExperimentCallbacks,
) error {
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
tk := new(TestKeys)
measurement.TestKeys = tk
tk.Agent = "redirect"
tk.ClientResolver = sess.ResolverIP()
// 1. Parse and verify URL
URL, err := url.Parse(string(measurement.Input))
if err != nil {
return ErrInputIsNotAnURL
}
if URL.Scheme != "http" && URL.Scheme != "https" {
return ErrUnsupportedInput
}
// 2. Perform the initial DNS lookup step
addrs, err := DNSDo(ctx, DNSConfig{Domain: URL.Hostname()})
endpoints := makeEndpoints(addrs, URL)
// 3. Find the testhelper
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
var testhelper *model.Service
for _, th := range testhelpers {
if th.Type == "https" {
testhelper = &th
break
}
}
if testhelper == nil {
return ErrNoAvailableTestHelpers
}
measurement.TestHelpers = map[string]interface{}{
"backend": testhelper,
}
// 4. Query the testhelper
resp, err := Control(ctx, sess, testhelper.Address, CtrlRequest{
HTTPRequest: URL.String(),
HTTPRequestHeaders: map[string][]string{
"Accept": {httpheader.Accept()},
"Accept-Language": {httpheader.AcceptLanguage()},
"User-Agent": {httpheader.UserAgent()},
},
Addrs: endpoints,
})
if err != nil || resp.URLMeasurements == nil {
return errors.New("no control response")
}
// 5. Go over the Control URL measurements and reproduce them without following redirects, one by one.
for _, controlURLMeasurement := range resp.URLMeasurements {
urlMeasurement := &URLMeasurement{
URL: controlURLMeasurement.URL,
Endpoints: []*EndpointMeasurement{},
}
URL, err = url.Parse(controlURLMeasurement.URL)
runtimex.PanicOnError(err, "url.Parse failed")
// DNS step
addrs, err = DNSDo(ctx, DNSConfig{Domain: URL.Hostname()})
urlMeasurement.DNS = &DNSMeasurement{
Domain: URL.Hostname(),
Addrs: addrs,
Failure: archival.NewFailure(err),
}
if controlURLMeasurement.Endpoints == nil {
tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement)
continue
}
// the testhelper tells us which endpoints to measure
for _, controlEndpoint := range controlURLMeasurement.Endpoints {
rt := controlEndpoint.HTTPRoundTripMeasurement
if rt == nil || rt.Request == nil {
continue
}
var endpointMeasurement *EndpointMeasurement
proto := controlEndpoint.Protocol
_, h3 := SupportedQUICVersions[proto]
switch {
case h3:
endpointMeasurement = m.measureEndpointH3(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers, proto)
case proto == "http":
endpointMeasurement = m.measureEndpointHTTP(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers)
case proto == "https":
endpointMeasurement = m.measureEndpointHTTPS(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers)
default:
panic("should not happen")
}
urlMeasurement.Endpoints = append(urlMeasurement.Endpoints, endpointMeasurement)
}
tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement)
}
return nil
}
func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement {
endpointMeasurement := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: "http",
}
// TCP connect step
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
endpointMeasurement.TCPConnectMeasurement = &TCPConnectMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
return endpointMeasurement
}
defer conn.Close()
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
URL: URL.String(),
},
}
transport := NewSingleTransport(conn)
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return endpointMeasurement
}
func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement {
endpointMeasurement := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: "https",
}
// TCP connect step
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
endpointMeasurement.TCPConnectMeasurement = &TCPConnectMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
return endpointMeasurement
}
defer conn.Close()
// TLS handshake step
tlsconn, err := TLSDo(conn, URL.Hostname())
endpointMeasurement.TLSHandshakeMeasurement = &TLSHandshakeMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
return endpointMeasurement
}
defer tlsconn.Close()
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
URL: URL.String(),
},
}
transport := NewSingleTransport(tlsconn)
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return endpointMeasurement
}
func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint string, headers http.Header, proto string) *EndpointMeasurement {
endpointMeasurement := &EndpointMeasurement{
Endpoint: endpoint,
Protocol: proto,
}
tlsConf := &tls.Config{
ServerName: URL.Hostname(),
NextProtos: []string{proto},
}
// QUIC handshake step
sess, err := QUICDo(ctx, QUICConfig{
Endpoint: endpoint,
TLSConf: tlsConf,
})
endpointMeasurement.QUICHandshakeMeasurement = &TLSHandshakeMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
return endpointMeasurement
}
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
URL: URL.String(),
},
}
transport := NewSingleH3Transport(sess, tlsConf, &quic.Config{})
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
StatusCode: int64(resp.StatusCode),
}
return endpointMeasurement
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with probe-cli
// therefore we should be careful when changing it.
type SummaryKeys struct {
Accessible bool `json:"accessible"`
Blocking string `json:"blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{}
return sk, nil
}
func makeEndpoints(addrs []string, URL *url.URL) []string {
endpoints := []string{}
if addrs == nil {
return endpoints
}
for _, addr := range addrs {
var port string
explicitPort := URL.Port()
scheme := URL.Scheme
switch {
case explicitPort != "":
port = explicitPort
case scheme == "http":
port = "80"
case scheme == "https":
port = "443"
default:
panic("should not happen")
}
endpoints = append(endpoints, net.JoinHostPort(addr, port))
}
return endpoints
}

33
internal/netxlite/resolver.go
View File

@ -4,6 +4,8 @@ import (
"context"
"net"
"time"
"golang.org/x/net/idna"
)
// Resolver performs domain name resolutions.
@ -80,3 +82,34 @@ func (r *ResolverLogger) Address() string {
}
return ""
}
// IDNAResolver is to support resolving Internationalized Domain Names.
// See RFC3492 for more information.
type IDNAResolver struct {
Resolver
}
// LookupHost implements Resolver.LookupHost
func (r *IDNAResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
host, err := idna.ToASCII(hostname)
if err != nil {
return nil, err
}
return r.Resolver.LookupHost(ctx, host)
}
// Network implements Resolver.Network.
func (r *IDNAResolver) Network() string {
if rn, ok := r.Resolver.(resolverNetworker); ok {
return rn.Network()
}
return "idna"
}
// Address implements Resolver.Address.
func (r *IDNAResolver) Address() string {
if ra, ok := r.Resolver.(resolverAddresser); ok {
return ra.Address()
}
return ""
}