cli: new testhelper and the websteps experiment prototype (#432)
This is the extension of https://github.com/ooni/probe-cli/pull/431, and my final deliverable for GSoC 2021. The diff introduces: 1) The new `testhelper` which supports testing multiple IP endpoints per domain and introduces HTTP/3 control measurements. The specification of the `testhelper` can be found at https://github.com/ooni/spec/pull/219. The `testhelper` algorithm consists of three main steps: * `InitialChecks` verifies that the input URL can be parsed, has an expected scheme, and contains a valid domain name. * `Explore` enumerates all the URLs that it discovers by redirection from the original URL, or by detecting h3 support at the target host. * `Generate` performs a step-by-step measurement of each discovered URL. 2) A prototype of the corresponding new experiment `websteps` which uses the control measurement of the `testhelper` to know which URLs to measure, and what to expect. The prototype does not yet have: * unit and integration tests, * an analysis tool to compare the control and the probe measurement. This PR is my final deliverable as it is the outcome of the trials, considerations and efforts of my GSoC weeks at OONI. It fully integrates HTTP/3 (QUIC) support which has been only used in the `urlgetter` experiment until now. Related issues: https://github.com/ooni/probe/issues/1729 and https://github.com/ooni/probe/issues/1733.
This commit is contained in:
kelmenhorst
GitHub
@@ -0,0 +1,38 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/errorsx"
|
||||
)
|
||||
|
||||
// CtrlRequest is the request sent by the probe
|
||||
type CtrlRequest struct {
|
||||
HTTPRequest string `json:"url"`
|
||||
HTTPRequestHeaders map[string][]string `json:"headers"`
|
||||
Addrs []string `json:"addrs"`
|
||||
}
|
||||
|
||||
// ControlResponse is the response from the control service.
|
||||
type ControlResponse struct {
|
||||
URLMeasurements []*URLMeasurement `json:"urls"`
|
||||
}
|
||||
|
||||
// Control performs the control request and returns the response.
|
||||
func Control(
|
||||
ctx context.Context, sess model.ExperimentSession,
|
||||
thAddr string, creq CtrlRequest) (out ControlResponse, err error) {
|
||||
clnt := httpx.Client{
|
||||
BaseURL: thAddr,
|
||||
HTTPClient: sess.DefaultHTTPClient(),
|
||||
Logger: sess.Logger(),
|
||||
}
|
||||
// make sure error is wrapped
|
||||
err = errorsx.SafeErrWrapperBuilder{
|
||||
Error: clnt.PostJSON(ctx, "/", creq, &out),
|
||||
Operation: errorsx.TopLevelOperation,
|
||||
}.MaybeBuild()
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
||||
)
|
||||
|
||||
type DNSConfig struct {
|
||||
Domain string
|
||||
Resolver netxlite.Resolver
|
||||
}
|
||||
|
||||
// DNSDo performs the DNS check.
|
||||
func DNSDo(ctx context.Context, config DNSConfig) ([]string, error) {
|
||||
resolver := config.Resolver
|
||||
if resolver == nil {
|
||||
childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google")
|
||||
runtimex.PanicOnError(err, "NewDNSClient failed")
|
||||
var resolver netxlite.Resolver = childResolver
|
||||
resolver = &netxlite.IDNAResolver{Resolver: resolver}
|
||||
}
|
||||
return config.Resolver.LookupHost(ctx, config.Domain)
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
|
||||
"github.com/ooni/probe-cli/v3/internal/errorsx"
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
||||
)
|
||||
|
||||
var ErrNoConnReuse = errors.New("cannot reuse connection")
|
||||
|
||||
func NewRequest(ctx context.Context, URL *url.URL, headers http.Header) *http.Request {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil)
|
||||
runtimex.PanicOnError(err, "NewRequestWithContect failed")
|
||||
for k, vs := range headers {
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// NewDialerResolver contructs a new dialer for TCP connections,
|
||||
// with default, errorwrapping and resolve functionalities
|
||||
func NewDialerResolver(resolver netxlite.Resolver) netxlite.Dialer {
|
||||
var d netxlite.Dialer = netxlite.DefaultDialer
|
||||
d = &errorsx.ErrorWrapperDialer{Dialer: d}
|
||||
d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d}
|
||||
return d
|
||||
}
|
||||
|
||||
// NewQUICDialerResolver creates a new QUICDialerResolver
|
||||
// with default, errorwrapping and resolve functionalities
|
||||
func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDialer {
|
||||
var ql quicdialer.QUICListener = &netxlite.QUICListenerStdlib{}
|
||||
ql = &errorsx.ErrorWrapperQUICListener{QUICListener: ql}
|
||||
var dialer netxlite.QUICContextDialer = &netxlite.QUICDialerQUICGo{
|
||||
QUICListener: ql,
|
||||
}
|
||||
dialer = &errorsx.ErrorWrapperQUICDialer{Dialer: dialer}
|
||||
dialer = &netxlite.QUICDialerResolver{Resolver: resolver, Dialer: dialer}
|
||||
return dialer
|
||||
}
|
||||
|
||||
// NewSingleH3Transport creates an http3.RoundTripper
|
||||
func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper {
|
||||
transport := &http3.RoundTripper{
|
||||
DisableCompression: true,
|
||||
TLSClientConfig: tlscfg,
|
||||
QuicConfig: qcfg,
|
||||
Dial: (&SingleDialerH3{qsess: &qsess}).Dial,
|
||||
}
|
||||
return transport
|
||||
}
|
||||
|
||||
// NewSingleTransport determines the appropriate HTTP Transport from the ALPN
|
||||
func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) {
|
||||
singledialer := &SingleDialer{conn: &conn}
|
||||
transport = http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.(*http.Transport).DialContext = singledialer.DialContext
|
||||
transport.(*http.Transport).DialTLSContext = singledialer.DialContext
|
||||
transport.(*http.Transport).DisableCompression = true
|
||||
transport.(*http.Transport).MaxConnsPerHost = 1
|
||||
|
||||
transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)}
|
||||
return transport
|
||||
}
|
||||
|
||||
type SingleDialer struct {
|
||||
sync.Mutex
|
||||
conn *net.Conn
|
||||
}
|
||||
|
||||
func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
if s.conn == nil {
|
||||
return nil, ErrNoConnReuse
|
||||
}
|
||||
c := s.conn
|
||||
s.conn = nil
|
||||
return *c, nil
|
||||
}
|
||||
|
||||
type SingleDialerH3 struct {
|
||||
sync.Mutex
|
||||
qsess *quic.EarlySession
|
||||
}
|
||||
|
||||
func (s *SingleDialerH3) Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
if s.qsess == nil {
|
||||
return nil, ErrNoConnReuse
|
||||
}
|
||||
qs := s.qsess
|
||||
s.qsess = nil
|
||||
return *qs, nil
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// HTTPDo performs the HTTP check.
|
||||
// Input:
|
||||
// req *http.Request
|
||||
// The same request than the one used by the Explore step.
|
||||
// This means that req contains the headers set by the original CtrlRequest, as well as,
|
||||
// in case of a redirect chain, additional headers that were added due to redirects
|
||||
// transport http.RoundTripper:
|
||||
// The transport to use, either http.Transport, or http3.RoundTripper.
|
||||
func HTTPDo(req *http.Request, transport http.RoundTripper) (*http.Response, []byte, error) {
|
||||
clnt := http.Client{
|
||||
CheckRedirect: func(r *http.Request, reqs []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
Transport: transport,
|
||||
}
|
||||
resp, err := clnt.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return resp, nil, nil
|
||||
}
|
||||
return resp, body, nil
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package websteps
|
||||
|
||||
import "net/http"
|
||||
|
||||
// RoundTrip describes a specific round trip.
|
||||
type RoundTrip struct {
|
||||
// proto is the protocol used, it can be "h2", "http/1.1", "h3", "h3-29"
|
||||
Proto string
|
||||
|
||||
// Request is the original HTTP request. The headers
|
||||
// also include cookies.
|
||||
Request *http.Request
|
||||
|
||||
// Response is the HTTP response.
|
||||
Response *http.Response
|
||||
|
||||
// sortIndex is an internal field using for sorting.
|
||||
SortIndex int
|
||||
}
|
||||
|
||||
// URLMeasurement is a measurement of a given URL that
|
||||
// includes connectivity measurement for each endpoint
|
||||
// implied by the given URL.
|
||||
type URLMeasurement struct {
|
||||
// URL is the URL we're using
|
||||
URL string `json:"url"`
|
||||
|
||||
// DNS contains the domain names resolved by the helper.
|
||||
DNS *DNSMeasurement `json:"dns"`
|
||||
|
||||
// RoundTrip is the related round trip.
|
||||
RoundTrip *RoundTrip `json:"-"`
|
||||
|
||||
// Endpoints contains endpoint measurements.
|
||||
Endpoints []*EndpointMeasurement `json:"endpoints"`
|
||||
}
|
||||
|
||||
// DNSMeasurement is a DNS measurement.
|
||||
type DNSMeasurement struct {
|
||||
// Domain is the domain we wanted to resolve.
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Addrs contains the resolved addresses.
|
||||
Addrs []string `json:"addrs"`
|
||||
|
||||
// Failure is the error that occurred.
|
||||
Failure *string `json:"failure"`
|
||||
}
|
||||
|
||||
// HTTPSEndpointMeasurement is the measurement of requesting a specific endpoint via HTTPS.
|
||||
type EndpointMeasurement struct {
|
||||
// Endpoint is the endpoint we're measuring.
|
||||
Endpoint string `json:"endpoint"`
|
||||
|
||||
// Protocol is the used protocol. It can be "http", "https", "h3", "h3-29" or other supported QUIC protocols
|
||||
Protocol string `json:"protocol"`
|
||||
|
||||
// TCPConnectMeasurement is the related TCP connect measurement, if applicable (nil for h3 requests)
|
||||
TCPConnectMeasurement *TCPConnectMeasurement `json:"tcp_connect"`
|
||||
|
||||
// QUICHandshakeMeasurement is the related QUIC(TLS 1.3) handshake measurement, if applicable (nil for http, https requests)
|
||||
QUICHandshakeMeasurement *TLSHandshakeMeasurement `json:"quic_handshake"`
|
||||
|
||||
// TLSHandshakeMeasurement is the related TLS handshake measurement, if applicable (nil for http, h3 requests)
|
||||
TLSHandshakeMeasurement *TLSHandshakeMeasurement `json:"tls_handshake"`
|
||||
|
||||
// HTTPRoundTripMeasurement is the related HTTP GET measurement.
|
||||
HTTPRoundTripMeasurement *HTTPRoundTripMeasurement `json:"http_round_trip"`
|
||||
}
|
||||
|
||||
// TCPConnectMeasurement is a TCP connect measurement.
|
||||
type TCPConnectMeasurement struct {
|
||||
// Failure is the error that occurred.
|
||||
Failure *string `json:"failure"`
|
||||
}
|
||||
|
||||
// TLSHandshakeMeasurement is a TLS handshake measurement.
|
||||
type TLSHandshakeMeasurement struct {
|
||||
// Failure is the error that occurred.
|
||||
Failure *string `json:"failure"`
|
||||
}
|
||||
|
||||
// HTTPRoundTripMeasurement contains a measured HTTP request and the corresponding response.
|
||||
type HTTPRoundTripMeasurement struct {
|
||||
Request *HTTPRequestMeasurement `json:"request"`
|
||||
Response *HTTPResponseMeasurement `json:"response"`
|
||||
}
|
||||
|
||||
// HTTPRequestMeasurement contains the headers of the measured HTTP Get request.
|
||||
type HTTPRequestMeasurement struct {
|
||||
Headers http.Header `json:"headers"`
|
||||
Method string `json:"method"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// HTTPResponseMeasurement contains the response of the measured HTTP Get request.
|
||||
type HTTPResponseMeasurement struct {
|
||||
BodyLength int64 `json:"body_length"`
|
||||
Failure *string `json:"failure"`
|
||||
Headers http.Header `json:"headers"`
|
||||
StatusCode int64 `json:"status_code"`
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
)
|
||||
|
||||
type QUICConfig struct {
|
||||
Endpoint string
|
||||
QUICDialer netxlite.QUICContextDialer
|
||||
Resolver netxlite.Resolver
|
||||
TLSConf *tls.Config
|
||||
}
|
||||
|
||||
// QUICDo performs the QUIC check.
|
||||
func QUICDo(ctx context.Context, config QUICConfig) (quic.EarlySession, error) {
|
||||
if config.QUICDialer != nil {
|
||||
return config.QUICDialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{})
|
||||
}
|
||||
resolver := config.Resolver
|
||||
if resolver == nil {
|
||||
resolver = &netxlite.ResolverSystem{}
|
||||
}
|
||||
dialer := NewQUICDialerResolver(resolver)
|
||||
return dialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{})
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
)
|
||||
|
||||
type TCPConfig struct {
|
||||
Dialer netxlite.Dialer
|
||||
Endpoint string
|
||||
Resolver netxlite.Resolver
|
||||
}
|
||||
|
||||
// TCPDo performs the TCP check.
|
||||
func TCPDo(ctx context.Context, config TCPConfig) (net.Conn, error) {
|
||||
if config.Dialer != nil {
|
||||
return config.Dialer.DialContext(ctx, "tcp", config.Endpoint)
|
||||
}
|
||||
resolver := config.Resolver
|
||||
if resolver == nil {
|
||||
resolver = &netxlite.ResolverSystem{}
|
||||
}
|
||||
dialer := NewDialerResolver(resolver)
|
||||
return dialer.DialContext(ctx, "tcp", config.Endpoint)
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
)
|
||||
|
||||
// TLSDo performs the TLS check.
|
||||
func TLSDo(conn net.Conn, hostname string) (*tls.Conn, error) {
|
||||
tlsConn := tls.Client(conn, &tls.Config{
|
||||
ServerName: hostname,
|
||||
NextProtos: []string{"h2", "http/1.1"},
|
||||
})
|
||||
err := tlsConn.Handshake()
|
||||
return tlsConn, err
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
|
||||
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
||||
)
|
||||
|
||||
const (
|
||||
testName = "websteps"
|
||||
testVersion = "0.0.1"
|
||||
)
|
||||
|
||||
// Config contains the experiment config.
|
||||
type Config struct{}
|
||||
|
||||
// TestKeys contains webconnectivity test keys.
|
||||
type TestKeys struct {
|
||||
Agent string `json:"agent"`
|
||||
ClientResolver string `json:"client_resolver"`
|
||||
URLMeasurements []*URLMeasurement
|
||||
}
|
||||
|
||||
// Measurer performs the measurement.
|
||||
type Measurer struct {
|
||||
Config Config
|
||||
}
|
||||
|
||||
// NewExperimentMeasurer creates a new ExperimentMeasurer.
|
||||
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
|
||||
return Measurer{Config: config}
|
||||
}
|
||||
|
||||
// ExperimentName implements ExperimentMeasurer.ExperExperimentName.
|
||||
func (m Measurer) ExperimentName() string {
|
||||
return testName
|
||||
}
|
||||
|
||||
// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion.
|
||||
func (m Measurer) ExperimentVersion() string {
|
||||
return testVersion
|
||||
}
|
||||
|
||||
// SupportedQUICVersions are the H3 over QUIC versions we currently support
|
||||
var SupportedQUICVersions = map[string]bool{
|
||||
"h3": true,
|
||||
"h3-29": true,
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
|
||||
ErrNoAvailableTestHelpers = errors.New("no available helpers")
|
||||
|
||||
// ErrNoInput indicates that no input was provided
|
||||
ErrNoInput = errors.New("no input provided")
|
||||
|
||||
// ErrInputIsNotAnURL indicates that the input is not an URL.
|
||||
ErrInputIsNotAnURL = errors.New("input is not an URL")
|
||||
|
||||
// ErrUnsupportedInput indicates that the input URL scheme is unsupported.
|
||||
ErrUnsupportedInput = errors.New("unsupported input scheme")
|
||||
)
|
||||
|
||||
// Run implements ExperimentMeasurer.Run.
|
||||
func (m Measurer) Run(
|
||||
ctx context.Context,
|
||||
sess model.ExperimentSession,
|
||||
measurement *model.Measurement,
|
||||
callbacks model.ExperimentCallbacks,
|
||||
) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
tk := new(TestKeys)
|
||||
measurement.TestKeys = tk
|
||||
tk.Agent = "redirect"
|
||||
tk.ClientResolver = sess.ResolverIP()
|
||||
|
||||
// 1. Parse and verify URL
|
||||
URL, err := url.Parse(string(measurement.Input))
|
||||
if err != nil {
|
||||
return ErrInputIsNotAnURL
|
||||
}
|
||||
if URL.Scheme != "http" && URL.Scheme != "https" {
|
||||
return ErrUnsupportedInput
|
||||
}
|
||||
// 2. Perform the initial DNS lookup step
|
||||
addrs, err := DNSDo(ctx, DNSConfig{Domain: URL.Hostname()})
|
||||
endpoints := makeEndpoints(addrs, URL)
|
||||
// 3. Find the testhelper
|
||||
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
|
||||
var testhelper *model.Service
|
||||
for _, th := range testhelpers {
|
||||
if th.Type == "https" {
|
||||
testhelper = &th
|
||||
break
|
||||
}
|
||||
}
|
||||
if testhelper == nil {
|
||||
return ErrNoAvailableTestHelpers
|
||||
}
|
||||
measurement.TestHelpers = map[string]interface{}{
|
||||
"backend": testhelper,
|
||||
}
|
||||
// 4. Query the testhelper
|
||||
resp, err := Control(ctx, sess, testhelper.Address, CtrlRequest{
|
||||
HTTPRequest: URL.String(),
|
||||
HTTPRequestHeaders: map[string][]string{
|
||||
"Accept": {httpheader.Accept()},
|
||||
"Accept-Language": {httpheader.AcceptLanguage()},
|
||||
"User-Agent": {httpheader.UserAgent()},
|
||||
},
|
||||
Addrs: endpoints,
|
||||
})
|
||||
if err != nil || resp.URLMeasurements == nil {
|
||||
return errors.New("no control response")
|
||||
}
|
||||
// 5. Go over the Control URL measurements and reproduce them without following redirects, one by one.
|
||||
for _, controlURLMeasurement := range resp.URLMeasurements {
|
||||
urlMeasurement := &URLMeasurement{
|
||||
URL: controlURLMeasurement.URL,
|
||||
Endpoints: []*EndpointMeasurement{},
|
||||
}
|
||||
URL, err = url.Parse(controlURLMeasurement.URL)
|
||||
runtimex.PanicOnError(err, "url.Parse failed")
|
||||
// DNS step
|
||||
addrs, err = DNSDo(ctx, DNSConfig{Domain: URL.Hostname()})
|
||||
urlMeasurement.DNS = &DNSMeasurement{
|
||||
Domain: URL.Hostname(),
|
||||
Addrs: addrs,
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if controlURLMeasurement.Endpoints == nil {
|
||||
tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement)
|
||||
continue
|
||||
}
|
||||
// the testhelper tells us which endpoints to measure
|
||||
for _, controlEndpoint := range controlURLMeasurement.Endpoints {
|
||||
rt := controlEndpoint.HTTPRoundTripMeasurement
|
||||
if rt == nil || rt.Request == nil {
|
||||
continue
|
||||
}
|
||||
var endpointMeasurement *EndpointMeasurement
|
||||
proto := controlEndpoint.Protocol
|
||||
_, h3 := SupportedQUICVersions[proto]
|
||||
switch {
|
||||
case h3:
|
||||
endpointMeasurement = m.measureEndpointH3(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers, proto)
|
||||
case proto == "http":
|
||||
endpointMeasurement = m.measureEndpointHTTP(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers)
|
||||
case proto == "https":
|
||||
endpointMeasurement = m.measureEndpointHTTPS(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers)
|
||||
default:
|
||||
panic("should not happen")
|
||||
}
|
||||
urlMeasurement.Endpoints = append(urlMeasurement.Endpoints, endpointMeasurement)
|
||||
}
|
||||
tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement {
|
||||
endpointMeasurement := &EndpointMeasurement{
|
||||
Endpoint: endpoint,
|
||||
Protocol: "http",
|
||||
}
|
||||
// TCP connect step
|
||||
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
|
||||
endpointMeasurement.TCPConnectMeasurement = &TCPConnectMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if err != nil {
|
||||
return endpointMeasurement
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// HTTP roundtrip step
|
||||
request := NewRequest(ctx, URL, headers)
|
||||
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
|
||||
Request: &HTTPRequestMeasurement{
|
||||
Headers: request.Header,
|
||||
Method: "GET",
|
||||
URL: URL.String(),
|
||||
},
|
||||
}
|
||||
transport := NewSingleTransport(conn)
|
||||
resp, body, err := HTTPDo(request, transport)
|
||||
if err != nil {
|
||||
// failed Response
|
||||
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
// successful Response
|
||||
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
|
||||
BodyLength: int64(len(body)),
|
||||
Failure: nil,
|
||||
Headers: resp.Header,
|
||||
StatusCode: int64(resp.StatusCode),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
|
||||
func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement {
|
||||
endpointMeasurement := &EndpointMeasurement{
|
||||
Endpoint: endpoint,
|
||||
Protocol: "https",
|
||||
}
|
||||
// TCP connect step
|
||||
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
|
||||
endpointMeasurement.TCPConnectMeasurement = &TCPConnectMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if err != nil {
|
||||
return endpointMeasurement
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// TLS handshake step
|
||||
tlsconn, err := TLSDo(conn, URL.Hostname())
|
||||
endpointMeasurement.TLSHandshakeMeasurement = &TLSHandshakeMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if err != nil {
|
||||
return endpointMeasurement
|
||||
}
|
||||
defer tlsconn.Close()
|
||||
|
||||
// HTTP roundtrip step
|
||||
request := NewRequest(ctx, URL, headers)
|
||||
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
|
||||
Request: &HTTPRequestMeasurement{
|
||||
Headers: request.Header,
|
||||
Method: "GET",
|
||||
URL: URL.String(),
|
||||
},
|
||||
}
|
||||
transport := NewSingleTransport(tlsconn)
|
||||
resp, body, err := HTTPDo(request, transport)
|
||||
if err != nil {
|
||||
// failed Response
|
||||
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
// successful Response
|
||||
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
|
||||
BodyLength: int64(len(body)),
|
||||
Failure: nil,
|
||||
Headers: resp.Header,
|
||||
StatusCode: int64(resp.StatusCode),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
|
||||
func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint string, headers http.Header, proto string) *EndpointMeasurement {
|
||||
endpointMeasurement := &EndpointMeasurement{
|
||||
Endpoint: endpoint,
|
||||
Protocol: proto,
|
||||
}
|
||||
tlsConf := &tls.Config{
|
||||
ServerName: URL.Hostname(),
|
||||
NextProtos: []string{proto},
|
||||
}
|
||||
// QUIC handshake step
|
||||
sess, err := QUICDo(ctx, QUICConfig{
|
||||
Endpoint: endpoint,
|
||||
TLSConf: tlsConf,
|
||||
})
|
||||
endpointMeasurement.QUICHandshakeMeasurement = &TLSHandshakeMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if err != nil {
|
||||
return endpointMeasurement
|
||||
}
|
||||
// HTTP roundtrip step
|
||||
request := NewRequest(ctx, URL, headers)
|
||||
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
|
||||
Request: &HTTPRequestMeasurement{
|
||||
Headers: request.Header,
|
||||
Method: "GET",
|
||||
URL: URL.String(),
|
||||
},
|
||||
}
|
||||
transport := NewSingleH3Transport(sess, tlsConf, &quic.Config{})
|
||||
resp, body, err := HTTPDo(request, transport)
|
||||
if err != nil {
|
||||
// failed Response
|
||||
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
// successful Response
|
||||
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
|
||||
BodyLength: int64(len(body)),
|
||||
Failure: nil,
|
||||
Headers: resp.Header,
|
||||
StatusCode: int64(resp.StatusCode),
|
||||
}
|
||||
return endpointMeasurement
|
||||
|
||||
}
|
||||
|
||||
// SummaryKeys contains summary keys for this experiment.
|
||||
//
|
||||
// Note that this structure is part of the ABI contract with probe-cli
|
||||
// therefore we should be careful when changing it.
|
||||
type SummaryKeys struct {
|
||||
Accessible bool `json:"accessible"`
|
||||
Blocking string `json:"blocking"`
|
||||
IsAnomaly bool `json:"-"`
|
||||
}
|
||||
|
||||
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
|
||||
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
|
||||
sk := SummaryKeys{}
|
||||
return sk, nil
|
||||
}
|
||||
|
||||
func makeEndpoints(addrs []string, URL *url.URL) []string {
|
||||
endpoints := []string{}
|
||||
if addrs == nil {
|
||||
return endpoints
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
var port string
|
||||
explicitPort := URL.Port()
|
||||
scheme := URL.Scheme
|
||||
switch {
|
||||
case explicitPort != "":
|
||||
port = explicitPort
|
||||
case scheme == "http":
|
||||
port = "80"
|
||||
case scheme == "https":
|
||||
port = "443"
|
||||
default:
|
||||
panic("should not happen")
|
||||
}
|
||||
endpoints = append(endpoints, net.JoinHostPort(addr, port))
|
||||
}
|
||||
return endpoints
|
||||
}
|
||||
Reference in New Issue
Block a user