cleanup: remove websteps summer 2021 implementation (#722)
See https://github.com/ooni/probe/issues/2094
This commit is contained in:
Simone Basso
GitHub
@@ -1,26 +0,0 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
)
|
||||
|
||||
// Control performs the control request and returns the response.
|
||||
func Control(
|
||||
ctx context.Context, sess model.ExperimentSession,
|
||||
thAddr string, resourcePath string, creq CtrlRequest) (out CtrlResponse, err error) {
|
||||
clnt := &httpx.APIClientTemplate{
|
||||
BaseURL: thAddr,
|
||||
HTTPClient: sess.DefaultHTTPClient(),
|
||||
Logger: sess.Logger(),
|
||||
}
|
||||
// make sure error is wrapped
|
||||
err = clnt.WithBodyLogging().Build().PostJSON(ctx, resourcePath, creq, &out)
|
||||
if err != nil {
|
||||
err = netxlite.NewTopLevelGenericErrWrapper(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
||||
)
|
||||
|
||||
type DNSConfig struct {
|
||||
Domain string
|
||||
Resolver model.Resolver
|
||||
}
|
||||
|
||||
// DNSDo performs the DNS check.
|
||||
func DNSDo(ctx context.Context, config DNSConfig) ([]string, error) {
|
||||
resolver := config.Resolver
|
||||
if resolver == nil {
|
||||
childResolver, err := netx.NewDNSClient(netx.Config{Logger: log.Log}, "doh://google")
|
||||
runtimex.PanicOnError(err, "NewDNSClient failed")
|
||||
resolver = childResolver
|
||||
resolver = &netxlite.ResolverIDNA{
|
||||
Resolver: resolver,
|
||||
}
|
||||
}
|
||||
return resolver.LookupHost(ctx, config.Domain)
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
oohttp "github.com/ooni/oohttp"
|
||||
"github.com/ooni/probe-cli/v3/internal/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
||||
)
|
||||
|
||||
var ErrNoConnReuse = errors.New("cannot reuse connection")
|
||||
|
||||
func NewRequest(ctx context.Context, URL *url.URL, headers http.Header) *http.Request {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil)
|
||||
runtimex.PanicOnError(err, "NewRequestWithContect failed")
|
||||
for k, vs := range headers {
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// NewDialerResolver contructs a new dialer for TCP connections,
|
||||
// with default, errorwrapping and resolve functionalities
|
||||
func NewDialerResolver(resolver model.Resolver) model.Dialer {
|
||||
var d model.Dialer = netxlite.DefaultDialer
|
||||
d = &netxlite.ErrorWrapperDialer{Dialer: d}
|
||||
d = &netxlite.DialerResolver{
|
||||
Resolver: resolver,
|
||||
Dialer: d,
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// NewQUICDialerResolver creates a new QUICDialerResolver
|
||||
// with default, errorwrapping and resolve functionalities
|
||||
func NewQUICDialerResolver(resolver model.Resolver) model.QUICDialer {
|
||||
var ql model.QUICListener = &netxlite.QUICListenerStdlib{}
|
||||
ql = &netxlite.ErrorWrapperQUICListener{QUICListener: ql}
|
||||
var dialer model.QUICDialer = &netxlite.QUICDialerQUICGo{
|
||||
QUICListener: ql,
|
||||
}
|
||||
dialer = &netxlite.ErrorWrapperQUICDialer{QUICDialer: dialer}
|
||||
dialer = &netxlite.QUICDialerResolver{
|
||||
Resolver: resolver,
|
||||
Dialer: dialer,
|
||||
}
|
||||
return dialer
|
||||
}
|
||||
|
||||
// NewSingleH3Transport creates an http3.RoundTripper.
|
||||
func NewSingleH3Transport(qconn quic.EarlyConnection, tlscfg *tls.Config, qcfg *quic.Config) http.RoundTripper {
|
||||
transport := &http3.RoundTripper{
|
||||
DisableCompression: true,
|
||||
TLSClientConfig: tlscfg,
|
||||
QuicConfig: qcfg,
|
||||
Dial: (&SingleDialerH3{qconn: &qconn}).Dial,
|
||||
}
|
||||
return transport
|
||||
}
|
||||
|
||||
// NewSingleTransport creates a new HTTP transport with a single-use dialer.
|
||||
func NewSingleTransport(conn net.Conn) http.RoundTripper {
|
||||
singledialer := &SingleDialer{conn: &conn}
|
||||
transport := newBaseTransport()
|
||||
transport.DialContext = singledialer.DialContext
|
||||
transport.DialTLSContext = singledialer.DialContext
|
||||
return transport
|
||||
}
|
||||
|
||||
// NewSingleTransport creates a new HTTP transport with a custom dialer and handshaker.
|
||||
func NewTransportWithDialer(dialer model.Dialer, tlsConfig *tls.Config, handshaker model.TLSHandshaker) http.RoundTripper {
|
||||
transport := newBaseTransport()
|
||||
transport.DialContext = dialer.DialContext
|
||||
transport.DialTLSContext = (&netxlite.TLSDialerLegacy{
|
||||
Config: tlsConfig,
|
||||
Dialer: dialer,
|
||||
TLSHandshaker: handshaker,
|
||||
}).DialTLSContext
|
||||
return transport
|
||||
}
|
||||
|
||||
// newBaseTransport creates a new HTTP transport with the default dialer.
|
||||
func newBaseTransport() (transport *oohttp.StdlibTransport) {
|
||||
base := oohttp.DefaultTransport.(*oohttp.Transport).Clone()
|
||||
base.DisableCompression = true
|
||||
base.MaxConnsPerHost = 1
|
||||
transport = &oohttp.StdlibTransport{Transport: base}
|
||||
return transport
|
||||
}
|
||||
|
||||
type SingleDialer struct {
|
||||
sync.Mutex
|
||||
conn *net.Conn
|
||||
}
|
||||
|
||||
func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
if s.conn == nil {
|
||||
return nil, ErrNoConnReuse
|
||||
}
|
||||
c := s.conn
|
||||
s.conn = nil
|
||||
return *c, nil
|
||||
}
|
||||
|
||||
type SingleDialerH3 struct {
|
||||
sync.Mutex
|
||||
qconn *quic.EarlyConnection
|
||||
}
|
||||
|
||||
func (s *SingleDialerH3) Dial(ctx context.Context, network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
if s.qconn == nil {
|
||||
return nil, ErrNoConnReuse
|
||||
}
|
||||
qs := s.qconn
|
||||
s.qconn = nil
|
||||
return *qs, nil
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
)
|
||||
|
||||
// HTTPDo performs the HTTP check.
|
||||
// Input:
|
||||
// req *http.Request
|
||||
// The same request than the one used by the Explore step.
|
||||
// This means that req contains the headers set by the original CtrlRequest, as well as,
|
||||
// in case of a redirect chain, additional headers that were added due to redirects
|
||||
// transport http.RoundTripper:
|
||||
// The transport to use, either http.Transport, or http3.RoundTripper.
|
||||
func HTTPDo(req *http.Request, transport http.RoundTripper) (*http.Response, []byte, error) {
|
||||
clnt := http.Client{
|
||||
CheckRedirect: func(r *http.Request, reqs []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
Transport: transport,
|
||||
}
|
||||
resp, err := clnt.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := netxlite.ReadAllContext(req.Context(), resp.Body)
|
||||
if err != nil {
|
||||
return resp, nil, nil
|
||||
}
|
||||
return resp, body, nil
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
package websteps
|
||||
|
||||
import "net/http"
|
||||
|
||||
// Websteps test helper spec messages:
|
||||
|
||||
// CtrlRequest is the request sent by the probe to the test helper.
|
||||
type CtrlRequest struct {
|
||||
// URL is the mandatory URL to measure.
|
||||
URL string `json:"url"`
|
||||
|
||||
// Headers contains optional headers.
|
||||
Headers map[string][]string `json:"headers"`
|
||||
|
||||
// Addrs contains the optional IP addresses resolved by the
|
||||
// probe for the domain inside URL.
|
||||
Addrs []string `json:"addrs"`
|
||||
}
|
||||
|
||||
// CtrlResponse is the response from the test helper.
|
||||
type CtrlResponse struct {
|
||||
// URLs contains the URLs we should measure. These URLs
|
||||
// derive from CtrlRequest.URL.
|
||||
URLs []*URLMeasurement `json:"urls"`
|
||||
}
|
||||
|
||||
// URLMeasurement contains all the URLs measured by the test helper.
|
||||
type URLMeasurement struct {
|
||||
// URL is the URL to which this measurement refers.
|
||||
URL string `json:"url"`
|
||||
|
||||
// DNS contains the domain names resolved by the test helper.
|
||||
DNS *DNSMeasurement `json:"dns"`
|
||||
|
||||
// Endpoints contains endpoint measurements.
|
||||
Endpoints []*EndpointMeasurement `json:"endpoints"`
|
||||
|
||||
// RoundTrip is the related round trip. This field MUST NOT be
|
||||
// exported as JSON, since it's only used internally by the test
|
||||
// helper and it's completely ignored by the probe.
|
||||
RoundTrip *RoundTripInfo `json:"-"`
|
||||
}
|
||||
|
||||
// RoundTripInfo contains info on a specific round trip. This data
|
||||
// structure is not part of the test helper protocol. We use it
|
||||
// _inside_ the test helper to describe the discovery phase where
|
||||
// we gather all the URLs that can derive from a given URL.
|
||||
type RoundTripInfo struct {
|
||||
// Proto is the protocol used, it can be "h2", "http/1.1", "h3".
|
||||
Proto string
|
||||
|
||||
// Request is the original HTTP request. Headers also include cookies.
|
||||
Request *http.Request
|
||||
|
||||
// Response is the HTTP response.
|
||||
Response *http.Response
|
||||
|
||||
// SortIndex is the index using for sorting round trips.
|
||||
SortIndex int
|
||||
}
|
||||
|
||||
// DNSMeasurement is a DNS measurement.
|
||||
type DNSMeasurement struct {
|
||||
// Domain is the domain we wanted to resolve.
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Failure is the error that occurred.
|
||||
Failure *string `json:"failure"`
|
||||
|
||||
// Addrs contains the resolved addresses.
|
||||
Addrs []string `json:"addrs"`
|
||||
}
|
||||
|
||||
// EndpointMeasurement is an HTTP measurement where we are using
|
||||
// a specific TCP/TLS/QUIC endpoint to get the URL.
|
||||
//
|
||||
// The specification describes this data structure as the sum of
|
||||
// three distinct types: HTTPEndpointMeasurement for "http",
|
||||
// HTTPSEndpointMeasurement for "https", and H3EndpointMeasurement
|
||||
// for "h3". We don't have sum types here, therefore we use the
|
||||
// Protocol field to indicate which fields are meaningful.
|
||||
type EndpointMeasurement struct {
|
||||
// Endpoint is the endpoint we're measuring.
|
||||
Endpoint string `json:"endpoint"`
|
||||
|
||||
// Protocol is one of "http", "https", and "h3".
|
||||
Protocol string `json:"protocol"`
|
||||
|
||||
// TCPConnect is the TCP connect measurement. This field
|
||||
// is only meaningful when protocol is "http" or "https."
|
||||
TCPConnect *TCPConnectMeasurement `json:"tcp_connect"`
|
||||
|
||||
// QUICHandshake is the QUIC handshake measurement. This field
|
||||
// is only meaningful when the protocol is "h3".
|
||||
QUICHandshake *QUICHandshakeMeasurement `json:"quic_handshake"`
|
||||
|
||||
// TLSHandshake is the TLS handshake measurement. This field
|
||||
// is only meaningful when the protocol is "https".
|
||||
TLSHandshake *TLSHandshakeMeasurement `json:"tls_handshake"`
|
||||
|
||||
// HTTPRoundTrip is the related HTTP GET measurement.
|
||||
HTTPRoundTrip *HTTPRoundTripMeasurement `json:"http_round_trip"`
|
||||
}
|
||||
|
||||
// TCPConnectMeasurement is a TCP connect measurement.
|
||||
type TCPConnectMeasurement struct {
|
||||
// Failure is the error that occurred.
|
||||
Failure *string `json:"failure"`
|
||||
}
|
||||
|
||||
// TLSHandshakeMeasurement is a TLS handshake measurement.
|
||||
type TLSHandshakeMeasurement struct {
|
||||
// Failure is the error that occurred.
|
||||
Failure *string `json:"failure"`
|
||||
}
|
||||
|
||||
// QUICHandshakeMeasurement is a QUIC handshake measurement.
|
||||
type QUICHandshakeMeasurement = TLSHandshakeMeasurement
|
||||
|
||||
// HTTPRoundTripMeasurement contains a measured HTTP request and
|
||||
// the corresponding response.
|
||||
type HTTPRoundTripMeasurement struct {
|
||||
// Request contains request data.
|
||||
Request *HTTPRequestMeasurement `json:"request"`
|
||||
|
||||
// Response contains response data.
|
||||
Response *HTTPResponseMeasurement `json:"response"`
|
||||
}
|
||||
|
||||
// HTTPRequestMeasurement contains request data.
|
||||
type HTTPRequestMeasurement struct {
|
||||
// Method is the request method.
|
||||
Method string `json:"method"`
|
||||
|
||||
// URL is the request URL.
|
||||
URL string `json:"url"`
|
||||
|
||||
// Headers contains request headers.
|
||||
Headers http.Header `json:"headers"`
|
||||
}
|
||||
|
||||
// HTTPResponseMeasurement contains response data.
|
||||
type HTTPResponseMeasurement struct {
|
||||
// BodyLength contains the body length in bytes.
|
||||
BodyLength int64 `json:"body_length"`
|
||||
|
||||
// Failure is the error that occurred.
|
||||
Failure *string `json:"failure"`
|
||||
|
||||
// Headers contains response headers.
|
||||
Headers http.Header `json:"headers"`
|
||||
|
||||
// StatusCode is the response status code.
|
||||
StatusCode int64 `json:"status_code"`
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/ooni/probe-cli/v3/internal/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
)
|
||||
|
||||
type QUICConfig struct {
|
||||
Endpoint string
|
||||
QUICDialer model.QUICDialer
|
||||
Resolver model.Resolver
|
||||
TLSConf *tls.Config
|
||||
}
|
||||
|
||||
// QUICDo performs the QUIC check.
|
||||
func QUICDo(ctx context.Context, config QUICConfig) (quic.EarlyConnection, error) {
|
||||
if config.QUICDialer != nil {
|
||||
return config.QUICDialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{})
|
||||
}
|
||||
resolver := config.Resolver
|
||||
if resolver == nil {
|
||||
resolver = &netxlite.ResolverSystem{}
|
||||
}
|
||||
dialer := NewQUICDialerResolver(resolver)
|
||||
return dialer.DialContext(ctx, "udp", config.Endpoint, config.TLSConf, &quic.Config{})
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
)
|
||||
|
||||
type TCPConfig struct {
|
||||
Dialer model.Dialer
|
||||
Endpoint string
|
||||
Resolver model.Resolver
|
||||
}
|
||||
|
||||
// TCPDo performs the TCP check.
|
||||
func TCPDo(ctx context.Context, config TCPConfig) (net.Conn, error) {
|
||||
if config.Dialer != nil {
|
||||
return config.Dialer.DialContext(ctx, "tcp", config.Endpoint)
|
||||
}
|
||||
resolver := config.Resolver
|
||||
if resolver == nil {
|
||||
resolver = &netxlite.ResolverSystem{}
|
||||
}
|
||||
dialer := NewDialerResolver(resolver)
|
||||
return dialer.DialContext(ctx, "tcp", config.Endpoint)
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
||||
utls "gitlab.com/yawning/utls.git"
|
||||
)
|
||||
|
||||
// TLSDo performs the TLS check.
|
||||
func TLSDo(ctx context.Context, conn net.Conn, hostname string) (net.Conn, error) {
|
||||
tlsConf := &tls.Config{
|
||||
ServerName: hostname,
|
||||
NextProtos: []string{"h2", "http/1.1"},
|
||||
}
|
||||
h := &netxlite.TLSHandshakerConfigurable{
|
||||
NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto),
|
||||
}
|
||||
tlsConn, _, err := h.Handshake(ctx, conn, tlsConf)
|
||||
return tlsConn, err
|
||||
}
|
||||
@@ -1,369 +0,0 @@
|
||||
// Package websteps implements the websteps experiment.
|
||||
//
|
||||
// Specifications:
|
||||
//
|
||||
// - test helper: https://github.com/ooni/spec/blob/master/backends/th-007-websteps.md
|
||||
//
|
||||
// - experiment: N/A.
|
||||
//
|
||||
// We are currently implementing:
|
||||
//
|
||||
// - version 202108.17.1114 of the test helper spec.
|
||||
//
|
||||
// - version N/A of the experiment spec.
|
||||
package websteps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
|
||||
"github.com/ooni/probe-cli/v3/internal/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
||||
)
|
||||
|
||||
const (
|
||||
testName = "websteps"
|
||||
testVersion = "0.0.1"
|
||||
)
|
||||
|
||||
// Config contains the experiment config.
|
||||
type Config struct{}
|
||||
|
||||
// TestKeys contains webconnectivity test keys.
|
||||
type TestKeys struct {
|
||||
Agent string `json:"agent"`
|
||||
ClientResolver string `json:"client_resolver"`
|
||||
URLMeasurements []*URLMeasurement
|
||||
}
|
||||
|
||||
// Measurer performs the measurement.
|
||||
type Measurer struct {
|
||||
Config Config
|
||||
}
|
||||
|
||||
// NewExperimentMeasurer creates a new ExperimentMeasurer.
|
||||
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
|
||||
return Measurer{Config: config}
|
||||
}
|
||||
|
||||
// ExperimentName implements ExperimentMeasurer.ExperExperimentName.
|
||||
func (m Measurer) ExperimentName() string {
|
||||
return testName
|
||||
}
|
||||
|
||||
// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion.
|
||||
func (m Measurer) ExperimentVersion() string {
|
||||
return testVersion
|
||||
}
|
||||
|
||||
// SupportedQUICVersions are the H3 over QUIC versions we currently support
|
||||
var SupportedQUICVersions = map[string]bool{
|
||||
"h3": true,
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
|
||||
ErrNoAvailableTestHelpers = errors.New("no available helpers")
|
||||
|
||||
// ErrNoInput indicates that no input was provided
|
||||
ErrNoInput = errors.New("no input provided")
|
||||
|
||||
// ErrInputIsNotAnURL indicates that the input is not an URL.
|
||||
ErrInputIsNotAnURL = errors.New("input is not an URL")
|
||||
|
||||
// ErrUnsupportedInput indicates that the input URL scheme is unsupported.
|
||||
ErrUnsupportedInput = errors.New("unsupported input scheme")
|
||||
)
|
||||
|
||||
// Run implements ExperimentMeasurer.Run.
|
||||
func (m Measurer) Run(
|
||||
ctx context.Context,
|
||||
sess model.ExperimentSession,
|
||||
measurement *model.Measurement,
|
||||
callbacks model.ExperimentCallbacks,
|
||||
) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
tk := new(TestKeys)
|
||||
measurement.TestKeys = tk
|
||||
tk.Agent = "redirect"
|
||||
tk.ClientResolver = sess.ResolverIP()
|
||||
|
||||
// 1. Parse and verify URL
|
||||
URL, err := url.Parse(string(measurement.Input))
|
||||
if err != nil {
|
||||
return ErrInputIsNotAnURL
|
||||
}
|
||||
if URL.Scheme != "http" && URL.Scheme != "https" {
|
||||
return ErrUnsupportedInput
|
||||
}
|
||||
// 2. Perform the initial DNS lookup step
|
||||
addrs, err := DNSDo(ctx, DNSConfig{Domain: URL.Hostname()})
|
||||
endpoints := makeEndpoints(addrs, URL)
|
||||
// 3. Find the testhelper
|
||||
// TODO(kelmenhorst,bassosimone): this is not used at the moment, but the hardcoded local address
|
||||
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
|
||||
var testhelper *model.OOAPIService
|
||||
for _, th := range testhelpers {
|
||||
if th.Type == "https" {
|
||||
testhelper = &th
|
||||
break
|
||||
}
|
||||
}
|
||||
if testhelper == nil {
|
||||
return ErrNoAvailableTestHelpers
|
||||
}
|
||||
measurement.TestHelpers = map[string]interface{}{
|
||||
"backend": testhelper,
|
||||
}
|
||||
// 4. Query the testhelper
|
||||
// TODO(kelmenhorst,bassosimone): remove hardcoded version here, this is only for testing purposes
|
||||
resp, err := Control(ctx, sess, "http://localhost:8080", "/api/unstable/websteps", CtrlRequest{
|
||||
URL: URL.String(),
|
||||
Headers: map[string][]string{
|
||||
"Accept": {httpheader.Accept()},
|
||||
"Accept-Language": {httpheader.AcceptLanguage()},
|
||||
"User-Agent": {httpheader.UserAgent()},
|
||||
},
|
||||
Addrs: endpoints,
|
||||
})
|
||||
if err != nil || resp.URLs == nil {
|
||||
return errors.New("no control response")
|
||||
}
|
||||
// 5. Go over the Control URL measurements and reproduce them without following redirects, one by one.
|
||||
for _, controlURLMeasurement := range resp.URLs {
|
||||
urlMeasurement := &URLMeasurement{
|
||||
URL: controlURLMeasurement.URL,
|
||||
Endpoints: []*EndpointMeasurement{},
|
||||
}
|
||||
URL, err = url.Parse(controlURLMeasurement.URL)
|
||||
runtimex.PanicOnError(err, "url.Parse failed")
|
||||
// DNS step
|
||||
addrs, err = DNSDo(ctx, DNSConfig{Domain: URL.Hostname()})
|
||||
urlMeasurement.DNS = &DNSMeasurement{
|
||||
Domain: URL.Hostname(),
|
||||
Addrs: addrs,
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if controlURLMeasurement.Endpoints == nil {
|
||||
tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement)
|
||||
continue
|
||||
}
|
||||
// the testhelper tells us which endpoints to measure
|
||||
for _, controlEndpoint := range controlURLMeasurement.Endpoints {
|
||||
rt := controlEndpoint.HTTPRoundTrip
|
||||
if rt == nil || rt.Request == nil {
|
||||
continue
|
||||
}
|
||||
var endpointMeasurement *EndpointMeasurement
|
||||
proto := controlEndpoint.Protocol
|
||||
_, h3 := SupportedQUICVersions[proto]
|
||||
switch {
|
||||
case h3:
|
||||
endpointMeasurement = m.measureEndpointH3(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers, proto)
|
||||
case proto == "http":
|
||||
endpointMeasurement = m.measureEndpointHTTP(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers)
|
||||
case proto == "https":
|
||||
endpointMeasurement = m.measureEndpointHTTPS(ctx, URL, controlEndpoint.Endpoint, rt.Request.Headers)
|
||||
default:
|
||||
panic("should not happen")
|
||||
}
|
||||
urlMeasurement.Endpoints = append(urlMeasurement.Endpoints, endpointMeasurement)
|
||||
}
|
||||
tk.URLMeasurements = append(tk.URLMeasurements, urlMeasurement)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement {
|
||||
endpointMeasurement := &EndpointMeasurement{
|
||||
Endpoint: endpoint,
|
||||
Protocol: "http",
|
||||
}
|
||||
// TCP connect step
|
||||
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
|
||||
endpointMeasurement.TCPConnect = &TCPConnectMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if err != nil {
|
||||
return endpointMeasurement
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// HTTP roundtrip step
|
||||
request := NewRequest(ctx, URL, headers)
|
||||
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
|
||||
Request: &HTTPRequestMeasurement{
|
||||
Headers: request.Header,
|
||||
Method: "GET",
|
||||
URL: URL.String(),
|
||||
},
|
||||
}
|
||||
transport := NewSingleTransport(conn)
|
||||
resp, body, err := HTTPDo(request, transport)
|
||||
if err != nil {
|
||||
// failed Response
|
||||
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
// successful Response
|
||||
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
|
||||
BodyLength: int64(len(body)),
|
||||
Failure: nil,
|
||||
Headers: resp.Header,
|
||||
StatusCode: int64(resp.StatusCode),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
|
||||
func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpoint string, headers http.Header) *EndpointMeasurement {
|
||||
endpointMeasurement := &EndpointMeasurement{
|
||||
Endpoint: endpoint,
|
||||
Protocol: "https",
|
||||
}
|
||||
// TCP connect step
|
||||
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
|
||||
endpointMeasurement.TCPConnect = &TCPConnectMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if err != nil {
|
||||
return endpointMeasurement
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// TLS handshake step
|
||||
tlsconn, err := TLSDo(ctx, conn, URL.Hostname())
|
||||
endpointMeasurement.TLSHandshake = &TLSHandshakeMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if err != nil {
|
||||
return endpointMeasurement
|
||||
}
|
||||
defer tlsconn.Close()
|
||||
|
||||
// HTTP roundtrip step
|
||||
request := NewRequest(ctx, URL, headers)
|
||||
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
|
||||
Request: &HTTPRequestMeasurement{
|
||||
Headers: request.Header,
|
||||
Method: "GET",
|
||||
URL: URL.String(),
|
||||
},
|
||||
}
|
||||
transport := NewSingleTransport(tlsconn)
|
||||
resp, body, err := HTTPDo(request, transport)
|
||||
if err != nil {
|
||||
// failed Response
|
||||
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
// successful Response
|
||||
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
|
||||
BodyLength: int64(len(body)),
|
||||
Failure: nil,
|
||||
Headers: resp.Header,
|
||||
StatusCode: int64(resp.StatusCode),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
|
||||
func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint string, headers http.Header, proto string) *EndpointMeasurement {
|
||||
endpointMeasurement := &EndpointMeasurement{
|
||||
Endpoint: endpoint,
|
||||
Protocol: proto,
|
||||
}
|
||||
tlsConf := &tls.Config{
|
||||
ServerName: URL.Hostname(),
|
||||
NextProtos: []string{proto},
|
||||
}
|
||||
// QUIC handshake step
|
||||
sess, err := QUICDo(ctx, QUICConfig{
|
||||
Endpoint: endpoint,
|
||||
TLSConf: tlsConf,
|
||||
})
|
||||
endpointMeasurement.QUICHandshake = &TLSHandshakeMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
if err != nil {
|
||||
return endpointMeasurement
|
||||
}
|
||||
// HTTP roundtrip step
|
||||
request := NewRequest(ctx, URL, headers)
|
||||
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
|
||||
Request: &HTTPRequestMeasurement{
|
||||
Headers: request.Header,
|
||||
Method: "GET",
|
||||
URL: URL.String(),
|
||||
},
|
||||
}
|
||||
transport := NewSingleH3Transport(sess, tlsConf, &quic.Config{})
|
||||
resp, body, err := HTTPDo(request, transport)
|
||||
if err != nil {
|
||||
// failed Response
|
||||
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
|
||||
Failure: archival.NewFailure(err),
|
||||
}
|
||||
return endpointMeasurement
|
||||
}
|
||||
// successful Response
|
||||
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
|
||||
BodyLength: int64(len(body)),
|
||||
Failure: nil,
|
||||
Headers: resp.Header,
|
||||
StatusCode: int64(resp.StatusCode),
|
||||
}
|
||||
return endpointMeasurement
|
||||
|
||||
}
|
||||
|
||||
// SummaryKeys contains summary keys for this experiment.
|
||||
//
|
||||
// Note that this structure is part of the ABI contract with ooniprobe
|
||||
// therefore we should be careful when changing it.
|
||||
type SummaryKeys struct {
|
||||
Accessible bool `json:"accessible"`
|
||||
Blocking string `json:"blocking"`
|
||||
IsAnomaly bool `json:"-"`
|
||||
}
|
||||
|
||||
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
|
||||
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
|
||||
sk := SummaryKeys{}
|
||||
return sk, nil
|
||||
}
|
||||
|
||||
func makeEndpoints(addrs []string, URL *url.URL) []string {
|
||||
endpoints := []string{}
|
||||
if addrs == nil {
|
||||
return endpoints
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
var port string
|
||||
explicitPort := URL.Port()
|
||||
scheme := URL.Scheme
|
||||
switch {
|
||||
case explicitPort != "":
|
||||
port = explicitPort
|
||||
case scheme == "http":
|
||||
port = "80"
|
||||
case scheme == "https":
|
||||
port = "443"
|
||||
default:
|
||||
panic("should not happen")
|
||||
}
|
||||
endpoints = append(endpoints, net.JoinHostPort(addr, port))
|
||||
}
|
||||
return endpoints
|
||||
}
|
||||
Reference in New Issue
Block a user