ooni-probe-cli/internal/engine/experiment/urlgetter/urlgetter.go

// Package urlgetter implements a nettest that fetches a URL.
//
// See https://github.com/ooni/spec/blob/master/nettests/ts-027-urlgetter.md.
//
// This package is now frozen. Please, use measurexlite for new code. New
// network experiments should not depend on this package. Please see
// https://github.com/ooni/probe-cli/blob/master/docs/design/dd-003-step-by-step.md
// for details about this.
package urlgetter

import (
	"context"
	"crypto/x509"
	"time"

	"github.com/ooni/probe-cli/v3/internal/model"
	"github.com/ooni/probe-cli/v3/internal/tracex"
)

const (
	testName    = "urlgetter"
	testVersion = "0.2.0"
)

// Config contains the experiment's configuration.
type Config struct {
	// not settable from command line
	CertPool *x509.CertPool
	Timeout  time.Duration

	// settable from command line
	DNSCache          string `ooni:"Add 'DOMAIN IP...' to cache"`
	DNSHTTPHost       string `ooni:"Force using specific HTTP Host header for DNS requests"`
	DNSTLSServerName  string `ooni:"Force TLS to using a specific SNI for encrypted DNS requests"`
	DNSTLSVersion     string `ooni:"Force specific TLS version used for DoT/DoH (e.g. 'TLSv1.3')"`
	FailOnHTTPError   bool   `ooni:"Fail HTTP request if status code is 400 or above"`
	HTTP3Enabled      bool   `ooni:"use http3 instead of http/1.1 or http2"`
	HTTPHost          string `ooni:"Force using specific HTTP Host header"`
	Method            string `ooni:"Force HTTP method different than GET"`
	NoFollowRedirects bool   `ooni:"Disable following redirects"`
	NoTLSVerify       bool   `ooni:"Disable TLS verification"`
	RejectDNSBogons   bool   `ooni:"Fail DNS lookup if response contains bogons"`
	ResolverURL       string `ooni:"URL describing the resolver to use"`
	TLSServerName     string `ooni:"Force TLS to using a specific SNI in Client Hello"`
	TLSVersion        string `ooni:"Force specific TLS version (e.g. 'TLSv1.3')"`
	Tunnel            string `ooni:"Run experiment over a tunnel, e.g. psiphon"`
	UserAgent         string `ooni:"Use the specified User-Agent"`
}

// TestKeys contains the experiment's result.
type TestKeys struct {
	// The following fields are part of the typical JSON emitted by OONI.
	Agent           string                   `json:"agent"`
	BootstrapTime   float64                  `json:"bootstrap_time,omitempty"`
	DNSCache        []string                 `json:"dns_cache,omitempty"`
	FailedOperation *string                  `json:"failed_operation"`
	Failure         *string                  `json:"failure"`
	NetworkEvents   []tracex.NetworkEvent    `json:"network_events"`
	Queries         []tracex.DNSQueryEntry   `json:"queries"`
	Requests        []tracex.RequestEntry    `json:"requests"`
	SOCKSProxy      string                   `json:"socksproxy,omitempty"`
	TCPConnect      []tracex.TCPConnectEntry `json:"tcp_connect"`
	TLSHandshakes   []tracex.TLSHandshake    `json:"tls_handshakes"`
	Tunnel          string                   `json:"tunnel,omitempty"`

	// The following fields are not serialised but are useful to simplify
	// analysing the measurements in telegram, whatsapp, etc.
	HTTPResponseStatus    int64    `json:"-"`
	HTTPResponseBody      string   `json:"-"`
	HTTPResponseLocations []string `json:"-"`
}

// RegisterExtensions registers the extensions used by the urlgetter
// experiment into the provided measurement.
func RegisterExtensions(m *model.Measurement) {
	tracex.ExtHTTP.AddTo(m)
	tracex.ExtDNS.AddTo(m)
	tracex.ExtNetevents.AddTo(m)
	tracex.ExtTCPConnect.AddTo(m)
	tracex.ExtTLSHandshake.AddTo(m)
	tracex.ExtTunnel.AddTo(m)
}

// Measurer performs the measurement.
type Measurer struct {
	Config
}

// ExperimentName implements model.ExperimentSession.ExperimentName
func (m Measurer) ExperimentName() string {
	return testName
}

// ExperimentVersion implements model.ExperimentSession.ExperimentVersion
func (m Measurer) ExperimentVersion() string {
	return testVersion
}

// Run implements model.ExperimentSession.Run
func (m Measurer) Run(
	ctx context.Context, sess model.ExperimentSession,
	measurement *model.Measurement, callbacks model.ExperimentCallbacks,
) error {
	// When using the urlgetter experiment directly, there is a nonconfigurable
	// default timeout that applies. When urlgetter is used as a library, it's
	// instead the responsibility of the user of urlgetter to set timeouts. Note
	// that this code is indeed only called when using urlgetter directly.
	if m.Config.Timeout <= 0 {
		m.Config.Timeout = 45 * time.Second
	}
	RegisterExtensions(measurement)
	g := Getter{
		Config:  m.Config,
		Session: sess,
		Target:  string(measurement.Input),
	}
	tk, _ := g.Get(ctx) // ignore error since we have the testkeys and we wanna submit them
	measurement.TestKeys = &tk
	return nil
}

// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
	return Measurer{Config: config}
}

// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with ooniprobe
// therefore we should be careful when changing it.
type SummaryKeys struct {
	IsAnomaly bool `json:"-"`
}

// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
	return SummaryKeys{IsAnomaly: false}, nil
}