From ba9151d4fa0afd7ab9a0bd30a89450af362a0102 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 30 Sep 2021 02:06:27 +0200 Subject: [PATCH] feat(webstepsx): websteps using measurex (#530) This diff adds the prototype websteps implementation that used to live at https://github.com/ooni/probe-cli/pull/506. The code is reasonably good already and it's pointing to a roaming test helper that I've properly configured. You can run websteps with: ``` ./miniooni -n websteps ``` This will go over the test list for your country. At this stage the mechanics of the experiment is set, but we still need to have a conversation on the following topics: 1. whether we're okay with reusing the data format used by other OONI experiments, or we would like to use a more compact data format (which may either be a more compact JSON or we can choose to always submit compressed measurements for websteps); 2. the extent to which we would like to keep the measurement as a collection of "the experiment saw this" and "the test helper saw that" and let the pipeline choose an overall score: this is clearly an option, but there is also the opposite option to build a summary of the measurement on the probe. Compared to the previous prototype of websteps, the main architectural change we have here is that we are following the point of view of the probe and the test helper is much more dumb. Basically, the probe will choose which redirection to follow and ask the test helper every time it discovers a new URL to measure it w/o redirections. Reference issue: https://github.com/ooni/probe/issues/1733 --- internal/cmd/oohelper/oohelper.go | 46 ++- internal/cmd/oohelperd/oohelperd.go | 2 + internal/engine/allexperiments.go | 8 +- internal/engine/experiment/webstepsx/doc.go | 6 + .../engine/experiment/webstepsx/measurer.go | 217 ++++++++++ internal/engine/experiment/webstepsx/th.go | 381 ++++++++++++++++++ 6 files changed, 650 insertions(+), 10 deletions(-) create mode 100644 internal/engine/experiment/webstepsx/doc.go create mode 100644 internal/engine/experiment/webstepsx/measurer.go create mode 100644 internal/engine/experiment/webstepsx/th.go diff --git a/internal/cmd/oohelper/oohelper.go b/internal/cmd/oohelper/oohelper.go index 46977bc..12d54a1 100644 --- a/internal/cmd/oohelper/oohelper.go +++ b/internal/cmd/oohelper/oohelper.go @@ -11,7 +11,9 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx" "github.com/ooni/probe-cli/v3/internal/engine/netx" + "github.com/ooni/probe-cli/v3/internal/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) @@ -20,8 +22,9 @@ var ( debug = flag.Bool("debug", false, "Toggle debug mode") httpClient *http.Client resolver netx.Resolver - server = flag.String("server", "https://wcth.ooni.io/", "URL of the test helper") + server = flag.String("server", "", "URL of the test helper") target = flag.String("target", "", "Target URL for the test helper") + fwebsteps = flag.Bool("websteps", false, "Use the websteps TH") ) func newhttpclient() *http.Client { @@ -43,18 +46,49 @@ func init() { } func main() { + defer cancel() logmap := map[bool]log.Level{ true: log.DebugLevel, false: log.InfoLevel, } flag.Parse() log.SetLevel(logmap[*debug]) - clnt := internal.OOClient{HTTPClient: httpClient, Resolver: resolver} - config := internal.OOConfig{TargetURL: *target, ServerURL: *server} - defer cancel() - cresp, err := clnt.Do(ctx, config) - runtimex.PanicOnError(err, "client.Do failed") + apimap := map[bool]func() interface{}{ + false: wcth, + true: webstepsth, + } + cresp := apimap[*fwebsteps]() data, err := json.MarshalIndent(cresp, "", " ") runtimex.PanicOnError(err, "json.MarshalIndent failed") fmt.Printf("%s\n", string(data)) } + +func webstepsth() interface{} { + serverURL := *server + if serverURL == "" { + serverURL = "https://1.th.ooni.org/api/v1/websteps" + } + clnt := &webstepsx.THClient{ + DNServers: []*measurex.ResolverInfo{{ + Network: "udp", + Address: "8.8.4.4:53", + }}, + HTTPClient: httpClient, + ServerURL: serverURL, + } + cresp, err := clnt.Run(ctx, *target) + runtimex.PanicOnError(err, "client.Run failed") + return cresp +} + +func wcth() interface{} { + serverURL := *server + if serverURL == "" { + serverURL = "https://wcth.ooni.io/" + } + clnt := internal.OOClient{HTTPClient: httpClient, Resolver: resolver} + config := internal.OOConfig{TargetURL: *target, ServerURL: serverURL} + cresp, err := clnt.Do(ctx, config) + runtimex.PanicOnError(err, "client.Do failed") + return cresp +} diff --git a/internal/cmd/oohelperd/oohelperd.go b/internal/cmd/oohelperd/oohelperd.go index e582962..2e095b5 100644 --- a/internal/cmd/oohelperd/oohelperd.go +++ b/internal/cmd/oohelperd/oohelperd.go @@ -11,6 +11,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/webconnectivity" "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/websteps" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx" "github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/runtimex" ) @@ -58,6 +59,7 @@ func main() { func testableMain() { mux := http.NewServeMux() mux.Handle("/api/unstable/websteps", &websteps.Handler{Config: &websteps.Config{}}) + mux.Handle("/api/v1/websteps", &webstepsx.THHandler{}) mux.Handle("/", webconnectivity.Handler{ Client: httpx, Dialer: dialer, diff --git a/internal/engine/allexperiments.go b/internal/engine/allexperiments.go index d587d6e..dd12d3d 100644 --- a/internal/engine/allexperiments.go +++ b/internal/engine/allexperiments.go @@ -23,7 +23,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/torsf" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx" "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" ) @@ -330,11 +330,11 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{ "websteps": func(session *Session) *ExperimentBuilder { return &ExperimentBuilder{ build: func(config interface{}) *Experiment { - return NewExperiment(session, websteps.NewExperimentMeasurer( - *config.(*websteps.Config), + return NewExperiment(session, webstepsx.NewExperimentMeasurer( + *config.(*webstepsx.Config), )) }, - config: &websteps.Config{}, + config: &webstepsx.Config{}, inputPolicy: InputOrQueryBackend, } }, diff --git a/internal/engine/experiment/webstepsx/doc.go b/internal/engine/experiment/webstepsx/doc.go new file mode 100644 index 0000000..5c3af36 --- /dev/null +++ b/internal/engine/experiment/webstepsx/doc.go @@ -0,0 +1,6 @@ +// Package webstepsx contains a websteps implementation +// based on the internal/measurex package. +// +// This implementation does not follow any existing spec +// rather we are modeling the spec on this one. +package webstepsx diff --git a/internal/engine/experiment/webstepsx/measurer.go b/internal/engine/experiment/webstepsx/measurer.go new file mode 100644 index 0000000..e5d4ad1 --- /dev/null +++ b/internal/engine/experiment/webstepsx/measurer.go @@ -0,0 +1,217 @@ +package webstepsx + +// +// Measurer +// +// This file contains the client implementation. +// + +import ( + "context" + "errors" + "net/http" + "net/url" + "time" + + "github.com/ooni/probe-cli/v3/internal/engine/model" + "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +const ( + testName = "websteps" + testVersion = "0.0.2" +) + +// Config contains the experiment config. +type Config struct{} + +// TestKeys contains the experiment's test keys. +type TestKeys struct { + *measurex.URLMeasurement +} + +// Measurer performs the measurement. +type Measurer struct { + Config Config +} + +var ( + _ model.ExperimentMeasurer = &Measurer{} + _ model.ExperimentMeasurerAsync = &Measurer{} +) + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return &Measurer{Config: config} +} + +// ExperimentName implements ExperimentMeasurer.ExperExperimentName. +func (mx *Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion. +func (mx *Measurer) ExperimentVersion() string { + return testVersion +} + +var ( + // ErrNoAvailableTestHelpers is emitted when there are no available test helpers. + ErrNoAvailableTestHelpers = errors.New("no available helpers") + + // ErrNoInput indicates that no input was provided. + ErrNoInput = errors.New("no input provided") + + // ErrInputIsNotAnURL indicates that the input is not an URL. + ErrInputIsNotAnURL = errors.New("input is not an URL") + + // ErrUnsupportedInput indicates that the input URL scheme is unsupported. + ErrUnsupportedInput = errors.New("unsupported input scheme") +) + +// RunAsync implements ExperimentMeasurerAsync.RunAsync. +func (mx *Measurer) RunAsync( + ctx context.Context, sess model.ExperimentSession, input string, + callbacks model.ExperimentCallbacks) (<-chan *model.ExperimentAsyncTestKeys, error) { + // 1. Parse and verify URL + URL, err := url.Parse(input) + if err != nil { + return nil, ErrInputIsNotAnURL + } + if URL.Scheme != "http" && URL.Scheme != "https" { + return nil, ErrUnsupportedInput + } + // 2. Find the testhelper + testhelpers, _ := sess.GetTestHelpersByName("web-connectivity") + var testhelper *model.Service + for _, th := range testhelpers { + if th.Type == "https" { + testhelper = &th + break + } + } + if testhelper == nil { + return nil, ErrNoAvailableTestHelpers + } + testhelper.Address = "https://1.th.ooni.org/api/v1/websteps" // TODO(bassosimone): remove! + out := make(chan *model.ExperimentAsyncTestKeys) + go mx.runAsync(ctx, sess, input, testhelper, out) + return out, nil +} + +var measurerResolvers = []*measurex.ResolverInfo{{ + Network: "system", + Address: "", +}, { + Network: "udp", + Address: "8.8.4.4:53", +}, { + Network: "udp", + Address: "1.1.1.1:53", +}} + +func (mx *Measurer) runAsync(ctx context.Context, sess model.ExperimentSession, + URL string, th *model.Service, out chan<- *model.ExperimentAsyncTestKeys) { + defer close(out) + helper := &measurerMeasureURLHelper{ + Clnt: sess.DefaultHTTPClient(), + Logger: sess.Logger(), + THURL: th.Address, + } + mmx := &measurex.Measurer{ + Begin: time.Now(), + HTTPClient: sess.DefaultHTTPClient(), + MeasureURLHelper: helper, + Logger: sess.Logger(), + Resolvers: measurerResolvers, + TLSHandshaker: netxlite.NewTLSHandshakerStdlib(sess.Logger()), + } + cookies := measurex.NewCookieJar() + in := mmx.MeasureURLAndFollowRedirections( + ctx, URL, measurex.NewHTTPRequestHeaderForMeasuring(), cookies) + for m := range in { + out <- &model.ExperimentAsyncTestKeys{ + MeasurementRuntime: m.TotalRuntime.Seconds(), + TestKeys: &TestKeys{URLMeasurement: m}, + Extensions: map[string]int64{ + archival.ExtHTTP.Name: archival.ExtHTTP.V, + archival.ExtDNS.Name: archival.ExtDNS.V, + archival.ExtNetevents.Name: archival.ExtNetevents.V, + archival.ExtTCPConnect.Name: archival.ExtTCPConnect.V, + archival.ExtTLSHandshake.Name: archival.ExtTLSHandshake.V, + }, + } + } +} + +// measurerMeasureURLHelper injects the TH into the normal +// URL measurement flow implemented by measurex. +type measurerMeasureURLHelper struct { + // Clnt is the MANDATORY client to use + Clnt measurex.HTTPClient + + // Logger is the MANDATORY Logger to use + Logger model.Logger + + // THURL is the MANDATORY TH URL. + THURL string +} + +func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints( + ctx context.Context, URL *url.URL, headers http.Header, + curEndpoints ...*measurex.HTTPEndpoint) ( + []*measurex.HTTPEndpoint, interface{}, error) { + cc := &THClientCall{ + Endpoints: measurex.HTTPEndpointsToEndpoints(curEndpoints), + HTTPClient: mth.Clnt, + Header: headers, + THURL: mth.THURL, + TargetURL: URL.String(), + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + ol := measurex.NewOperationLogger( + mth.Logger, "THClientCall %s", URL.String()) + resp, err := cc.Call(ctx) + ol.Stop(err) + if err != nil { + return nil, resp, err + } + var out []*measurex.HTTPEndpoint + for _, epnt := range resp.Endpoints { + out = append(out, &measurex.HTTPEndpoint{ + Domain: URL.Hostname(), + Network: epnt.Network, + Address: epnt.Address, + SNI: URL.Hostname(), + ALPN: measurex.ALPNForHTTPEndpoint(epnt.Network), + URL: URL, + Header: headers, + }) + } + return out, resp, nil +} + +// Run implements ExperimentMeasurer.Run. +func (mx *Measurer) Run(ctx context.Context, sess model.ExperimentSession, + measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { + return errors.New("sync run is not implemented") +} + +// SummaryKeys contains summary keys for this experiment. +// +// Note that this structure is part of the ABI contract with probe-cli +// therefore we should be careful when changing it. +type SummaryKeys struct { + Accessible bool `json:"accessible"` + Blocking string `json:"blocking"` + IsAnomaly bool `json:"-"` +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (mx *Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + sk := SummaryKeys{} + return sk, nil +} diff --git a/internal/engine/experiment/webstepsx/th.go b/internal/engine/experiment/webstepsx/th.go new file mode 100644 index 0000000..20978b4 --- /dev/null +++ b/internal/engine/experiment/webstepsx/th.go @@ -0,0 +1,381 @@ +package webstepsx + +// +// TH (Test Helper) +// +// This file contains an implementation of the +// (proposed) websteps test helper spec. +// + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/version" +) + +// +// Messages exchanged by the TH client and server +// + +// THClientRequest is the request received by the test helper. +type THClientRequest struct { + // Endpoints is a list of endpoints to measure. + Endpoints []*measurex.Endpoint + + // URL is the URL we want to measure. + URL string + + // HTTPRequestHeaders contains the request headers. + HTTPRequestHeaders http.Header +} + +// THServerResponse is the response from the test helper. +type THServerResponse struct { + // DNS contains all the DNS related measurements. + DNS []*measurex.DNSMeasurement `json:"dns"` + + // Endpoints contains a measurement for each endpoint + // that was discovered by the probe or the TH. + Endpoints []*measurex.HTTPEndpointMeasurement `json:"endpoints"` +} + +// thMaxAcceptableBodySize is the maximum acceptable body size by TH code. +const thMaxAcceptableBodySize = 1 << 20 + +// +// TH client implementation +// + +// THClient is the high-level API to invoke the TH. This API +// should be used by command line clients. +type THClient struct { + // DNSServers is the MANDATORY list of DNS-over-UDP + // servers to use to discover endpoints locally. + DNServers []*measurex.ResolverInfo + + // HTTPClient is the MANDATORY HTTP client to + // use for contacting the TH. + HTTPClient measurex.HTTPClient + + // ServerURL is the MANDATORY URL of the TH HTTP endpoint. + ServerURL string +} + +// Run calls the TH and returns the response or an error. +// +// Arguments: +// +// - ctx is the context with timeout/deadline/cancellation +// +// - URL is the URL the TH server should measure for us +// +// Algorithm: +// +// - use DNSServers to discover extra endpoints for the target URL +// +// - call the TH using the HTTPClient and the ServerURL +// +// - return response or error. +func (c *THClient) Run(ctx context.Context, URL string) (*THServerResponse, error) { + parsed, err := url.Parse(URL) + if err != nil { + return nil, err + } + mx := measurex.NewMeasurerWithDefaultSettings() + var dns []*measurex.DNSMeasurement + for m := range mx.LookupURLHostParallel(ctx, parsed, c.DNServers...) { + dns = append(dns, m) + } + endpoints, err := measurex.AllEndpointsForURL(parsed, dns...) + if err != nil { + return nil, err + } + return (&THClientCall{ + Endpoints: endpoints, + HTTPClient: c.HTTPClient, + Header: measurex.NewHTTPRequestHeaderForMeasuring(), + THURL: c.ServerURL, + TargetURL: URL, + }).Call(ctx) +} + +// THClientCall allows to perform a single TH client call. Make sure +// you fill all the fields marked as MANDATORY before use. +type THClientCall struct { + // Endpoints contains the MANDATORY endpoints we discovered. + Endpoints []*measurex.Endpoint + + // HTTPClient is the MANDATORY HTTP client to + // use for contacting the TH. + HTTPClient measurex.HTTPClient + + // Header contains the MANDATORY request headers. + Header http.Header + + // THURL is the MANDATORY test helper URL. + THURL string + + // TargetURL is the MANDATORY URL to measure. + TargetURL string +} + +// Call performs the specified TH call and returns either a response or an error. +func (c *THClientCall) Call(ctx context.Context) (*THServerResponse, error) { + creq := &THClientRequest{ + Endpoints: c.Endpoints, + URL: c.TargetURL, + HTTPRequestHeaders: c.Header, + } + reqBody, err := json.Marshal(creq) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext( + ctx, "POST", c.THURL, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", fmt.Sprintf("ooniprobe-cli/%s", version.Version)) + return c.httpClientDo(req) +} + +// errTHRequestFailed is the error returned if the TH response is not 200 Ok. +var errTHRequestFailed = errors.New("th: request failed") + +func (c *THClientCall) httpClientDo(req *http.Request) (*THServerResponse, error) { + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { // THHandler returns either 400 or 200 + return nil, errTHRequestFailed + } + r := io.LimitReader(resp.Body, thMaxAcceptableBodySize) + respBody, err := netxlite.ReadAllContext(req.Context(), r) + if err != nil { + return nil, err + } + var sresp THServerResponse + if err := json.Unmarshal(respBody, &sresp); err != nil { + return nil, err + } + return &sresp, nil +} + +// +// TH server implementation +// + +// THHandler implements the test helper API. +// +// This handler exposes a unique HTTP endpoint that you need to +// mount to the desired path when creating the server. +// +// The canonical mount point for the HTTP endpoint is /api/v1/websteps. +// +// Accepted methods and request body: +// +// - we only accept POST; +// +// - we expect a THClientRequest as the body. +// +// Status code and response body: +// +// - on success, status is 200 and THServerResponse is the body; +// +// - on failure, status is 400 and there is no body. +// +type THHandler struct{} + +// ServerHTTP implements http.Handler.ServeHTTP. +func (h *THHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + w.Header().Add("Server", fmt.Sprintf("oohelperd/%s", version.Version)) + if req.Method != "POST" { + w.WriteHeader(400) + return + } + reader := io.LimitReader(req.Body, thMaxAcceptableBodySize) + data, err := netxlite.ReadAllContext(req.Context(), reader) + if err != nil { + w.WriteHeader(400) + return + } + var creq THClientRequest + if err := json.Unmarshal(data, &creq); err != nil { + w.WriteHeader(400) + return + } + cresp, err := h.singleStep(req.Context(), &creq) + if err != nil { + w.WriteHeader(400) + return + } + // We assume that the following call cannot fail because it's a + // clearly serializable data structure. + data, err = json.Marshal(cresp) + runtimex.PanicOnError(err, "json.Marshal failed") + w.Header().Add("Content-Type", "application/json") + w.Write(data) +} + +// singleStep performs a singleStep measurement. +// +// The function name derives from the definition (we invented) +// of "web steps". Each redirection is a step. For each step you +// need to figure out the endpoints to use with the DNS. After +// that, you need to check all endpoints. Because here we do not +// perform redirection, this is just a single "step". +// +// The algorithm is the following: +// +// 1. parse the URL and return error if it does not parse or +// the scheme is neither HTTP nor HTTPS; +// +// 2. discover additional endpoints using a suitable DoH +// resolver and the URL's hostname as the domain; +// +// 3. measure each discovered endpoint. +// +// The return value is either a THServerResponse or an error. +func (h *THHandler) singleStep( + ctx context.Context, req *THClientRequest) (*THServerResponse, error) { + mx := measurex.NewMeasurerWithDefaultSettings() + mx.MeasureURLHelper = &thMeasureURLHelper{req.Endpoints} + mx.Resolvers = []*measurex.ResolverInfo{{ + Network: measurex.ResolverForeign, + ForeignResolver: thResolver, + }} + jar := measurex.NewCookieJar() + meas, err := mx.MeasureURL(ctx, req.URL, req.HTTPRequestHeaders, jar) + if err != nil { + return nil, err + } + return &THServerResponse{ + DNS: meas.DNS, + Endpoints: h.simplifyEndpoints(meas.Endpoints), + }, nil +} + +func (h *THHandler) simplifyEndpoints( + in []*measurex.HTTPEndpointMeasurement) (out []*measurex.HTTPEndpointMeasurement) { + for _, epnt := range in { + out = append(out, &measurex.HTTPEndpointMeasurement{ + URL: epnt.URL, + Network: epnt.Network, + Address: epnt.Address, + Measurement: h.simplifyMeasurement(epnt.Measurement), + }) + } + return +} + +func (h *THHandler) simplifyMeasurement(in *measurex.Measurement) (out *measurex.Measurement) { + out = &measurex.Measurement{ + Connect: in.Connect, + TLSHandshake: h.simplifyHandshake(in.TLSHandshake), + QUICHandshake: h.simplifyHandshake(in.QUICHandshake), + LookupHost: in.LookupHost, + LookupHTTPSSvc: in.LookupHTTPSSvc, + HTTPRoundTrip: h.simplifyHTTPRoundTrip(in.HTTPRoundTrip), + } + return +} + +func (h *THHandler) simplifyHandshake( + in []*measurex.TLSHandshakeEvent) (out []*measurex.TLSHandshakeEvent) { + for _, ev := range in { + out = append(out, &measurex.TLSHandshakeEvent{ + CipherSuite: ev.CipherSuite, + Failure: ev.Failure, + NegotiatedProto: ev.NegotiatedProto, + TLSVersion: ev.TLSVersion, + PeerCerts: nil, + Finished: 0, + RemoteAddr: ev.RemoteAddr, + SNI: ev.SNI, + ALPN: ev.ALPN, + SkipVerify: ev.SkipVerify, + Oddity: ev.Oddity, + Network: ev.Network, + Started: 0, + }) + } + return +} + +func (h *THHandler) simplifyHTTPRoundTrip( + in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) { + for _, ev := range in { + out = append(out, &measurex.HTTPRoundTripEvent{ + Failure: ev.Failure, + Request: ev.Request, + Response: h.simplifyHTTPResponse(ev.Response), + Finished: 0, + Started: 0, + Oddity: ev.Oddity, + }) + } + return +} + +func (h *THHandler) simplifyHTTPResponse( + in *measurex.HTTPResponse) (out *measurex.HTTPResponse) { + if in != nil { + out = &measurex.HTTPResponse{ + Code: in.Code, + Headers: in.Headers, + Body: nil, + BodyIsTruncated: in.BodyIsTruncated, + BodyLength: in.BodyLength, + BodyIsUTF8: in.BodyIsUTF8, + } + } + return +} + +type thMeasureURLHelper struct { + epnts []*measurex.Endpoint +} + +func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints( + ctx context.Context, URL *url.URL, headers http.Header, + serverEpnts ...*measurex.HTTPEndpoint) ( + epnts []*measurex.HTTPEndpoint, thMeaurement interface{}, err error) { + for _, epnt := range thh.epnts { + epnts = append(epnts, &measurex.HTTPEndpoint{ + Domain: URL.Hostname(), + Network: epnt.Network, + Address: epnt.Address, + SNI: URL.Hostname(), + ALPN: measurex.ALPNForHTTPEndpoint(epnt.Network), + URL: URL, + Header: headers, // but overriden later anyway + }) + } + return +} + +// thResolverURL is the DNS resolver URL used by the TH. We use an +// encrypted resolver to reduce the risk that there is DNS-over-UDP +// censorship in the place where we deploy the TH. +const thResolverURL = "https://dns.google/dns-query" + +// thResolver is the DNS resolver used by the TH. +// +// Here we're using github.com/apex/log as the logger, which +// is fine because this is backend only code. +var thResolver = netxlite.WrapResolver(log.Log, netxlite.NewSerialResolver( + netxlite.NewDNSOverHTTPS(http.DefaultClient, thResolverURL), +))