diff --git a/internal/cmd/oohelperd/dns.go b/internal/cmd/oohelperd/dns.go index 6b64940..2e4cb24 100644 --- a/internal/cmd/oohelperd/dns.go +++ b/internal/cmd/oohelperd/dns.go @@ -50,7 +50,11 @@ func dnsDo(ctx context.Context, config *dnsConfig) { addrs = []string{} // fix: the old test helper did that } failure := dnsMapFailure(newfailure(err)) - config.Out <- ctrlDNSResult{Failure: failure, Addrs: addrs} + config.Out <- ctrlDNSResult{ + Failure: failure, + Addrs: addrs, + ASNs: []int64{}, // unused by the TH and not serialized + } } // dnsMapFailure attempts to map netxlite failures to the strings diff --git a/internal/cmd/oohelperd/handler.go b/internal/cmd/oohelperd/handler.go index 691f893..f952fe1 100644 --- a/internal/cmd/oohelperd/handler.go +++ b/internal/cmd/oohelperd/handler.go @@ -29,6 +29,9 @@ type handler struct { // NewResolver is the MANDATORY factory for creating a new resolver. NewResolver func() model.Resolver + + // NewTLSHandshaker is the MANDATORY factory for creating a new TLS handshaker. + NewTLSHandshaker func() model.TLSHandshaker } var _ http.Handler = &handler{} diff --git a/internal/cmd/oohelperd/handler_test.go b/internal/cmd/oohelperd/handler_test.go index ac37ae1..945de86 100644 --- a/internal/cmd/oohelperd/handler_test.go +++ b/internal/cmd/oohelperd/handler_test.go @@ -63,6 +63,9 @@ func TestHandlerWorkingAsIntended(t *testing.T) { NewResolver: func() model.Resolver { return netxlite.NewUnwrappedStdlibResolver() }, + NewTLSHandshaker: func() model.TLSHandshaker { + return netxlite.NewTLSHandshakerStdlib(model.DiscardLogger) + }, } srv := httptest.NewServer(handler) defer srv.Close() @@ -76,25 +79,37 @@ func TestHandlerWorkingAsIntended(t *testing.T) { parseBody bool } expectations := []expectationSpec{{ - name: "check for invalid method", - reqMethod: "GET", - respStatusCode: 400, + name: "check for invalid method", + reqMethod: "GET", + reqContentType: "", + reqBody: "", + respStatusCode: 400, + respContentType: "", + parseBody: false, }, { - name: "check for invalid content-type", - reqMethod: "POST", - respStatusCode: 400, + name: "check for invalid content-type", + reqMethod: "POST", + reqContentType: "", + reqBody: "", + respStatusCode: 400, + respContentType: "", + parseBody: false, }, { - name: "check for invalid request body", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: "{", - respStatusCode: 400, + name: "check for invalid request body", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: "{", + respStatusCode: 400, + respContentType: "", + parseBody: false, }, { - name: "with measurement failure", - reqMethod: "POST", - reqContentType: "application/json", - reqBody: `{"http_request": "http://[::1]aaaa"}`, - respStatusCode: 400, + name: "with measurement failure", + reqMethod: "POST", + reqContentType: "application/json", + reqBody: `{"http_request": "http://[::1]aaaa"}`, + respStatusCode: 400, + respContentType: "", + parseBody: false, }, { name: "with reasonably good request", reqMethod: "POST", diff --git a/internal/cmd/oohelperd/http.go b/internal/cmd/oohelperd/http.go index 596404c..43a34c2 100644 --- a/internal/cmd/oohelperd/http.go +++ b/internal/cmd/oohelperd/http.go @@ -18,6 +18,9 @@ import ( "github.com/ooni/probe-cli/v3/internal/tracex" ) +// TODO(bassosimone): we should refactor the TH to use step-by-step such that we +// can use an existing connection for the HTTP-measuring task + // ctrlHTTPResponse is the result of the HTTP check performed by // the Web Connectivity test helper. type ctrlHTTPResponse = webconnectivity.ControlHTTPRequestResult @@ -51,11 +54,13 @@ func httpDo(ctx context.Context, config *httpConfig) { defer config.Wg.Done() req, err := http.NewRequestWithContext(ctx, "GET", config.URL, nil) if err != nil { - config.Out <- ctrlHTTPResponse{ // fix: emit -1 like the old test helper does + // fix: emit -1 like the old test helper does + config.Out <- ctrlHTTPResponse{ BodyLength: -1, Failure: httpMapFailure(err), - StatusCode: -1, + Title: "", Headers: map[string]string{}, + StatusCode: -1, } return } @@ -73,11 +78,13 @@ func httpDo(ctx context.Context, config *httpConfig) { defer clnt.CloseIdleConnections() resp, err := clnt.Do(req) if err != nil { - config.Out <- ctrlHTTPResponse{ // fix: emit -1 like old test helper does + // fix: emit -1 like the old test helper does + config.Out <- ctrlHTTPResponse{ BodyLength: -1, Failure: httpMapFailure(err), - StatusCode: -1, + Title: "", Headers: map[string]string{}, + StatusCode: -1, } return } diff --git a/internal/cmd/oohelperd/ipinfo.go b/internal/cmd/oohelperd/ipinfo.go index b542174..6cbdf98 100644 --- a/internal/cmd/oohelperd/ipinfo.go +++ b/internal/cmd/oohelperd/ipinfo.go @@ -52,6 +52,9 @@ type endpointInfo struct { // Epnt is the endpoint to measure Epnt string + + // TLS indicates whether we should try using TLS + TLS bool } // ipInfoToEndpoints takes in input the [ipinfo] returned by newIPInfo @@ -86,6 +89,7 @@ func ipInfoToEndpoints(URL *url.URL, ipinfo map[string]*webconnectivity.ControlI out = append(out, endpointInfo{ Addr: addr, Epnt: epnt, + TLS: port == "443", }) } } diff --git a/internal/cmd/oohelperd/ipinfo_test.go b/internal/cmd/oohelperd/ipinfo_test.go index ca2c902..ab1a965 100644 --- a/internal/cmd/oohelperd/ipinfo_test.go +++ b/internal/cmd/oohelperd/ipinfo_test.go @@ -142,15 +142,19 @@ func Test_ipInfoToEndpoints(t *testing.T) { want: []endpointInfo{{ Addr: "8.8.4.4", Epnt: "8.8.4.4:443", + TLS: true, }, { Addr: "8.8.4.4", Epnt: "8.8.4.4:80", + TLS: false, }, { Addr: "8.8.8.8", Epnt: "8.8.8.8:443", + TLS: true, }, { Addr: "8.8.8.8", Epnt: "8.8.8.8:80", + TLS: false, }}, }, { name: "with bogons and explicit port", @@ -176,9 +180,11 @@ func Test_ipInfoToEndpoints(t *testing.T) { want: []endpointInfo{{ Addr: "8.8.4.4", Epnt: "8.8.4.4:5432", + TLS: false, }, { Addr: "8.8.8.8", Epnt: "8.8.8.8:5432", + TLS: false, }}, }, { name: "with addresses and some bogons, no port, and unknown scheme", @@ -224,9 +230,11 @@ func Test_ipInfoToEndpoints(t *testing.T) { want: []endpointInfo{{ Addr: "8.8.4.4", Epnt: "8.8.4.4:443", + TLS: true, }, { Addr: "8.8.8.8", Epnt: "8.8.8.8:443", + TLS: true, }}, }} for _, tt := range tests { diff --git a/internal/cmd/oohelperd/main.go b/internal/cmd/oohelperd/main.go index 1bee0f6..b7904d0 100644 --- a/internal/cmd/oohelperd/main.go +++ b/internal/cmd/oohelperd/main.go @@ -62,6 +62,9 @@ func main() { return netxlite.NewDialerWithResolver(log.Log, newResolver()) }, NewResolver: newResolver, + NewTLSHandshaker: func() model.TLSHandshaker { + return netxlite.NewTLSHandshakerStdlib(log.Log) + }, }) srv := &http.Server{Addr: *endpoint, Handler: mux} listener, err := net.Listen("tcp", *endpoint) diff --git a/internal/cmd/oohelperd/measure.go b/internal/cmd/oohelperd/measure.go index 6b277cb..c65ae4b 100644 --- a/internal/cmd/oohelperd/measure.go +++ b/internal/cmd/oohelperd/measure.go @@ -47,7 +47,13 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp wg.Wait() // start assembling the response - cresp := &ctrlResponse{} + cresp := &ctrlResponse{ + TCPConnect: map[string]webconnectivity.ControlTCPConnectResult{}, + TLSHandshake: map[string]webconnectivity.ControlTLSHandshakeResult{}, + HTTPRequest: webconnectivity.ControlHTTPRequestResult{}, + DNS: webconnectivity.ControlDNSResult{}, + IPInfo: map[string]*webconnectivity.ControlIPInfo{}, + } select { case cresp.DNS = <-dnsch: default: @@ -56,6 +62,7 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp cresp.DNS = ctrlDNSResult{ Failure: nil, Addrs: []string{}, + ASNs: []int64{}, // unused by the TH and not serialized } } @@ -68,11 +75,14 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp for _, endpoint := range endpoints { wg.Add(1) go tcpDo(ctx, &tcpConfig{ - Address: endpoint.Addr, - Endpoint: endpoint.Epnt, - NewDialer: config.NewDialer, - Out: tcpconnch, - Wg: wg, + Address: endpoint.Addr, + EnableTLS: endpoint.TLS, + Endpoint: endpoint.Epnt, + NewDialer: config.NewDialer, + NewTSLHandshaker: config.NewTLSHandshaker, + URLHostname: URL.Hostname(), + Out: tcpconnch, + Wg: wg, }) } @@ -93,12 +103,17 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp // continue assembling the response cresp.HTTPRequest = <-httpch - cresp.TCPConnect = make(map[string]ctrlTCPResult) Loop: for { select { case tcpconn := <-tcpconnch: cresp.TCPConnect[tcpconn.Endpoint] = tcpconn.TCP + if tcpconn.TLS != nil { + cresp.TLSHandshake[tcpconn.Endpoint] = *tcpconn.TLS + if info := cresp.IPInfo[tcpconn.Address]; info != nil && tcpconn.TLS.Failure == nil { + info.Flags |= webconnectivity.ControlIPInfoFlagValidForDomain + } + } default: break Loop } diff --git a/internal/cmd/oohelperd/tcpconnect.go b/internal/cmd/oohelperd/tcpconnect.go index a0e5adb..fa20e29 100644 --- a/internal/cmd/oohelperd/tcpconnect.go +++ b/internal/cmd/oohelperd/tcpconnect.go @@ -1,11 +1,12 @@ package main // -// TCP connect measurements +// TCP connect (and optionally TLS handshake) measurements // import ( "context" + "crypto/tls" "sync" "time" @@ -18,6 +19,9 @@ import ( // ctrlTCPResult is the result of the TCP check performed by the test helper. type ctrlTCPResult = webconnectivity.ControlTCPConnectResult +// ctrlTLSResult is the result of the TLS check performed by the test helper. +type ctrlTLSResult = webconnectivity.ControlTLSHandshakeResult + // tcpResultPair contains the endpoint and the corresponding result. type tcpResultPair struct { // Address is the IP address we measured. @@ -28,6 +32,9 @@ type tcpResultPair struct { // TCP contains the TCP results. TCP ctrlTCPResult + + // TLS contains the TLS results + TLS *ctrlTLSResult } // tcpConfig configures the TCP connect check. @@ -35,22 +42,31 @@ type tcpConfig struct { // Address is the MANDATORY address to measure. Address string + // EnableTLS OPTIONALLY enables TLS. + EnableTLS bool + // Endpoint is the MANDATORY endpoint to connect to. Endpoint string // NewDialer is the MANDATORY factory for creating a new dialer. NewDialer func() model.Dialer + // NewTSLHandshaker is the MANDATORY factory for creating a new handshaker. + NewTSLHandshaker func() model.TLSHandshaker + // Out is the MANDATORY where we'll post the TCP measurement results. Out chan *tcpResultPair + // URLHostname is the MANDATORY URL.Hostname() to use. + URLHostname string + // Wg is MANDATORY and is used to sync with the parent. Wg *sync.WaitGroup } // tcpDo performs the TCP check. func tcpDo(ctx context.Context, config *tcpConfig) { - const timeout = 10 * time.Second + const timeout = 15 * time.Second ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() defer config.Wg.Done() @@ -58,6 +74,7 @@ func tcpDo(ctx context.Context, config *tcpConfig) { Address: config.Address, Endpoint: config.Endpoint, TCP: webconnectivity.ControlTCPConnectResult{}, + TLS: nil, // means: not measured } defer func() { config.Out <- out @@ -67,7 +84,23 @@ func tcpDo(ctx context.Context, config *tcpConfig) { conn, err := dialer.DialContext(ctx, "tcp", config.Endpoint) out.TCP.Failure = tcpMapFailure(newfailure(err)) out.TCP.Status = err == nil - measurexlite.MaybeClose(conn) + defer measurexlite.MaybeClose(conn) + if err != nil || !config.EnableTLS { + return + } + tlsConfig := &tls.Config{ + NextProtos: []string{"h2", "http/1.1"}, + RootCAs: netxlite.NewDefaultCertPool(), + ServerName: config.URLHostname, + } + thx := config.NewTSLHandshaker() + tlsConn, _, err := thx.Handshake(ctx, conn, tlsConfig) + out.TLS = &ctrlTLSResult{ + ServerName: config.URLHostname, + Status: err == nil, + Failure: newfailure(err), + } + measurexlite.MaybeClose(tlsConn) } // tcpMapFailure attempts to map netxlite failures to the strings diff --git a/internal/engine/experiment/webconnectivity/control.go b/internal/engine/experiment/webconnectivity/control.go index 91089ea..90b5bdb 100644 --- a/internal/engine/experiment/webconnectivity/control.go +++ b/internal/engine/experiment/webconnectivity/control.go @@ -9,6 +9,9 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) +// TODO(bassosimone): these struct definitions should be moved outside the +// specific implementation of Web Connectivity v0.4. + // ControlRequest is the request that we send to the control type ControlRequest struct { HTTPRequest string `json:"http_request"` @@ -23,6 +26,14 @@ type ControlTCPConnectResult struct { Failure *string `json:"failure"` } +// ControlTLSHandshakeResult is the result of the TLS handshake +// attempt performed by the control vantage point. +type ControlTLSHandshakeResult struct { + ServerName string `json:"server_name"` + Status bool `json:"status"` + Failure *string `json:"failure"` +} + // ControlHTTPRequestResult is the result of the HTTP request // performed by the control vantage point. type ControlHTTPRequestResult struct { @@ -67,14 +78,19 @@ const ( // ControlIPInfoFlagIsBogon indicates that the address is a bogon ControlIPInfoFlagIsBogon + + // ControlIPInfoFlagValidForDomain indicates that an IP address + // is valid for the domain because it works with TLS + ControlIPInfoFlagValidForDomain ) // ControlResponse is the response from the control service. type ControlResponse struct { - TCPConnect map[string]ControlTCPConnectResult `json:"tcp_connect"` - HTTPRequest ControlHTTPRequestResult `json:"http_request"` - DNS ControlDNSResult `json:"dns"` - IPInfo map[string]*ControlIPInfo `json:"ip_info"` + TCPConnect map[string]ControlTCPConnectResult `json:"tcp_connect"` + TLSHandshake map[string]ControlTLSHandshakeResult `json:"tls_handshake"` + HTTPRequest ControlHTTPRequestResult `json:"http_request"` + DNS ControlDNSResult `json:"dns"` + IPInfo map[string]*ControlIPInfo `json:"ip_info"` } // Control performs the control request and returns the response.