feat(oohelperd): measure TLS for :443 endpoints (#886)

This diff improves oohelperd to measure :443 endpoints with TLS.

Part of https://github.com/ooni/probe/issues/2237.

internal/cmd/oohelperd/dns.go

@@ -50,7 +50,11 @@ func dnsDo(ctx context.Context, config *dnsConfig) {
 		addrs = []string{} // fix: the old test helper did that
 	}
 	failure := dnsMapFailure(newfailure(err))
-	config.Out <- ctrlDNSResult{Failure: failure, Addrs: addrs}
+	config.Out <- ctrlDNSResult{
+		Failure: failure,
+		Addrs:   addrs,
+		ASNs:    []int64{}, // unused by the TH and not serialized
+	}
 }
 
 // dnsMapFailure attempts to map netxlite failures to the strings

internal/cmd/oohelperd/handler.go

@@ -29,6 +29,9 @@ type handler struct {
 
 	// NewResolver is the MANDATORY factory for creating a new resolver.
 	NewResolver func() model.Resolver
+
+	// NewTLSHandshaker is the MANDATORY factory for creating a new TLS handshaker.
+	NewTLSHandshaker func() model.TLSHandshaker
 }
 
 var _ http.Handler = &handler{}

internal/cmd/oohelperd/handler_test.go

@@ -63,6 +63,9 @@ func TestHandlerWorkingAsIntended(t *testing.T) {
 		NewResolver: func() model.Resolver {
 			return netxlite.NewUnwrappedStdlibResolver()
 		},
+		NewTLSHandshaker: func() model.TLSHandshaker {
+			return netxlite.NewTLSHandshakerStdlib(model.DiscardLogger)
+		},
 	}
 	srv := httptest.NewServer(handler)
 	defer srv.Close()
@@ -76,25 +79,37 @@ func TestHandlerWorkingAsIntended(t *testing.T) {
 		parseBody       bool
 	}
 	expectations := []expectationSpec{{
-		name:           "check for invalid method",
+		name:            "check for invalid method",
-		reqMethod:      "GET",
+		reqMethod:       "GET",
-		respStatusCode: 400,
+		reqContentType:  "",
+		reqBody:         "",
+		respStatusCode:  400,
+		respContentType: "",
+		parseBody:       false,
 	}, {
-		name:           "check for invalid content-type",
+		name:            "check for invalid content-type",
-		reqMethod:      "POST",
+		reqMethod:       "POST",
-		respStatusCode: 400,
+		reqContentType:  "",
+		reqBody:         "",
+		respStatusCode:  400,
+		respContentType: "",
+		parseBody:       false,
 	}, {
-		name:           "check for invalid request body",
+		name:            "check for invalid request body",
-		reqMethod:      "POST",
+		reqMethod:       "POST",
-		reqContentType: "application/json",
+		reqContentType:  "application/json",
-		reqBody:        "{",
+		reqBody:         "{",
-		respStatusCode: 400,
+		respStatusCode:  400,
+		respContentType: "",
+		parseBody:       false,
 	}, {
-		name:           "with measurement failure",
+		name:            "with measurement failure",
-		reqMethod:      "POST",
+		reqMethod:       "POST",
-		reqContentType: "application/json",
+		reqContentType:  "application/json",
-		reqBody:        `{"http_request": "http://[::1]aaaa"}`,
+		reqBody:         `{"http_request": "http://[::1]aaaa"}`,
-		respStatusCode: 400,
+		respStatusCode:  400,
+		respContentType: "",
+		parseBody:       false,
 	}, {
 		name:            "with reasonably good request",
 		reqMethod:       "POST",

internal/cmd/oohelperd/http.go

@@ -18,6 +18,9 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/tracex"
 )
 
+// TODO(bassosimone): we should refactor the TH to use step-by-step such that we
+// can use an existing connection for the HTTP-measuring task
+
 // ctrlHTTPResponse is the result of the HTTP check performed by
 // the Web Connectivity test helper.
 type ctrlHTTPResponse = webconnectivity.ControlHTTPRequestResult
@@ -51,11 +54,13 @@ func httpDo(ctx context.Context, config *httpConfig) {
 	defer config.Wg.Done()
 	req, err := http.NewRequestWithContext(ctx, "GET", config.URL, nil)
 	if err != nil {
-		config.Out <- ctrlHTTPResponse{ // fix: emit -1 like the old test helper does
+		// fix: emit -1 like the old test helper does
+		config.Out <- ctrlHTTPResponse{
 			BodyLength: -1,
 			Failure:    httpMapFailure(err),
-			StatusCode: -1,
+			Title:      "",
 			Headers:    map[string]string{},
+			StatusCode: -1,
 		}
 		return
 	}
@@ -73,11 +78,13 @@ func httpDo(ctx context.Context, config *httpConfig) {
 	defer clnt.CloseIdleConnections()
 	resp, err := clnt.Do(req)
 	if err != nil {
-		config.Out <- ctrlHTTPResponse{ // fix: emit -1 like old test helper does
+		// fix: emit -1 like the old test helper does
+		config.Out <- ctrlHTTPResponse{
 			BodyLength: -1,
 			Failure:    httpMapFailure(err),
-			StatusCode: -1,
+			Title:      "",
 			Headers:    map[string]string{},
+			StatusCode: -1,
 		}
 		return
 	}

internal/cmd/oohelperd/ipinfo.go

@@ -52,6 +52,9 @@ type endpointInfo struct {
 
 	// Epnt is the endpoint to measure
 	Epnt string
+
+	// TLS indicates whether we should try using TLS
+	TLS bool
 }
 
 // ipInfoToEndpoints takes in input the [ipinfo] returned by newIPInfo
@@ -86,6 +89,7 @@ func ipInfoToEndpoints(URL *url.URL, ipinfo map[string]*webconnectivity.ControlI
 			out = append(out, endpointInfo{
 				Addr: addr,
 				Epnt: epnt,
+				TLS:  port == "443",
 			})
 		}
 	}

internal/cmd/oohelperd/ipinfo_test.go

@@ -142,15 +142,19 @@ func Test_ipInfoToEndpoints(t *testing.T) {
 		want: []endpointInfo{{
 			Addr: "8.8.4.4",
 			Epnt: "8.8.4.4:443",
+			TLS:  true,
 		}, {
 			Addr: "8.8.4.4",
 			Epnt: "8.8.4.4:80",
+			TLS:  false,
 		}, {
 			Addr: "8.8.8.8",
 			Epnt: "8.8.8.8:443",
+			TLS:  true,
 		}, {
 			Addr: "8.8.8.8",
 			Epnt: "8.8.8.8:80",
+			TLS:  false,
 		}},
 	}, {
 		name: "with bogons and explicit port",
@@ -176,9 +180,11 @@ func Test_ipInfoToEndpoints(t *testing.T) {
 		want: []endpointInfo{{
 			Addr: "8.8.4.4",
 			Epnt: "8.8.4.4:5432",
+			TLS:  false,
 		}, {
 			Addr: "8.8.8.8",
 			Epnt: "8.8.8.8:5432",
+			TLS:  false,
 		}},
 	}, {
 		name: "with addresses and some bogons, no port, and unknown scheme",
@@ -224,9 +230,11 @@ func Test_ipInfoToEndpoints(t *testing.T) {
 		want: []endpointInfo{{
 			Addr: "8.8.4.4",
 			Epnt: "8.8.4.4:443",
+			TLS:  true,
 		}, {
 			Addr: "8.8.8.8",
 			Epnt: "8.8.8.8:443",
+			TLS:  true,
 		}},
 	}}
 	for _, tt := range tests {

internal/cmd/oohelperd/main.go

@@ -62,6 +62,9 @@ func main() {
 			return netxlite.NewDialerWithResolver(log.Log, newResolver())
 		},
 		NewResolver: newResolver,
+		NewTLSHandshaker: func() model.TLSHandshaker {
+			return netxlite.NewTLSHandshakerStdlib(log.Log)
+		},
 	})
 	srv := &http.Server{Addr: *endpoint, Handler: mux}
 	listener, err := net.Listen("tcp", *endpoint)

internal/cmd/oohelperd/measure.go

@@ -47,7 +47,13 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp
 	wg.Wait()
 
 	// start assembling the response
-	cresp := &ctrlResponse{}
+	cresp := &ctrlResponse{
+		TCPConnect:   map[string]webconnectivity.ControlTCPConnectResult{},
+		TLSHandshake: map[string]webconnectivity.ControlTLSHandshakeResult{},
+		HTTPRequest:  webconnectivity.ControlHTTPRequestResult{},
+		DNS:          webconnectivity.ControlDNSResult{},
+		IPInfo:       map[string]*webconnectivity.ControlIPInfo{},
+	}
 	select {
 	case cresp.DNS = <-dnsch:
 	default:
@@ -56,6 +62,7 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp
 		cresp.DNS = ctrlDNSResult{
 			Failure: nil,
 			Addrs:   []string{},
+			ASNs:    []int64{}, // unused by the TH and not serialized
 		}
 	}
 
@@ -68,11 +75,14 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp
 	for _, endpoint := range endpoints {
 		wg.Add(1)
 		go tcpDo(ctx, &tcpConfig{
-			Address:   endpoint.Addr,
+			Address:          endpoint.Addr,
-			Endpoint:  endpoint.Epnt,
+			EnableTLS:        endpoint.TLS,
-			NewDialer: config.NewDialer,
+			Endpoint:         endpoint.Epnt,
-			Out:       tcpconnch,
+			NewDialer:        config.NewDialer,
-			Wg:        wg,
+			NewTSLHandshaker: config.NewTLSHandshaker,
+			URLHostname:      URL.Hostname(),
+			Out:              tcpconnch,
+			Wg:               wg,
 		})
 	}
 
@@ -93,12 +103,17 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp
 
 	// continue assembling the response
 	cresp.HTTPRequest = <-httpch
-	cresp.TCPConnect = make(map[string]ctrlTCPResult)
 Loop:
 	for {
 		select {
 		case tcpconn := <-tcpconnch:
 			cresp.TCPConnect[tcpconn.Endpoint] = tcpconn.TCP
+			if tcpconn.TLS != nil {
+				cresp.TLSHandshake[tcpconn.Endpoint] = *tcpconn.TLS
+				if info := cresp.IPInfo[tcpconn.Address]; info != nil && tcpconn.TLS.Failure == nil {
+					info.Flags |= webconnectivity.ControlIPInfoFlagValidForDomain
+				}
+			}
 		default:
 			break Loop
 		}

internal/cmd/oohelperd/tcpconnect.go

@@ -1,11 +1,12 @@
 package main
 
 //
-// TCP connect measurements
+// TCP connect (and optionally TLS handshake) measurements
 //
 
 import (
 	"context"
+	"crypto/tls"
 	"sync"
 	"time"
 
@@ -18,6 +19,9 @@ import (
 // ctrlTCPResult is the result of the TCP check performed by the test helper.
 type ctrlTCPResult = webconnectivity.ControlTCPConnectResult
 
+// ctrlTLSResult is the result of the TLS check performed by the test helper.
+type ctrlTLSResult = webconnectivity.ControlTLSHandshakeResult
+
 // tcpResultPair contains the endpoint and the corresponding result.
 type tcpResultPair struct {
 	// Address is the IP address we measured.
@@ -28,6 +32,9 @@ type tcpResultPair struct {
 
 	// TCP contains the TCP results.
 	TCP ctrlTCPResult
+
+	// TLS contains the TLS results
+	TLS *ctrlTLSResult
 }
 
 // tcpConfig configures the TCP connect check.
@@ -35,22 +42,31 @@ type tcpConfig struct {
 	// Address is the MANDATORY address to measure.
 	Address string
 
+	// EnableTLS OPTIONALLY enables TLS.
+	EnableTLS bool
+
 	// Endpoint is the MANDATORY endpoint to connect to.
 	Endpoint string
 
 	// NewDialer is the MANDATORY factory for creating a new dialer.
 	NewDialer func() model.Dialer
 
+	// NewTSLHandshaker is the MANDATORY factory for creating a new handshaker.
+	NewTSLHandshaker func() model.TLSHandshaker
+
 	// Out is the MANDATORY where we'll post the TCP measurement results.
 	Out chan *tcpResultPair
 
+	// URLHostname is the MANDATORY URL.Hostname() to use.
+	URLHostname string
+
 	// Wg is MANDATORY and is used to sync with the parent.
 	Wg *sync.WaitGroup
 }
 
 // tcpDo performs the TCP check.
 func tcpDo(ctx context.Context, config *tcpConfig) {
-	const timeout = 10 * time.Second
+	const timeout = 15 * time.Second
 	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 	defer config.Wg.Done()
@@ -58,6 +74,7 @@ func tcpDo(ctx context.Context, config *tcpConfig) {
 		Address:  config.Address,
 		Endpoint: config.Endpoint,
 		TCP:      webconnectivity.ControlTCPConnectResult{},
+		TLS:      nil, // means: not measured
 	}
 	defer func() {
 		config.Out <- out
@@ -67,7 +84,23 @@ func tcpDo(ctx context.Context, config *tcpConfig) {
 	conn, err := dialer.DialContext(ctx, "tcp", config.Endpoint)
 	out.TCP.Failure = tcpMapFailure(newfailure(err))
 	out.TCP.Status = err == nil
-	measurexlite.MaybeClose(conn)
+	defer measurexlite.MaybeClose(conn)
+	if err != nil || !config.EnableTLS {
+		return
+	}
+	tlsConfig := &tls.Config{
+		NextProtos: []string{"h2", "http/1.1"},
+		RootCAs:    netxlite.NewDefaultCertPool(),
+		ServerName: config.URLHostname,
+	}
+	thx := config.NewTSLHandshaker()
+	tlsConn, _, err := thx.Handshake(ctx, conn, tlsConfig)
+	out.TLS = &ctrlTLSResult{
+		ServerName: config.URLHostname,
+		Status:     err == nil,
+		Failure:    newfailure(err),
+	}
+	measurexlite.MaybeClose(tlsConn)
 }
 
 // tcpMapFailure attempts to map netxlite failures to the strings

internal/engine/experiment/webconnectivity/control.go

@@ -9,6 +9,9 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
 )
 
+// TODO(bassosimone): these struct definitions should be moved outside the
+// specific implementation of Web Connectivity v0.4.
+
 // ControlRequest is the request that we send to the control
 type ControlRequest struct {
 	HTTPRequest        string              `json:"http_request"`
@@ -23,6 +26,14 @@ type ControlTCPConnectResult struct {
 	Failure *string `json:"failure"`
 }
 
+// ControlTLSHandshakeResult is the result of the TLS handshake
+// attempt performed by the control vantage point.
+type ControlTLSHandshakeResult struct {
+	ServerName string  `json:"server_name"`
+	Status     bool    `json:"status"`
+	Failure    *string `json:"failure"`
+}
+
 // ControlHTTPRequestResult is the result of the HTTP request
 // performed by the control vantage point.
 type ControlHTTPRequestResult struct {
@@ -67,14 +78,19 @@ const (
 
 	// ControlIPInfoFlagIsBogon indicates that the address is a bogon
 	ControlIPInfoFlagIsBogon
+
+	// ControlIPInfoFlagValidForDomain indicates that an IP address
+	// is valid for the domain because it works with TLS
+	ControlIPInfoFlagValidForDomain
 )
 
 // ControlResponse is the response from the control service.
 type ControlResponse struct {
-	TCPConnect  map[string]ControlTCPConnectResult `json:"tcp_connect"`
+	TCPConnect   map[string]ControlTCPConnectResult   `json:"tcp_connect"`
-	HTTPRequest ControlHTTPRequestResult           `json:"http_request"`
+	TLSHandshake map[string]ControlTLSHandshakeResult `json:"tls_handshake"`
-	DNS         ControlDNSResult                   `json:"dns"`
+	HTTPRequest  ControlHTTPRequestResult             `json:"http_request"`
-	IPInfo      map[string]*ControlIPInfo          `json:"ip_info"`
+	DNS          ControlDNSResult                     `json:"dns"`
+	IPInfo       map[string]*ControlIPInfo            `json:"ip_info"`
 }
 
 // Control performs the control request and returns the response.