enable utls for websteps (#442)

This diff enables `websteps` to use uTLS for TLS parroting. It integrates the `oohttp.StdlibTransport` wrapper which uses the `ooni/oohttp` fork. `oohttp` supports TLS-like connections like `utls.Conn`.
As a prototype, the testhelper and `websteps` code now uses the `utls.HelloChrome_Auto` fingerprint, i.e. the simulated TLS fingerprint of the Google Chrome browser.

It is a further contribution for my GSoC project.

Reference issue: https://github.com/ooni/probe/issues/1733

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/miekg/dns v1.1.42
 	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/montanaflynn/stats v0.6.6
+	github.com/ooni/oohttp v0.0.0-20210818104219-f8ceac6f2622
 	github.com/ooni/probe-assets v0.3.1
 	github.com/ooni/psiphon v0.8.0
 	github.com/oschwald/geoip2-golang v1.5.0
@@ -41,7 +42,7 @@ require (
 	gitlab.com/yawning/obfs4.git v0.0.0-20210511220700-e330d1b7024b
 	gitlab.com/yawning/utls.git v0.0.12-1
 	golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect
-	golang.org/x/net v0.0.0-20210510120150-4163338589ed
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d
 	golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2
 	gopkg.in/AlecAivazis/survey.v1 v1.8.8
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect

go.sum

@@ -385,6 +385,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/ooni/oohttp v0.0.0-20210818104219-f8ceac6f2622 h1:Wpu4o3J3fLD4BPqA3CmrnbXVAAWKEjramvfhDUFZp+E=
+github.com/ooni/oohttp v0.0.0-20210818104219-f8ceac6f2622/go.mod h1:kgtoj+Dn4bmx09hEUgbPI7YX0gkWlu+fz2I0S5auyX4=
 github.com/ooni/probe-assets v0.3.1 h1:6PDcoJTICJxL8PdeM0+a3ZfkTWrFfCn90fUqTWR0LDA=
 github.com/ooni/probe-assets v0.3.1/go.mod h1:N0PyNM3aadlYDDCFXAPzs54HC54+MZA/4/xnCtd9EAo=
 github.com/ooni/psiphon v0.8.0 h1:digldztBlINi3HWuxdK4gFhkiaheAoDVjZN/ApZHWBM=
@@ -693,8 +695,8 @@ golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
-golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

internal/cmd/oohelperd/internal/websteps/explore.go

@@ -8,8 +8,10 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
 	"github.com/ooni/probe-cli/v3/internal/runtimex"
+	utls "gitlab.com/yawning/utls.git"
 )
 
 // Explore is the second step of the test helper algorithm. Its objective
@@ -104,7 +106,10 @@ func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http.
 	tlsConf := &tls.Config{
 		NextProtos: []string{"h2", "http/1.1"},
 	}
-	transport := netxlite.NewHTTPTransport(NewDialerResolver(e.resolver), tlsConf, &netxlite.TLSHandshakerConfigurable{})
+	handshaker := &netxlite.TLSHandshakerConfigurable{
+		NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto),
+	}
+	transport := websteps.NewTransportWithDialer(websteps.NewDialerResolver(e.resolver), tlsConf, handshaker)
 	// TODO(bassosimone): here we should use runtimex.PanicOnError
 	jarjar, _ := cookiejar.New(nil)
 	clnt := &http.Client{
@@ -126,7 +131,7 @@ func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http.
 // getH3 uses HTTP/3 to get the given URL and returns the final
 // response after redirection, and an error. If the error is nil, the final response is valid.
 func (e *DefaultExplorer) getH3(h3URL *h3URL, headers map[string][]string) (*http.Response, error) {
-	dialer := NewQUICDialerResolver(e.resolver)
+	dialer := websteps.NewQUICDialerResolver(e.resolver)
 	tlsConf := &tls.Config{
 		NextProtos: []string{h3URL.proto},
 	}

internal/cmd/oohelperd/internal/websteps/factory.go

@@ -1,96 +0,0 @@
-package websteps
-
-import (
-	"context"
-	"crypto/tls"
-	"errors"
-	"net"
-	"net/http"
-	"sync"
-
-	"github.com/apex/log"
-	"github.com/lucas-clemente/quic-go"
-	"github.com/lucas-clemente/quic-go/http3"
-	"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
-	"github.com/ooni/probe-cli/v3/internal/errorsx"
-	"github.com/ooni/probe-cli/v3/internal/netxlite"
-)
-
-var ErrNoConnReuse = errors.New("cannot reuse connection")
-
-// NewDialerResolver contructs a new dialer for TCP connections,
-// with default, errorwrapping and resolve functionalities
-func NewDialerResolver(resolver netxlite.Resolver) netxlite.Dialer {
-	var d netxlite.Dialer = netxlite.DefaultDialer
-	d = &errorsx.ErrorWrapperDialer{Dialer: d}
-	d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d}
-	return d
-}
-
-// NewQUICDialerResolver creates a new QUICDialerResolver
-// with default, errorwrapping and resolve functionalities
-func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDialer {
-	var ql quicdialer.QUICListener = &netxlite.QUICListenerStdlib{}
-	ql = &errorsx.ErrorWrapperQUICListener{QUICListener: ql}
-	var dialer netxlite.QUICContextDialer = &netxlite.QUICDialerQUICGo{
-		QUICListener: ql,
-	}
-	dialer = &errorsx.ErrorWrapperQUICDialer{Dialer: dialer}
-	dialer = &netxlite.QUICDialerResolver{Resolver: resolver, Dialer: dialer}
-	return dialer
-}
-
-// NewSingleH3Transport creates an http3.RoundTripper
-func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper {
-	transport := &http3.RoundTripper{
-		DisableCompression: true,
-		TLSClientConfig:    tlscfg,
-		QuicConfig:         qcfg,
-		Dial:               (&SingleDialerH3{qsess: &qsess}).Dial,
-	}
-	return transport
-}
-
-// NewSingleTransport determines the appropriate HTTP Transport from the ALPN
-func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) {
-	singledialer := &SingleDialer{conn: &conn}
-	transport = http.DefaultTransport.(*http.Transport).Clone()
-	transport.(*http.Transport).DialContext = singledialer.DialContext
-	transport.(*http.Transport).DialTLSContext = singledialer.DialContext
-	transport.(*http.Transport).DisableCompression = true
-	transport.(*http.Transport).MaxConnsPerHost = 1
-	transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)}
-	return transport
-}
-
-type SingleDialer struct {
-	sync.Mutex
-	conn *net.Conn
-}
-
-func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) {
-	s.Lock()
-	defer s.Unlock()
-	if s.conn == nil {
-		return nil, ErrNoConnReuse
-	}
-	c := s.conn
-	s.conn = nil
-	return *c, nil
-}
-
-type SingleDialerH3 struct {
-	sync.Mutex
-	qsess *quic.EarlySession
-}
-
-func (s *SingleDialerH3) Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
-	s.Lock()
-	defer s.Unlock()
-	if s.qsess == nil {
-		return nil, ErrNoConnReuse
-	}
-	qs := s.qsess
-	s.qsess = nil
-	return *qs, nil
-}

internal/cmd/oohelperd/internal/websteps/generate.go

@@ -130,7 +130,7 @@ func (g *DefaultGenerator) GenerateHTTPEndpoint(ctx context.Context, rt *RoundTr
 			URL:     rt.Request.URL.String(),
 		},
 	}
-	transport := NewSingleTransport(tcpConn)
+	transport := websteps.NewSingleTransport(tcpConn)
 	if g.transport != nil {
 		transport = g.transport
 	}
@@ -176,7 +176,7 @@ func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundT
 	}
 	defer tcpConn.Close()
 
-	tlsConn, err = TLSDo(tcpConn, rt.Request.URL.Hostname())
+	tlsConn, err = TLSDo(ctx, tcpConn, rt.Request.URL.Hostname())
 	currentEndpoint.TLSHandshake = &TLSHandshakeMeasurement{
 		Failure: newfailure(err),
 	}
@@ -193,7 +193,7 @@ func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundT
 			URL:     rt.Request.URL.String(),
 		},
 	}
-	transport := NewSingleTransport(tlsConn)
+	transport := websteps.NewSingleTransport(tlsConn)
 	if g.transport != nil {
 		transport = g.transport
 	}
@@ -248,7 +248,7 @@ func (g *DefaultGenerator) GenerateH3Endpoint(ctx context.Context, rt *RoundTrip
 			URL:     rt.Request.URL.String(),
 		},
 	}
-	var transport http.RoundTripper = NewSingleH3Transport(sess, tlsConf, &quic.Config{})
+	var transport http.RoundTripper = websteps.NewSingleH3Transport(sess, tlsConf, &quic.Config{})
 	if g.transport != nil {
 		transport = g.transport
 	}

internal/cmd/oohelperd/internal/websteps/generate_test.go

@@ -252,7 +252,10 @@ func TestGenerateHTTPS(t *testing.T) {
 		t.Fatal("TCPConnectMeasurement should not be nil")
 	}
 	if endpointMeasurement.TLSHandshake == nil {
-		t.Fatal("TCPConnectMeasurement should not be nil")
+		t.Fatal("TLSHandshakeMeasurement should not be nil")
+	}
+	if endpointMeasurement.TLSHandshake.Failure != nil {
+		t.Fatal("unexpected failure at TLSHandshakeMeasurement")
 	}
 	if endpointMeasurement.HTTPRoundTrip == nil {
 		t.Fatal("HTTPRoundTripMeasurement should not be nil")
@@ -283,7 +286,10 @@ func TestGenerateHTTPSTLSFailure(t *testing.T) {
 		t.Fatal("TCPConnectMeasurement should not be nil")
 	}
 	if endpointMeasurement.TLSHandshake == nil {
-		t.Fatal("TCPConnectMeasurement should not be nil")
+		t.Fatal("TLSHandshakeMeasurement should not be nil")
+	}
+	if endpointMeasurement.TLSHandshake.Failure == nil {
+		t.Fatal("expected failure at TLSHandshakeMeasurement")
 	}
 	if endpointMeasurement.HTTPRoundTrip != nil {
 		t.Fatal("HTTPRoundTripMeasurement should be nil")

internal/engine/experiment/websteps/factory.go

@@ -9,9 +9,9 @@ import (
 	"net/url"
 	"sync"
 
-	"github.com/apex/log"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/lucas-clemente/quic-go/http3"
+	oohttp "github.com/ooni/oohttp"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
 	"github.com/ooni/probe-cli/v3/internal/errorsx"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
@@ -53,8 +53,8 @@ func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDiale
 	return dialer
 }
 
-// NewSingleH3Transport creates an http3.RoundTripper
+// NewSingleH3Transport creates an http3.RoundTripper.
-func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper {
+func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) http.RoundTripper {
 	transport := &http3.RoundTripper{
 		DisableCompression: true,
 		TLSClientConfig:    tlscfg,
@@ -64,16 +64,33 @@ func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *qui
 	return transport
 }
 
-// NewSingleTransport determines the appropriate HTTP Transport from the ALPN
+// NewSingleTransport creates a new HTTP transport with a single-use dialer.
-func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) {
+func NewSingleTransport(conn net.Conn) http.RoundTripper {
 	singledialer := &SingleDialer{conn: &conn}
-	transport = http.DefaultTransport.(*http.Transport).Clone()
+	transport := newBaseTransport()
-	transport.(*http.Transport).DialContext = singledialer.DialContext
+	transport.DialContext = singledialer.DialContext
-	transport.(*http.Transport).DialTLSContext = singledialer.DialContext
+	transport.DialTLSContext = singledialer.DialContext
-	transport.(*http.Transport).DisableCompression = true
+	return transport
-	transport.(*http.Transport).MaxConnsPerHost = 1
+}
 
-	transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)}
+// NewSingleTransport creates a new HTTP transport with a custom dialer and handshaker.
+func NewTransportWithDialer(dialer netxlite.Dialer, tlsConfig *tls.Config, handshaker netxlite.TLSHandshaker) http.RoundTripper {
+	transport := newBaseTransport()
+	transport.DialContext = dialer.DialContext
+	transport.DialTLSContext = (&netxlite.TLSDialer{
+		Config:        tlsConfig,
+		Dialer:        dialer,
+		TLSHandshaker: handshaker,
+	}).DialTLSContext
+	return transport
+}
+
+// newBaseTransport creates a new HTTP transport with the default dialer.
+func newBaseTransport() (transport *oohttp.StdlibTransport) {
+	base := oohttp.DefaultTransport.(*oohttp.Transport).Clone()
+	base.DisableCompression = true
+	base.MaxConnsPerHost = 1
+	transport = &oohttp.StdlibTransport{Transport: base}
 	return transport
 }
 

internal/engine/experiment/websteps/tls.go

@@ -1,16 +1,23 @@
 package websteps
 
 import (
+	"context"
 	"crypto/tls"
 	"net"
+
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	utls "gitlab.com/yawning/utls.git"
 )
 
 // TLSDo performs the TLS check.
-func TLSDo(conn net.Conn, hostname string) (*tls.Conn, error) {
+func TLSDo(ctx context.Context, conn net.Conn, hostname string) (net.Conn, error) {
-	tlsConn := tls.Client(conn, &tls.Config{
+	tlsConf := &tls.Config{
 		ServerName: hostname,
 		NextProtos: []string{"h2", "http/1.1"},
-	})
+	}
-	err := tlsConn.Handshake()
+	h := &netxlite.TLSHandshakerConfigurable{
+		NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto),
+	}
+	tlsConn, _, err := h.Handshake(ctx, conn, tlsConf)
 	return tlsConn, err
 }

internal/engine/experiment/websteps/websteps.go

@@ -241,7 +241,7 @@ func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpo
 	defer conn.Close()
 
 	// TLS handshake step
-	tlsconn, err := TLSDo(conn, URL.Hostname())
+	tlsconn, err := TLSDo(ctx, conn, URL.Hostname())
 	endpointMeasurement.TLSHandshake = &TLSHandshakeMeasurement{
 		Failure: archival.NewFailure(err),
 	}

internal/netxlite/tlshandshaker.go

@@ -135,6 +135,7 @@ func (c *UTLSConn) ConnectionState() tls.ConnectionState {
 		DidResume:                   uState.DidResume,
 		CipherSuite:                 uState.CipherSuite,
 		NegotiatedProtocol:          uState.NegotiatedProtocol,
+		NegotiatedProtocolIsMutual:  true,
 		ServerName:                  uState.ServerName,
 		PeerCertificates:            uState.PeerCertificates,
 		VerifiedChains:              uState.VerifiedChains,