c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

enable utls for websteps (#442)

Browse Source 
This diff enables `websteps` to use uTLS for TLS parroting. It integrates the `oohttp.StdlibTransport` wrapper which uses the `ooni/oohttp` fork. `oohttp` supports TLS-like connections like `utls.Conn`.
As a prototype, the testhelper and `websteps` code now uses the `utls.HelloChrome_Auto` fingerprint, i.e. the simulated TLS fingerprint of the Google Chrome browser.

It is a further contribution for my GSoC project.

Reference issue: https://github.com/ooni/probe/issues/1733
This commit is contained in:
kelmenhorst
2021-08-18 16:10:27 +02:00
committed by GitHub
parent 21a2b315fe
commit 1874f7a7c2
10 changed files with 66 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/cmd/oohelperd/internal/websteps/explore.go
+7 -2
View File
@@ -8,8 +8,10 @@ import (
"sort"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
utls "gitlab.com/yawning/utls.git"
)
// Explore is the second step of the test helper algorithm. Its objective
@@ -104,7 +106,10 @@ func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http.
tlsConf := &tls.Config{
NextProtos: []string{"h2", "http/1.1"},
}
transport := netxlite.NewHTTPTransport(NewDialerResolver(e.resolver), tlsConf, &netxlite.TLSHandshakerConfigurable{})
handshaker := &netxlite.TLSHandshakerConfigurable{
NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto),
}
transport := websteps.NewTransportWithDialer(websteps.NewDialerResolver(e.resolver), tlsConf, handshaker)
// TODO(bassosimone): here we should use runtimex.PanicOnError
jarjar, _ := cookiejar.New(nil)
clnt := &http.Client{
@@ -126,7 +131,7 @@ func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http.
// getH3 uses HTTP/3 to get the given URL and returns the final
// response after redirection, and an error. If the error is nil, the final response is valid.
func (e *DefaultExplorer) getH3(h3URL *h3URL, headers map[string][]string) (*http.Response, error) {
dialer := NewQUICDialerResolver(e.resolver)
dialer := websteps.NewQUICDialerResolver(e.resolver)
tlsConf := &tls.Config{
NextProtos: []string{h3URL.proto},
}
internal/cmd/oohelperd/internal/websteps/factory.go
-96
View File
@@ -1,96 +0,0 @@
package websteps
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"sync"
"github.com/apex/log"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
"github.com/ooni/probe-cli/v3/internal/errorsx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
var ErrNoConnReuse = errors.New("cannot reuse connection")
// NewDialerResolver contructs a new dialer for TCP connections,
// with default, errorwrapping and resolve functionalities
func NewDialerResolver(resolver netxlite.Resolver) netxlite.Dialer {
var d netxlite.Dialer = netxlite.DefaultDialer
d = &errorsx.ErrorWrapperDialer{Dialer: d}
d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d}
return d
}
// NewQUICDialerResolver creates a new QUICDialerResolver
// with default, errorwrapping and resolve functionalities
func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDialer {
var ql quicdialer.QUICListener = &netxlite.QUICListenerStdlib{}
ql = &errorsx.ErrorWrapperQUICListener{QUICListener: ql}
var dialer netxlite.QUICContextDialer = &netxlite.QUICDialerQUICGo{
QUICListener: ql,
}
dialer = &errorsx.ErrorWrapperQUICDialer{Dialer: dialer}
dialer = &netxlite.QUICDialerResolver{Resolver: resolver, Dialer: dialer}
return dialer
}
// NewSingleH3Transport creates an http3.RoundTripper
func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper {
transport := &http3.RoundTripper{
DisableCompression: true,
TLSClientConfig: tlscfg,
QuicConfig: qcfg,
Dial: (&SingleDialerH3{qsess: &qsess}).Dial,
}
return transport
}
// NewSingleTransport determines the appropriate HTTP Transport from the ALPN
func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) {
singledialer := &SingleDialer{conn: &conn}
transport = http.DefaultTransport.(*http.Transport).Clone()
transport.(*http.Transport).DialContext = singledialer.DialContext
transport.(*http.Transport).DialTLSContext = singledialer.DialContext
transport.(*http.Transport).DisableCompression = true
transport.(*http.Transport).MaxConnsPerHost = 1
transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)}
return transport
}
type SingleDialer struct {
sync.Mutex
conn *net.Conn
}
func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) {
s.Lock()
defer s.Unlock()
if s.conn == nil {
return nil, ErrNoConnReuse
}
c := s.conn
s.conn = nil
return *c, nil
}
type SingleDialerH3 struct {
sync.Mutex
qsess *quic.EarlySession
}
func (s *SingleDialerH3) Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) {
s.Lock()
defer s.Unlock()
if s.qsess == nil {
return nil, ErrNoConnReuse
}
qs := s.qsess
s.qsess = nil
return *qs, nil
}
internal/cmd/oohelperd/internal/websteps/generate.go
+4 -4
View File
@@ -130,7 +130,7 @@ func (g *DefaultGenerator) GenerateHTTPEndpoint(ctx context.Context, rt *RoundTr
URL: rt.Request.URL.String(),
},
}
transport := NewSingleTransport(tcpConn)
transport := websteps.NewSingleTransport(tcpConn)
if g.transport != nil {
transport = g.transport
}
@@ -176,7 +176,7 @@ func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundT
}
defer tcpConn.Close()
tlsConn, err = TLSDo(tcpConn, rt.Request.URL.Hostname())
tlsConn, err = TLSDo(ctx, tcpConn, rt.Request.URL.Hostname())
currentEndpoint.TLSHandshake = &TLSHandshakeMeasurement{
Failure: newfailure(err),
}
@@ -193,7 +193,7 @@ func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundT
URL: rt.Request.URL.String(),
},
}
transport := NewSingleTransport(tlsConn)
transport := websteps.NewSingleTransport(tlsConn)
if g.transport != nil {
transport = g.transport
}
@@ -248,7 +248,7 @@ func (g *DefaultGenerator) GenerateH3Endpoint(ctx context.Context, rt *RoundTrip
URL: rt.Request.URL.String(),
},
}
var transport http.RoundTripper = NewSingleH3Transport(sess, tlsConf, &quic.Config{})
var transport http.RoundTripper = websteps.NewSingleH3Transport(sess, tlsConf, &quic.Config{})
if g.transport != nil {
transport = g.transport
}
internal/cmd/oohelperd/internal/websteps/generate_test.go
+8 -2
View File
@@ -252,7 +252,10 @@ func TestGenerateHTTPS(t *testing.T) {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshake == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
t.Fatal("TLSHandshakeMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshake.Failure != nil {
t.Fatal("unexpected failure at TLSHandshakeMeasurement")
}
if endpointMeasurement.HTTPRoundTrip == nil {
t.Fatal("HTTPRoundTripMeasurement should not be nil")
@@ -283,7 +286,10 @@ func TestGenerateHTTPSTLSFailure(t *testing.T) {
t.Fatal("TCPConnectMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshake == nil {
t.Fatal("TCPConnectMeasurement should not be nil")
t.Fatal("TLSHandshakeMeasurement should not be nil")
}
if endpointMeasurement.TLSHandshake.Failure == nil {
t.Fatal("expected failure at TLSHandshakeMeasurement")
}
if endpointMeasurement.HTTPRoundTrip != nil {
t.Fatal("HTTPRoundTripMeasurement should be nil")
internal/engine/experiment/websteps/factory.go
+28 -11
View File
@@ -9,9 +9,9 @@ import (
"net/url"
"sync"
"github.com/apex/log"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
oohttp "github.com/ooni/oohttp"
"github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer"
"github.com/ooni/probe-cli/v3/internal/errorsx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
@@ -53,8 +53,8 @@ func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDiale
return dialer
}
// NewSingleH3Transport creates an http3.RoundTripper
func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper {
// NewSingleH3Transport creates an http3.RoundTripper.
func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) http.RoundTripper {
transport := &http3.RoundTripper{
DisableCompression: true,
TLSClientConfig: tlscfg,
@@ -64,16 +64,33 @@ func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *qui
return transport
}
// NewSingleTransport determines the appropriate HTTP Transport from the ALPN
func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) {
// NewSingleTransport creates a new HTTP transport with a single-use dialer.
func NewSingleTransport(conn net.Conn) http.RoundTripper {
singledialer := &SingleDialer{conn: &conn}
transport = http.DefaultTransport.(*http.Transport).Clone()
transport.(*http.Transport).DialContext = singledialer.DialContext
transport.(*http.Transport).DialTLSContext = singledialer.DialContext
transport.(*http.Transport).DisableCompression = true
transport.(*http.Transport).MaxConnsPerHost = 1
transport := newBaseTransport()
transport.DialContext = singledialer.DialContext
transport.DialTLSContext = singledialer.DialContext
return transport
}
transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)}
// NewSingleTransport creates a new HTTP transport with a custom dialer and handshaker.
func NewTransportWithDialer(dialer netxlite.Dialer, tlsConfig *tls.Config, handshaker netxlite.TLSHandshaker) http.RoundTripper {
transport := newBaseTransport()
transport.DialContext = dialer.DialContext
transport.DialTLSContext = (&netxlite.TLSDialer{
Config: tlsConfig,
Dialer: dialer,
TLSHandshaker: handshaker,
}).DialTLSContext
return transport
}
// newBaseTransport creates a new HTTP transport with the default dialer.
func newBaseTransport() (transport *oohttp.StdlibTransport) {
base := oohttp.DefaultTransport.(*oohttp.Transport).Clone()
base.DisableCompression = true
base.MaxConnsPerHost = 1
transport = &oohttp.StdlibTransport{Transport: base}
return transport
}
internal/engine/experiment/websteps/tls.go
+11 -4
View File
@@ -1,16 +1,23 @@
package websteps
import (
"context"
"crypto/tls"
"net"
"github.com/ooni/probe-cli/v3/internal/netxlite"
utls "gitlab.com/yawning/utls.git"
)
// TLSDo performs the TLS check.
func TLSDo(conn net.Conn, hostname string) (*tls.Conn, error) {
tlsConn := tls.Client(conn, &tls.Config{
func TLSDo(ctx context.Context, conn net.Conn, hostname string) (net.Conn, error) {
tlsConf := &tls.Config{
ServerName: hostname,
NextProtos: []string{"h2", "http/1.1"},
})
err := tlsConn.Handshake()
}
h := &netxlite.TLSHandshakerConfigurable{
NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto),
}
tlsConn, _, err := h.Handshake(ctx, conn, tlsConf)
return tlsConn, err
}
internal/engine/experiment/websteps/websteps.go
+1 -1
View File
@@ -241,7 +241,7 @@ func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpo
defer conn.Close()
// TLS handshake step
tlsconn, err := TLSDo(conn, URL.Hostname())
tlsconn, err := TLSDo(ctx, conn, URL.Hostname())
endpointMeasurement.TLSHandshake = &TLSHandshakeMeasurement{
Failure: archival.NewFailure(err),
}
internal/netxlite/tlshandshaker.go
+1
View File
@@ -135,6 +135,7 @@ func (c *UTLSConn) ConnectionState() tls.ConnectionState {
DidResume: uState.DidResume,
CipherSuite: uState.CipherSuite,
NegotiatedProtocol: uState.NegotiatedProtocol,
NegotiatedProtocolIsMutual: true,
ServerName: uState.ServerName,
PeerCertificates: uState.PeerCertificates,
VerifiedChains: uState.VerifiedChains,