c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: migrate apitool from netx to netxlite (#496)

Browse Source 
I discovered which transport were used by apitool and made sure he gets the same transports now. While there, I discovered an issue with ooni/oohttp that has been fixed with https://github.com/ooni/oohttp/commit/cba9b1ce5ecf7a20217ad8571ad7a85e490c4789.

Part of https://github.com/ooni/probe/issues/1591
This commit is contained in:
Simone Basso
2021-09-09 01:19:17 +02:00
committed by GitHub
parent 00a85cb7f0
commit 1d79d70b43
15 changed files with 149 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/netxlite/http.go
+37 -6
View File
@@ -106,6 +106,9 @@ func (txp *httpTransportConnectionsCloser) CloseIdleConnections() {
// necessary to perform sane measurements with tracing. We will be
// able to possibly relax this requirement after we change the
// way in which we perform measurements.
//
// The returned transport will set a default user agent if the
// request has not already set a user agent.
func NewHTTPTransport(logger Logger, dialer Dialer, tlsDialer TLSDialer) HTTPTransport {
// Using oohttp to support any TLS library.
txp := oohttp.DefaultTransport.(*oohttp.Transport).Clone()
@@ -137,13 +140,15 @@ func NewHTTPTransport(logger Logger, dialer Dialer, tlsDialer TLSDialer) HTTPTra
// Ensure we correctly forward CloseIdleConnections and compose
// with a logging transport thus enabling logging.
return &httpTransportLogger{
HTTPTransport: &httpTransportConnectionsCloser{
HTTPTransport: &oohttp.StdlibTransport{Transport: txp},
Dialer: dialer,
TLSDialer: tlsDialer,
return &httpUserAgentTransport{
HTTPTransport: &httpTransportLogger{
HTTPTransport: &httpTransportConnectionsCloser{
HTTPTransport: &oohttp.StdlibTransport{Transport: txp},
Dialer: dialer,
TLSDialer: tlsDialer,
},
Logger: logger,
},
Logger: logger,
}
}
@@ -233,3 +238,29 @@ func (c *httpTLSConnWithReadTimeout) Read(b []byte) (int, error) {
defer c.TLSConn.SetReadDeadline(time.Time{})
return c.TLSConn.Read(b)
}
// httpUserAgentTransport is a transport that ensures that we always
// set an OONI specific default User-Agent header.
type httpUserAgentTransport struct {
HTTPTransport
}
const defaultHTTPUserAgent = "miniooni/0.1.0-dev"
func (txp *httpUserAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", defaultHTTPUserAgent)
}
return txp.HTTPTransport.RoundTrip(req)
}
var _ HTTPTransport = &httpUserAgentTransport{}
// NewHTTPTransportStdlib creates a new HTTPTransport that uses
// the Go standard library for all operations, including DNS
// resolutions and TLS handshakes.
func NewHTTPTransportStdlib(logger Logger) HTTPTransport {
dialer := NewDialerWithResolver(logger, NewResolverStdlib(logger))
tlsDialer := NewTLSDialer(dialer, NewTLSHandshakerStdlib(logger))
return NewHTTPTransport(logger, dialer, tlsDialer)
}
internal/netxlite/http_test.go
+78 -2
View File
@@ -202,7 +202,7 @@ func TestNewHTTPTransport(t *testing.T) {
called.Add(1)
},
},
Resolver: NewResolverSystem(log.Log),
Resolver: NewResolverStdlib(log.Log),
}
td := NewTLSDialer(d, NewTLSHandshakerStdlib(log.Log))
txp := NewHTTPTransport(log.Log, d, td)
@@ -224,7 +224,8 @@ func TestNewHTTPTransport(t *testing.T) {
d := &mocks.Dialer{}
td := &mocks.TLSDialer{}
txp := NewHTTPTransport(log.Log, d, td)
logger := txp.(*httpTransportLogger)
ua := txp.(*httpUserAgentTransport)
logger := ua.HTTPTransport.(*httpTransportLogger)
if logger.Logger != log.Log {
t.Fatal("invalid logger")
}
@@ -423,3 +424,78 @@ func TestHTTPTLSDialerWithReadTimeout(t *testing.T) {
}
})
}
func TestHTTPUserAgentTransport(t *testing.T) {
t.Run("CloseIdleConnections", func(t *testing.T) {
var called bool
txp := &httpUserAgentTransport{
HTTPTransport: &mocks.HTTPTransport{
MockCloseIdleConnections: func() {
called = true
},
},
}
txp.CloseIdleConnections()
if !called {
t.Fatal("not called")
}
})
t.Run("RoundTrip", func(t *testing.T) {
t.Run("without an user-agent", func(t *testing.T) {
var ua string
txp := &httpUserAgentTransport{
HTTPTransport: &mocks.HTTPTransport{
MockRoundTrip: func(req *http.Request) (*http.Response, error) {
ua = req.Header.Get("User-Agent")
return &http.Response{}, nil
},
},
}
txp.RoundTrip(&http.Request{Header: make(http.Header)})
if ua != defaultHTTPUserAgent {
t.Fatal("not the expected user-agent")
}
})
t.Run("with an user-agent", func(t *testing.T) {
var ua string
expected := "antani/1.0"
txp := &httpUserAgentTransport{
HTTPTransport: &mocks.HTTPTransport{
MockRoundTrip: func(req *http.Request) (*http.Response, error) {
ua = req.Header.Get("User-Agent")
return &http.Response{}, nil
},
},
}
txp.RoundTrip(&http.Request{
Header: http.Header{
"User-Agent": {expected},
},
})
if ua != expected {
t.Fatal("not the expected user-agent")
}
})
})
}
func TestNewHTTPTransportStdlib(t *testing.T) {
// What to test about this factory?
txp := NewHTTPTransportStdlib(log.Log)
ctx, cancel := context.WithCancel(context.Background())
cancel() // immediately!
req, err := http.NewRequestWithContext(ctx, "GET", "http://x.org", nil)
if err != nil {
t.Fatal(err)
}
resp, err := txp.RoundTrip(req)
if !errors.Is(err, context.Canceled) {
t.Fatal("unexpected err", err)
}
if resp != nil {
t.Fatal("unexpected resp")
}
txp.CloseIdleConnections()
}
internal/netxlite/integration_test.go
+4 -4
View File
@@ -21,7 +21,7 @@ func TestResolver(t *testing.T) {
t.Run("works as intended", func(t *testing.T) {
// TODO(bassosimone): this is actually an integration
// test but how to test this case?
r := netxlite.NewResolverSystem(log.Log)
r := netxlite.NewResolverStdlib(log.Log)
defer r.CloseIdleConnections()
addrs, err := r.LookupHost(context.Background(), "dns.google.com")
if err != nil {
@@ -39,7 +39,7 @@ func TestHTTPTransport(t *testing.T) {
}
t.Run("works as intended", func(t *testing.T) {
d := netxlite.NewDialerWithResolver(log.Log, netxlite.NewResolverSystem(log.Log))
d := netxlite.NewDialerWithResolver(log.Log, netxlite.NewResolverStdlib(log.Log))
td := netxlite.NewTLSDialer(d, netxlite.NewTLSHandshakerStdlib(log.Log))
txp := netxlite.NewHTTPTransport(log.Log, d, td)
client := &http.Client{Transport: txp}
@@ -61,7 +61,7 @@ func TestHTTP3Transport(t *testing.T) {
d := netxlite.NewQUICDialerWithResolver(
netxlite.NewQUICListener(),
log.Log,
netxlite.NewResolverSystem(log.Log),
netxlite.NewResolverStdlib(log.Log),
)
txp := netxlite.NewHTTP3Transport(log.Log, d, &tls.Config{})
client := &http.Client{Transport: txp}
@@ -119,7 +119,7 @@ func TestQUICDialer(t *testing.T) {
t.Run("can guess the SNI and ALPN when using a domain name for web", func(t *testing.T) {
d := netxlite.NewQUICDialerWithResolver(
netxlite.NewQUICListener(), log.Log,
netxlite.NewResolverSystem(log.Log),
netxlite.NewResolverStdlib(log.Log),
)
ctx := context.Background()
sess, err := d.DialContext(
internal/netxlite/legacy.go
+1
View File
@@ -32,6 +32,7 @@ type (
TLSHandshakerLogger = tlsHandshakerLogger
DialerSystem = dialerSystem
TLSDialerLegacy = tlsDialer
UserAgentTransport = httpUserAgentTransport
)
// ResolverLegacy performs domain name resolutions.
internal/netxlite/quic_test.go
+4 -4
View File
@@ -256,7 +256,7 @@ func TestQUICDialerResolver(t *testing.T) {
t.Run("on success", func(t *testing.T) {
tlsConfig := &tls.Config{}
dialer := &quicDialerResolver{
Resolver: NewResolverSystem(log.Log),
Resolver: NewResolverStdlib(log.Log),
Dialer: &quicDialerQUICGo{
QUICListener: &quicListenerStdlib{},
}}
@@ -275,7 +275,7 @@ func TestQUICDialerResolver(t *testing.T) {
t.Run("with missing port", func(t *testing.T) {
tlsConfig := &tls.Config{}
dialer := &quicDialerResolver{
Resolver: NewResolverSystem(log.Log),
Resolver: NewResolverStdlib(log.Log),
Dialer: &quicDialerQUICGo{}}
sess, err := dialer.DialContext(
context.Background(), "udp", "www.google.com",
@@ -312,7 +312,7 @@ func TestQUICDialerResolver(t *testing.T) {
// to establish a connection leads to a failure
tlsConf := &tls.Config{}
dialer := &quicDialerResolver{
Resolver: NewResolverSystem(log.Log),
Resolver: NewResolverStdlib(log.Log),
Dialer: &quicDialerQUICGo{
QUICListener: &quicListenerStdlib{},
}}
@@ -336,7 +336,7 @@ func TestQUICDialerResolver(t *testing.T) {
var gotTLSConfig *tls.Config
tlsConfig := &tls.Config{}
dialer := &quicDialerResolver{
Resolver: NewResolverSystem(log.Log),
Resolver: NewResolverStdlib(log.Log),
Dialer: &mocks.QUICDialer{
MockDialContext: func(ctx context.Context, network, address string,
tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlySession, error) {
internal/netxlite/resolver.go
+2 -2
View File
@@ -25,7 +25,7 @@ type Resolver interface {
CloseIdleConnections()
}
// NewResolverSystem creates a new resolver using system
// NewResolverStdlib creates a new resolver using system
// facilities for resolving domain names (e.g., getaddrinfo).
//
// The resolver will provide the following guarantees:
@@ -41,7 +41,7 @@ type Resolver interface {
//
// 5. enforces reasonable timeouts (
// see https://github.com/ooni/probe/issues/1726).
func NewResolverSystem(logger Logger) Resolver {
func NewResolverStdlib(logger Logger) Resolver {
return &resolverIDNA{
Resolver: &resolverLogger{
Resolver: &resolverShortCircuitIPAddr{
internal/netxlite/resolver_test.go
+1 -1
View File
@@ -16,7 +16,7 @@ import (
)
func TestNewResolverSystem(t *testing.T) {
resolver := NewResolverSystem(log.Log)
resolver := NewResolverStdlib(log.Log)
idna := resolver.(*resolverIDNA)
logger := idna.Resolver.(*resolverLogger)
if logger.Logger != log.Log {