ooni-probe-cli/internal/engine/experiment/urlgetter/getter_integration_test.go
Simone Basso bf3c8bcdc3
[forwardport] fix(netx): stop collecting HTTP performance metrics (#689)
This diff forward ports b6db4f64dc83a2a27ee3ce6bba5ac93db922832d, whose
original log message is the following:

- - -

We're now using ooni/oohttp as our HTTP library in most cases.

A limitation of this library is that net/http/httptrace does not
work very well and reliably because (1) we need to use oohttp's
version of that code and (2) we cannot observe net events.

I noticed this fact because an integration test for collecting
HTTP performance metrics was broken.

The best solution here is to remove this functionality, since
it was basically unused in the repository. Only some integration
tests inside urlgetter bothered with these metrics.

A more clinical fix would have been to use ooni/oohttp/httptrace
instead of net/http/httptrace in the stdlib, but it does not
seem to be a good idea, given that those metrics were not used.

With this diff applied, we'll further reduce the number of locally
failing integration tests to just jafar-specific tests.

This diff WILL need to be forwardported to `master`.
2022-02-09 15:08:19 +01:00

740 lines
20 KiB
Go

package urlgetter_test
import (
"context"
"errors"
"net/http"
"strings"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/mockable"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestGetterWithVeryShortTimeout(t *testing.T) {
g := urlgetter.Getter{
Config: urlgetter.Config{
Timeout: 1,
},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(context.Background())
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatal("not the error we expected")
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != netxlite.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || *tk.Failure != "generic_timeout_error" {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 3 {
t.Fatal("not the NetworkEvents we expected")
}
if tk.NetworkEvents[0].Operation != "http_transaction_start" {
t.Fatal("not the NetworkEvents[0].Operation we expected")
}
if tk.NetworkEvents[1].Operation != "http_request_metadata" {
t.Fatal("not the NetworkEvents[1].Operation we expected")
}
if tk.NetworkEvents[2].Operation != "http_transaction_done" {
t.Fatal("not the NetworkEvents[2].Operation we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextVanilla(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // faily immediately
g := urlgetter.Getter{
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != netxlite.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 3 {
t.Fatal("not the NetworkEvents we expected")
}
if tk.NetworkEvents[0].Operation != "http_transaction_start" {
t.Fatal("not the NetworkEvents[0].Operation we expected")
}
if tk.NetworkEvents[1].Operation != "http_request_metadata" {
t.Fatal("not the NetworkEvents[1].Operation we expected")
}
if tk.NetworkEvents[2].Operation != "http_transaction_done" {
t.Fatal("not the NetworkEvents[2].Operation we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextAndMethod(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // faily immediately
g := urlgetter.Getter{
Config: urlgetter.Config{Method: "POST"},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != netxlite.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 3 {
t.Fatal("not the NetworkEvents we expected")
}
if tk.NetworkEvents[0].Operation != "http_transaction_start" {
t.Fatal("not the NetworkEvents[0].Operation we expected")
}
if tk.NetworkEvents[1].Operation != "http_request_metadata" {
t.Fatal("not the NetworkEvents[1].Operation we expected")
}
if tk.NetworkEvents[2].Operation != "http_transaction_done" {
t.Fatal("not the NetworkEvents[2].Operation we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "POST" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextNoFollowRedirects(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // faily immediately
g := urlgetter.Getter{
Config: urlgetter.Config{
NoFollowRedirects: true,
},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if tk.Agent != "agent" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != netxlite.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 3 {
t.Fatal("not the NetworkEvents we expected")
}
if tk.NetworkEvents[0].Operation != "http_transaction_start" {
t.Fatal("not the NetworkEvents[0].Operation we expected")
}
if tk.NetworkEvents[1].Operation != "http_request_metadata" {
t.Fatal("not the NetworkEvents[1].Operation we expected")
}
if tk.NetworkEvents[2].Operation != "http_transaction_done" {
t.Fatal("not the NetworkEvents[2].Operation we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextCannotStartTunnel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
g := urlgetter.Getter{
Config: urlgetter.Config{
Tunnel: "psiphon",
},
Session: &mockable.Session{MockableLogger: log.Log},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatalf("not the error we expected: %+v", err)
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != netxlite.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || *tk.Failure != "interrupted" {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 0 {
t.Fatal("not the NetworkEvents we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 0 {
t.Fatal("not the Requests we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "psiphon" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextUnknownResolverURL(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // faily immediately
g := urlgetter.Getter{
Config: urlgetter.Config{
ResolverURL: "antani://8.8.8.8:53",
},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if err == nil || err.Error() != "unknown_failure: unsupported resolver scheme" {
t.Fatal("not the error we expected")
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != netxlite.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || *tk.Failure != "unknown_failure: unsupported resolver scheme" {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 0 {
t.Fatal("not the NetworkEvents we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 0 {
t.Fatal("not the Requests we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterIntegrationHTTPS(t *testing.T) {
ctx := context.Background()
g := urlgetter.Getter{
Config: urlgetter.Config{
NoFollowRedirects: true, // reduce number of events
},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if err != nil {
t.Fatal(err)
}
if tk.Agent != "agent" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation != nil {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure != nil {
t.Fatal("not the Failure we expected")
}
var (
httpTransactionStart bool
httpRequestMetadata bool
resolveStart bool
resolveDone bool
connect bool
tlsHandshakeStart bool
tlsHandshakeDone bool
httpResponseMetadata bool
httpResponseBodySnapshot bool
httpTransactionDone bool
)
for _, ev := range tk.NetworkEvents {
switch ev.Operation {
case "http_transaction_start":
httpTransactionStart = true
case "http_request_metadata":
httpRequestMetadata = true
case "resolve_start":
resolveStart = true
case "resolve_done":
resolveDone = true
case netxlite.ConnectOperation:
connect = true
case "tls_handshake_start":
tlsHandshakeStart = true
case "tls_handshake_done":
tlsHandshakeDone = true
case "http_response_metadata":
httpResponseMetadata = true
case "http_response_body_snapshot":
httpResponseBodySnapshot = true
case "http_transaction_done":
httpTransactionDone = true
}
}
ok := true
ok = ok && httpTransactionStart
ok = ok && httpRequestMetadata
ok = ok && resolveStart
ok = ok && resolveDone
ok = ok && connect
ok = ok && tlsHandshakeStart
ok = ok && tlsHandshakeDone
ok = ok && httpResponseMetadata
ok = ok && httpResponseBodySnapshot
ok = ok && httpTransactionDone
if !ok {
t.Fatal("not the NetworkEvents we expected")
}
if len(tk.Queries) != 2 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 1 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 1 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 200 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if len(tk.HTTPResponseBody) <= 0 {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterIntegrationRedirect(t *testing.T) {
ctx := context.Background()
g := urlgetter.Getter{
Config: urlgetter.Config{NoFollowRedirects: true},
Session: &mockable.Session{},
Target: "http://web.whatsapp.com",
}
tk, err := g.Get(ctx)
if err != nil {
t.Fatal(err)
}
if tk.HTTPResponseStatus != 302 {
t.Fatal("unexpected status code")
}
if len(tk.HTTPResponseLocations) != 1 {
t.Fatal("missing redirect URL")
}
if tk.HTTPResponseLocations[0] != "https://web.whatsapp.com/" {
t.Fatal("invalid redirect URL")
}
}
func TestGetterIntegrationTLSHandshake(t *testing.T) {
ctx := context.Background()
g := urlgetter.Getter{
Config: urlgetter.Config{
NoFollowRedirects: true, // reduce number of events
},
Session: &mockable.Session{},
Target: "tlshandshake://www.google.com:443",
}
tk, err := g.Get(ctx)
if err != nil {
t.Fatal(err)
}
if tk.Agent != "agent" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation != nil {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure != nil {
t.Fatal("not the Failure we expected")
}
var (
httpTransactionStart bool
httpRequestMetadata bool
resolveStart bool
resolveDone bool
connect bool
tlsHandshakeStart bool
tlsHandshakeDone bool
httpResponseMetadata bool
httpResponseBodySnapshot bool
httpTransactionDone bool
)
for _, ev := range tk.NetworkEvents {
switch ev.Operation {
case "http_transaction_start":
httpTransactionStart = true
case "http_request_metadata":
httpRequestMetadata = true
case "resolve_start":
resolveStart = true
case "resolve_done":
resolveDone = true
case netxlite.ConnectOperation:
connect = true
case "tls_handshake_start":
tlsHandshakeStart = true
case "tls_handshake_done":
tlsHandshakeDone = true
case "http_response_metadata":
httpResponseMetadata = true
case "http_response_body_snapshot":
httpResponseBodySnapshot = true
case "http_transaction_done":
httpTransactionDone = true
}
}
ok := true
ok = ok && !httpTransactionStart
ok = ok && !httpRequestMetadata
ok = ok && resolveStart
ok = ok && resolveDone
ok = ok && connect
ok = ok && tlsHandshakeStart
ok = ok && tlsHandshakeDone
ok = ok && !httpResponseMetadata
ok = ok && !httpResponseBodySnapshot
ok = ok && !httpTransactionDone
if !ok {
t.Fatal("not the NetworkEvents we expected")
}
if len(tk.Queries) != 2 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 1 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 0 {
t.Fatal("not the Requests we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 1 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterHTTPSWithTunnel(t *testing.T) {
// quick enough (0.4s) to run with every run
ctx := context.Background()
g := urlgetter.Getter{
Config: urlgetter.Config{
NoFollowRedirects: true, // reduce number of events
Tunnel: "fake",
},
Session: &mockable.Session{
MockableHTTPClient: http.DefaultClient,
MockableLogger: log.Log,
},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if err != nil {
t.Fatal(err)
}
if tk.Agent != "agent" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime <= 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation != nil {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure != nil {
t.Fatal("not the Failure we expected")
}
var (
httpTransactionStart bool
httpRequestMetadata bool
resolveStart bool
resolveDone bool
connect bool
tlsHandshakeStart bool
tlsHandshakeDone bool
httpResponseMetadata bool
httpResponseBodySnapshot bool
httpTransactionDone bool
)
for _, ev := range tk.NetworkEvents {
switch ev.Operation {
case "http_transaction_start":
httpTransactionStart = true
case "http_request_metadata":
httpRequestMetadata = true
case "resolve_start":
resolveStart = true
case "resolve_done":
resolveDone = true
case netxlite.ConnectOperation:
connect = true
case "tls_handshake_start":
tlsHandshakeStart = true
case "tls_handshake_done":
tlsHandshakeDone = true
case "http_response_metadata":
httpResponseMetadata = true
case "http_response_body_snapshot":
httpResponseBodySnapshot = true
case "http_transaction_done":
httpTransactionDone = true
}
}
ok := true
ok = ok && httpTransactionStart
ok = ok && httpRequestMetadata
ok = ok && resolveStart == false
ok = ok && resolveDone == false
ok = ok && connect
ok = ok && tlsHandshakeStart
ok = ok && tlsHandshakeDone
ok = ok && httpResponseMetadata
ok = ok && httpResponseBodySnapshot
ok = ok && httpTransactionDone
if !ok {
t.Fatalf("not the NetworkEvents we expected: %+v", tk.NetworkEvents)
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 1 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy == "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 1 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "fake" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 200 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if len(tk.HTTPResponseBody) <= 0 {
t.Fatal("not the HTTPResponseBody we expected")
}
}