urlgetter: fix tunnel test (#299)

* urlgetter: fix tunnel test

This diff fixes the urlgetter test suite to make sure we
are correctly testing for tunnel creation.

While there, improve the way in which we create a testing
directory and add a test for that.

Part of https://github.com/ooni/probe/issues/985.

* fix comment

* fix comment

internal/engine/experiment/urlgetter/getter.go

@@ -2,10 +2,8 @@ package urlgetter
 
 import (
 	"context"
-	"fmt"
+	"io/ioutil"
 	"net/url"
-	"path/filepath"
-	"sync"
 	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
@@ -36,6 +34,9 @@ type Getter struct {
 	// Target is the thing to measure in this run. This field must
 	// be set otherwise the code won't know what to do.
 	Target string
+
+	// testIOUtilTempDir allows us to mock ioutil.TempDir
+	testIOUtilTempDir func(dir, pattern string) (string, error)
 }
 
 // Get performs the action described by g using the given context
@@ -82,18 +83,13 @@ func (g Getter) Get(ctx context.Context) (TestKeys, error) {
 	return tk, err
 }
 
-// TODO(bassosimone): this mechanism where we count breaks tests
+// ioutilTempDir calls either g.testIOUtilTempDir or ioutil.TempDir
-// because now tests are not idempotent anymore. Therefore, we
+func (g Getter) ioutilTempDir(dir, pattern string) (string, error) {
-// SHOULD be creating a temporary directory instead.
+	if g.testIOUtilTempDir != nil {
-
+		return g.testIOUtilTempDir(dir, pattern)
-var (
+	}
-	// tunnelDirCount counts the number of tunnels started by
+	return ioutil.TempDir(dir, pattern)
-	// the urlgetter package so far.
+}
-	tunnelDirCount int64
-
-	// tunnelDirMu protects tunnelDirCount
-	tunnelDirMu sync.Mutex
-)
 
 func (g Getter) get(ctx context.Context, saver *trace.Saver) (TestKeys, error) {
 	tk := TestKeys{
@@ -112,17 +108,16 @@ func (g Getter) get(ctx context.Context, saver *trace.Saver) (TestKeys, error) {
 		// Every new instance of the tunnel goes into a separate
 		// directory within the temporary directory. Calling
 		// Session.Close will delete such a directory.
-		tunnelDirMu.Lock()
+		tundir, err := g.ioutilTempDir(g.Session.TempDir(), "urlgetter-tunnel-")
-		count := tunnelDirCount
+		if err != nil {
-		tunnelDirCount++
+			return tk, err
-		tunnelDirMu.Unlock()
+		}
 		tun, err := tunnel.Start(ctx, &tunnel.Config{
 			Name:      g.Config.Tunnel,
 			Session:   g.Session,
 			TorArgs:   g.Session.TorArgs(),
 			TorBinary: g.Session.TorBinary(),
-			TunnelDir: filepath.Join(
+			TunnelDir: tundir,
-				g.Session.TempDir(), fmt.Sprintf("urlgetter-tunnel-%d", count)),
 		})
 		if err != nil {
 			return tk, err

internal/engine/experiment/urlgetter/getter_integration_test.go

@@ -0,0 +1,775 @@
+package urlgetter_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+func TestGetterWithVeryShortTimeout(t *testing.T) {
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			Timeout: 1,
+		},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(context.Background())
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || *tk.Failure != "generic_timeout_error" {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 3 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
+		t.Fatal("not the NetworkEvents[0].Operation we expected")
+	}
+	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
+		t.Fatal("not the NetworkEvents[1].Operation we expected")
+	}
+	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
+		t.Fatal("not the NetworkEvents[2].Operation we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextVanilla(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // faily immediately
+	g := urlgetter.Getter{
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 3 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
+		t.Fatal("not the NetworkEvents[0].Operation we expected")
+	}
+	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
+		t.Fatal("not the NetworkEvents[1].Operation we expected")
+	}
+	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
+		t.Fatal("not the NetworkEvents[2].Operation we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextAndMethod(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // faily immediately
+	g := urlgetter.Getter{
+		Config:  urlgetter.Config{Method: "POST"},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 3 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
+		t.Fatal("not the NetworkEvents[0].Operation we expected")
+	}
+	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
+		t.Fatal("not the NetworkEvents[1].Operation we expected")
+	}
+	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
+		t.Fatal("not the NetworkEvents[2].Operation we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "POST" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextNoFollowRedirects(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // faily immediately
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true,
+		},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 3 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
+		t.Fatal("not the NetworkEvents[0].Operation we expected")
+	}
+	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
+		t.Fatal("not the NetworkEvents[1].Operation we expected")
+	}
+	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
+		t.Fatal("not the NetworkEvents[2].Operation we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextCannotStartTunnel(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // fail immediately
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			Tunnel: "psiphon",
+		},
+		Session: &mockable.Session{MockableLogger: log.Log},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || *tk.Failure != "interrupted" {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 0 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "psiphon" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterWithCancelledContextUnknownResolverURL(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // faily immediately
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			ResolverURL: "antani://8.8.8.8:53",
+		},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if err == nil || err.Error() != "unknown_failure: unsupported resolver scheme" {
+		t.Fatal("not the error we expected")
+	}
+	if tk.Agent != "redirect" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure == nil || *tk.Failure != "unknown_failure: unsupported resolver scheme" {
+		t.Fatal("not the Failure we expected")
+	}
+	if len(tk.NetworkEvents) != 0 {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 0 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterIntegrationHTTPS(t *testing.T) {
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true, // reduce number of events
+		},
+		Session: &mockable.Session{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure != nil {
+		t.Fatal("not the Failure we expected")
+	}
+	var (
+		httpTransactionStart     bool
+		httpRequestMetadata      bool
+		resolveStart             bool
+		resolveDone              bool
+		connect                  bool
+		tlsHandshakeStart        bool
+		tlsHandshakeDone         bool
+		httpWroteHeaders         bool
+		httpWroteRequest         bool
+		httpFirstResponseByte    bool
+		httpResponseMetadata     bool
+		httpResponseBodySnapshot bool
+		httpTransactionDone      bool
+	)
+	for _, ev := range tk.NetworkEvents {
+		switch ev.Operation {
+		case "http_transaction_start":
+			httpTransactionStart = true
+		case "http_request_metadata":
+			httpRequestMetadata = true
+		case "resolve_start":
+			resolveStart = true
+		case "resolve_done":
+			resolveDone = true
+		case errorx.ConnectOperation:
+			connect = true
+		case "tls_handshake_start":
+			tlsHandshakeStart = true
+		case "tls_handshake_done":
+			tlsHandshakeDone = true
+		case "http_wrote_headers":
+			httpWroteHeaders = true
+		case "http_wrote_request":
+			httpWroteRequest = true
+		case "http_first_response_byte":
+			httpFirstResponseByte = true
+		case "http_response_metadata":
+			httpResponseMetadata = true
+		case "http_response_body_snapshot":
+			httpResponseBodySnapshot = true
+		case "http_transaction_done":
+			httpTransactionDone = true
+		}
+	}
+	ok := true
+	ok = ok && httpTransactionStart
+	ok = ok && httpRequestMetadata
+	ok = ok && resolveStart
+	ok = ok && resolveDone
+	ok = ok && connect
+	ok = ok && tlsHandshakeStart
+	ok = ok && tlsHandshakeDone
+	ok = ok && httpWroteHeaders
+	ok = ok && httpWroteRequest
+	ok = ok && httpFirstResponseByte
+	ok = ok && httpResponseMetadata
+	ok = ok && httpResponseBodySnapshot
+	ok = ok && httpTransactionDone
+	if !ok {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 2 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 1 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 1 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 200 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if len(tk.HTTPResponseBody) <= 0 {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterIntegrationRedirect(t *testing.T) {
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config:  urlgetter.Config{NoFollowRedirects: true},
+		Session: &mockable.Session{},
+		Target:  "http://web.whatsapp.com",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.HTTPResponseStatus != 302 {
+		t.Fatal("unexpected status code")
+	}
+	if len(tk.HTTPResponseLocations) != 1 {
+		t.Fatal("missing redirect URL")
+	}
+	if tk.HTTPResponseLocations[0] != "https://web.whatsapp.com/" {
+		t.Fatal("invalid redirect URL")
+	}
+}
+
+func TestGetterIntegrationTLSHandshake(t *testing.T) {
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true, // reduce number of events
+		},
+		Session: &mockable.Session{},
+		Target:  "tlshandshake://www.google.com:443",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure != nil {
+		t.Fatal("not the Failure we expected")
+	}
+	var (
+		httpTransactionStart     bool
+		httpRequestMetadata      bool
+		resolveStart             bool
+		resolveDone              bool
+		connect                  bool
+		tlsHandshakeStart        bool
+		tlsHandshakeDone         bool
+		httpWroteHeaders         bool
+		httpWroteRequest         bool
+		httpFirstResponseByte    bool
+		httpResponseMetadata     bool
+		httpResponseBodySnapshot bool
+		httpTransactionDone      bool
+	)
+	for _, ev := range tk.NetworkEvents {
+		switch ev.Operation {
+		case "http_transaction_start":
+			httpTransactionStart = true
+		case "http_request_metadata":
+			httpRequestMetadata = true
+		case "resolve_start":
+			resolveStart = true
+		case "resolve_done":
+			resolveDone = true
+		case errorx.ConnectOperation:
+			connect = true
+		case "tls_handshake_start":
+			tlsHandshakeStart = true
+		case "tls_handshake_done":
+			tlsHandshakeDone = true
+		case "http_wrote_headers":
+			httpWroteHeaders = true
+		case "http_wrote_request":
+			httpWroteRequest = true
+		case "http_first_response_byte":
+			httpFirstResponseByte = true
+		case "http_response_metadata":
+			httpResponseMetadata = true
+		case "http_response_body_snapshot":
+			httpResponseBodySnapshot = true
+		case "http_transaction_done":
+			httpTransactionDone = true
+		}
+	}
+	ok := true
+	ok = ok && !httpTransactionStart
+	ok = ok && !httpRequestMetadata
+	ok = ok && resolveStart
+	ok = ok && resolveDone
+	ok = ok && connect
+	ok = ok && tlsHandshakeStart
+	ok = ok && tlsHandshakeDone
+	ok = ok && !httpWroteHeaders
+	ok = ok && !httpWroteRequest
+	ok = ok && !httpFirstResponseByte
+	ok = ok && !httpResponseMetadata
+	ok = ok && !httpResponseBodySnapshot
+	ok = ok && !httpTransactionDone
+	if !ok {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 2 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 1 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 0 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 1 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 0 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if tk.HTTPResponseBody != "" {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}
+
+func TestGetterHTTPSWithTunnel(t *testing.T) {
+	// quick enough (0.4s) to run with every run
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true, // reduce number of events
+			Tunnel:            "fake",
+		},
+		Session: &mockable.Session{
+			MockableHTTPClient: http.DefaultClient,
+			MockableLogger:     log.Log,
+		},
+		Target: "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime <= 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("not the FailedOperation we expected")
+	}
+	if tk.Failure != nil {
+		t.Fatal("not the Failure we expected")
+	}
+	var (
+		httpTransactionStart     bool
+		httpRequestMetadata      bool
+		resolveStart             bool
+		resolveDone              bool
+		connect                  bool
+		tlsHandshakeStart        bool
+		tlsHandshakeDone         bool
+		httpWroteHeaders         bool
+		httpWroteRequest         bool
+		httpFirstResponseByte    bool
+		httpResponseMetadata     bool
+		httpResponseBodySnapshot bool
+		httpTransactionDone      bool
+	)
+	for _, ev := range tk.NetworkEvents {
+		switch ev.Operation {
+		case "http_transaction_start":
+			httpTransactionStart = true
+		case "http_request_metadata":
+			httpRequestMetadata = true
+		case "resolve_start":
+			resolveStart = true
+		case "resolve_done":
+			resolveDone = true
+		case errorx.ConnectOperation:
+			connect = true
+		case "tls_handshake_start":
+			tlsHandshakeStart = true
+		case "tls_handshake_done":
+			tlsHandshakeDone = true
+		case "http_wrote_headers":
+			httpWroteHeaders = true
+		case "http_wrote_request":
+			httpWroteRequest = true
+		case "http_first_response_byte":
+			httpFirstResponseByte = true
+		case "http_response_metadata":
+			httpResponseMetadata = true
+		case "http_response_body_snapshot":
+			httpResponseBodySnapshot = true
+		case "http_transaction_done":
+			httpTransactionDone = true
+		}
+	}
+	ok := true
+	ok = ok && httpTransactionStart
+	ok = ok && httpRequestMetadata
+	ok = ok && resolveStart == false
+	ok = ok && resolveDone == false
+	ok = ok && connect
+	ok = ok && tlsHandshakeStart
+	ok = ok && tlsHandshakeDone
+	ok = ok && httpWroteHeaders
+	ok = ok && httpWroteRequest
+	ok = ok && httpFirstResponseByte
+	ok = ok && httpResponseMetadata
+	ok = ok && httpResponseBodySnapshot
+	ok = ok && httpTransactionDone
+	if !ok {
+		t.Fatalf("not the NetworkEvents we expected: %+v", tk.NetworkEvents)
+	}
+	if len(tk.Queries) != 0 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 1 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy == "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 1 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "fake" {
+		t.Fatal("not the Tunnel we expected")
+	}
+	if tk.HTTPResponseStatus != 200 {
+		t.Fatal("not the HTTPResponseStatus we expected")
+	}
+	if len(tk.HTTPResponseBody) <= 0 {
+		t.Fatal("not the HTTPResponseBody we expected")
+	}
+}

internal/engine/experiment/urlgetter/getter_test.go

@@ -1,780 +1,73 @@
-package urlgetter_test
+package urlgetter
 
 import (
 	"context"
 	"errors"
 	"net/http"
-	"strings"
 	"testing"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
-	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
 )
 
-func TestGetterWithVeryShortTimeout(t *testing.T) {
+func TestGetterHTTPSWithTunnelCannotCreateTempDir(t *testing.T) {
-	g := urlgetter.Getter{
+	expected := errors.New("mocked error")
-		Config: urlgetter.Config{
-			Timeout: 1,
-		},
-		Session: &mockable.Session{},
-		Target:  "https://www.google.com",
-	}
-	tk, err := g.Get(context.Background())
-	if !errors.Is(err, context.DeadlineExceeded) {
-		t.Fatal("not the error we expected")
-	}
-	if tk.Agent != "redirect" {
-		t.Fatal("not the Agent we expected")
-	}
-	if tk.BootstrapTime != 0 {
-		t.Fatal("not the BootstrapTime we expected")
-	}
-	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
-		t.Fatal("not the FailedOperation we expected")
-	}
-	if tk.Failure == nil || *tk.Failure != "generic_timeout_error" {
-		t.Fatal("not the Failure we expected")
-	}
-	if len(tk.NetworkEvents) != 3 {
-		t.Fatal("not the NetworkEvents we expected")
-	}
-	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
-		t.Fatal("not the NetworkEvents[0].Operation we expected")
-	}
-	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
-		t.Fatal("not the NetworkEvents[1].Operation we expected")
-	}
-	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
-		t.Fatal("not the NetworkEvents[2].Operation we expected")
-	}
-	if len(tk.Queries) != 0 {
-		t.Fatal("not the Queries we expected")
-	}
-	if len(tk.TCPConnect) != 0 {
-		t.Fatal("not the TCPConnect we expected")
-	}
-	if len(tk.Requests) != 1 {
-		t.Fatal("not the Requests we expected")
-	}
-	if tk.Requests[0].Request.Method != "GET" {
-		t.Fatal("not the Method we expected")
-	}
-	if tk.Requests[0].Request.URL != "https://www.google.com" {
-		t.Fatal("not the URL we expected")
-	}
-	if tk.SOCKSProxy != "" {
-		t.Fatal("not the SOCKSProxy we expected")
-	}
-	if len(tk.TLSHandshakes) != 0 {
-		t.Fatal("not the TLSHandshakes we expected")
-	}
-	if tk.Tunnel != "" {
-		t.Fatal("not the Tunnel we expected")
-	}
-	if tk.HTTPResponseStatus != 0 {
-		t.Fatal("not the HTTPResponseStatus we expected")
-	}
-	if tk.HTTPResponseBody != "" {
-		t.Fatal("not the HTTPResponseBody we expected")
-	}
-}
-
-func TestGetterWithCancelledContextVanilla(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel() // faily immediately
-	g := urlgetter.Getter{
-		Session: &mockable.Session{},
-		Target:  "https://www.google.com",
-	}
-	tk, err := g.Get(ctx)
-	if !errors.Is(err, context.Canceled) {
-		t.Fatal("not the error we expected")
-	}
-	if tk.Agent != "redirect" {
-		t.Fatal("not the Agent we expected")
-	}
-	if tk.BootstrapTime != 0 {
-		t.Fatal("not the BootstrapTime we expected")
-	}
-	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
-		t.Fatal("not the FailedOperation we expected")
-	}
-	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
-		t.Fatal("not the Failure we expected")
-	}
-	if len(tk.NetworkEvents) != 3 {
-		t.Fatal("not the NetworkEvents we expected")
-	}
-	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
-		t.Fatal("not the NetworkEvents[0].Operation we expected")
-	}
-	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
-		t.Fatal("not the NetworkEvents[1].Operation we expected")
-	}
-	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
-		t.Fatal("not the NetworkEvents[2].Operation we expected")
-	}
-	if len(tk.Queries) != 0 {
-		t.Fatal("not the Queries we expected")
-	}
-	if len(tk.TCPConnect) != 0 {
-		t.Fatal("not the TCPConnect we expected")
-	}
-	if len(tk.Requests) != 1 {
-		t.Fatal("not the Requests we expected")
-	}
-	if tk.Requests[0].Request.Method != "GET" {
-		t.Fatal("not the Method we expected")
-	}
-	if tk.Requests[0].Request.URL != "https://www.google.com" {
-		t.Fatal("not the URL we expected")
-	}
-	if tk.SOCKSProxy != "" {
-		t.Fatal("not the SOCKSProxy we expected")
-	}
-	if len(tk.TLSHandshakes) != 0 {
-		t.Fatal("not the TLSHandshakes we expected")
-	}
-	if tk.Tunnel != "" {
-		t.Fatal("not the Tunnel we expected")
-	}
-	if tk.HTTPResponseStatus != 0 {
-		t.Fatal("not the HTTPResponseStatus we expected")
-	}
-	if tk.HTTPResponseBody != "" {
-		t.Fatal("not the HTTPResponseBody we expected")
-	}
-}
-
-func TestGetterWithCancelledContextAndMethod(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel() // faily immediately
-	g := urlgetter.Getter{
-		Config:  urlgetter.Config{Method: "POST"},
-		Session: &mockable.Session{},
-		Target:  "https://www.google.com",
-	}
-	tk, err := g.Get(ctx)
-	if !errors.Is(err, context.Canceled) {
-		t.Fatal("not the error we expected")
-	}
-	if tk.Agent != "redirect" {
-		t.Fatal("not the Agent we expected")
-	}
-	if tk.BootstrapTime != 0 {
-		t.Fatal("not the BootstrapTime we expected")
-	}
-	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
-		t.Fatal("not the FailedOperation we expected")
-	}
-	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
-		t.Fatal("not the Failure we expected")
-	}
-	if len(tk.NetworkEvents) != 3 {
-		t.Fatal("not the NetworkEvents we expected")
-	}
-	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
-		t.Fatal("not the NetworkEvents[0].Operation we expected")
-	}
-	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
-		t.Fatal("not the NetworkEvents[1].Operation we expected")
-	}
-	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
-		t.Fatal("not the NetworkEvents[2].Operation we expected")
-	}
-	if len(tk.Queries) != 0 {
-		t.Fatal("not the Queries we expected")
-	}
-	if len(tk.TCPConnect) != 0 {
-		t.Fatal("not the TCPConnect we expected")
-	}
-	if len(tk.Requests) != 1 {
-		t.Fatal("not the Requests we expected")
-	}
-	if tk.Requests[0].Request.Method != "POST" {
-		t.Fatal("not the Method we expected")
-	}
-	if tk.Requests[0].Request.URL != "https://www.google.com" {
-		t.Fatal("not the URL we expected")
-	}
-	if tk.SOCKSProxy != "" {
-		t.Fatal("not the SOCKSProxy we expected")
-	}
-	if len(tk.TLSHandshakes) != 0 {
-		t.Fatal("not the TLSHandshakes we expected")
-	}
-	if tk.Tunnel != "" {
-		t.Fatal("not the Tunnel we expected")
-	}
-	if tk.HTTPResponseStatus != 0 {
-		t.Fatal("not the HTTPResponseStatus we expected")
-	}
-	if tk.HTTPResponseBody != "" {
-		t.Fatal("not the HTTPResponseBody we expected")
-	}
-}
-
-func TestGetterWithCancelledContextNoFollowRedirects(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel() // faily immediately
-	g := urlgetter.Getter{
-		Config: urlgetter.Config{
-			NoFollowRedirects: true,
-		},
-		Session: &mockable.Session{},
-		Target:  "https://www.google.com",
-	}
-	tk, err := g.Get(ctx)
-	if !errors.Is(err, context.Canceled) {
-		t.Fatal("not the error we expected")
-	}
-	if tk.Agent != "agent" {
-		t.Fatal("not the Agent we expected")
-	}
-	if tk.BootstrapTime != 0 {
-		t.Fatal("not the BootstrapTime we expected")
-	}
-	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
-		t.Fatal("not the FailedOperation we expected")
-	}
-	if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
-		t.Fatal("not the Failure we expected")
-	}
-	if len(tk.NetworkEvents) != 3 {
-		t.Fatal("not the NetworkEvents we expected")
-	}
-	if tk.NetworkEvents[0].Operation != "http_transaction_start" {
-		t.Fatal("not the NetworkEvents[0].Operation we expected")
-	}
-	if tk.NetworkEvents[1].Operation != "http_request_metadata" {
-		t.Fatal("not the NetworkEvents[1].Operation we expected")
-	}
-	if tk.NetworkEvents[2].Operation != "http_transaction_done" {
-		t.Fatal("not the NetworkEvents[2].Operation we expected")
-	}
-	if len(tk.Queries) != 0 {
-		t.Fatal("not the Queries we expected")
-	}
-	if len(tk.TCPConnect) != 0 {
-		t.Fatal("not the TCPConnect we expected")
-	}
-	if len(tk.Requests) != 1 {
-		t.Fatal("not the Requests we expected")
-	}
-	if tk.Requests[0].Request.Method != "GET" {
-		t.Fatal("not the Method we expected")
-	}
-	if tk.Requests[0].Request.URL != "https://www.google.com" {
-		t.Fatal("not the URL we expected")
-	}
-	if tk.SOCKSProxy != "" {
-		t.Fatal("not the SOCKSProxy we expected")
-	}
-	if len(tk.TLSHandshakes) != 0 {
-		t.Fatal("not the TLSHandshakes we expected")
-	}
-	if tk.Tunnel != "" {
-		t.Fatal("not the Tunnel we expected")
-	}
-	if tk.HTTPResponseStatus != 0 {
-		t.Fatal("not the HTTPResponseStatus we expected")
-	}
-	if tk.HTTPResponseBody != "" {
-		t.Fatal("not the HTTPResponseBody we expected")
-	}
-}
-
-func TestGetterWithCancelledContextCannotStartTunnel(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel() // fail immediately
-	g := urlgetter.Getter{
-		Config: urlgetter.Config{
-			Tunnel: "psiphon",
-		},
-		Session: &mockable.Session{MockableLogger: log.Log},
-		Target:  "https://www.google.com",
-	}
-	tk, err := g.Get(ctx)
-	if !errors.Is(err, context.Canceled) {
-		t.Fatalf("not the error we expected: %+v", err)
-	}
-	if tk.Agent != "redirect" {
-		t.Fatal("not the Agent we expected")
-	}
-	if tk.BootstrapTime != 0 {
-		t.Fatal("not the BootstrapTime we expected")
-	}
-	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
-		t.Fatal("not the FailedOperation we expected")
-	}
-	if tk.Failure == nil || *tk.Failure != "interrupted" {
-		t.Fatal("not the Failure we expected")
-	}
-	if len(tk.NetworkEvents) != 0 {
-		t.Fatal("not the NetworkEvents we expected")
-	}
-	if len(tk.Queries) != 0 {
-		t.Fatal("not the Queries we expected")
-	}
-	if len(tk.TCPConnect) != 0 {
-		t.Fatal("not the TCPConnect we expected")
-	}
-	if len(tk.Requests) != 0 {
-		t.Fatal("not the Requests we expected")
-	}
-	if tk.SOCKSProxy != "" {
-		t.Fatal("not the SOCKSProxy we expected")
-	}
-	if len(tk.TLSHandshakes) != 0 {
-		t.Fatal("not the TLSHandshakes we expected")
-	}
-	if tk.Tunnel != "psiphon" {
-		t.Fatal("not the Tunnel we expected")
-	}
-	if tk.HTTPResponseStatus != 0 {
-		t.Fatal("not the HTTPResponseStatus we expected")
-	}
-	if tk.HTTPResponseBody != "" {
-		t.Fatal("not the HTTPResponseBody we expected")
-	}
-}
-
-func TestGetterWithCancelledContextUnknownResolverURL(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel() // faily immediately
-	g := urlgetter.Getter{
-		Config: urlgetter.Config{
-			ResolverURL: "antani://8.8.8.8:53",
-		},
-		Session: &mockable.Session{},
-		Target:  "https://www.google.com",
-	}
-	tk, err := g.Get(ctx)
-	if err == nil || err.Error() != "unknown_failure: unsupported resolver scheme" {
-		t.Fatal("not the error we expected")
-	}
-	if tk.Agent != "redirect" {
-		t.Fatal("not the Agent we expected")
-	}
-	if tk.BootstrapTime != 0 {
-		t.Fatal("not the BootstrapTime we expected")
-	}
-	if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
-		t.Fatal("not the FailedOperation we expected")
-	}
-	if tk.Failure == nil || *tk.Failure != "unknown_failure: unsupported resolver scheme" {
-		t.Fatal("not the Failure we expected")
-	}
-	if len(tk.NetworkEvents) != 0 {
-		t.Fatal("not the NetworkEvents we expected")
-	}
-	if len(tk.Queries) != 0 {
-		t.Fatal("not the Queries we expected")
-	}
-	if len(tk.TCPConnect) != 0 {
-		t.Fatal("not the TCPConnect we expected")
-	}
-	if len(tk.Requests) != 0 {
-		t.Fatal("not the Requests we expected")
-	}
-	if tk.SOCKSProxy != "" {
-		t.Fatal("not the SOCKSProxy we expected")
-	}
-	if len(tk.TLSHandshakes) != 0 {
-		t.Fatal("not the TLSHandshakes we expected")
-	}
-	if tk.Tunnel != "" {
-		t.Fatal("not the Tunnel we expected")
-	}
-	if tk.HTTPResponseStatus != 0 {
-		t.Fatal("not the HTTPResponseStatus we expected")
-	}
-	if tk.HTTPResponseBody != "" {
-		t.Fatal("not the HTTPResponseBody we expected")
-	}
-}
-
-func TestGetterIntegrationHTTPS(t *testing.T) {
 	ctx := context.Background()
-	g := urlgetter.Getter{
+	g := Getter{
-		Config: urlgetter.Config{
+		Config: Config{
 			NoFollowRedirects: true, // reduce number of events
-		},
+			Tunnel:            "fake",
-		Session: &mockable.Session{},
-		Target:  "https://www.google.com",
-	}
-	tk, err := g.Get(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if tk.Agent != "agent" {
-		t.Fatal("not the Agent we expected")
-	}
-	if tk.BootstrapTime != 0 {
-		t.Fatal("not the BootstrapTime we expected")
-	}
-	if tk.FailedOperation != nil {
-		t.Fatal("not the FailedOperation we expected")
-	}
-	if tk.Failure != nil {
-		t.Fatal("not the Failure we expected")
-	}
-	var (
-		httpTransactionStart     bool
-		httpRequestMetadata      bool
-		resolveStart             bool
-		resolveDone              bool
-		connect                  bool
-		tlsHandshakeStart        bool
-		tlsHandshakeDone         bool
-		httpWroteHeaders         bool
-		httpWroteRequest         bool
-		httpFirstResponseByte    bool
-		httpResponseMetadata     bool
-		httpResponseBodySnapshot bool
-		httpTransactionDone      bool
-	)
-	for _, ev := range tk.NetworkEvents {
-		switch ev.Operation {
-		case "http_transaction_start":
-			httpTransactionStart = true
-		case "http_request_metadata":
-			httpRequestMetadata = true
-		case "resolve_start":
-			resolveStart = true
-		case "resolve_done":
-			resolveDone = true
-		case errorx.ConnectOperation:
-			connect = true
-		case "tls_handshake_start":
-			tlsHandshakeStart = true
-		case "tls_handshake_done":
-			tlsHandshakeDone = true
-		case "http_wrote_headers":
-			httpWroteHeaders = true
-		case "http_wrote_request":
-			httpWroteRequest = true
-		case "http_first_response_byte":
-			httpFirstResponseByte = true
-		case "http_response_metadata":
-			httpResponseMetadata = true
-		case "http_response_body_snapshot":
-			httpResponseBodySnapshot = true
-		case "http_transaction_done":
-			httpTransactionDone = true
-		}
-	}
-	ok := true
-	ok = ok && httpTransactionStart
-	ok = ok && httpRequestMetadata
-	ok = ok && resolveStart
-	ok = ok && resolveDone
-	ok = ok && connect
-	ok = ok && tlsHandshakeStart
-	ok = ok && tlsHandshakeDone
-	ok = ok && httpWroteHeaders
-	ok = ok && httpWroteRequest
-	ok = ok && httpFirstResponseByte
-	ok = ok && httpResponseMetadata
-	ok = ok && httpResponseBodySnapshot
-	ok = ok && httpTransactionDone
-	if !ok {
-		t.Fatal("not the NetworkEvents we expected")
-	}
-	if len(tk.Queries) != 2 {
-		t.Fatal("not the Queries we expected")
-	}
-	if len(tk.TCPConnect) != 1 {
-		t.Fatal("not the TCPConnect we expected")
-	}
-	if len(tk.Requests) != 1 {
-		t.Fatal("not the Requests we expected")
-	}
-	if tk.Requests[0].Request.Method != "GET" {
-		t.Fatal("not the Method we expected")
-	}
-	if tk.Requests[0].Request.URL != "https://www.google.com" {
-		t.Fatal("not the URL we expected")
-	}
-	if tk.SOCKSProxy != "" {
-		t.Fatal("not the SOCKSProxy we expected")
-	}
-	if len(tk.TLSHandshakes) != 1 {
-		t.Fatal("not the TLSHandshakes we expected")
-	}
-	if tk.Tunnel != "" {
-		t.Fatal("not the Tunnel we expected")
-	}
-	if tk.HTTPResponseStatus != 200 {
-		t.Fatal("not the HTTPResponseStatus we expected")
-	}
-	if len(tk.HTTPResponseBody) <= 0 {
-		t.Fatal("not the HTTPResponseBody we expected")
-	}
-}
-
-func TestGetterIntegrationRedirect(t *testing.T) {
-	ctx := context.Background()
-	g := urlgetter.Getter{
-		Config:  urlgetter.Config{NoFollowRedirects: true},
-		Session: &mockable.Session{},
-		Target:  "http://web.whatsapp.com",
-	}
-	tk, err := g.Get(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if tk.HTTPResponseStatus != 302 {
-		t.Fatal("unexpected status code")
-	}
-	if len(tk.HTTPResponseLocations) != 1 {
-		t.Fatal("missing redirect URL")
-	}
-	if tk.HTTPResponseLocations[0] != "https://web.whatsapp.com/" {
-		t.Fatal("invalid redirect URL")
-	}
-}
-
-func TestGetterIntegrationTLSHandshake(t *testing.T) {
-	ctx := context.Background()
-	g := urlgetter.Getter{
-		Config: urlgetter.Config{
-			NoFollowRedirects: true, // reduce number of events
-		},
-		Session: &mockable.Session{},
-		Target:  "tlshandshake://www.google.com:443",
-	}
-	tk, err := g.Get(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if tk.Agent != "agent" {
-		t.Fatal("not the Agent we expected")
-	}
-	if tk.BootstrapTime != 0 {
-		t.Fatal("not the BootstrapTime we expected")
-	}
-	if tk.FailedOperation != nil {
-		t.Fatal("not the FailedOperation we expected")
-	}
-	if tk.Failure != nil {
-		t.Fatal("not the Failure we expected")
-	}
-	var (
-		httpTransactionStart     bool
-		httpRequestMetadata      bool
-		resolveStart             bool
-		resolveDone              bool
-		connect                  bool
-		tlsHandshakeStart        bool
-		tlsHandshakeDone         bool
-		httpWroteHeaders         bool
-		httpWroteRequest         bool
-		httpFirstResponseByte    bool
-		httpResponseMetadata     bool
-		httpResponseBodySnapshot bool
-		httpTransactionDone      bool
-	)
-	for _, ev := range tk.NetworkEvents {
-		switch ev.Operation {
-		case "http_transaction_start":
-			httpTransactionStart = true
-		case "http_request_metadata":
-			httpRequestMetadata = true
-		case "resolve_start":
-			resolveStart = true
-		case "resolve_done":
-			resolveDone = true
-		case errorx.ConnectOperation:
-			connect = true
-		case "tls_handshake_start":
-			tlsHandshakeStart = true
-		case "tls_handshake_done":
-			tlsHandshakeDone = true
-		case "http_wrote_headers":
-			httpWroteHeaders = true
-		case "http_wrote_request":
-			httpWroteRequest = true
-		case "http_first_response_byte":
-			httpFirstResponseByte = true
-		case "http_response_metadata":
-			httpResponseMetadata = true
-		case "http_response_body_snapshot":
-			httpResponseBodySnapshot = true
-		case "http_transaction_done":
-			httpTransactionDone = true
-		}
-	}
-	ok := true
-	ok = ok && !httpTransactionStart
-	ok = ok && !httpRequestMetadata
-	ok = ok && resolveStart
-	ok = ok && resolveDone
-	ok = ok && connect
-	ok = ok && tlsHandshakeStart
-	ok = ok && tlsHandshakeDone
-	ok = ok && !httpWroteHeaders
-	ok = ok && !httpWroteRequest
-	ok = ok && !httpFirstResponseByte
-	ok = ok && !httpResponseMetadata
-	ok = ok && !httpResponseBodySnapshot
-	ok = ok && !httpTransactionDone
-	if !ok {
-		t.Fatal("not the NetworkEvents we expected")
-	}
-	if len(tk.Queries) != 2 {
-		t.Fatal("not the Queries we expected")
-	}
-	if len(tk.TCPConnect) != 1 {
-		t.Fatal("not the TCPConnect we expected")
-	}
-	if len(tk.Requests) != 0 {
-		t.Fatal("not the Requests we expected")
-	}
-	if tk.SOCKSProxy != "" {
-		t.Fatal("not the SOCKSProxy we expected")
-	}
-	if len(tk.TLSHandshakes) != 1 {
-		t.Fatal("not the TLSHandshakes we expected")
-	}
-	if tk.Tunnel != "" {
-		t.Fatal("not the Tunnel we expected")
-	}
-	if tk.HTTPResponseStatus != 0 {
-		t.Fatal("not the HTTPResponseStatus we expected")
-	}
-	if tk.HTTPResponseBody != "" {
-		t.Fatal("not the HTTPResponseBody we expected")
-	}
-}
-
-func TestGetterIntegrationHTTPSWithTunnel(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skip test in short mode")
-	}
-	// TODO(bassosimone): this test is broken. It now requires a
-	// real Session to work as intended. We didn't notice until now
-	// because integration tests do not run for every PR.
-	ctx := context.Background()
-	g := urlgetter.Getter{
-		Config: urlgetter.Config{
-			NoFollowRedirects: true, // reduce number of events
-			Tunnel:            "psiphon",
 		},
 		Session: &mockable.Session{
 			MockableHTTPClient: http.DefaultClient,
 			MockableLogger:     log.Log,
 		},
 		Target: "https://www.google.com",
+		testIOUtilTempDir: func(dir, pattern string) (string, error) {
+			return "", expected
+		},
 	}
 	tk, err := g.Get(ctx)
-	if err != nil {
+	if !errors.Is(err, expected) {
-		t.Fatal(err)
+		t.Fatal("not the error we expected", err)
 	}
 	if tk.Agent != "agent" {
 		t.Fatal("not the Agent we expected")
 	}
-	if tk.BootstrapTime <= 0 {
+	if tk.BootstrapTime != 0 {
 		t.Fatal("not the BootstrapTime we expected")
 	}
-	if tk.FailedOperation != nil {
+	if tk.FailedOperation == nil || *tk.FailedOperation != "top_level" {
 		t.Fatal("not the FailedOperation we expected")
 	}
-	if tk.Failure != nil {
+	if tk.Failure == nil || *tk.Failure != "unknown_failure: mocked error" {
 		t.Fatal("not the Failure we expected")
 	}
-	var (
+	if len(tk.NetworkEvents) != 0 {
-		httpTransactionStart     bool
+		t.Fatal("not the NetworkEvents we expected")
-		httpRequestMetadata      bool
-		resolveStart             bool
-		resolveDone              bool
-		connect                  bool
-		tlsHandshakeStart        bool
-		tlsHandshakeDone         bool
-		httpWroteHeaders         bool
-		httpWroteRequest         bool
-		httpFirstResponseByte    bool
-		httpResponseMetadata     bool
-		httpResponseBodySnapshot bool
-		httpTransactionDone      bool
-	)
-	for _, ev := range tk.NetworkEvents {
-		switch ev.Operation {
-		case "http_transaction_start":
-			httpTransactionStart = true
-		case "http_request_metadata":
-			httpRequestMetadata = true
-		case "resolve_start":
-			resolveStart = true
-		case "resolve_done":
-			resolveDone = true
-		case errorx.ConnectOperation:
-			connect = true
-		case "tls_handshake_start":
-			tlsHandshakeStart = true
-		case "tls_handshake_done":
-			tlsHandshakeDone = true
-		case "http_wrote_headers":
-			httpWroteHeaders = true
-		case "http_wrote_request":
-			httpWroteRequest = true
-		case "http_first_response_byte":
-			httpFirstResponseByte = true
-		case "http_response_metadata":
-			httpResponseMetadata = true
-		case "http_response_body_snapshot":
-			httpResponseBodySnapshot = true
-		case "http_transaction_done":
-			httpTransactionDone = true
-		}
-	}
-	ok := true
-	ok = ok && httpTransactionStart
-	ok = ok && httpRequestMetadata
-	ok = ok && resolveStart == false
-	ok = ok && resolveDone == false
-	ok = ok && connect
-	ok = ok && tlsHandshakeStart
-	ok = ok && tlsHandshakeDone
-	ok = ok && httpWroteHeaders
-	ok = ok && httpWroteRequest
-	ok = ok && httpFirstResponseByte
-	ok = ok && httpResponseMetadata
-	ok = ok && httpResponseBodySnapshot
-	ok = ok && httpTransactionDone
-	if !ok {
-		t.Fatalf("not the NetworkEvents we expected: %+v", tk.NetworkEvents)
 	}
 	if len(tk.Queries) != 0 {
 		t.Fatal("not the Queries we expected")
 	}
-	if len(tk.TCPConnect) != 1 {
+	if len(tk.TCPConnect) != 0 {
 		t.Fatal("not the TCPConnect we expected")
 	}
-	if len(tk.Requests) != 1 {
+	if len(tk.Requests) != 0 {
 		t.Fatal("not the Requests we expected")
 	}
-	if tk.Requests[0].Request.Method != "GET" {
+	if tk.SOCKSProxy != "" {
-		t.Fatal("not the Method we expected")
-	}
-	if tk.Requests[0].Request.URL != "https://www.google.com" {
-		t.Fatal("not the URL we expected")
-	}
-	if tk.SOCKSProxy == "" {
 		t.Fatal("not the SOCKSProxy we expected")
 	}
-	if len(tk.TLSHandshakes) != 1 {
+	if len(tk.TLSHandshakes) != 0 {
 		t.Fatal("not the TLSHandshakes we expected")
 	}
-	if tk.Tunnel != "psiphon" {
+	if tk.Tunnel != "fake" {
 		t.Fatal("not the Tunnel we expected")
 	}
-	if tk.HTTPResponseStatus != 200 {
+	if tk.HTTPResponseStatus != 0 {
 		t.Fatal("not the HTTPResponseStatus we expected")
 	}
-	if len(tk.HTTPResponseBody) <= 0 {
+	if len(tk.HTTPResponseBody) != 0 {
 		t.Fatal("not the HTTPResponseBody we expected")
 	}
 }

internal/engine/tunnel/fake.go

@@ -40,6 +40,16 @@ func (t *fakeTunnel) SOCKS5ProxyURL() *url.URL {
 
 // fakeStart starts the fake tunnel.
 func fakeStart(ctx context.Context, config *Config) (Tunnel, error) {
+	// do the same things other tunnels do:
+	//
+	// 1. abort if context is cancelled
+	//
+	// 2. check for tunnelDir being not empty
+	//
+	// 3. attempt to create tunnelDir
+	//
+	// after that, it's all fake and we just create a simple
+	// socks5 server that we can use
 	select {
 	case <-ctx.Done():
 		return nil, ctx.Err() // simplifies unit testing this code
@@ -48,6 +58,9 @@ func fakeStart(ctx context.Context, config *Config) (Tunnel, error) {
 	if config.TunnelDir == "" {
 		return nil, ErrEmptyTunnelDir
 	}
+	if err := config.mkdirAll(config.TunnelDir, 0700); err != nil {
+		return nil, err
+	}
 	server, err := config.socks5New(&socks5.Config{})
 	if err != nil {
 		return nil, err

internal/engine/tunnel/fake_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"net"
+	"os"
 	"testing"
 
 	"github.com/armon/go-socks5"
@@ -41,6 +42,25 @@ func TestFakeWithEmptyTunnelDir(t *testing.T) {
 	}
 }
 
+func TestFakeWithFailingMkdirAll(t *testing.T) {
+	expected := errors.New("mocked error")
+	ctx := context.Background()
+	sess := &mockable.Session{}
+	tunnel, err := fakeStart(ctx, &Config{
+		Session:   sess,
+		TunnelDir: "testdata",
+		testMkdirAll: func(dir string, mode os.FileMode) error {
+			return expected
+		},
+	})
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+	if tunnel != nil {
+		t.Fatal("expected nil tunnel here")
+	}
+}
+
 func TestFakeSocks5NewFails(t *testing.T) {
 	expected := errors.New("mocked error")
 	ctx := context.Background()