ooni-probe-cli/internal/engine/experiment/urlgetter/runner_test.go

package urlgetter_test

import (
	"context"
	"errors"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/ooni/probe-cli/v3/internal/atomicx"
	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
	"github.com/ooni/probe-cli/v3/internal/model"
)

func TestRunnerWithInvalidURLScheme(t *testing.T) {
	r := urlgetter.Runner{Target: "antani://www.google.com"}
	err := r.Run(context.Background())
	if err == nil || err.Error() != "unknown targetURL scheme" {
		t.Fatal("not the error we expected")
	}
}

func TestRunnerHTTPWithContextCanceled(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	r := urlgetter.Runner{Target: "https://www.google.com"}
	err := r.Run(ctx)
	if !errors.Is(err, context.Canceled) {
		t.Fatal("not the error we expected")
	}
}

func TestRunnerDNSLookupWithContextCanceled(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	r := urlgetter.Runner{Target: "dnslookup://www.google.com"}
	err := r.Run(ctx)
	if err == nil || err.Error() != "interrupted" {
		t.Fatal("not the error we expected")
	}
}

func TestRunnerTLSHandshakeWithContextCanceled(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	r := urlgetter.Runner{Target: "tlshandshake://www.google.com:443"}
	err := r.Run(ctx)
	if err == nil || err.Error() != "interrupted" {
		t.Fatal("not the error we expected")
	}
}

func TestRunnerTCPConnectWithContextCanceled(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	r := urlgetter.Runner{Target: "tcpconnect://www.google.com:443"}
	err := r.Run(ctx)
	if err == nil || err.Error() != "interrupted" {
		t.Fatal("not the error we expected")
	}
}

func TestRunnerWithInvalidURL(t *testing.T) {
	r := urlgetter.Runner{Target: "\t"}
	err := r.Run(context.Background())
	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
		t.Fatal("not the error we expected")
	}
}

func TestRunnerWithEmptyHostname(t *testing.T) {
	r := urlgetter.Runner{Target: "http:///foo.txt"}
	err := r.Run(context.Background())
	if err == nil || !strings.HasSuffix(err.Error(), "no Host in request URL") {
		t.Fatal("not the error we expected")
	}
}

func TestRunnerTLSHandshakeSuccess(t *testing.T) {
	if testing.Short() {
		t.Skip("skip test in short mode")
	}
	r := urlgetter.Runner{Target: "tlshandshake://www.google.com:443"}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
}

func TestRunnerTCPConnectSuccess(t *testing.T) {
	if testing.Short() {
		t.Skip("skip test in short mode")
	}
	r := urlgetter.Runner{Target: "tcpconnect://www.google.com:443"}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
}

func TestRunnerDNSLookupSuccess(t *testing.T) {
	if testing.Short() {
		t.Skip("skip test in short mode")
	}
	r := urlgetter.Runner{Target: "dnslookup://www.google.com"}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
}

func TestRunnerHTTPSSuccess(t *testing.T) {
	if testing.Short() {
		t.Skip("skip test in short mode")
	}
	r := urlgetter.Runner{Target: "https://www.google.com"}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
}

func TestRunnerHTTPSetHostHeader(t *testing.T) {
	var host string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		host = r.Host
		w.WriteHeader(200)
	}))
	defer server.Close()
	r := urlgetter.Runner{
		Config: urlgetter.Config{
			HTTPHost: "x.org",
		},
		Target: server.URL,
	}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if host != "x.org" {
		t.Fatal("not the host we expected")
	}
}

func TestRunnerHTTPNoRedirect(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Location", "http:///") // cause failure if we redirect
		w.WriteHeader(302)
	}))
	defer server.Close()
	r := urlgetter.Runner{
		Config: urlgetter.Config{
			NoFollowRedirects: true,
		},
		Target: server.URL,
	}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
}

func TestRunnerHTTPWithConnectionClosedByServer(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		hijacker, ok := w.(http.Hijacker)
		if !ok {
			panic("hijacking not supported by this server")
		}
		conn, _, _ := hijacker.Hijack()
		conn.Write([]byte("HTTP/1.1 200 Ok\r\n"))
		conn.Write([]byte("Content-Length: 1024\r\n"))
		conn.Write([]byte("\r\n"))
		conn.Write([]byte("123456789"))
		conn.Close()
	}))
	defer server.Close()
	r := urlgetter.Runner{
		Config: urlgetter.Config{
			NoFollowRedirects: true,
		},
		Target: server.URL,
	}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
}

func TestRunnerHTTPWeHandle400Correctly(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(400)
	}))
	defer server.Close()
	r := urlgetter.Runner{
		Config: urlgetter.Config{
			FailOnHTTPError:   true,
			NoFollowRedirects: true,
		},
		Target: server.URL,
	}
	err := r.Run(context.Background())
	if !errors.Is(err, urlgetter.ErrHTTPRequestFailed) {
		t.Fatal("not the error we expected")
	}
}

func TestRunnerHTTPWithConnectionClosedByServerAnd400(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		hijacker, ok := w.(http.Hijacker)
		if !ok {
			panic("hijacking not supported by this server")
		}
		conn, _, _ := hijacker.Hijack()
		conn.Write([]byte("HTTP/1.1 400 Bad Request\r\n"))
		conn.Write([]byte("Content-Length: 1024\r\n"))
		conn.Write([]byte("\r\n"))
		conn.Write([]byte("123456789"))
		conn.Close()
	}))
	defer server.Close()
	r := urlgetter.Runner{
		Config: urlgetter.Config{
			FailOnHTTPError:   true,
			NoFollowRedirects: true,
		},
		Target: server.URL,
	}
	err := r.Run(context.Background())
	if !errors.Is(err, urlgetter.ErrHTTPRequestFailed) {
		t.Fatal("not the error we expected", err)
	}
}

func TestRunnerWeCanForceUserAgent(t *testing.T) {
	expected := "antani/1.23.4-dev"
	found := &atomicx.Int64{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("User-Agent") == expected {
			found.Add(1)
		}
		w.WriteHeader(200)
	}))
	defer server.Close()
	r := urlgetter.Runner{
		Config: urlgetter.Config{
			FailOnHTTPError:   true,
			NoFollowRedirects: true,
			UserAgent:         expected,
		},
		Target: server.URL,
	}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if found.Load() != 1 {
		t.Fatal("we didn't override the user agent")
	}
}

func TestRunnerDefaultUserAgent(t *testing.T) {
	expected := model.HTTPHeaderUserAgent
	found := &atomicx.Int64{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("User-Agent") == expected {
			found.Add(1)
		}
		w.WriteHeader(200)
	}))
	defer server.Close()
	r := urlgetter.Runner{
		Config: urlgetter.Config{
			FailOnHTTPError:   true,
			NoFollowRedirects: true,
		},
		Target: server.URL,
	}
	err := r.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if found.Load() != 1 {
		t.Fatal("we didn't override the user agent")
	}
}