b5da8be183
* chore(netxlite): add currently failing test case This diff introduces a test cases that will fail because of the reason explained in https://github.com/ooni/probe/issues/1965. * chore(netxlite/iox_test.go): add failing unit tests These tests directly show how the Go implementation of ReadAll and Copy has the issue of checking for io.EOF equality. * fix(netxlite): make {ReadAll,Copy}Context robust to wrapped io.EOF The fix is simple: we just need to check for `errors.Is(err, io.EOF)` after either io.ReadAll or io.Copy has returned. When this condition is true, we need to convert the error back to `nil` as it ought to be. While there, observe that the unit tests I committed in the previous commit are wrongly asserting that the error must be wrapped. This assertion is not correct, because in both cases we have just ensured that the returned error is `nil` (i.e., success). See https://github.com/ooni/probe/issues/1965. * cleanup: remove previous workaround for wrapped io.EOF These workarounds were partial, meaning that they would cover some cases in which the issue occurred but not all of them. Handling the problem in `netxlite.{ReadAll,Copy}Context` is the right thing to do _as long as_ we always use these functions instead of `io.{ReadAll,Copy}`. This is why it's now important to ensure we clearly mention that inside of the `CONTRIBUTING.md` guide and to also ensure that we're not using these functions in the code base. * fix(urlgetter): repair tests who assumed to see EOF error Now that we have established that we should normalize EOF when reading bodies like the stdlib does and now that it's clear why our behavior diverged from the stdlib, we also need to repair all the tests that assumed this incorrect behavior. * fix(all): don't use io{,util}.{Copy,ReadAll} * feat: add checks to ensure we don't use io.{Copy,ReadAll} * doc(netxlite): document we know how to deal w/ wrapped io.EOF * fix(nocopyreadall.bash): add exception for i/n/iox.go
287 lines
7.2 KiB
Go
287 lines
7.2 KiB
Go
package urlgetter_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/ooni/probe-cli/v3/internal/atomicx"
|
|
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
|
|
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
|
)
|
|
|
|
func TestRunnerWithInvalidURLScheme(t *testing.T) {
|
|
r := urlgetter.Runner{Target: "antani://www.google.com"}
|
|
err := r.Run(context.Background())
|
|
if err == nil || err.Error() != "unknown targetURL scheme" {
|
|
t.Fatal("not the error we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerHTTPWithContextCanceled(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
r := urlgetter.Runner{Target: "https://www.google.com"}
|
|
err := r.Run(ctx)
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatal("not the error we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerDNSLookupWithContextCanceled(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
r := urlgetter.Runner{Target: "dnslookup://www.google.com"}
|
|
err := r.Run(ctx)
|
|
if err == nil || err.Error() != "interrupted" {
|
|
t.Fatal("not the error we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerTLSHandshakeWithContextCanceled(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
r := urlgetter.Runner{Target: "tlshandshake://www.google.com:443"}
|
|
err := r.Run(ctx)
|
|
if err == nil || err.Error() != "interrupted" {
|
|
t.Fatal("not the error we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerTCPConnectWithContextCanceled(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
r := urlgetter.Runner{Target: "tcpconnect://www.google.com:443"}
|
|
err := r.Run(ctx)
|
|
if err == nil || err.Error() != "interrupted" {
|
|
t.Fatal("not the error we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerWithInvalidURL(t *testing.T) {
|
|
r := urlgetter.Runner{Target: "\t"}
|
|
err := r.Run(context.Background())
|
|
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
|
|
t.Fatal("not the error we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerWithEmptyHostname(t *testing.T) {
|
|
r := urlgetter.Runner{Target: "http:///foo.txt"}
|
|
err := r.Run(context.Background())
|
|
if err == nil || !strings.HasSuffix(err.Error(), "no Host in request URL") {
|
|
t.Fatal("not the error we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerTLSHandshakeSuccess(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skip test in short mode")
|
|
}
|
|
r := urlgetter.Runner{Target: "tlshandshake://www.google.com:443"}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerTCPConnectSuccess(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skip test in short mode")
|
|
}
|
|
r := urlgetter.Runner{Target: "tcpconnect://www.google.com:443"}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerDNSLookupSuccess(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skip test in short mode")
|
|
}
|
|
r := urlgetter.Runner{Target: "dnslookup://www.google.com"}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerHTTPSSuccess(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("skip test in short mode")
|
|
}
|
|
r := urlgetter.Runner{Target: "https://www.google.com"}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerHTTPSetHostHeader(t *testing.T) {
|
|
var host string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
host = r.Host
|
|
w.WriteHeader(200)
|
|
}))
|
|
defer server.Close()
|
|
r := urlgetter.Runner{
|
|
Config: urlgetter.Config{
|
|
HTTPHost: "x.org",
|
|
},
|
|
Target: server.URL,
|
|
}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if host != "x.org" {
|
|
t.Fatal("not the host we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerHTTPNoRedirect(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Add("Location", "http:///") // cause failure if we redirect
|
|
w.WriteHeader(302)
|
|
}))
|
|
defer server.Close()
|
|
r := urlgetter.Runner{
|
|
Config: urlgetter.Config{
|
|
NoFollowRedirects: true,
|
|
},
|
|
Target: server.URL,
|
|
}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerHTTPWithConnectionClosedByServer(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
hijacker, ok := w.(http.Hijacker)
|
|
if !ok {
|
|
panic("hijacking not supported by this server")
|
|
}
|
|
conn, _, _ := hijacker.Hijack()
|
|
conn.Write([]byte("HTTP/1.1 200 Ok\r\n"))
|
|
conn.Write([]byte("Content-Length: 1024\r\n"))
|
|
conn.Write([]byte("\r\n"))
|
|
conn.Write([]byte("123456789"))
|
|
conn.Close()
|
|
}))
|
|
defer server.Close()
|
|
r := urlgetter.Runner{
|
|
Config: urlgetter.Config{
|
|
NoFollowRedirects: true,
|
|
},
|
|
Target: server.URL,
|
|
}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerHTTPWeHandle400Correctly(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(400)
|
|
}))
|
|
defer server.Close()
|
|
r := urlgetter.Runner{
|
|
Config: urlgetter.Config{
|
|
FailOnHTTPError: true,
|
|
NoFollowRedirects: true,
|
|
},
|
|
Target: server.URL,
|
|
}
|
|
err := r.Run(context.Background())
|
|
if !errors.Is(err, urlgetter.ErrHTTPRequestFailed) {
|
|
t.Fatal("not the error we expected")
|
|
}
|
|
}
|
|
|
|
func TestRunnerHTTPWithConnectionClosedByServerAnd400(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
hijacker, ok := w.(http.Hijacker)
|
|
if !ok {
|
|
panic("hijacking not supported by this server")
|
|
}
|
|
conn, _, _ := hijacker.Hijack()
|
|
conn.Write([]byte("HTTP/1.1 400 Bad Request\r\n"))
|
|
conn.Write([]byte("Content-Length: 1024\r\n"))
|
|
conn.Write([]byte("\r\n"))
|
|
conn.Write([]byte("123456789"))
|
|
conn.Close()
|
|
}))
|
|
defer server.Close()
|
|
r := urlgetter.Runner{
|
|
Config: urlgetter.Config{
|
|
FailOnHTTPError: true,
|
|
NoFollowRedirects: true,
|
|
},
|
|
Target: server.URL,
|
|
}
|
|
err := r.Run(context.Background())
|
|
if !errors.Is(err, urlgetter.ErrHTTPRequestFailed) {
|
|
t.Fatal("not the error we expected", err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerWeCanForceUserAgent(t *testing.T) {
|
|
expected := "antani/1.23.4-dev"
|
|
found := &atomicx.Int64{}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Header.Get("User-Agent") == expected {
|
|
found.Add(1)
|
|
}
|
|
w.WriteHeader(200)
|
|
}))
|
|
defer server.Close()
|
|
r := urlgetter.Runner{
|
|
Config: urlgetter.Config{
|
|
FailOnHTTPError: true,
|
|
NoFollowRedirects: true,
|
|
UserAgent: expected,
|
|
},
|
|
Target: server.URL,
|
|
}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if found.Load() != 1 {
|
|
t.Fatal("we didn't override the user agent")
|
|
}
|
|
}
|
|
|
|
func TestRunnerDefaultUserAgent(t *testing.T) {
|
|
expected := httpheader.UserAgent()
|
|
found := &atomicx.Int64{}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Header.Get("User-Agent") == expected {
|
|
found.Add(1)
|
|
}
|
|
w.WriteHeader(200)
|
|
}))
|
|
defer server.Close()
|
|
r := urlgetter.Runner{
|
|
Config: urlgetter.Config{
|
|
FailOnHTTPError: true,
|
|
NoFollowRedirects: true,
|
|
},
|
|
Target: server.URL,
|
|
}
|
|
err := r.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if found.Load() != 1 {
|
|
t.Fatal("we didn't override the user agent")
|
|
}
|
|
}
|