c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(httpx): use mocks to implement tests (#650)

Browse Source 
* refactor(httpx): use mocks to implement tests

While there, make sure no test depends on external services by
replacing such tests with httptest.

See https://github.com/ooni/probe/issues/1951.

* fix(httpx): ensure we honour the context
This commit is contained in:
Simone Basso 2022-01-05 16:13:42 +01:00 committed by GitHub
parent 93f084598e
commit ee0aa18616
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 332 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
internal/engine/httpx/fake_test.go
View File

@ -1,45 +0,0 @@
package httpx
import (
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
type FakeTransport struct {
Err error
Func func(*http.Request) (*http.Response, error)
Resp *http.Response
}
func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond)
if txp.Func != nil {
return txp.Func(req)
}
if req.Body != nil {
netxlite.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {
return nil, txp.Err
}
txp.Resp.Request = req // non thread safe but it doesn't matter
return txp.Resp, nil
}
func (txp FakeTransport) CloseIdleConnections() {}
type FakeBody struct {
Err error
}
func (fb FakeBody) Read(p []byte) (int, error) {
time.Sleep(10 * time.Microsecond)
return 0, fb.Err
}
func (fb FakeBody) Close() error {
return nil
}

4
internal/engine/httpx/httpx.go
View File

@ -137,7 +137,7 @@ func (c *apiClient) newRequest(ctx context.Context, method, resourcePath string,
if query != nil { if query != nil {
URL.RawQuery = query.Encode() URL.RawQuery = query.Encode()
} }
request, err := http.NewRequest(method, URL.String(), body) request, err := http.NewRequestWithContext(ctx, method, URL.String(), body)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -149,7 +149,7 @@ func (c *apiClient) newRequest(ctx context.Context, method, resourcePath string,
request.Header.Set("Accept", c.Accept) request.Header.Set("Accept", c.Accept)
} }
request.Header.Set("User-Agent", c.UserAgent) request.Header.Set("User-Agent", c.UserAgent)
return request.WithContext(ctx), nil return request, nil
} }
// ErrRequestFailed indicates that the server returned >= 400. // ErrRequestFailed indicates that the server returned >= 400.

632
internal/engine/httpx/httpx_test.go
View File

@ -2,7 +2,9 @@ package httpx
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt"
"io" "io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -10,13 +12,16 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/apex/log"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/fakefill" "github.com/ooni/probe-cli/v3/internal/fakefill"
"github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/version"
) )
const userAgent = "miniooni/0.1.0-dev" // userAgent is the user agent used by this test suite
var userAgent = fmt.Sprintf("ooniprobe-cli/%s", version.Version)
func TestAPIClientTemplate(t *testing.T) { func TestAPIClientTemplate(t *testing.T) {
t.Run("normal constructor", func(t *testing.T) { t.Run("normal constructor", func(t *testing.T) {
@ -63,331 +68,354 @@ func TestAPIClientTemplate(t *testing.T) {
}) })
} }
func newClient() *apiClient { // newAPIClient is an helper factory creating a client for testing.
func newAPIClient() *apiClient {
return &apiClient{ return &apiClient{
BaseURL: "https://httpbin.org", BaseURL: "https://example.com",
HTTPClient: http.DefaultClient, HTTPClient: http.DefaultClient,
Logger: log.Log, Logger: model.DiscardLogger,
UserAgent: userAgent, UserAgent: userAgent,
} }
} }
func TestNewRequestWithJSONBodyJSONMarshalFailure(t *testing.T) { // fakeRequest is a fake request we serialize.
client := newClient() type fakeRequest struct {
req, err := client.newRequestWithJSONBody( Name string
context.Background(), "GET", "/", nil, make(chan interface{}), Age int
) Sleeping bool
if err == nil || !strings.HasPrefix(err.Error(), "json: unsupported type") { Attributes map[string][]string
t.Fatal("not the error we expected")
}
if req != nil {
t.Fatal("expected nil request here")
}
} }
func TestNewRequestWithJSONBodyNewRequestFailure(t *testing.T) { func TestAPIClient(t *testing.T) {
client := newClient() t.Run("newRequestWithJSONBody", func(t *testing.T) {
client.BaseURL = "\t\t\t" // cause URL parse error t.Run("JSON marshal failure", func(t *testing.T) {
req, err := client.newRequestWithJSONBody( client := newAPIClient()
context.Background(), "GET", "/", nil, nil, req, err := client.newRequestWithJSONBody(
) context.Background(), "GET", "/", nil, make(chan interface{}),
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") { )
t.Fatal("not the error we expected") if err == nil || !strings.HasPrefix(err.Error(), "json: unsupported type") {
} t.Fatal("not the error we expected", err)
if req != nil { }
t.Fatal("expected nil request here") if req != nil {
} t.Fatal("expected nil request here")
} }
})
func TestNewRequestWithQuery(t *testing.T) { t.Run("newRequest failure", func(t *testing.T) {
client := newClient() client := newAPIClient()
q := url.Values{} client.BaseURL = "\t\t\t" // cause URL parse error
q.Add("antani", "mascetti") req, err := client.newRequestWithJSONBody(
q.Add("melandri", "conte") context.Background(), "GET", "/", nil, nil,
req, err := client.newRequest( )
context.Background(), "GET", "/", q, nil, if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
) t.Fatal("not the error we expected")
if err != nil { }
t.Fatal(err) if req != nil {
} t.Fatal("expected nil request here")
if req.URL.Query().Get("antani") != "mascetti" { }
t.Fatal("expected different query string here") })
}
if req.URL.Query().Get("melandri") != "conte" {
t.Fatal("expected different query string here")
}
}
func TestNewRequestNewRequestFailure(t *testing.T) { t.Run("sets the content-type properly", func(t *testing.T) {
client := newClient() var jsonReq fakeRequest
req, err := client.newRequest( ff := &fakefill.Filler{}
context.Background(), "\t\t\t", "/", nil, nil, ff.Fill(&jsonReq)
) client := newAPIClient()
if err == nil || !strings.HasPrefix(err.Error(), "net/http: invalid method") { req, err := client.newRequestWithJSONBody(
t.Fatal("not the error we expected") context.Background(), "GET", "/", nil, jsonReq,
} )
if req != nil { if err != nil {
t.Fatal("expected nil request here") t.Fatal(err)
} }
} if req.Header.Get("Content-Type") != "application/json" {
t.Fatal("did not set content-type properly")
}
})
})
func TestNewRequestCloudfronting(t *testing.T) { t.Run("newRequest", func(t *testing.T) {
client := newClient() t.Run("with invalid method", func(t *testing.T) {
client.Host = "www.x.org" client := newAPIClient()
req, err := client.newRequest( req, err := client.newRequest(
context.Background(), "GET", "/", nil, nil, context.Background(), "\t\t\t", "/", nil, nil,
) )
if err != nil { if err == nil || !strings.HasPrefix(err.Error(), "net/http: invalid method") {
t.Fatal(err) t.Fatal("not the error we expected")
} }
if req.Host != client.Host { if req != nil {
t.Fatal("expected different req.Host here") t.Fatal("expected nil request here")
} }
} })
func TestNewRequestAcceptIsSet(t *testing.T) { t.Run("with query", func(t *testing.T) {
client := newClient() client := newAPIClient()
client.Accept = "application/xml" q := url.Values{}
req, err := client.newRequestWithJSONBody( q.Add("antani", "mascetti")
context.Background(), "GET", "/", nil, []string{}, q.Add("melandri", "conte")
) req, err := client.newRequest(
if err != nil { context.Background(), "GET", "/", q, nil,
t.Fatal(err) )
} if err != nil {
if req.Header.Get("Accept") != "application/xml" { t.Fatal(err)
t.Fatal("expected different Accept here") }
} if req.URL.Query().Get("antani") != "mascetti" {
} t.Fatal("expected different query string here")
}
if req.URL.Query().Get("melandri") != "conte" {
t.Fatal("expected different query string here")
}
})
func TestNewRequestContentTypeIsSet(t *testing.T) { t.Run("with authorization", func(t *testing.T) {
client := newClient() client := newAPIClient()
req, err := client.newRequestWithJSONBody( client.Authorization = "deadbeef"
context.Background(), "GET", "/", nil, []string{}, req, err := client.newRequest(
) context.Background(), "GET", "/", nil, nil,
if err != nil { )
t.Fatal(err) if err != nil {
} t.Fatal(err)
if req.Header.Get("Content-Type") != "application/json" { }
t.Fatal("expected different Content-Type here") if req.Header.Get("Authorization") != client.Authorization {
} t.Fatal("expected different Authorization here")
} }
})
func TestNewRequestAuthorizationHeader(t *testing.T) { t.Run("with accept", func(t *testing.T) {
client := newClient() client := newAPIClient()
client.Authorization = "deadbeef" client.Accept = "application/xml"
req, err := client.newRequest( req, err := client.newRequestWithJSONBody(
context.Background(), "GET", "/", nil, nil, context.Background(), "GET", "/", nil, []string{},
) )
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if req.Header.Get("Authorization") != client.Authorization { if req.Header.Get("Accept") != "application/xml" {
t.Fatal("expected different Authorization here") t.Fatal("expected different Accept here")
} }
} })
func TestNewRequestUserAgentIsSet(t *testing.T) { t.Run("with custom host header", func(t *testing.T) {
client := newClient() client := newAPIClient()
req, err := client.newRequest( client.Host = "www.x.org"
context.Background(), "GET", "/", nil, nil, req, err := client.newRequest(
) context.Background(), "GET", "/", nil, nil,
if err != nil { )
t.Fatal(err) if err != nil {
} t.Fatal(err)
if req.Header.Get("User-Agent") != userAgent { }
t.Fatal("expected different User-Agent here") if req.Host != client.Host {
} t.Fatal("expected different req.Host here")
} }
})
func TestClientDoJSONClientDoFailure(t *testing.T) { t.Run("with user agent", func(t *testing.T) {
expected := errors.New("mocked error") client := newAPIClient()
client := newClient() req, err := client.newRequest(
client.HTTPClient = &http.Client{Transport: FakeTransport{ context.Background(), "GET", "/", nil, nil,
Err: expected, )
}} if err != nil {
err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil) t.Fatal(err)
if !errors.Is(err, expected) { }
t.Fatal("not the error we expected") if req.Header.Get("User-Agent") != userAgent {
} t.Fatal("expected different User-Agent here")
} }
})
})
func TestClientDoJSONResponseNotSuccessful(t *testing.T) { t.Run("doJSON", func(t *testing.T) {
client := newClient() t.Run("do failure", func(t *testing.T) {
client.HTTPClient = &http.Client{Transport: FakeTransport{ expected := errors.New("mocked error")
Resp: &http.Response{ client := newAPIClient()
StatusCode: 401, client.HTTPClient = &mocks.HTTPClient{
Body: FakeBody{}, MockDo: func(req *http.Request) (*http.Response, error) {
}, return nil, expected
}} },
err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil) }
if err == nil || !strings.HasPrefix(err.Error(), "httpx: request failed") { err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
t.Fatal("not the error we expected") if !errors.Is(err, expected) {
} t.Fatal("not the error we expected")
} }
})
func TestClientDoJSONResponseReadingBodyError(t *testing.T) { t.Run("response is not successful (i.e., >= 400)", func(t *testing.T) {
expected := errors.New("mocked error") client := newAPIClient()
client := newClient() client.HTTPClient = &mocks.HTTPClient{
client.HTTPClient = &http.Client{Transport: FakeTransport{ MockDo: func(req *http.Request) (*http.Response, error) {
Resp: &http.Response{ return &http.Response{
StatusCode: 200, StatusCode: 401,
Body: FakeBody{ Body: io.NopCloser(strings.NewReader("{}")),
Err: expected, }, nil
}, },
}, }
}} err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil) if !errors.Is(err, ErrRequestFailed) {
if !errors.Is(err, expected) { t.Fatal("not the error we expected", err)
t.Fatal("not the error we expected") }
} })
}
func TestClientDoJSONResponseIsNotJSON(t *testing.T) { t.Run("cannot read body", func(t *testing.T) {
client := newClient() expected := errors.New("mocked error")
client.HTTPClient = &http.Client{Transport: FakeTransport{ client := newAPIClient()
Resp: &http.Response{ client.HTTPClient = &mocks.HTTPClient{
StatusCode: 200, MockDo: func(req *http.Request) (*http.Response, error) {
Body: FakeBody{ return &http.Response{
Err: io.EOF, StatusCode: 200,
}, Body: io.NopCloser(&mocks.Reader{
}, MockRead: func(b []byte) (int, error) {
}} return 0, expected
err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil) },
if err == nil || err.Error() != "unexpected end of JSON input" { }),
t.Fatal("not the error we expected") }, nil
} },
} }
err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
})
type httpbinheaders struct { t.Run("response is not JSON", func(t *testing.T) {
Headers map[string]string `json:"headers"` client := newAPIClient()
} client.HTTPClient = &mocks.HTTPClient{
MockDo: func(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: 200,
Body: io.NopCloser(strings.NewReader("[")),
}, nil
},
}
err := client.doJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
if err == nil || err.Error() != "unexpected end of JSON input" {
t.Fatal("not the error we expected")
}
})
})
func TestReadJSONSuccess(t *testing.T) { t.Run("GetJSON", func(t *testing.T) {
var headers httpbinheaders t.Run("successful case", func(t *testing.T) {
err := newClient().GetJSON(context.Background(), "/headers", &headers) server := httptest.NewServer(http.HandlerFunc(
if err != nil { func(w http.ResponseWriter, r *http.Request) {
t.Fatal(err) w.Write([]byte(`["foo", "bar"]`))
} },
if headers.Headers["Host"] != "httpbin.org" { ))
t.Fatal("unexpected Host header") defer server.Close()
} ctx := context.Background()
if headers.Headers["User-Agent"] != "miniooni/0.1.0-dev" { var result []string
t.Fatal("unexpected Host header") err := (&apiClient{
} BaseURL: server.URL,
} HTTPClient: http.DefaultClient,
Logger: model.DiscardLogger,
}).GetJSON(ctx, "/", &result)
if err != nil {
t.Fatal(err)
}
if len(result) != 2 || result[0] != "foo" || result[1] != "bar" {
t.Fatal("invalid result", result)
}
})
type httpbinpost struct { t.Run("failure case", func(t *testing.T) {
Data string `json:"data"` var headers []string
} client := newAPIClient()
client.BaseURL = "\t\t\t\t"
err := client.GetJSON(context.Background(), "/", &headers)
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("not the error we expected")
}
})
})
func TestCreateJSONSuccess(t *testing.T) { t.Run("PostJSON", func(t *testing.T) {
headers := httpbinheaders{ t.Run("successful case", func(t *testing.T) {
Headers: map[string]string{ server := httptest.NewServer(http.HandlerFunc(
"Foo": "bar", func(w http.ResponseWriter, r *http.Request) {
}, var incoming []string
} data, err := netxlite.ReadAllContext(r.Context(), r.Body)
var response httpbinpost if err != nil {
err := newClient().PostJSON(context.Background(), "/post", &headers, &response) w.WriteHeader(500)
if err != nil { return
t.Fatal(err) }
} if err := json.Unmarshal(data, &incoming); err != nil {
if response.Data != `{"headers":{"Foo":"bar"}}` { w.WriteHeader(500)
t.Fatal(response.Data) return
} }
} w.Write(data)
},
))
defer server.Close()
ctx := context.Background()
incoming := []string{"foo", "bar"}
var result []string
err := (&apiClient{
BaseURL: server.URL,
HTTPClient: http.DefaultClient,
Logger: model.DiscardLogger,
}).PostJSON(ctx, "/", incoming, &result)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(incoming, result); diff != "" {
t.Fatal(diff)
}
})
func TestReadJSONFailure(t *testing.T) { t.Run("failure case", func(t *testing.T) {
var headers httpbinheaders incoming := []string{"foo", "bar"}
client := newClient() var result []string
client.BaseURL = "\t\t\t\t" client := newAPIClient()
err := client.GetJSON(context.Background(), "/headers", &headers) client.BaseURL = "\t\t\t\t"
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") { err := client.PostJSON(context.Background(), "/", incoming, &result)
t.Fatal("not the error we expected") if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
} t.Fatal("not the error we expected")
} }
})
})
func TestCreateJSONFailure(t *testing.T) { t.Run("FetchResource", func(t *testing.T) {
var headers httpbinheaders t.Run("successful case", func(t *testing.T) {
client := newClient() server := httptest.NewServer(http.HandlerFunc(
client.BaseURL = "\t\t\t\t" func(w http.ResponseWriter, r *http.Request) {
err := client.PostJSON(context.Background(), "/headers", &headers, &headers) w.Write([]byte("deadbeef"))
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") { },
t.Fatal("not the error we expected") ))
} defer server.Close()
} ctx := context.Background()
data, err := (&apiClient{
BaseURL: server.URL,
HTTPClient: http.DefaultClient,
Logger: model.DiscardLogger,
}).FetchResource(ctx, "/")
if err != nil {
t.Fatal(err)
}
if string(data) != "deadbeef" {
t.Fatal("invalid data")
}
})
func TestFetchResourceIntegration(t *testing.T) { t.Run("failure case", func(t *testing.T) {
log.SetLevel(log.DebugLevel) client := newAPIClient()
ctx := context.Background() client.BaseURL = "\t\t\t\t"
data, err := (&apiClient{ data, err := client.FetchResource(context.Background(), "/")
BaseURL: "http://facebook.com/", if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
HTTPClient: http.DefaultClient, t.Fatal("not the error we expected")
Logger: log.Log, }
UserAgent: "ooniprobe-engine/0.1.0", if data != nil {
}).FetchResource(ctx, "/robots.txt") t.Fatal("unexpected data")
if err != nil { }
t.Fatal(err) })
} })
if len(data) <= 0 {
t.Fatal("Did not expect an empty resource")
}
}
func TestFetchResourceExpiredContext(t *testing.T) { t.Run("we honour context", func(t *testing.T) {
log.SetLevel(log.DebugLevel) // It should suffice to check one of the public methods here
ctx, cancel := context.WithCancel(context.Background()) client := newAPIClient()
cancel() ctx, cancel := context.WithCancel(context.Background())
data, err := (&apiClient{ cancel() // test should fail
BaseURL: "http://facebook.com/", data, err := client.FetchResource(ctx, "/")
HTTPClient: http.DefaultClient, if !errors.Is(err, context.Canceled) {
Logger: log.Log, t.Fatal("unexpected err", err)
UserAgent: "ooniprobe-engine/0.1.0", }
}).FetchResource(ctx, "/robots.txt") if data != nil {
if !errors.Is(err, context.Canceled) { t.Fatal("unexpected data")
t.Fatal("not the error we expected") }
} })
if len(data) != 0 {
t.Fatal("expected an empty resource")
}
}
func TestFetchResourceInvalidURL(t *testing.T) {
log.SetLevel(log.DebugLevel)
ctx := context.Background()
data, err := (&apiClient{
BaseURL: "http://\t/",
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "ooniprobe-engine/0.1.0",
}).FetchResource(ctx, "/robots.txt")
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("not the error we expected")
}
if len(data) != 0 {
t.Fatal("expected an empty resource")
}
}
func TestFetchResource400(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(400)
},
))
defer server.Close()
log.SetLevel(log.DebugLevel)
ctx := context.Background()
data, err := (&apiClient{
Authorization: "foobar",
BaseURL: server.URL,
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "ooniprobe-engine/0.1.0",
}).FetchResource(ctx, "")
if err == nil || !strings.HasSuffix(err.Error(), "400 Bad Request") {
t.Fatal("not the error we expected")
}
if len(data) != 0 {
t.Fatal("expected an empty resource")
}
} }