From 0fdc9cafb50fb04af0088582c6fb979c2426b8b0 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 15 Jun 2021 11:57:40 +0200 Subject: [PATCH] fix(all): introduce and use iox.ReadAllContext (#379) * fix(all): introduce and use iox.ReadAllContext This improvement over the ioutil.ReadAll utility returns early if the context expires. This enables us to unblock stuck code in case there's censorship confounding the TCP stack. See https://github.com/ooni/probe/issues/1417. Compared to the functionality postulated in the above mentioned issue, I choose to be more generic and separate limiting the maximum body size (not implemented here) from using the context to return early when reading a body (or any other reader). After implementing iox.ReadAllContext, I made sure we always use it everywhere in the tree instead of ioutil.ReadAll. This includes many parts of the codebase where in theory we don't need iox.ReadAllContext. Though, changing all the places makes checking whether we're not using ioutil.ReadAll where we should not be using it easy: `git grep` should return no lines. * Update internal/iox/iox_test.go * fix(ndt7): treat context errors as non-errors The rationale is explained by the comment documenting reduceErr. * Update internal/engine/experiment/ndt7/download.go --- CONTRIBUTING.md | 2 + cmd/ooniprobe/internal/database/database.go | 5 +- internal/cmd/jafar/badproxy/badproxy.go | 5 +- .../cmd/jafar/httpproxy/httpproxy_test.go | 8 +-- .../cmd/jafar/uncensored/uncensored_test.go | 5 +- internal/cmd/oohelper/internal/client.go | 4 +- internal/cmd/oohelper/internal/fake_test.go | 4 +- internal/cmd/oohelperd/internal/fake_test.go | 4 +- internal/cmd/oohelperd/internal/http.go | 4 +- internal/cmd/oohelperd/internal/internal.go | 11 +-- .../cmd/oohelperd/internal/internal_test.go | 5 +- internal/engine/experiment/dash/collect.go | 4 +- internal/engine/experiment/dash/dash.go | 8 +-- internal/engine/experiment/dash/download.go | 6 +- internal/engine/experiment/dash/fake_test.go | 3 +- internal/engine/experiment/dash/negotiate.go | 4 +- internal/engine/experiment/dash/spec.go | 6 -- internal/engine/experiment/hhfm/fake_test.go | 5 +- internal/engine/experiment/hhfm/hhfm.go | 4 +- internal/engine/experiment/ndt7/download.go | 31 +++++++- .../engine/experiment/ndt7/download_test.go | 17 +++++ internal/engine/geolocate/fake_test.go | 5 +- internal/engine/httpx/fake_test.go | 5 +- internal/engine/httpx/jsonapi.go | 5 +- .../engine/internal/mlablocate/mlablocate.go | 4 +- .../engine/internal/mlablocatev2/fake_test.go | 5 +- .../internal/mlablocatev2/mlablocatev2.go | 4 +- internal/engine/legacy/netx/dialer.go | 2 +- internal/engine/legacy/netx/http_test.go | 6 +- .../netx/oldhttptransport/bodytracer.go | 4 +- .../netx/oldhttptransport/bodytracer_test.go | 6 +- .../oldhttptransport/httptransport_test.go | 6 +- .../netx/oldhttptransport/tracetripper.go | 43 ++++++------ .../oldhttptransport/tracetripper_test.go | 10 +-- .../oldhttptransport/transactioner_test.go | 5 +- .../legacy/netxlogger/netxlogger_test.go | 5 +- .../legacy/oonitemplates/oonitemplates.go | 5 +- .../oonitemplates/oonitemplates_test.go | 4 +- internal/engine/netx/fake_test.go | 5 +- .../netx/httptransport/bytecounter_test.go | 6 +- .../engine/netx/httptransport/fake_test.go | 5 +- .../engine/netx/httptransport/logging_test.go | 4 +- internal/engine/netx/httptransport/saver.go | 11 +-- .../engine/netx/httptransport/saver_test.go | 6 +- internal/engine/netx/integration_test.go | 4 +- internal/engine/netx/resolver/dnsoverhttps.go | 4 +- internal/engine/netx/tlsx/certifi.go | 2 +- internal/engine/netx/tlsx/generate.go | 6 +- .../engine/probeservices/collector_test.go | 3 +- .../probeservices/probeservices_test.go | 4 +- internal/engine/session_psiphon.go | 4 +- internal/iox/example_test.go | 21 ++++++ internal/iox/iox.go | 47 +++++++++++++ internal/iox/iox_test.go | 70 +++++++++++++++++++ internal/ooapi/apis.go | 35 ++++++---- internal/ooapi/apis_test.go | 26 +++---- internal/ooapi/caching.go | 2 +- internal/ooapi/caching_test.go | 2 +- internal/ooapi/callers.go | 2 +- internal/ooapi/clientcall.go | 2 +- internal/ooapi/clientcall_test.go | 22 +++--- internal/ooapi/cloners.go | 2 +- internal/ooapi/fake_test.go | 5 +- internal/ooapi/fakeapi_test.go | 2 +- internal/ooapi/internal/generator/apis.go | 3 +- internal/ooapi/internal/generator/apistest.go | 4 +- .../internal/generator/clientcalltest.go | 4 +- .../ooapi/internal/generator/responses.go | 7 +- internal/ooapi/login.go | 2 +- internal/ooapi/login_test.go | 2 +- internal/ooapi/loginhandler_test.go | 6 +- internal/ooapi/requests.go | 2 +- internal/ooapi/responses.go | 49 ++++++------- internal/ooapi/swagger_test.go | 4 +- internal/ooapi/swaggerdiff_test.go | 5 +- internal/ptx/ptx_test.go | 4 +- 76 files changed, 440 insertions(+), 223 deletions(-) create mode 100644 internal/iox/example_test.go create mode 100644 internal/iox/iox.go create mode 100644 internal/iox/iox_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c74f759..effd47f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,8 @@ run `go mod tidy` to minimize such changes. - use `./internal/fsx.OpenFile` when you need to open a file +- use `./internal/iox.ReadAllContext` instead of `ioutil.ReadAll` + ## Code testing requirements Make sure all tests pass with `go test -race ./...` run from the diff --git a/cmd/ooniprobe/internal/database/database.go b/cmd/ooniprobe/internal/database/database.go index 9b59047..2589a8c 100644 --- a/cmd/ooniprobe/internal/database/database.go +++ b/cmd/ooniprobe/internal/database/database.go @@ -1,11 +1,12 @@ package database import ( + "context" "database/sql" "embed" - "io/ioutil" "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/iox" migrate "github.com/rubenv/sql-migrate" "upper.io/db.v3/lib/sqlbuilder" "upper.io/db.v3/sqlite" @@ -19,7 +20,7 @@ func readAsset(path string) ([]byte, error) { if err != nil { return nil, err } - return ioutil.ReadAll(filep) + return iox.ReadAllContext(context.Background(), filep) } func readAssetDir(path string) ([]string, error) { diff --git a/internal/cmd/jafar/badproxy/badproxy.go b/internal/cmd/jafar/badproxy/badproxy.go index e3060c0..2eebb3f 100644 --- a/internal/cmd/jafar/badproxy/badproxy.go +++ b/internal/cmd/jafar/badproxy/badproxy.go @@ -6,16 +6,17 @@ package badproxy import ( + "context" "crypto/rsa" "crypto/tls" "crypto/x509" "io" - "io/ioutil" "net" "strings" "time" "github.com/google/martian/v3/mitm" + "github.com/ooni/probe-cli/v3/internal/iox" ) // CensoringProxy is a proxy that does not behave correctly. @@ -56,7 +57,7 @@ func (p *CensoringProxy) serve(conn net.Conn) { } else { const maxread = 1 << 17 reader := io.LimitReader(conn, maxread) - ioutil.ReadAll(reader) + iox.ReadAllContext(context.Background(), reader) } conn.Close() } diff --git a/internal/cmd/jafar/httpproxy/httpproxy_test.go b/internal/cmd/jafar/httpproxy/httpproxy_test.go index 8318ade..a701ce1 100644 --- a/internal/cmd/jafar/httpproxy/httpproxy_test.go +++ b/internal/cmd/jafar/httpproxy/httpproxy_test.go @@ -3,12 +3,12 @@ package httpproxy import ( "bytes" "context" - "io/ioutil" "net" "net/http" "testing" "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestPass(t *testing.T) { @@ -89,7 +89,7 @@ func checkrequest( t.Fatal("unexpected value of status code") } t.Log(resp) - values, _ := resp.Header["Via"] + values := resp.Header["Via"] var foundProduct bool for _, value := range values { if value == product { @@ -102,7 +102,7 @@ func checkrequest( if !foundProduct && expectVia { t.Fatal("Via header not found") } - proxiedData, err := ioutil.ReadAll(resp.Body) + proxiedData, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } @@ -120,7 +120,7 @@ func checkbody(t *testing.T, proxiedData []byte, host string) { t.Fatal("unexpected status code") } defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/cmd/jafar/uncensored/uncensored_test.go b/internal/cmd/jafar/uncensored/uncensored_test.go index b58c055..731318c 100644 --- a/internal/cmd/jafar/uncensored/uncensored_test.go +++ b/internal/cmd/jafar/uncensored/uncensored_test.go @@ -3,10 +3,11 @@ package uncensored import ( "bytes" "context" - "io/ioutil" "net/http" "net/url" "testing" + + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestGood(t *testing.T) { @@ -55,7 +56,7 @@ func TestGood(t *testing.T) { if resp.StatusCode != 200 { t.Fatal("invalid status-code") } - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/cmd/oohelper/internal/client.go b/internal/cmd/oohelper/internal/client.go index 74d665f..468b284 100644 --- a/internal/cmd/oohelper/internal/client.go +++ b/internal/cmd/oohelper/internal/client.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net" "net/http" "net/url" @@ -15,6 +14,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/engine/httpheader" "github.com/ooni/probe-cli/v3/internal/engine/netx" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/version" ) @@ -126,7 +126,7 @@ func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, erro if resp.StatusCode != 200 { return nil, ErrHTTPStatusCode } - data, err = ioutil.ReadAll(resp.Body) + data, err = iox.ReadAllContext(ctx, resp.Body) if err != nil { return nil, err } diff --git a/internal/cmd/oohelper/internal/fake_test.go b/internal/cmd/oohelper/internal/fake_test.go index 13bcdb9..df7987b 100644 --- a/internal/cmd/oohelper/internal/fake_test.go +++ b/internal/cmd/oohelper/internal/fake_test.go @@ -3,13 +3,13 @@ package internal import ( "context" "io" - "io/ioutil" "net" "net/http" "time" "github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/engine/netx" + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeResolver struct { @@ -63,7 +63,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.Func(req) } if req.Body != nil { - ioutil.ReadAll(req.Body) + iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if txp.Err != nil { diff --git a/internal/cmd/oohelperd/internal/fake_test.go b/internal/cmd/oohelperd/internal/fake_test.go index 95abe2c..46c2654 100644 --- a/internal/cmd/oohelperd/internal/fake_test.go +++ b/internal/cmd/oohelperd/internal/fake_test.go @@ -3,13 +3,13 @@ package internal import ( "context" "io" - "io/ioutil" "net" "net/http" "time" "github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/engine/netx" + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeResolver struct { @@ -63,7 +63,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.Func(req) } if req.Body != nil { - ioutil.ReadAll(req.Body) + iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if txp.Err != nil { diff --git a/internal/cmd/oohelperd/internal/http.go b/internal/cmd/oohelperd/internal/http.go index 3985c44..3f25bf8 100644 --- a/internal/cmd/oohelperd/internal/http.go +++ b/internal/cmd/oohelperd/internal/http.go @@ -3,12 +3,12 @@ package internal import ( "context" "io" - "io/ioutil" "net/http" "strings" "sync" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/iox" ) // CtrlHTTPResponse is the result of the HTTP check performed by @@ -56,7 +56,7 @@ func HTTPDo(ctx context.Context, config *HTTPConfig) { headers[k] = resp.Header.Get(k) } reader := &io.LimitedReader{R: resp.Body, N: config.MaxAcceptableBody} - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) config.Out <- CtrlHTTPResponse{ BodyLength: int64(len(data)), Failure: newfailure(err), diff --git a/internal/cmd/oohelperd/internal/internal.go b/internal/cmd/oohelperd/internal/internal.go index 35adad6..0bf5b44 100644 --- a/internal/cmd/oohelperd/internal/internal.go +++ b/internal/cmd/oohelperd/internal/internal.go @@ -4,10 +4,10 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "github.com/ooni/probe-cli/v3/internal/engine/netx" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/version" ) @@ -32,7 +32,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } reader := &io.LimitedReader{R: req.Body, N: h.MaxAcceptableBody} - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(req.Context(), reader) if err != nil { w.WriteHeader(400) return @@ -42,12 +42,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.WriteHeader(400) return } - measureConfig := MeasureConfig{ - Client: h.Client, - Dialer: h.Dialer, - MaxAcceptableBody: h.MaxAcceptableBody, - Resolver: h.Resolver, - } + measureConfig := MeasureConfig(h) cresp, err := Measure(req.Context(), measureConfig, &creq) if err != nil { w.WriteHeader(400) diff --git a/internal/cmd/oohelperd/internal/internal_test.go b/internal/cmd/oohelperd/internal/internal_test.go index 1064f24..34cdffd 100644 --- a/internal/cmd/oohelperd/internal/internal_test.go +++ b/internal/cmd/oohelperd/internal/internal_test.go @@ -1,9 +1,9 @@ package internal_test import ( + "context" "encoding/json" "errors" - "io/ioutil" "net" "net/http" "net/http/httptest" @@ -12,6 +12,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal" "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" + "github.com/ooni/probe-cli/v3/internal/iox" ) const simplerequest = `{ @@ -126,7 +127,7 @@ func TestWorkingAsIntended(t *testing.T) { if v := resp.Header.Get("content-type"); v != expect.respContentType { t.Fatalf("unexpected content-type: %s", v) } - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/experiment/dash/collect.go b/internal/engine/experiment/dash/collect.go index 9e15dcc..641e222 100644 --- a/internal/engine/experiment/dash/collect.go +++ b/internal/engine/experiment/dash/collect.go @@ -16,7 +16,7 @@ type collectDeps interface { JSONMarshal(v interface{}) ([]byte, error) Logger() model.Logger NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error) - ReadAll(r io.Reader) ([]byte, error) + ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) Scheme() string UserAgent() string } @@ -47,7 +47,7 @@ func collect(ctx context.Context, fqdn, authorization string, return errHTTPRequestFailed } defer resp.Body.Close() - data, err = deps.ReadAll(resp.Body) + data, err = deps.ReadAllContext(ctx, resp.Body) if err != nil { return err } diff --git a/internal/engine/experiment/dash/dash.go b/internal/engine/experiment/dash/dash.go index 5b285bd..08313b3 100644 --- a/internal/engine/experiment/dash/dash.go +++ b/internal/engine/experiment/dash/dash.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "runtime" "time" @@ -20,6 +19,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/humanize" + "github.com/ooni/probe-cli/v3/internal/iox" ) const ( @@ -86,8 +86,8 @@ func (r runner) NewHTTPRequest(meth, url string, body io.Reader) (*http.Request, return http.NewRequest(meth, url, body) } -func (r runner) ReadAll(reader io.Reader) ([]byte, error) { - return ioutil.ReadAll(reader) +func (r runner) ReadAllContext(ctx context.Context, reader io.Reader) ([]byte, error) { + return iox.ReadAllContext(ctx, reader) } func (r runner) Scheme() string { @@ -179,7 +179,7 @@ func (r runner) measure( current.ConnectTime = connectTime r.tk.ReceiverData = append(r.tk.ReceiverData, current) total += current.Received - avgspeed := 8 * float64(total) / time.Now().Sub(begin).Seconds() + avgspeed := 8 * float64(total) / time.Since(begin).Seconds() percentage := float64(current.Iteration) / float64(numIterations) message := fmt.Sprintf("streaming: speed: %s", humanize.SI(avgspeed, "bit/s")) r.callbacks.OnProgress(percentage, message) diff --git a/internal/engine/experiment/dash/download.go b/internal/engine/experiment/dash/download.go index 1d860d2..02574d7 100644 --- a/internal/engine/experiment/dash/download.go +++ b/internal/engine/experiment/dash/download.go @@ -12,7 +12,7 @@ import ( type downloadDeps interface { HTTPClient() *http.Client NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error) - ReadAll(r io.Reader) ([]byte, error) + ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) Scheme() string UserAgent() string } @@ -57,7 +57,7 @@ func download(ctx context.Context, config downloadConfig) (downloadResult, error return result, errHTTPRequestFailed } defer resp.Body.Close() - data, err := config.deps.ReadAll(resp.Body) + data, err := config.deps.ReadAllContext(ctx, resp.Body) if err != nil { return result, err } @@ -66,7 +66,7 @@ func download(ctx context.Context, config downloadConfig) (downloadResult, error // turns out that Neubot and MK do the same. So, we do what they do. At // the same time, we are currently not able to include the overhead that // is caused by HTTP headers etc. So, we're a bit less precise. - result.elapsed = time.Now().Sub(savedTicks).Seconds() + result.elapsed = time.Since(savedTicks).Seconds() result.received = int64(len(data)) result.requestTicks = savedTicks.Sub(config.begin).Seconds() result.timestamp = time.Now().Unix() diff --git a/internal/engine/experiment/dash/fake_test.go b/internal/engine/experiment/dash/fake_test.go index baa9b5c..7f35618 100644 --- a/internal/engine/experiment/dash/fake_test.go +++ b/internal/engine/experiment/dash/fake_test.go @@ -1,6 +1,7 @@ package dash import ( + "context" "io" "net/http" "time" @@ -36,7 +37,7 @@ func (d FakeDeps) NewHTTPRequest( return d.newHTTPRequestResult, d.newHTTPRequestErr } -func (d FakeDeps) ReadAll(r io.Reader) ([]byte, error) { +func (d FakeDeps) ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) { return d.readAllResult, d.readAllErr } diff --git a/internal/engine/experiment/dash/negotiate.go b/internal/engine/experiment/dash/negotiate.go index 71968c4..1464099 100644 --- a/internal/engine/experiment/dash/negotiate.go +++ b/internal/engine/experiment/dash/negotiate.go @@ -16,7 +16,7 @@ type negotiateDeps interface { JSONMarshal(v interface{}) ([]byte, error) Logger() model.Logger NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error) - ReadAll(r io.Reader) ([]byte, error) + ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) Scheme() string UserAgent() string } @@ -48,7 +48,7 @@ func negotiate( return negotiateResp, errHTTPRequestFailed } defer resp.Body.Close() - data, err = deps.ReadAll(resp.Body) + data, err = deps.ReadAllContext(ctx, resp.Body) if err != nil { return negotiateResp, err } diff --git a/internal/engine/experiment/dash/spec.go b/internal/engine/experiment/dash/spec.go index b689b69..6b3060e 100644 --- a/internal/engine/experiment/dash/spec.go +++ b/internal/engine/experiment/dash/spec.go @@ -1,12 +1,6 @@ package dash const ( - // currentServerSchemaVersion is the version of the server schema that - // will be adopted by this implementation. Version 3 is the one that is - // Neubot uses. We needed to bump the version because Web100 is not on - // M-Lab anymore and hence we need to make a breaking change. - currentServerSchemaVersion = 4 - // negotiatePath is the URL path used to negotiate negotiatePath = "/negotiate/dash" diff --git a/internal/engine/experiment/hhfm/fake_test.go b/internal/engine/experiment/hhfm/fake_test.go index 3138c97..ee0c7b8 100644 --- a/internal/engine/experiment/hhfm/fake_test.go +++ b/internal/engine/experiment/hhfm/fake_test.go @@ -2,10 +2,11 @@ package hhfm_test import ( "context" - "io/ioutil" "net" "net/http" "time" + + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeDialer struct { @@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.Func(req) } if req.Body != nil { - ioutil.ReadAll(req.Body) + iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if txp.Err != nil { diff --git a/internal/engine/experiment/hhfm/hhfm.go b/internal/engine/experiment/hhfm/hhfm.go index a2eee07..d6b69e0 100644 --- a/internal/engine/experiment/hhfm/hhfm.go +++ b/internal/engine/experiment/hhfm/hhfm.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net" "net/http" "sort" @@ -21,6 +20,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" "github.com/ooni/probe-cli/v3/internal/engine/netx/dialer" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/randx" ) @@ -198,7 +198,7 @@ func transact(txp Transport, req *http.Request, return nil, nil, urlgetter.ErrHTTPRequestFailed } callbacks.OnProgress(0.75, "reading response body...") - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(req.Context(), resp.Body) callbacks.OnProgress(1.00, fmt.Sprintf("got reseponse body... %+v", err)) if err != nil { return nil, nil, err diff --git a/internal/engine/experiment/ndt7/download.go b/internal/engine/experiment/ndt7/download.go index 5adc0bb..447cde2 100644 --- a/internal/engine/experiment/ndt7/download.go +++ b/internal/engine/experiment/ndt7/download.go @@ -2,11 +2,12 @@ package ndt7 import ( "context" + "errors" "io" - "io/ioutil" "time" "github.com/gorilla/websocket" + "github.com/ooni/probe-cli/v3/internal/iox" ) type downloadManager struct { @@ -33,6 +34,30 @@ func newDownloadManager( } func (mgr downloadManager) run(ctx context.Context) error { + return mgr.reduceErr(mgr.doRun(ctx)) +} + +// reduceErr treats as non-errors the errors caused by the context +// so that we can focus instead on network errors. +// +// This function was introduced by https://github.com/ooni/probe-cli/pull/379 +// since before such a PR we did not see context interrupting +// errors when we were reading messages. Since before such a PR +// we used to return `nil` on context errors, this function is +// here to keep the previous behavior by filtering the error +// returned when reading messages, given that now reading messages +// can fail midway because we use iox.ReadAllContext. +func (mgr downloadManager) reduceErr(err error) error { + if errors.Is(err, context.Canceled) { + return nil + } + if errors.Is(err, context.DeadlineExceeded) { + return nil + } + return err +} + +func (mgr downloadManager) doRun(ctx context.Context) error { var total int64 start := time.Now() if err := mgr.conn.SetReadDeadline(start.Add(mgr.maxRuntime)); err != nil { @@ -47,7 +72,7 @@ func (mgr downloadManager) run(ctx context.Context) error { return err } if kind == websocket.TextMessage { - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return err } @@ -57,7 +82,7 @@ func (mgr downloadManager) run(ctx context.Context) error { } continue } - n, err := io.Copy(ioutil.Discard, reader) + n, err := io.Copy(io.Discard, reader) if err != nil { return err } diff --git a/internal/engine/experiment/ndt7/download_test.go b/internal/engine/experiment/ndt7/download_test.go index bc393a3..fb3b1ed 100644 --- a/internal/engine/experiment/ndt7/download_test.go +++ b/internal/engine/experiment/ndt7/download_test.go @@ -143,3 +143,20 @@ type goodJSONReader struct{} func (r *goodJSONReader) Read(p []byte) (int, error) { return copy(p, []byte(`{}`)), io.EOF } + +func TestDownloadReduceErr(t *testing.T) { + mgr := downloadManager{} + if mgr.reduceErr(context.Canceled) != nil { + t.Fatal("reduceErr not working as intended for context.Canceled") + } + if mgr.reduceErr(context.DeadlineExceeded) != nil { + t.Fatal("reduceErr not working as intended for context.DeadlineExceeded") + } + if mgr.reduceErr(nil) != nil { + t.Fatal("reduceErr not working as intended for nil") + } + expected := errors.New("mocked error") + if mgr.reduceErr(expected) != expected { + t.Fatal("reduceErr not working as intended for other errors") + } +} diff --git a/internal/engine/geolocate/fake_test.go b/internal/engine/geolocate/fake_test.go index 93978c5..93c8589 100644 --- a/internal/engine/geolocate/fake_test.go +++ b/internal/engine/geolocate/fake_test.go @@ -1,9 +1,10 @@ package geolocate import ( - "io/ioutil" "net/http" "time" + + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeTransport struct { @@ -14,7 +15,7 @@ type FakeTransport struct { func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { time.Sleep(10 * time.Microsecond) if req.Body != nil { - ioutil.ReadAll(req.Body) + iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if txp.Err != nil { diff --git a/internal/engine/httpx/fake_test.go b/internal/engine/httpx/fake_test.go index 2ae6cfd..a5c94ea 100644 --- a/internal/engine/httpx/fake_test.go +++ b/internal/engine/httpx/fake_test.go @@ -1,9 +1,10 @@ package httpx import ( - "io/ioutil" "net/http" "time" + + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeTransport struct { @@ -18,7 +19,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.Func(req) } if req.Body != nil { - ioutil.ReadAll(req.Body) + iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if txp.Err != nil { diff --git a/internal/engine/httpx/jsonapi.go b/internal/engine/httpx/jsonapi.go index 1d4dcae..0845295 100644 --- a/internal/engine/httpx/jsonapi.go +++ b/internal/engine/httpx/jsonapi.go @@ -7,9 +7,10 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "net/url" + + "github.com/ooni/probe-cli/v3/internal/iox" ) // Logger is the definition of Logger used by this package. @@ -100,7 +101,7 @@ func (c Client) Do(request *http.Request) ([]byte, error) { if response.StatusCode >= 400 { return nil, fmt.Errorf("httpx: request failed: %s", response.Status) } - return ioutil.ReadAll(response.Body) + return iox.ReadAllContext(request.Context(), response.Body) } // DoJSON performs the provided request and unmarshals the JSON response body diff --git a/internal/engine/internal/mlablocate/mlablocate.go b/internal/engine/internal/mlablocate/mlablocate.go index c8591c6..6574ae7 100644 --- a/internal/engine/internal/mlablocate/mlablocate.go +++ b/internal/engine/internal/mlablocate/mlablocate.go @@ -6,11 +6,11 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net/http" "net/url" "github.com/ooni/probe-cli/v3/internal/engine/model" + "github.com/ooni/probe-cli/v3/internal/iox" ) // Client is a locate.measurementlab.net client. @@ -63,7 +63,7 @@ func (c *Client) Query(ctx context.Context, tool string) (Result, error) { if resp.StatusCode != 200 { return Result{}, fmt.Errorf("mlablocate: non-200 status code: %d", resp.StatusCode) } - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(ctx, resp.Body) if err != nil { return Result{}, err } diff --git a/internal/engine/internal/mlablocatev2/fake_test.go b/internal/engine/internal/mlablocatev2/fake_test.go index 0fc63ac..c0228d8 100644 --- a/internal/engine/internal/mlablocatev2/fake_test.go +++ b/internal/engine/internal/mlablocatev2/fake_test.go @@ -1,9 +1,10 @@ package mlablocatev2 import ( - "io/ioutil" "net/http" "time" + + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeTransport struct { @@ -18,7 +19,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.Func(req) } if req.Body != nil { - ioutil.ReadAll(req.Body) + iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if txp.Err != nil { diff --git a/internal/engine/internal/mlablocatev2/mlablocatev2.go b/internal/engine/internal/mlablocatev2/mlablocatev2.go index cda8cf2..36777d3 100644 --- a/internal/engine/internal/mlablocatev2/mlablocatev2.go +++ b/internal/engine/internal/mlablocatev2/mlablocatev2.go @@ -6,12 +6,12 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net/http" "net/url" "regexp" "github.com/ooni/probe-cli/v3/internal/engine/model" + "github.com/ooni/probe-cli/v3/internal/iox" ) const ( @@ -102,7 +102,7 @@ func (c Client) query(ctx context.Context, path string) (resultRecord, error) { if resp.StatusCode != 200 { return resultRecord{}, fmt.Errorf("%w: %d", ErrRequestFailed, resp.StatusCode) } - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(ctx, resp.Body) if err != nil { return resultRecord{}, err } diff --git a/internal/engine/legacy/netx/dialer.go b/internal/engine/legacy/netx/dialer.go index 56721f0..ec3ff93 100644 --- a/internal/engine/legacy/netx/dialer.go +++ b/internal/engine/legacy/netx/dialer.go @@ -135,7 +135,7 @@ func (d *Dialer) SetCABundle(path string) error { return err } pool := x509.NewCertPool() - if pool.AppendCertsFromPEM(cert) == false { + if !pool.AppendCertsFromPEM(cert) { return errors.New("AppendCertsFromPEM failed") } d.TLSConfig.RootCAs = pool diff --git a/internal/engine/legacy/netx/http_test.go b/internal/engine/legacy/netx/http_test.go index 4e8551c..52794ca 100644 --- a/internal/engine/legacy/netx/http_test.go +++ b/internal/engine/legacy/netx/http_test.go @@ -4,7 +4,6 @@ import ( "context" "crypto/x509" "errors" - "io/ioutil" "net" "net/http" "net/http/httptest" @@ -15,6 +14,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" + "github.com/ooni/probe-cli/v3/internal/iox" ) func dowithclient(t *testing.T, client *netx.HTTPClient) { @@ -24,7 +24,7 @@ func dowithclient(t *testing.T, client *netx.HTTPClient) { t.Fatal(err) } defer resp.Body.Close() - _, err = ioutil.ReadAll(resp.Body) + _, err = iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } @@ -126,7 +126,7 @@ func httpProxyTestMain(t *testing.T, client *http.Client, expect int) { t.Fatal(err) } defer resp.Body.Close() - _, err = ioutil.ReadAll(resp.Body) + _, err = iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/legacy/netx/oldhttptransport/bodytracer.go b/internal/engine/legacy/netx/oldhttptransport/bodytracer.go index ed46c64..8d39628 100644 --- a/internal/engine/legacy/netx/oldhttptransport/bodytracer.go +++ b/internal/engine/legacy/netx/oldhttptransport/bodytracer.go @@ -63,7 +63,7 @@ func (bw *bodyWrapper) Read(b []byte) (n int, err error) { // bytes read (0 <= n <= len(p)) and any error encountered." Data: b[:n], Error: err, - DurationSinceBeginning: time.Now().Sub(bw.root.Beginning), + DurationSinceBeginning: time.Since(bw.root.Beginning), TransactionID: bw.tid, }, }) @@ -74,7 +74,7 @@ func (bw *bodyWrapper) Close() (err error) { err = bw.ReadCloser.Close() bw.root.Handler.OnMeasurement(modelx.Measurement{ HTTPResponseDone: &modelx.HTTPResponseDoneEvent{ - DurationSinceBeginning: time.Now().Sub(bw.root.Beginning), + DurationSinceBeginning: time.Since(bw.root.Beginning), TransactionID: bw.tid, }, }) diff --git a/internal/engine/legacy/netx/oldhttptransport/bodytracer_test.go b/internal/engine/legacy/netx/oldhttptransport/bodytracer_test.go index 8d8e2c0..a33d71b 100644 --- a/internal/engine/legacy/netx/oldhttptransport/bodytracer_test.go +++ b/internal/engine/legacy/netx/oldhttptransport/bodytracer_test.go @@ -1,9 +1,11 @@ package oldhttptransport import ( - "io/ioutil" + "context" "net/http" "testing" + + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestBodyTracerSuccess(t *testing.T) { @@ -15,7 +17,7 @@ func TestBodyTracerSuccess(t *testing.T) { t.Fatal(err) } defer resp.Body.Close() - _, err = ioutil.ReadAll(resp.Body) + _, err = iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/legacy/netx/oldhttptransport/httptransport_test.go b/internal/engine/legacy/netx/oldhttptransport/httptransport_test.go index b1c06ae..835dc3d 100644 --- a/internal/engine/legacy/netx/oldhttptransport/httptransport_test.go +++ b/internal/engine/legacy/netx/oldhttptransport/httptransport_test.go @@ -1,9 +1,11 @@ package oldhttptransport import ( - "io/ioutil" + "context" "net/http" "testing" + + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestGood(t *testing.T) { @@ -15,7 +17,7 @@ func TestGood(t *testing.T) { t.Fatal(err) } defer resp.Body.Close() - _, err = ioutil.ReadAll(resp.Body) + _, err = iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/legacy/netx/oldhttptransport/tracetripper.go b/internal/engine/legacy/netx/oldhttptransport/tracetripper.go index baad6dd..ff88aa6 100644 --- a/internal/engine/legacy/netx/oldhttptransport/tracetripper.go +++ b/internal/engine/legacy/netx/oldhttptransport/tracetripper.go @@ -2,9 +2,9 @@ package oldhttptransport import ( "bytes" + "context" "crypto/tls" "io" - "io/ioutil" "net/http" "net/http/httptrace" "sync" @@ -16,21 +16,22 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/transactionid" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" + "github.com/ooni/probe-cli/v3/internal/iox" ) // TraceTripper performs single HTTP transactions. type TraceTripper struct { - readAllErrs *atomicx.Int64 - readAll func(r io.Reader) ([]byte, error) - roundTripper http.RoundTripper + readAllErrs *atomicx.Int64 + readAllContext func(ctx context.Context, r io.Reader) ([]byte, error) + roundTripper http.RoundTripper } // NewTraceTripper creates a new Transport. func NewTraceTripper(roundTripper http.RoundTripper) *TraceTripper { return &TraceTripper{ - readAllErrs: &atomicx.Int64{}, - readAll: ioutil.ReadAll, - roundTripper: roundTripper, + readAllErrs: &atomicx.Int64{}, + readAllContext: iox.ReadAllContext, + roundTripper: roundTripper, } } @@ -57,10 +58,10 @@ func (c *readCloseWrapper) Close() error { } func readSnap( - source *io.ReadCloser, limit int64, - readAll func(r io.Reader) ([]byte, error), + ctx context.Context, source *io.ReadCloser, limit int64, + readAllContext func(ctx context.Context, r io.Reader) ([]byte, error), ) (data []byte, err error) { - data, err = readAll(io.LimitReader(*source, limit)) + data, err = readAllContext(ctx, io.LimitReader(*source, limit)) if err == nil { *source = newReadCloseWrapper( io.MultiReader(bytes.NewReader(data), *source), @@ -79,7 +80,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { root.Handler.OnMeasurement(modelx.Measurement{ HTTPRoundTripStart: &modelx.HTTPRoundTripStartEvent{ DialID: dialid.ContextDialID(req.Context()), - DurationSinceBeginning: time.Now().Sub(root.Beginning), + DurationSinceBeginning: time.Since(root.Beginning), Method: req.Method, TransactionID: tid, URL: req.URL.String(), @@ -98,7 +99,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { // Save a snapshot of the request body if req.Body != nil { - requestBody, err = readSnap(&req.Body, snapSize, t.readAll) + requestBody, err = readSnap(req.Context(), &req.Body, snapSize, t.readAllContext) if err != nil { return nil, err } @@ -114,7 +115,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { // configured in the http.Transport root.Handler.OnMeasurement(modelx.Measurement{ TLSHandshakeStart: &modelx.TLSHandshakeStartEvent{ - DurationSinceBeginning: time.Now().Sub(root.Beginning), + DurationSinceBeginning: time.Since(root.Beginning), TransactionID: tid, }, }) @@ -127,7 +128,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { Operation: errorx.TLSHandshakeOperation, TransactionID: tid, }.MaybeBuild() - durationSinceBeginning := time.Now().Sub(root.Beginning) + durationSinceBeginning := time.Since(root.Beginning) // Event emitted by net/http when DialTLS is not // configured in the http.Transport root.Handler.OnMeasurement(modelx.Measurement{ @@ -149,7 +150,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { info.Conn.LocalAddr().Network(), info.Conn.LocalAddr().String(), ), - DurationSinceBeginning: time.Now().Sub(root.Beginning), + DurationSinceBeginning: time.Since(root.Beginning), TransactionID: tid, }, }) @@ -165,7 +166,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { requestHeadersMu.Unlock() root.Handler.OnMeasurement(modelx.Measurement{ HTTPRequestHeader: &modelx.HTTPRequestHeaderEvent{ - DurationSinceBeginning: time.Now().Sub(root.Beginning), + DurationSinceBeginning: time.Since(root.Beginning), Key: key, TransactionID: tid, Value: values, @@ -175,7 +176,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { WroteHeaders: func() { root.Handler.OnMeasurement(modelx.Measurement{ HTTPRequestHeadersDone: &modelx.HTTPRequestHeadersDoneEvent{ - DurationSinceBeginning: time.Now().Sub(root.Beginning), + DurationSinceBeginning: time.Since(root.Beginning), Headers: requestHeaders, // [*] Method: req.Method, // [*] TransactionID: tid, @@ -193,7 +194,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { }.MaybeBuild() root.Handler.OnMeasurement(modelx.Measurement{ HTTPRequestDone: &modelx.HTTPRequestDoneEvent{ - DurationSinceBeginning: time.Now().Sub(root.Beginning), + DurationSinceBeginning: time.Since(root.Beginning), Error: err, TransactionID: tid, }, @@ -202,7 +203,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { GotFirstResponseByte: func() { root.Handler.OnMeasurement(modelx.Measurement{ HTTPResponseStart: &modelx.HTTPResponseStartEvent{ - DurationSinceBeginning: time.Now().Sub(root.Beginning), + DurationSinceBeginning: time.Since(root.Beginning), TransactionID: tid, }, }) @@ -233,7 +234,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { // [*] Require less event joining work by providing info that // makes this event alone actionable for OONI event := &modelx.HTTPRoundTripDoneEvent{ - DurationSinceBeginning: time.Now().Sub(root.Beginning), + DurationSinceBeginning: time.Since(root.Beginning), Error: err, RequestBodySnap: requestBody, RequestHeaders: requestHeaders, // [*] @@ -248,7 +249,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) { event.ResponseProto = resp.Proto // Save a snapshot of the response body var data []byte - data, err = readSnap(&resp.Body, snapSize, t.readAll) + data, err = readSnap(req.Context(), &resp.Body, snapSize, t.readAllContext) if err != nil { t.readAllErrs.Add(1) resp = nil // this is how net/http likes it diff --git a/internal/engine/legacy/netx/oldhttptransport/tracetripper_test.go b/internal/engine/legacy/netx/oldhttptransport/tracetripper_test.go index f53e4ab..990bc4a 100644 --- a/internal/engine/legacy/netx/oldhttptransport/tracetripper_test.go +++ b/internal/engine/legacy/netx/oldhttptransport/tracetripper_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "io/ioutil" "net/http" "net/http/httptrace" "sync" @@ -14,6 +13,7 @@ import ( "github.com/miekg/dns" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestTraceTripperSuccess(t *testing.T) { @@ -25,7 +25,7 @@ func TestTraceTripperSuccess(t *testing.T) { t.Fatal(err) } defer resp.Body.Close() - _, err = ioutil.ReadAll(resp.Body) + _, err = iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } @@ -47,7 +47,7 @@ func (h *roundTripHandler) OnMeasurement(m modelx.Measurement) { func TestTraceTripperReadAllFailure(t *testing.T) { transport := NewTraceTripper(http.DefaultTransport) - transport.readAll = func(r io.Reader) ([]byte, error) { + transport.readAllContext = func(ctx context.Context, r io.Reader) ([]byte, error) { return nil, io.EOF } client := &http.Client{Transport: transport} @@ -156,7 +156,7 @@ func TestTraceTripperWithCorrectSnaps(t *testing.T) { // Read the whole response body, parse it as valid DNS // reply and verify we obtained what we expected - replyData, err := ioutil.ReadAll(resp.Body) + replyData, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } @@ -230,7 +230,7 @@ func TestTraceTripperWithReadAllFailingForBody(t *testing.T) { // use such transport to configure an ordinary client transport := NewTraceTripper(http.DefaultTransport) errorMocked := errors.New("mocked error") - transport.readAll = func(r io.Reader) ([]byte, error) { + transport.readAllContext = func(ctx context.Context, r io.Reader) ([]byte, error) { return nil, errorMocked } const snapSize = 15 diff --git a/internal/engine/legacy/netx/oldhttptransport/transactioner_test.go b/internal/engine/legacy/netx/oldhttptransport/transactioner_test.go index 1eb9cc0..d7804fe 100644 --- a/internal/engine/legacy/netx/oldhttptransport/transactioner_test.go +++ b/internal/engine/legacy/netx/oldhttptransport/transactioner_test.go @@ -1,11 +1,12 @@ package oldhttptransport import ( - "io/ioutil" + "context" "net/http" "testing" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/transactionid" + "github.com/ooni/probe-cli/v3/internal/iox" ) type transactionerCheckTransactionID struct { @@ -33,7 +34,7 @@ func TestTransactionerSuccess(t *testing.T) { t.Fatal(err) } defer resp.Body.Close() - _, err = ioutil.ReadAll(resp.Body) + _, err = iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/legacy/netxlogger/netxlogger_test.go b/internal/engine/legacy/netxlogger/netxlogger_test.go index 00dd38b..3bf64ba 100644 --- a/internal/engine/legacy/netxlogger/netxlogger_test.go +++ b/internal/engine/legacy/netxlogger/netxlogger_test.go @@ -1,7 +1,7 @@ package netxlogger import ( - "io/ioutil" + "context" "net/http" "testing" "time" @@ -10,6 +10,7 @@ import ( "github.com/apex/log/handlers/discard" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestGood(t *testing.T) { @@ -32,7 +33,7 @@ func TestGood(t *testing.T) { t.Fatal("expected non-nil resp here") } defer resp.Body.Close() - _, err = ioutil.ReadAll(resp.Body) + _, err = iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/legacy/oonitemplates/oonitemplates.go b/internal/engine/legacy/oonitemplates/oonitemplates.go index cb7ba09..3be2291 100644 --- a/internal/engine/legacy/oonitemplates/oonitemplates.go +++ b/internal/engine/legacy/oonitemplates/oonitemplates.go @@ -24,6 +24,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/runtimex" "gitlab.com/yawning/obfs4.git/transports" obfs4base "gitlab.com/yawning/obfs4.git/transports/base" @@ -90,7 +91,7 @@ func (m *connmapper) scramble(cid int64) int64 { // See https://stackoverflow.com/a/38140573/4354461 m.mu.Lock() defer m.mu.Unlock() - if value, found := m.table[cid]; found == true { + if value, found := m.table[cid]; found { return value } var factor int64 = 1 @@ -366,7 +367,7 @@ func HTTPDo( config.MaxResponseBodySnapSize, ), ) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) mu.Lock() results.BodySnap, results.Error = data, err mu.Unlock() diff --git a/internal/engine/legacy/oonitemplates/oonitemplates_test.go b/internal/engine/legacy/oonitemplates/oonitemplates_test.go index 683ab96..ae78760 100644 --- a/internal/engine/legacy/oonitemplates/oonitemplates_test.go +++ b/internal/engine/legacy/oonitemplates/oonitemplates_test.go @@ -377,7 +377,7 @@ type faketransport struct { } func (txp *faketransport) Name() string { - return txp.Name() + return txp.txp.Name() } func (txp *faketransport) ClientFactory(stateDir string) (obfs4base.ClientFactory, error) { @@ -385,7 +385,7 @@ func (txp *faketransport) ClientFactory(stateDir string) (obfs4base.ClientFactor } func (txp *faketransport) ServerFactory(stateDir string, args *goptlib.Args) (obfs4base.ServerFactory, error) { - return txp.ServerFactory(stateDir, args) + return txp.txp.ServerFactory(stateDir, args) } func TestConnmapper(t *testing.T) { diff --git a/internal/engine/netx/fake_test.go b/internal/engine/netx/fake_test.go index 2c3b464..ba3f816 100644 --- a/internal/engine/netx/fake_test.go +++ b/internal/engine/netx/fake_test.go @@ -2,10 +2,11 @@ package netx import ( "context" - "io/ioutil" "net" "net/http" "time" + + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeDialer struct { @@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.Func(req) } if req.Body != nil { - ioutil.ReadAll(req.Body) + iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if txp.Err != nil { diff --git a/internal/engine/netx/httptransport/bytecounter_test.go b/internal/engine/netx/httptransport/bytecounter_test.go index dd5005f..a4a7713 100644 --- a/internal/engine/netx/httptransport/bytecounter_test.go +++ b/internal/engine/netx/httptransport/bytecounter_test.go @@ -1,6 +1,7 @@ package httptransport_test import ( + "context" "errors" "io" "io/ioutil" @@ -10,6 +11,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestByteCounterFailure(t *testing.T) { @@ -68,7 +70,7 @@ func TestByteCounterSuccess(t *testing.T) { if err != nil { t.Fatal(err) } - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } @@ -104,7 +106,7 @@ func TestByteCounterSuccessWithEOF(t *testing.T) { if err != nil { t.Fatal(err) } - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/netx/httptransport/fake_test.go b/internal/engine/netx/httptransport/fake_test.go index 4f2a25b..e5bbb0f 100644 --- a/internal/engine/netx/httptransport/fake_test.go +++ b/internal/engine/netx/httptransport/fake_test.go @@ -2,10 +2,11 @@ package httptransport import ( "context" - "io/ioutil" "net" "net/http" "time" + + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeDialer struct { @@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { return txp.Func(req) } if req.Body != nil { - ioutil.ReadAll(req.Body) + iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if txp.Err != nil { diff --git a/internal/engine/netx/httptransport/logging_test.go b/internal/engine/netx/httptransport/logging_test.go index c9c8a16..ae632f5 100644 --- a/internal/engine/netx/httptransport/logging_test.go +++ b/internal/engine/netx/httptransport/logging_test.go @@ -1,6 +1,7 @@ package httptransport_test import ( + "context" "errors" "io" "io/ioutil" @@ -11,6 +12,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestLoggingFailure(t *testing.T) { @@ -72,6 +74,6 @@ func TestLoggingSuccess(t *testing.T) { if err != nil { t.Fatal(err) } - ioutil.ReadAll(resp.Body) + iox.ReadAllContext(context.Background(), resp.Body) resp.Body.Close() } diff --git a/internal/engine/netx/httptransport/saver.go b/internal/engine/netx/httptransport/saver.go index f75bfdb..c94d4df 100644 --- a/internal/engine/netx/httptransport/saver.go +++ b/internal/engine/netx/httptransport/saver.go @@ -2,14 +2,15 @@ package httptransport import ( "bytes" + "context" "errors" "io" - "io/ioutil" "net/http" "net/http/httptrace" "time" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/iox" ) // SaverPerformanceHTTPTransport is a RoundTripper that saves @@ -109,7 +110,7 @@ func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response, snapsize = txp.SnapshotSize } if req.Body != nil { - data, err := saverSnapRead(req.Body, snapsize) + data, err := saverSnapRead(req.Context(), req.Body, snapsize) if err != nil { return nil, err } @@ -125,7 +126,7 @@ func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response, if err != nil { return nil, err } - data, err := saverSnapRead(resp.Body, snapsize) + data, err := saverSnapRead(req.Context(), resp.Body, snapsize) err = ignoreExpectedEOF(err, resp) if err != nil { resp.Body.Close() @@ -157,8 +158,8 @@ func ignoreExpectedEOF(err error, resp *http.Response) error { return err } -func saverSnapRead(r io.ReadCloser, snapsize int) ([]byte, error) { - return ioutil.ReadAll(io.LimitReader(r, int64(snapsize))) +func saverSnapRead(ctx context.Context, r io.ReadCloser, snapsize int) ([]byte, error) { + return iox.ReadAllContext(ctx, io.LimitReader(r, int64(snapsize))) } func saverCompose(data []byte, r io.ReadCloser) io.ReadCloser { diff --git a/internal/engine/netx/httptransport/saver_test.go b/internal/engine/netx/httptransport/saver_test.go index 8846107..fd484a0 100644 --- a/internal/engine/netx/httptransport/saver_test.go +++ b/internal/engine/netx/httptransport/saver_test.go @@ -1,6 +1,7 @@ package httptransport_test import ( + "context" "errors" "io/ioutil" "net/http" @@ -10,6 +11,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestSaverPerformanceNoMultipleEvents(t *testing.T) { @@ -245,7 +247,7 @@ func TestSaverBodySuccess(t *testing.T) { txp := httptransport.SaverBodyHTTPTransport{ RoundTripper: httptransport.FakeTransport{ Func: func(req *http.Request) (*http.Response, error) { - data, err := ioutil.ReadAll(req.Body) + data, err := iox.ReadAllContext(context.Background(), req.Body) if err != nil { t.Fatal(err) } @@ -274,7 +276,7 @@ func TestSaverBodySuccess(t *testing.T) { t.Fatal("unexpected status code") } defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/netx/integration_test.go b/internal/engine/netx/integration_test.go index 0b7db6d..9f3bb37 100644 --- a/internal/engine/netx/integration_test.go +++ b/internal/engine/netx/integration_test.go @@ -3,7 +3,6 @@ package netx_test import ( "context" "errors" - "io/ioutil" "net/http" "testing" @@ -12,6 +11,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestSuccess(t *testing.T) { @@ -38,7 +38,7 @@ func TestSuccess(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = ioutil.ReadAll(resp.Body); err != nil { + if _, err = iox.ReadAllContext(context.Background(), resp.Body); err != nil { t.Fatal(err) } if err = resp.Body.Close(); err != nil { diff --git a/internal/engine/netx/resolver/dnsoverhttps.go b/internal/engine/netx/resolver/dnsoverhttps.go index 8041024..e0686b1 100644 --- a/internal/engine/netx/resolver/dnsoverhttps.go +++ b/internal/engine/netx/resolver/dnsoverhttps.go @@ -4,11 +4,11 @@ import ( "bytes" "context" "errors" - "io/ioutil" "net/http" "time" "github.com/ooni/probe-cli/v3/internal/engine/httpheader" + "github.com/ooni/probe-cli/v3/internal/iox" ) // DNSOverHTTPS is a DNS over HTTPS RoundTripper. Requests are submitted over @@ -56,7 +56,7 @@ func (t DNSOverHTTPS) RoundTrip(ctx context.Context, query []byte) ([]byte, erro if resp.Header.Get("content-type") != "application/dns-message" { return nil, errors.New("doh: invalid content-type") } - return ioutil.ReadAll(resp.Body) + return iox.ReadAllContext(ctx, resp.Body) } // RequiresPadding returns true for DoH according to RFC8467 diff --git a/internal/engine/netx/tlsx/certifi.go b/internal/engine/netx/tlsx/certifi.go index 23b4ed7..1af3da1 100644 --- a/internal/engine/netx/tlsx/certifi.go +++ b/internal/engine/netx/tlsx/certifi.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-06-08 14:36:18.474117 +0200 CEST m=+0.459588210 +// 2021-06-15 10:55:55.638897 +0200 CEST m=+4.257631084 // https://curl.haxx.se/ca/cacert.pem package tlsx diff --git a/internal/engine/netx/tlsx/generate.go b/internal/engine/netx/tlsx/generate.go index b9ae011..154b93e 100644 --- a/internal/engine/netx/tlsx/generate.go +++ b/internal/engine/netx/tlsx/generate.go @@ -12,14 +12,16 @@ package main import ( + "context" "crypto/x509" - "io/ioutil" "log" "net/http" "os" "strings" "text/template" "time" + + "github.com/ooni/probe-cli/v3/internal/iox" ) var tmpl = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. @@ -50,7 +52,7 @@ func main() { } defer resp.Body.Close() - bundle, err := ioutil.ReadAll(resp.Body) + bundle, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { log.Fatal(err) } diff --git a/internal/engine/probeservices/collector_test.go b/internal/engine/probeservices/collector_test.go index c95b6dd..ed0dceb 100644 --- a/internal/engine/probeservices/collector_test.go +++ b/internal/engine/probeservices/collector_test.go @@ -15,6 +15,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" + "github.com/ooni/probe-cli/v3/internal/iox" ) type fakeTestKeys struct { @@ -233,7 +234,7 @@ func TestEndToEnd(t *testing.T) { return } if r.RequestURI == "/report/_id" { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { panic(err) } diff --git a/internal/engine/probeservices/probeservices_test.go b/internal/engine/probeservices/probeservices_test.go index fdb9123..3ea93d3 100644 --- a/internal/engine/probeservices/probeservices_test.go +++ b/internal/engine/probeservices/probeservices_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "io" - "io/ioutil" "net/http" "regexp" "strings" @@ -17,6 +16,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" "github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra" + "github.com/ooni/probe-cli/v3/internal/iox" ) func newclient() *probeservices.Client { @@ -165,7 +165,7 @@ func TestCloudfront(t *testing.T) { if resp.StatusCode != 200 { t.Fatal("unexpected status code") } - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(req.Context(), resp.Body) if err != nil { t.Fatal(err) } diff --git a/internal/engine/session_psiphon.go b/internal/engine/session_psiphon.go index a6ee33e..050e264 100644 --- a/internal/engine/session_psiphon.go +++ b/internal/engine/session_psiphon.go @@ -6,9 +6,9 @@ import ( "bytes" "context" _ "embed" - "io/ioutil" "filippo.io/age" + "github.com/ooni/probe-cli/v3/internal/iox" ) //go:embed psiphon-config.json.age @@ -34,7 +34,7 @@ func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]b if err != nil { return nil, err } - return ioutil.ReadAll(output) + return iox.ReadAllContext(ctx, output) } // FetchPsiphonConfig decrypts psiphonConfigJSONAge using diff --git a/internal/iox/example_test.go b/internal/iox/example_test.go new file mode 100644 index 0000000..3c3abc4 --- /dev/null +++ b/internal/iox/example_test.go @@ -0,0 +1,21 @@ +package iox_test + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/ooni/probe-cli/v3/internal/iox" +) + +func ExampleReadAllContext() { + r := strings.NewReader("deadbeef") + ctx := context.Background() + out, err := iox.ReadAllContext(ctx, r) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%d\n", len(out)) + // Output: 8 +} diff --git a/internal/iox/iox.go b/internal/iox/iox.go new file mode 100644 index 0000000..f083d53 --- /dev/null +++ b/internal/iox/iox.go @@ -0,0 +1,47 @@ +// Package iox contains io extensions. +package iox + +import ( + "context" + "io" +) + +// ReadAllContext reads the whole reader r in a +// background goroutine. This function will return +// earlier if the context is cancelled. In which case +// we will continue reading from r in the background +// goroutine, and we will discard the result. To stop +// the long-running goroutine, you need to close the +// connection bound to the r reader, if possible. +func ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) { + datach, errch := make(chan []byte, 1), make(chan error, 1) // buffers + go func() { + data, err := io.ReadAll(r) + if err != nil { + errch <- err + return + } + datach <- data + }() + select { + case data := <-datach: + return data, nil + case <-ctx.Done(): + return nil, ctx.Err() + case err := <-errch: + return nil, err + } +} + +// MockableReader allows to mock any io.Reader. +type MockableReader struct { + MockRead func(b []byte) (int, error) +} + +// MockableReader implements an io.Reader. +var _ io.Reader = &MockableReader{} + +// Read implements io.Reader.Read. +func (r *MockableReader) Read(b []byte) (int, error) { + return r.MockRead(b) +} diff --git a/internal/iox/iox_test.go b/internal/iox/iox_test.go new file mode 100644 index 0000000..ccd2d0b --- /dev/null +++ b/internal/iox/iox_test.go @@ -0,0 +1,70 @@ +package iox + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +func TestReadAllContextCommonCase(t *testing.T) { + r := strings.NewReader("deadbeef") + ctx := context.Background() + out, err := ReadAllContext(ctx, r) + if err != nil { + t.Fatal(err) + } + if len(out) != 8 { + t.Fatal("not the expected number of bytes") + } +} + +func TestReadAllContextWithError(t *testing.T) { + expected := errors.New("mocked error") + r := &MockableReader{ + MockRead: func(b []byte) (int, error) { + return 0, expected + }, + } + ctx := context.Background() + out, err := ReadAllContext(ctx, r) + if !errors.Is(err, expected) { + t.Fatal("not the error we expected", err) + } + if len(out) != 0 { + t.Fatal("not the expected number of bytes") + } +} + +func TestReadAllContextWithCancelledContext(t *testing.T) { + r := strings.NewReader("deadbeef") + ctx, cancel := context.WithCancel(context.Background()) + cancel() // fail immediately + out, err := ReadAllContext(ctx, r) + if !errors.Is(err, context.Canceled) { + t.Fatal("not the error we expected", err) + } + if len(out) != 0 { + t.Fatal("not the expected number of bytes") + } +} + +func TestReadAllContextWithErrorAndCancelledContext(t *testing.T) { + expected := errors.New("mocked error") + r := &MockableReader{ + MockRead: func(b []byte) (int, error) { + time.Sleep(time.Millisecond) + return 0, expected + }, + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() // fail immediately + out, err := ReadAllContext(ctx, r) + if !errors.Is(err, context.Canceled) { + t.Fatal("not the error we expected", err) + } + if len(out) != 0 { + t.Fatal("not the expected number of bytes") + } +} diff --git a/internal/ooapi/apis.go b/internal/ooapi/apis.go index dfe60c9..bb76b32 100644 --- a/internal/ooapi/apis.go +++ b/internal/ooapi/apis.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:00.422051399 +0200 CEST m=+0.000129449 +// 2021-06-15 10:55:55.967236 +0200 CEST m=+0.000374501 package ooapi @@ -59,7 +59,8 @@ func (api *simpleCheckReportIDAPI) Call(ctx context.Context, req *apimodel.Check if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleCheckInAPI implements the CheckIn API. @@ -109,7 +110,8 @@ func (api *simpleCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequ if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleLoginAPI implements the Login API. @@ -159,7 +161,8 @@ func (api *simpleLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleMeasurementMetaAPI implements the MeasurementMeta API. @@ -209,7 +212,8 @@ func (api *simpleMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.Mea if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleRegisterAPI implements the Register API. @@ -259,7 +263,8 @@ func (api *simpleRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRe if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleTestHelpersAPI implements the TestHelpers API. @@ -309,7 +314,8 @@ func (api *simpleTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHel if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simplePsiphonConfigAPI implements the PsiphonConfig API. @@ -377,7 +383,8 @@ func (api *simplePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.Psiph if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleTorTargetsAPI implements the TorTargets API. @@ -445,7 +452,8 @@ func (api *simpleTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTarge if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleURLsAPI implements the URLs API. @@ -495,7 +503,8 @@ func (api *simpleURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) ( if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleOpenReportAPI implements the OpenReport API. @@ -545,7 +554,8 @@ func (api *simpleOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenRepo if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } // simpleSubmitMeasurementAPI implements the SubmitMeasurement API. @@ -603,5 +613,6 @@ func (api *simpleSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.S if api.UserAgent != "" { httpReq.Header.Add("User-Agent", api.UserAgent) } - return api.newResponse(api.httpClient().Do(httpReq)) + httpResp, err := api.httpClient().Do(httpReq) + return api.newResponse(ctx, httpResp, err) } diff --git a/internal/ooapi/apis_test.go b/internal/ooapi/apis_test.go index 6171074..acc52bb 100644 --- a/internal/ooapi/apis_test.go +++ b/internal/ooapi/apis_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:01.254917327 +0200 CEST m=+0.000217025 +// 2021-06-15 10:55:56.308361 +0200 CEST m=+0.000215751 package ooapi @@ -9,7 +9,6 @@ import ( "context" "encoding/json" "errors" - "io/ioutil" "net/http" "net/http/httptest" "net/url" @@ -18,6 +17,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel" ) @@ -177,7 +177,7 @@ func (h *handleCheckReportID) ServeHTTP(w http.ResponseWriter, r *http.Request) } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -440,7 +440,7 @@ func (h *handleCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Request) { } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -685,7 +685,7 @@ func (h *handleLogin) ServeHTTP(w http.ResponseWriter, r *http.Request) { } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -912,7 +912,7 @@ func (h *handleMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *http.Request } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -1175,7 +1175,7 @@ func (h *handleRegister) ServeHTTP(w http.ResponseWriter, r *http.Request) { } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -1402,7 +1402,7 @@ func (h *handleTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.Request) { } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -1671,7 +1671,7 @@ func (h *handlePsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http.Request) } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -1942,7 +1942,7 @@ func (h *handleTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Request) { } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -2192,7 +2192,7 @@ func (h *handleURLs) ServeHTTP(w http.ResponseWriter, r *http.Request) { } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -2437,7 +2437,7 @@ func (h *handleOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Request) { } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -2682,7 +2682,7 @@ func (h *handleSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r *http.Reque } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return diff --git a/internal/ooapi/caching.go b/internal/ooapi/caching.go index 640cdcc..cf05d5c 100644 --- a/internal/ooapi/caching.go +++ b/internal/ooapi/caching.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:01.869492095 +0200 CEST m=+0.000168945 +// 2021-06-15 10:55:56.587245 +0200 CEST m=+0.000218959 package ooapi diff --git a/internal/ooapi/caching_test.go b/internal/ooapi/caching_test.go index 0c6c1c7..e5085d6 100644 --- a/internal/ooapi/caching_test.go +++ b/internal/ooapi/caching_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:02.497717446 +0200 CEST m=+0.000113904 +// 2021-06-15 10:55:56.851681 +0200 CEST m=+0.000186126 package ooapi diff --git a/internal/ooapi/callers.go b/internal/ooapi/callers.go index 87a9fe8..7a4ae4c 100644 --- a/internal/ooapi/callers.go +++ b/internal/ooapi/callers.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:03.02266641 +0200 CEST m=+0.000097757 +// 2021-06-15 10:55:57.174427 +0200 CEST m=+0.000217876 package ooapi diff --git a/internal/ooapi/clientcall.go b/internal/ooapi/clientcall.go index 6e094ea..60dddd6 100644 --- a/internal/ooapi/clientcall.go +++ b/internal/ooapi/clientcall.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:03.586305848 +0200 CEST m=+0.000123000 +// 2021-06-15 10:55:57.445485 +0200 CEST m=+0.000219751 package ooapi diff --git a/internal/ooapi/clientcall_test.go b/internal/ooapi/clientcall_test.go index 438b655..fa0210f 100644 --- a/internal/ooapi/clientcall_test.go +++ b/internal/ooapi/clientcall_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:04.198485035 +0200 CEST m=+0.000114145 +// 2021-06-15 10:55:57.708833 +0200 CEST m=+0.000193418 package ooapi @@ -8,7 +8,6 @@ package ooapi import ( "context" "encoding/json" - "io/ioutil" "net/http" "net/http/httptest" "net/url" @@ -16,6 +15,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/kvstore" "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel" ) @@ -42,7 +42,7 @@ func (h *handleClientCallCheckReportID) ServeHTTP(w http.ResponseWriter, r *http } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -135,7 +135,7 @@ func (h *handleClientCallCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Reque } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -227,7 +227,7 @@ func (h *handleClientCallMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *ht } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -320,7 +320,7 @@ func (h *handleClientCallTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.R } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -435,7 +435,7 @@ func (h *handleClientCallPsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -550,7 +550,7 @@ func (h *handleClientCallTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Re } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -643,7 +643,7 @@ func (h *handleClientCallURLs) ServeHTTP(w http.ResponseWriter, r *http.Request) } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -736,7 +736,7 @@ func (h *handleClientCallOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Re } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -828,7 +828,7 @@ func (h *handleClientCallSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r * } h.count++ if r.Body != nil { - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return diff --git a/internal/ooapi/cloners.go b/internal/ooapi/cloners.go index e058269..2eced32 100644 --- a/internal/ooapi/cloners.go +++ b/internal/ooapi/cloners.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:04.793154609 +0200 CEST m=+0.000108739 +// 2021-06-15 10:55:57.987741 +0200 CEST m=+0.000211126 package ooapi diff --git a/internal/ooapi/fake_test.go b/internal/ooapi/fake_test.go index 9b0b708..9738d6c 100644 --- a/internal/ooapi/fake_test.go +++ b/internal/ooapi/fake_test.go @@ -3,9 +3,10 @@ package ooapi import ( "context" "io" - "io/ioutil" "net/http" "time" + + "github.com/ooni/probe-cli/v3/internal/iox" ) type FakeCodec struct { @@ -30,7 +31,7 @@ type FakeHTTPClient struct { func (c *FakeHTTPClient) Do(req *http.Request) (*http.Response, error) { time.Sleep(10 * time.Microsecond) if req.Body != nil { - _, _ = ioutil.ReadAll(req.Body) + _, _ = iox.ReadAllContext(req.Context(), req.Body) req.Body.Close() } if c.Err != nil { diff --git a/internal/ooapi/fakeapi_test.go b/internal/ooapi/fakeapi_test.go index 12d2e14..7b040e6 100644 --- a/internal/ooapi/fakeapi_test.go +++ b/internal/ooapi/fakeapi_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:05.331414434 +0200 CEST m=+0.000124504 +// 2021-06-15 10:55:58.234786 +0200 CEST m=+0.000218167 package ooapi diff --git a/internal/ooapi/internal/generator/apis.go b/internal/ooapi/internal/generator/apis.go index 1d15cda..cbaedd9 100644 --- a/internal/ooapi/internal/generator/apis.go +++ b/internal/ooapi/internal/generator/apis.go @@ -156,7 +156,8 @@ func (d *Descriptor) genNewAPI(sb *strings.Builder) { fmt.Fprint(sb, "\tif api.UserAgent != \"\" {\n") fmt.Fprint(sb, "\t\thttpReq.Header.Add(\"User-Agent\", api.UserAgent)\n") fmt.Fprint(sb, "\t}\n") - fmt.Fprint(sb, "\treturn api.newResponse(api.httpClient().Do(httpReq))\n") + fmt.Fprint(sb, "\thttpResp, err := api.httpClient().Do(httpReq)\n") + fmt.Fprint(sb, "\treturn api.newResponse(ctx, httpResp, err)\n") fmt.Fprint(sb, "}\n\n") } diff --git a/internal/ooapi/internal/generator/apistest.go b/internal/ooapi/internal/generator/apistest.go index feeb7c8..02dba5b 100644 --- a/internal/ooapi/internal/generator/apistest.go +++ b/internal/ooapi/internal/generator/apistest.go @@ -231,7 +231,7 @@ func (d *Descriptor) genTestRoundTrip(sb *strings.Builder) { fmt.Fprint(sb, "\t}\n") fmt.Fprint(sb, "\th.count++\n") fmt.Fprint(sb, "\tif r.Body != nil {\n") - fmt.Fprint(sb, "\t\tdata, err := ioutil.ReadAll(r.Body)\n") + fmt.Fprint(sb, "\t\tdata, err := iox.ReadAllContext(r.Context(), r.Body)\n") fmt.Fprint(sb, "\t\tif err != nil {\n") fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n") fmt.Fprintf(sb, "\t\t\treturn\n") @@ -443,7 +443,6 @@ func GenAPIsTestGo(file string) { fmt.Fprint(&sb, "\t\"context\"\n") fmt.Fprint(&sb, "\t\"encoding/json\"\n") fmt.Fprint(&sb, "\t\"errors\"\n") - fmt.Fprint(&sb, "\t\"io/ioutil\"\n") fmt.Fprint(&sb, "\t\"net/http/httptest\"\n") fmt.Fprint(&sb, "\t\"net/http\"\n") fmt.Fprint(&sb, "\t\"net/url\"\n") @@ -452,6 +451,7 @@ func GenAPIsTestGo(file string) { fmt.Fprint(&sb, "\t\"sync\"\n") fmt.Fprint(&sb, "\n") fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n") + fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n") fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n") fmt.Fprint(&sb, ")\n") for _, desc := range Descriptors { diff --git a/internal/ooapi/internal/generator/clientcalltest.go b/internal/ooapi/internal/generator/clientcalltest.go index 0443d80..01fa47c 100644 --- a/internal/ooapi/internal/generator/clientcalltest.go +++ b/internal/ooapi/internal/generator/clientcalltest.go @@ -57,7 +57,7 @@ func (d *Descriptor) genTestClientCallRoundTrip(sb *strings.Builder) { fmt.Fprint(sb, "\t}\n") fmt.Fprint(sb, "\th.count++\n") fmt.Fprint(sb, "\tif r.Body != nil {\n") - fmt.Fprint(sb, "\t\tdata, err := ioutil.ReadAll(r.Body)\n") + fmt.Fprint(sb, "\t\tdata, err := iox.ReadAllContext(r.Context(), r.Body)\n") fmt.Fprint(sb, "\t\tif err != nil {\n") fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n") fmt.Fprintf(sb, "\t\t\treturn\n") @@ -161,7 +161,6 @@ func GenClientCallTestGo(file string) { fmt.Fprint(&sb, "import (\n") fmt.Fprint(&sb, "\t\"context\"\n") fmt.Fprint(&sb, "\t\"encoding/json\"\n") - fmt.Fprint(&sb, "\t\"io/ioutil\"\n") fmt.Fprint(&sb, "\t\"net/http/httptest\"\n") fmt.Fprint(&sb, "\t\"net/http\"\n") fmt.Fprint(&sb, "\t\"net/url\"\n") @@ -169,6 +168,7 @@ func GenClientCallTestGo(file string) { fmt.Fprint(&sb, "\t\"sync\"\n") fmt.Fprint(&sb, "\n") fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n") + fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n") fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/kvstore\"\n") fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n") fmt.Fprint(&sb, ")\n") diff --git a/internal/ooapi/internal/generator/responses.go b/internal/ooapi/internal/generator/responses.go index da81013..3a7ad30 100644 --- a/internal/ooapi/internal/generator/responses.go +++ b/internal/ooapi/internal/generator/responses.go @@ -9,7 +9,7 @@ import ( func (d *Descriptor) genNewResponse(sb *strings.Builder) { fmt.Fprintf(sb, - "func (api *%s) newResponse(resp *http.Response, err error) (%s, error) {\n", + "func (api *%s) newResponse(ctx context.Context, resp *http.Response, err error) (%s, error) {\n", d.APIStructName(), d.ResponseTypeName()) fmt.Fprint(sb, "\tif err != nil {\n") @@ -23,7 +23,7 @@ func (d *Descriptor) genNewResponse(sb *strings.Builder) { fmt.Fprint(sb, "\t}\n") fmt.Fprint(sb, "\tdefer resp.Body.Close()\n") fmt.Fprint(sb, "\treader := io.LimitReader(resp.Body, 4<<20)\n") - fmt.Fprint(sb, "\tdata, err := ioutil.ReadAll(reader)\n") + fmt.Fprint(sb, "\tdata, err := iox.ReadAllContext(ctx, reader)\n") fmt.Fprint(sb, "\tif err != nil {\n") fmt.Fprint(sb, "\t\treturn nil, err\n") fmt.Fprint(sb, "\t}\n") @@ -67,10 +67,11 @@ func GenResponsesGo(file string) { fmt.Fprint(&sb, "package ooapi\n\n") fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file) fmt.Fprint(&sb, "import (\n") + fmt.Fprint(&sb, "\t\"context\"\n") fmt.Fprint(&sb, "\t\"io\"\n") - fmt.Fprint(&sb, "\t\"io/ioutil\"\n") fmt.Fprint(&sb, "\t\"net/http\"\n") fmt.Fprint(&sb, "\n") + fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n") fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n") fmt.Fprint(&sb, ")\n\n") for _, desc := range Descriptors { diff --git a/internal/ooapi/login.go b/internal/ooapi/login.go index 2c52aca..f629eec 100644 --- a/internal/ooapi/login.go +++ b/internal/ooapi/login.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:06.213159282 +0200 CEST m=+0.000104135 +// 2021-06-15 10:55:58.487888 +0200 CEST m=+0.000221710 package ooapi diff --git a/internal/ooapi/login_test.go b/internal/ooapi/login_test.go index 8131618..037a4e7 100644 --- a/internal/ooapi/login_test.go +++ b/internal/ooapi/login_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:06.893164862 +0200 CEST m=+0.000136881 +// 2021-06-15 10:55:58.755431 +0200 CEST m=+0.000217917 package ooapi diff --git a/internal/ooapi/loginhandler_test.go b/internal/ooapi/loginhandler_test.go index 3ceac37..51242a9 100644 --- a/internal/ooapi/loginhandler_test.go +++ b/internal/ooapi/loginhandler_test.go @@ -2,7 +2,6 @@ package ooapi import ( "encoding/json" - "io/ioutil" "net/http" "strings" "sync" @@ -10,6 +9,7 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel" ) @@ -72,7 +72,7 @@ func (lh *LoginHandler) register(w http.ResponseWriter, r *http.Request) { w.WriteHeader(400) return } - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return @@ -113,7 +113,7 @@ func (lh *LoginHandler) login(w http.ResponseWriter, r *http.Request) { w.WriteHeader(400) return } - data, err := ioutil.ReadAll(r.Body) + data, err := iox.ReadAllContext(r.Context(), r.Body) if err != nil { w.WriteHeader(400) return diff --git a/internal/ooapi/requests.go b/internal/ooapi/requests.go index 7f23cdb..4e2ff33 100644 --- a/internal/ooapi/requests.go +++ b/internal/ooapi/requests.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:07.590481334 +0200 CEST m=+0.000085230 +// 2021-06-15 10:55:59.020364 +0200 CEST m=+0.000217085 package ooapi diff --git a/internal/ooapi/responses.go b/internal/ooapi/responses.go index da6078c..4401eb9 100644 --- a/internal/ooapi/responses.go +++ b/internal/ooapi/responses.go @@ -1,19 +1,20 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:08.237841277 +0200 CEST m=+0.000121556 +// 2021-06-15 10:55:59.329509 +0200 CEST m=+0.000220043 package ooapi //go:generate go run ./internal/generator -file responses.go import ( + "context" "io" - "io/ioutil" "net/http" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel" ) -func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) { +func (api *simpleCheckReportIDAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) { if err != nil { return nil, err } @@ -25,7 +26,7 @@ func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) ( } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -36,7 +37,7 @@ func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) ( return out, nil } -func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckInResponse, error) { +func (api *simpleCheckInAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.CheckInResponse, error) { if err != nil { return nil, err } @@ -48,7 +49,7 @@ func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimo } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -59,7 +60,7 @@ func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimo return out, nil } -func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimodel.LoginResponse, error) { +func (api *simpleLoginAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.LoginResponse, error) { if err != nil { return nil, err } @@ -71,7 +72,7 @@ func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimode } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -82,7 +83,7 @@ func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimode return out, nil } -func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) { +func (api *simpleMeasurementMetaAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) { if err != nil { return nil, err } @@ -94,7 +95,7 @@ func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error) } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -105,7 +106,7 @@ func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error) return out, nil } -func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apimodel.RegisterResponse, error) { +func (api *simpleRegisterAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.RegisterResponse, error) { if err != nil { return nil, err } @@ -117,7 +118,7 @@ func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apim } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -128,7 +129,7 @@ func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apim return out, nil } -func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (apimodel.TestHelpersResponse, error) { +func (api *simpleTestHelpersAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.TestHelpersResponse, error) { if err != nil { return nil, err } @@ -140,7 +141,7 @@ func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (ap } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -154,7 +155,7 @@ func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (ap return out, nil } -func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) { +func (api *simplePsiphonConfigAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) { if err != nil { return nil, err } @@ -166,7 +167,7 @@ func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) ( } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -180,7 +181,7 @@ func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) ( return out, nil } -func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (apimodel.TorTargetsResponse, error) { +func (api *simpleTorTargetsAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.TorTargetsResponse, error) { if err != nil { return nil, err } @@ -192,7 +193,7 @@ func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (api } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -206,7 +207,7 @@ func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (api return out, nil } -func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel.URLsResponse, error) { +func (api *simpleURLsAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.URLsResponse, error) { if err != nil { return nil, err } @@ -218,7 +219,7 @@ func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -229,7 +230,7 @@ func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel return out, nil } -func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*apimodel.OpenReportResponse, error) { +func (api *simpleOpenReportAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.OpenReportResponse, error) { if err != nil { return nil, err } @@ -241,7 +242,7 @@ func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*ap } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } @@ -252,7 +253,7 @@ func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*ap return out, nil } -func (api *simpleSubmitMeasurementAPI) newResponse(resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) { +func (api *simpleSubmitMeasurementAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) { if err != nil { return nil, err } @@ -264,7 +265,7 @@ func (api *simpleSubmitMeasurementAPI) newResponse(resp *http.Response, err erro } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 4<<20) - data, err := ioutil.ReadAll(reader) + data, err := iox.ReadAllContext(ctx, reader) if err != nil { return nil, err } diff --git a/internal/ooapi/swagger_test.go b/internal/ooapi/swagger_test.go index c44a93e..5539e5e 100644 --- a/internal/ooapi/swagger_test.go +++ b/internal/ooapi/swagger_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-05-12 09:15:08.888123159 +0200 CEST m=+0.000883704 +// 2021-06-15 10:55:59.636856 +0200 CEST m=+0.000624459 package ooapi @@ -9,7 +9,7 @@ const swagger = `{ "swagger": "2.0", "info": { "title": "OONI API specification", - "version": "0.20210512.5071508" + "version": "0.20210615.6085559" }, "host": "api.ooni.io", "basePath": "/", diff --git a/internal/ooapi/swaggerdiff_test.go b/internal/ooapi/swaggerdiff_test.go index d805223..31ea27c 100644 --- a/internal/ooapi/swaggerdiff_test.go +++ b/internal/ooapi/swaggerdiff_test.go @@ -1,9 +1,9 @@ package ooapi import ( + "context" "encoding/json" "fmt" - "io/ioutil" "log" "net/http" "sort" @@ -13,6 +13,7 @@ import ( "github.com/hexops/gotextdiff" "github.com/hexops/gotextdiff/myers" "github.com/hexops/gotextdiff/span" + "github.com/ooni/probe-cli/v3/internal/iox" "github.com/ooni/probe-cli/v3/internal/ooapi/internal/openapi" ) @@ -36,7 +37,7 @@ func getServerModel(serverURL string) *openapi.Swagger { log.Fatal(err) } defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { log.Fatal(err) } diff --git a/internal/ptx/ptx_test.go b/internal/ptx/ptx_test.go index c7ef744..64f5bc6 100644 --- a/internal/ptx/ptx_test.go +++ b/internal/ptx/ptx_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -15,6 +14,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/engine/netx/mockablex" + "github.com/ooni/probe-cli/v3/internal/iox" ) func TestListenerLoggerWorks(t *testing.T) { @@ -57,7 +57,7 @@ func TestListenerWorksWithFakeDialer(t *testing.T) { if err != nil { t.Fatal(err) } - data, err := ioutil.ReadAll(resp.Body) + data, err := iox.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) }