fix(all): introduce and use iox.ReadAllContext (#379)

* fix(all): introduce and use iox.ReadAllContext

This improvement over the ioutil.ReadAll utility returns early
if the context expires. This enables us to unblock stuck code in
case there's censorship confounding the TCP stack.

See https://github.com/ooni/probe/issues/1417.

Compared to the functionality postulated in the above mentioned
issue, I choose to be more generic and separate limiting the
maximum body size (not implemented here) from using the context
to return early when reading a body (or any other reader).

After implementing iox.ReadAllContext, I made sure we always
use it everywhere in the tree instead of ioutil.ReadAll.

This includes many parts of the codebase where in theory we don't
need iox.ReadAllContext. Though, changing all the places makes
checking whether we're not using ioutil.ReadAll where we should
not be using it easy: `git grep` should return no lines.

* Update internal/iox/iox_test.go

* fix(ndt7): treat context errors as non-errors

The rationale is explained by the comment documenting reduceErr.

* Update internal/engine/experiment/ndt7/download.go
This commit is contained in:
Simone Basso 2021-06-15 11:57:40 +02:00 committed by GitHub
parent 576e035b20
commit 0fdc9cafb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
76 changed files with 440 additions and 223 deletions

View File

@ -63,6 +63,8 @@ run `go mod tidy` to minimize such changes.
- use `./internal/fsx.OpenFile` when you need to open a file - use `./internal/fsx.OpenFile` when you need to open a file
- use `./internal/iox.ReadAllContext` instead of `ioutil.ReadAll`
## Code testing requirements ## Code testing requirements
Make sure all tests pass with `go test -race ./...` run from the Make sure all tests pass with `go test -race ./...` run from the

View File

@ -1,11 +1,12 @@
package database package database
import ( import (
"context"
"database/sql" "database/sql"
"embed" "embed"
"io/ioutil"
"github.com/apex/log" "github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/iox"
migrate "github.com/rubenv/sql-migrate" migrate "github.com/rubenv/sql-migrate"
"upper.io/db.v3/lib/sqlbuilder" "upper.io/db.v3/lib/sqlbuilder"
"upper.io/db.v3/sqlite" "upper.io/db.v3/sqlite"
@ -19,7 +20,7 @@ func readAsset(path string) ([]byte, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return ioutil.ReadAll(filep) return iox.ReadAllContext(context.Background(), filep)
} }
func readAssetDir(path string) ([]string, error) { func readAssetDir(path string) ([]string, error) {

View File

@ -6,16 +6,17 @@
package badproxy package badproxy
import ( import (
"context"
"crypto/rsa" "crypto/rsa"
"crypto/tls" "crypto/tls"
"crypto/x509" "crypto/x509"
"io" "io"
"io/ioutil"
"net" "net"
"strings" "strings"
"time" "time"
"github.com/google/martian/v3/mitm" "github.com/google/martian/v3/mitm"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
// CensoringProxy is a proxy that does not behave correctly. // CensoringProxy is a proxy that does not behave correctly.
@ -56,7 +57,7 @@ func (p *CensoringProxy) serve(conn net.Conn) {
} else { } else {
const maxread = 1 << 17 const maxread = 1 << 17
reader := io.LimitReader(conn, maxread) reader := io.LimitReader(conn, maxread)
ioutil.ReadAll(reader) iox.ReadAllContext(context.Background(), reader)
} }
conn.Close() conn.Close()
} }

View File

@ -3,12 +3,12 @@ package httpproxy
import ( import (
"bytes" "bytes"
"context" "context"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"testing" "testing"
"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestPass(t *testing.T) { func TestPass(t *testing.T) {
@ -89,7 +89,7 @@ func checkrequest(
t.Fatal("unexpected value of status code") t.Fatal("unexpected value of status code")
} }
t.Log(resp) t.Log(resp)
values, _ := resp.Header["Via"] values := resp.Header["Via"]
var foundProduct bool var foundProduct bool
for _, value := range values { for _, value := range values {
if value == product { if value == product {
@ -102,7 +102,7 @@ func checkrequest(
if !foundProduct && expectVia { if !foundProduct && expectVia {
t.Fatal("Via header not found") t.Fatal("Via header not found")
} }
proxiedData, err := ioutil.ReadAll(resp.Body) proxiedData, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -120,7 +120,7 @@ func checkbody(t *testing.T, proxiedData []byte, host string) {
t.Fatal("unexpected status code") t.Fatal("unexpected status code")
} }
defer resp.Body.Close() defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -3,10 +3,11 @@ package uncensored
import ( import (
"bytes" "bytes"
"context" "context"
"io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"testing" "testing"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestGood(t *testing.T) { func TestGood(t *testing.T) {
@ -55,7 +56,7 @@ func TestGood(t *testing.T) {
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
t.Fatal("invalid status-code") t.Fatal("invalid status-code")
} }
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -6,7 +6,6 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
@ -15,6 +14,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/httpheader" "github.com/ooni/probe-cli/v3/internal/engine/httpheader"
"github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version" "github.com/ooni/probe-cli/v3/internal/version"
) )
@ -126,7 +126,7 @@ func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, erro
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
return nil, ErrHTTPStatusCode return nil, ErrHTTPStatusCode
} }
data, err = ioutil.ReadAll(resp.Body) data, err = iox.ReadAllContext(ctx, resp.Body)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -3,13 +3,13 @@ package internal
import ( import (
"context" "context"
"io" "io"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeResolver struct { type FakeResolver struct {
@ -63,7 +63,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req) return txp.Func(req)
} }
if req.Body != nil { if req.Body != nil {
ioutil.ReadAll(req.Body) iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if txp.Err != nil { if txp.Err != nil {

View File

@ -3,13 +3,13 @@ package internal
import ( import (
"context" "context"
"io" "io"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeResolver struct { type FakeResolver struct {
@ -63,7 +63,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req) return txp.Func(req)
} }
if req.Body != nil { if req.Body != nil {
ioutil.ReadAll(req.Body) iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if txp.Err != nil { if txp.Err != nil {

View File

@ -3,12 +3,12 @@ package internal
import ( import (
"context" "context"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"strings" "strings"
"sync" "sync"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
// CtrlHTTPResponse is the result of the HTTP check performed by // CtrlHTTPResponse is the result of the HTTP check performed by
@ -56,7 +56,7 @@ func HTTPDo(ctx context.Context, config *HTTPConfig) {
headers[k] = resp.Header.Get(k) headers[k] = resp.Header.Get(k)
} }
reader := &io.LimitedReader{R: resp.Body, N: config.MaxAcceptableBody} reader := &io.LimitedReader{R: resp.Body, N: config.MaxAcceptableBody}
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
config.Out <- CtrlHTTPResponse{ config.Out <- CtrlHTTPResponse{
BodyLength: int64(len(data)), BodyLength: int64(len(data)),
Failure: newfailure(err), Failure: newfailure(err),

View File

@ -4,10 +4,10 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/version" "github.com/ooni/probe-cli/v3/internal/version"
) )
@ -32,7 +32,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
reader := &io.LimitedReader{R: req.Body, N: h.MaxAcceptableBody} reader := &io.LimitedReader{R: req.Body, N: h.MaxAcceptableBody}
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(req.Context(), reader)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -42,12 +42,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(400) w.WriteHeader(400)
return return
} }
measureConfig := MeasureConfig{ measureConfig := MeasureConfig(h)
Client: h.Client,
Dialer: h.Dialer,
MaxAcceptableBody: h.MaxAcceptableBody,
Resolver: h.Resolver,
}
cresp, err := Measure(req.Context(), measureConfig, &creq) cresp, err := Measure(req.Context(), measureConfig, &creq)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)

View File

@ -1,9 +1,9 @@
package internal_test package internal_test
import ( import (
"context"
"encoding/json" "encoding/json"
"errors" "errors"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -12,6 +12,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal" "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
const simplerequest = `{ const simplerequest = `{
@ -126,7 +127,7 @@ func TestWorkingAsIntended(t *testing.T) {
if v := resp.Header.Get("content-type"); v != expect.respContentType { if v := resp.Header.Get("content-type"); v != expect.respContentType {
t.Fatalf("unexpected content-type: %s", v) t.Fatalf("unexpected content-type: %s", v)
} }
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -16,7 +16,7 @@ type collectDeps interface {
JSONMarshal(v interface{}) ([]byte, error) JSONMarshal(v interface{}) ([]byte, error)
Logger() model.Logger Logger() model.Logger
NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error) NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
ReadAll(r io.Reader) ([]byte, error) ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error)
Scheme() string Scheme() string
UserAgent() string UserAgent() string
} }
@ -47,7 +47,7 @@ func collect(ctx context.Context, fqdn, authorization string,
return errHTTPRequestFailed return errHTTPRequestFailed
} }
defer resp.Body.Close() defer resp.Body.Close()
data, err = deps.ReadAll(resp.Body) data, err = deps.ReadAllContext(ctx, resp.Body)
if err != nil { if err != nil {
return err return err
} }

View File

@ -9,7 +9,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"runtime" "runtime"
"time" "time"
@ -20,6 +19,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/humanize" "github.com/ooni/probe-cli/v3/internal/humanize"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
const ( const (
@ -86,8 +86,8 @@ func (r runner) NewHTTPRequest(meth, url string, body io.Reader) (*http.Request,
return http.NewRequest(meth, url, body) return http.NewRequest(meth, url, body)
} }
func (r runner) ReadAll(reader io.Reader) ([]byte, error) { func (r runner) ReadAllContext(ctx context.Context, reader io.Reader) ([]byte, error) {
return ioutil.ReadAll(reader) return iox.ReadAllContext(ctx, reader)
} }
func (r runner) Scheme() string { func (r runner) Scheme() string {
@ -179,7 +179,7 @@ func (r runner) measure(
current.ConnectTime = connectTime current.ConnectTime = connectTime
r.tk.ReceiverData = append(r.tk.ReceiverData, current) r.tk.ReceiverData = append(r.tk.ReceiverData, current)
total += current.Received total += current.Received
avgspeed := 8 * float64(total) / time.Now().Sub(begin).Seconds() avgspeed := 8 * float64(total) / time.Since(begin).Seconds()
percentage := float64(current.Iteration) / float64(numIterations) percentage := float64(current.Iteration) / float64(numIterations)
message := fmt.Sprintf("streaming: speed: %s", humanize.SI(avgspeed, "bit/s")) message := fmt.Sprintf("streaming: speed: %s", humanize.SI(avgspeed, "bit/s"))
r.callbacks.OnProgress(percentage, message) r.callbacks.OnProgress(percentage, message)

View File

@ -12,7 +12,7 @@ import (
type downloadDeps interface { type downloadDeps interface {
HTTPClient() *http.Client HTTPClient() *http.Client
NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error) NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
ReadAll(r io.Reader) ([]byte, error) ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error)
Scheme() string Scheme() string
UserAgent() string UserAgent() string
} }
@ -57,7 +57,7 @@ func download(ctx context.Context, config downloadConfig) (downloadResult, error
return result, errHTTPRequestFailed return result, errHTTPRequestFailed
} }
defer resp.Body.Close() defer resp.Body.Close()
data, err := config.deps.ReadAll(resp.Body) data, err := config.deps.ReadAllContext(ctx, resp.Body)
if err != nil { if err != nil {
return result, err return result, err
} }
@ -66,7 +66,7 @@ func download(ctx context.Context, config downloadConfig) (downloadResult, error
// turns out that Neubot and MK do the same. So, we do what they do. At // turns out that Neubot and MK do the same. So, we do what they do. At
// the same time, we are currently not able to include the overhead that // the same time, we are currently not able to include the overhead that
// is caused by HTTP headers etc. So, we're a bit less precise. // is caused by HTTP headers etc. So, we're a bit less precise.
result.elapsed = time.Now().Sub(savedTicks).Seconds() result.elapsed = time.Since(savedTicks).Seconds()
result.received = int64(len(data)) result.received = int64(len(data))
result.requestTicks = savedTicks.Sub(config.begin).Seconds() result.requestTicks = savedTicks.Sub(config.begin).Seconds()
result.timestamp = time.Now().Unix() result.timestamp = time.Now().Unix()

View File

@ -1,6 +1,7 @@
package dash package dash
import ( import (
"context"
"io" "io"
"net/http" "net/http"
"time" "time"
@ -36,7 +37,7 @@ func (d FakeDeps) NewHTTPRequest(
return d.newHTTPRequestResult, d.newHTTPRequestErr return d.newHTTPRequestResult, d.newHTTPRequestErr
} }
func (d FakeDeps) ReadAll(r io.Reader) ([]byte, error) { func (d FakeDeps) ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) {
return d.readAllResult, d.readAllErr return d.readAllResult, d.readAllErr
} }

View File

@ -16,7 +16,7 @@ type negotiateDeps interface {
JSONMarshal(v interface{}) ([]byte, error) JSONMarshal(v interface{}) ([]byte, error)
Logger() model.Logger Logger() model.Logger
NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error) NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
ReadAll(r io.Reader) ([]byte, error) ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error)
Scheme() string Scheme() string
UserAgent() string UserAgent() string
} }
@ -48,7 +48,7 @@ func negotiate(
return negotiateResp, errHTTPRequestFailed return negotiateResp, errHTTPRequestFailed
} }
defer resp.Body.Close() defer resp.Body.Close()
data, err = deps.ReadAll(resp.Body) data, err = deps.ReadAllContext(ctx, resp.Body)
if err != nil { if err != nil {
return negotiateResp, err return negotiateResp, err
} }

View File

@ -1,12 +1,6 @@
package dash package dash
const ( const (
// currentServerSchemaVersion is the version of the server schema that
// will be adopted by this implementation. Version 3 is the one that is
// Neubot uses. We needed to bump the version because Web100 is not on
// M-Lab anymore and hence we need to make a breaking change.
currentServerSchemaVersion = 4
// negotiatePath is the URL path used to negotiate // negotiatePath is the URL path used to negotiate
negotiatePath = "/negotiate/dash" negotiatePath = "/negotiate/dash"

View File

@ -2,10 +2,11 @@ package hhfm_test
import ( import (
"context" "context"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeDialer struct { type FakeDialer struct {
@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req) return txp.Func(req)
} }
if req.Body != nil { if req.Body != nil {
ioutil.ReadAll(req.Body) iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if txp.Err != nil { if txp.Err != nil {

View File

@ -9,7 +9,6 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"sort" "sort"
@ -21,6 +20,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival" "github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer" "github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/randx" "github.com/ooni/probe-cli/v3/internal/randx"
) )
@ -198,7 +198,7 @@ func transact(txp Transport, req *http.Request,
return nil, nil, urlgetter.ErrHTTPRequestFailed return nil, nil, urlgetter.ErrHTTPRequestFailed
} }
callbacks.OnProgress(0.75, "reading response body...") callbacks.OnProgress(0.75, "reading response body...")
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(req.Context(), resp.Body)
callbacks.OnProgress(1.00, fmt.Sprintf("got reseponse body... %+v", err)) callbacks.OnProgress(1.00, fmt.Sprintf("got reseponse body... %+v", err))
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err

View File

@ -2,11 +2,12 @@ package ndt7
import ( import (
"context" "context"
"errors"
"io" "io"
"io/ioutil"
"time" "time"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type downloadManager struct { type downloadManager struct {
@ -33,6 +34,30 @@ func newDownloadManager(
} }
func (mgr downloadManager) run(ctx context.Context) error { func (mgr downloadManager) run(ctx context.Context) error {
return mgr.reduceErr(mgr.doRun(ctx))
}
// reduceErr treats as non-errors the errors caused by the context
// so that we can focus instead on network errors.
//
// This function was introduced by https://github.com/ooni/probe-cli/pull/379
// since before such a PR we did not see context interrupting
// errors when we were reading messages. Since before such a PR
// we used to return `nil` on context errors, this function is
// here to keep the previous behavior by filtering the error
// returned when reading messages, given that now reading messages
// can fail midway because we use iox.ReadAllContext.
func (mgr downloadManager) reduceErr(err error) error {
if errors.Is(err, context.Canceled) {
return nil
}
if errors.Is(err, context.DeadlineExceeded) {
return nil
}
return err
}
func (mgr downloadManager) doRun(ctx context.Context) error {
var total int64 var total int64
start := time.Now() start := time.Now()
if err := mgr.conn.SetReadDeadline(start.Add(mgr.maxRuntime)); err != nil { if err := mgr.conn.SetReadDeadline(start.Add(mgr.maxRuntime)); err != nil {
@ -47,7 +72,7 @@ func (mgr downloadManager) run(ctx context.Context) error {
return err return err
} }
if kind == websocket.TextMessage { if kind == websocket.TextMessage {
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return err return err
} }
@ -57,7 +82,7 @@ func (mgr downloadManager) run(ctx context.Context) error {
} }
continue continue
} }
n, err := io.Copy(ioutil.Discard, reader) n, err := io.Copy(io.Discard, reader)
if err != nil { if err != nil {
return err return err
} }

View File

@ -143,3 +143,20 @@ type goodJSONReader struct{}
func (r *goodJSONReader) Read(p []byte) (int, error) { func (r *goodJSONReader) Read(p []byte) (int, error) {
return copy(p, []byte(`{}`)), io.EOF return copy(p, []byte(`{}`)), io.EOF
} }
func TestDownloadReduceErr(t *testing.T) {
mgr := downloadManager{}
if mgr.reduceErr(context.Canceled) != nil {
t.Fatal("reduceErr not working as intended for context.Canceled")
}
if mgr.reduceErr(context.DeadlineExceeded) != nil {
t.Fatal("reduceErr not working as intended for context.DeadlineExceeded")
}
if mgr.reduceErr(nil) != nil {
t.Fatal("reduceErr not working as intended for nil")
}
expected := errors.New("mocked error")
if mgr.reduceErr(expected) != expected {
t.Fatal("reduceErr not working as intended for other errors")
}
}

View File

@ -1,9 +1,10 @@
package geolocate package geolocate
import ( import (
"io/ioutil"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeTransport struct { type FakeTransport struct {
@ -14,7 +15,7 @@ type FakeTransport struct {
func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond) time.Sleep(10 * time.Microsecond)
if req.Body != nil { if req.Body != nil {
ioutil.ReadAll(req.Body) iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if txp.Err != nil { if txp.Err != nil {

View File

@ -1,9 +1,10 @@
package httpx package httpx
import ( import (
"io/ioutil"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeTransport struct { type FakeTransport struct {
@ -18,7 +19,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req) return txp.Func(req)
} }
if req.Body != nil { if req.Body != nil {
ioutil.ReadAll(req.Body) iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if txp.Err != nil { if txp.Err != nil {

View File

@ -7,9 +7,10 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
// Logger is the definition of Logger used by this package. // Logger is the definition of Logger used by this package.
@ -100,7 +101,7 @@ func (c Client) Do(request *http.Request) ([]byte, error) {
if response.StatusCode >= 400 { if response.StatusCode >= 400 {
return nil, fmt.Errorf("httpx: request failed: %s", response.Status) return nil, fmt.Errorf("httpx: request failed: %s", response.Status)
} }
return ioutil.ReadAll(response.Body) return iox.ReadAllContext(request.Context(), response.Body)
} }
// DoJSON performs the provided request and unmarshals the JSON response body // DoJSON performs the provided request and unmarshals the JSON response body

View File

@ -6,11 +6,11 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
// Client is a locate.measurementlab.net client. // Client is a locate.measurementlab.net client.
@ -63,7 +63,7 @@ func (c *Client) Query(ctx context.Context, tool string) (Result, error) {
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
return Result{}, fmt.Errorf("mlablocate: non-200 status code: %d", resp.StatusCode) return Result{}, fmt.Errorf("mlablocate: non-200 status code: %d", resp.StatusCode)
} }
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(ctx, resp.Body)
if err != nil { if err != nil {
return Result{}, err return Result{}, err
} }

View File

@ -1,9 +1,10 @@
package mlablocatev2 package mlablocatev2
import ( import (
"io/ioutil"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeTransport struct { type FakeTransport struct {
@ -18,7 +19,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req) return txp.Func(req)
} }
if req.Body != nil { if req.Body != nil {
ioutil.ReadAll(req.Body) iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if txp.Err != nil { if txp.Err != nil {

View File

@ -6,12 +6,12 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"regexp" "regexp"
"github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
const ( const (
@ -102,7 +102,7 @@ func (c Client) query(ctx context.Context, path string) (resultRecord, error) {
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
return resultRecord{}, fmt.Errorf("%w: %d", ErrRequestFailed, resp.StatusCode) return resultRecord{}, fmt.Errorf("%w: %d", ErrRequestFailed, resp.StatusCode)
} }
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(ctx, resp.Body)
if err != nil { if err != nil {
return resultRecord{}, err return resultRecord{}, err
} }

View File

@ -135,7 +135,7 @@ func (d *Dialer) SetCABundle(path string) error {
return err return err
} }
pool := x509.NewCertPool() pool := x509.NewCertPool()
if pool.AppendCertsFromPEM(cert) == false { if !pool.AppendCertsFromPEM(cert) {
return errors.New("AppendCertsFromPEM failed") return errors.New("AppendCertsFromPEM failed")
} }
d.TLSConfig.RootCAs = pool d.TLSConfig.RootCAs = pool

View File

@ -4,7 +4,6 @@ import (
"context" "context"
"crypto/x509" "crypto/x509"
"errors" "errors"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -15,6 +14,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func dowithclient(t *testing.T, client *netx.HTTPClient) { func dowithclient(t *testing.T, client *netx.HTTPClient) {
@ -24,7 +24,7 @@ func dowithclient(t *testing.T, client *netx.HTTPClient) {
t.Fatal(err) t.Fatal(err)
} }
defer resp.Body.Close() defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body) _, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -126,7 +126,7 @@ func httpProxyTestMain(t *testing.T, client *http.Client, expect int) {
t.Fatal(err) t.Fatal(err)
} }
defer resp.Body.Close() defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body) _, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -63,7 +63,7 @@ func (bw *bodyWrapper) Read(b []byte) (n int, err error) {
// bytes read (0 <= n <= len(p)) and any error encountered." // bytes read (0 <= n <= len(p)) and any error encountered."
Data: b[:n], Data: b[:n],
Error: err, Error: err,
DurationSinceBeginning: time.Now().Sub(bw.root.Beginning), DurationSinceBeginning: time.Since(bw.root.Beginning),
TransactionID: bw.tid, TransactionID: bw.tid,
}, },
}) })
@ -74,7 +74,7 @@ func (bw *bodyWrapper) Close() (err error) {
err = bw.ReadCloser.Close() err = bw.ReadCloser.Close()
bw.root.Handler.OnMeasurement(modelx.Measurement{ bw.root.Handler.OnMeasurement(modelx.Measurement{
HTTPResponseDone: &modelx.HTTPResponseDoneEvent{ HTTPResponseDone: &modelx.HTTPResponseDoneEvent{
DurationSinceBeginning: time.Now().Sub(bw.root.Beginning), DurationSinceBeginning: time.Since(bw.root.Beginning),
TransactionID: bw.tid, TransactionID: bw.tid,
}, },
}) })

View File

@ -1,9 +1,11 @@
package oldhttptransport package oldhttptransport
import ( import (
"io/ioutil" "context"
"net/http" "net/http"
"testing" "testing"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestBodyTracerSuccess(t *testing.T) { func TestBodyTracerSuccess(t *testing.T) {
@ -15,7 +17,7 @@ func TestBodyTracerSuccess(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer resp.Body.Close() defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body) _, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -1,9 +1,11 @@
package oldhttptransport package oldhttptransport
import ( import (
"io/ioutil" "context"
"net/http" "net/http"
"testing" "testing"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestGood(t *testing.T) { func TestGood(t *testing.T) {
@ -15,7 +17,7 @@ func TestGood(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer resp.Body.Close() defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body) _, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -2,9 +2,9 @@ package oldhttptransport
import ( import (
"bytes" "bytes"
"context"
"crypto/tls" "crypto/tls"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"net/http/httptrace" "net/http/httptrace"
"sync" "sync"
@ -16,21 +16,22 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/transactionid" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/transactionid"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
// TraceTripper performs single HTTP transactions. // TraceTripper performs single HTTP transactions.
type TraceTripper struct { type TraceTripper struct {
readAllErrs *atomicx.Int64 readAllErrs *atomicx.Int64
readAll func(r io.Reader) ([]byte, error) readAllContext func(ctx context.Context, r io.Reader) ([]byte, error)
roundTripper http.RoundTripper roundTripper http.RoundTripper
} }
// NewTraceTripper creates a new Transport. // NewTraceTripper creates a new Transport.
func NewTraceTripper(roundTripper http.RoundTripper) *TraceTripper { func NewTraceTripper(roundTripper http.RoundTripper) *TraceTripper {
return &TraceTripper{ return &TraceTripper{
readAllErrs: &atomicx.Int64{}, readAllErrs: &atomicx.Int64{},
readAll: ioutil.ReadAll, readAllContext: iox.ReadAllContext,
roundTripper: roundTripper, roundTripper: roundTripper,
} }
} }
@ -57,10 +58,10 @@ func (c *readCloseWrapper) Close() error {
} }
func readSnap( func readSnap(
source *io.ReadCloser, limit int64, ctx context.Context, source *io.ReadCloser, limit int64,
readAll func(r io.Reader) ([]byte, error), readAllContext func(ctx context.Context, r io.Reader) ([]byte, error),
) (data []byte, err error) { ) (data []byte, err error) {
data, err = readAll(io.LimitReader(*source, limit)) data, err = readAllContext(ctx, io.LimitReader(*source, limit))
if err == nil { if err == nil {
*source = newReadCloseWrapper( *source = newReadCloseWrapper(
io.MultiReader(bytes.NewReader(data), *source), io.MultiReader(bytes.NewReader(data), *source),
@ -79,7 +80,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
root.Handler.OnMeasurement(modelx.Measurement{ root.Handler.OnMeasurement(modelx.Measurement{
HTTPRoundTripStart: &modelx.HTTPRoundTripStartEvent{ HTTPRoundTripStart: &modelx.HTTPRoundTripStartEvent{
DialID: dialid.ContextDialID(req.Context()), DialID: dialid.ContextDialID(req.Context()),
DurationSinceBeginning: time.Now().Sub(root.Beginning), DurationSinceBeginning: time.Since(root.Beginning),
Method: req.Method, Method: req.Method,
TransactionID: tid, TransactionID: tid,
URL: req.URL.String(), URL: req.URL.String(),
@ -98,7 +99,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
// Save a snapshot of the request body // Save a snapshot of the request body
if req.Body != nil { if req.Body != nil {
requestBody, err = readSnap(&req.Body, snapSize, t.readAll) requestBody, err = readSnap(req.Context(), &req.Body, snapSize, t.readAllContext)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -114,7 +115,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
// configured in the http.Transport // configured in the http.Transport
root.Handler.OnMeasurement(modelx.Measurement{ root.Handler.OnMeasurement(modelx.Measurement{
TLSHandshakeStart: &modelx.TLSHandshakeStartEvent{ TLSHandshakeStart: &modelx.TLSHandshakeStartEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning), DurationSinceBeginning: time.Since(root.Beginning),
TransactionID: tid, TransactionID: tid,
}, },
}) })
@ -127,7 +128,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
Operation: errorx.TLSHandshakeOperation, Operation: errorx.TLSHandshakeOperation,
TransactionID: tid, TransactionID: tid,
}.MaybeBuild() }.MaybeBuild()
durationSinceBeginning := time.Now().Sub(root.Beginning) durationSinceBeginning := time.Since(root.Beginning)
// Event emitted by net/http when DialTLS is not // Event emitted by net/http when DialTLS is not
// configured in the http.Transport // configured in the http.Transport
root.Handler.OnMeasurement(modelx.Measurement{ root.Handler.OnMeasurement(modelx.Measurement{
@ -149,7 +150,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
info.Conn.LocalAddr().Network(), info.Conn.LocalAddr().Network(),
info.Conn.LocalAddr().String(), info.Conn.LocalAddr().String(),
), ),
DurationSinceBeginning: time.Now().Sub(root.Beginning), DurationSinceBeginning: time.Since(root.Beginning),
TransactionID: tid, TransactionID: tid,
}, },
}) })
@ -165,7 +166,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
requestHeadersMu.Unlock() requestHeadersMu.Unlock()
root.Handler.OnMeasurement(modelx.Measurement{ root.Handler.OnMeasurement(modelx.Measurement{
HTTPRequestHeader: &modelx.HTTPRequestHeaderEvent{ HTTPRequestHeader: &modelx.HTTPRequestHeaderEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning), DurationSinceBeginning: time.Since(root.Beginning),
Key: key, Key: key,
TransactionID: tid, TransactionID: tid,
Value: values, Value: values,
@ -175,7 +176,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
WroteHeaders: func() { WroteHeaders: func() {
root.Handler.OnMeasurement(modelx.Measurement{ root.Handler.OnMeasurement(modelx.Measurement{
HTTPRequestHeadersDone: &modelx.HTTPRequestHeadersDoneEvent{ HTTPRequestHeadersDone: &modelx.HTTPRequestHeadersDoneEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning), DurationSinceBeginning: time.Since(root.Beginning),
Headers: requestHeaders, // [*] Headers: requestHeaders, // [*]
Method: req.Method, // [*] Method: req.Method, // [*]
TransactionID: tid, TransactionID: tid,
@ -193,7 +194,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
}.MaybeBuild() }.MaybeBuild()
root.Handler.OnMeasurement(modelx.Measurement{ root.Handler.OnMeasurement(modelx.Measurement{
HTTPRequestDone: &modelx.HTTPRequestDoneEvent{ HTTPRequestDone: &modelx.HTTPRequestDoneEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning), DurationSinceBeginning: time.Since(root.Beginning),
Error: err, Error: err,
TransactionID: tid, TransactionID: tid,
}, },
@ -202,7 +203,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
GotFirstResponseByte: func() { GotFirstResponseByte: func() {
root.Handler.OnMeasurement(modelx.Measurement{ root.Handler.OnMeasurement(modelx.Measurement{
HTTPResponseStart: &modelx.HTTPResponseStartEvent{ HTTPResponseStart: &modelx.HTTPResponseStartEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning), DurationSinceBeginning: time.Since(root.Beginning),
TransactionID: tid, TransactionID: tid,
}, },
}) })
@ -233,7 +234,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
// [*] Require less event joining work by providing info that // [*] Require less event joining work by providing info that
// makes this event alone actionable for OONI // makes this event alone actionable for OONI
event := &modelx.HTTPRoundTripDoneEvent{ event := &modelx.HTTPRoundTripDoneEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning), DurationSinceBeginning: time.Since(root.Beginning),
Error: err, Error: err,
RequestBodySnap: requestBody, RequestBodySnap: requestBody,
RequestHeaders: requestHeaders, // [*] RequestHeaders: requestHeaders, // [*]
@ -248,7 +249,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
event.ResponseProto = resp.Proto event.ResponseProto = resp.Proto
// Save a snapshot of the response body // Save a snapshot of the response body
var data []byte var data []byte
data, err = readSnap(&resp.Body, snapSize, t.readAll) data, err = readSnap(req.Context(), &resp.Body, snapSize, t.readAllContext)
if err != nil { if err != nil {
t.readAllErrs.Add(1) t.readAllErrs.Add(1)
resp = nil // this is how net/http likes it resp = nil // this is how net/http likes it

View File

@ -5,7 +5,6 @@ import (
"context" "context"
"errors" "errors"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"net/http/httptrace" "net/http/httptrace"
"sync" "sync"
@ -14,6 +13,7 @@ import (
"github.com/miekg/dns" "github.com/miekg/dns"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestTraceTripperSuccess(t *testing.T) { func TestTraceTripperSuccess(t *testing.T) {
@ -25,7 +25,7 @@ func TestTraceTripperSuccess(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer resp.Body.Close() defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body) _, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -47,7 +47,7 @@ func (h *roundTripHandler) OnMeasurement(m modelx.Measurement) {
func TestTraceTripperReadAllFailure(t *testing.T) { func TestTraceTripperReadAllFailure(t *testing.T) {
transport := NewTraceTripper(http.DefaultTransport) transport := NewTraceTripper(http.DefaultTransport)
transport.readAll = func(r io.Reader) ([]byte, error) { transport.readAllContext = func(ctx context.Context, r io.Reader) ([]byte, error) {
return nil, io.EOF return nil, io.EOF
} }
client := &http.Client{Transport: transport} client := &http.Client{Transport: transport}
@ -156,7 +156,7 @@ func TestTraceTripperWithCorrectSnaps(t *testing.T) {
// Read the whole response body, parse it as valid DNS // Read the whole response body, parse it as valid DNS
// reply and verify we obtained what we expected // reply and verify we obtained what we expected
replyData, err := ioutil.ReadAll(resp.Body) replyData, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -230,7 +230,7 @@ func TestTraceTripperWithReadAllFailingForBody(t *testing.T) {
// use such transport to configure an ordinary client // use such transport to configure an ordinary client
transport := NewTraceTripper(http.DefaultTransport) transport := NewTraceTripper(http.DefaultTransport)
errorMocked := errors.New("mocked error") errorMocked := errors.New("mocked error")
transport.readAll = func(r io.Reader) ([]byte, error) { transport.readAllContext = func(ctx context.Context, r io.Reader) ([]byte, error) {
return nil, errorMocked return nil, errorMocked
} }
const snapSize = 15 const snapSize = 15

View File

@ -1,11 +1,12 @@
package oldhttptransport package oldhttptransport
import ( import (
"io/ioutil" "context"
"net/http" "net/http"
"testing" "testing"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/transactionid" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/transactionid"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type transactionerCheckTransactionID struct { type transactionerCheckTransactionID struct {
@ -33,7 +34,7 @@ func TestTransactionerSuccess(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer resp.Body.Close() defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body) _, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -1,7 +1,7 @@
package netxlogger package netxlogger
import ( import (
"io/ioutil" "context"
"net/http" "net/http"
"testing" "testing"
"time" "time"
@ -10,6 +10,7 @@ import (
"github.com/apex/log/handlers/discard" "github.com/apex/log/handlers/discard"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestGood(t *testing.T) { func TestGood(t *testing.T) {
@ -32,7 +33,7 @@ func TestGood(t *testing.T) {
t.Fatal("expected non-nil resp here") t.Fatal("expected non-nil resp here")
} }
defer resp.Body.Close() defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body) _, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -24,6 +24,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx" "github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/runtimex"
"gitlab.com/yawning/obfs4.git/transports" "gitlab.com/yawning/obfs4.git/transports"
obfs4base "gitlab.com/yawning/obfs4.git/transports/base" obfs4base "gitlab.com/yawning/obfs4.git/transports/base"
@ -90,7 +91,7 @@ func (m *connmapper) scramble(cid int64) int64 {
// See https://stackoverflow.com/a/38140573/4354461 // See https://stackoverflow.com/a/38140573/4354461
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
if value, found := m.table[cid]; found == true { if value, found := m.table[cid]; found {
return value return value
} }
var factor int64 = 1 var factor int64 = 1
@ -366,7 +367,7 @@ func HTTPDo(
config.MaxResponseBodySnapSize, config.MaxResponseBodySnapSize,
), ),
) )
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
mu.Lock() mu.Lock()
results.BodySnap, results.Error = data, err results.BodySnap, results.Error = data, err
mu.Unlock() mu.Unlock()

View File

@ -377,7 +377,7 @@ type faketransport struct {
} }
func (txp *faketransport) Name() string { func (txp *faketransport) Name() string {
return txp.Name() return txp.txp.Name()
} }
func (txp *faketransport) ClientFactory(stateDir string) (obfs4base.ClientFactory, error) { func (txp *faketransport) ClientFactory(stateDir string) (obfs4base.ClientFactory, error) {
@ -385,7 +385,7 @@ func (txp *faketransport) ClientFactory(stateDir string) (obfs4base.ClientFactor
} }
func (txp *faketransport) ServerFactory(stateDir string, args *goptlib.Args) (obfs4base.ServerFactory, error) { func (txp *faketransport) ServerFactory(stateDir string, args *goptlib.Args) (obfs4base.ServerFactory, error) {
return txp.ServerFactory(stateDir, args) return txp.txp.ServerFactory(stateDir, args)
} }
func TestConnmapper(t *testing.T) { func TestConnmapper(t *testing.T) {

View File

@ -2,10 +2,11 @@ package netx
import ( import (
"context" "context"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeDialer struct { type FakeDialer struct {
@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req) return txp.Func(req)
} }
if req.Body != nil { if req.Body != nil {
ioutil.ReadAll(req.Body) iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if txp.Err != nil { if txp.Err != nil {

View File

@ -1,6 +1,7 @@
package httptransport_test package httptransport_test
import ( import (
"context"
"errors" "errors"
"io" "io"
"io/ioutil" "io/ioutil"
@ -10,6 +11,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestByteCounterFailure(t *testing.T) { func TestByteCounterFailure(t *testing.T) {
@ -68,7 +70,7 @@ func TestByteCounterSuccess(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -104,7 +106,7 @@ func TestByteCounterSuccessWithEOF(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -2,10 +2,11 @@ package httptransport
import ( import (
"context" "context"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeDialer struct { type FakeDialer struct {
@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req) return txp.Func(req)
} }
if req.Body != nil { if req.Body != nil {
ioutil.ReadAll(req.Body) iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if txp.Err != nil { if txp.Err != nil {

View File

@ -1,6 +1,7 @@
package httptransport_test package httptransport_test
import ( import (
"context"
"errors" "errors"
"io" "io"
"io/ioutil" "io/ioutil"
@ -11,6 +12,7 @@ import (
"github.com/apex/log" "github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestLoggingFailure(t *testing.T) { func TestLoggingFailure(t *testing.T) {
@ -72,6 +74,6 @@ func TestLoggingSuccess(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
ioutil.ReadAll(resp.Body) iox.ReadAllContext(context.Background(), resp.Body)
resp.Body.Close() resp.Body.Close()
} }

View File

@ -2,14 +2,15 @@ package httptransport
import ( import (
"bytes" "bytes"
"context"
"errors" "errors"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"net/http/httptrace" "net/http/httptrace"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
// SaverPerformanceHTTPTransport is a RoundTripper that saves // SaverPerformanceHTTPTransport is a RoundTripper that saves
@ -109,7 +110,7 @@ func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response,
snapsize = txp.SnapshotSize snapsize = txp.SnapshotSize
} }
if req.Body != nil { if req.Body != nil {
data, err := saverSnapRead(req.Body, snapsize) data, err := saverSnapRead(req.Context(), req.Body, snapsize)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -125,7 +126,7 @@ func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response,
if err != nil { if err != nil {
return nil, err return nil, err
} }
data, err := saverSnapRead(resp.Body, snapsize) data, err := saverSnapRead(req.Context(), resp.Body, snapsize)
err = ignoreExpectedEOF(err, resp) err = ignoreExpectedEOF(err, resp)
if err != nil { if err != nil {
resp.Body.Close() resp.Body.Close()
@ -157,8 +158,8 @@ func ignoreExpectedEOF(err error, resp *http.Response) error {
return err return err
} }
func saverSnapRead(r io.ReadCloser, snapsize int) ([]byte, error) { func saverSnapRead(ctx context.Context, r io.ReadCloser, snapsize int) ([]byte, error) {
return ioutil.ReadAll(io.LimitReader(r, int64(snapsize))) return iox.ReadAllContext(ctx, io.LimitReader(r, int64(snapsize)))
} }
func saverCompose(data []byte, r io.ReadCloser) io.ReadCloser { func saverCompose(data []byte, r io.ReadCloser) io.ReadCloser {

View File

@ -1,6 +1,7 @@
package httptransport_test package httptransport_test
import ( import (
"context"
"errors" "errors"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -10,6 +11,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestSaverPerformanceNoMultipleEvents(t *testing.T) { func TestSaverPerformanceNoMultipleEvents(t *testing.T) {
@ -245,7 +247,7 @@ func TestSaverBodySuccess(t *testing.T) {
txp := httptransport.SaverBodyHTTPTransport{ txp := httptransport.SaverBodyHTTPTransport{
RoundTripper: httptransport.FakeTransport{ RoundTripper: httptransport.FakeTransport{
Func: func(req *http.Request) (*http.Response, error) { Func: func(req *http.Request) (*http.Response, error) {
data, err := ioutil.ReadAll(req.Body) data, err := iox.ReadAllContext(context.Background(), req.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -274,7 +276,7 @@ func TestSaverBodySuccess(t *testing.T) {
t.Fatal("unexpected status code") t.Fatal("unexpected status code")
} }
defer resp.Body.Close() defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -3,7 +3,6 @@ package netx_test
import ( import (
"context" "context"
"errors" "errors"
"io/ioutil"
"net/http" "net/http"
"testing" "testing"
@ -12,6 +11,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestSuccess(t *testing.T) { func TestSuccess(t *testing.T) {
@ -38,7 +38,7 @@ func TestSuccess(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err = ioutil.ReadAll(resp.Body); err != nil { if _, err = iox.ReadAllContext(context.Background(), resp.Body); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err = resp.Body.Close(); err != nil { if err = resp.Body.Close(); err != nil {

View File

@ -4,11 +4,11 @@ import (
"bytes" "bytes"
"context" "context"
"errors" "errors"
"io/ioutil"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/engine/httpheader" "github.com/ooni/probe-cli/v3/internal/engine/httpheader"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
// DNSOverHTTPS is a DNS over HTTPS RoundTripper. Requests are submitted over // DNSOverHTTPS is a DNS over HTTPS RoundTripper. Requests are submitted over
@ -56,7 +56,7 @@ func (t DNSOverHTTPS) RoundTrip(ctx context.Context, query []byte) ([]byte, erro
if resp.Header.Get("content-type") != "application/dns-message" { if resp.Header.Get("content-type") != "application/dns-message" {
return nil, errors.New("doh: invalid content-type") return nil, errors.New("doh: invalid content-type")
} }
return ioutil.ReadAll(resp.Body) return iox.ReadAllContext(ctx, resp.Body)
} }
// RequiresPadding returns true for DoH according to RFC8467 // RequiresPadding returns true for DoH according to RFC8467

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-06-08 14:36:18.474117 +0200 CEST m=+0.459588210 // 2021-06-15 10:55:55.638897 +0200 CEST m=+4.257631084
// https://curl.haxx.se/ca/cacert.pem // https://curl.haxx.se/ca/cacert.pem
package tlsx package tlsx

View File

@ -12,14 +12,16 @@
package main package main
import ( import (
"context"
"crypto/x509" "crypto/x509"
"io/ioutil"
"log" "log"
"net/http" "net/http"
"os" "os"
"strings" "strings"
"text/template" "text/template"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
var tmpl = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. var tmpl = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
@ -50,7 +52,7 @@ func main() {
} }
defer resp.Body.Close() defer resp.Body.Close()
bundle, err := ioutil.ReadAll(resp.Body) bundle, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

View File

@ -15,6 +15,7 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices" "github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type fakeTestKeys struct { type fakeTestKeys struct {
@ -233,7 +234,7 @@ func TestEndToEnd(t *testing.T) {
return return
} }
if r.RequestURI == "/report/_id" { if r.RequestURI == "/report/_id" {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
panic(err) panic(err)
} }

View File

@ -4,7 +4,6 @@ import (
"context" "context"
"errors" "errors"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"regexp" "regexp"
"strings" "strings"
@ -17,6 +16,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices" "github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra" "github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func newclient() *probeservices.Client { func newclient() *probeservices.Client {
@ -165,7 +165,7 @@ func TestCloudfront(t *testing.T) {
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
t.Fatal("unexpected status code") t.Fatal("unexpected status code")
} }
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(req.Context(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -6,9 +6,9 @@ import (
"bytes" "bytes"
"context" "context"
_ "embed" _ "embed"
"io/ioutil"
"filippo.io/age" "filippo.io/age"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
//go:embed psiphon-config.json.age //go:embed psiphon-config.json.age
@ -34,7 +34,7 @@ func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]b
if err != nil { if err != nil {
return nil, err return nil, err
} }
return ioutil.ReadAll(output) return iox.ReadAllContext(ctx, output)
} }
// FetchPsiphonConfig decrypts psiphonConfigJSONAge using // FetchPsiphonConfig decrypts psiphonConfigJSONAge using

View File

@ -0,0 +1,21 @@
package iox_test
import (
"context"
"fmt"
"log"
"strings"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func ExampleReadAllContext() {
r := strings.NewReader("deadbeef")
ctx := context.Background()
out, err := iox.ReadAllContext(ctx, r)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%d\n", len(out))
// Output: 8
}

47
internal/iox/iox.go Normal file
View File

@ -0,0 +1,47 @@
// Package iox contains io extensions.
package iox
import (
"context"
"io"
)
// ReadAllContext reads the whole reader r in a
// background goroutine. This function will return
// earlier if the context is cancelled. In which case
// we will continue reading from r in the background
// goroutine, and we will discard the result. To stop
// the long-running goroutine, you need to close the
// connection bound to the r reader, if possible.
func ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) {
datach, errch := make(chan []byte, 1), make(chan error, 1) // buffers
go func() {
data, err := io.ReadAll(r)
if err != nil {
errch <- err
return
}
datach <- data
}()
select {
case data := <-datach:
return data, nil
case <-ctx.Done():
return nil, ctx.Err()
case err := <-errch:
return nil, err
}
}
// MockableReader allows to mock any io.Reader.
type MockableReader struct {
MockRead func(b []byte) (int, error)
}
// MockableReader implements an io.Reader.
var _ io.Reader = &MockableReader{}
// Read implements io.Reader.Read.
func (r *MockableReader) Read(b []byte) (int, error) {
return r.MockRead(b)
}

70
internal/iox/iox_test.go Normal file
View File

@ -0,0 +1,70 @@
package iox
import (
"context"
"errors"
"strings"
"testing"
"time"
)
func TestReadAllContextCommonCase(t *testing.T) {
r := strings.NewReader("deadbeef")
ctx := context.Background()
out, err := ReadAllContext(ctx, r)
if err != nil {
t.Fatal(err)
}
if len(out) != 8 {
t.Fatal("not the expected number of bytes")
}
}
func TestReadAllContextWithError(t *testing.T) {
expected := errors.New("mocked error")
r := &MockableReader{
MockRead: func(b []byte) (int, error) {
return 0, expected
},
}
ctx := context.Background()
out, err := ReadAllContext(ctx, r)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected", err)
}
if len(out) != 0 {
t.Fatal("not the expected number of bytes")
}
}
func TestReadAllContextWithCancelledContext(t *testing.T) {
r := strings.NewReader("deadbeef")
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
out, err := ReadAllContext(ctx, r)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected", err)
}
if len(out) != 0 {
t.Fatal("not the expected number of bytes")
}
}
func TestReadAllContextWithErrorAndCancelledContext(t *testing.T) {
expected := errors.New("mocked error")
r := &MockableReader{
MockRead: func(b []byte) (int, error) {
time.Sleep(time.Millisecond)
return 0, expected
},
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
out, err := ReadAllContext(ctx, r)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected", err)
}
if len(out) != 0 {
t.Fatal("not the expected number of bytes")
}
}

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:00.422051399 +0200 CEST m=+0.000129449 // 2021-06-15 10:55:55.967236 +0200 CEST m=+0.000374501
package ooapi package ooapi
@ -59,7 +59,8 @@ func (api *simpleCheckReportIDAPI) Call(ctx context.Context, req *apimodel.Check
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleCheckInAPI implements the CheckIn API. // simpleCheckInAPI implements the CheckIn API.
@ -109,7 +110,8 @@ func (api *simpleCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequ
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleLoginAPI implements the Login API. // simpleLoginAPI implements the Login API.
@ -159,7 +161,8 @@ func (api *simpleLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest)
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleMeasurementMetaAPI implements the MeasurementMeta API. // simpleMeasurementMetaAPI implements the MeasurementMeta API.
@ -209,7 +212,8 @@ func (api *simpleMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.Mea
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleRegisterAPI implements the Register API. // simpleRegisterAPI implements the Register API.
@ -259,7 +263,8 @@ func (api *simpleRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRe
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleTestHelpersAPI implements the TestHelpers API. // simpleTestHelpersAPI implements the TestHelpers API.
@ -309,7 +314,8 @@ func (api *simpleTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHel
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simplePsiphonConfigAPI implements the PsiphonConfig API. // simplePsiphonConfigAPI implements the PsiphonConfig API.
@ -377,7 +383,8 @@ func (api *simplePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.Psiph
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleTorTargetsAPI implements the TorTargets API. // simpleTorTargetsAPI implements the TorTargets API.
@ -445,7 +452,8 @@ func (api *simpleTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTarge
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleURLsAPI implements the URLs API. // simpleURLsAPI implements the URLs API.
@ -495,7 +503,8 @@ func (api *simpleURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleOpenReportAPI implements the OpenReport API. // simpleOpenReportAPI implements the OpenReport API.
@ -545,7 +554,8 @@ func (api *simpleOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenRepo
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }
// simpleSubmitMeasurementAPI implements the SubmitMeasurement API. // simpleSubmitMeasurementAPI implements the SubmitMeasurement API.
@ -603,5 +613,6 @@ func (api *simpleSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.S
if api.UserAgent != "" { if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent) httpReq.Header.Add("User-Agent", api.UserAgent)
} }
return api.newResponse(api.httpClient().Do(httpReq)) httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
} }

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:01.254917327 +0200 CEST m=+0.000217025 // 2021-06-15 10:55:56.308361 +0200 CEST m=+0.000215751
package ooapi package ooapi
@ -9,7 +9,6 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
@ -18,6 +17,7 @@ import (
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel" "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
) )
@ -177,7 +177,7 @@ func (h *handleCheckReportID) ServeHTTP(w http.ResponseWriter, r *http.Request)
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -440,7 +440,7 @@ func (h *handleCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -685,7 +685,7 @@ func (h *handleLogin) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -912,7 +912,7 @@ func (h *handleMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *http.Request
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -1175,7 +1175,7 @@ func (h *handleRegister) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -1402,7 +1402,7 @@ func (h *handleTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -1671,7 +1671,7 @@ func (h *handlePsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http.Request)
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -1942,7 +1942,7 @@ func (h *handleTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -2192,7 +2192,7 @@ func (h *handleURLs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -2437,7 +2437,7 @@ func (h *handleOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -2682,7 +2682,7 @@ func (h *handleSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r *http.Reque
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:01.869492095 +0200 CEST m=+0.000168945 // 2021-06-15 10:55:56.587245 +0200 CEST m=+0.000218959
package ooapi package ooapi

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:02.497717446 +0200 CEST m=+0.000113904 // 2021-06-15 10:55:56.851681 +0200 CEST m=+0.000186126
package ooapi package ooapi

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:03.02266641 +0200 CEST m=+0.000097757 // 2021-06-15 10:55:57.174427 +0200 CEST m=+0.000217876
package ooapi package ooapi

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:03.586305848 +0200 CEST m=+0.000123000 // 2021-06-15 10:55:57.445485 +0200 CEST m=+0.000219751
package ooapi package ooapi

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:04.198485035 +0200 CEST m=+0.000114145 // 2021-06-15 10:55:57.708833 +0200 CEST m=+0.000193418
package ooapi package ooapi
@ -8,7 +8,6 @@ package ooapi
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
@ -16,6 +15,7 @@ import (
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/kvstore" "github.com/ooni/probe-cli/v3/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel" "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
) )
@ -42,7 +42,7 @@ func (h *handleClientCallCheckReportID) ServeHTTP(w http.ResponseWriter, r *http
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -135,7 +135,7 @@ func (h *handleClientCallCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Reque
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -227,7 +227,7 @@ func (h *handleClientCallMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *ht
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -320,7 +320,7 @@ func (h *handleClientCallTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.R
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -435,7 +435,7 @@ func (h *handleClientCallPsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -550,7 +550,7 @@ func (h *handleClientCallTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Re
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -643,7 +643,7 @@ func (h *handleClientCallURLs) ServeHTTP(w http.ResponseWriter, r *http.Request)
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -736,7 +736,7 @@ func (h *handleClientCallOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Re
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -828,7 +828,7 @@ func (h *handleClientCallSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r *
} }
h.count++ h.count++
if r.Body != nil { if r.Body != nil {
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:04.793154609 +0200 CEST m=+0.000108739 // 2021-06-15 10:55:57.987741 +0200 CEST m=+0.000211126
package ooapi package ooapi

View File

@ -3,9 +3,10 @@ package ooapi
import ( import (
"context" "context"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"time" "time"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
type FakeCodec struct { type FakeCodec struct {
@ -30,7 +31,7 @@ type FakeHTTPClient struct {
func (c *FakeHTTPClient) Do(req *http.Request) (*http.Response, error) { func (c *FakeHTTPClient) Do(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond) time.Sleep(10 * time.Microsecond)
if req.Body != nil { if req.Body != nil {
_, _ = ioutil.ReadAll(req.Body) _, _ = iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close() req.Body.Close()
} }
if c.Err != nil { if c.Err != nil {

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:05.331414434 +0200 CEST m=+0.000124504 // 2021-06-15 10:55:58.234786 +0200 CEST m=+0.000218167
package ooapi package ooapi

View File

@ -156,7 +156,8 @@ func (d *Descriptor) genNewAPI(sb *strings.Builder) {
fmt.Fprint(sb, "\tif api.UserAgent != \"\" {\n") fmt.Fprint(sb, "\tif api.UserAgent != \"\" {\n")
fmt.Fprint(sb, "\t\thttpReq.Header.Add(\"User-Agent\", api.UserAgent)\n") fmt.Fprint(sb, "\t\thttpReq.Header.Add(\"User-Agent\", api.UserAgent)\n")
fmt.Fprint(sb, "\t}\n") fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn api.newResponse(api.httpClient().Do(httpReq))\n") fmt.Fprint(sb, "\thttpResp, err := api.httpClient().Do(httpReq)\n")
fmt.Fprint(sb, "\treturn api.newResponse(ctx, httpResp, err)\n")
fmt.Fprint(sb, "}\n\n") fmt.Fprint(sb, "}\n\n")
} }

View File

@ -231,7 +231,7 @@ func (d *Descriptor) genTestRoundTrip(sb *strings.Builder) {
fmt.Fprint(sb, "\t}\n") fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\th.count++\n") fmt.Fprint(sb, "\th.count++\n")
fmt.Fprint(sb, "\tif r.Body != nil {\n") fmt.Fprint(sb, "\tif r.Body != nil {\n")
fmt.Fprint(sb, "\t\tdata, err := ioutil.ReadAll(r.Body)\n") fmt.Fprint(sb, "\t\tdata, err := iox.ReadAllContext(r.Context(), r.Body)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n") fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n") fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\t\treturn\n") fmt.Fprintf(sb, "\t\t\treturn\n")
@ -443,7 +443,6 @@ func GenAPIsTestGo(file string) {
fmt.Fprint(&sb, "\t\"context\"\n") fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"encoding/json\"\n") fmt.Fprint(&sb, "\t\"encoding/json\"\n")
fmt.Fprint(&sb, "\t\"errors\"\n") fmt.Fprint(&sb, "\t\"errors\"\n")
fmt.Fprint(&sb, "\t\"io/ioutil\"\n")
fmt.Fprint(&sb, "\t\"net/http/httptest\"\n") fmt.Fprint(&sb, "\t\"net/http/httptest\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n") fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\t\"net/url\"\n") fmt.Fprint(&sb, "\t\"net/url\"\n")
@ -452,6 +451,7 @@ func GenAPIsTestGo(file string) {
fmt.Fprint(&sb, "\t\"sync\"\n") fmt.Fprint(&sb, "\t\"sync\"\n")
fmt.Fprint(&sb, "\n") fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n") fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n") fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n") fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors { for _, desc := range Descriptors {

View File

@ -57,7 +57,7 @@ func (d *Descriptor) genTestClientCallRoundTrip(sb *strings.Builder) {
fmt.Fprint(sb, "\t}\n") fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\th.count++\n") fmt.Fprint(sb, "\th.count++\n")
fmt.Fprint(sb, "\tif r.Body != nil {\n") fmt.Fprint(sb, "\tif r.Body != nil {\n")
fmt.Fprint(sb, "\t\tdata, err := ioutil.ReadAll(r.Body)\n") fmt.Fprint(sb, "\t\tdata, err := iox.ReadAllContext(r.Context(), r.Body)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n") fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n") fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\t\treturn\n") fmt.Fprintf(sb, "\t\t\treturn\n")
@ -161,7 +161,6 @@ func GenClientCallTestGo(file string) {
fmt.Fprint(&sb, "import (\n") fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n") fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"encoding/json\"\n") fmt.Fprint(&sb, "\t\"encoding/json\"\n")
fmt.Fprint(&sb, "\t\"io/ioutil\"\n")
fmt.Fprint(&sb, "\t\"net/http/httptest\"\n") fmt.Fprint(&sb, "\t\"net/http/httptest\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n") fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\t\"net/url\"\n") fmt.Fprint(&sb, "\t\"net/url\"\n")
@ -169,6 +168,7 @@ func GenClientCallTestGo(file string) {
fmt.Fprint(&sb, "\t\"sync\"\n") fmt.Fprint(&sb, "\t\"sync\"\n")
fmt.Fprint(&sb, "\n") fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n") fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/kvstore\"\n") fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/kvstore\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n") fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n") fmt.Fprint(&sb, ")\n")

View File

@ -9,7 +9,7 @@ import (
func (d *Descriptor) genNewResponse(sb *strings.Builder) { func (d *Descriptor) genNewResponse(sb *strings.Builder) {
fmt.Fprintf(sb, fmt.Fprintf(sb,
"func (api *%s) newResponse(resp *http.Response, err error) (%s, error) {\n", "func (api *%s) newResponse(ctx context.Context, resp *http.Response, err error) (%s, error) {\n",
d.APIStructName(), d.ResponseTypeName()) d.APIStructName(), d.ResponseTypeName())
fmt.Fprint(sb, "\tif err != nil {\n") fmt.Fprint(sb, "\tif err != nil {\n")
@ -23,7 +23,7 @@ func (d *Descriptor) genNewResponse(sb *strings.Builder) {
fmt.Fprint(sb, "\t}\n") fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tdefer resp.Body.Close()\n") fmt.Fprint(sb, "\tdefer resp.Body.Close()\n")
fmt.Fprint(sb, "\treader := io.LimitReader(resp.Body, 4<<20)\n") fmt.Fprint(sb, "\treader := io.LimitReader(resp.Body, 4<<20)\n")
fmt.Fprint(sb, "\tdata, err := ioutil.ReadAll(reader)\n") fmt.Fprint(sb, "\tdata, err := iox.ReadAllContext(ctx, reader)\n")
fmt.Fprint(sb, "\tif err != nil {\n") fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n") fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n") fmt.Fprint(sb, "\t}\n")
@ -67,10 +67,11 @@ func GenResponsesGo(file string) {
fmt.Fprint(&sb, "package ooapi\n\n") fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file) fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n") fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"io\"\n") fmt.Fprint(&sb, "\t\"io\"\n")
fmt.Fprint(&sb, "\t\"io/ioutil\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n") fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\n") fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n") fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n\n") fmt.Fprint(&sb, ")\n\n")
for _, desc := range Descriptors { for _, desc := range Descriptors {

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:06.213159282 +0200 CEST m=+0.000104135 // 2021-06-15 10:55:58.487888 +0200 CEST m=+0.000221710
package ooapi package ooapi

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:06.893164862 +0200 CEST m=+0.000136881 // 2021-06-15 10:55:58.755431 +0200 CEST m=+0.000217917
package ooapi package ooapi

View File

@ -2,7 +2,6 @@ package ooapi
import ( import (
"encoding/json" "encoding/json"
"io/ioutil"
"net/http" "net/http"
"strings" "strings"
"sync" "sync"
@ -10,6 +9,7 @@ import (
"time" "time"
"github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel" "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
) )
@ -72,7 +72,7 @@ func (lh *LoginHandler) register(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(400) w.WriteHeader(400)
return return
} }
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return
@ -113,7 +113,7 @@ func (lh *LoginHandler) login(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(400) w.WriteHeader(400)
return return
} }
data, err := ioutil.ReadAll(r.Body) data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil { if err != nil {
w.WriteHeader(400) w.WriteHeader(400)
return return

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:07.590481334 +0200 CEST m=+0.000085230 // 2021-06-15 10:55:59.020364 +0200 CEST m=+0.000217085
package ooapi package ooapi

View File

@ -1,19 +1,20 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:08.237841277 +0200 CEST m=+0.000121556 // 2021-06-15 10:55:59.329509 +0200 CEST m=+0.000220043
package ooapi package ooapi
//go:generate go run ./internal/generator -file responses.go //go:generate go run ./internal/generator -file responses.go
import ( import (
"context"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel" "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
) )
func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) { func (api *simpleCheckReportIDAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -25,7 +26,7 @@ func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) (
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -36,7 +37,7 @@ func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) (
return out, nil return out, nil
} }
func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckInResponse, error) { func (api *simpleCheckInAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.CheckInResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -48,7 +49,7 @@ func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimo
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -59,7 +60,7 @@ func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimo
return out, nil return out, nil
} }
func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimodel.LoginResponse, error) { func (api *simpleLoginAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.LoginResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -71,7 +72,7 @@ func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimode
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -82,7 +83,7 @@ func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimode
return out, nil return out, nil
} }
func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) { func (api *simpleMeasurementMetaAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -94,7 +95,7 @@ func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error)
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -105,7 +106,7 @@ func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error)
return out, nil return out, nil
} }
func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apimodel.RegisterResponse, error) { func (api *simpleRegisterAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.RegisterResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -117,7 +118,7 @@ func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apim
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -128,7 +129,7 @@ func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apim
return out, nil return out, nil
} }
func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (apimodel.TestHelpersResponse, error) { func (api *simpleTestHelpersAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.TestHelpersResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -140,7 +141,7 @@ func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (ap
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -154,7 +155,7 @@ func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (ap
return out, nil return out, nil
} }
func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) { func (api *simplePsiphonConfigAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -166,7 +167,7 @@ func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) (
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -180,7 +181,7 @@ func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) (
return out, nil return out, nil
} }
func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (apimodel.TorTargetsResponse, error) { func (api *simpleTorTargetsAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.TorTargetsResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -192,7 +193,7 @@ func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (api
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -206,7 +207,7 @@ func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (api
return out, nil return out, nil
} }
func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel.URLsResponse, error) { func (api *simpleURLsAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.URLsResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -218,7 +219,7 @@ func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -229,7 +230,7 @@ func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel
return out, nil return out, nil
} }
func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*apimodel.OpenReportResponse, error) { func (api *simpleOpenReportAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.OpenReportResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -241,7 +242,7 @@ func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*ap
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -252,7 +253,7 @@ func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*ap
return out, nil return out, nil
} }
func (api *simpleSubmitMeasurementAPI) newResponse(resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) { func (api *simpleSubmitMeasurementAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -264,7 +265,7 @@ func (api *simpleSubmitMeasurementAPI) newResponse(resp *http.Response, err erro
} }
defer resp.Body.Close() defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20) reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader) data, err := iox.ReadAllContext(ctx, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:08.888123159 +0200 CEST m=+0.000883704 // 2021-06-15 10:55:59.636856 +0200 CEST m=+0.000624459
package ooapi package ooapi
@ -9,7 +9,7 @@ const swagger = `{
"swagger": "2.0", "swagger": "2.0",
"info": { "info": {
"title": "OONI API specification", "title": "OONI API specification",
"version": "0.20210512.5071508" "version": "0.20210615.6085559"
}, },
"host": "api.ooni.io", "host": "api.ooni.io",
"basePath": "/", "basePath": "/",

View File

@ -1,9 +1,9 @@
package ooapi package ooapi
import ( import (
"context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil"
"log" "log"
"net/http" "net/http"
"sort" "sort"
@ -13,6 +13,7 @@ import (
"github.com/hexops/gotextdiff" "github.com/hexops/gotextdiff"
"github.com/hexops/gotextdiff/myers" "github.com/hexops/gotextdiff/myers"
"github.com/hexops/gotextdiff/span" "github.com/hexops/gotextdiff/span"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/ooapi/internal/openapi" "github.com/ooni/probe-cli/v3/internal/ooapi/internal/openapi"
) )
@ -36,7 +37,7 @@ func getServerModel(serverURL string) *openapi.Swagger {
log.Fatal(err) log.Fatal(err)
} }
defer resp.Body.Close() defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

View File

@ -4,7 +4,6 @@ import (
"context" "context"
"errors" "errors"
"io" "io"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
@ -15,6 +14,7 @@ import (
"github.com/apex/log" "github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/mockablex" "github.com/ooni/probe-cli/v3/internal/engine/netx/mockablex"
"github.com/ooni/probe-cli/v3/internal/iox"
) )
func TestListenerLoggerWorks(t *testing.T) { func TestListenerLoggerWorks(t *testing.T) {
@ -57,7 +57,7 @@ func TestListenerWorksWithFakeDialer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
data, err := ioutil.ReadAll(resp.Body) data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }