c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(all): introduce and use iox.ReadAllContext (#379)

Browse Source 
* fix(all): introduce and use iox.ReadAllContext

This improvement over the ioutil.ReadAll utility returns early
if the context expires. This enables us to unblock stuck code in
case there's censorship confounding the TCP stack.

See https://github.com/ooni/probe/issues/1417.

Compared to the functionality postulated in the above mentioned
issue, I choose to be more generic and separate limiting the
maximum body size (not implemented here) from using the context
to return early when reading a body (or any other reader).

After implementing iox.ReadAllContext, I made sure we always
use it everywhere in the tree instead of ioutil.ReadAll.

This includes many parts of the codebase where in theory we don't
need iox.ReadAllContext. Though, changing all the places makes
checking whether we're not using ioutil.ReadAll where we should
not be using it easy: `git grep` should return no lines.

* Update internal/iox/iox_test.go

* fix(ndt7): treat context errors as non-errors

The rationale is explained by the comment documenting reduceErr.

* Update internal/engine/experiment/ndt7/download.go
This commit is contained in:
Simone Basso 2021-06-15 11:57:40 +02:00 committed by GitHub
parent 576e035b20
commit 0fdc9cafb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
76 changed files with 440 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CONTRIBUTING.md
View File

@ -63,6 +63,8 @@ run `go mod tidy` to minimize such changes.
- use `./internal/fsx.OpenFile` when you need to open a file
- use `./internal/iox.ReadAllContext` instead of `ioutil.ReadAll`
## Code testing requirements
Make sure all tests pass with `go test -race ./...` run from the

5
cmd/ooniprobe/internal/database/database.go
View File

@ -1,11 +1,12 @@
package database
import (
"context"
"database/sql"
"embed"
"io/ioutil"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/iox"
migrate "github.com/rubenv/sql-migrate"
"upper.io/db.v3/lib/sqlbuilder"
"upper.io/db.v3/sqlite"
@ -19,7 +20,7 @@ func readAsset(path string) ([]byte, error) {
if err != nil {
return nil, err
}
return ioutil.ReadAll(filep)
return iox.ReadAllContext(context.Background(), filep)
}
func readAssetDir(path string) ([]string, error) {

5
internal/cmd/jafar/badproxy/badproxy.go
View File

@ -6,16 +6,17 @@
package badproxy
import (
"context"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"io"
"io/ioutil"
"net"
"strings"
"time"
"github.com/google/martian/v3/mitm"
"github.com/ooni/probe-cli/v3/internal/iox"
)
// CensoringProxy is a proxy that does not behave correctly.
@ -56,7 +57,7 @@ func (p *CensoringProxy) serve(conn net.Conn) {
} else {
const maxread = 1 << 17
reader := io.LimitReader(conn, maxread)
ioutil.ReadAll(reader)
iox.ReadAllContext(context.Background(), reader)
}
conn.Close()
}

8
internal/cmd/jafar/httpproxy/httpproxy_test.go
View File

@ -3,12 +3,12 @@ package httpproxy
import (
"bytes"
"context"
"io/ioutil"
"net"
"net/http"
"testing"
"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestPass(t *testing.T) {
@ -89,7 +89,7 @@ func checkrequest(
t.Fatal("unexpected value of status code")
}
t.Log(resp)
values, _ := resp.Header["Via"]
values := resp.Header["Via"]
var foundProduct bool
for _, value := range values {
if value == product {
@ -102,7 +102,7 @@ func checkrequest(
if !foundProduct && expectVia {
t.Fatal("Via header not found")
}
proxiedData, err := ioutil.ReadAll(resp.Body)
proxiedData, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
@ -120,7 +120,7 @@ func checkbody(t *testing.T, proxiedData []byte, host string) {
t.Fatal("unexpected status code")
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

5
internal/cmd/jafar/uncensored/uncensored_test.go
View File

@ -3,10 +3,11 @@ package uncensored
import (
"bytes"
"context"
"io/ioutil"
"net/http"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestGood(t *testing.T) {
@ -55,7 +56,7 @@ func TestGood(t *testing.T) {
if resp.StatusCode != 200 {
t.Fatal("invalid status-code")
}
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

4
internal/cmd/oohelper/internal/client.go
View File

@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
@ -15,6 +14,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version"
)
@ -126,7 +126,7 @@ func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, erro
if resp.StatusCode != 200 {
return nil, ErrHTTPStatusCode
}
data, err = ioutil.ReadAll(resp.Body)
data, err = iox.ReadAllContext(ctx, resp.Body)
if err != nil {
return nil, err
}

4
internal/cmd/oohelper/internal/fake_test.go
View File

@ -3,13 +3,13 @@ package internal
import (
"context"
"io"
"io/ioutil"
"net"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeResolver struct {
@ -63,7 +63,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req)
}
if req.Body != nil {
ioutil.ReadAll(req.Body)
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {

4
internal/cmd/oohelperd/internal/fake_test.go
View File

@ -3,13 +3,13 @@ package internal
import (
"context"
"io"
"io/ioutil"
"net"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeResolver struct {
@ -63,7 +63,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req)
}
if req.Body != nil {
ioutil.ReadAll(req.Body)
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {

4
internal/cmd/oohelperd/internal/http.go
View File

@ -3,12 +3,12 @@ package internal
import (
"context"
"io"
"io/ioutil"
"net/http"
"strings"
"sync"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/iox"
)
// CtrlHTTPResponse is the result of the HTTP check performed by
@ -56,7 +56,7 @@ func HTTPDo(ctx context.Context, config *HTTPConfig) {
headers[k] = resp.Header.Get(k)
}
reader := &io.LimitedReader{R: resp.Body, N: config.MaxAcceptableBody}
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
config.Out <- CtrlHTTPResponse{
BodyLength: int64(len(data)),
Failure: newfailure(err),

11
internal/cmd/oohelperd/internal/internal.go
View File

@ -4,10 +4,10 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/version"
)
@ -32,7 +32,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
reader := &io.LimitedReader{R: req.Body, N: h.MaxAcceptableBody}
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(req.Context(), reader)
if err != nil {
w.WriteHeader(400)
return
@ -42,12 +42,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(400)
return
}
measureConfig := MeasureConfig{
Client: h.Client,
Dialer: h.Dialer,
MaxAcceptableBody: h.MaxAcceptableBody,
Resolver: h.Resolver,
}
measureConfig := MeasureConfig(h)
cresp, err := Measure(req.Context(), measureConfig, &creq)
if err != nil {
w.WriteHeader(400)

5
internal/cmd/oohelperd/internal/internal_test.go
View File

@ -1,9 +1,9 @@
package internal_test
import (
"context"
"encoding/json"
"errors"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
@ -12,6 +12,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
"github.com/ooni/probe-cli/v3/internal/iox"
)
const simplerequest = `{
@ -126,7 +127,7 @@ func TestWorkingAsIntended(t *testing.T) {
if v := resp.Header.Get("content-type"); v != expect.respContentType {
t.Fatalf("unexpected content-type: %s", v)
}
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

4
internal/engine/experiment/dash/collect.go
View File

@ -16,7 +16,7 @@ type collectDeps interface {
JSONMarshal(v interface{}) ([]byte, error)
Logger() model.Logger
NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
ReadAll(r io.Reader) ([]byte, error)
ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error)
Scheme() string
UserAgent() string
}
@ -47,7 +47,7 @@ func collect(ctx context.Context, fqdn, authorization string,
return errHTTPRequestFailed
}
defer resp.Body.Close()
data, err = deps.ReadAll(resp.Body)
data, err = deps.ReadAllContext(ctx, resp.Body)
if err != nil {
return err
}

8
internal/engine/experiment/dash/dash.go
View File

@ -9,7 +9,6 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"runtime"
"time"
@ -20,6 +19,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/humanize"
"github.com/ooni/probe-cli/v3/internal/iox"
)
const (
@ -86,8 +86,8 @@ func (r runner) NewHTTPRequest(meth, url string, body io.Reader) (*http.Request,
return http.NewRequest(meth, url, body)
}
func (r runner) ReadAll(reader io.Reader) ([]byte, error) {
return ioutil.ReadAll(reader)
func (r runner) ReadAllContext(ctx context.Context, reader io.Reader) ([]byte, error) {
return iox.ReadAllContext(ctx, reader)
}
func (r runner) Scheme() string {
@ -179,7 +179,7 @@ func (r runner) measure(
current.ConnectTime = connectTime
r.tk.ReceiverData = append(r.tk.ReceiverData, current)
total += current.Received
avgspeed := 8 * float64(total) / time.Now().Sub(begin).Seconds()
avgspeed := 8 * float64(total) / time.Since(begin).Seconds()
percentage := float64(current.Iteration) / float64(numIterations)
message := fmt.Sprintf("streaming: speed: %s", humanize.SI(avgspeed, "bit/s"))
r.callbacks.OnProgress(percentage, message)

6
internal/engine/experiment/dash/download.go
View File

@ -12,7 +12,7 @@ import (
type downloadDeps interface {
HTTPClient() *http.Client
NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
ReadAll(r io.Reader) ([]byte, error)
ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error)
Scheme() string
UserAgent() string
}
@ -57,7 +57,7 @@ func download(ctx context.Context, config downloadConfig) (downloadResult, error
return result, errHTTPRequestFailed
}
defer resp.Body.Close()
data, err := config.deps.ReadAll(resp.Body)
data, err := config.deps.ReadAllContext(ctx, resp.Body)
if err != nil {
return result, err
}
@ -66,7 +66,7 @@ func download(ctx context.Context, config downloadConfig) (downloadResult, error
// turns out that Neubot and MK do the same. So, we do what they do. At
// the same time, we are currently not able to include the overhead that
// is caused by HTTP headers etc. So, we're a bit less precise.
result.elapsed = time.Now().Sub(savedTicks).Seconds()
result.elapsed = time.Since(savedTicks).Seconds()
result.received = int64(len(data))
result.requestTicks = savedTicks.Sub(config.begin).Seconds()
result.timestamp = time.Now().Unix()

3
internal/engine/experiment/dash/fake_test.go
View File

@ -1,6 +1,7 @@
package dash
import (
"context"
"io"
"net/http"
"time"
@ -36,7 +37,7 @@ func (d FakeDeps) NewHTTPRequest(
return d.newHTTPRequestResult, d.newHTTPRequestErr
}
func (d FakeDeps) ReadAll(r io.Reader) ([]byte, error) {
func (d FakeDeps) ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) {
return d.readAllResult, d.readAllErr
}

4
internal/engine/experiment/dash/negotiate.go
View File

@ -16,7 +16,7 @@ type negotiateDeps interface {
JSONMarshal(v interface{}) ([]byte, error)
Logger() model.Logger
NewHTTPRequest(method string, url string, body io.Reader) (*http.Request, error)
ReadAll(r io.Reader) ([]byte, error)
ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error)
Scheme() string
UserAgent() string
}
@ -48,7 +48,7 @@ func negotiate(
return negotiateResp, errHTTPRequestFailed
}
defer resp.Body.Close()
data, err = deps.ReadAll(resp.Body)
data, err = deps.ReadAllContext(ctx, resp.Body)
if err != nil {
return negotiateResp, err
}

6
internal/engine/experiment/dash/spec.go
View File

@ -1,12 +1,6 @@
package dash
const (
// currentServerSchemaVersion is the version of the server schema that
// will be adopted by this implementation. Version 3 is the one that is
// Neubot uses. We needed to bump the version because Web100 is not on
// M-Lab anymore and hence we need to make a breaking change.
currentServerSchemaVersion = 4
// negotiatePath is the URL path used to negotiate
negotiatePath = "/negotiate/dash"

5
internal/engine/experiment/hhfm/fake_test.go
View File

@ -2,10 +2,11 @@ package hhfm_test
import (
"context"
"io/ioutil"
"net"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeDialer struct {
@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req)
}
if req.Body != nil {
ioutil.ReadAll(req.Body)
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {

4
internal/engine/experiment/hhfm/hhfm.go
View File

@ -9,7 +9,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"sort"
@ -21,6 +20,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/randx"
)
@ -198,7 +198,7 @@ func transact(txp Transport, req *http.Request,
return nil, nil, urlgetter.ErrHTTPRequestFailed
}
callbacks.OnProgress(0.75, "reading response body...")
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(req.Context(), resp.Body)
callbacks.OnProgress(1.00, fmt.Sprintf("got reseponse body... %+v", err))
if err != nil {
return nil, nil, err

31
internal/engine/experiment/ndt7/download.go
View File

@ -2,11 +2,12 @@ package ndt7
import (
"context"
"errors"
"io"
"io/ioutil"
"time"
"github.com/gorilla/websocket"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type downloadManager struct {
@ -33,6 +34,30 @@ func newDownloadManager(
}
func (mgr downloadManager) run(ctx context.Context) error {
return mgr.reduceErr(mgr.doRun(ctx))
}
// reduceErr treats as non-errors the errors caused by the context
// so that we can focus instead on network errors.
//
// This function was introduced by https://github.com/ooni/probe-cli/pull/379
// since before such a PR we did not see context interrupting
// errors when we were reading messages. Since before such a PR
// we used to return `nil` on context errors, this function is
// here to keep the previous behavior by filtering the error
// returned when reading messages, given that now reading messages
// can fail midway because we use iox.ReadAllContext.
func (mgr downloadManager) reduceErr(err error) error {
if errors.Is(err, context.Canceled) {
return nil
}
if errors.Is(err, context.DeadlineExceeded) {
return nil
}
return err
}
func (mgr downloadManager) doRun(ctx context.Context) error {
var total int64
start := time.Now()
if err := mgr.conn.SetReadDeadline(start.Add(mgr.maxRuntime)); err != nil {
@ -47,7 +72,7 @@ func (mgr downloadManager) run(ctx context.Context) error {
return err
}
if kind == websocket.TextMessage {
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return err
}
@ -57,7 +82,7 @@ func (mgr downloadManager) run(ctx context.Context) error {
}
continue
}
n, err := io.Copy(ioutil.Discard, reader)
n, err := io.Copy(io.Discard, reader)
if err != nil {
return err
}

17
internal/engine/experiment/ndt7/download_test.go
View File

@ -143,3 +143,20 @@ type goodJSONReader struct{}
func (r *goodJSONReader) Read(p []byte) (int, error) {
return copy(p, []byte(`{}`)), io.EOF
}
func TestDownloadReduceErr(t *testing.T) {
mgr := downloadManager{}
if mgr.reduceErr(context.Canceled) != nil {
t.Fatal("reduceErr not working as intended for context.Canceled")
}
if mgr.reduceErr(context.DeadlineExceeded) != nil {
t.Fatal("reduceErr not working as intended for context.DeadlineExceeded")
}
if mgr.reduceErr(nil) != nil {
t.Fatal("reduceErr not working as intended for nil")
}
expected := errors.New("mocked error")
if mgr.reduceErr(expected) != expected {
t.Fatal("reduceErr not working as intended for other errors")
}
}

5
internal/engine/geolocate/fake_test.go
View File

@ -1,9 +1,10 @@
package geolocate
import (
"io/ioutil"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeTransport struct {
@ -14,7 +15,7 @@ type FakeTransport struct {
func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond)
if req.Body != nil {
ioutil.ReadAll(req.Body)
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {

5
internal/engine/httpx/fake_test.go
View File

@ -1,9 +1,10 @@
package httpx
import (
"io/ioutil"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeTransport struct {
@ -18,7 +19,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req)
}
if req.Body != nil {
ioutil.ReadAll(req.Body)
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {

5
internal/engine/httpx/jsonapi.go
View File

@ -7,9 +7,10 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"github.com/ooni/probe-cli/v3/internal/iox"
)
// Logger is the definition of Logger used by this package.
@ -100,7 +101,7 @@ func (c Client) Do(request *http.Request) ([]byte, error) {
if response.StatusCode >= 400 {
return nil, fmt.Errorf("httpx: request failed: %s", response.Status)
}
return ioutil.ReadAll(response.Body)
return iox.ReadAllContext(request.Context(), response.Body)
}
// DoJSON performs the provided request and unmarshals the JSON response body

4
internal/engine/internal/mlablocate/mlablocate.go
View File

@ -6,11 +6,11 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/iox"
)
// Client is a locate.measurementlab.net client.
@ -63,7 +63,7 @@ func (c *Client) Query(ctx context.Context, tool string) (Result, error) {
if resp.StatusCode != 200 {
return Result{}, fmt.Errorf("mlablocate: non-200 status code: %d", resp.StatusCode)
}
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(ctx, resp.Body)
if err != nil {
return Result{}, err
}

5
internal/engine/internal/mlablocatev2/fake_test.go
View File

@ -1,9 +1,10 @@
package mlablocatev2
import (
"io/ioutil"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeTransport struct {
@ -18,7 +19,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req)
}
if req.Body != nil {
ioutil.ReadAll(req.Body)
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {

4
internal/engine/internal/mlablocatev2/mlablocatev2.go
View File

@ -6,12 +6,12 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"regexp"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/iox"
)
const (
@ -102,7 +102,7 @@ func (c Client) query(ctx context.Context, path string) (resultRecord, error) {
if resp.StatusCode != 200 {
return resultRecord{}, fmt.Errorf("%w: %d", ErrRequestFailed, resp.StatusCode)
}
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(ctx, resp.Body)
if err != nil {
return resultRecord{}, err
}

2
internal/engine/legacy/netx/dialer.go
View File

@ -135,7 +135,7 @@ func (d *Dialer) SetCABundle(path string) error {
return err
}
pool := x509.NewCertPool()
if pool.AppendCertsFromPEM(cert) == false {
if !pool.AppendCertsFromPEM(cert) {
return errors.New("AppendCertsFromPEM failed")
}
d.TLSConfig.RootCAs = pool

6
internal/engine/legacy/netx/http_test.go
View File

@ -4,7 +4,6 @@ import (
"context"
"crypto/x509"
"errors"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
@ -15,6 +14,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func dowithclient(t *testing.T, client *netx.HTTPClient) {
@ -24,7 +24,7 @@ func dowithclient(t *testing.T, client *netx.HTTPClient) {
t.Fatal(err)
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
_, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
@ -126,7 +126,7 @@ func httpProxyTestMain(t *testing.T, client *http.Client, expect int) {
t.Fatal(err)
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
_, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

4
internal/engine/legacy/netx/oldhttptransport/bodytracer.go
View File

@ -63,7 +63,7 @@ func (bw *bodyWrapper) Read(b []byte) (n int, err error) {
// bytes read (0 <= n <= len(p)) and any error encountered."
Data: b[:n],
Error: err,
DurationSinceBeginning: time.Now().Sub(bw.root.Beginning),
DurationSinceBeginning: time.Since(bw.root.Beginning),
TransactionID: bw.tid,
},
})
@ -74,7 +74,7 @@ func (bw *bodyWrapper) Close() (err error) {
err = bw.ReadCloser.Close()
bw.root.Handler.OnMeasurement(modelx.Measurement{
HTTPResponseDone: &modelx.HTTPResponseDoneEvent{
DurationSinceBeginning: time.Now().Sub(bw.root.Beginning),
DurationSinceBeginning: time.Since(bw.root.Beginning),
TransactionID: bw.tid,
},
})

6
internal/engine/legacy/netx/oldhttptransport/bodytracer_test.go
View File

@ -1,9 +1,11 @@
package oldhttptransport
import (
"io/ioutil"
"context"
"net/http"
"testing"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestBodyTracerSuccess(t *testing.T) {
@ -15,7 +17,7 @@ func TestBodyTracerSuccess(t *testing.T) {
t.Fatal(err)
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
_, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

6
internal/engine/legacy/netx/oldhttptransport/httptransport_test.go
View File

@ -1,9 +1,11 @@
package oldhttptransport
import (
"io/ioutil"
"context"
"net/http"
"testing"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestGood(t *testing.T) {
@ -15,7 +17,7 @@ func TestGood(t *testing.T) {
t.Fatal(err)
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
_, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

35
internal/engine/legacy/netx/oldhttptransport/tracetripper.go
View File

@ -2,9 +2,9 @@ package oldhttptransport
import (
"bytes"
"context"
"crypto/tls"
"io"
"io/ioutil"
"net/http"
"net/http/httptrace"
"sync"
@ -16,12 +16,13 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/transactionid"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/iox"
)
// TraceTripper performs single HTTP transactions.
type TraceTripper struct {
readAllErrs *atomicx.Int64
readAll func(r io.Reader) ([]byte, error)
readAllContext func(ctx context.Context, r io.Reader) ([]byte, error)
roundTripper http.RoundTripper
}
@ -29,7 +30,7 @@ type TraceTripper struct {
func NewTraceTripper(roundTripper http.RoundTripper) *TraceTripper {
return &TraceTripper{
readAllErrs: &atomicx.Int64{},
readAll: ioutil.ReadAll,
readAllContext: iox.ReadAllContext,
roundTripper: roundTripper,
}
}
@ -57,10 +58,10 @@ func (c *readCloseWrapper) Close() error {
}
func readSnap(
source *io.ReadCloser, limit int64,
readAll func(r io.Reader) ([]byte, error),
ctx context.Context, source *io.ReadCloser, limit int64,
readAllContext func(ctx context.Context, r io.Reader) ([]byte, error),
) (data []byte, err error) {
data, err = readAll(io.LimitReader(*source, limit))
data, err = readAllContext(ctx, io.LimitReader(*source, limit))
if err == nil {
*source = newReadCloseWrapper(
io.MultiReader(bytes.NewReader(data), *source),
@ -79,7 +80,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
root.Handler.OnMeasurement(modelx.Measurement{
HTTPRoundTripStart: &modelx.HTTPRoundTripStartEvent{
DialID: dialid.ContextDialID(req.Context()),
DurationSinceBeginning: time.Now().Sub(root.Beginning),
DurationSinceBeginning: time.Since(root.Beginning),
Method: req.Method,
TransactionID: tid,
URL: req.URL.String(),
@ -98,7 +99,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
// Save a snapshot of the request body
if req.Body != nil {
requestBody, err = readSnap(&req.Body, snapSize, t.readAll)
requestBody, err = readSnap(req.Context(), &req.Body, snapSize, t.readAllContext)
if err != nil {
return nil, err
}
@ -114,7 +115,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
// configured in the http.Transport
root.Handler.OnMeasurement(modelx.Measurement{
TLSHandshakeStart: &modelx.TLSHandshakeStartEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning),
DurationSinceBeginning: time.Since(root.Beginning),
TransactionID: tid,
},
})
@ -127,7 +128,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
Operation: errorx.TLSHandshakeOperation,
TransactionID: tid,
}.MaybeBuild()
durationSinceBeginning := time.Now().Sub(root.Beginning)
durationSinceBeginning := time.Since(root.Beginning)
// Event emitted by net/http when DialTLS is not
// configured in the http.Transport
root.Handler.OnMeasurement(modelx.Measurement{
@ -149,7 +150,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
info.Conn.LocalAddr().Network(),
info.Conn.LocalAddr().String(),
),
DurationSinceBeginning: time.Now().Sub(root.Beginning),
DurationSinceBeginning: time.Since(root.Beginning),
TransactionID: tid,
},
})
@ -165,7 +166,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
requestHeadersMu.Unlock()
root.Handler.OnMeasurement(modelx.Measurement{
HTTPRequestHeader: &modelx.HTTPRequestHeaderEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning),
DurationSinceBeginning: time.Since(root.Beginning),
Key: key,
TransactionID: tid,
Value: values,
@ -175,7 +176,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
WroteHeaders: func() {
root.Handler.OnMeasurement(modelx.Measurement{
HTTPRequestHeadersDone: &modelx.HTTPRequestHeadersDoneEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning),
DurationSinceBeginning: time.Since(root.Beginning),
Headers: requestHeaders, // [*]
Method: req.Method, // [*]
TransactionID: tid,
@ -193,7 +194,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
}.MaybeBuild()
root.Handler.OnMeasurement(modelx.Measurement{
HTTPRequestDone: &modelx.HTTPRequestDoneEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning),
DurationSinceBeginning: time.Since(root.Beginning),
Error: err,
TransactionID: tid,
},
@ -202,7 +203,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
GotFirstResponseByte: func() {
root.Handler.OnMeasurement(modelx.Measurement{
HTTPResponseStart: &modelx.HTTPResponseStartEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning),
DurationSinceBeginning: time.Since(root.Beginning),
TransactionID: tid,
},
})
@ -233,7 +234,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
// [*] Require less event joining work by providing info that
// makes this event alone actionable for OONI
event := &modelx.HTTPRoundTripDoneEvent{
DurationSinceBeginning: time.Now().Sub(root.Beginning),
DurationSinceBeginning: time.Since(root.Beginning),
Error: err,
RequestBodySnap: requestBody,
RequestHeaders: requestHeaders, // [*]
@ -248,7 +249,7 @@ func (t *TraceTripper) RoundTrip(req *http.Request) (*http.Response, error) {
event.ResponseProto = resp.Proto
// Save a snapshot of the response body
var data []byte
data, err = readSnap(&resp.Body, snapSize, t.readAll)
data, err = readSnap(req.Context(), &resp.Body, snapSize, t.readAllContext)
if err != nil {
t.readAllErrs.Add(1)
resp = nil // this is how net/http likes it

10
internal/engine/legacy/netx/oldhttptransport/tracetripper_test.go
View File

@ -5,7 +5,6 @@ import (
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"net/http/httptrace"
"sync"
@ -14,6 +13,7 @@ import (
"github.com/miekg/dns"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestTraceTripperSuccess(t *testing.T) {
@ -25,7 +25,7 @@ func TestTraceTripperSuccess(t *testing.T) {
t.Fatal(err)
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
_, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
@ -47,7 +47,7 @@ func (h *roundTripHandler) OnMeasurement(m modelx.Measurement) {
func TestTraceTripperReadAllFailure(t *testing.T) {
transport := NewTraceTripper(http.DefaultTransport)
transport.readAll = func(r io.Reader) ([]byte, error) {
transport.readAllContext = func(ctx context.Context, r io.Reader) ([]byte, error) {
return nil, io.EOF
}
client := &http.Client{Transport: transport}
@ -156,7 +156,7 @@ func TestTraceTripperWithCorrectSnaps(t *testing.T) {
// Read the whole response body, parse it as valid DNS
// reply and verify we obtained what we expected
replyData, err := ioutil.ReadAll(resp.Body)
replyData, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
@ -230,7 +230,7 @@ func TestTraceTripperWithReadAllFailingForBody(t *testing.T) {
// use such transport to configure an ordinary client
transport := NewTraceTripper(http.DefaultTransport)
errorMocked := errors.New("mocked error")
transport.readAll = func(r io.Reader) ([]byte, error) {
transport.readAllContext = func(ctx context.Context, r io.Reader) ([]byte, error) {
return nil, errorMocked
}
const snapSize = 15

5
internal/engine/legacy/netx/oldhttptransport/transactioner_test.go
View File

@ -1,11 +1,12 @@
package oldhttptransport
import (
"io/ioutil"
"context"
"net/http"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/transactionid"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type transactionerCheckTransactionID struct {
@ -33,7 +34,7 @@ func TestTransactionerSuccess(t *testing.T) {
t.Fatal(err)
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
_, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

5
internal/engine/legacy/netxlogger/netxlogger_test.go
View File

@ -1,7 +1,7 @@
package netxlogger
import (
"io/ioutil"
"context"
"net/http"
"testing"
"time"
@ -10,6 +10,7 @@ import (
"github.com/apex/log/handlers/discard"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestGood(t *testing.T) {
@ -32,7 +33,7 @@ func TestGood(t *testing.T) {
t.Fatal("expected non-nil resp here")
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
_, err = iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

5
internal/engine/legacy/oonitemplates/oonitemplates.go
View File

@ -24,6 +24,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/handlers"
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netx/modelx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"gitlab.com/yawning/obfs4.git/transports"
obfs4base "gitlab.com/yawning/obfs4.git/transports/base"
@ -90,7 +91,7 @@ func (m *connmapper) scramble(cid int64) int64 {
// See https://stackoverflow.com/a/38140573/4354461
m.mu.Lock()
defer m.mu.Unlock()
if value, found := m.table[cid]; found == true {
if value, found := m.table[cid]; found {
return value
}
var factor int64 = 1
@ -366,7 +367,7 @@ func HTTPDo(
config.MaxResponseBodySnapSize,
),
)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
mu.Lock()
results.BodySnap, results.Error = data, err
mu.Unlock()

4
internal/engine/legacy/oonitemplates/oonitemplates_test.go
View File

@ -377,7 +377,7 @@ type faketransport struct {
}
func (txp *faketransport) Name() string {
return txp.Name()
return txp.txp.Name()
}
func (txp *faketransport) ClientFactory(stateDir string) (obfs4base.ClientFactory, error) {
@ -385,7 +385,7 @@ func (txp *faketransport) ClientFactory(stateDir string) (obfs4base.ClientFactor
}
func (txp *faketransport) ServerFactory(stateDir string, args *goptlib.Args) (obfs4base.ServerFactory, error) {
return txp.ServerFactory(stateDir, args)
return txp.txp.ServerFactory(stateDir, args)
}
func TestConnmapper(t *testing.T) {

5
internal/engine/netx/fake_test.go
View File

@ -2,10 +2,11 @@ package netx
import (
"context"
"io/ioutil"
"net"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeDialer struct {
@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req)
}
if req.Body != nil {
ioutil.ReadAll(req.Body)
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {

6
internal/engine/netx/httptransport/bytecounter_test.go
View File

@ -1,6 +1,7 @@
package httptransport_test
import (
"context"
"errors"
"io"
"io/ioutil"
@ -10,6 +11,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestByteCounterFailure(t *testing.T) {
@ -68,7 +70,7 @@ func TestByteCounterSuccess(t *testing.T) {
if err != nil {
t.Fatal(err)
}
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
@ -104,7 +106,7 @@ func TestByteCounterSuccessWithEOF(t *testing.T) {
if err != nil {
t.Fatal(err)
}
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

5
internal/engine/netx/httptransport/fake_test.go
View File

@ -2,10 +2,11 @@ package httptransport
import (
"context"
"io/ioutil"
"net"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeDialer struct {
@ -30,7 +31,7 @@ func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return txp.Func(req)
}
if req.Body != nil {
ioutil.ReadAll(req.Body)
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {

4
internal/engine/netx/httptransport/logging_test.go
View File

@ -1,6 +1,7 @@
package httptransport_test
import (
"context"
"errors"
"io"
"io/ioutil"
@ -11,6 +12,7 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestLoggingFailure(t *testing.T) {
@ -72,6 +74,6 @@ func TestLoggingSuccess(t *testing.T) {
if err != nil {
t.Fatal(err)
}
ioutil.ReadAll(resp.Body)
iox.ReadAllContext(context.Background(), resp.Body)
resp.Body.Close()
}

11
internal/engine/netx/httptransport/saver.go
View File

@ -2,14 +2,15 @@ package httptransport
import (
"bytes"
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"net/http/httptrace"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/iox"
)
// SaverPerformanceHTTPTransport is a RoundTripper that saves
@ -109,7 +110,7 @@ func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response,
snapsize = txp.SnapshotSize
}
if req.Body != nil {
data, err := saverSnapRead(req.Body, snapsize)
data, err := saverSnapRead(req.Context(), req.Body, snapsize)
if err != nil {
return nil, err
}
@ -125,7 +126,7 @@ func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response,
if err != nil {
return nil, err
}
data, err := saverSnapRead(resp.Body, snapsize)
data, err := saverSnapRead(req.Context(), resp.Body, snapsize)
err = ignoreExpectedEOF(err, resp)
if err != nil {
resp.Body.Close()
@ -157,8 +158,8 @@ func ignoreExpectedEOF(err error, resp *http.Response) error {
return err
}
func saverSnapRead(r io.ReadCloser, snapsize int) ([]byte, error) {
return ioutil.ReadAll(io.LimitReader(r, int64(snapsize)))
func saverSnapRead(ctx context.Context, r io.ReadCloser, snapsize int) ([]byte, error) {
return iox.ReadAllContext(ctx, io.LimitReader(r, int64(snapsize)))
}
func saverCompose(data []byte, r io.ReadCloser) io.ReadCloser {

6
internal/engine/netx/httptransport/saver_test.go
View File

@ -1,6 +1,7 @@
package httptransport_test
import (
"context"
"errors"
"io/ioutil"
"net/http"
@ -10,6 +11,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestSaverPerformanceNoMultipleEvents(t *testing.T) {
@ -245,7 +247,7 @@ func TestSaverBodySuccess(t *testing.T) {
txp := httptransport.SaverBodyHTTPTransport{
RoundTripper: httptransport.FakeTransport{
Func: func(req *http.Request) (*http.Response, error) {
data, err := ioutil.ReadAll(req.Body)
data, err := iox.ReadAllContext(context.Background(), req.Body)
if err != nil {
t.Fatal(err)
}
@ -274,7 +276,7 @@ func TestSaverBodySuccess(t *testing.T) {
t.Fatal("unexpected status code")
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}

4
internal/engine/netx/integration_test.go
View File

@ -3,7 +3,6 @@ package netx_test
import (
"context"
"errors"
"io/ioutil"
"net/http"
"testing"
@ -12,6 +11,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestSuccess(t *testing.T) {
@ -38,7 +38,7 @@ func TestSuccess(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err = ioutil.ReadAll(resp.Body); err != nil {
if _, err = iox.ReadAllContext(context.Background(), resp.Body); err != nil {
t.Fatal(err)
}
if err = resp.Body.Close(); err != nil {

4
internal/engine/netx/resolver/dnsoverhttps.go
View File

@ -4,11 +4,11 @@ import (
"bytes"
"context"
"errors"
"io/ioutil"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
"github.com/ooni/probe-cli/v3/internal/iox"
)
// DNSOverHTTPS is a DNS over HTTPS RoundTripper. Requests are submitted over
@ -56,7 +56,7 @@ func (t DNSOverHTTPS) RoundTrip(ctx context.Context, query []byte) ([]byte, erro
if resp.Header.Get("content-type") != "application/dns-message" {
return nil, errors.New("doh: invalid content-type")
}
return ioutil.ReadAll(resp.Body)
return iox.ReadAllContext(ctx, resp.Body)
}
// RequiresPadding returns true for DoH according to RFC8467

2
internal/engine/netx/tlsx/certifi.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-06-08 14:36:18.474117 +0200 CEST m=+0.459588210
// 2021-06-15 10:55:55.638897 +0200 CEST m=+4.257631084
// https://curl.haxx.se/ca/cacert.pem
package tlsx

6
internal/engine/netx/tlsx/generate.go
View File

@ -12,14 +12,16 @@
package main
import (
"context"
"crypto/x509"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"text/template"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
var tmpl = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
@ -50,7 +52,7 @@ func main() {
}
defer resp.Body.Close()
bundle, err := ioutil.ReadAll(resp.Body)
bundle, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
log.Fatal(err)
}

3
internal/engine/probeservices/collector_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type fakeTestKeys struct {
@ -233,7 +234,7 @@ func TestEndToEnd(t *testing.T) {
return
}
if r.RequestURI == "/report/_id" {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
panic(err)
}

4
internal/engine/probeservices/probeservices_test.go
View File

@ -4,7 +4,6 @@ import (
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"regexp"
"strings"
@ -17,6 +16,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func newclient() *probeservices.Client {
@ -165,7 +165,7 @@ func TestCloudfront(t *testing.T) {
if resp.StatusCode != 200 {
t.Fatal("unexpected status code")
}
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(req.Context(), resp.Body)
if err != nil {
t.Fatal(err)
}

4
internal/engine/session_psiphon.go
View File

@ -6,9 +6,9 @@ import (
"bytes"
"context"
_ "embed"
"io/ioutil"
"filippo.io/age"
"github.com/ooni/probe-cli/v3/internal/iox"
)
//go:embed psiphon-config.json.age
@ -34,7 +34,7 @@ func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]b
if err != nil {
return nil, err
}
return ioutil.ReadAll(output)
return iox.ReadAllContext(ctx, output)
}
// FetchPsiphonConfig decrypts psiphonConfigJSONAge using

21
internal/iox/example_test.go Normal file
View File

@ -0,0 +1,21 @@
package iox_test
import (
"context"
"fmt"
"log"
"strings"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func ExampleReadAllContext() {
r := strings.NewReader("deadbeef")
ctx := context.Background()
out, err := iox.ReadAllContext(ctx, r)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%d\n", len(out))
// Output: 8
}

47
internal/iox/iox.go Normal file
View File

@ -0,0 +1,47 @@
// Package iox contains io extensions.
package iox
import (
"context"
"io"
)
// ReadAllContext reads the whole reader r in a
// background goroutine. This function will return
// earlier if the context is cancelled. In which case
// we will continue reading from r in the background
// goroutine, and we will discard the result. To stop
// the long-running goroutine, you need to close the
// connection bound to the r reader, if possible.
func ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) {
datach, errch := make(chan []byte, 1), make(chan error, 1) // buffers
go func() {
data, err := io.ReadAll(r)
if err != nil {
errch <- err
return
}
datach <- data
}()
select {
case data := <-datach:
return data, nil
case <-ctx.Done():
return nil, ctx.Err()
case err := <-errch:
return nil, err
}
}
// MockableReader allows to mock any io.Reader.
type MockableReader struct {
MockRead func(b []byte) (int, error)
}
// MockableReader implements an io.Reader.
var _ io.Reader = &MockableReader{}
// Read implements io.Reader.Read.
func (r *MockableReader) Read(b []byte) (int, error) {
return r.MockRead(b)
}

70
internal/iox/iox_test.go Normal file
View File

@ -0,0 +1,70 @@
package iox
import (
"context"
"errors"
"strings"
"testing"
"time"
)
func TestReadAllContextCommonCase(t *testing.T) {
r := strings.NewReader("deadbeef")
ctx := context.Background()
out, err := ReadAllContext(ctx, r)
if err != nil {
t.Fatal(err)
}
if len(out) != 8 {
t.Fatal("not the expected number of bytes")
}
}
func TestReadAllContextWithError(t *testing.T) {
expected := errors.New("mocked error")
r := &MockableReader{
MockRead: func(b []byte) (int, error) {
return 0, expected
},
}
ctx := context.Background()
out, err := ReadAllContext(ctx, r)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected", err)
}
if len(out) != 0 {
t.Fatal("not the expected number of bytes")
}
}
func TestReadAllContextWithCancelledContext(t *testing.T) {
r := strings.NewReader("deadbeef")
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
out, err := ReadAllContext(ctx, r)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected", err)
}
if len(out) != 0 {
t.Fatal("not the expected number of bytes")
}
}
func TestReadAllContextWithErrorAndCancelledContext(t *testing.T) {
expected := errors.New("mocked error")
r := &MockableReader{
MockRead: func(b []byte) (int, error) {
time.Sleep(time.Millisecond)
return 0, expected
},
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
out, err := ReadAllContext(ctx, r)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected", err)
}
if len(out) != 0 {
t.Fatal("not the expected number of bytes")
}
}

35
internal/ooapi/apis.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:00.422051399 +0200 CEST m=+0.000129449
// 2021-06-15 10:55:55.967236 +0200 CEST m=+0.000374501
package ooapi
@ -59,7 +59,8 @@ func (api *simpleCheckReportIDAPI) Call(ctx context.Context, req *apimodel.Check
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleCheckInAPI implements the CheckIn API.
@ -109,7 +110,8 @@ func (api *simpleCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequ
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleLoginAPI implements the Login API.
@ -159,7 +161,8 @@ func (api *simpleLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest)
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleMeasurementMetaAPI implements the MeasurementMeta API.
@ -209,7 +212,8 @@ func (api *simpleMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.Mea
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleRegisterAPI implements the Register API.
@ -259,7 +263,8 @@ func (api *simpleRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRe
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleTestHelpersAPI implements the TestHelpers API.
@ -309,7 +314,8 @@ func (api *simpleTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHel
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simplePsiphonConfigAPI implements the PsiphonConfig API.
@ -377,7 +383,8 @@ func (api *simplePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.Psiph
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleTorTargetsAPI implements the TorTargets API.
@ -445,7 +452,8 @@ func (api *simpleTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTarge
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleURLsAPI implements the URLs API.
@ -495,7 +503,8 @@ func (api *simpleURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleOpenReportAPI implements the OpenReport API.
@ -545,7 +554,8 @@ func (api *simpleOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenRepo
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleSubmitMeasurementAPI implements the SubmitMeasurement API.
@ -603,5 +613,6 @@ func (api *simpleSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.S
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
return api.newResponse(api.httpClient().Do(httpReq))
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}

26
internal/ooapi/apis_test.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:01.254917327 +0200 CEST m=+0.000217025
// 2021-06-15 10:55:56.308361 +0200 CEST m=+0.000215751
package ooapi
@ -9,7 +9,6 @@ import (
"context"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
@ -18,6 +17,7 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
@ -177,7 +177,7 @@ func (h *handleCheckReportID) ServeHTTP(w http.ResponseWriter, r *http.Request)
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -440,7 +440,7 @@ func (h *handleCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -685,7 +685,7 @@ func (h *handleLogin) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -912,7 +912,7 @@ func (h *handleMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *http.Request
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -1175,7 +1175,7 @@ func (h *handleRegister) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -1402,7 +1402,7 @@ func (h *handleTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -1671,7 +1671,7 @@ func (h *handlePsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http.Request)
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -1942,7 +1942,7 @@ func (h *handleTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -2192,7 +2192,7 @@ func (h *handleURLs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -2437,7 +2437,7 @@ func (h *handleOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -2682,7 +2682,7 @@ func (h *handleSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r *http.Reque
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return

2
internal/ooapi/caching.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:01.869492095 +0200 CEST m=+0.000168945
// 2021-06-15 10:55:56.587245 +0200 CEST m=+0.000218959
package ooapi

2
internal/ooapi/caching_test.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:02.497717446 +0200 CEST m=+0.000113904
// 2021-06-15 10:55:56.851681 +0200 CEST m=+0.000186126
package ooapi

2
internal/ooapi/callers.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:03.02266641 +0200 CEST m=+0.000097757
// 2021-06-15 10:55:57.174427 +0200 CEST m=+0.000217876
package ooapi

2
internal/ooapi/clientcall.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:03.586305848 +0200 CEST m=+0.000123000
// 2021-06-15 10:55:57.445485 +0200 CEST m=+0.000219751
package ooapi

22
internal/ooapi/clientcall_test.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:04.198485035 +0200 CEST m=+0.000114145
// 2021-06-15 10:55:57.708833 +0200 CEST m=+0.000193418
package ooapi
@ -8,7 +8,6 @@ package ooapi
import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
@ -16,6 +15,7 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
@ -42,7 +42,7 @@ func (h *handleClientCallCheckReportID) ServeHTTP(w http.ResponseWriter, r *http
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -135,7 +135,7 @@ func (h *handleClientCallCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Reque
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -227,7 +227,7 @@ func (h *handleClientCallMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *ht
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -320,7 +320,7 @@ func (h *handleClientCallTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.R
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -435,7 +435,7 @@ func (h *handleClientCallPsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -550,7 +550,7 @@ func (h *handleClientCallTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Re
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -643,7 +643,7 @@ func (h *handleClientCallURLs) ServeHTTP(w http.ResponseWriter, r *http.Request)
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -736,7 +736,7 @@ func (h *handleClientCallOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Re
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -828,7 +828,7 @@ func (h *handleClientCallSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r *
}
h.count++
if r.Body != nil {
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return

2
internal/ooapi/cloners.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:04.793154609 +0200 CEST m=+0.000108739
// 2021-06-15 10:55:57.987741 +0200 CEST m=+0.000211126
package ooapi

5
internal/ooapi/fake_test.go
View File

@ -3,9 +3,10 @@ package ooapi
import (
"context"
"io"
"io/ioutil"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeCodec struct {
@ -30,7 +31,7 @@ type FakeHTTPClient struct {
func (c *FakeHTTPClient) Do(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond)
if req.Body != nil {
_, _ = ioutil.ReadAll(req.Body)
_, _ = iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if c.Err != nil {

2
internal/ooapi/fakeapi_test.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:05.331414434 +0200 CEST m=+0.000124504
// 2021-06-15 10:55:58.234786 +0200 CEST m=+0.000218167
package ooapi

3
internal/ooapi/internal/generator/apis.go
View File

@ -156,7 +156,8 @@ func (d *Descriptor) genNewAPI(sb *strings.Builder) {
fmt.Fprint(sb, "\tif api.UserAgent != \"\" {\n")
fmt.Fprint(sb, "\t\thttpReq.Header.Add(\"User-Agent\", api.UserAgent)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn api.newResponse(api.httpClient().Do(httpReq))\n")
fmt.Fprint(sb, "\thttpResp, err := api.httpClient().Do(httpReq)\n")
fmt.Fprint(sb, "\treturn api.newResponse(ctx, httpResp, err)\n")
fmt.Fprint(sb, "}\n\n")
}

4
internal/ooapi/internal/generator/apistest.go
View File

@ -231,7 +231,7 @@ func (d *Descriptor) genTestRoundTrip(sb *strings.Builder) {
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\th.count++\n")
fmt.Fprint(sb, "\tif r.Body != nil {\n")
fmt.Fprint(sb, "\t\tdata, err := ioutil.ReadAll(r.Body)\n")
fmt.Fprint(sb, "\t\tdata, err := iox.ReadAllContext(r.Context(), r.Body)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\t\treturn\n")
@ -443,7 +443,6 @@ func GenAPIsTestGo(file string) {
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"encoding/json\"\n")
fmt.Fprint(&sb, "\t\"errors\"\n")
fmt.Fprint(&sb, "\t\"io/ioutil\"\n")
fmt.Fprint(&sb, "\t\"net/http/httptest\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\t\"net/url\"\n")
@ -452,6 +451,7 @@ func GenAPIsTestGo(file string) {
fmt.Fprint(&sb, "\t\"sync\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {

4
internal/ooapi/internal/generator/clientcalltest.go
View File

@ -57,7 +57,7 @@ func (d *Descriptor) genTestClientCallRoundTrip(sb *strings.Builder) {
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\th.count++\n")
fmt.Fprint(sb, "\tif r.Body != nil {\n")
fmt.Fprint(sb, "\t\tdata, err := ioutil.ReadAll(r.Body)\n")
fmt.Fprint(sb, "\t\tdata, err := iox.ReadAllContext(r.Context(), r.Body)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\t\treturn\n")
@ -161,7 +161,6 @@ func GenClientCallTestGo(file string) {
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"encoding/json\"\n")
fmt.Fprint(&sb, "\t\"io/ioutil\"\n")
fmt.Fprint(&sb, "\t\"net/http/httptest\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\t\"net/url\"\n")
@ -169,6 +168,7 @@ func GenClientCallTestGo(file string) {
fmt.Fprint(&sb, "\t\"sync\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/kvstore\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")

7
internal/ooapi/internal/generator/responses.go
View File

@ -9,7 +9,7 @@ import (
func (d *Descriptor) genNewResponse(sb *strings.Builder) {
fmt.Fprintf(sb,
"func (api *%s) newResponse(resp *http.Response, err error) (%s, error) {\n",
"func (api *%s) newResponse(ctx context.Context, resp *http.Response, err error) (%s, error) {\n",
d.APIStructName(), d.ResponseTypeName())
fmt.Fprint(sb, "\tif err != nil {\n")
@ -23,7 +23,7 @@ func (d *Descriptor) genNewResponse(sb *strings.Builder) {
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tdefer resp.Body.Close()\n")
fmt.Fprint(sb, "\treader := io.LimitReader(resp.Body, 4<<20)\n")
fmt.Fprint(sb, "\tdata, err := ioutil.ReadAll(reader)\n")
fmt.Fprint(sb, "\tdata, err := iox.ReadAllContext(ctx, reader)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
@ -67,10 +67,11 @@ func GenResponsesGo(file string) {
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"io\"\n")
fmt.Fprint(&sb, "\t\"io/ioutil\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/iox\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n\n")
for _, desc := range Descriptors {

2
internal/ooapi/login.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:06.213159282 +0200 CEST m=+0.000104135
// 2021-06-15 10:55:58.487888 +0200 CEST m=+0.000221710
package ooapi

2
internal/ooapi/login_test.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:06.893164862 +0200 CEST m=+0.000136881
// 2021-06-15 10:55:58.755431 +0200 CEST m=+0.000217917
package ooapi

6
internal/ooapi/loginhandler_test.go
View File

@ -2,7 +2,6 @@ package ooapi
import (
"encoding/json"
"io/ioutil"
"net/http"
"strings"
"sync"
@ -10,6 +9,7 @@ import (
"time"
"github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
@ -72,7 +72,7 @@ func (lh *LoginHandler) register(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(400)
return
}
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
@ -113,7 +113,7 @@ func (lh *LoginHandler) login(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(400)
return
}
data, err := ioutil.ReadAll(r.Body)
data, err := iox.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return

2
internal/ooapi/requests.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:07.590481334 +0200 CEST m=+0.000085230
// 2021-06-15 10:55:59.020364 +0200 CEST m=+0.000217085
package ooapi

49
internal/ooapi/responses.go
View File

@ -1,19 +1,20 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:08.237841277 +0200 CEST m=+0.000121556
// 2021-06-15 10:55:59.329509 +0200 CEST m=+0.000220043
package ooapi
//go:generate go run ./internal/generator -file responses.go
import (
"context"
"io"
"io/ioutil"
"net/http"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) {
func (api *simpleCheckReportIDAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) {
if err != nil {
return nil, err
}
@ -25,7 +26,7 @@ func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) (
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -36,7 +37,7 @@ func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) (
return out, nil
}
func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckInResponse, error) {
func (api *simpleCheckInAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.CheckInResponse, error) {
if err != nil {
return nil, err
}
@ -48,7 +49,7 @@ func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimo
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -59,7 +60,7 @@ func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimo
return out, nil
}
func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimodel.LoginResponse, error) {
func (api *simpleLoginAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.LoginResponse, error) {
if err != nil {
return nil, err
}
@ -71,7 +72,7 @@ func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimode
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -82,7 +83,7 @@ func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimode
return out, nil
}
func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) {
func (api *simpleMeasurementMetaAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) {
if err != nil {
return nil, err
}
@ -94,7 +95,7 @@ func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -105,7 +106,7 @@ func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error)
return out, nil
}
func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apimodel.RegisterResponse, error) {
func (api *simpleRegisterAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.RegisterResponse, error) {
if err != nil {
return nil, err
}
@ -117,7 +118,7 @@ func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apim
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -128,7 +129,7 @@ func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apim
return out, nil
}
func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (apimodel.TestHelpersResponse, error) {
func (api *simpleTestHelpersAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.TestHelpersResponse, error) {
if err != nil {
return nil, err
}
@ -140,7 +141,7 @@ func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (ap
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -154,7 +155,7 @@ func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (ap
return out, nil
}
func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) {
func (api *simplePsiphonConfigAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) {
if err != nil {
return nil, err
}
@ -166,7 +167,7 @@ func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) (
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -180,7 +181,7 @@ func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) (
return out, nil
}
func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (apimodel.TorTargetsResponse, error) {
func (api *simpleTorTargetsAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.TorTargetsResponse, error) {
if err != nil {
return nil, err
}
@ -192,7 +193,7 @@ func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (api
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -206,7 +207,7 @@ func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (api
return out, nil
}
func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel.URLsResponse, error) {
func (api *simpleURLsAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.URLsResponse, error) {
if err != nil {
return nil, err
}
@ -218,7 +219,7 @@ func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -229,7 +230,7 @@ func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel
return out, nil
}
func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*apimodel.OpenReportResponse, error) {
func (api *simpleOpenReportAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.OpenReportResponse, error) {
if err != nil {
return nil, err
}
@ -241,7 +242,7 @@ func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*ap
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
@ -252,7 +253,7 @@ func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*ap
return out, nil
}
func (api *simpleSubmitMeasurementAPI) newResponse(resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) {
func (api *simpleSubmitMeasurementAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) {
if err != nil {
return nil, err
}
@ -264,7 +265,7 @@ func (api *simpleSubmitMeasurementAPI) newResponse(resp *http.Response, err erro
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := ioutil.ReadAll(reader)
data, err := iox.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}

4
internal/ooapi/swagger_test.go
View File

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2021-05-12 09:15:08.888123159 +0200 CEST m=+0.000883704
// 2021-06-15 10:55:59.636856 +0200 CEST m=+0.000624459
package ooapi
@ -9,7 +9,7 @@ const swagger = `{
"swagger": "2.0",
"info": {
"title": "OONI API specification",
"version": "0.20210512.5071508"
"version": "0.20210615.6085559"
},
"host": "api.ooni.io",
"basePath": "/",

5
internal/ooapi/swaggerdiff_test.go
View File

@ -1,9 +1,9 @@
package ooapi
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"sort"
@ -13,6 +13,7 @@ import (
"github.com/hexops/gotextdiff"
"github.com/hexops/gotextdiff/myers"
"github.com/hexops/gotextdiff/span"
"github.com/ooni/probe-cli/v3/internal/iox"
"github.com/ooni/probe-cli/v3/internal/ooapi/internal/openapi"
)
@ -36,7 +37,7 @@ func getServerModel(serverURL string) *openapi.Swagger {
log.Fatal(err)
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
log.Fatal(err)
}

4
internal/ptx/ptx_test.go
View File

@ -4,7 +4,6 @@ import (
"context"
"errors"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
@ -15,6 +14,7 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/mockablex"
"github.com/ooni/probe-cli/v3/internal/iox"
)
func TestListenerLoggerWorks(t *testing.T) {
@ -57,7 +57,7 @@ func TestListenerWorksWithFakeDialer(t *testing.T) {
if err != nil {
t.Fatal(err)
}
data, err := ioutil.ReadAll(resp.Body)
data, err := iox.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}