From 0f61778e7af9f34fc021e44d068dabf0d51c4921 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 18 Mar 2021 08:44:58 +0100 Subject: [PATCH 01/28] oonimkall: mobile api for running WebConnectivity (#223) * poc: mobile api for running WebConnectivity Yesterday we had a team meeting where we discussed the importance of stopping using MK wrappers for running experiments. Here's a PoC showing how we can do that for WebConnectivity. It seems to me we probably want to add more tests and merge this code such that we can experiment with it quite soon. There seems to be opportunities for auto-generating code, BTW. While working on this PoC, I've also took a chance to revamp the external documentation of pkg/oonimkall. * feat: start making the code more abstract * chore: write unit tests for new code * fix(oonimkall): improve naming * refactor: cosmetic changes * fix: explain --- pkg/oonimkall/experiment.go | 96 +++++++++++ pkg/oonimkall/experiment_test.go | 93 +++++++++++ pkg/oonimkall/session_integration_test.go | 34 ++-- pkg/oonimkall/webconnectivity.go | 91 ++++++++++ .../webconnectivity_integration_test.go | 28 ++++ pkg/oonimkall/webconnectivity_test.go | 156 ++++++++++++++++++ 6 files changed, 481 insertions(+), 17 deletions(-) create mode 100644 pkg/oonimkall/experiment.go create mode 100644 pkg/oonimkall/experiment_test.go create mode 100644 pkg/oonimkall/webconnectivity.go create mode 100644 pkg/oonimkall/webconnectivity_integration_test.go create mode 100644 pkg/oonimkall/webconnectivity_test.go diff --git a/pkg/oonimkall/experiment.go b/pkg/oonimkall/experiment.go new file mode 100644 index 0000000..559af1c --- /dev/null +++ b/pkg/oonimkall/experiment.go @@ -0,0 +1,96 @@ +package oonimkall + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/model" +) + +// experimentSession is the abstract representation of +// a session according to an experiment. +type experimentSession interface { + // lock locks the session + lock() + + // maybeLookupBackends lookups the backends + maybeLookupBackends(ctx context.Context) error + + // maybeLookupLocations lookups the probe location + maybeLookupLocation(ctx context.Context) error + + // newExperimentBuilder creates a new experiment builder + newExperimentBuilder(name string) (experimentBuilder, error) + + // unlock unlocks the session + unlock() +} + +// lock implements experimentSession.lock +func (sess *Session) lock() { + sess.mtx.Lock() +} + +// maybeLookupBackends implements experimentSession.maybeLookupBackends +func (sess *Session) maybeLookupBackends(ctx context.Context) error { + return sess.sessp.MaybeLookupBackendsContext(ctx) +} + +// maybeLookupLocation implements experimentSession.maybeLookupLocation +func (sess *Session) maybeLookupLocation(ctx context.Context) error { + return sess.sessp.MaybeLookupLocationContext(ctx) +} + +// newExperimentBuilder implements experimentSession.newExperimentBuilder +func (sess *Session) newExperimentBuilder(name string) (experimentBuilder, error) { + eb, err := sess.sessp.NewExperimentBuilder(name) + if err != nil { + return nil, err + } + return &experimentBuilderWrapper{eb: eb}, nil +} + +// unlock implements experimentSession.unlock +func (sess *Session) unlock() { + sess.mtx.Unlock() +} + +// experimentBuilder is the representation of an experiment +// builder that we use inside this package. +type experimentBuilder interface { + // newExperiment creates a new experiment instance + newExperiment() experiment + + // setCallbacks sets the experiment callbacks + setCallbacks(ExperimentCallbacks) +} + +// experimentBuilderWrapper wraps *ExperimentBuilder +type experimentBuilderWrapper struct { + eb *engine.ExperimentBuilder +} + +// newExperiment implements experimentBuilder.newExperiment +func (eb *experimentBuilderWrapper) newExperiment() experiment { + return eb.eb.NewExperiment() +} + +// setCallbacks implements experimentBuilder.setCallbacks +func (eb *experimentBuilderWrapper) setCallbacks(cb ExperimentCallbacks) { + eb.eb.SetCallbacks(cb) +} + +// experiment is the representation of an experiment that +// we use inside this package for running nettests. +type experiment interface { + // MeasureWithContext runs the measurement with the given input + // and context. It returns a measurement or an error. + MeasureWithContext(ctx context.Context, input string) ( + measurement *model.Measurement, err error) + + // KibiBytesSent returns the number of KiB sent. + KibiBytesSent() float64 + + // KibiBytesReceived returns the number of KiB received. + KibiBytesReceived() float64 +} diff --git a/pkg/oonimkall/experiment_test.go b/pkg/oonimkall/experiment_test.go new file mode 100644 index 0000000..3a9a732 --- /dev/null +++ b/pkg/oonimkall/experiment_test.go @@ -0,0 +1,93 @@ +package oonimkall + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/ooni/probe-cli/v3/internal/engine/model" +) + +// FakeExperimentCallbacks contains fake ExperimentCallbacks. +type FakeExperimentCallbacks struct{} + +// OnProgress implements ExperimentCallbacks.OnProgress +func (cb *FakeExperimentCallbacks) OnProgress(percentage float64, message string) {} + +// FakeExperimentSession is a fake experimentSession +type FakeExperimentSession struct { + ExperimentBuilder experimentBuilder + LockCount int32 + LookupBackendsErr error + LookupLocationErr error + NewExperimentBuilderErr error + UnlockCount int32 +} + +// lock implements experimentSession.lock +func (sess *FakeExperimentSession) lock() { + atomic.AddInt32(&sess.LockCount, 1) +} + +// maybeLookupBackends implements experimentSession.maybeLookupBackends +func (sess *FakeExperimentSession) maybeLookupBackends(ctx context.Context) error { + return sess.LookupBackendsErr +} + +// maybeLookupLocation implements experimentSession.maybeLookupLocation +func (sess *FakeExperimentSession) maybeLookupLocation(ctx context.Context) error { + return sess.LookupLocationErr +} + +// newExperimentBuilder implements experimentSession.newExperimentBuilder +func (sess *FakeExperimentSession) newExperimentBuilder(name string) (experimentBuilder, error) { + return sess.ExperimentBuilder, sess.NewExperimentBuilderErr +} + +// unlock implements experimentSession.unlock +func (sess *FakeExperimentSession) unlock() { + atomic.AddInt32(&sess.UnlockCount, 1) +} + +// FakeExperimentBuilder is a fake experimentBuilder +type FakeExperimentBuilder struct { + Callbacks ExperimentCallbacks + Experiment experiment + mu sync.Mutex +} + +// newExperiment implements experimentBuilder.newExperiment +func (eb *FakeExperimentBuilder) newExperiment() experiment { + return eb.Experiment +} + +// setCallbacks implements experimentBuilder.setCallbacks +func (eb *FakeExperimentBuilder) setCallbacks(cb ExperimentCallbacks) { + defer eb.mu.Unlock() + eb.mu.Lock() + eb.Callbacks = cb +} + +// FakeExperiment is a fake experiment +type FakeExperiment struct { + Err error + Measurement *model.Measurement + Received float64 + Sent float64 +} + +// MeasureWithContext implements experiment.MeasureWithContext. +func (e *FakeExperiment) MeasureWithContext(ctx context.Context, input string) ( + measurement *model.Measurement, err error) { + return e.Measurement, e.Err +} + +// KibiBytesSent implements experiment.KibiBytesSent +func (e *FakeExperiment) KibiBytesSent() float64 { + return e.Sent +} + +// KibiBytesReceived implements experiment.KibiBytesReceived +func (e *FakeExperiment) KibiBytesReceived() float64 { + return e.Received +} diff --git a/pkg/oonimkall/session_integration_test.go b/pkg/oonimkall/session_integration_test.go index 03303e2..26f1bb6 100644 --- a/pkg/oonimkall/session_integration_test.go +++ b/pkg/oonimkall/session_integration_test.go @@ -18,7 +18,7 @@ import ( "github.com/ooni/probe-cli/v3/pkg/oonimkall" ) -func NewSessionWithAssetsDir(assetsDir string) (*oonimkall.Session, error) { +func NewSessionForTestingWithAssetsDir(assetsDir string) (*oonimkall.Session, error) { return oonimkall.NewSession(&oonimkall.SessionConfig{ AssetsDir: assetsDir, ProbeServicesURL: "https://ams-pg-test.ooni.org/", @@ -29,8 +29,8 @@ func NewSessionWithAssetsDir(assetsDir string) (*oonimkall.Session, error) { }) } -func NewSession() (*oonimkall.Session, error) { - return NewSessionWithAssetsDir("../testdata/oonimkall/assets") +func NewSessionForTesting() (*oonimkall.Session, error) { + return NewSessionForTestingWithAssetsDir("../testdata/oonimkall/assets") } func TestNewSessionWithInvalidStateDir(t *testing.T) { @@ -72,7 +72,7 @@ func TestMaybeUpdateResourcesWithCancelledContext(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(dir) - sess, err := NewSessionWithAssetsDir(dir) + sess, err := NewSessionForTestingWithAssetsDir(dir) if err != nil { t.Fatal(err) } @@ -104,7 +104,7 @@ func TestGeolocateWithCancelledContext(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -123,7 +123,7 @@ func TestGeolocateGood(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -163,7 +163,7 @@ func TestSubmitWithCancelledContext(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -182,7 +182,7 @@ func TestSubmitWithInvalidJSON(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -234,7 +234,7 @@ func TestSubmitMeasurementGood(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -248,7 +248,7 @@ func TestSubmitCancelContextAfterFirstSubmission(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -267,7 +267,7 @@ func TestSubmitCancelContextAfterFirstSubmission(t *testing.T) { } func TestCheckInSuccess(t *testing.T) { - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -315,7 +315,7 @@ func TestCheckInSuccess(t *testing.T) { } func TestCheckInLookupLocationFailure(t *testing.T) { - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -342,7 +342,7 @@ func TestCheckInLookupLocationFailure(t *testing.T) { } func TestCheckInNewProbeServicesFailure(t *testing.T) { - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -371,7 +371,7 @@ func TestCheckInNewProbeServicesFailure(t *testing.T) { } func TestCheckInCheckInFailure(t *testing.T) { - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -400,7 +400,7 @@ func TestCheckInCheckInFailure(t *testing.T) { } func TestCheckInNoParams(t *testing.T) { - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -423,7 +423,7 @@ func TestCheckInNoParams(t *testing.T) { } func TestFetchURLListSuccess(t *testing.T) { - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } @@ -448,7 +448,7 @@ func TestFetchURLListSuccess(t *testing.T) { } func TestFetchURLListWithCC(t *testing.T) { - sess, err := NewSession() + sess, err := NewSessionForTesting() if err != nil { t.Fatal(err) } diff --git a/pkg/oonimkall/webconnectivity.go b/pkg/oonimkall/webconnectivity.go new file mode 100644 index 0000000..4a393e2 --- /dev/null +++ b/pkg/oonimkall/webconnectivity.go @@ -0,0 +1,91 @@ +package oonimkall + +import ( + "context" + "encoding/json" + + "github.com/ooni/probe-cli/v3/internal/engine/runtimex" +) + +// WebConnectivityConfig contains settings for WebConnectivity. +type WebConnectivityConfig struct { + // Callbacks contains the experiment callbacks. This field is + // optional. Leave it empty and we'll use a default set of + // callbacks that use the session logger. + Callbacks ExperimentCallbacks + + // Input contains the URL to measure. This field must be set + // by the user, otherwise the experiment fails. + Input string +} + +// WebConnectivityResults contains the results of WebConnectivity. +type WebConnectivityResults struct { + // KibiBytesReceived contains the KiB received. + KibiBytesReceived float64 + + // KibiBytesSent contains the KiB sent. + KibiBytesSent float64 + + // Measurement contains the resulting measurement. + Measurement string +} + +// webConnectivityRunner is the type that runs +// the WebConnectivity experiment. +type webConnectivityRunner struct { + sess experimentSession +} + +// run runs the WebConnectivity experiment to completion. Both arguments +// must be correctly initialized. The return value is either a valid +// results with a nil error, or nil results with an error. +func (r *webConnectivityRunner) run(ctx context.Context, config *WebConnectivityConfig) (*WebConnectivityResults, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() // helps with testing + default: + // fallthrough + } + // TODO(bassosimone): I suspect most of the code for running + // experiments is going to be quite redundant. Autogen? + defer r.sess.unlock() + r.sess.lock() + if err := r.sess.maybeLookupBackends(ctx); err != nil { + return nil, err + } + if err := r.sess.maybeLookupLocation(ctx); err != nil { + return nil, err + } + builder, err := r.sess.newExperimentBuilder("web_connectivity") + if err != nil { + return nil, err + } + if config.Callbacks != nil { + builder.setCallbacks(config.Callbacks) + } + exp := builder.newExperiment() + measurement, err := exp.MeasureWithContext(ctx, config.Input) + if err != nil { + return nil, err + } + data, err := json.Marshal(measurement) + runtimex.PanicOnError(err, "json.Marshal should not fail here") + return &WebConnectivityResults{ + KibiBytesReceived: exp.KibiBytesReceived(), + KibiBytesSent: exp.KibiBytesSent(), + Measurement: string(data), + }, nil +} + +// WebConnectivity runs the WebConnectivity experiment. Both ctx and config +// MUST NOT be nil. Returns either an error or the experiment results. +// +// This function locks the session until it's done. That is, no other operation +// can be performed as long as this function is pending. +// +// This API is currently experimental. We do not promise that we will bump +// the major version number when changing it. +func (sess *Session) WebConnectivity(ctx *Context, config *WebConnectivityConfig) (*WebConnectivityResults, error) { + return (&webConnectivityRunner{sess: sess}).run(ctx.ctx, config) +} diff --git a/pkg/oonimkall/webconnectivity_integration_test.go b/pkg/oonimkall/webconnectivity_integration_test.go new file mode 100644 index 0000000..b4b39f8 --- /dev/null +++ b/pkg/oonimkall/webconnectivity_integration_test.go @@ -0,0 +1,28 @@ +package oonimkall_test + +import ( + "testing" + + "github.com/ooni/probe-cli/v3/pkg/oonimkall" +) + +func TestSessionWebConnectivity(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + sess, err := NewSessionForTesting() + if err != nil { + t.Fatal(err) + } + ctx := sess.NewContext() + config := &oonimkall.WebConnectivityConfig{ + Input: "https://www.google.com", + } + results, err := sess.WebConnectivity(ctx, config) + if err != nil { + t.Fatal(err) + } + t.Logf("bytes received: %f", results.KibiBytesReceived) + t.Logf("bytes sent: %f", results.KibiBytesSent) + t.Logf("measurement: %d bytes", len(results.Measurement)) +} diff --git a/pkg/oonimkall/webconnectivity_test.go b/pkg/oonimkall/webconnectivity_test.go new file mode 100644 index 0000000..7326b13 --- /dev/null +++ b/pkg/oonimkall/webconnectivity_test.go @@ -0,0 +1,156 @@ +package oonimkall + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/engine/model" +) + +func TestWebConnectivityRunnerWithMaybeLookupBackendsFailure(t *testing.T) { + errMocked := errors.New("mocked error") + sess := &FakeExperimentSession{LookupBackendsErr: errMocked} + runner := &webConnectivityRunner{sess: sess} + ctx := context.Background() + config := &WebConnectivityConfig{Input: "https://ooni.org"} + out, err := runner.run(ctx, config) + if !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } + if out != nil { + t.Fatal("expected nil here") + } + if sess.LockCount != 1 || sess.UnlockCount != 1 { + t.Fatal("invalid locking pattern") + } +} + +func TestWebConnectivityRunnerWithMaybeLookupLocationFailure(t *testing.T) { + errMocked := errors.New("mocked error") + sess := &FakeExperimentSession{LookupLocationErr: errMocked} + runner := &webConnectivityRunner{sess: sess} + ctx := context.Background() + config := &WebConnectivityConfig{Input: "https://ooni.org"} + out, err := runner.run(ctx, config) + if !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } + if out != nil { + t.Fatal("expected nil here") + } + if sess.LockCount != 1 || sess.UnlockCount != 1 { + t.Fatal("invalid locking pattern") + } +} + +func TestWebConnectivityRunnerWithNewExperimentBuilderFailure(t *testing.T) { + errMocked := errors.New("mocked error") + sess := &FakeExperimentSession{NewExperimentBuilderErr: errMocked} + runner := &webConnectivityRunner{sess: sess} + ctx := context.Background() + config := &WebConnectivityConfig{Input: "https://ooni.org"} + out, err := runner.run(ctx, config) + if !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } + if out != nil { + t.Fatal("expected nil here") + } + if sess.LockCount != 1 || sess.UnlockCount != 1 { + t.Fatal("invalid locking pattern") + } +} + +func TestWebConnectivityRunnerWithMeasureFailure(t *testing.T) { + errMocked := errors.New("mocked error") + cbs := &FakeExperimentCallbacks{} + e := &FakeExperiment{Err: errMocked} + eb := &FakeExperimentBuilder{Experiment: e} + sess := &FakeExperimentSession{ExperimentBuilder: eb} + runner := &webConnectivityRunner{sess: sess} + ctx := context.Background() + config := &WebConnectivityConfig{ + Callbacks: cbs, + Input: "https://ooni.org", + } + out, err := runner.run(ctx, config) + if !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } + if out != nil { + t.Fatal("expected nil here") + } + if sess.LockCount != 1 || sess.UnlockCount != 1 { + t.Fatal("invalid locking pattern") + } + if eb.Callbacks != cbs { + t.Fatal("unexpected callbacks") + } +} + +func TestWebConnectivityRunnerWithNoError(t *testing.T) { + // We create a measurement with non default fields. One of them is + // enough to check that we are getting in output the non default + // data structure that was preconfigured in the mocks. + m := &model.Measurement{Input: "https://ooni.org"} + cbs := &FakeExperimentCallbacks{} + e := &FakeExperiment{Measurement: m, Sent: 10, Received: 128} + eb := &FakeExperimentBuilder{Experiment: e} + sess := &FakeExperimentSession{ExperimentBuilder: eb} + runner := &webConnectivityRunner{sess: sess} + ctx := context.Background() + config := &WebConnectivityConfig{ + Callbacks: cbs, + Input: "https://ooni.org", + } + out, err := runner.run(ctx, config) + if err != nil { + t.Fatal(err) + } + if out == nil { + t.Fatal("expected non-nil here") + } + if sess.LockCount != 1 || sess.UnlockCount != 1 { + t.Fatal("invalid locking pattern") + } + if eb.Callbacks != cbs { + t.Fatal("unexpected callbacks") + } + if out.KibiBytesSent != 10 || out.KibiBytesReceived != 128 { + t.Fatal("invalid bytes sent or received") + } + var mm *model.Measurement + mdata := []byte(out.Measurement) + if err := json.Unmarshal(mdata, &mm); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(m, mm); diff != "" { + t.Fatal(diff) + } +} + +func TestWebConnectivityRunWithCancelledContext(t *testing.T) { + sess, err := NewSession(&SessionConfig{ + AssetsDir: "../testdata/oonimkall/assets", + ProbeServicesURL: "https://ams-pg-test.ooni.org/", + SoftwareName: "oonimkall-test", + SoftwareVersion: "0.1.0", + StateDir: "../testdata/oonimkall/state", + TempDir: "../testdata/", + }) + if err != nil { + t.Fatal(err) + } + ctx := sess.NewContext() + ctx.Cancel() // kill it immediately + out, err := sess.WebConnectivity(ctx, &WebConnectivityConfig{}) + if !errors.Is(err, context.Canceled) { + t.Fatal("not the error we expected", err) + } + if out != nil { + t.Fatal("expected nil output here") + } +} From c22828d3690d5e50a89ee41687a5972ac989cd90 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 18 Mar 2021 08:47:07 +0100 Subject: [PATCH 02/28] chore: bless 3.9.0-alpha (#257) --- internal/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version/version.go b/internal/version/version.go index 0b5eeb3..848e907 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -3,5 +3,5 @@ package version const ( // Version is the software version - Version = "3.8.0" + Version = "3.9.0-alpha" ) From 28ce79eff1ff61d3a5407e8f9c0e51e639e69e78 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 19 Mar 2021 09:30:42 +0100 Subject: [PATCH 03/28] feat(ooapi): add toplevel client and simplify API (#248) * feat(ooapi): add toplevel client and simplify API This diff should simplify using ooapi from other packages by adding more abstraction that wraps the existing code. Part of https://github.com/ooni/probe/issues/1355. * fix(ooapi): use correct comment for cloners See https://github.com/ooni/probe-cli/pull/248#discussion_r590663843 * fix(ooapi): make sure the documentation is current See https://github.com/ooni/probe-cli/pull/248#discussion_r590665773 * fix(ooapi): automate copying APIs See https://github.com/ooni/probe-cli/pull/248#discussion_r590665837 * feat(ooapi): add unit tests for clientcall.go See https://github.com/ooni/probe-cli/pull/248#discussion_r590666297 * fix(ooapi): rewrite integration tests to use toplevel API See https://github.com/ooni/probe-cli/pull/248#discussion_r590665084 --- internal/engine/netx/gocertifi/certifi.go | 4 +- internal/engine/ooapi/apis.go | 168 ++-- internal/engine/ooapi/apis_test.go | 204 ++-- internal/engine/ooapi/caching.go | 32 +- internal/engine/ooapi/caching_test.go | 50 +- internal/engine/ooapi/callers.go | 68 +- internal/engine/ooapi/client.go | 13 + internal/engine/ooapi/clientcall.go | 214 +++++ internal/engine/ooapi/clientcall_test.go | 898 ++++++++++++++++++ internal/engine/ooapi/cloners.go | 18 +- internal/engine/ooapi/dependencies.go | 4 +- internal/engine/ooapi/doc.go | 133 +-- internal/engine/ooapi/errors.go | 16 +- internal/engine/ooapi/fakeapi_test.go | 36 +- internal/engine/ooapi/httpclient_test.go | 21 + internal/engine/ooapi/integration_test.go | 77 +- .../engine/ooapi/internal/generator/apis.go | 4 +- .../ooapi/internal/generator/caching.go | 18 +- .../ooapi/internal/generator/cachingtest.go | 36 +- .../ooapi/internal/generator/clientcall.go | 104 ++ .../internal/generator/clientcalltest.go | 181 ++++ .../ooapi/internal/generator/cloners.go | 2 +- .../ooapi/internal/generator/fakeapitest.go | 18 +- .../ooapi/internal/generator/generator.go | 4 + .../engine/ooapi/internal/generator/login.go | 4 +- .../ooapi/internal/generator/logintest.go | 130 +-- .../ooapi/internal/generator/reflect.go | 22 +- internal/engine/ooapi/kvstore_test.go | 6 +- internal/engine/ooapi/login.go | 70 +- internal/engine/ooapi/login_test.go | 278 +++--- internal/engine/ooapi/requests.go | 24 +- internal/engine/ooapi/responses.go | 24 +- internal/engine/ooapi/swagger_test.go | 4 +- 33 files changed, 2090 insertions(+), 795 deletions(-) create mode 100644 internal/engine/ooapi/client.go create mode 100644 internal/engine/ooapi/clientcall.go create mode 100644 internal/engine/ooapi/clientcall_test.go create mode 100644 internal/engine/ooapi/httpclient_test.go create mode 100644 internal/engine/ooapi/internal/generator/clientcall.go create mode 100644 internal/engine/ooapi/internal/generator/clientcalltest.go diff --git a/internal/engine/netx/gocertifi/certifi.go b/internal/engine/netx/gocertifi/certifi.go index f04d006..7968d85 100644 --- a/internal/engine/netx/gocertifi/certifi.go +++ b/internal/engine/netx/gocertifi/certifi.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-03 11:48:43.129132377 +0100 CET m=+2.301468593 +// 2021-03-10 12:20:35.042573349 +0100 CET m=+0.416591177 // https://curl.haxx.se/ca/cacert.pem package gocertifi @@ -3241,7 +3241,7 @@ kpzNNIaRkPpkUZ3+/uul9XXeifdy ` // CACerts builds an X.509 certificate pool containing the -// certificate bundle from https://curl.haxx.se/ca/cacert.pem fetch on 2021-03-03 11:48:43.129132377 +0100 CET m=+2.301468593. +// certificate bundle from https://curl.haxx.se/ca/cacert.pem fetch on 2021-03-10 12:20:35.042573349 +0100 CET m=+0.416591177. // Returns nil on error along with an appropriate error code. func CACerts() (*x509.CertPool, error) { pool := x509.NewCertPool() diff --git a/internal/engine/ooapi/apis.go b/internal/engine/ooapi/apis.go index 3c079be..0bf2126 100644 --- a/internal/engine/ooapi/apis.go +++ b/internal/engine/ooapi/apis.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:50.431349269 +0100 CET m=+0.000196051 +// 2021-03-10 13:17:31.904167818 +0100 CET m=+0.000077877 package ooapi @@ -12,8 +12,8 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -// CheckReportIDAPI implements the CheckReportID API. -type CheckReportIDAPI struct { +// simpleCheckReportIDAPI implements the CheckReportID API. +type simpleCheckReportIDAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -21,28 +21,28 @@ type CheckReportIDAPI struct { UserAgent string // optional } -func (api *CheckReportIDAPI) baseURL() string { +func (api *simpleCheckReportIDAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *CheckReportIDAPI) requestMaker() RequestMaker { +func (api *simpleCheckReportIDAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *CheckReportIDAPI) jsonCodec() JSONCodec { +func (api *simpleCheckReportIDAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *CheckReportIDAPI) httpClient() HTTPClient { +func (api *simpleCheckReportIDAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -50,7 +50,7 @@ func (api *CheckReportIDAPI) httpClient() HTTPClient { } // Call calls the CheckReportID API. -func (api *CheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error) { +func (api *simpleCheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -62,8 +62,8 @@ func (api *CheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckReport return api.newResponse(api.httpClient().Do(httpReq)) } -// CheckInAPI implements the CheckIn API. -type CheckInAPI struct { +// simpleCheckInAPI implements the CheckIn API. +type simpleCheckInAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -71,28 +71,28 @@ type CheckInAPI struct { UserAgent string // optional } -func (api *CheckInAPI) baseURL() string { +func (api *simpleCheckInAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *CheckInAPI) requestMaker() RequestMaker { +func (api *simpleCheckInAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *CheckInAPI) jsonCodec() JSONCodec { +func (api *simpleCheckInAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *CheckInAPI) httpClient() HTTPClient { +func (api *simpleCheckInAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -100,7 +100,7 @@ func (api *CheckInAPI) httpClient() HTTPClient { } // Call calls the CheckIn API. -func (api *CheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error) { +func (api *simpleCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -112,8 +112,8 @@ func (api *CheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequest) ( return api.newResponse(api.httpClient().Do(httpReq)) } -// LoginAPI implements the Login API. -type LoginAPI struct { +// simpleLoginAPI implements the Login API. +type simpleLoginAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -121,28 +121,28 @@ type LoginAPI struct { UserAgent string // optional } -func (api *LoginAPI) baseURL() string { +func (api *simpleLoginAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *LoginAPI) requestMaker() RequestMaker { +func (api *simpleLoginAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *LoginAPI) jsonCodec() JSONCodec { +func (api *simpleLoginAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *LoginAPI) httpClient() HTTPClient { +func (api *simpleLoginAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -150,7 +150,7 @@ func (api *LoginAPI) httpClient() HTTPClient { } // Call calls the Login API. -func (api *LoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error) { +func (api *simpleLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -162,8 +162,8 @@ func (api *LoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) (*api return api.newResponse(api.httpClient().Do(httpReq)) } -// MeasurementMetaAPI implements the MeasurementMeta API. -type MeasurementMetaAPI struct { +// simpleMeasurementMetaAPI implements the MeasurementMeta API. +type simpleMeasurementMetaAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -171,28 +171,28 @@ type MeasurementMetaAPI struct { UserAgent string // optional } -func (api *MeasurementMetaAPI) baseURL() string { +func (api *simpleMeasurementMetaAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *MeasurementMetaAPI) requestMaker() RequestMaker { +func (api *simpleMeasurementMetaAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *MeasurementMetaAPI) jsonCodec() JSONCodec { +func (api *simpleMeasurementMetaAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *MeasurementMetaAPI) httpClient() HTTPClient { +func (api *simpleMeasurementMetaAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -200,7 +200,7 @@ func (api *MeasurementMetaAPI) httpClient() HTTPClient { } // Call calls the MeasurementMeta API. -func (api *MeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) { +func (api *simpleMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -212,8 +212,8 @@ func (api *MeasurementMetaAPI) Call(ctx context.Context, req *apimodel.Measureme return api.newResponse(api.httpClient().Do(httpReq)) } -// RegisterAPI implements the Register API. -type RegisterAPI struct { +// simpleRegisterAPI implements the Register API. +type simpleRegisterAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -221,28 +221,28 @@ type RegisterAPI struct { UserAgent string // optional } -func (api *RegisterAPI) baseURL() string { +func (api *simpleRegisterAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *RegisterAPI) requestMaker() RequestMaker { +func (api *simpleRegisterAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *RegisterAPI) jsonCodec() JSONCodec { +func (api *simpleRegisterAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *RegisterAPI) httpClient() HTTPClient { +func (api *simpleRegisterAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -250,7 +250,7 @@ func (api *RegisterAPI) httpClient() HTTPClient { } // Call calls the Register API. -func (api *RegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error) { +func (api *simpleRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -262,8 +262,8 @@ func (api *RegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRequest) return api.newResponse(api.httpClient().Do(httpReq)) } -// TestHelpersAPI implements the TestHelpers API. -type TestHelpersAPI struct { +// simpleTestHelpersAPI implements the TestHelpers API. +type simpleTestHelpersAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -271,28 +271,28 @@ type TestHelpersAPI struct { UserAgent string // optional } -func (api *TestHelpersAPI) baseURL() string { +func (api *simpleTestHelpersAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *TestHelpersAPI) requestMaker() RequestMaker { +func (api *simpleTestHelpersAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *TestHelpersAPI) jsonCodec() JSONCodec { +func (api *simpleTestHelpersAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *TestHelpersAPI) httpClient() HTTPClient { +func (api *simpleTestHelpersAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -300,7 +300,7 @@ func (api *TestHelpersAPI) httpClient() HTTPClient { } // Call calls the TestHelpers API. -func (api *TestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error) { +func (api *simpleTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -312,8 +312,8 @@ func (api *TestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelpersRe return api.newResponse(api.httpClient().Do(httpReq)) } -// PsiphonConfigAPI implements the PsiphonConfig API. -type PsiphonConfigAPI struct { +// simplePsiphonConfigAPI implements the PsiphonConfig API. +type simplePsiphonConfigAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -324,8 +324,8 @@ type PsiphonConfigAPI struct { // WithToken returns a copy of the API where the // value of the Token field is replaced with token. -func (api *PsiphonConfigAPI) WithToken(token string) PsiphonConfigCaller { - out := &PsiphonConfigAPI{} +func (api *simplePsiphonConfigAPI) WithToken(token string) callerForPsiphonConfigAPI { + out := &simplePsiphonConfigAPI{} out.BaseURL = api.BaseURL out.HTTPClient = api.HTTPClient out.JSONCodec = api.JSONCodec @@ -335,28 +335,28 @@ func (api *PsiphonConfigAPI) WithToken(token string) PsiphonConfigCaller { return out } -func (api *PsiphonConfigAPI) baseURL() string { +func (api *simplePsiphonConfigAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *PsiphonConfigAPI) requestMaker() RequestMaker { +func (api *simplePsiphonConfigAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *PsiphonConfigAPI) jsonCodec() JSONCodec { +func (api *simplePsiphonConfigAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *PsiphonConfigAPI) httpClient() HTTPClient { +func (api *simplePsiphonConfigAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -364,7 +364,7 @@ func (api *PsiphonConfigAPI) httpClient() HTTPClient { } // Call calls the PsiphonConfig API. -func (api *PsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) { +func (api *simplePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -380,8 +380,8 @@ func (api *PsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConf return api.newResponse(api.httpClient().Do(httpReq)) } -// TorTargetsAPI implements the TorTargets API. -type TorTargetsAPI struct { +// simpleTorTargetsAPI implements the TorTargets API. +type simpleTorTargetsAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -392,8 +392,8 @@ type TorTargetsAPI struct { // WithToken returns a copy of the API where the // value of the Token field is replaced with token. -func (api *TorTargetsAPI) WithToken(token string) TorTargetsCaller { - out := &TorTargetsAPI{} +func (api *simpleTorTargetsAPI) WithToken(token string) callerForTorTargetsAPI { + out := &simpleTorTargetsAPI{} out.BaseURL = api.BaseURL out.HTTPClient = api.HTTPClient out.JSONCodec = api.JSONCodec @@ -403,28 +403,28 @@ func (api *TorTargetsAPI) WithToken(token string) TorTargetsCaller { return out } -func (api *TorTargetsAPI) baseURL() string { +func (api *simpleTorTargetsAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *TorTargetsAPI) requestMaker() RequestMaker { +func (api *simpleTorTargetsAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *TorTargetsAPI) jsonCodec() JSONCodec { +func (api *simpleTorTargetsAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *TorTargetsAPI) httpClient() HTTPClient { +func (api *simpleTorTargetsAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -432,7 +432,7 @@ func (api *TorTargetsAPI) httpClient() HTTPClient { } // Call calls the TorTargets API. -func (api *TorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) { +func (api *simpleTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -448,8 +448,8 @@ func (api *TorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequ return api.newResponse(api.httpClient().Do(httpReq)) } -// URLsAPI implements the URLs API. -type URLsAPI struct { +// simpleURLsAPI implements the URLs API. +type simpleURLsAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -457,28 +457,28 @@ type URLsAPI struct { UserAgent string // optional } -func (api *URLsAPI) baseURL() string { +func (api *simpleURLsAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *URLsAPI) requestMaker() RequestMaker { +func (api *simpleURLsAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *URLsAPI) jsonCodec() JSONCodec { +func (api *simpleURLsAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *URLsAPI) httpClient() HTTPClient { +func (api *simpleURLsAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -486,7 +486,7 @@ func (api *URLsAPI) httpClient() HTTPClient { } // Call calls the URLs API. -func (api *URLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error) { +func (api *simpleURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -498,8 +498,8 @@ func (api *URLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*apimo return api.newResponse(api.httpClient().Do(httpReq)) } -// OpenReportAPI implements the OpenReport API. -type OpenReportAPI struct { +// simpleOpenReportAPI implements the OpenReport API. +type simpleOpenReportAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional @@ -507,28 +507,28 @@ type OpenReportAPI struct { UserAgent string // optional } -func (api *OpenReportAPI) baseURL() string { +func (api *simpleOpenReportAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *OpenReportAPI) requestMaker() RequestMaker { +func (api *simpleOpenReportAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *OpenReportAPI) jsonCodec() JSONCodec { +func (api *simpleOpenReportAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *OpenReportAPI) httpClient() HTTPClient { +func (api *simpleOpenReportAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -536,7 +536,7 @@ func (api *OpenReportAPI) httpClient() HTTPClient { } // Call calls the OpenReport API. -func (api *OpenReportAPI) Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error) { +func (api *simpleOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err @@ -548,45 +548,45 @@ func (api *OpenReportAPI) Call(ctx context.Context, req *apimodel.OpenReportRequ return api.newResponse(api.httpClient().Do(httpReq)) } -// SubmitMeasurementAPI implements the SubmitMeasurement API. -type SubmitMeasurementAPI struct { +// simpleSubmitMeasurementAPI implements the SubmitMeasurement API. +type simpleSubmitMeasurementAPI struct { BaseURL string // optional HTTPClient HTTPClient // optional JSONCodec JSONCodec // optional RequestMaker RequestMaker // optional - TemplateExecutor TemplateExecutor // optional + TemplateExecutor templateExecutor // optional UserAgent string // optional } -func (api *SubmitMeasurementAPI) baseURL() string { +func (api *simpleSubmitMeasurementAPI) baseURL() string { if api.BaseURL != "" { return api.BaseURL } return "https://ps1.ooni.io" } -func (api *SubmitMeasurementAPI) requestMaker() RequestMaker { +func (api *simpleSubmitMeasurementAPI) requestMaker() RequestMaker { if api.RequestMaker != nil { return api.RequestMaker } return &defaultRequestMaker{} } -func (api *SubmitMeasurementAPI) jsonCodec() JSONCodec { +func (api *simpleSubmitMeasurementAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *SubmitMeasurementAPI) templateExecutor() TemplateExecutor { +func (api *simpleSubmitMeasurementAPI) templateExecutor() templateExecutor { if api.TemplateExecutor != nil { return api.TemplateExecutor } return &defaultTemplateExecutor{} } -func (api *SubmitMeasurementAPI) httpClient() HTTPClient { +func (api *simpleSubmitMeasurementAPI) httpClient() HTTPClient { if api.HTTPClient != nil { return api.HTTPClient } @@ -594,7 +594,7 @@ func (api *SubmitMeasurementAPI) httpClient() HTTPClient { } // Call calls the SubmitMeasurement API. -func (api *SubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error) { +func (api *simpleSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error) { httpReq, err := api.newRequest(ctx, req) if err != nil { return nil, err diff --git a/internal/engine/ooapi/apis_test.go b/internal/engine/ooapi/apis_test.go index ace748d..f613123 100644 --- a/internal/engine/ooapi/apis_test.go +++ b/internal/engine/ooapi/apis_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:50.81792142 +0100 CET m=+0.000095792 +// 2021-03-10 13:17:32.206416809 +0100 CET m=+0.000074280 package ooapi @@ -22,7 +22,7 @@ import ( ) func TestCheckReportIDInvalidURL(t *testing.T) { - api := &CheckReportIDAPI{ + api := &simpleCheckReportIDAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -41,7 +41,7 @@ func TestCheckReportIDInvalidURL(t *testing.T) { func TestCheckReportIDWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &CheckReportIDAPI{ + api := &simpleCheckReportIDAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -59,7 +59,7 @@ func TestCheckReportIDWithHTTPErr(t *testing.T) { func TestCheckReportIDWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &CheckReportIDAPI{ + api := &simpleCheckReportIDAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -77,7 +77,7 @@ func TestCheckReportIDWithNewRequestErr(t *testing.T) { func TestCheckReportIDWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &CheckReportIDAPI{ + api := &simpleCheckReportIDAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -95,7 +95,7 @@ func TestCheckReportIDWith401(t *testing.T) { func TestCheckReportIDWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &CheckReportIDAPI{ + api := &simpleCheckReportIDAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -117,7 +117,7 @@ func TestCheckReportIDWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &CheckReportIDAPI{ + api := &simpleCheckReportIDAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -139,7 +139,7 @@ func TestCheckReportIDWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &CheckReportIDAPI{ + api := &simpleCheckReportIDAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -209,7 +209,7 @@ func TestCheckReportIDRoundTrip(t *testing.T) { req := &apimodel.CheckReportIDRequest{} ff := &fakeFill{} ff.fill(&req) - api := &CheckReportIDAPI{BaseURL: srvr.URL} + api := &simpleCheckReportIDAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -252,7 +252,7 @@ func TestCheckReportIDMandatoryFields(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{ StatusCode: 500, }} - api := &CheckReportIDAPI{ + api := &simpleCheckReportIDAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -267,7 +267,7 @@ func TestCheckReportIDMandatoryFields(t *testing.T) { } func TestCheckInInvalidURL(t *testing.T) { - api := &CheckInAPI{ + api := &simpleCheckInAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -286,7 +286,7 @@ func TestCheckInInvalidURL(t *testing.T) { func TestCheckInWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &CheckInAPI{ + api := &simpleCheckInAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -304,7 +304,7 @@ func TestCheckInWithHTTPErr(t *testing.T) { func TestCheckInMarshalErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &CheckInAPI{ + api := &simpleCheckInAPI{ JSONCodec: &FakeCodec{EncodeErr: errMocked}, } ctx := context.Background() @@ -322,7 +322,7 @@ func TestCheckInMarshalErr(t *testing.T) { func TestCheckInWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &CheckInAPI{ + api := &simpleCheckInAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -340,7 +340,7 @@ func TestCheckInWithNewRequestErr(t *testing.T) { func TestCheckInWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &CheckInAPI{ + api := &simpleCheckInAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -358,7 +358,7 @@ func TestCheckInWith401(t *testing.T) { func TestCheckInWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &CheckInAPI{ + api := &simpleCheckInAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -380,7 +380,7 @@ func TestCheckInWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &CheckInAPI{ + api := &simpleCheckInAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -402,7 +402,7 @@ func TestCheckInWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &CheckInAPI{ + api := &simpleCheckInAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -472,7 +472,7 @@ func TestCheckInRoundTrip(t *testing.T) { req := &apimodel.CheckInRequest{} ff := &fakeFill{} ff.fill(&req) - api := &CheckInAPI{BaseURL: srvr.URL} + api := &simpleCheckInAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -512,7 +512,7 @@ func TestCheckInRoundTrip(t *testing.T) { } func TestLoginInvalidURL(t *testing.T) { - api := &LoginAPI{ + api := &simpleLoginAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -531,7 +531,7 @@ func TestLoginInvalidURL(t *testing.T) { func TestLoginWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &LoginAPI{ + api := &simpleLoginAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -549,7 +549,7 @@ func TestLoginWithHTTPErr(t *testing.T) { func TestLoginMarshalErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &LoginAPI{ + api := &simpleLoginAPI{ JSONCodec: &FakeCodec{EncodeErr: errMocked}, } ctx := context.Background() @@ -567,7 +567,7 @@ func TestLoginMarshalErr(t *testing.T) { func TestLoginWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &LoginAPI{ + api := &simpleLoginAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -585,7 +585,7 @@ func TestLoginWithNewRequestErr(t *testing.T) { func TestLoginWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &LoginAPI{ + api := &simpleLoginAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -603,7 +603,7 @@ func TestLoginWith401(t *testing.T) { func TestLoginWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &LoginAPI{ + api := &simpleLoginAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -625,7 +625,7 @@ func TestLoginWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &LoginAPI{ + api := &simpleLoginAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -647,7 +647,7 @@ func TestLoginWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &LoginAPI{ + api := &simpleLoginAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -717,7 +717,7 @@ func TestLoginRoundTrip(t *testing.T) { req := &apimodel.LoginRequest{} ff := &fakeFill{} ff.fill(&req) - api := &LoginAPI{BaseURL: srvr.URL} + api := &simpleLoginAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -757,7 +757,7 @@ func TestLoginRoundTrip(t *testing.T) { } func TestMeasurementMetaInvalidURL(t *testing.T) { - api := &MeasurementMetaAPI{ + api := &simpleMeasurementMetaAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -776,7 +776,7 @@ func TestMeasurementMetaInvalidURL(t *testing.T) { func TestMeasurementMetaWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &MeasurementMetaAPI{ + api := &simpleMeasurementMetaAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -794,7 +794,7 @@ func TestMeasurementMetaWithHTTPErr(t *testing.T) { func TestMeasurementMetaWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &MeasurementMetaAPI{ + api := &simpleMeasurementMetaAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -812,7 +812,7 @@ func TestMeasurementMetaWithNewRequestErr(t *testing.T) { func TestMeasurementMetaWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &MeasurementMetaAPI{ + api := &simpleMeasurementMetaAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -830,7 +830,7 @@ func TestMeasurementMetaWith401(t *testing.T) { func TestMeasurementMetaWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &MeasurementMetaAPI{ + api := &simpleMeasurementMetaAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -852,7 +852,7 @@ func TestMeasurementMetaWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &MeasurementMetaAPI{ + api := &simpleMeasurementMetaAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -874,7 +874,7 @@ func TestMeasurementMetaWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &MeasurementMetaAPI{ + api := &simpleMeasurementMetaAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -944,7 +944,7 @@ func TestMeasurementMetaRoundTrip(t *testing.T) { req := &apimodel.MeasurementMetaRequest{} ff := &fakeFill{} ff.fill(&req) - api := &MeasurementMetaAPI{BaseURL: srvr.URL} + api := &simpleMeasurementMetaAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -987,7 +987,7 @@ func TestMeasurementMetaMandatoryFields(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{ StatusCode: 500, }} - api := &MeasurementMetaAPI{ + api := &simpleMeasurementMetaAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1002,7 +1002,7 @@ func TestMeasurementMetaMandatoryFields(t *testing.T) { } func TestRegisterInvalidURL(t *testing.T) { - api := &RegisterAPI{ + api := &simpleRegisterAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -1021,7 +1021,7 @@ func TestRegisterInvalidURL(t *testing.T) { func TestRegisterWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &RegisterAPI{ + api := &simpleRegisterAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1039,7 +1039,7 @@ func TestRegisterWithHTTPErr(t *testing.T) { func TestRegisterMarshalErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &RegisterAPI{ + api := &simpleRegisterAPI{ JSONCodec: &FakeCodec{EncodeErr: errMocked}, } ctx := context.Background() @@ -1057,7 +1057,7 @@ func TestRegisterMarshalErr(t *testing.T) { func TestRegisterWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &RegisterAPI{ + api := &simpleRegisterAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -1075,7 +1075,7 @@ func TestRegisterWithNewRequestErr(t *testing.T) { func TestRegisterWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &RegisterAPI{ + api := &simpleRegisterAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1093,7 +1093,7 @@ func TestRegisterWith401(t *testing.T) { func TestRegisterWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &RegisterAPI{ + api := &simpleRegisterAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1115,7 +1115,7 @@ func TestRegisterWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &RegisterAPI{ + api := &simpleRegisterAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1137,7 +1137,7 @@ func TestRegisterWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &RegisterAPI{ + api := &simpleRegisterAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -1207,7 +1207,7 @@ func TestRegisterRoundTrip(t *testing.T) { req := &apimodel.RegisterRequest{} ff := &fakeFill{} ff.fill(&req) - api := &RegisterAPI{BaseURL: srvr.URL} + api := &simpleRegisterAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -1247,7 +1247,7 @@ func TestRegisterRoundTrip(t *testing.T) { } func TestTestHelpersInvalidURL(t *testing.T) { - api := &TestHelpersAPI{ + api := &simpleTestHelpersAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -1266,7 +1266,7 @@ func TestTestHelpersInvalidURL(t *testing.T) { func TestTestHelpersWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &TestHelpersAPI{ + api := &simpleTestHelpersAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1284,7 +1284,7 @@ func TestTestHelpersWithHTTPErr(t *testing.T) { func TestTestHelpersWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &TestHelpersAPI{ + api := &simpleTestHelpersAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -1302,7 +1302,7 @@ func TestTestHelpersWithNewRequestErr(t *testing.T) { func TestTestHelpersWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &TestHelpersAPI{ + api := &simpleTestHelpersAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1320,7 +1320,7 @@ func TestTestHelpersWith401(t *testing.T) { func TestTestHelpersWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &TestHelpersAPI{ + api := &simpleTestHelpersAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1342,7 +1342,7 @@ func TestTestHelpersWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &TestHelpersAPI{ + api := &simpleTestHelpersAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1364,7 +1364,7 @@ func TestTestHelpersWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &TestHelpersAPI{ + api := &simpleTestHelpersAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -1434,7 +1434,7 @@ func TestTestHelpersRoundTrip(t *testing.T) { req := &apimodel.TestHelpersRequest{} ff := &fakeFill{} ff.fill(&req) - api := &TestHelpersAPI{BaseURL: srvr.URL} + api := &simpleTestHelpersAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -1478,7 +1478,7 @@ func TestTestHelpersResponseLiteralNull(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`null`)}, }} - api := &TestHelpersAPI{ + api := &simpleTestHelpersAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -1495,7 +1495,7 @@ func TestTestHelpersResponseLiteralNull(t *testing.T) { } func TestPsiphonConfigInvalidURL(t *testing.T) { - api := &PsiphonConfigAPI{ + api := &simplePsiphonConfigAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -1512,7 +1512,7 @@ func TestPsiphonConfigInvalidURL(t *testing.T) { } func TestPsiphonConfigWithMissingToken(t *testing.T) { - api := &PsiphonConfigAPI{} // no token + api := &simplePsiphonConfigAPI{} // no token ctx := context.Background() req := &apimodel.PsiphonConfigRequest{} ff := &fakeFill{} @@ -1529,7 +1529,7 @@ func TestPsiphonConfigWithMissingToken(t *testing.T) { func TestPsiphonConfigWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &PsiphonConfigAPI{ + api := &simplePsiphonConfigAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1548,7 +1548,7 @@ func TestPsiphonConfigWithHTTPErr(t *testing.T) { func TestPsiphonConfigWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &PsiphonConfigAPI{ + api := &simplePsiphonConfigAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, Token: "fakeToken", } @@ -1567,7 +1567,7 @@ func TestPsiphonConfigWithNewRequestErr(t *testing.T) { func TestPsiphonConfigWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &PsiphonConfigAPI{ + api := &simplePsiphonConfigAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1586,7 +1586,7 @@ func TestPsiphonConfigWith401(t *testing.T) { func TestPsiphonConfigWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &PsiphonConfigAPI{ + api := &simplePsiphonConfigAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1609,7 +1609,7 @@ func TestPsiphonConfigWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &PsiphonConfigAPI{ + api := &simplePsiphonConfigAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1632,7 +1632,7 @@ func TestPsiphonConfigWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &PsiphonConfigAPI{ + api := &simplePsiphonConfigAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, Token: "fakeToken", @@ -1703,7 +1703,7 @@ func TestPsiphonConfigRoundTrip(t *testing.T) { req := &apimodel.PsiphonConfigRequest{} ff := &fakeFill{} ff.fill(&req) - api := &PsiphonConfigAPI{BaseURL: srvr.URL} + api := &simplePsiphonConfigAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) ff.fill(&api.Token) // issue request @@ -1748,7 +1748,7 @@ func TestPsiphonConfigResponseLiteralNull(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`null`)}, }} - api := &PsiphonConfigAPI{ + api := &simplePsiphonConfigAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1766,7 +1766,7 @@ func TestPsiphonConfigResponseLiteralNull(t *testing.T) { } func TestTorTargetsInvalidURL(t *testing.T) { - api := &TorTargetsAPI{ + api := &simpleTorTargetsAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -1783,7 +1783,7 @@ func TestTorTargetsInvalidURL(t *testing.T) { } func TestTorTargetsWithMissingToken(t *testing.T) { - api := &TorTargetsAPI{} // no token + api := &simpleTorTargetsAPI{} // no token ctx := context.Background() req := &apimodel.TorTargetsRequest{} ff := &fakeFill{} @@ -1800,7 +1800,7 @@ func TestTorTargetsWithMissingToken(t *testing.T) { func TestTorTargetsWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &TorTargetsAPI{ + api := &simpleTorTargetsAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1819,7 +1819,7 @@ func TestTorTargetsWithHTTPErr(t *testing.T) { func TestTorTargetsWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &TorTargetsAPI{ + api := &simpleTorTargetsAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, Token: "fakeToken", } @@ -1838,7 +1838,7 @@ func TestTorTargetsWithNewRequestErr(t *testing.T) { func TestTorTargetsWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &TorTargetsAPI{ + api := &simpleTorTargetsAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1857,7 +1857,7 @@ func TestTorTargetsWith401(t *testing.T) { func TestTorTargetsWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &TorTargetsAPI{ + api := &simpleTorTargetsAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1880,7 +1880,7 @@ func TestTorTargetsWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &TorTargetsAPI{ + api := &simpleTorTargetsAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -1903,7 +1903,7 @@ func TestTorTargetsWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &TorTargetsAPI{ + api := &simpleTorTargetsAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, Token: "fakeToken", @@ -1974,7 +1974,7 @@ func TestTorTargetsRoundTrip(t *testing.T) { req := &apimodel.TorTargetsRequest{} ff := &fakeFill{} ff.fill(&req) - api := &TorTargetsAPI{BaseURL: srvr.URL} + api := &simpleTorTargetsAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) ff.fill(&api.Token) // issue request @@ -2019,7 +2019,7 @@ func TestTorTargetsResponseLiteralNull(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`null`)}, }} - api := &TorTargetsAPI{ + api := &simpleTorTargetsAPI{ HTTPClient: clnt, Token: "fakeToken", } @@ -2037,7 +2037,7 @@ func TestTorTargetsResponseLiteralNull(t *testing.T) { } func TestURLsInvalidURL(t *testing.T) { - api := &URLsAPI{ + api := &simpleURLsAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -2056,7 +2056,7 @@ func TestURLsInvalidURL(t *testing.T) { func TestURLsWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &URLsAPI{ + api := &simpleURLsAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2074,7 +2074,7 @@ func TestURLsWithHTTPErr(t *testing.T) { func TestURLsWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &URLsAPI{ + api := &simpleURLsAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -2092,7 +2092,7 @@ func TestURLsWithNewRequestErr(t *testing.T) { func TestURLsWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &URLsAPI{ + api := &simpleURLsAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2110,7 +2110,7 @@ func TestURLsWith401(t *testing.T) { func TestURLsWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &URLsAPI{ + api := &simpleURLsAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2132,7 +2132,7 @@ func TestURLsWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &URLsAPI{ + api := &simpleURLsAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2154,7 +2154,7 @@ func TestURLsWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &URLsAPI{ + api := &simpleURLsAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -2224,7 +2224,7 @@ func TestURLsRoundTrip(t *testing.T) { req := &apimodel.URLsRequest{} ff := &fakeFill{} ff.fill(&req) - api := &URLsAPI{BaseURL: srvr.URL} + api := &simpleURLsAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -2264,7 +2264,7 @@ func TestURLsRoundTrip(t *testing.T) { } func TestOpenReportInvalidURL(t *testing.T) { - api := &OpenReportAPI{ + api := &simpleOpenReportAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -2283,7 +2283,7 @@ func TestOpenReportInvalidURL(t *testing.T) { func TestOpenReportWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &OpenReportAPI{ + api := &simpleOpenReportAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2301,7 +2301,7 @@ func TestOpenReportWithHTTPErr(t *testing.T) { func TestOpenReportMarshalErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &OpenReportAPI{ + api := &simpleOpenReportAPI{ JSONCodec: &FakeCodec{EncodeErr: errMocked}, } ctx := context.Background() @@ -2319,7 +2319,7 @@ func TestOpenReportMarshalErr(t *testing.T) { func TestOpenReportWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &OpenReportAPI{ + api := &simpleOpenReportAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -2337,7 +2337,7 @@ func TestOpenReportWithNewRequestErr(t *testing.T) { func TestOpenReportWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &OpenReportAPI{ + api := &simpleOpenReportAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2355,7 +2355,7 @@ func TestOpenReportWith401(t *testing.T) { func TestOpenReportWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &OpenReportAPI{ + api := &simpleOpenReportAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2377,7 +2377,7 @@ func TestOpenReportWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &OpenReportAPI{ + api := &simpleOpenReportAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2399,7 +2399,7 @@ func TestOpenReportWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &OpenReportAPI{ + api := &simpleOpenReportAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -2469,7 +2469,7 @@ func TestOpenReportRoundTrip(t *testing.T) { req := &apimodel.OpenReportRequest{} ff := &fakeFill{} ff.fill(&req) - api := &OpenReportAPI{BaseURL: srvr.URL} + api := &simpleOpenReportAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -2509,7 +2509,7 @@ func TestOpenReportRoundTrip(t *testing.T) { } func TestSubmitMeasurementInvalidURL(t *testing.T) { - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ BaseURL: "\t", // invalid } ctx := context.Background() @@ -2528,7 +2528,7 @@ func TestSubmitMeasurementInvalidURL(t *testing.T) { func TestSubmitMeasurementWithHTTPErr(t *testing.T) { errMocked := errors.New("mocked error") clnt := &FakeHTTPClient{Err: errMocked} - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2546,7 +2546,7 @@ func TestSubmitMeasurementWithHTTPErr(t *testing.T) { func TestSubmitMeasurementMarshalErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ JSONCodec: &FakeCodec{EncodeErr: errMocked}, } ctx := context.Background() @@ -2564,7 +2564,7 @@ func TestSubmitMeasurementMarshalErr(t *testing.T) { func TestSubmitMeasurementWithNewRequestErr(t *testing.T) { errMocked := errors.New("mocked error") - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ RequestMaker: &FakeRequestMaker{Err: errMocked}, } ctx := context.Background() @@ -2582,7 +2582,7 @@ func TestSubmitMeasurementWithNewRequestErr(t *testing.T) { func TestSubmitMeasurementWith401(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}} - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2600,7 +2600,7 @@ func TestSubmitMeasurementWith401(t *testing.T) { func TestSubmitMeasurementWith400(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}} - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2622,7 +2622,7 @@ func TestSubmitMeasurementWithResponseBodyReadErr(t *testing.T) { StatusCode: 200, Body: &FakeBody{Err: errMocked}, }} - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ HTTPClient: clnt, } ctx := context.Background() @@ -2644,7 +2644,7 @@ func TestSubmitMeasurementWithUnmarshalFailure(t *testing.T) { StatusCode: 200, Body: &FakeBody{Data: []byte(`{}`)}, }} - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ HTTPClient: clnt, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } @@ -2714,7 +2714,7 @@ func TestSubmitMeasurementRoundTrip(t *testing.T) { req := &apimodel.SubmitMeasurementRequest{} ff := &fakeFill{} ff.fill(&req) - api := &SubmitMeasurementAPI{BaseURL: srvr.URL} + api := &simpleSubmitMeasurementAPI{BaseURL: srvr.URL} ff.fill(&api.UserAgent) // issue request ctx := context.Background() @@ -2758,7 +2758,7 @@ func TestSubmitMeasurementTemplateErr(t *testing.T) { clnt := &FakeHTTPClient{Resp: &http.Response{ StatusCode: 500, }} - api := &SubmitMeasurementAPI{ + api := &simpleSubmitMeasurementAPI{ HTTPClient: clnt, TemplateExecutor: &FakeTemplateExecutor{Err: errMocked}, } diff --git a/internal/engine/ooapi/caching.go b/internal/engine/ooapi/caching.go index 2b2e333..725d2ab 100644 --- a/internal/engine/ooapi/caching.go +++ b/internal/engine/ooapi/caching.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:51.194159684 +0100 CET m=+0.000175181 +// 2021-03-10 13:17:32.455000394 +0100 CET m=+0.000119352 package ooapi @@ -12,20 +12,20 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -// MeasurementMetaCache implements caching for MeasurementMetaAPI. -type MeasurementMetaCache struct { - API MeasurementMetaCaller // mandatory - GobCodec GobCodec // optional - KVStore KVStore // mandatory +// withCacheMeasurementMetaAPI implements caching for simpleMeasurementMetaAPI. +type withCacheMeasurementMetaAPI struct { + API callerForMeasurementMetaAPI // mandatory + GobCodec GobCodec // optional + KVStore KVStore // mandatory } -type cacheEntryForMeasurementMeta struct { +type cacheEntryForMeasurementMetaAPI struct { Req *apimodel.MeasurementMetaRequest Resp *apimodel.MeasurementMetaResponse } // Call calls the API and implements caching. -func (c *MeasurementMetaCache) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) { +func (c *withCacheMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) { if resp, _ := c.readcache(req); resp != nil { return resp, nil } @@ -39,26 +39,26 @@ func (c *MeasurementMetaCache) Call(ctx context.Context, req *apimodel.Measureme return resp, nil } -func (c *MeasurementMetaCache) gobCodec() GobCodec { +func (c *withCacheMeasurementMetaAPI) gobCodec() GobCodec { if c.GobCodec != nil { return c.GobCodec } return &defaultGobCodec{} } -func (c *MeasurementMetaCache) getcache() ([]cacheEntryForMeasurementMeta, error) { +func (c *withCacheMeasurementMetaAPI) getcache() ([]cacheEntryForMeasurementMetaAPI, error) { data, err := c.KVStore.Get("MeasurementMeta.cache") if err != nil { return nil, err } - var out []cacheEntryForMeasurementMeta + var out []cacheEntryForMeasurementMetaAPI if err := c.gobCodec().Decode(data, &out); err != nil { return nil, err } return out, nil } -func (c *MeasurementMetaCache) setcache(in []cacheEntryForMeasurementMeta) error { +func (c *withCacheMeasurementMetaAPI) setcache(in []cacheEntryForMeasurementMetaAPI) error { data, err := c.gobCodec().Encode(in) if err != nil { return err @@ -66,7 +66,7 @@ func (c *MeasurementMetaCache) setcache(in []cacheEntryForMeasurementMeta) error return c.KVStore.Set("MeasurementMeta.cache", data) } -func (c *MeasurementMetaCache) readcache(req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) { +func (c *withCacheMeasurementMetaAPI) readcache(req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) { cache, err := c.getcache() if err != nil { return nil, err @@ -79,9 +79,9 @@ func (c *MeasurementMetaCache) readcache(req *apimodel.MeasurementMetaRequest) ( return nil, errCacheNotFound } -func (c *MeasurementMetaCache) writecache(req *apimodel.MeasurementMetaRequest, resp *apimodel.MeasurementMetaResponse) error { +func (c *withCacheMeasurementMetaAPI) writecache(req *apimodel.MeasurementMetaRequest, resp *apimodel.MeasurementMetaResponse) error { cache, _ := c.getcache() - out := []cacheEntryForMeasurementMeta{{Req: req, Resp: resp}} + out := []cacheEntryForMeasurementMetaAPI{{Req: req, Resp: resp}} const toomany = 64 for idx, cur := range cache { if reflect.DeepEqual(req, cur.Req) { @@ -95,4 +95,4 @@ func (c *MeasurementMetaCache) writecache(req *apimodel.MeasurementMetaRequest, return c.setcache(out) } -var _ MeasurementMetaCaller = &MeasurementMetaCache{} +var _ callerForMeasurementMetaAPI = &withCacheMeasurementMetaAPI{} diff --git a/internal/engine/ooapi/caching_test.go b/internal/engine/ooapi/caching_test.go index 592d69c..b75e46f 100644 --- a/internal/engine/ooapi/caching_test.go +++ b/internal/engine/ooapi/caching_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:51.49660021 +0100 CET m=+0.000217672 +// 2021-03-10 13:17:32.727159555 +0100 CET m=+0.000080664 package ooapi @@ -14,15 +14,15 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -func TestCacheMeasurementMetaAPISuccess(t *testing.T) { +func TestCachesimpleMeasurementMetaAPISuccess(t *testing.T) { ff := &fakeFill{} var expect *apimodel.MeasurementMetaResponse ff.fill(&expect) - cache := &MeasurementMetaCache{ + cache := &withCacheMeasurementMetaAPI{ API: &FakeMeasurementMetaAPI{ Response: expect, }, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.MeasurementMetaRequest ff.fill(&req) @@ -39,12 +39,12 @@ func TestCacheMeasurementMetaAPISuccess(t *testing.T) { } } -func TestCacheMeasurementMetaAPIWriteCacheError(t *testing.T) { +func TestCachesimpleMeasurementMetaAPIWriteCacheError(t *testing.T) { errMocked := errors.New("mocked error") ff := &fakeFill{} var expect *apimodel.MeasurementMetaResponse ff.fill(&expect) - cache := &MeasurementMetaCache{ + cache := &withCacheMeasurementMetaAPI{ API: &FakeMeasurementMetaAPI{ Response: expect, }, @@ -62,14 +62,14 @@ func TestCacheMeasurementMetaAPIWriteCacheError(t *testing.T) { } } -func TestCacheMeasurementMetaAPIFailureWithNoCache(t *testing.T) { +func TestCachesimpleMeasurementMetaAPIFailureWithNoCache(t *testing.T) { errMocked := errors.New("mocked error") ff := &fakeFill{} - cache := &MeasurementMetaCache{ + cache := &withCacheMeasurementMetaAPI{ API: &FakeMeasurementMetaAPI{ Err: errMocked, }, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.MeasurementMetaRequest ff.fill(&req) @@ -83,16 +83,16 @@ func TestCacheMeasurementMetaAPIFailureWithNoCache(t *testing.T) { } } -func TestCacheMeasurementMetaAPIFailureWithPreviousCache(t *testing.T) { +func TestCachesimpleMeasurementMetaAPIFailureWithPreviousCache(t *testing.T) { ff := &fakeFill{} var expect *apimodel.MeasurementMetaResponse ff.fill(&expect) fakeapi := &FakeMeasurementMetaAPI{ Response: expect, } - cache := &MeasurementMetaCache{ + cache := &withCacheMeasurementMetaAPI{ API: fakeapi, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.MeasurementMetaRequest ff.fill(&req) @@ -127,12 +127,12 @@ func TestCacheMeasurementMetaAPIFailureWithPreviousCache(t *testing.T) { } } -func TestCacheMeasurementMetaAPISetcacheWithEncodeError(t *testing.T) { +func TestCachesimpleMeasurementMetaAPISetcacheWithEncodeError(t *testing.T) { ff := &fakeFill{} errMocked := errors.New("mocked error") - var in []cacheEntryForMeasurementMeta + var in []cacheEntryForMeasurementMetaAPI ff.fill(&in) - cache := &MeasurementMetaCache{ + cache := &withCacheMeasurementMetaAPI{ GobCodec: &FakeCodec{EncodeErr: errMocked}, } err := cache.setcache(in) @@ -141,12 +141,12 @@ func TestCacheMeasurementMetaAPISetcacheWithEncodeError(t *testing.T) { } } -func TestCacheMeasurementMetaAPIReadCacheNotFound(t *testing.T) { +func TestCachesimpleMeasurementMetaAPIReadCacheNotFound(t *testing.T) { ff := &fakeFill{} - var incache []cacheEntryForMeasurementMeta + var incache []cacheEntryForMeasurementMetaAPI ff.fill(&incache) - cache := &MeasurementMetaCache{ - KVStore: &memkvstore{}, + cache := &withCacheMeasurementMetaAPI{ + KVStore: &MemKVStore{}, } err := cache.setcache(incache) if err != nil { @@ -163,7 +163,7 @@ func TestCacheMeasurementMetaAPIReadCacheNotFound(t *testing.T) { } } -func TestCacheMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) { +func TestCachesimpleMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) { ff := &fakeFill{} var req *apimodel.MeasurementMetaRequest ff.fill(&req) @@ -171,8 +171,8 @@ func TestCacheMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) { ff.fill(&resp1) var resp2 *apimodel.MeasurementMetaResponse ff.fill(&resp2) - cache := &MeasurementMetaCache{ - KVStore: &memkvstore{}, + cache := &withCacheMeasurementMetaAPI{ + KVStore: &MemKVStore{}, } err := cache.writecache(req, resp1) if err != nil { @@ -194,10 +194,10 @@ func TestCacheMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) { } } -func TestCacheMeasurementMetaAPICacheSizeLimited(t *testing.T) { +func TestCachesimpleMeasurementMetaAPICacheSizeLimited(t *testing.T) { ff := &fakeFill{} - cache := &MeasurementMetaCache{ - KVStore: &memkvstore{}, + cache := &withCacheMeasurementMetaAPI{ + KVStore: &MemKVStore{}, } var prev int for { diff --git a/internal/engine/ooapi/callers.go b/internal/engine/ooapi/callers.go index 89a1b94..d539634 100644 --- a/internal/engine/ooapi/callers.go +++ b/internal/engine/ooapi/callers.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:51.773813223 +0100 CET m=+0.000114768 +// 2021-03-10 13:17:32.990745473 +0100 CET m=+0.000093232 package ooapi @@ -11,68 +11,68 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -// CheckReportIDCaller represents any type exposing a method -// like CheckReportIDAPI.Call. -type CheckReportIDCaller interface { +// callerForCheckReportIDAPI represents any type exposing a method +// like simpleCheckReportIDAPI.Call. +type callerForCheckReportIDAPI interface { Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error) } -// CheckInCaller represents any type exposing a method -// like CheckInAPI.Call. -type CheckInCaller interface { +// callerForCheckInAPI represents any type exposing a method +// like simpleCheckInAPI.Call. +type callerForCheckInAPI interface { Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error) } -// LoginCaller represents any type exposing a method -// like LoginAPI.Call. -type LoginCaller interface { +// callerForLoginAPI represents any type exposing a method +// like simpleLoginAPI.Call. +type callerForLoginAPI interface { Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error) } -// MeasurementMetaCaller represents any type exposing a method -// like MeasurementMetaAPI.Call. -type MeasurementMetaCaller interface { +// callerForMeasurementMetaAPI represents any type exposing a method +// like simpleMeasurementMetaAPI.Call. +type callerForMeasurementMetaAPI interface { Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) } -// RegisterCaller represents any type exposing a method -// like RegisterAPI.Call. -type RegisterCaller interface { +// callerForRegisterAPI represents any type exposing a method +// like simpleRegisterAPI.Call. +type callerForRegisterAPI interface { Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error) } -// TestHelpersCaller represents any type exposing a method -// like TestHelpersAPI.Call. -type TestHelpersCaller interface { +// callerForTestHelpersAPI represents any type exposing a method +// like simpleTestHelpersAPI.Call. +type callerForTestHelpersAPI interface { Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error) } -// PsiphonConfigCaller represents any type exposing a method -// like PsiphonConfigAPI.Call. -type PsiphonConfigCaller interface { +// callerForPsiphonConfigAPI represents any type exposing a method +// like simplePsiphonConfigAPI.Call. +type callerForPsiphonConfigAPI interface { Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) } -// TorTargetsCaller represents any type exposing a method -// like TorTargetsAPI.Call. -type TorTargetsCaller interface { +// callerForTorTargetsAPI represents any type exposing a method +// like simpleTorTargetsAPI.Call. +type callerForTorTargetsAPI interface { Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) } -// URLsCaller represents any type exposing a method -// like URLsAPI.Call. -type URLsCaller interface { +// callerForURLsAPI represents any type exposing a method +// like simpleURLsAPI.Call. +type callerForURLsAPI interface { Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error) } -// OpenReportCaller represents any type exposing a method -// like OpenReportAPI.Call. -type OpenReportCaller interface { +// callerForOpenReportAPI represents any type exposing a method +// like simpleOpenReportAPI.Call. +type callerForOpenReportAPI interface { Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error) } -// SubmitMeasurementCaller represents any type exposing a method -// like SubmitMeasurementAPI.Call. -type SubmitMeasurementCaller interface { +// callerForSubmitMeasurementAPI represents any type exposing a method +// like simpleSubmitMeasurementAPI.Call. +type callerForSubmitMeasurementAPI interface { Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error) } diff --git a/internal/engine/ooapi/client.go b/internal/engine/ooapi/client.go new file mode 100644 index 0000000..4a9137f --- /dev/null +++ b/internal/engine/ooapi/client.go @@ -0,0 +1,13 @@ +package ooapi + +// Client is a client for speaking with the OONI API. Make sure you +// fill in the mandatory fields. +type Client struct { + BaseURL string // optional + GobCodec GobCodec // optional + HTTPClient HTTPClient // optional + JSONCodec JSONCodec // optional + KVStore KVStore // mandatory + RequestMaker RequestMaker // optional + UserAgent string // optional +} diff --git a/internal/engine/ooapi/clientcall.go b/internal/engine/ooapi/clientcall.go new file mode 100644 index 0000000..e6658aa --- /dev/null +++ b/internal/engine/ooapi/clientcall.go @@ -0,0 +1,214 @@ +// Code generated by go generate; DO NOT EDIT. +// 2021-03-10 13:17:33.307415377 +0100 CET m=+0.000150212 + +package ooapi + +//go:generate go run ./internal/generator -file clientcall.go + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" +) + +func (c *Client) newCheckReportIDCaller() callerForCheckReportIDAPI { + return &simpleCheckReportIDAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + } +} + +// CheckReportID calls the CheckReportID API. +func (c *Client) CheckReportID( + ctx context.Context, req *apimodel.CheckReportIDRequest, +) (*apimodel.CheckReportIDResponse, error) { + api := c.newCheckReportIDCaller() + return api.Call(ctx, req) +} + +func (c *Client) newCheckInCaller() callerForCheckInAPI { + return &simpleCheckInAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + } +} + +// CheckIn calls the CheckIn API. +func (c *Client) CheckIn( + ctx context.Context, req *apimodel.CheckInRequest, +) (*apimodel.CheckInResponse, error) { + api := c.newCheckInCaller() + return api.Call(ctx, req) +} + +func (c *Client) newMeasurementMetaCaller() callerForMeasurementMetaAPI { + return &withCacheMeasurementMetaAPI{ + API: &simpleMeasurementMetaAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + }, + GobCodec: c.GobCodec, + KVStore: c.KVStore, + } +} + +// MeasurementMeta calls the MeasurementMeta API. +func (c *Client) MeasurementMeta( + ctx context.Context, req *apimodel.MeasurementMetaRequest, +) (*apimodel.MeasurementMetaResponse, error) { + api := c.newMeasurementMetaCaller() + return api.Call(ctx, req) +} + +func (c *Client) newTestHelpersCaller() callerForTestHelpersAPI { + return &simpleTestHelpersAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + } +} + +// TestHelpers calls the TestHelpers API. +func (c *Client) TestHelpers( + ctx context.Context, req *apimodel.TestHelpersRequest, +) (apimodel.TestHelpersResponse, error) { + api := c.newTestHelpersCaller() + return api.Call(ctx, req) +} + +func (c *Client) newPsiphonConfigCaller() callerForPsiphonConfigAPI { + return &withLoginPsiphonConfigAPI{ + API: &simplePsiphonConfigAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + }, + JSONCodec: c.JSONCodec, + KVStore: c.KVStore, + RegisterAPI: &simpleRegisterAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + }, + LoginAPI: &simpleLoginAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + }, + } +} + +// PsiphonConfig calls the PsiphonConfig API. +func (c *Client) PsiphonConfig( + ctx context.Context, req *apimodel.PsiphonConfigRequest, +) (apimodel.PsiphonConfigResponse, error) { + api := c.newPsiphonConfigCaller() + return api.Call(ctx, req) +} + +func (c *Client) newTorTargetsCaller() callerForTorTargetsAPI { + return &withLoginTorTargetsAPI{ + API: &simpleTorTargetsAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + }, + JSONCodec: c.JSONCodec, + KVStore: c.KVStore, + RegisterAPI: &simpleRegisterAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + }, + LoginAPI: &simpleLoginAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + }, + } +} + +// TorTargets calls the TorTargets API. +func (c *Client) TorTargets( + ctx context.Context, req *apimodel.TorTargetsRequest, +) (apimodel.TorTargetsResponse, error) { + api := c.newTorTargetsCaller() + return api.Call(ctx, req) +} + +func (c *Client) newURLsCaller() callerForURLsAPI { + return &simpleURLsAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + } +} + +// URLs calls the URLs API. +func (c *Client) URLs( + ctx context.Context, req *apimodel.URLsRequest, +) (*apimodel.URLsResponse, error) { + api := c.newURLsCaller() + return api.Call(ctx, req) +} + +func (c *Client) newOpenReportCaller() callerForOpenReportAPI { + return &simpleOpenReportAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + } +} + +// OpenReport calls the OpenReport API. +func (c *Client) OpenReport( + ctx context.Context, req *apimodel.OpenReportRequest, +) (*apimodel.OpenReportResponse, error) { + api := c.newOpenReportCaller() + return api.Call(ctx, req) +} + +func (c *Client) newSubmitMeasurementCaller() callerForSubmitMeasurementAPI { + return &simpleSubmitMeasurementAPI{ + BaseURL: c.BaseURL, + HTTPClient: c.HTTPClient, + JSONCodec: c.JSONCodec, + RequestMaker: c.RequestMaker, + UserAgent: c.UserAgent, + } +} + +// SubmitMeasurement calls the SubmitMeasurement API. +func (c *Client) SubmitMeasurement( + ctx context.Context, req *apimodel.SubmitMeasurementRequest, +) (*apimodel.SubmitMeasurementResponse, error) { + api := c.newSubmitMeasurementCaller() + return api.Call(ctx, req) +} diff --git a/internal/engine/ooapi/clientcall_test.go b/internal/engine/ooapi/clientcall_test.go new file mode 100644 index 0000000..82096b4 --- /dev/null +++ b/internal/engine/ooapi/clientcall_test.go @@ -0,0 +1,898 @@ +// Code generated by go generate; DO NOT EDIT. +// 2021-03-10 13:17:33.590560357 +0100 CET m=+0.000145982 + +package ooapi + +//go:generate go run ./internal/generator -file clientcall_test.go + +import ( + "context" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" +) + +type handleClientCallCheckReportID struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp *apimodel.CheckReportIDResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallCheckReportID) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out *apimodel.CheckReportIDResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestCheckReportIDClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallCheckReportID{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.CheckReportIDRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.CheckReportID(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "GET" { + t.Fatal("invalid method") + } + // check the query + api := &simpleCheckReportIDAPI{BaseURL: srvr.URL} + httpReq, err := api.newRequest(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" { + t.Fatal(diff) + } +} + +type handleClientCallCheckIn struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp *apimodel.CheckInResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out *apimodel.CheckInResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestCheckInClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallCheckIn{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.CheckInRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.CheckIn(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "POST" { + t.Fatal("invalid method") + } + // check the body + if handler.contentType != "application/json" { + t.Fatal("invalid content-type header") + } + got := &apimodel.CheckInRequest{} + if err := json.Unmarshal(handler.body, &got); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(req, got); diff != "" { + t.Fatal(diff) + } +} + +type handleClientCallMeasurementMeta struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp *apimodel.MeasurementMetaResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out *apimodel.MeasurementMetaResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestMeasurementMetaClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallMeasurementMeta{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.MeasurementMetaRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.MeasurementMeta(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "GET" { + t.Fatal("invalid method") + } + // check the query + api := &simpleMeasurementMetaAPI{BaseURL: srvr.URL} + httpReq, err := api.newRequest(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" { + t.Fatal(diff) + } +} + +type handleClientCallTestHelpers struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp apimodel.TestHelpersResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out apimodel.TestHelpersResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestTestHelpersClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallTestHelpers{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.TestHelpersRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.TestHelpers(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "GET" { + t.Fatal("invalid method") + } + // check the query + api := &simpleTestHelpersAPI{BaseURL: srvr.URL} + httpReq, err := api.newRequest(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" { + t.Fatal(diff) + } +} + +type handleClientCallPsiphonConfig struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp apimodel.PsiphonConfigResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallPsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + if r.URL.Path == "/api/v1/register" { + var out apimodel.RegisterResponse + ff.fill(&out) + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) + return + } + if r.URL.Path == "/api/v1/login" { + var out apimodel.LoginResponse + ff.fill(&out) + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) + return + } + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out apimodel.PsiphonConfigResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestPsiphonConfigClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallPsiphonConfig{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.PsiphonConfigRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.PsiphonConfig(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "GET" { + t.Fatal("invalid method") + } + // check the query + api := &simplePsiphonConfigAPI{BaseURL: srvr.URL} + httpReq, err := api.newRequest(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" { + t.Fatal(diff) + } +} + +type handleClientCallTorTargets struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp apimodel.TorTargetsResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + if r.URL.Path == "/api/v1/register" { + var out apimodel.RegisterResponse + ff.fill(&out) + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) + return + } + if r.URL.Path == "/api/v1/login" { + var out apimodel.LoginResponse + ff.fill(&out) + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) + return + } + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out apimodel.TorTargetsResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestTorTargetsClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallTorTargets{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.TorTargetsRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.TorTargets(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "GET" { + t.Fatal("invalid method") + } + // check the query + api := &simpleTorTargetsAPI{BaseURL: srvr.URL} + httpReq, err := api.newRequest(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" { + t.Fatal(diff) + } +} + +type handleClientCallURLs struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp *apimodel.URLsResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallURLs) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out *apimodel.URLsResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestURLsClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallURLs{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.URLsRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.URLs(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "GET" { + t.Fatal("invalid method") + } + // check the query + api := &simpleURLsAPI{BaseURL: srvr.URL} + httpReq, err := api.newRequest(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" { + t.Fatal(diff) + } +} + +type handleClientCallOpenReport struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp *apimodel.OpenReportResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out *apimodel.OpenReportResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestOpenReportClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallOpenReport{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.OpenReportRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.OpenReport(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "POST" { + t.Fatal("invalid method") + } + // check the body + if handler.contentType != "application/json" { + t.Fatal("invalid content-type header") + } + got := &apimodel.OpenReportRequest{} + if err := json.Unmarshal(handler.body, &got); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(req, got); diff != "" { + t.Fatal(diff) + } +} + +type handleClientCallSubmitMeasurement struct { + accept string + body []byte + contentType string + count int32 + method string + mu sync.Mutex + resp *apimodel.SubmitMeasurementResponse + url *url.URL + userAgent string +} + +func (h *handleClientCallSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ff := fakeFill{} + defer h.mu.Unlock() + h.mu.Lock() + if h.count > 0 { + w.WriteHeader(400) + return + } + h.count++ + if r.Body != nil { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(400) + return + } + h.body = data + } + h.method = r.Method + h.url = r.URL + h.accept = r.Header.Get("Accept") + h.contentType = r.Header.Get("Content-Type") + h.userAgent = r.Header.Get("User-Agent") + var out *apimodel.SubmitMeasurementResponse + ff.fill(&out) + h.resp = out + data, err := json.Marshal(out) + if err != nil { + w.WriteHeader(400) + return + } + w.Write(data) +} + +func TestSubmitMeasurementClientCallRoundTrip(t *testing.T) { + // setup + handler := &handleClientCallSubmitMeasurement{} + srvr := httptest.NewServer(handler) + defer srvr.Close() + req := &apimodel.SubmitMeasurementRequest{} + ff := &fakeFill{} + ff.fill(&req) + clnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL} + ff.fill(&clnt.UserAgent) + // issue request + ctx := context.Background() + resp, err := clnt.SubmitMeasurement(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected non-nil response here") + } + // compare our response and server's one + if diff := cmp.Diff(handler.resp, resp); diff != "" { + t.Fatal(diff) + } + // check whether headers are OK + if handler.accept != "application/json" { + t.Fatal("invalid accept header") + } + if handler.userAgent != clnt.UserAgent { + t.Fatal("invalid user-agent header") + } + // check whether the method is OK + if handler.method != "POST" { + t.Fatal("invalid method") + } + // check the body + if handler.contentType != "application/json" { + t.Fatal("invalid content-type header") + } + got := &apimodel.SubmitMeasurementRequest{} + if err := json.Unmarshal(handler.body, &got); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(req, got); diff != "" { + t.Fatal(diff) + } +} diff --git a/internal/engine/ooapi/cloners.go b/internal/engine/ooapi/cloners.go index d461e5a..903318b 100644 --- a/internal/engine/ooapi/cloners.go +++ b/internal/engine/ooapi/cloners.go @@ -1,18 +1,18 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:52.108352268 +0100 CET m=+0.000275862 +// 2021-03-10 13:17:33.849313529 +0100 CET m=+0.000136772 package ooapi //go:generate go run ./internal/generator -file cloners.go -// PsiphonConfigCaller represents any type exposing a method -// like PsiphonConfigAPI.WithToken. -type PsiphonConfigCloner interface { - WithToken(token string) PsiphonConfigCaller +// clonerForPsiphonConfigAPI represents any type exposing a method +// like simplePsiphonConfigAPI.WithToken. +type clonerForPsiphonConfigAPI interface { + WithToken(token string) callerForPsiphonConfigAPI } -// TorTargetsCaller represents any type exposing a method -// like TorTargetsAPI.WithToken. -type TorTargetsCloner interface { - WithToken(token string) TorTargetsCaller +// clonerForTorTargetsAPI represents any type exposing a method +// like simpleTorTargetsAPI.WithToken. +type clonerForTorTargetsAPI interface { + WithToken(token string) callerForTorTargetsAPI } diff --git a/internal/engine/ooapi/dependencies.go b/internal/engine/ooapi/dependencies.go index 0b1bbf1..2abf27e 100644 --- a/internal/engine/ooapi/dependencies.go +++ b/internal/engine/ooapi/dependencies.go @@ -21,8 +21,8 @@ type RequestMaker interface { NewRequest(ctx context.Context, method, URL string, body io.Reader) (*http.Request, error) } -// TemplateExecutor parses and executes a text template. -type TemplateExecutor interface { +// templateExecutor parses and executes a text template. +type templateExecutor interface { // Execute takes in input a template string and some piece of data. It // returns either a string where template parameters have been replaced, // on success, or an error, on failure. diff --git a/internal/engine/ooapi/doc.go b/internal/engine/ooapi/doc.go index 1f8f6fd..0117043 100644 --- a/internal/engine/ooapi/doc.go +++ b/internal/engine/ooapi/doc.go @@ -1,108 +1,19 @@ -// Package ooapi contains clients for the OONI API. We -// automatically generate the code in this package from -// the apimodel and internal/generator packages. For -// each OONI API, we define up to three data structures: +// Package ooapi contains a client for the OONI API. We +// automatically generate the code in this package from the +// apimodel and internal/generator packages. // -// 1. a data structure representing the API; +// Usage // -// 2. a caching data structure, if the API -// supports caching; +// You need to create a Client. Make sure you set all +// the mandatory fields. You will then have a function +// for every supported OONI API. This function will +// take in input a context and a request. You need to +// fill the request, of course. The return value is +// either a response or an error. // -// 3. an auto-login data structure, if the API -// requires login. -// -// The rest of this documentation page describes these -// three data structures and the design and architecture -// of this package. Refer to subpackages for more -// information on how to specify an API. -// -// API data structure -// -// For each API, this package defines a data structure -// representing the API. For example, for the TorTargets API, -// we define the TorTargetsAPI data structure. -// -// The API data structure defines a method named Call that -// allows calling the specified API. Call takes as arguments -// a context and the request for the API and returns the -// API response or an error. -// -// Request and response messages live inside the apimodel -// subpackage. We name them after the API. Thus, for -// the TorTargets API, the request is TorTargetsRequest, -// and the response is TorTargetsResponse. -// -// API data structures are cheap to create and do not -// mutate. They should be used in place and then forgotten -// off once the API call is complete. -// -// Unless explicitly indicated, the zero value of every -// API data structure is a valid API data structure. -// -// In terms of dependencies, APIs certainly need an http.Client -// to communicate with the OONI backend. To represent such a -// client, we use the HTTPClient interface. If you do not tell -// an API which http.Client to use, we will default to the -// standard library's http.DefaultClient. -// -// An API also depends on a JSONCodec. That is, on a data -// structures that encodes data to/from JSON. If you do not -// specify explicitly a JSONCodec, we will use the Go -// standard library's JSON implementation. -// -// When an API requires authentication, you need to tell -// it which authentication token to use. This gives you -// control over obtaining the token and is the low-level -// way of interacting with authenticated APIs. We recommend -// using the auto-login wrappers instead (see below). -// -// Authenticated APIs also define the WithToken method. This -// method takes as argument a token and returns a copy of the -// original API using the given token. We use this method -// to implement auto-login wrappers. -// -// For each API, we also define two interfaces: -// -// 1. the Caller interface represents the possibility of -// calling a specific API with the correct arguments; -// -// 2. the Cloner interface represents the possibility of -// calling WithToken on the given API. -// -// They abstract the interaction between the API type and -// its caching and auto-login wrappers. -// -// Caching -// -// If an API supports caching, we define a type whose name -// ends in Cache. The TorTargets API cache, for example, -// is TorTargetsCache. These caching types wrap the API type -// and provide the caching functionality. -// -// Because the cache needs to read from and write to the -// disk, a caching type needs a KVStore. A KVStore is -// an interface that allow you to bind a specific key to -// a given blob of bytes and to retrieve such bytes later. -// -// Caches use the gob data format from the Go standard -// library (`encoding/gob`). We abstract this dependency -// using the GobCodec interface. By default, when you -// do not specify a GobCodec we use the implementation -// of gob from the Go standard library. -// -// See the example describing caching for more information -// on how to use caching. -// -// Auto-login -// -// If an API supports auto-login, we define a type whose -// name ends with WithLogin. The TorTargets auto-login struct, -// for example, is called TorTargetsAPIWithLogin. -// -// Auto-login wrappers need to store persistent data. We -// use a KVStore for that (see above). We encode login data -// using JSON. To this end, we use a JSONCodec (also -// described above). +// If an API requires login, we will automatically +// perform the login. If an API uses caching, we will +// automatically use the cache. // // See the example describing auto-login for more information // on how to use auto-login. @@ -142,22 +53,4 @@ // The ./internal/generator contains code to generate most // code in this package. In particular, the spec.go file is // the specification of the APIs. -// -// Notable generated files -// -// - apis.go: contains APIs (e.g., TorTargetsAPI); -// -// - caching.go: contains caching wrappers for every API -// that declares that it needs a cache (e.g., TorTargetsCache); -// -// - callers.go: contains Callers; -// -// - cloners.go: contains the Cloners; -// -// - login.go: contains auto-login wrappers (e.g., -// TorTargetsAPIWithLogin); -// -// - requests.go: contains code to generate http.Requests. -// -// - responses.go: code to parse http.Responses. package ooapi diff --git a/internal/engine/ooapi/errors.go b/internal/engine/ooapi/errors.go index 35ed7cc..5227ed4 100644 --- a/internal/engine/ooapi/errors.go +++ b/internal/engine/ooapi/errors.go @@ -2,13 +2,13 @@ package ooapi import "errors" -// Errors defined by this package. In addition to these errors, this -// package may of course return any other stdlib specific error. +// Errors defined by this package. var ( - ErrEmptyField = errors.New("apiclient: empty field") - ErrHTTPFailure = errors.New("apiclient: http request failed") - ErrJSONLiteralNull = errors.New("apiclient: server returned us a literal null") - ErrMissingToken = errors.New("apiclient: missing auth token") - ErrUnauthorized = errors.New("apiclient: not authorized") - errCacheNotFound = errors.New("apiclient: not found in cache") + ErrAPICallFailed = errors.New("ooapi: API call failed") + ErrEmptyField = errors.New("ooapi: empty field") + ErrHTTPFailure = errors.New("ooapi: http request failed") + ErrJSONLiteralNull = errors.New("ooapi: server returned us a literal null") + ErrMissingToken = errors.New("ooapi: missing auth token") + ErrUnauthorized = errors.New("ooapi: not authorized") + errCacheNotFound = errors.New("ooapi: not found in cache") ) diff --git a/internal/engine/ooapi/fakeapi_test.go b/internal/engine/ooapi/fakeapi_test.go index 3084483..81d096c 100644 --- a/internal/engine/ooapi/fakeapi_test.go +++ b/internal/engine/ooapi/fakeapi_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:52.357709034 +0100 CET m=+0.000208565 +// 2021-03-10 13:17:34.086220417 +0100 CET m=+0.000080714 package ooapi @@ -24,7 +24,7 @@ func (fapi *FakeCheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckR } var ( - _ CheckReportIDCaller = &FakeCheckReportIDAPI{} + _ callerForCheckReportIDAPI = &FakeCheckReportIDAPI{} ) type FakeCheckInAPI struct { @@ -39,7 +39,7 @@ func (fapi *FakeCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInReque } var ( - _ CheckInCaller = &FakeCheckInAPI{} + _ callerForCheckInAPI = &FakeCheckInAPI{} ) type FakeLoginAPI struct { @@ -54,7 +54,7 @@ func (fapi *FakeLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) } var ( - _ LoginCaller = &FakeLoginAPI{} + _ callerForLoginAPI = &FakeLoginAPI{} ) type FakeMeasurementMetaAPI struct { @@ -69,7 +69,7 @@ func (fapi *FakeMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.Meas } var ( - _ MeasurementMetaCaller = &FakeMeasurementMetaAPI{} + _ callerForMeasurementMetaAPI = &FakeMeasurementMetaAPI{} ) type FakeRegisterAPI struct { @@ -84,7 +84,7 @@ func (fapi *FakeRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterReq } var ( - _ RegisterCaller = &FakeRegisterAPI{} + _ callerForRegisterAPI = &FakeRegisterAPI{} ) type FakeTestHelpersAPI struct { @@ -99,11 +99,11 @@ func (fapi *FakeTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelp } var ( - _ TestHelpersCaller = &FakeTestHelpersAPI{} + _ callerForTestHelpersAPI = &FakeTestHelpersAPI{} ) type FakePsiphonConfigAPI struct { - WithResult PsiphonConfigCaller + WithResult callerForPsiphonConfigAPI Err error Response apimodel.PsiphonConfigResponse CountCall int32 @@ -114,17 +114,17 @@ func (fapi *FakePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.Psipho return fapi.Response, fapi.Err } -func (fapi *FakePsiphonConfigAPI) WithToken(token string) PsiphonConfigCaller { +func (fapi *FakePsiphonConfigAPI) WithToken(token string) callerForPsiphonConfigAPI { return fapi.WithResult } var ( - _ PsiphonConfigCaller = &FakePsiphonConfigAPI{} - _ PsiphonConfigCloner = &FakePsiphonConfigAPI{} + _ callerForPsiphonConfigAPI = &FakePsiphonConfigAPI{} + _ clonerForPsiphonConfigAPI = &FakePsiphonConfigAPI{} ) type FakeTorTargetsAPI struct { - WithResult TorTargetsCaller + WithResult callerForTorTargetsAPI Err error Response apimodel.TorTargetsResponse CountCall int32 @@ -135,13 +135,13 @@ func (fapi *FakeTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTarget return fapi.Response, fapi.Err } -func (fapi *FakeTorTargetsAPI) WithToken(token string) TorTargetsCaller { +func (fapi *FakeTorTargetsAPI) WithToken(token string) callerForTorTargetsAPI { return fapi.WithResult } var ( - _ TorTargetsCaller = &FakeTorTargetsAPI{} - _ TorTargetsCloner = &FakeTorTargetsAPI{} + _ callerForTorTargetsAPI = &FakeTorTargetsAPI{} + _ clonerForTorTargetsAPI = &FakeTorTargetsAPI{} ) type FakeURLsAPI struct { @@ -156,7 +156,7 @@ func (fapi *FakeURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (* } var ( - _ URLsCaller = &FakeURLsAPI{} + _ callerForURLsAPI = &FakeURLsAPI{} ) type FakeOpenReportAPI struct { @@ -171,7 +171,7 @@ func (fapi *FakeOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenRepor } var ( - _ OpenReportCaller = &FakeOpenReportAPI{} + _ callerForOpenReportAPI = &FakeOpenReportAPI{} ) type FakeSubmitMeasurementAPI struct { @@ -186,5 +186,5 @@ func (fapi *FakeSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.Su } var ( - _ SubmitMeasurementCaller = &FakeSubmitMeasurementAPI{} + _ callerForSubmitMeasurementAPI = &FakeSubmitMeasurementAPI{} ) diff --git a/internal/engine/ooapi/httpclient_test.go b/internal/engine/ooapi/httpclient_test.go new file mode 100644 index 0000000..c6acb46 --- /dev/null +++ b/internal/engine/ooapi/httpclient_test.go @@ -0,0 +1,21 @@ +package ooapi + +import ( + "net/http" + "testing" +) + +type VerboseHTTPClient struct { + T *testing.T +} + +func (c *VerboseHTTPClient) Do(req *http.Request) (*http.Response, error) { + c.T.Logf("> %s %s", req.Method, req.URL.String()) + resp, err := http.DefaultClient.Do(req) + if err != nil { + c.T.Logf("< %s", err.Error()) + return nil, err + } + c.T.Logf("< %d", resp.StatusCode) + return resp, nil +} diff --git a/internal/engine/ooapi/integration_test.go b/internal/engine/ooapi/integration_test.go index 43465be..59e0eea 100644 --- a/internal/engine/ooapi/integration_test.go +++ b/internal/engine/ooapi/integration_test.go @@ -1,28 +1,13 @@ -package ooapi +package ooapi_test import ( "context" - "net/http" "testing" + "github.com/ooni/probe-cli/v3/internal/engine/ooapi" "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -type VerboseHTTPClient struct { - t *testing.T -} - -func (c *VerboseHTTPClient) Do(req *http.Request) (*http.Response, error) { - c.t.Logf("> %s %s", req.Method, req.URL.String()) - resp, err := http.DefaultClient.Do(req) - if err != nil { - c.t.Logf("< %s", err.Error()) - return nil, err - } - c.t.Logf("< %d", resp.StatusCode) - return resp, nil -} - func TestWithRealServerDoCheckIn(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") @@ -40,12 +25,10 @@ func TestWithRealServerDoCheckIn(t *testing.T) { CategoryCodes: []string{"NEWS", "CULTR"}, }, } - httpClnt := &VerboseHTTPClient{t: t} - api := &CheckInAPI{ - HTTPClient: httpClnt, - } + httpClnt := &ooapi.VerboseHTTPClient{T: t} + clnt := &ooapi.Client{HTTPClient: httpClnt, KVStore: &ooapi.MemKVStore{}} ctx := context.Background() - resp, err := api.Call(ctx, req) + resp, err := clnt.CheckIn(ctx, req) if err != nil { t.Fatal(err) } @@ -67,9 +50,9 @@ func TestWithRealServerDoCheckReportID(t *testing.T) { req := &apimodel.CheckReportIDRequest{ ReportID: "20210223T093606Z_ndt_JO_8376_n1_kDYToqrugDY54Soy", } - api := &CheckReportIDAPI{} + clnt := &ooapi.Client{KVStore: &ooapi.MemKVStore{}} ctx := context.Background() - resp, err := api.Call(ctx, req) + resp, err := clnt.CheckReportID(ctx, req) if err != nil { t.Fatal(err) } @@ -86,9 +69,9 @@ func TestWithRealServerDoMeasurementMeta(t *testing.T) { req := &apimodel.MeasurementMetaRequest{ ReportID: "20210223T093606Z_ndt_JO_8376_n1_kDYToqrugDY54Soy", } - api := &MeasurementMetaAPI{} + clnt := &ooapi.Client{KVStore: &ooapi.MemKVStore{}} ctx := context.Background() - resp, err := api.Call(ctx, req) + resp, err := clnt.MeasurementMeta(ctx, req) if err != nil { t.Fatal(err) } @@ -113,9 +96,9 @@ func TestWithRealServerDoOpenReport(t *testing.T) { TestStartTime: "2018-11-01 15:33:20", TestVersion: "0.1.0", } - api := &OpenReportAPI{} + clnt := &ooapi.Client{KVStore: &ooapi.MemKVStore{}} ctx := context.Background() - resp, err := api.Call(ctx, req) + resp, err := clnt.OpenReport(ctx, req) if err != nil { t.Fatal(err) } @@ -130,21 +113,10 @@ func TestWithRealServerDoPsiphonConfig(t *testing.T) { t.Skip("skip test in short mode") } req := &apimodel.PsiphonConfigRequest{} - httpClnt := &VerboseHTTPClient{t: t} - api := &PsiphonConfigAPIWithLogin{ - API: &PsiphonConfigAPI{ - HTTPClient: httpClnt, - }, - KVStore: &memkvstore{}, - RegisterAPI: &RegisterAPI{ - HTTPClient: httpClnt, - }, - LoginAPI: &LoginAPI{ - HTTPClient: httpClnt, - }, - } + httpClnt := &ooapi.VerboseHTTPClient{T: t} + clnt := &ooapi.Client{HTTPClient: httpClnt, KVStore: &ooapi.MemKVStore{}} ctx := context.Background() - resp, err := api.Call(ctx, req) + resp, err := clnt.PsiphonConfig(ctx, req) if err != nil { t.Fatal(err) } @@ -159,21 +131,10 @@ func TestWithRealServerDoTorTargets(t *testing.T) { t.Skip("skip test in short mode") } req := &apimodel.TorTargetsRequest{} - httpClnt := &VerboseHTTPClient{t: t} - api := &TorTargetsAPIWithLogin{ - API: &TorTargetsAPI{ - HTTPClient: httpClnt, - }, - KVStore: &memkvstore{}, - RegisterAPI: &RegisterAPI{ - HTTPClient: httpClnt, - }, - LoginAPI: &LoginAPI{ - HTTPClient: httpClnt, - }, - } + httpClnt := &ooapi.VerboseHTTPClient{T: t} + clnt := &ooapi.Client{HTTPClient: httpClnt, KVStore: &ooapi.MemKVStore{}} ctx := context.Background() - resp, err := api.Call(ctx, req) + resp, err := clnt.TorTargets(ctx, req) if err != nil { t.Fatal(err) } @@ -191,9 +152,9 @@ func TestWithRealServerDoURLs(t *testing.T) { CountryCode: "IT", Limit: 3, } - api := &URLsAPI{} + clnt := &ooapi.Client{KVStore: &ooapi.MemKVStore{}} ctx := context.Background() - resp, err := api.Call(ctx, req) + resp, err := clnt.URLs(ctx, req) if err != nil { t.Fatal(err) } diff --git a/internal/engine/ooapi/internal/generator/apis.go b/internal/engine/ooapi/internal/generator/apis.go index e69b1f7..56536db 100644 --- a/internal/engine/ooapi/internal/generator/apis.go +++ b/internal/engine/ooapi/internal/generator/apis.go @@ -54,7 +54,7 @@ var apiFields = []apiField{{ comment: "optional", }, { name: "TemplateExecutor", - kind: "TemplateExecutor", + kind: "templateExecutor", comment: "optional", ifTemplate: true, }, { @@ -120,7 +120,7 @@ func (d *Descriptor) genNewAPI(sb *strings.Builder) { if d.URLPath.IsTemplate { fmt.Fprintf( - sb, "func (api *%s) templateExecutor() TemplateExecutor {\n", + sb, "func (api *%s) templateExecutor() templateExecutor {\n", d.APIStructName()) fmt.Fprint(sb, "\tif api.TemplateExecutor != nil {\n") fmt.Fprint(sb, "\t\treturn api.TemplateExecutor\n") diff --git a/internal/engine/ooapi/internal/generator/caching.go b/internal/engine/ooapi/internal/generator/caching.go index ca1dd6a..75a2bc9 100644 --- a/internal/engine/ooapi/internal/generator/caching.go +++ b/internal/engine/ooapi/internal/generator/caching.go @@ -8,8 +8,8 @@ import ( func (d *Descriptor) genNewCache(sb *strings.Builder) { fmt.Fprintf(sb, "// %s implements caching for %s.\n", - d.CacheStructName(), d.APIStructName()) - fmt.Fprintf(sb, "type %s struct {\n", d.CacheStructName()) + d.WithCacheAPIStructName(), d.APIStructName()) + fmt.Fprintf(sb, "type %s struct {\n", d.WithCacheAPIStructName()) fmt.Fprintf(sb, "\tAPI %s // mandatory\n", d.CallerInterfaceName()) fmt.Fprint(sb, "\tGobCodec GobCodec // optional\n") fmt.Fprint(sb, "\tKVStore KVStore // mandatory\n") @@ -22,7 +22,7 @@ func (d *Descriptor) genNewCache(sb *strings.Builder) { fmt.Fprintf(sb, "// Call calls the API and implements caching.\n") fmt.Fprintf(sb, "func (c *%s) Call(ctx context.Context, req %s) (%s, error) {\n", - d.CacheStructName(), d.RequestTypeName(), d.ResponseTypeName()) + d.WithCacheAPIStructName(), d.RequestTypeName(), d.ResponseTypeName()) if d.CachePolicy == CacheAlways { fmt.Fprint(sb, "\tif resp, _ := c.readcache(req); resp != nil {\n") fmt.Fprint(sb, "\t\treturn resp, nil\n") @@ -43,7 +43,7 @@ func (d *Descriptor) genNewCache(sb *strings.Builder) { fmt.Fprint(sb, "\treturn resp, nil\n") fmt.Fprint(sb, "}\n\n") - fmt.Fprintf(sb, "func (c *%s) gobCodec() GobCodec {\n", d.CacheStructName()) + fmt.Fprintf(sb, "func (c *%s) gobCodec() GobCodec {\n", d.WithCacheAPIStructName()) fmt.Fprint(sb, "\tif c.GobCodec != nil {\n") fmt.Fprint(sb, "\t\treturn c.GobCodec\n") fmt.Fprint(sb, "\t}\n") @@ -51,7 +51,7 @@ func (d *Descriptor) genNewCache(sb *strings.Builder) { fmt.Fprint(sb, "}\n\n") fmt.Fprintf(sb, "func (c *%s) getcache() ([]%s, error) {\n", - d.CacheStructName(), d.CacheEntryName()) + d.WithCacheAPIStructName(), d.CacheEntryName()) fmt.Fprintf(sb, "\tdata, err := c.KVStore.Get(\"%s\")\n", d.CacheKey()) fmt.Fprint(sb, "\tif err != nil {\n") fmt.Fprint(sb, "\t\treturn nil, err\n") @@ -64,7 +64,7 @@ func (d *Descriptor) genNewCache(sb *strings.Builder) { fmt.Fprint(sb, "}\n\n") fmt.Fprintf(sb, "func (c *%s) setcache(in []%s) error {\n", - d.CacheStructName(), d.CacheEntryName()) + d.WithCacheAPIStructName(), d.CacheEntryName()) fmt.Fprint(sb, "\tdata, err := c.gobCodec().Encode(in)\n") fmt.Fprint(sb, "\tif err != nil {\n") fmt.Fprint(sb, "\t\treturn err\n") @@ -73,7 +73,7 @@ func (d *Descriptor) genNewCache(sb *strings.Builder) { fmt.Fprint(sb, "}\n\n") fmt.Fprintf(sb, "func (c *%s) readcache(req %s) (%s, error) {\n", - d.CacheStructName(), d.RequestTypeName(), d.ResponseTypeName()) + d.WithCacheAPIStructName(), d.RequestTypeName(), d.ResponseTypeName()) fmt.Fprint(sb, "\tcache, err := c.getcache()\n") fmt.Fprint(sb, "\tif err != nil {\n") fmt.Fprint(sb, "\t\treturn nil, err\n") @@ -87,7 +87,7 @@ func (d *Descriptor) genNewCache(sb *strings.Builder) { fmt.Fprint(sb, "}\n\n") fmt.Fprintf(sb, "func (c *%s) writecache(req %s, resp %s) error {\n", - d.CacheStructName(), d.RequestTypeName(), d.ResponseTypeName()) + d.WithCacheAPIStructName(), d.RequestTypeName(), d.ResponseTypeName()) fmt.Fprint(sb, "\tcache, _ := c.getcache()\n") fmt.Fprintf(sb, "\tout := []%s{{Req: req, Resp: resp}}\n", d.CacheEntryName()) fmt.Fprint(sb, "\tconst toomany = 64\n") @@ -104,7 +104,7 @@ func (d *Descriptor) genNewCache(sb *strings.Builder) { fmt.Fprint(sb, "}\n\n") fmt.Fprintf(sb, "var _ %s = &%s{}\n\n", d.CallerInterfaceName(), - d.CacheStructName()) + d.WithCacheAPIStructName()) } // GenCachingGo generates caching.go. diff --git a/internal/engine/ooapi/internal/generator/cachingtest.go b/internal/engine/ooapi/internal/generator/cachingtest.go index a556885..c56459d 100644 --- a/internal/engine/ooapi/internal/generator/cachingtest.go +++ b/internal/engine/ooapi/internal/generator/cachingtest.go @@ -11,11 +11,11 @@ func (d *Descriptor) genTestCacheSuccess(sb *strings.Builder) { fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") - fmt.Fprintf(sb, "\tcache := &%s{\n", d.CacheStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\tResponse: expect,\n") fmt.Fprint(sb, "\t\t},\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) fmt.Fprint(sb, "\tff.fill(&req)\n") @@ -39,8 +39,8 @@ func (d *Descriptor) genTestWriteCacheError(sb *strings.Builder) { fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") - fmt.Fprintf(sb, "\tcache := &%s{\n", d.CacheStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\tResponse: expect,\n") fmt.Fprint(sb, "\t\t},\n") fmt.Fprint(sb, "\t\tKVStore: &FakeKVStore{SetError: errMocked},\n") @@ -62,11 +62,11 @@ func (d *Descriptor) genTestFailureWithNoCache(sb *strings.Builder) { fmt.Fprintf(sb, "func TestCache%sFailureWithNoCache(t *testing.T) {\n", d.APIStructName()) fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n") fmt.Fprint(sb, "\tff := &fakeFill{}\n") - fmt.Fprintf(sb, "\tcache := &%s{\n", d.CacheStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\tErr: errMocked,\n") fmt.Fprint(sb, "\t\t},\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) fmt.Fprint(sb, "\tff.fill(&req)\n") @@ -87,12 +87,12 @@ func (d *Descriptor) genTestFailureWithPreviousCache(sb *strings.Builder) { fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") - fmt.Fprintf(sb, "\tfakeapi := &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\tfakeapi := &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\tResponse: expect,\n") fmt.Fprint(sb, "\t}\n") - fmt.Fprintf(sb, "\tcache := &%s{\n", d.CacheStructName()) + fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName()) fmt.Fprint(sb, "\t\tAPI: fakeapi,\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) fmt.Fprint(sb, "\tff.fill(&req)\n") @@ -134,7 +134,7 @@ func (d *Descriptor) genTestSetcacheWithEncodeError(sb *strings.Builder) { fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n") fmt.Fprintf(sb, "\tvar in []%s\n", d.CacheEntryName()) fmt.Fprint(sb, "\tff.fill(&in)\n") - fmt.Fprintf(sb, "\tcache := &%s{\n", d.CacheStructName()) + fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName()) fmt.Fprint(sb, "\t\tGobCodec: &FakeCodec{EncodeErr: errMocked},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\terr := cache.setcache(in)\n") @@ -155,8 +155,8 @@ func (d *Descriptor) genTestReadCacheNotFound(sb *strings.Builder) { fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar incache []%s\n", d.CacheEntryName()) fmt.Fprint(sb, "\tff.fill(&incache)\n") - fmt.Fprintf(sb, "\tcache := &%s{\n", d.CacheStructName()) - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName()) + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\terr := cache.setcache(incache)\n") fmt.Fprintf(sb, "\tif err != nil {\n") @@ -183,8 +183,8 @@ func (d *Descriptor) genTestWriteCacheDuplicate(sb *strings.Builder) { fmt.Fprint(sb, "\tff.fill(&resp1)\n") fmt.Fprintf(sb, "\tvar resp2 %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&resp2)\n") - fmt.Fprintf(sb, "\tcache := &%s{\n", d.CacheStructName()) - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName()) + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\terr := cache.writecache(req, resp1)\n") fmt.Fprintf(sb, "\tif err != nil {\n") @@ -216,8 +216,8 @@ func (d *Descriptor) genTestCachSizeLimited(sb *strings.Builder) { } fmt.Fprintf(sb, "func TestCache%sCacheSizeLimited(t *testing.T) {\n", d.APIStructName()) fmt.Fprint(sb, "\tff := &fakeFill{}\n") - fmt.Fprintf(sb, "\tcache := &%s{\n", d.CacheStructName()) - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName()) + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar prev int\n") fmt.Fprintf(sb, "\tfor {\n") diff --git a/internal/engine/ooapi/internal/generator/clientcall.go b/internal/engine/ooapi/internal/generator/clientcall.go new file mode 100644 index 0000000..eeb688d --- /dev/null +++ b/internal/engine/ooapi/internal/generator/clientcall.go @@ -0,0 +1,104 @@ +package main + +import ( + "fmt" + "strings" + "time" +) + +func (d *Descriptor) clientMakeAPIBase(sb *strings.Builder) { + fmt.Fprintf(sb, "&%s{\n", d.APIStructName()) + for _, field := range apiFields { + if field.ifLogin || field.ifTemplate { + continue + } + fmt.Fprintf(sb, "\t%s: c.%s,\n", field.name, field.name) + } + fmt.Fprint(sb, "}") +} + +func (d *Descriptor) clientMakeAPI(sb *strings.Builder) { + if d.RequiresLogin && d.CachePolicy != CacheNone { + panic("we don't support requiresLogin with caching") + } + if d.RequiresLogin { + fmt.Fprintf(sb, "&%s{\n", d.WithLoginAPIStructName()) + fmt.Fprint(sb, "\tAPI:") + d.clientMakeAPIBase(sb) + fmt.Fprint(sb, ",\n") + fmt.Fprint(sb, "\tJSONCodec: c.JSONCodec,\n") + fmt.Fprint(sb, "\tKVStore: c.KVStore,\n") + fmt.Fprint(sb, "\tRegisterAPI: &simpleRegisterAPI{\n") + for _, field := range apiFields { + if field.ifLogin || field.ifTemplate { + continue + } + fmt.Fprintf(sb, "\t%s: c.%s,\n", field.name, field.name) + } + fmt.Fprint(sb, "\t},\n") + fmt.Fprint(sb, "\tLoginAPI: &simpleLoginAPI{\n") + for _, field := range apiFields { + if field.ifLogin || field.ifTemplate { + continue + } + fmt.Fprintf(sb, "\t%s: c.%s,\n", field.name, field.name) + } + fmt.Fprint(sb, "\t},\n") + fmt.Fprint(sb, "}\n") + return + } + if d.CachePolicy != CacheNone { + fmt.Fprintf(sb, "&%s{\n", d.WithCacheAPIStructName()) + fmt.Fprint(sb, "\tAPI:") + d.clientMakeAPIBase(sb) + fmt.Fprint(sb, ",\n") + fmt.Fprint(sb, "\tGobCodec: c.GobCodec,\n") + fmt.Fprint(sb, "\tKVStore: c.KVStore,\n") + fmt.Fprint(sb, "}\n") + return + } + d.clientMakeAPIBase(sb) + fmt.Fprint(sb, "\n") +} + +func (d *Descriptor) genClientNewCaller(sb *strings.Builder) { + fmt.Fprintf(sb, "func (c *Client) new%sCaller() ", d.Name) + fmt.Fprintf(sb, "%s {\n", d.CallerInterfaceName()) + fmt.Fprint(sb, "\treturn ") + d.clientMakeAPI(sb) + fmt.Fprint(sb, "}\n\n") +} + +func (d *Descriptor) genClientCall(sb *strings.Builder) { + fmt.Fprintf(sb, "// %s calls the %s API.\n", d.Name, d.Name) + fmt.Fprintf(sb, "func (c *Client) %s(\n", d.Name) + fmt.Fprintf(sb, "ctx context.Context, req %s,\n) ", d.RequestTypeName()) + fmt.Fprintf(sb, "(%s, error) {\n", d.ResponseTypeName()) + fmt.Fprintf(sb, "\tapi := c.new%sCaller()\n", d.Name) + fmt.Fprint(sb, "\treturn api.Call(ctx, req)\n") + fmt.Fprint(sb, "}\n\n") +} + +// GenClientCallGo generates clientcall.go. +func GenClientCallGo(file string) { + var sb strings.Builder + fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n") + fmt.Fprintf(&sb, "// %s\n\n", time.Now()) + fmt.Fprint(&sb, "package ooapi\n\n") + fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file) + fmt.Fprint(&sb, "import (\n") + fmt.Fprint(&sb, "\t\"context\"\n") + fmt.Fprint(&sb, "\n") + fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel\"\n") + fmt.Fprint(&sb, ")\n") + for _, desc := range Descriptors { + switch desc.Name { + case "Register", "Login": + // We don't want to generate these APIs as toplevel. + continue + } + desc.genClientNewCaller(&sb) + desc.genClientCall(&sb) + } + writefile(file, &sb) +} diff --git a/internal/engine/ooapi/internal/generator/clientcalltest.go b/internal/engine/ooapi/internal/generator/clientcalltest.go new file mode 100644 index 0000000..f840eff --- /dev/null +++ b/internal/engine/ooapi/internal/generator/clientcalltest.go @@ -0,0 +1,181 @@ +package main + +import ( + "fmt" + "strings" + "time" +) + +func (d *Descriptor) genTestClientCallRoundTrip(sb *strings.Builder) { + // generate the type of the handler + fmt.Fprintf(sb, "type handleClientCall%s struct {\n", d.Name) + fmt.Fprint(sb, "\taccept string\n") + fmt.Fprint(sb, "\tbody []byte\n") + fmt.Fprint(sb, "\tcontentType string\n") + fmt.Fprint(sb, "\tcount int32\n") + fmt.Fprint(sb, "\tmethod string\n") + fmt.Fprint(sb, "\tmu sync.Mutex\n") + fmt.Fprintf(sb, "\tresp %s\n", d.ResponseTypeName()) + fmt.Fprint(sb, "\turl *url.URL\n") + fmt.Fprint(sb, "\tuserAgent string\n") + fmt.Fprint(sb, "}\n\n") + + // generate the handling function + fmt.Fprintf(sb, + "func (h *handleClientCall%s) ServeHTTP(w http.ResponseWriter, r *http.Request) {", + d.Name) + fmt.Fprint(sb, "\tff := fakeFill{}\n") + if d.RequiresLogin { + fmt.Fprintf(sb, "\tif r.URL.Path == \"/api/v1/register\" {\n") + fmt.Fprintf(sb, "\t\tvar out apimodel.RegisterResponse\n") + fmt.Fprintf(sb, "\t\tff.fill(&out)\n") + fmt.Fprintf(sb, "\t\tdata, err := json.Marshal(out)\n") + fmt.Fprintf(sb, "\t\tif err != nil {\n") + fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n") + fmt.Fprintf(sb, "\t\t\treturn\n") + fmt.Fprintf(sb, "\t\t}\n") + fmt.Fprintf(sb, "\t\tw.Write(data)\n") + fmt.Fprintf(sb, "\t\treturn\n") + fmt.Fprintf(sb, "\t}\n") + fmt.Fprintf(sb, "\tif r.URL.Path == \"/api/v1/login\" {\n") + fmt.Fprintf(sb, "\t\tvar out apimodel.LoginResponse\n") + fmt.Fprintf(sb, "\t\tff.fill(&out)\n") + fmt.Fprintf(sb, "\t\tdata, err := json.Marshal(out)\n") + fmt.Fprintf(sb, "\t\tif err != nil {\n") + fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n") + fmt.Fprintf(sb, "\t\t\treturn\n") + fmt.Fprintf(sb, "\t\t}\n") + fmt.Fprintf(sb, "\t\tw.Write(data)\n") + fmt.Fprintf(sb, "\t\treturn\n") + fmt.Fprintf(sb, "\t}\n") + } + fmt.Fprint(sb, "\tdefer h.mu.Unlock()\n") + fmt.Fprint(sb, "\th.mu.Lock()\n") + fmt.Fprint(sb, "\tif h.count > 0 {\n") + fmt.Fprint(sb, "\t\tw.WriteHeader(400)\n") + fmt.Fprint(sb, "\t\treturn\n") + fmt.Fprint(sb, "\t}\n") + fmt.Fprint(sb, "\th.count++\n") + fmt.Fprint(sb, "\tif r.Body != nil {\n") + fmt.Fprint(sb, "\t\tdata, err := ioutil.ReadAll(r.Body)\n") + fmt.Fprint(sb, "\t\tif err != nil {\n") + fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n") + fmt.Fprintf(sb, "\t\t\treturn\n") + fmt.Fprint(sb, "\t\t}\n") + fmt.Fprint(sb, "\t\th.body = data\n") + fmt.Fprint(sb, "\t}\n") + fmt.Fprint(sb, "\th.method = r.Method\n") + fmt.Fprint(sb, "\th.url = r.URL\n") + fmt.Fprint(sb, "\th.accept = r.Header.Get(\"Accept\")\n") + fmt.Fprint(sb, "\th.contentType = r.Header.Get(\"Content-Type\")\n") + fmt.Fprint(sb, "\th.userAgent = r.Header.Get(\"User-Agent\")\n") + fmt.Fprintf(sb, "\tvar out %s\n", d.ResponseTypeName()) + fmt.Fprint(sb, "\tff.fill(&out)\n") + fmt.Fprintf(sb, "\th.resp = out\n") + fmt.Fprintf(sb, "\tdata, err := json.Marshal(out)\n") + fmt.Fprintf(sb, "\tif err != nil {\n") + fmt.Fprintf(sb, "\t\tw.WriteHeader(400)\n") + fmt.Fprintf(sb, "\t\treturn\n") + fmt.Fprintf(sb, "\t}\n") + fmt.Fprintf(sb, "\tw.Write(data)\n") + fmt.Fprintf(sb, "\t}\n\n") + + // generate the test itself + fmt.Fprintf(sb, "func Test%sClientCallRoundTrip(t *testing.T) {\n", d.Name) + + fmt.Fprint(sb, "\t// setup\n") + fmt.Fprintf(sb, "\thandler := &handleClientCall%s{}\n", d.Name) + fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n") + fmt.Fprint(sb, "\tdefer srvr.Close()\n") + fmt.Fprintf(sb, "\treq := &%s{}\n", d.RequestTypeNameAsStruct()) + fmt.Fprint(sb, "\tff := &fakeFill{}\n") + fmt.Fprint(sb, "\tff.fill(&req)\n") + fmt.Fprint(sb, "\tclnt := &Client{KVStore: &MemKVStore{}, BaseURL: srvr.URL}\n") + fmt.Fprint(sb, "\tff.fill(&clnt.UserAgent)\n") + + fmt.Fprint(sb, "\t// issue request\n") + fmt.Fprint(sb, "\tctx := context.Background()\n") + fmt.Fprintf(sb, "\tresp, err := clnt.%s(ctx, req)\n", d.Name) + fmt.Fprint(sb, "\tif err != nil {\n") + fmt.Fprint(sb, "\t\tt.Fatal(err)\n") + fmt.Fprint(sb, "\t}\n") + fmt.Fprint(sb, "\tif resp == nil {\n") + fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response here\")\n") + fmt.Fprint(sb, "\t}\n") + + fmt.Fprint(sb, "\t// compare our response and server's one\n") + fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.resp, resp); diff != \"\" {") + fmt.Fprint(sb, "\t\tt.Fatal(diff)\n") + fmt.Fprint(sb, "\t}\n") + + fmt.Fprint(sb, "\t// check whether headers are OK\n") + fmt.Fprint(sb, "\tif handler.accept != \"application/json\" {\n") + fmt.Fprint(sb, "\t\tt.Fatal(\"invalid accept header\")\n") + fmt.Fprint(sb, "\t}\n") + fmt.Fprint(sb, "\tif handler.userAgent != clnt.UserAgent {\n") + fmt.Fprint(sb, "\t\tt.Fatal(\"invalid user-agent header\")\n") + fmt.Fprint(sb, "\t}\n") + + fmt.Fprint(sb, "\t// check whether the method is OK\n") + fmt.Fprintf(sb, "\tif handler.method != \"%s\" {\n", d.Method) + fmt.Fprint(sb, "\t\tt.Fatal(\"invalid method\")\n") + fmt.Fprint(sb, "\t}\n") + + if d.Method == "POST" { + fmt.Fprint(sb, "\t// check the body\n") + fmt.Fprint(sb, "\tif handler.contentType != \"application/json\" {\n") + fmt.Fprint(sb, "\t\tt.Fatal(\"invalid content-type header\")\n") + fmt.Fprint(sb, "\t}\n") + fmt.Fprintf(sb, "\tgot := &%s{}\n", d.RequestTypeNameAsStruct()) + fmt.Fprintf(sb, "\tif err := json.Unmarshal(handler.body, &got); err != nil {\n") + fmt.Fprint(sb, "\t\tt.Fatal(err)\n") + fmt.Fprint(sb, "\t}\n") + fmt.Fprint(sb, "\tif diff := cmp.Diff(req, got); diff != \"\" {\n") + fmt.Fprint(sb, "\t\tt.Fatal(diff)\n") + fmt.Fprint(sb, "\t}\n") + } else { + fmt.Fprint(sb, "\t// check the query\n") + fmt.Fprintf(sb, "\tapi := &%s{BaseURL: srvr.URL}\n", d.APIStructName()) + fmt.Fprint(sb, "\thttpReq, err := api.newRequest(context.Background(), req)\n") + fmt.Fprint(sb, "\tif err != nil {\n") + fmt.Fprint(sb, "\t\tt.Fatal(err)\n") + fmt.Fprint(sb, "\t}\n") + fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != \"\" {\n") + fmt.Fprint(sb, "\t\tt.Fatal(diff)\n") + fmt.Fprint(sb, "\t}\n") + fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != \"\" {\n") + fmt.Fprint(sb, "\t\tt.Fatal(diff)\n") + fmt.Fprint(sb, "\t}\n") + } + + fmt.Fprint(sb, "}\n\n") +} + +// GenClientCallTestGo generates clientcall_test.go. +func GenClientCallTestGo(file string) { + var sb strings.Builder + fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n") + fmt.Fprintf(&sb, "// %s\n\n", time.Now()) + fmt.Fprint(&sb, "package ooapi\n\n") + fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file) + fmt.Fprint(&sb, "import (\n") + fmt.Fprint(&sb, "\t\"context\"\n") + fmt.Fprint(&sb, "\t\"encoding/json\"\n") + fmt.Fprint(&sb, "\t\"io/ioutil\"\n") + fmt.Fprint(&sb, "\t\"net/http/httptest\"\n") + fmt.Fprint(&sb, "\t\"net/http\"\n") + fmt.Fprint(&sb, "\t\"net/url\"\n") + fmt.Fprint(&sb, "\t\"testing\"\n") + fmt.Fprint(&sb, "\t\"sync\"\n") + fmt.Fprint(&sb, "\n") + fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n") + fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel\"\n") + fmt.Fprint(&sb, ")\n") + for _, desc := range Descriptors { + if desc.Name == "Login" || desc.Name == "Register" { + continue // they cannot be called directly + } + desc.genTestClientCallRoundTrip(&sb) + } + writefile(file, &sb) +} diff --git a/internal/engine/ooapi/internal/generator/cloners.go b/internal/engine/ooapi/internal/generator/cloners.go index 1db6d92..f271c42 100644 --- a/internal/engine/ooapi/internal/generator/cloners.go +++ b/internal/engine/ooapi/internal/generator/cloners.go @@ -8,7 +8,7 @@ import ( func (d *Descriptor) genNewCloner(sb *strings.Builder) { fmt.Fprintf(sb, "// %s represents any type exposing a method\n", - d.CallerInterfaceName()) + d.ClonerInterfaceName()) fmt.Fprintf(sb, "// like %s.WithToken.\n", d.APIStructName()) fmt.Fprintf(sb, "type %s interface {\n", d.ClonerInterfaceName()) fmt.Fprintf(sb, "\tWithToken(token string) %s\n", d.CallerInterfaceName()) diff --git a/internal/engine/ooapi/internal/generator/fakeapitest.go b/internal/engine/ooapi/internal/generator/fakeapitest.go index deb2b56..48b12c2 100644 --- a/internal/engine/ooapi/internal/generator/fakeapitest.go +++ b/internal/engine/ooapi/internal/generator/fakeapitest.go @@ -7,7 +7,7 @@ import ( ) func (d *Descriptor) genNewFakeAPI(sb *strings.Builder) { - fmt.Fprintf(sb, "type Fake%s struct {\n", d.APIStructName()) + fmt.Fprintf(sb, "type %s struct {\n", d.FakeAPIStructName()) if d.RequiresLogin { fmt.Fprintf(sb, "\tWithResult %s\n", d.CallerInterfaceName()) } @@ -16,25 +16,25 @@ func (d *Descriptor) genNewFakeAPI(sb *strings.Builder) { fmt.Fprint(sb, "\tCountCall int32\n") fmt.Fprint(sb, "}\n\n") - fmt.Fprintf(sb, "func (fapi *Fake%s) Call(ctx context.Context, req %s) (%s, error) {\n", - d.APIStructName(), d.RequestTypeName(), d.ResponseTypeName()) + fmt.Fprintf(sb, "func (fapi *%s) Call(ctx context.Context, req %s) (%s, error) {\n", + d.FakeAPIStructName(), d.RequestTypeName(), d.ResponseTypeName()) fmt.Fprint(sb, "\tatomic.AddInt32(&fapi.CountCall, 1)\n") fmt.Fprint(sb, "\treturn fapi.Response, fapi.Err\n") fmt.Fprint(sb, "}\n\n") if d.RequiresLogin { - fmt.Fprintf(sb, "func (fapi *Fake%s) WithToken(token string) %s {\n", - d.APIStructName(), d.CallerInterfaceName()) + fmt.Fprintf(sb, "func (fapi *%s) WithToken(token string) %s {\n", + d.FakeAPIStructName(), d.CallerInterfaceName()) fmt.Fprint(sb, "\treturn fapi.WithResult\n") fmt.Fprint(sb, "}\n\n") } fmt.Fprint(sb, "var (\n") - fmt.Fprintf(sb, "\t_ %s = &Fake%s{}\n", d.CallerInterfaceName(), - d.APIStructName()) + fmt.Fprintf(sb, "\t_ %s = &%s{}\n", d.CallerInterfaceName(), + d.FakeAPIStructName()) if d.RequiresLogin { - fmt.Fprintf(sb, "\t_ %s = &Fake%s{}\n", d.ClonerInterfaceName(), - d.APIStructName()) + fmt.Fprintf(sb, "\t_ %s = &%s{}\n", d.ClonerInterfaceName(), + d.FakeAPIStructName()) } fmt.Fprint(sb, ")\n\n") } diff --git a/internal/engine/ooapi/internal/generator/generator.go b/internal/engine/ooapi/internal/generator/generator.go index 6158767..be621ad 100644 --- a/internal/engine/ooapi/internal/generator/generator.go +++ b/internal/engine/ooapi/internal/generator/generator.go @@ -47,6 +47,10 @@ func main() { GenCachingTestGo(file) case "login_test.go": GenLoginTestGo(file) + case "clientcall.go": + GenClientCallGo(file) + case "clientcall_test.go": + GenClientCallTestGo(file) default: panic(fmt.Sprintf("don't know how to create this file: %s", file)) } diff --git a/internal/engine/ooapi/internal/generator/login.go b/internal/engine/ooapi/internal/generator/login.go index 4328d2e..61dedba 100644 --- a/internal/engine/ooapi/internal/generator/login.go +++ b/internal/engine/ooapi/internal/generator/login.go @@ -13,8 +13,8 @@ func (d *Descriptor) genNewLogin(sb *strings.Builder) { fmt.Fprintf(sb, "\tAPI %s // mandatory\n", d.ClonerInterfaceName()) fmt.Fprint(sb, "\tJSONCodec JSONCodec // optional\n") fmt.Fprint(sb, "\tKVStore KVStore // mandatory\n") - fmt.Fprint(sb, "\tRegisterAPI RegisterCaller // mandatory\n") - fmt.Fprint(sb, "\tLoginAPI LoginCaller // mandatory\n") + fmt.Fprint(sb, "\tRegisterAPI callerForRegisterAPI // mandatory\n") + fmt.Fprint(sb, "\tLoginAPI callerForLoginAPI // mandatory\n") fmt.Fprint(sb, "}\n\n") fmt.Fprintf(sb, "// Call logins, if needed, then calls the API.\n") diff --git a/internal/engine/ooapi/internal/generator/logintest.go b/internal/engine/ooapi/internal/generator/logintest.go index d74c9f5..7960f3c 100644 --- a/internal/engine/ooapi/internal/generator/logintest.go +++ b/internal/engine/ooapi/internal/generator/logintest.go @@ -7,7 +7,7 @@ import ( ) func (d *Descriptor) genTestRegisterAndLoginSuccess(sb *strings.Builder) { - fmt.Fprintf(sb, "func TestRegisterAndLogin%sSuccess(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func TestRegisterAndLogin%sSuccess(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") @@ -26,14 +26,14 @@ func (d *Descriptor) genTestRegisterAndLoginSuccess(sb *strings.Builder) { fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) - fmt.Fprintf(sb, "\t\t\tWithResult: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) + fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n") fmt.Fprint(sb, "\t\t\t},\n") fmt.Fprint(sb, "\t\t},\n") fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -62,7 +62,7 @@ func (d *Descriptor) genTestRegisterAndLoginSuccess(sb *strings.Builder) { } func (d *Descriptor) genTestContinueUsingToken(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sContinueUsingToken(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sContinueUsingToken(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") @@ -81,14 +81,14 @@ func (d *Descriptor) genTestContinueUsingToken(sb *strings.Builder) { fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) - fmt.Fprintf(sb, "\t\t\tWithResult: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) + fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n") fmt.Fprint(sb, "\t\t\t},\n") fmt.Fprint(sb, "\t\t},\n") fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -150,7 +150,7 @@ func (d *Descriptor) genTestContinueUsingToken(sb *strings.Builder) { } func (d *Descriptor) genTestWithValidButExpiredToken(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sWithValidButExpiredToken(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sWithValidButExpiredToken(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") @@ -168,14 +168,14 @@ func (d *Descriptor) genTestWithValidButExpiredToken(sb *strings.Builder) { fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) - fmt.Fprintf(sb, "\t\t\tWithResult: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) + fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n") fmt.Fprint(sb, "\t\t\t},\n") fmt.Fprint(sb, "\t\t},\n") fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tls := &loginState{\n") @@ -214,7 +214,7 @@ func (d *Descriptor) genTestWithValidButExpiredToken(sb *strings.Builder) { } func (d *Descriptor) genTestWithRegisterAPIError(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sWithRegisterAPIError(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sWithRegisterAPIError(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") @@ -225,13 +225,13 @@ func (d *Descriptor) genTestWithRegisterAPIError(sb *strings.Builder) { fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) - fmt.Fprintf(sb, "\t\t\tWithResult: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) + fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n") fmt.Fprint(sb, "\t\t\t},\n") fmt.Fprint(sb, "\t\t},\n") fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -253,7 +253,7 @@ func (d *Descriptor) genTestWithRegisterAPIError(sb *strings.Builder) { } func (d *Descriptor) genTestWithLoginFailure(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sWithLoginFailure(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sWithLoginFailure(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") @@ -270,14 +270,14 @@ func (d *Descriptor) genTestWithLoginFailure(sb *strings.Builder) { fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) - fmt.Fprintf(sb, "\t\t\tWithResult: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) + fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n") fmt.Fprint(sb, "\t\t\t},\n") fmt.Fprint(sb, "\t\t},\n") fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -303,7 +303,7 @@ func (d *Descriptor) genTestWithLoginFailure(sb *strings.Builder) { } func (d *Descriptor) genTestRegisterAndLoginThenFail(sb *strings.Builder) { - fmt.Fprintf(sb, "func TestRegisterAndLogin%sThenFail(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func TestRegisterAndLogin%sThenFail(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") @@ -323,14 +323,14 @@ func (d *Descriptor) genTestRegisterAndLoginThenFail(sb *strings.Builder) { fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n") fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) - fmt.Fprintf(sb, "\t\t\tWithResult: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) + fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\t\tErr: errMocked,\n") fmt.Fprint(sb, "\t\t\t},\n") fmt.Fprint(sb, "\t\t},\n") fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -356,22 +356,22 @@ func (d *Descriptor) genTestRegisterAndLoginThenFail(sb *strings.Builder) { } func (d *Descriptor) genTestTheDatabaseIsReplaced(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sTheDatabaseIsReplaced(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sTheDatabaseIsReplaced(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprint(sb, "\thandler := &LoginHandler{t: t}\n") fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n") fmt.Fprint(sb, "\tdefer srvr.Close()\n") - fmt.Fprint(sb, "\tregisterAPI := &RegisterAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") - fmt.Fprint(sb, "\t\tloginAPI := &LoginAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName()) - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") @@ -379,7 +379,7 @@ func (d *Descriptor) genTestTheDatabaseIsReplaced(sb *strings.Builder) { fmt.Fprintf(sb, "\tAPI : baseAPI,\n") fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -430,22 +430,22 @@ func (d *Descriptor) genTestTheDatabaseIsReplaced(sb *strings.Builder) { } func (d *Descriptor) genTestTheDatabaseIsReplacedThenFailure(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sTheDatabaseIsReplacedThenFailure(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sTheDatabaseIsReplacedThenFailure(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprint(sb, "\thandler := &LoginHandler{t: t}\n") fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n") fmt.Fprint(sb, "\tdefer srvr.Close()\n") - fmt.Fprint(sb, "\tregisterAPI := &RegisterAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") - fmt.Fprint(sb, "\t\tloginAPI := &LoginAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName()) - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") @@ -453,7 +453,7 @@ func (d *Descriptor) genTestTheDatabaseIsReplacedThenFailure(sb *strings.Builder fmt.Fprintf(sb, "\tAPI : baseAPI,\n") fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -506,7 +506,7 @@ func (d *Descriptor) genTestTheDatabaseIsReplacedThenFailure(sb *strings.Builder } func (d *Descriptor) genTestRegisterAndLoginCannotWriteState(sb *strings.Builder) { - fmt.Fprintf(sb, "func TestRegisterAndLogin%sCannotWriteState(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func TestRegisterAndLogin%sCannotWriteState(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") @@ -526,14 +526,14 @@ func (d *Descriptor) genTestRegisterAndLoginCannotWriteState(sb *strings.Builder fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n") fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName()) - fmt.Fprintf(sb, "\t\tAPI: &Fake%s{\n", d.APIStructName()) - fmt.Fprintf(sb, "\t\t\tWithResult: &Fake%s{\n", d.APIStructName()) + fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName()) + fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName()) fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n") fmt.Fprint(sb, "\t\t\t},\n") fmt.Fprint(sb, "\t\t},\n") fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t\tJSONCodec: &FakeCodec{\n") fmt.Fprint(sb, "\t\t\tEncodeErr: errMocked,\n") fmt.Fprint(sb, "\t\t},\n") @@ -562,7 +562,7 @@ func (d *Descriptor) genTestRegisterAndLoginCannotWriteState(sb *strings.Builder } func (d *Descriptor) genTestReadStateDecodeFailure(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sReadStateDecodeFailure(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sReadStateDecodeFailure(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName()) fmt.Fprint(sb, "\tff.fill(&expect)\n") @@ -570,7 +570,7 @@ func (d *Descriptor) genTestReadStateDecodeFailure(sb *strings.Builder) { fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n") fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName()) - fmt.Fprint(sb, "\t\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\t\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t\tJSONCodec: &FakeCodec{DecodeErr: errMocked},\n") fmt.Fprint(sb, "\t}\n") @@ -596,22 +596,22 @@ func (d *Descriptor) genTestReadStateDecodeFailure(sb *strings.Builder) { } func (d *Descriptor) genTestClockIsOffThenSuccess(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sClockIsOffThenSuccess(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sClockIsOffThenSuccess(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprint(sb, "\thandler := &LoginHandler{t: t}\n") fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n") fmt.Fprint(sb, "\tdefer srvr.Close()\n") - fmt.Fprint(sb, "\tregisterAPI := &RegisterAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") - fmt.Fprint(sb, "\t\tloginAPI := &LoginAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName()) - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") @@ -619,7 +619,7 @@ func (d *Descriptor) genTestClockIsOffThenSuccess(sb *strings.Builder) { fmt.Fprintf(sb, "\tAPI : baseAPI,\n") fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -672,22 +672,22 @@ func (d *Descriptor) genTestClockIsOffThenSuccess(sb *strings.Builder) { } func (d *Descriptor) genTestClockIsOffThen401(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sClockIsOffThen401(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sClockIsOffThen401(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprint(sb, "\thandler := &LoginHandler{t: t}\n") fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n") fmt.Fprint(sb, "\tdefer srvr.Close()\n") - fmt.Fprint(sb, "\tregisterAPI := &RegisterAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") - fmt.Fprint(sb, "\t\tloginAPI := &LoginAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName()) - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") @@ -695,7 +695,7 @@ func (d *Descriptor) genTestClockIsOffThen401(sb *strings.Builder) { fmt.Fprintf(sb, "\tAPI : baseAPI,\n") fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) @@ -749,22 +749,22 @@ func (d *Descriptor) genTestClockIsOffThen401(sb *strings.Builder) { } func (d *Descriptor) genTestClockIsOffThen500(sb *strings.Builder) { - fmt.Fprintf(sb, "func Test%sClockIsOffThen500(t *testing.T) {\n", d.APIStructName()) + fmt.Fprintf(sb, "func Test%sClockIsOffThen500(t *testing.T) {\n", d.Name) fmt.Fprint(sb, "\tff := &fakeFill{}\n") fmt.Fprint(sb, "\thandler := &LoginHandler{t: t}\n") fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n") fmt.Fprint(sb, "\tdefer srvr.Close()\n") - fmt.Fprint(sb, "\tregisterAPI := &RegisterAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") - fmt.Fprint(sb, "\t\tloginAPI := &LoginAPI{\n") - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t\t}\n") fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName()) - fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{t: t},\n") + fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n") fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n") fmt.Fprint(sb, "\t}\n") @@ -772,7 +772,7 @@ func (d *Descriptor) genTestClockIsOffThen500(sb *strings.Builder) { fmt.Fprintf(sb, "\tAPI : baseAPI,\n") fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n") fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n") - fmt.Fprint(sb, "\tKVStore: &memkvstore{},\n") + fmt.Fprint(sb, "\tKVStore: &MemKVStore{},\n") fmt.Fprint(sb, "\t}\n") fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName()) diff --git a/internal/engine/ooapi/internal/generator/reflect.go b/internal/engine/ooapi/internal/generator/reflect.go index 9b2f27b..767dbce 100644 --- a/internal/engine/ooapi/internal/generator/reflect.go +++ b/internal/engine/ooapi/internal/generator/reflect.go @@ -23,37 +23,43 @@ func (d *Descriptor) ResponseTypeName() string { // APIStructName returns the correct struct type name // for the API we're currently processing. func (d *Descriptor) APIStructName() string { - return fmt.Sprintf("%sAPI", d.Name) + return fmt.Sprintf("simple%sAPI", d.Name) +} + +// FakeAPIStructName returns the correct struct type name +// for the fake for the API we're currently processing. +func (d *Descriptor) FakeAPIStructName() string { + return fmt.Sprintf("Fake%sAPI", d.Name) } // WithLoginAPIStructName returns the correct struct type name // for the WithLoginAPI we're currently processing. func (d *Descriptor) WithLoginAPIStructName() string { - return fmt.Sprintf("%sAPIWithLogin", d.Name) + return fmt.Sprintf("withLogin%sAPI", d.Name) } // CallerInterfaceName returns the correct caller interface name // for the API we're currently processing. func (d *Descriptor) CallerInterfaceName() string { - return fmt.Sprintf("%sCaller", d.Name) + return fmt.Sprintf("callerFor%sAPI", d.Name) } // ClonerInterfaceName returns the correct cloner interface name // for the API we're currently processing. func (d *Descriptor) ClonerInterfaceName() string { - return fmt.Sprintf("%sCloner", d.Name) + return fmt.Sprintf("clonerFor%sAPI", d.Name) } -// CacheStructName returns the correct struct type name for +// WithCacheAPIStructName returns the correct struct type name for // the cache for the API we're currently processing. -func (d *Descriptor) CacheStructName() string { - return fmt.Sprintf("%sCache", d.Name) +func (d *Descriptor) WithCacheAPIStructName() string { + return fmt.Sprintf("withCache%sAPI", d.Name) } // CacheEntryName returns the correct struct type name for the // cache entry for the API we're currently processing. func (d *Descriptor) CacheEntryName() string { - return fmt.Sprintf("cacheEntryFor%s", d.Name) + return fmt.Sprintf("cacheEntryFor%sAPI", d.Name) } // CacheKey returns the correct cache key for the API diff --git a/internal/engine/ooapi/kvstore_test.go b/internal/engine/ooapi/kvstore_test.go index 31c7e71..625c9d7 100644 --- a/internal/engine/ooapi/kvstore_test.go +++ b/internal/engine/ooapi/kvstore_test.go @@ -8,12 +8,12 @@ import ( var errMemkvstoreNotFound = errors.New("memkvstore: not found") -type memkvstore struct { +type MemKVStore struct { m map[string][]byte mu sync.Mutex } -func (kvs *memkvstore) Get(key string) ([]byte, error) { +func (kvs *MemKVStore) Get(key string) ([]byte, error) { defer kvs.mu.Unlock() kvs.mu.Lock() out, good := kvs.m[key] @@ -23,7 +23,7 @@ func (kvs *memkvstore) Get(key string) ([]byte, error) { return out, nil } -func (kvs *memkvstore) Set(key string, value []byte) error { +func (kvs *MemKVStore) Set(key string, value []byte) error { defer kvs.mu.Unlock() kvs.mu.Lock() if kvs.m == nil { diff --git a/internal/engine/ooapi/login.go b/internal/engine/ooapi/login.go index 49f9833..53a1a72 100644 --- a/internal/engine/ooapi/login.go +++ b/internal/engine/ooapi/login.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:52.62521737 +0100 CET m=+0.000161706 +// 2021-03-10 13:17:34.342751918 +0100 CET m=+0.000196026 package ooapi @@ -12,17 +12,17 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -// PsiphonConfigAPIWithLogin implements login for PsiphonConfigAPI. -type PsiphonConfigAPIWithLogin struct { - API PsiphonConfigCloner // mandatory - JSONCodec JSONCodec // optional - KVStore KVStore // mandatory - RegisterAPI RegisterCaller // mandatory - LoginAPI LoginCaller // mandatory +// withLoginPsiphonConfigAPI implements login for simplePsiphonConfigAPI. +type withLoginPsiphonConfigAPI struct { + API clonerForPsiphonConfigAPI // mandatory + JSONCodec JSONCodec // optional + KVStore KVStore // mandatory + RegisterAPI callerForRegisterAPI // mandatory + LoginAPI callerForLoginAPI // mandatory } // Call logins, if needed, then calls the API. -func (api *PsiphonConfigAPIWithLogin) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) { +func (api *withLoginPsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) { token, err := api.maybeLogin(ctx) if err != nil { return nil, err @@ -56,14 +56,14 @@ func (api *PsiphonConfigAPIWithLogin) Call(ctx context.Context, req *apimodel.Ps return resp, nil } -func (api *PsiphonConfigAPIWithLogin) jsonCodec() JSONCodec { +func (api *withLoginPsiphonConfigAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *PsiphonConfigAPIWithLogin) readstate() (*loginState, error) { +func (api *withLoginPsiphonConfigAPI) readstate() (*loginState, error) { data, err := api.KVStore.Get(loginKey) if err != nil { return nil, err @@ -75,7 +75,7 @@ func (api *PsiphonConfigAPIWithLogin) readstate() (*loginState, error) { return &ls, nil } -func (api *PsiphonConfigAPIWithLogin) writestate(ls *loginState) error { +func (api *withLoginPsiphonConfigAPI) writestate(ls *loginState) error { data, err := api.jsonCodec().Encode(*ls) if err != nil { return err @@ -83,7 +83,7 @@ func (api *PsiphonConfigAPIWithLogin) writestate(ls *loginState) error { return api.KVStore.Set(loginKey, data) } -func (api *PsiphonConfigAPIWithLogin) doRegister(ctx context.Context, password string) (string, error) { +func (api *withLoginPsiphonConfigAPI) doRegister(ctx context.Context, password string) (string, error) { req := newRegisterRequest(password) ls := &loginState{} resp, err := api.RegisterAPI.Call(ctx, req) @@ -95,7 +95,7 @@ func (api *PsiphonConfigAPIWithLogin) doRegister(ctx context.Context, password s return api.doLogin(ctx, ls) } -func (api *PsiphonConfigAPIWithLogin) forceRegister(ctx context.Context) (string, error) { +func (api *withLoginPsiphonConfigAPI) forceRegister(ctx context.Context) (string, error) { var password string // If we already have a previous password, let us keep // using it. This will allow a new version of the API to @@ -115,7 +115,7 @@ func (api *PsiphonConfigAPIWithLogin) forceRegister(ctx context.Context) (string return api.doRegister(ctx, password) } -func (api *PsiphonConfigAPIWithLogin) forceLogin(ctx context.Context) (string, error) { +func (api *withLoginPsiphonConfigAPI) forceLogin(ctx context.Context) (string, error) { ls, err := api.readstate() if err != nil { return "", err @@ -123,7 +123,7 @@ func (api *PsiphonConfigAPIWithLogin) forceLogin(ctx context.Context) (string, e return api.doLogin(ctx, ls) } -func (api *PsiphonConfigAPIWithLogin) maybeLogin(ctx context.Context) (string, error) { +func (api *withLoginPsiphonConfigAPI) maybeLogin(ctx context.Context) (string, error) { ls, _ := api.readstate() if ls == nil || !ls.credentialsValid() { return api.forceRegister(ctx) @@ -134,7 +134,7 @@ func (api *PsiphonConfigAPIWithLogin) maybeLogin(ctx context.Context) (string, e return ls.Token, nil } -func (api *PsiphonConfigAPIWithLogin) doLogin(ctx context.Context, ls *loginState) (string, error) { +func (api *withLoginPsiphonConfigAPI) doLogin(ctx context.Context, ls *loginState) (string, error) { req := &apimodel.LoginRequest{ ClientID: ls.ClientID, Password: ls.Password, @@ -151,19 +151,19 @@ func (api *PsiphonConfigAPIWithLogin) doLogin(ctx context.Context, ls *loginStat return ls.Token, nil } -var _ PsiphonConfigCaller = &PsiphonConfigAPIWithLogin{} +var _ callerForPsiphonConfigAPI = &withLoginPsiphonConfigAPI{} -// TorTargetsAPIWithLogin implements login for TorTargetsAPI. -type TorTargetsAPIWithLogin struct { - API TorTargetsCloner // mandatory - JSONCodec JSONCodec // optional - KVStore KVStore // mandatory - RegisterAPI RegisterCaller // mandatory - LoginAPI LoginCaller // mandatory +// withLoginTorTargetsAPI implements login for simpleTorTargetsAPI. +type withLoginTorTargetsAPI struct { + API clonerForTorTargetsAPI // mandatory + JSONCodec JSONCodec // optional + KVStore KVStore // mandatory + RegisterAPI callerForRegisterAPI // mandatory + LoginAPI callerForLoginAPI // mandatory } // Call logins, if needed, then calls the API. -func (api *TorTargetsAPIWithLogin) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) { +func (api *withLoginTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) { token, err := api.maybeLogin(ctx) if err != nil { return nil, err @@ -197,14 +197,14 @@ func (api *TorTargetsAPIWithLogin) Call(ctx context.Context, req *apimodel.TorTa return resp, nil } -func (api *TorTargetsAPIWithLogin) jsonCodec() JSONCodec { +func (api *withLoginTorTargetsAPI) jsonCodec() JSONCodec { if api.JSONCodec != nil { return api.JSONCodec } return &defaultJSONCodec{} } -func (api *TorTargetsAPIWithLogin) readstate() (*loginState, error) { +func (api *withLoginTorTargetsAPI) readstate() (*loginState, error) { data, err := api.KVStore.Get(loginKey) if err != nil { return nil, err @@ -216,7 +216,7 @@ func (api *TorTargetsAPIWithLogin) readstate() (*loginState, error) { return &ls, nil } -func (api *TorTargetsAPIWithLogin) writestate(ls *loginState) error { +func (api *withLoginTorTargetsAPI) writestate(ls *loginState) error { data, err := api.jsonCodec().Encode(*ls) if err != nil { return err @@ -224,7 +224,7 @@ func (api *TorTargetsAPIWithLogin) writestate(ls *loginState) error { return api.KVStore.Set(loginKey, data) } -func (api *TorTargetsAPIWithLogin) doRegister(ctx context.Context, password string) (string, error) { +func (api *withLoginTorTargetsAPI) doRegister(ctx context.Context, password string) (string, error) { req := newRegisterRequest(password) ls := &loginState{} resp, err := api.RegisterAPI.Call(ctx, req) @@ -236,7 +236,7 @@ func (api *TorTargetsAPIWithLogin) doRegister(ctx context.Context, password stri return api.doLogin(ctx, ls) } -func (api *TorTargetsAPIWithLogin) forceRegister(ctx context.Context) (string, error) { +func (api *withLoginTorTargetsAPI) forceRegister(ctx context.Context) (string, error) { var password string // If we already have a previous password, let us keep // using it. This will allow a new version of the API to @@ -256,7 +256,7 @@ func (api *TorTargetsAPIWithLogin) forceRegister(ctx context.Context) (string, e return api.doRegister(ctx, password) } -func (api *TorTargetsAPIWithLogin) forceLogin(ctx context.Context) (string, error) { +func (api *withLoginTorTargetsAPI) forceLogin(ctx context.Context) (string, error) { ls, err := api.readstate() if err != nil { return "", err @@ -264,7 +264,7 @@ func (api *TorTargetsAPIWithLogin) forceLogin(ctx context.Context) (string, erro return api.doLogin(ctx, ls) } -func (api *TorTargetsAPIWithLogin) maybeLogin(ctx context.Context) (string, error) { +func (api *withLoginTorTargetsAPI) maybeLogin(ctx context.Context) (string, error) { ls, _ := api.readstate() if ls == nil || !ls.credentialsValid() { return api.forceRegister(ctx) @@ -275,7 +275,7 @@ func (api *TorTargetsAPIWithLogin) maybeLogin(ctx context.Context) (string, erro return ls.Token, nil } -func (api *TorTargetsAPIWithLogin) doLogin(ctx context.Context, ls *loginState) (string, error) { +func (api *withLoginTorTargetsAPI) doLogin(ctx context.Context, ls *loginState) (string, error) { req := &apimodel.LoginRequest{ ClientID: ls.ClientID, Password: ls.Password, @@ -292,4 +292,4 @@ func (api *TorTargetsAPIWithLogin) doLogin(ctx context.Context, ls *loginState) return ls.Token, nil } -var _ TorTargetsCaller = &TorTargetsAPIWithLogin{} +var _ callerForTorTargetsAPI = &withLoginTorTargetsAPI{} diff --git a/internal/engine/ooapi/login_test.go b/internal/engine/ooapi/login_test.go index c5b3e44..209d1ec 100644 --- a/internal/engine/ooapi/login_test.go +++ b/internal/engine/ooapi/login_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:52.9205436 +0100 CET m=+0.000137951 +// 2021-03-10 13:17:34.605701732 +0100 CET m=+0.000131680 package ooapi @@ -16,7 +16,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -func TestRegisterAndLoginPsiphonConfigAPISuccess(t *testing.T) { +func TestRegisterAndLoginPsiphonConfigSuccess(t *testing.T) { ff := &fakeFill{} var expect apimodel.PsiphonConfigResponse ff.fill(&expect) @@ -31,7 +31,7 @@ func TestRegisterAndLoginPsiphonConfigAPISuccess(t *testing.T) { Token: "antani-antani-token", }, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: &FakePsiphonConfigAPI{ WithResult: &FakePsiphonConfigAPI{ Response: expect, @@ -39,7 +39,7 @@ func TestRegisterAndLoginPsiphonConfigAPISuccess(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -62,7 +62,7 @@ func TestRegisterAndLoginPsiphonConfigAPISuccess(t *testing.T) { } } -func TestPsiphonConfigAPIContinueUsingToken(t *testing.T) { +func TestPsiphonConfigContinueUsingToken(t *testing.T) { ff := &fakeFill{} var expect apimodel.PsiphonConfigResponse ff.fill(&expect) @@ -77,7 +77,7 @@ func TestPsiphonConfigAPIContinueUsingToken(t *testing.T) { Token: "antani-antani-token", }, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: &FakePsiphonConfigAPI{ WithResult: &FakePsiphonConfigAPI{ Response: expect, @@ -85,7 +85,7 @@ func TestPsiphonConfigAPIContinueUsingToken(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -135,7 +135,7 @@ func TestPsiphonConfigAPIContinueUsingToken(t *testing.T) { } } -func TestPsiphonConfigAPIWithValidButExpiredToken(t *testing.T) { +func TestPsiphonConfigWithValidButExpiredToken(t *testing.T) { ff := &fakeFill{} var expect apimodel.PsiphonConfigResponse ff.fill(&expect) @@ -149,7 +149,7 @@ func TestPsiphonConfigAPIWithValidButExpiredToken(t *testing.T) { Token: "antani-antani-token", }, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: &FakePsiphonConfigAPI{ WithResult: &FakePsiphonConfigAPI{ Response: expect, @@ -157,7 +157,7 @@ func TestPsiphonConfigAPIWithValidButExpiredToken(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } ls := &loginState{ ClientID: "antani-antani", @@ -189,7 +189,7 @@ func TestPsiphonConfigAPIWithValidButExpiredToken(t *testing.T) { } } -func TestPsiphonConfigAPIWithRegisterAPIError(t *testing.T) { +func TestPsiphonConfigWithRegisterAPIError(t *testing.T) { ff := &fakeFill{} var expect apimodel.PsiphonConfigResponse ff.fill(&expect) @@ -197,14 +197,14 @@ func TestPsiphonConfigAPIWithRegisterAPIError(t *testing.T) { registerAPI := &FakeRegisterAPI{ Err: errMocked, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: &FakePsiphonConfigAPI{ WithResult: &FakePsiphonConfigAPI{ Response: expect, }, }, RegisterAPI: registerAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -221,7 +221,7 @@ func TestPsiphonConfigAPIWithRegisterAPIError(t *testing.T) { } } -func TestPsiphonConfigAPIWithLoginFailure(t *testing.T) { +func TestPsiphonConfigWithLoginFailure(t *testing.T) { ff := &fakeFill{} var expect apimodel.PsiphonConfigResponse ff.fill(&expect) @@ -234,7 +234,7 @@ func TestPsiphonConfigAPIWithLoginFailure(t *testing.T) { loginAPI := &FakeLoginAPI{ Err: errMocked, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: &FakePsiphonConfigAPI{ WithResult: &FakePsiphonConfigAPI{ Response: expect, @@ -242,7 +242,7 @@ func TestPsiphonConfigAPIWithLoginFailure(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -262,7 +262,7 @@ func TestPsiphonConfigAPIWithLoginFailure(t *testing.T) { } } -func TestRegisterAndLoginPsiphonConfigAPIThenFail(t *testing.T) { +func TestRegisterAndLoginPsiphonConfigThenFail(t *testing.T) { ff := &fakeFill{} var expect apimodel.PsiphonConfigResponse ff.fill(&expect) @@ -278,7 +278,7 @@ func TestRegisterAndLoginPsiphonConfigAPIThenFail(t *testing.T) { }, } errMocked := errors.New("mocked error") - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: &FakePsiphonConfigAPI{ WithResult: &FakePsiphonConfigAPI{ Err: errMocked, @@ -286,7 +286,7 @@ func TestRegisterAndLoginPsiphonConfigAPIThenFail(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -306,28 +306,28 @@ func TestRegisterAndLoginPsiphonConfigAPIThenFail(t *testing.T) { } } -func TestPsiphonConfigAPITheDatabaseIsReplaced(t *testing.T) { +func TestPsiphonConfigTheDatabaseIsReplaced(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &PsiphonConfigAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simplePsiphonConfigAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -366,7 +366,7 @@ func TestPsiphonConfigAPITheDatabaseIsReplaced(t *testing.T) { } } -func TestRegisterAndLoginPsiphonConfigAPICannotWriteState(t *testing.T) { +func TestRegisterAndLoginPsiphonConfigCannotWriteState(t *testing.T) { ff := &fakeFill{} var expect apimodel.PsiphonConfigResponse ff.fill(&expect) @@ -382,7 +382,7 @@ func TestRegisterAndLoginPsiphonConfigAPICannotWriteState(t *testing.T) { }, } errMocked := errors.New("mocked error") - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: &FakePsiphonConfigAPI{ WithResult: &FakePsiphonConfigAPI{ Response: expect, @@ -390,7 +390,7 @@ func TestRegisterAndLoginPsiphonConfigAPICannotWriteState(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, JSONCodec: &FakeCodec{ EncodeErr: errMocked, }, @@ -413,13 +413,13 @@ func TestRegisterAndLoginPsiphonConfigAPICannotWriteState(t *testing.T) { } } -func TestPsiphonConfigAPIReadStateDecodeFailure(t *testing.T) { +func TestPsiphonConfigReadStateDecodeFailure(t *testing.T) { ff := &fakeFill{} var expect apimodel.PsiphonConfigResponse ff.fill(&expect) errMocked := errors.New("mocked error") - login := &PsiphonConfigAPIWithLogin{ - KVStore: &memkvstore{}, + login := &withLoginPsiphonConfigAPI{ + KVStore: &MemKVStore{}, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } ls := &loginState{ @@ -440,28 +440,28 @@ func TestPsiphonConfigAPIReadStateDecodeFailure(t *testing.T) { } } -func TestPsiphonConfigAPITheDatabaseIsReplacedThenFailure(t *testing.T) { +func TestPsiphonConfigTheDatabaseIsReplacedThenFailure(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &PsiphonConfigAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simplePsiphonConfigAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -502,28 +502,28 @@ func TestPsiphonConfigAPITheDatabaseIsReplacedThenFailure(t *testing.T) { } } -func TestPsiphonConfigAPIClockIsOffThenSuccess(t *testing.T) { +func TestPsiphonConfigClockIsOffThenSuccess(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &PsiphonConfigAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simplePsiphonConfigAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -564,28 +564,28 @@ func TestPsiphonConfigAPIClockIsOffThenSuccess(t *testing.T) { } } -func TestPsiphonConfigAPIClockIsOffThen401(t *testing.T) { +func TestPsiphonConfigClockIsOffThen401(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &PsiphonConfigAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simplePsiphonConfigAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -627,28 +627,28 @@ func TestPsiphonConfigAPIClockIsOffThen401(t *testing.T) { } } -func TestPsiphonConfigAPIClockIsOffThen500(t *testing.T) { +func TestPsiphonConfigClockIsOffThen500(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &PsiphonConfigAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simplePsiphonConfigAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &PsiphonConfigAPIWithLogin{ + login := &withLoginPsiphonConfigAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.PsiphonConfigRequest ff.fill(&req) @@ -690,7 +690,7 @@ func TestPsiphonConfigAPIClockIsOffThen500(t *testing.T) { } } -func TestRegisterAndLoginTorTargetsAPISuccess(t *testing.T) { +func TestRegisterAndLoginTorTargetsSuccess(t *testing.T) { ff := &fakeFill{} var expect apimodel.TorTargetsResponse ff.fill(&expect) @@ -705,7 +705,7 @@ func TestRegisterAndLoginTorTargetsAPISuccess(t *testing.T) { Token: "antani-antani-token", }, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: &FakeTorTargetsAPI{ WithResult: &FakeTorTargetsAPI{ Response: expect, @@ -713,7 +713,7 @@ func TestRegisterAndLoginTorTargetsAPISuccess(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -736,7 +736,7 @@ func TestRegisterAndLoginTorTargetsAPISuccess(t *testing.T) { } } -func TestTorTargetsAPIContinueUsingToken(t *testing.T) { +func TestTorTargetsContinueUsingToken(t *testing.T) { ff := &fakeFill{} var expect apimodel.TorTargetsResponse ff.fill(&expect) @@ -751,7 +751,7 @@ func TestTorTargetsAPIContinueUsingToken(t *testing.T) { Token: "antani-antani-token", }, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: &FakeTorTargetsAPI{ WithResult: &FakeTorTargetsAPI{ Response: expect, @@ -759,7 +759,7 @@ func TestTorTargetsAPIContinueUsingToken(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -809,7 +809,7 @@ func TestTorTargetsAPIContinueUsingToken(t *testing.T) { } } -func TestTorTargetsAPIWithValidButExpiredToken(t *testing.T) { +func TestTorTargetsWithValidButExpiredToken(t *testing.T) { ff := &fakeFill{} var expect apimodel.TorTargetsResponse ff.fill(&expect) @@ -823,7 +823,7 @@ func TestTorTargetsAPIWithValidButExpiredToken(t *testing.T) { Token: "antani-antani-token", }, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: &FakeTorTargetsAPI{ WithResult: &FakeTorTargetsAPI{ Response: expect, @@ -831,7 +831,7 @@ func TestTorTargetsAPIWithValidButExpiredToken(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } ls := &loginState{ ClientID: "antani-antani", @@ -863,7 +863,7 @@ func TestTorTargetsAPIWithValidButExpiredToken(t *testing.T) { } } -func TestTorTargetsAPIWithRegisterAPIError(t *testing.T) { +func TestTorTargetsWithRegisterAPIError(t *testing.T) { ff := &fakeFill{} var expect apimodel.TorTargetsResponse ff.fill(&expect) @@ -871,14 +871,14 @@ func TestTorTargetsAPIWithRegisterAPIError(t *testing.T) { registerAPI := &FakeRegisterAPI{ Err: errMocked, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: &FakeTorTargetsAPI{ WithResult: &FakeTorTargetsAPI{ Response: expect, }, }, RegisterAPI: registerAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -895,7 +895,7 @@ func TestTorTargetsAPIWithRegisterAPIError(t *testing.T) { } } -func TestTorTargetsAPIWithLoginFailure(t *testing.T) { +func TestTorTargetsWithLoginFailure(t *testing.T) { ff := &fakeFill{} var expect apimodel.TorTargetsResponse ff.fill(&expect) @@ -908,7 +908,7 @@ func TestTorTargetsAPIWithLoginFailure(t *testing.T) { loginAPI := &FakeLoginAPI{ Err: errMocked, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: &FakeTorTargetsAPI{ WithResult: &FakeTorTargetsAPI{ Response: expect, @@ -916,7 +916,7 @@ func TestTorTargetsAPIWithLoginFailure(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -936,7 +936,7 @@ func TestTorTargetsAPIWithLoginFailure(t *testing.T) { } } -func TestRegisterAndLoginTorTargetsAPIThenFail(t *testing.T) { +func TestRegisterAndLoginTorTargetsThenFail(t *testing.T) { ff := &fakeFill{} var expect apimodel.TorTargetsResponse ff.fill(&expect) @@ -952,7 +952,7 @@ func TestRegisterAndLoginTorTargetsAPIThenFail(t *testing.T) { }, } errMocked := errors.New("mocked error") - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: &FakeTorTargetsAPI{ WithResult: &FakeTorTargetsAPI{ Err: errMocked, @@ -960,7 +960,7 @@ func TestRegisterAndLoginTorTargetsAPIThenFail(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -980,28 +980,28 @@ func TestRegisterAndLoginTorTargetsAPIThenFail(t *testing.T) { } } -func TestTorTargetsAPITheDatabaseIsReplaced(t *testing.T) { +func TestTorTargetsTheDatabaseIsReplaced(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &TorTargetsAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simpleTorTargetsAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -1040,7 +1040,7 @@ func TestTorTargetsAPITheDatabaseIsReplaced(t *testing.T) { } } -func TestRegisterAndLoginTorTargetsAPICannotWriteState(t *testing.T) { +func TestRegisterAndLoginTorTargetsCannotWriteState(t *testing.T) { ff := &fakeFill{} var expect apimodel.TorTargetsResponse ff.fill(&expect) @@ -1056,7 +1056,7 @@ func TestRegisterAndLoginTorTargetsAPICannotWriteState(t *testing.T) { }, } errMocked := errors.New("mocked error") - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: &FakeTorTargetsAPI{ WithResult: &FakeTorTargetsAPI{ Response: expect, @@ -1064,7 +1064,7 @@ func TestRegisterAndLoginTorTargetsAPICannotWriteState(t *testing.T) { }, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, JSONCodec: &FakeCodec{ EncodeErr: errMocked, }, @@ -1087,13 +1087,13 @@ func TestRegisterAndLoginTorTargetsAPICannotWriteState(t *testing.T) { } } -func TestTorTargetsAPIReadStateDecodeFailure(t *testing.T) { +func TestTorTargetsReadStateDecodeFailure(t *testing.T) { ff := &fakeFill{} var expect apimodel.TorTargetsResponse ff.fill(&expect) errMocked := errors.New("mocked error") - login := &TorTargetsAPIWithLogin{ - KVStore: &memkvstore{}, + login := &withLoginTorTargetsAPI{ + KVStore: &MemKVStore{}, JSONCodec: &FakeCodec{DecodeErr: errMocked}, } ls := &loginState{ @@ -1114,28 +1114,28 @@ func TestTorTargetsAPIReadStateDecodeFailure(t *testing.T) { } } -func TestTorTargetsAPITheDatabaseIsReplacedThenFailure(t *testing.T) { +func TestTorTargetsTheDatabaseIsReplacedThenFailure(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &TorTargetsAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simpleTorTargetsAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -1176,28 +1176,28 @@ func TestTorTargetsAPITheDatabaseIsReplacedThenFailure(t *testing.T) { } } -func TestTorTargetsAPIClockIsOffThenSuccess(t *testing.T) { +func TestTorTargetsClockIsOffThenSuccess(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &TorTargetsAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simpleTorTargetsAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -1238,28 +1238,28 @@ func TestTorTargetsAPIClockIsOffThenSuccess(t *testing.T) { } } -func TestTorTargetsAPIClockIsOffThen401(t *testing.T) { +func TestTorTargetsClockIsOffThen401(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &TorTargetsAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simpleTorTargetsAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) @@ -1301,28 +1301,28 @@ func TestTorTargetsAPIClockIsOffThen401(t *testing.T) { } } -func TestTorTargetsAPIClockIsOffThen500(t *testing.T) { +func TestTorTargetsClockIsOffThen500(t *testing.T) { ff := &fakeFill{} handler := &LoginHandler{t: t} srvr := httptest.NewServer(handler) defer srvr.Close() - registerAPI := &RegisterAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + registerAPI := &simpleRegisterAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - loginAPI := &LoginAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + loginAPI := &simpleLoginAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - baseAPI := &TorTargetsAPI{ - HTTPClient: &VerboseHTTPClient{t: t}, + baseAPI := &simpleTorTargetsAPI{ + HTTPClient: &VerboseHTTPClient{T: t}, BaseURL: srvr.URL, } - login := &TorTargetsAPIWithLogin{ + login := &withLoginTorTargetsAPI{ API: baseAPI, RegisterAPI: registerAPI, LoginAPI: loginAPI, - KVStore: &memkvstore{}, + KVStore: &MemKVStore{}, } var req *apimodel.TorTargetsRequest ff.fill(&req) diff --git a/internal/engine/ooapi/requests.go b/internal/engine/ooapi/requests.go index 806d5f2..78a920f 100644 --- a/internal/engine/ooapi/requests.go +++ b/internal/engine/ooapi/requests.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:53.210720456 +0100 CET m=+0.000083649 +// 2021-03-10 13:17:35.068399906 +0100 CET m=+0.000095438 package ooapi @@ -14,7 +14,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -func (api *CheckReportIDAPI) newRequest(ctx context.Context, req *apimodel.CheckReportIDRequest) (*http.Request, error) { +func (api *simpleCheckReportIDAPI) newRequest(ctx context.Context, req *apimodel.CheckReportIDRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -29,7 +29,7 @@ func (api *CheckReportIDAPI) newRequest(ctx context.Context, req *apimodel.Check return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil) } -func (api *CheckInAPI) newRequest(ctx context.Context, req *apimodel.CheckInRequest) (*http.Request, error) { +func (api *simpleCheckInAPI) newRequest(ctx context.Context, req *apimodel.CheckInRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -47,7 +47,7 @@ func (api *CheckInAPI) newRequest(ctx context.Context, req *apimodel.CheckInRequ return out, nil } -func (api *LoginAPI) newRequest(ctx context.Context, req *apimodel.LoginRequest) (*http.Request, error) { +func (api *simpleLoginAPI) newRequest(ctx context.Context, req *apimodel.LoginRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -65,7 +65,7 @@ func (api *LoginAPI) newRequest(ctx context.Context, req *apimodel.LoginRequest) return out, nil } -func (api *MeasurementMetaAPI) newRequest(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*http.Request, error) { +func (api *simpleMeasurementMetaAPI) newRequest(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -86,7 +86,7 @@ func (api *MeasurementMetaAPI) newRequest(ctx context.Context, req *apimodel.Mea return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil) } -func (api *RegisterAPI) newRequest(ctx context.Context, req *apimodel.RegisterRequest) (*http.Request, error) { +func (api *simpleRegisterAPI) newRequest(ctx context.Context, req *apimodel.RegisterRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -104,7 +104,7 @@ func (api *RegisterAPI) newRequest(ctx context.Context, req *apimodel.RegisterRe return out, nil } -func (api *TestHelpersAPI) newRequest(ctx context.Context, req *apimodel.TestHelpersRequest) (*http.Request, error) { +func (api *simpleTestHelpersAPI) newRequest(ctx context.Context, req *apimodel.TestHelpersRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -113,7 +113,7 @@ func (api *TestHelpersAPI) newRequest(ctx context.Context, req *apimodel.TestHel return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil) } -func (api *PsiphonConfigAPI) newRequest(ctx context.Context, req *apimodel.PsiphonConfigRequest) (*http.Request, error) { +func (api *simplePsiphonConfigAPI) newRequest(ctx context.Context, req *apimodel.PsiphonConfigRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -122,7 +122,7 @@ func (api *PsiphonConfigAPI) newRequest(ctx context.Context, req *apimodel.Psiph return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil) } -func (api *TorTargetsAPI) newRequest(ctx context.Context, req *apimodel.TorTargetsRequest) (*http.Request, error) { +func (api *simpleTorTargetsAPI) newRequest(ctx context.Context, req *apimodel.TorTargetsRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -131,7 +131,7 @@ func (api *TorTargetsAPI) newRequest(ctx context.Context, req *apimodel.TorTarge return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil) } -func (api *URLsAPI) newRequest(ctx context.Context, req *apimodel.URLsRequest) (*http.Request, error) { +func (api *simpleURLsAPI) newRequest(ctx context.Context, req *apimodel.URLsRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -151,7 +151,7 @@ func (api *URLsAPI) newRequest(ctx context.Context, req *apimodel.URLsRequest) ( return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil) } -func (api *OpenReportAPI) newRequest(ctx context.Context, req *apimodel.OpenReportRequest) (*http.Request, error) { +func (api *simpleOpenReportAPI) newRequest(ctx context.Context, req *apimodel.OpenReportRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err @@ -169,7 +169,7 @@ func (api *OpenReportAPI) newRequest(ctx context.Context, req *apimodel.OpenRepo return out, nil } -func (api *SubmitMeasurementAPI) newRequest(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*http.Request, error) { +func (api *simpleSubmitMeasurementAPI) newRequest(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*http.Request, error) { URL, err := url.Parse(api.baseURL()) if err != nil { return nil, err diff --git a/internal/engine/ooapi/responses.go b/internal/engine/ooapi/responses.go index b148aea..01d682c 100644 --- a/internal/engine/ooapi/responses.go +++ b/internal/engine/ooapi/responses.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:53.567815989 +0100 CET m=+0.000158731 +// 2021-03-10 13:17:35.43791782 +0100 CET m=+0.000136842 package ooapi @@ -13,7 +13,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/ooapi/apimodel" ) -func (api *CheckReportIDAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) { +func (api *simpleCheckReportIDAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) { if err != nil { return nil, err } @@ -36,7 +36,7 @@ func (api *CheckReportIDAPI) newResponse(resp *http.Response, err error) (*apimo return out, nil } -func (api *CheckInAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckInResponse, error) { +func (api *simpleCheckInAPI) newResponse(resp *http.Response, err error) (*apimodel.CheckInResponse, error) { if err != nil { return nil, err } @@ -59,7 +59,7 @@ func (api *CheckInAPI) newResponse(resp *http.Response, err error) (*apimodel.Ch return out, nil } -func (api *LoginAPI) newResponse(resp *http.Response, err error) (*apimodel.LoginResponse, error) { +func (api *simpleLoginAPI) newResponse(resp *http.Response, err error) (*apimodel.LoginResponse, error) { if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (api *LoginAPI) newResponse(resp *http.Response, err error) (*apimodel.Logi return out, nil } -func (api *MeasurementMetaAPI) newResponse(resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) { +func (api *simpleMeasurementMetaAPI) newResponse(resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) { if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (api *MeasurementMetaAPI) newResponse(resp *http.Response, err error) (*api return out, nil } -func (api *RegisterAPI) newResponse(resp *http.Response, err error) (*apimodel.RegisterResponse, error) { +func (api *simpleRegisterAPI) newResponse(resp *http.Response, err error) (*apimodel.RegisterResponse, error) { if err != nil { return nil, err } @@ -128,7 +128,7 @@ func (api *RegisterAPI) newResponse(resp *http.Response, err error) (*apimodel.R return out, nil } -func (api *TestHelpersAPI) newResponse(resp *http.Response, err error) (apimodel.TestHelpersResponse, error) { +func (api *simpleTestHelpersAPI) newResponse(resp *http.Response, err error) (apimodel.TestHelpersResponse, error) { if err != nil { return nil, err } @@ -154,7 +154,7 @@ func (api *TestHelpersAPI) newResponse(resp *http.Response, err error) (apimodel return out, nil } -func (api *PsiphonConfigAPI) newResponse(resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) { +func (api *simplePsiphonConfigAPI) newResponse(resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) { if err != nil { return nil, err } @@ -180,7 +180,7 @@ func (api *PsiphonConfigAPI) newResponse(resp *http.Response, err error) (apimod return out, nil } -func (api *TorTargetsAPI) newResponse(resp *http.Response, err error) (apimodel.TorTargetsResponse, error) { +func (api *simpleTorTargetsAPI) newResponse(resp *http.Response, err error) (apimodel.TorTargetsResponse, error) { if err != nil { return nil, err } @@ -206,7 +206,7 @@ func (api *TorTargetsAPI) newResponse(resp *http.Response, err error) (apimodel. return out, nil } -func (api *URLsAPI) newResponse(resp *http.Response, err error) (*apimodel.URLsResponse, error) { +func (api *simpleURLsAPI) newResponse(resp *http.Response, err error) (*apimodel.URLsResponse, error) { if err != nil { return nil, err } @@ -229,7 +229,7 @@ func (api *URLsAPI) newResponse(resp *http.Response, err error) (*apimodel.URLsR return out, nil } -func (api *OpenReportAPI) newResponse(resp *http.Response, err error) (*apimodel.OpenReportResponse, error) { +func (api *simpleOpenReportAPI) newResponse(resp *http.Response, err error) (*apimodel.OpenReportResponse, error) { if err != nil { return nil, err } @@ -252,7 +252,7 @@ func (api *OpenReportAPI) newResponse(resp *http.Response, err error) (*apimodel return out, nil } -func (api *SubmitMeasurementAPI) newResponse(resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) { +func (api *simpleSubmitMeasurementAPI) newResponse(resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) { if err != nil { return nil, err } diff --git a/internal/engine/ooapi/swagger_test.go b/internal/engine/ooapi/swagger_test.go index 5cf18ab..e775b2f 100644 --- a/internal/engine/ooapi/swagger_test.go +++ b/internal/engine/ooapi/swagger_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-02-26 15:45:53.881261959 +0100 CET m=+0.000594905 +// 2021-03-10 13:17:35.72281221 +0100 CET m=+0.000577472 package ooapi @@ -9,7 +9,7 @@ const swagger = `{ "swagger": "2.0", "info": { "title": "OONI API specification", - "version": "0.20210226.2144553" + "version": "0.20210310.3121735" }, "host": "api.ooni.io", "basePath": "/", From 70d7c1a22c832c351b6a93148b010106fa472fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 22 Mar 2021 14:31:50 +0100 Subject: [PATCH 04/28] Add signal to the im test group (#259) * Add signal to the im test group * fix(ipconfig_test.go): disable when running in CI Reference issue: https://github.com/ooni/probe/issues/1418 * fix(geolocate): remove unused variable Came across this while looking into this issue with the CI that is now failing. Guess fixing it here comes across as leaving the camp slightly less in a bad shape than how I found it. Co-authored-by: Simone Basso --- cmd/ooniprobe/internal/nettests/groups.go | 2 +- internal/engine/geolocate/ipconfig_test.go | 6 ++++++ internal/engine/geolocate/iplookup.go | 3 --- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/ooniprobe/internal/nettests/groups.go b/cmd/ooniprobe/internal/nettests/groups.go index 5c4d9bc..2390432 100644 --- a/cmd/ooniprobe/internal/nettests/groups.go +++ b/cmd/ooniprobe/internal/nettests/groups.go @@ -37,6 +37,7 @@ var All = map[string]Group{ FacebookMessenger{}, Telegram{}, WhatsApp{}, + Signal{}, }, UnattendedOK: true, }, @@ -54,7 +55,6 @@ var All = map[string]Group{ Nettests: []Nettest{ DNSCheck{}, STUNReachability{}, - Signal{}, }, }, } diff --git a/internal/engine/geolocate/ipconfig_test.go b/internal/engine/geolocate/ipconfig_test.go index ed9b262..d9d6c9b 100644 --- a/internal/engine/geolocate/ipconfig_test.go +++ b/internal/engine/geolocate/ipconfig_test.go @@ -4,6 +4,7 @@ import ( "context" "net" "net/http" + "os" "testing" "github.com/apex/log" @@ -11,6 +12,11 @@ import ( ) func TestIPLookupWorksUsingIPConfig(t *testing.T) { + if os.Getenv("CI") == "true" { + // See https://github.com/ooni/probe-cli/pull/259/checks?check_run_id=2166066881#step:5:123 + // as well as https://github.com/ooni/probe/issues/1418. + t.Skip("This test does not work with GitHub Actions") + } ip, err := ipConfigIPLookup( context.Background(), http.DefaultClient, diff --git a/internal/engine/geolocate/iplookup.go b/internal/engine/geolocate/iplookup.go index acf0222..01462bd 100644 --- a/internal/engine/geolocate/iplookup.go +++ b/internal/engine/geolocate/iplookup.go @@ -7,7 +7,6 @@ import ( "math/rand" "net" "net/http" - "sync" "time" "github.com/ooni/probe-cli/v3/internal/engine/internal/multierror" @@ -61,8 +60,6 @@ var ( fn: ubuntuIPLookup, }, } - - once sync.Once ) type ipLookupClient struct { From fc19c9901acf5c7df45139885b79f603f6a9adb3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 23 Mar 2021 16:46:46 +0100 Subject: [PATCH 05/28] fix(webconnectivity): expose network events (#258) * fix(webconnectivity): expose network events By not exposing network events in webconnectivity, we are missing several interesting, explanatory data points. This diff fixes the issue by: 1. enriching the definition of network events to include extra data useful for performing (manual) data analysis; 2. adding a tags field to network events such that we can add tags to specific events and understand where they come from; 3. exposing all the (tagged) network events that happen when running a webconnectivity experiment. See https://github.com/ooni/probe-engine/issues/1157. * progress * more work towards landing this diff * Apply suggestions from code review --- .../experiment/webconnectivity/connects.go | 4 +- .../experiment/webconnectivity/dnslookup.go | 5 ++- .../experiment/webconnectivity/httpget.go | 3 ++ .../webconnectivity/webconnectivity.go | 45 +++++++++++++++++-- .../webconnectivity/webconnectivity_test.go | 2 +- internal/engine/netx/archival/archival.go | 24 +++++----- 6 files changed, 67 insertions(+), 16 deletions(-) diff --git a/internal/engine/experiment/webconnectivity/connects.go b/internal/engine/experiment/webconnectivity/connects.go index 4f047d2..1500a95 100644 --- a/internal/engine/experiment/webconnectivity/connects.go +++ b/internal/engine/experiment/webconnectivity/connects.go @@ -3,6 +3,7 @@ package webconnectivity import ( "context" "net/url" + "time" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/model" @@ -10,6 +11,7 @@ import ( // ConnectsConfig contains the config for Connects type ConnectsConfig struct { + Begin time.Time Session model.ExperimentSession TargetURL *url.URL URLGetterURLs []string @@ -28,7 +30,7 @@ type ConnectsResult struct { // check whether the resolved endpoints are reachable. func Connects(ctx context.Context, config ConnectsConfig) (out ConnectsResult) { out.AllKeys = []urlgetter.TestKeys{} - multi := urlgetter.Multi{Session: config.Session} + multi := urlgetter.Multi{Begin: config.Begin, Session: config.Session} inputs := []urlgetter.MultiInput{} for _, url := range config.URLGetterURLs { inputs = append(inputs, urlgetter.MultiInput{ diff --git a/internal/engine/experiment/webconnectivity/dnslookup.go b/internal/engine/experiment/webconnectivity/dnslookup.go index 6d0a324..e5624d1 100644 --- a/internal/engine/experiment/webconnectivity/dnslookup.go +++ b/internal/engine/experiment/webconnectivity/dnslookup.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" "sort" + "time" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/model" @@ -12,6 +13,7 @@ import ( // DNSLookupConfig contains settings for the DNS lookup. type DNSLookupConfig struct { + Begin time.Time Session model.ExperimentSession URL *url.URL } @@ -27,7 +29,8 @@ type DNSLookupResult struct { func DNSLookup(ctx context.Context, config DNSLookupConfig) (out DNSLookupResult) { target := fmt.Sprintf("dnslookup://%s", config.URL.Hostname()) config.Session.Logger().Infof("%s...", target) - result, err := urlgetter.Getter{Session: config.Session, Target: target}.Get(ctx) + result, err := urlgetter.Getter{ + Begin: config.Begin, Session: config.Session, Target: target}.Get(ctx) out.Addrs = make(map[string]int64) for _, query := range result.Queries { for _, answer := range query.Answers { diff --git a/internal/engine/experiment/webconnectivity/httpget.go b/internal/engine/experiment/webconnectivity/httpget.go index 35200ca..0c06d84 100644 --- a/internal/engine/experiment/webconnectivity/httpget.go +++ b/internal/engine/experiment/webconnectivity/httpget.go @@ -6,6 +6,7 @@ import ( "net" "net/url" "strings" + "time" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/model" @@ -14,6 +15,7 @@ import ( // HTTPGetConfig contains the config for HTTPGet type HTTPGetConfig struct { Addresses []string + Begin time.Time Session model.ExperimentSession TargetURL *url.URL } @@ -54,6 +56,7 @@ func HTTPGet(ctx context.Context, config HTTPGetConfig) (out HTTPGetResult) { config.Session.Logger().Infof("GET %s...", target) domain := config.TargetURL.Hostname() result, err := urlgetter.Getter{ + Begin: config.Begin, Config: urlgetter.Config{ DNSCache: HTTPGetMakeDNSCache(domain, addresses), }, diff --git a/internal/engine/experiment/webconnectivity/webconnectivity.go b/internal/engine/experiment/webconnectivity/webconnectivity.go index 7f76ed4..4b68cfa 100644 --- a/internal/engine/experiment/webconnectivity/webconnectivity.go +++ b/internal/engine/experiment/webconnectivity/webconnectivity.go @@ -19,7 +19,7 @@ import ( const ( testName = "web_connectivity" - testVersion = "0.3.0" + testVersion = "0.4.0" ) // Config contains the experiment config. @@ -32,6 +32,15 @@ type TestKeys struct { Retries *int64 `json:"retries"` // unused SOCKSProxy *string `json:"socksproxy"` // unused + // For now mostly TCP/TLS "connect" experiment but we are + // considering adding more events. An open question is + // currently how to properly tag these events so that it + // is rather obvious where they come from. + // + // See https://github.com/ooni/probe/issues/1413. + NetworkEvents []archival.NetworkEvent `json:"network_events"` + TLSHandshakes []archival.TLSHandshake `json:"tls_handshakes"` + // DNS experiment Queries []archival.DNSQueryEntry `json:"queries"` DNSExperimentFailure *string `json:"dns_experiment_failure"` @@ -42,7 +51,7 @@ type TestKeys struct { ControlRequest ControlRequest `json:"-"` Control ControlResponse `json:"control"` - // TCP connect experiment + // TCP/TLS "connect" experiment TCPConnect []archival.TCPConnectEntry `json:"tcp_connect"` TCPConnectSuccesses int `json:"-"` TCPConnectAttempts int `json:"-"` @@ -90,6 +99,19 @@ var ( ErrUnsupportedInput = errors.New("unsupported input scheme") ) +// Tags describing the section of this experiment in which +// the data has been collected. +const ( + // DNSExperimentTag is a tag indicating the DNS experiment. + DNSExperimentTag = "dns_experiment" + + // TCPTLSExperimentTag is a tag indicating the connect experiment. + TCPTLSExperimentTag = "tcptls_experiment" + + // HTTPExperimentTag is a tag indicating the HTTP experiment. + HTTPExperimentTag = "http_experiment" +) + // Run implements ExperimentMeasurer.Run. func (m Measurer) Run( ctx context.Context, @@ -129,7 +151,9 @@ func (m Measurer) Run( "backend": testhelper, } // 2. perform the DNS lookup step - dnsResult := DNSLookup(ctx, DNSLookupConfig{Session: sess, URL: URL}) + dnsResult := DNSLookup(ctx, DNSLookupConfig{ + Begin: measurement.MeasurementStartTimeSaved, + Session: sess, URL: URL}) tk.Queries = append(tk.Queries, dnsResult.TestKeys.Queries...) tk.DNSExperimentFailure = dnsResult.Failure epnts := NewEndpoints(URL, dnsResult.Addresses()) @@ -152,7 +176,13 @@ func (m Measurer) Run( sess.Logger().Infof("DNS analysis result: %+v", internal.StringPointerToString( tk.DNSAnalysisResult.DNSConsistency)) // 5. perform TCP/TLS connects + // + // TODO(bassosimone): here we should also follow the IP addresses + // returned by the control experiment. + // + // See https://github.com/ooni/probe/issues/1414 connectsResult := Connects(ctx, ConnectsConfig{ + Begin: measurement.MeasurementStartTimeSaved, Session: sess, TargetURL: URL, URLGetterURLs: epnts.URLs(), @@ -164,12 +194,21 @@ func (m Measurer) Run( // sad that we're storing analysis result inside the measurement tk.TCPConnect = append(tk.TCPConnect, ComputeTCPBlocking( tcpkeys.TCPConnect, tk.Control.TCPConnect)...) + for _, ev := range tcpkeys.NetworkEvents { + ev.Tags = []string{TCPTLSExperimentTag} + tk.NetworkEvents = append(tk.NetworkEvents, ev) + } + for _, ev := range tcpkeys.TLSHandshakes { + ev.Tags = []string{TCPTLSExperimentTag} + tk.TLSHandshakes = append(tk.TLSHandshakes, ev) + } } tk.TCPConnectAttempts = connectsResult.Total tk.TCPConnectSuccesses = connectsResult.Successes // 6. perform HTTP/HTTPS measurement httpResult := HTTPGet(ctx, HTTPGetConfig{ Addresses: dnsResult.Addresses(), + Begin: measurement.MeasurementStartTimeSaved, Session: sess, TargetURL: URL, }) diff --git a/internal/engine/experiment/webconnectivity/webconnectivity_test.go b/internal/engine/experiment/webconnectivity/webconnectivity_test.go index 8a04618..f61ba0b 100644 --- a/internal/engine/experiment/webconnectivity/webconnectivity_test.go +++ b/internal/engine/experiment/webconnectivity/webconnectivity_test.go @@ -21,7 +21,7 @@ func TestNewExperimentMeasurer(t *testing.T) { if measurer.ExperimentName() != "web_connectivity" { t.Fatal("unexpected name") } - if measurer.ExperimentVersion() != "0.3.0" { + if measurer.ExperimentVersion() != "0.4.0" { t.Fatal("unexpected version") } } diff --git a/internal/engine/netx/archival/archival.go b/internal/engine/netx/archival/archival.go index ec1fdac..67ef5f8 100644 --- a/internal/engine/netx/archival/archival.go +++ b/internal/engine/netx/archival/archival.go @@ -463,17 +463,20 @@ func (qtype dnsQueryType) makequeryentry(begin time.Time, ev trace.Event) DNSQue } } -// NetworkEvent is a network event. +// NetworkEvent is a network event. It contains all the possible fields +// and most fields are optional. They are only added when it makes sense +// for them to be there _and_ we have data to show. type NetworkEvent struct { - Address string `json:"address,omitempty"` - ConnID int64 `json:"conn_id,omitempty"` - DialID int64 `json:"dial_id,omitempty"` - Failure *string `json:"failure"` - NumBytes int64 `json:"num_bytes,omitempty"` - Operation string `json:"operation"` - Proto string `json:"proto,omitempty"` - T float64 `json:"t"` - TransactionID int64 `json:"transaction_id,omitempty"` + Address string `json:"address,omitempty"` + ConnID int64 `json:"conn_id,omitempty"` + DialID int64 `json:"dial_id,omitempty"` + Failure *string `json:"failure"` + NumBytes int64 `json:"num_bytes,omitempty"` + Operation string `json:"operation"` + Proto string `json:"proto,omitempty"` + T float64 `json:"t"` + Tags []string `json:"tags,omitempty"` + TransactionID int64 `json:"transaction_id,omitempty"` } // NewNetworkEventsList returns a list of DNS queries. @@ -547,6 +550,7 @@ type TLSHandshake struct { PeerCertificates []MaybeBinaryValue `json:"peer_certificates"` ServerName string `json:"server_name"` T float64 `json:"t"` + Tags []string `json:"tags"` TLSVersion string `json:"tls_version"` TransactionID int64 `json:"transaction_id,omitempty"` } From 9c820cf8558d60e5348fed8cef09f67c188fe81f Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 24 Mar 2021 11:39:45 +0100 Subject: [PATCH 06/28] fix(ooniprobe): send batch output to stdout (#261) Rationale explained in detail in a documentation comment. Reference issue: https://github.com/ooni/probe/issues/1384. --- cmd/ooniprobe/internal/log/handlers/batch/batch.go | 13 +++++++++++-- cmd/ooniprobe/internal/log/handlers/cli/cli.go | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/ooniprobe/internal/log/handlers/batch/batch.go b/cmd/ooniprobe/internal/log/handlers/batch/batch.go index 2b11371..2d06a4f 100644 --- a/cmd/ooniprobe/internal/log/handlers/batch/batch.go +++ b/cmd/ooniprobe/internal/log/handlers/batch/batch.go @@ -9,8 +9,17 @@ import ( "github.com/apex/log" ) -// Default handler outputting to stderr. -var Default = New(os.Stderr) +// Default handler outputting to stdout. We want to emit the batch +// output on the standard output, for two reasons: +// +// 1. because third party libraries MAY log on the stderr and +// their logs are most likely not JSON; +// +// 2. because this enables piping to `jq` or other tools in +// a much more natural way than when emitting on stderr. +// +// See https://github.com/ooni/probe/issues/1384. +var Default = New(os.Stdout) // Handler implementation. type Handler struct { diff --git a/cmd/ooniprobe/internal/log/handlers/cli/cli.go b/cmd/ooniprobe/internal/log/handlers/cli/cli.go index bd34346..575c922 100644 --- a/cmd/ooniprobe/internal/log/handlers/cli/cli.go +++ b/cmd/ooniprobe/internal/log/handlers/cli/cli.go @@ -14,7 +14,7 @@ import ( "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils" ) -// Default handler outputting to stderr. +// Default handler outputting to stdout. var Default = New(os.Stdout) // start time. From 576059b3fab5d307e26b3c04a4fb5fc3ad6d77db Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 24 Mar 2021 12:35:53 +0100 Subject: [PATCH 07/28] fix: apply staticcheck suggestions (#262) Preliminary work done as part of https://github.com/ooni/probe/issues/1299 --- internal/engine/experimentbuilder.go | 2 +- internal/engine/probeservices/benchselect.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/engine/experimentbuilder.go b/internal/engine/experimentbuilder.go index 994efc8..332e079 100644 --- a/internal/engine/experimentbuilder.go +++ b/internal/engine/experimentbuilder.go @@ -193,7 +193,7 @@ func canonicalizeExperimentName(name string) string { } func newExperimentBuilder(session *Session, name string) (*ExperimentBuilder, error) { - factory, _ := experimentsByName[canonicalizeExperimentName(name)] + factory := experimentsByName[canonicalizeExperimentName(name)] if factory == nil { return nil, fmt.Errorf("no such experiment: %s", name) } diff --git a/internal/engine/probeservices/benchselect.go b/internal/engine/probeservices/benchselect.go index 6ff54dd..4f1a77d 100644 --- a/internal/engine/probeservices/benchselect.go +++ b/internal/engine/probeservices/benchselect.go @@ -85,7 +85,7 @@ func (c *Candidate) try(ctx context.Context, sess Session) { } start := time.Now() testhelpers, err := client.GetTestHelpers(ctx) - c.Duration = time.Now().Sub(start) + c.Duration = time.Since(start) c.Err = err c.TestHelpers = testhelpers sess.Logger().Debugf("probe services: %+v: %+v %s", c.Endpoint, err, c.Duration) From 3b029ee0d6574974aa567f1fcc79b3ba66b989bb Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 25 Mar 2021 12:02:02 +0100 Subject: [PATCH 08/28] feat(ExperimentOrchestraClient): add CheckIn (#263) We use ExperimentOrchestraClient in several places to help us calling probe-services APIs. We need to call CheckIn because we want to use CheckIn in InputLoader. (We also want to remove the URLs API, but that is not something doable now, since the mobile app is still using this API via the wrappers at pkg/oonimkall.) Work part of https://github.com/ooni/probe/issues/1299. --- internal/engine/inputloader_test.go | 4 ++++ internal/engine/internal/mockable/mockable.go | 8 ++++++++ internal/engine/model/experiment.go | 9 +++++++++ pkg/oonimkall/task_integration_test.go | 2 +- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/internal/engine/inputloader_test.go b/internal/engine/inputloader_test.go index 38bf483..9e231ef 100644 --- a/internal/engine/inputloader_test.go +++ b/internal/engine/inputloader_test.go @@ -80,6 +80,10 @@ func TestInputLoaderNewOrchestraClientFailure(t *testing.T) { type InputLoaderBrokenOrchestraClient struct{} +func (InputLoaderBrokenOrchestraClient) CheckIn(ctx context.Context, config model.CheckInConfig) (*model.CheckInInfo, error) { + return nil, io.EOF +} + func (InputLoaderBrokenOrchestraClient) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { return nil, io.EOF } diff --git a/internal/engine/internal/mockable/mockable.go b/internal/engine/internal/mockable/mockable.go index 1ce50af..032e24d 100644 --- a/internal/engine/internal/mockable/mockable.go +++ b/internal/engine/internal/mockable/mockable.go @@ -162,6 +162,8 @@ var _ torx.Session = &Session{} // ExperimentOrchestraClient is the experiment's view of // a client for querying the OONI orchestra. type ExperimentOrchestraClient struct { + MockableCheckInInfo *model.CheckInInfo + MockableCheckInErr error MockableFetchPsiphonConfigResult []byte MockableFetchPsiphonConfigErr error MockableFetchTorTargetsResult map[string]model.TorTarget @@ -170,6 +172,12 @@ type ExperimentOrchestraClient struct { MockableFetchURLListErr error } +// CheckIn implements ExperimentOrchestraClient.CheckIn. +func (c ExperimentOrchestraClient) CheckIn( + ctx context.Context, config model.CheckInConfig) (*model.CheckInInfo, error) { + return c.MockableCheckInInfo, c.MockableCheckInErr +} + // FetchPsiphonConfig implements ExperimentOrchestraClient.FetchPsiphonConfig func (c ExperimentOrchestraClient) FetchPsiphonConfig( ctx context.Context) ([]byte, error) { diff --git a/internal/engine/model/experiment.go b/internal/engine/model/experiment.go index 18e8add..18e5f32 100644 --- a/internal/engine/model/experiment.go +++ b/internal/engine/model/experiment.go @@ -8,8 +8,17 @@ import ( // ExperimentOrchestraClient is the experiment's view of // a client for querying the OONI orchestra API. type ExperimentOrchestraClient interface { + // CheckIn calls the check-in API. + CheckIn(ctx context.Context, config CheckInConfig) (*CheckInInfo, error) + + // FetchPsiphonConfig returns psiphon config from the API. FetchPsiphonConfig(ctx context.Context) ([]byte, error) + + // FetchTorTargets returns tor targets from the API. FetchTorTargets(ctx context.Context, cc string) (map[string]TorTarget, error) + + // FetchURLList returns URLs from the API. + // This method is deprecated and will be removed soon. FetchURLList(ctx context.Context, config URLListConfig) ([]URLInfo, error) } diff --git a/pkg/oonimkall/task_integration_test.go b/pkg/oonimkall/task_integration_test.go index 9848a54..f8d1081 100644 --- a/pkg/oonimkall/task_integration_test.go +++ b/pkg/oonimkall/task_integration_test.go @@ -315,7 +315,7 @@ func TestMaxRuntime(t *testing.T) { // In case there are further timeouts, e.g. in the sessionresolver, the // time used by the experiment will be much more. This is for example the // case in https://github.com/ooni/probe-engine/issues/1005. - if time.Now().Sub(begin) > 10*time.Second { + if time.Since(begin) > 10*time.Second { t.Fatal("expected shorter runtime") } } From c94721d9e53e1f1266661f0bb2e0b8132a5e7fb7 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 25 Mar 2021 15:18:29 +0100 Subject: [PATCH 09/28] fix(sessionresolver): proxy check conditional on existing proxy (#264) There was a face-palming error in the implementation causing the proxy check to be implemented also without a proxy. This meant that we were ALWAYS skipping http3 and system resolvers. The bug has been introduced in 3.8.0. So, the currently released version of the probe, sadly, has this beheavior :-(. Reference issue https://github.com/ooni/probe/issues/1426. --- .../internal/sessionresolver/resolvermaker.go | 2 +- .../sessionresolver/sessionresolver.go | 69 ++++++++++++++++--- .../sessionresolver/sessionresolver_test.go | 2 +- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/internal/engine/internal/sessionresolver/resolvermaker.go b/internal/engine/internal/sessionresolver/resolvermaker.go index 1719c6e..0874d7b 100644 --- a/internal/engine/internal/sessionresolver/resolvermaker.go +++ b/internal/engine/internal/sessionresolver/resolvermaker.go @@ -97,7 +97,7 @@ func (r *Resolver) newresolver(URL string) (childResolver, error) { func (r *Resolver) getresolver(URL string) (childResolver, error) { defer r.mu.Unlock() r.mu.Lock() - if re, found := r.res[URL]; found == true { + if re, found := r.res[URL]; found { return re, nil // already created } re, err := r.newresolver(URL) diff --git a/internal/engine/internal/sessionresolver/sessionresolver.go b/internal/engine/internal/sessionresolver/sessionresolver.go index 9b9af01..8e7be14 100644 --- a/internal/engine/internal/sessionresolver/sessionresolver.go +++ b/internal/engine/internal/sessionresolver/sessionresolver.go @@ -17,6 +17,10 @@ // but it will of course be the most popular resolver if anything else // is failing us. (We will still occasionally probe for other working // resolvers and increase their score on success.) +// +// We also support a socks5 proxy. When such a proxy is configured, +// the code WILL skip http3 resolvers AS WELL AS the system +// resolver, in an attempt to avoid leaking your queries. package sessionresolver import ( @@ -34,18 +38,60 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/runtimex" ) -// Resolver is the session resolver. You should create an instance of -// this structure and use it in session.go. +// Resolver is the session resolver. Resolver will try to use +// a bunch of DoT/DoH resolvers before falling back to the +// system resolver. The relative priorities of the resolver +// are stored onto the KVStore such that we can remember them +// and therefore we can generally give preference to underlying +// DoT/DoH resolvers that work better. +// +// You MUST NOT modify public fields of this structure once it +// has been created, because that MAY lead to data races. +// +// You should create an instance of this structure and use +// it in internal/engine/session.go. type Resolver struct { - ByteCounter *bytecounter.Counter // optional - KVStore KVStore // optional - Logger Logger // optional - ProxyURL *url.URL // optional - codec codec + // ByteCounter is the optional byte counter. It will count + // the bytes used by any child resolver except for the + // system resolver, whose bytes ARE NOT counted. If this + // field is not set, then we won't count the bytes. + ByteCounter *bytecounter.Counter + + // KVStore is the optional key-value store where you + // want us to write statistics about which resolver is + // working better in your network. If this field is + // not set, then we'll use a in-memory store. + KVStore KVStore + + // Logger is the optional logger you want us to use + // to emit log messages. + Logger Logger + + // ProxyURL is the optional URL of the socks5 proxy + // we should be using. If not set, then we WON'T use + // any proxy. If set, then we WON'T use any http3 + // based resolvers and we WON'T use the system resolver. + ProxyURL *url.URL + + // codec is the optional codec to use. If not set, we + // will construct a default codec. + codec codec + + // dnsClientMaker is the optional dnsclientmaker to + // use. If not set, we will use the default. dnsClientMaker dnsclientmaker - mu sync.Mutex - once sync.Once - res map[string]childResolver + + // mu provides synchronisation of internal fields. + mu sync.Mutex + + // once ensures that CloseIdleConnection is + // run just once. + once sync.Once + + // res maps a URL to a child resolver. We will + // construct child resolvers just once and we + // will track them into this field. + res map[string]childResolver } // CloseIdleConnections closes the idle connections, if any. This @@ -73,7 +119,8 @@ func (r *Resolver) LookupHost(ctx context.Context, hostname string) ([]string, e defer r.writestate(state) me := multierror.New(ErrLookupHost) for _, e := range state { - if r.shouldSkipWithProxy(e) { + if r.ProxyURL != nil && r.shouldSkipWithProxy(e) { + r.logger().Infof("sessionresolver: skipping with proxy: %+v", e) continue // we cannot proxy this URL so ignore it } addrs, err := r.lookupHost(ctx, e, hostname) diff --git a/internal/engine/internal/sessionresolver/sessionresolver_test.go b/internal/engine/internal/sessionresolver/sessionresolver_test.go index b2b28d3..8b46c1f 100644 --- a/internal/engine/internal/sessionresolver/sessionresolver_test.go +++ b/internal/engine/internal/sessionresolver/sessionresolver_test.go @@ -294,7 +294,7 @@ func TestResolverWorksWithProxy(t *testing.T) { <-done // check results if !errors.Is(err, ErrLookupHost) { - t.Fatal("not the error we expected") + t.Fatal("not the error we expected", err) } if addrs != nil { t.Fatal("expected nil addrs") From 0115d6c4704a3d43e70504d6325cd7b9438dd455 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 26 Mar 2021 09:34:27 +0100 Subject: [PATCH 10/28] refactor(inputloader): better docs and naming (#265) * refactor(inputloader): better docs and naming Work done as part of https://github.com/ooni/probe/issues/1299. * fix: correct a typo --- .../internal/nettests/web_connectivity.go | 2 +- internal/engine/allexperiments.go | 8 +-- .../engine/experiment_integration_test.go | 2 +- internal/engine/experimentbuilder.go | 4 +- internal/engine/inputloader.go | 60 +++++++++++++------ .../engine/inputloader_integration_test.go | 8 +-- internal/engine/inputloader_test.go | 4 +- .../internal/sessionresolver/resolvermaker.go | 2 +- pkg/oonimkall/internal/tasks/runner.go | 4 +- 9 files changed, 59 insertions(+), 35 deletions(-) diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index 923fab4..1286845 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -10,7 +10,7 @@ import ( func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) { inputloader := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOrQueryTestLists, + InputPolicy: engine.InputOrQueryBackend, Session: ctl.Session, SourceFiles: ctl.InputFiles, StaticInputs: ctl.Inputs, diff --git a/internal/engine/allexperiments.go b/internal/engine/allexperiments.go index 5b6ebfb..2178269 100644 --- a/internal/engine/allexperiments.go +++ b/internal/engine/allexperiments.go @@ -152,7 +152,7 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{ )) }, config: &httphostheader.Config{}, - inputPolicy: InputOrQueryTestLists, + inputPolicy: InputOrQueryBackend, } }, @@ -237,7 +237,7 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{ )) }, config: &sniblocking.Config{}, - inputPolicy: InputOrQueryTestLists, + inputPolicy: InputOrQueryBackend, } }, @@ -273,7 +273,7 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{ )) }, config: &tlstool.Config{}, - inputPolicy: InputOrQueryTestLists, + inputPolicy: InputOrQueryBackend, } }, @@ -309,7 +309,7 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{ )) }, config: &webconnectivity.Config{}, - inputPolicy: InputOrQueryTestLists, + inputPolicy: InputOrQueryBackend, } }, diff --git a/internal/engine/experiment_integration_test.go b/internal/engine/experiment_integration_test.go index e689b00..f7b8a68 100644 --- a/internal/engine/experiment_integration_test.go +++ b/internal/engine/experiment_integration_test.go @@ -142,7 +142,7 @@ func TestNeedsInput(t *testing.T) { if err != nil { t.Fatal(err) } - if builder.InputPolicy() != InputOrQueryTestLists { + if builder.InputPolicy() != InputOrQueryBackend { t.Fatal("web_connectivity certainly needs input") } } diff --git a/internal/engine/experimentbuilder.go b/internal/engine/experimentbuilder.go index 332e079..97d8299 100644 --- a/internal/engine/experimentbuilder.go +++ b/internal/engine/experimentbuilder.go @@ -16,12 +16,12 @@ import ( type InputPolicy string const ( - // InputOrQueryTestLists indicates that the experiment requires + // InputOrQueryBackend indicates that the experiment requires // external input to run and that this kind of input is URLs // from the citizenlab/test-lists repository. If this input // not provided to the experiment, then the code that runs the // experiment is supposed to fetch from URLs from OONI's backends. - InputOrQueryTestLists = InputPolicy("or_query_test_lists") + InputOrQueryBackend = InputPolicy("or_query_backend") // InputStrictlyRequired indicates that the experiment // requires input and we currently don't have an API for diff --git a/internal/engine/inputloader.go b/internal/engine/inputloader.go index d606d22..ccd1583 100644 --- a/internal/engine/inputloader.go +++ b/internal/engine/inputloader.go @@ -10,14 +10,15 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/model" ) -// The following errors are returned by the InputLoader. +// These errors are returned by the InputLoader. var ( - ErrNoInputExpected = errors.New("we did not expect any input") - ErrInputRequired = errors.New("no input provided") ErrDetectedEmptyFile = errors.New("file did not contain any input") + ErrInputRequired = errors.New("no input provided") + ErrNoInputExpected = errors.New("we did not expect any input") ) -// InputLoaderSession is the session according to an InputLoader. +// InputLoaderSession is the session according to an InputLoader. We +// introduce this abstraction because it helps us with testing. type InputLoaderSession interface { MaybeLookupLocationContext(ctx context.Context) error NewOrchestraClient(ctx context.Context) (model.ExperimentOrchestraClient, error) @@ -25,8 +26,8 @@ type InputLoaderSession interface { } // InputLoader loads input according to the specified policy -// from the specified sources and OONI services. The behaviour -// depends on the input policy as described below. +// either from command line and input files or from OONI services. The +// behaviour depends on the input policy as described below. // // InputNone // @@ -40,16 +41,16 @@ type InputLoaderSession interface { // input, we return it. Otherwise we return a single, empty entry // that causes experiments that don't require input to run once. // -// InputOrQueryTestLists +// InputOrQueryBackend // // We gather input from StaticInput and SourceFiles. If there is // input, we return it. Otherwise, we use OONI's probe services -// to gather input using the test lists API. +// to gather input using the best API for the task. // // InputStrictlyRequired // -// Like InputOrQueryTestLists but, if there is no input, it's an -// user error and we just abort running the experiment. +// We gather input from StaticInput and SourceFiles. If there is +// input, we return it. Otherwise, we return an error. type InputLoader interface { // Load attempts to load input using the specified input loader. We will // return a list of URLs because this is the only input we support. @@ -64,7 +65,8 @@ type InputLoaderConfig struct { // SourceFiles contains optional files to read input // from. Each file should contain a single input string - // per line. We will fail if any file is unreadable. + // per line. We will fail if any file is unreadable + // as well as if any file is empty. SourceFiles []string // InputPolicy specifies the input policy for the @@ -94,10 +96,17 @@ func NewInputLoader(config InputLoaderConfig) InputLoader { return inputLoader{InputLoaderConfig: config} } +// TODO(bassosimone): it seems there's no reason to return an +// interface from the constructor. Generally, "Effective Go" +// recommends that an interface is used by the receiver rather +// than by the sender. We should follow that rule of thumb. + +// inputLoader is the concrete implementation of InputLoader. type inputLoader struct { InputLoaderConfig } +// verifies that inputLoader is an InputLoader. var _ InputLoader = inputLoader{} // Load attempts to load input using the specified input loader. We will @@ -106,8 +115,8 @@ func (il inputLoader) Load(ctx context.Context) ([]model.URLInfo, error) { switch il.InputPolicy { case InputOptional: return il.loadOptional() - case InputOrQueryTestLists: - return il.loadOrQueryTestList(ctx) + case InputOrQueryBackend: + return il.loadOrQueryBackend(ctx) case InputStrictlyRequired: return il.loadStrictlyRequired(ctx) default: @@ -115,21 +124,26 @@ func (il inputLoader) Load(ctx context.Context) ([]model.URLInfo, error) { } } +// loadNone implements the InputNone policy. func (il inputLoader) loadNone() ([]model.URLInfo, error) { if len(il.StaticInputs) > 0 || len(il.SourceFiles) > 0 { return nil, ErrNoInputExpected } + // Note that we need to return a single empty entry. return []model.URLInfo{{}}, nil } +// loadOptional implements the InputOptional policy. func (il inputLoader) loadOptional() ([]model.URLInfo, error) { inputs, err := il.loadLocal() if err == nil && len(inputs) <= 0 { + // Note that we need to return a single empty entry. inputs = []model.URLInfo{{}} } return inputs, err } +// loadStrictlyRequired implements the InputStrictlyRequired policy. func (il inputLoader) loadStrictlyRequired(ctx context.Context) ([]model.URLInfo, error) { inputs, err := il.loadLocal() if err != nil || len(inputs) > 0 { @@ -138,14 +152,16 @@ func (il inputLoader) loadStrictlyRequired(ctx context.Context) ([]model.URLInfo return nil, ErrInputRequired } -func (il inputLoader) loadOrQueryTestList(ctx context.Context) ([]model.URLInfo, error) { +// loadOrQueryBackend implements the InputOrQueryBackend policy. +func (il inputLoader) loadOrQueryBackend(ctx context.Context) ([]model.URLInfo, error) { inputs, err := il.loadLocal() if err != nil || len(inputs) > 0 { return inputs, err } - return il.loadRemote(loadRemoteConfig{ctx: ctx, session: il.Session}) + return il.loadRemote(inputLoaderLoadRemoteConfig{ctx: ctx, session: il.Session}) } +// loadLocal loads inputs from StaticInputs and SourceFiles. func (il inputLoader) loadLocal() ([]model.URLInfo, error) { inputs := []model.URLInfo{} for _, input := range il.StaticInputs { @@ -165,7 +181,12 @@ func (il inputLoader) loadLocal() ([]model.URLInfo, error) { return inputs, nil } -func (il inputLoader) readfile(filepath string, open func(string) (fsx.File, error)) ([]model.URLInfo, error) { +// inputLoaderOpenFn is the type of the function to open a file. +type inputLoaderOpenFn func(filepath string) (fsx.File, error) + +// readfile reads inputs from the specified file. The open argument should be +// compatibile with stdlib's fs.Open and helps us with unit testing. +func (il inputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]model.URLInfo, error) { inputs := []model.URLInfo{} filep, err := open(filepath) if err != nil { @@ -188,12 +209,15 @@ func (il inputLoader) readfile(filepath string, open func(string) (fsx.File, err return inputs, nil } -type loadRemoteConfig struct { +// inputLoaderLoadRemoteConfig contains configuration for loading the input from +// a remote source (which currently is _only_ the OONI backend). +type inputLoaderLoadRemoteConfig struct { ctx context.Context session InputLoaderSession } -func (il inputLoader) loadRemote(conf loadRemoteConfig) ([]model.URLInfo, error) { +// loadRemote loads inputs from a remote source. +func (il inputLoader) loadRemote(conf inputLoaderLoadRemoteConfig) ([]model.URLInfo, error) { if err := conf.session.MaybeLookupLocationContext(conf.ctx); err != nil { return nil, err } diff --git a/internal/engine/inputloader_integration_test.go b/internal/engine/inputloader_integration_test.go index c289256..744157a 100644 --- a/internal/engine/inputloader_integration_test.go +++ b/internal/engine/inputloader_integration_test.go @@ -211,7 +211,7 @@ func TestInputLoaderInputOrQueryTestListsWithInput(t *testing.T) { "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, - InputPolicy: engine.InputOrQueryTestLists, + InputPolicy: engine.InputOrQueryBackend, }) ctx := context.Background() out, err := il.Load(ctx) @@ -247,7 +247,7 @@ func TestInputLoaderInputOrQueryTestListsWithNoInputAndCancelledContext(t *testi } defer sess.Close() il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOrQueryTestLists, + InputPolicy: engine.InputOrQueryBackend, Session: sess, }) ctx, cancel := context.WithCancel(context.Background()) @@ -282,7 +282,7 @@ func TestInputLoaderInputOrQueryTestListsWithNoInput(t *testing.T) { } defer sess.Close() il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOrQueryTestLists, + InputPolicy: engine.InputOrQueryBackend, Session: sess, URLLimit: 30, }) @@ -298,7 +298,7 @@ func TestInputLoaderInputOrQueryTestListsWithNoInput(t *testing.T) { func TestInputLoaderInputOrQueryTestListsWithEmptyFile(t *testing.T) { il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOrQueryTestLists, + InputPolicy: engine.InputOrQueryBackend, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader3.txt", // we want it before inputloader2.txt diff --git a/internal/engine/inputloader_test.go b/internal/engine/inputloader_test.go index 9e231ef..d40483e 100644 --- a/internal/engine/inputloader_test.go +++ b/internal/engine/inputloader_test.go @@ -65,7 +65,7 @@ func (InputLoaderBrokenSession) ProbeCC() string { func TestInputLoaderNewOrchestraClientFailure(t *testing.T) { il := inputLoader{} - lrc := loadRemoteConfig{ + lrc := inputLoaderLoadRemoteConfig{ ctx: context.Background(), session: InputLoaderBrokenSession{}, } @@ -98,7 +98,7 @@ func (InputLoaderBrokenOrchestraClient) FetchURLList(ctx context.Context, config func TestInputLoaderFetchURLListFailure(t *testing.T) { il := inputLoader{} - lrc := loadRemoteConfig{ + lrc := inputLoaderLoadRemoteConfig{ ctx: context.Background(), session: InputLoaderBrokenSession{ OrchestraClient: InputLoaderBrokenOrchestraClient{}, diff --git a/internal/engine/internal/sessionresolver/resolvermaker.go b/internal/engine/internal/sessionresolver/resolvermaker.go index 0874d7b..2452de0 100644 --- a/internal/engine/internal/sessionresolver/resolvermaker.go +++ b/internal/engine/internal/sessionresolver/resolvermaker.go @@ -21,7 +21,7 @@ const systemResolverURL = "system:///" // allmakers contains all the makers in a list. We use the http3 // prefix to indicate we wanna use http3. The code will translate -// this to https and set the proper next options. +// this to https and set the proper netx options. var allmakers = []*resolvermaker{{ url: "https://cloudflare-dns.com/dns-query", }, { diff --git a/pkg/oonimkall/internal/tasks/runner.go b/pkg/oonimkall/internal/tasks/runner.go index cc106c8..34345e2 100644 --- a/pkg/oonimkall/internal/tasks/runner.go +++ b/pkg/oonimkall/internal/tasks/runner.go @@ -176,7 +176,7 @@ func (r *Runner) Run(ctx context.Context) { builder.SetCallbacks(&runnerCallbacks{emitter: r.emitter}) if len(r.settings.Inputs) <= 0 { switch builder.InputPolicy() { - case engine.InputOrQueryTestLists, engine.InputStrictlyRequired: + case engine.InputOrQueryBackend, engine.InputStrictlyRequired: r.emitter.EmitFailureStartup("no input provided") return } @@ -209,7 +209,7 @@ func (r *Runner) Run(ctx context.Context) { // this policy in the future, but for now this covers in a // reasonable way web connectivity, so we should be ok. switch builder.InputPolicy() { - case engine.InputOrQueryTestLists, engine.InputStrictlyRequired: + case engine.InputOrQueryBackend, engine.InputStrictlyRequired: var cancel context.CancelFunc ctx, cancel = context.WithTimeout( ctx, time.Duration(r.settings.Options.MaxRuntime)*time.Second, From e0b0dfedc1e77ae725ce16175fcacb999107e31d Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 29 Mar 2021 15:04:41 +0200 Subject: [PATCH 11/28] feat(session): expose CheckIn method (#266) * feat(session): expose CheckIn method It seems to me the right thing to do is to query the CheckIn API from the Session rather than querying it from InputLoader. Then, InputLoader could just take a reference to a Session-like interface that allows this functionality. So, this diff exposes the Session.CheckIn method. Doing that, in turn, required some refactoring to allow for more and better unit tests. While doing that, I also noticed that Session required a mutex to be a well-behaving type, so I did that. While doing that, I also tried to cover all the lines in session.go and, as part of that, I have removed unused code. Reference issue: https://github.com/ooni/probe/issues/1299. * fix: reinstate comment I shan't have removed * fix: repair broken test * fix: a bit more coverage, annotations, etc. * Update internal/engine/session.go * Update internal/engine/session_integration_test.go * Update internal/engine/session_internal_test.go --- internal/engine/experiment.go | 7 +- internal/engine/session.go | 193 ++++++++++++++++--- internal/engine/session_integration_test.go | 91 +++++---- internal/engine/session_internal_test.go | 195 +++++++++++++++++++- pkg/oonimkall/session.go | 4 +- pkg/oonimkall/session_integration_test.go | 12 +- pkg/oonimkall/task_integration_test.go | 6 +- 7 files changed, 440 insertions(+), 68 deletions(-) diff --git a/internal/engine/experiment.go b/internal/engine/experiment.go index 3669123..0ba160c 100644 --- a/internal/engine/experiment.go +++ b/internal/engine/experiment.go @@ -1,4 +1,4 @@ -// Package engine contains the engine API +// Package engine contains the engine API. package engine import ( @@ -188,10 +188,7 @@ func (e *Experiment) OpenReportContext(ctx context.Context) error { Counter: e.byteCounter, }, } - if e.session.selectedProbeService == nil { - return errors.New("no probe services selected") - } - client, err := probeservices.NewClient(e.session, *e.session.selectedProbeService) + client, err := e.session.NewProbeServicesClient(ctx) if err != nil { e.session.logger.Debugf("%+v", err) return err diff --git a/internal/engine/session.go b/internal/engine/session.go index e80d99d..862e14a 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -40,7 +40,7 @@ type SessionConfig struct { TorBinary string } -// Session is a measurement session +// Session is a measurement session. type Session struct { assetsDir string availableProbeServices []model.Service @@ -63,6 +63,32 @@ type Session struct { tunnelMu sync.Mutex tunnelName string tunnel tunnel.Tunnel + + // mu provides mutual exclusion. + mu sync.Mutex + + // testLookupLocationContext is a an optional hook for testing + // allowing us to mock LookupLocationContext. + testLookupLocationContext func(ctx context.Context) (*geolocate.Results, error) + + // testMaybeLookupBackendsContext is an optional hook for testing + // allowing us to mock MaybeLookupBackendsContext. + testMaybeLookupBackendsContext func(ctx context.Context) error + + // testMaybeLookupLocationContext is an optional hook for testing + // allowing us to mock MaybeLookupLocationContext. + testMaybeLookupLocationContext func(ctx context.Context) error + + // testNewProbeServicesClientForCheckIn is an optional hook for testing + // allowing us to mock NewProbeServicesClient when calling CheckIn. + testNewProbeServicesClientForCheckIn func(ctx context.Context) ( + sessionProbeServicesClientForCheckIn, error) +} + +// sessionProbeServicesClientForCheckIn returns the probe services +// client that we should be using for performing the check-in. +type sessionProbeServicesClientForCheckIn interface { + CheckIn(ctx context.Context, config model.CheckInConfig) (*model.CheckInInfo, error) } // NewSession creates a new session or returns an error @@ -138,12 +164,98 @@ func (s *Session) KibiBytesSent() float64 { return s.byteCounter.KibiBytesSent() } +// CheckIn calls the check-in API. The input arguments MUST NOT +// be nil. Before querying the API, this function will ensure +// that the config structure does not contain any field that +// SHOULD be initialized and is not initialized. Whenever there +// is a field that is not initialized, we will attempt to set +// a reasonable default value for such a field. This list describes +// the current defaults we'll choose: +// +// - Platform: if empty, set to Session.Platform(); +// +// - ProbeASN: if empty, set to Session.ProbeASNString(); +// +// - ProbeCC: if empty, set to Session.ProbeCC(); +// +// - RunType: if empty, set to "timed"; +// +// - SoftwareName: if empty, set to Session.SoftwareName(); +// +// - SoftwareVersion: if empty, set to Session.SoftwareVersion(); +// +// - WebConnectivity.CategoryCodes: if nil, we will allocate +// an empty array (the API does not like nil). +// +// Because we MAY need to know the current ASN and CC, this +// function MAY call MaybeLookupLocationContext. +// +// The return value is either the check-in response or an error. +func (s *Session) CheckIn( + ctx context.Context, config *model.CheckInConfig) (*model.CheckInInfo, error) { + if err := s.maybeLookupLocationContext(ctx); err != nil { + return nil, err + } + client, err := s.newProbeServicesClientForCheckIn(ctx) + if err != nil { + return nil, err + } + if config.Platform == "" { + config.Platform = s.Platform() + } + if config.ProbeASN == "" { + config.ProbeASN = s.ProbeASNString() + } + if config.ProbeCC == "" { + config.ProbeCC = s.ProbeCC() + } + if config.RunType == "" { + config.RunType = "timed" // most conservative choice + } + if config.SoftwareName == "" { + config.SoftwareName = s.SoftwareName() + } + if config.SoftwareVersion == "" { + config.SoftwareVersion = s.SoftwareVersion() + } + if config.WebConnectivity.CategoryCodes == nil { + config.WebConnectivity.CategoryCodes = []string{} + } + return client.CheckIn(ctx, *config) +} + +// maybeLookupLocationContext is a wrapper for MaybeLookupLocationContext that calls +// the configurable testMaybeLookupLocationContext mock, if configured, and the +// real MaybeLookupLocationContext API otherwise. +func (s *Session) maybeLookupLocationContext(ctx context.Context) error { + if s.testMaybeLookupLocationContext != nil { + return s.testMaybeLookupLocationContext(ctx) + } + return s.MaybeLookupLocationContext(ctx) +} + +// newProbeServicesClientForCheckIn is a wrapper for NewProbeServicesClientForCheckIn +// that calls the configurable testNewProbeServicesClientForCheckIn mock, if +// configured, and the real NewProbeServicesClient API otherwise. +func (s *Session) newProbeServicesClientForCheckIn( + ctx context.Context) (sessionProbeServicesClientForCheckIn, error) { + if s.testNewProbeServicesClientForCheckIn != nil { + return s.testNewProbeServicesClientForCheckIn(ctx) + } + client, err := s.NewProbeServicesClient(ctx) + if err != nil { + return nil, err + } + return client, nil +} + // Close ensures that we close all the idle connections that the HTTP clients // we are currently using may have created. It will also remove the temp dir // that contains data from this session. Not calling this function may likely // cause memory leaks in your application because of open idle connections, // as well as excessive usage of disk space. func (s *Session) Close() error { + // TODO(bassosimone): introduce a sync.Once to make this method idempotent. s.httpDefaultTransport.CloseIdleConnections() s.resolver.CloseIdleConnections() s.logger.Infof("%s", s.resolver.Stats()) @@ -161,6 +273,8 @@ func (s *Session) CountryDatabasePath() string { // GetTestHelpersByName returns the available test helpers that // use the specified name, or false if there's none. func (s *Session) GetTestHelpersByName(name string) ([]model.Service, bool) { + defer s.mu.Unlock() + s.mu.Lock() services, ok := s.availableTestHelpers[name] return services, ok } @@ -187,12 +301,7 @@ func (s *Session) MaybeLookupLocation() error { // MaybeLookupBackends is a caching OONI backends lookup call. func (s *Session) MaybeLookupBackends() error { - return s.maybeLookupBackends(context.Background()) -} - -// MaybeLookupBackendsContext is like MaybeLookupBackends but with context. -func (s *Session) MaybeLookupBackendsContext(ctx context.Context) (err error) { - return s.maybeLookupBackends(ctx) + return s.MaybeLookupBackendsContext(context.Background()) } // ErrAlreadyUsingProxy indicates that we cannot create a tunnel with @@ -213,6 +322,7 @@ var ErrAlreadyUsingProxy = errors.New( // // The tunnel will be closed by session.Close(). func (s *Session) MaybeStartTunnel(ctx context.Context, name string) error { + // TODO(bassosimone): see if we can unify tunnelMu and mu. s.tunnelMu.Lock() defer s.tunnelMu.Unlock() if s.tunnel != nil && s.tunnelName == name { @@ -258,11 +368,15 @@ func (s *Session) NewExperimentBuilder(name string) (*ExperimentBuilder, error) // OONI probe services. This function will benchmark the available // probe services, and select the fastest. In case all probe services // seem to be down, we try again applying circumvention tactics. +// This function will fail IMMEDIATELY if given a cancelled context. func (s *Session) NewProbeServicesClient(ctx context.Context) (*probeservices.Client, error) { - if err := s.maybeLookupBackends(ctx); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() // helps with testing + } + if err := s.maybeLookupBackendsContext(ctx); err != nil { return nil, err } - if err := s.MaybeLookupLocationContext(ctx); err != nil { + if err := s.maybeLookupLocationContext(ctx); err != nil { return nil, err } if s.selectedProbeServiceHook != nil { @@ -313,6 +427,8 @@ func (s *Session) ProbeASNString() string { // ProbeASN returns the probe ASN as an integer. func (s *Session) ProbeASN() uint { + defer s.mu.Unlock() + s.mu.Lock() asn := geolocate.DefaultProbeASN if s.location != nil { asn = s.location.ASN @@ -322,6 +438,8 @@ func (s *Session) ProbeASN() uint { // ProbeCC returns the probe CC. func (s *Session) ProbeCC() string { + defer s.mu.Unlock() + s.mu.Lock() cc := geolocate.DefaultProbeCC if s.location != nil { cc = s.location.CountryCode @@ -331,6 +449,8 @@ func (s *Session) ProbeCC() string { // ProbeNetworkName returns the probe network name. func (s *Session) ProbeNetworkName() string { + defer s.mu.Unlock() + s.mu.Lock() nn := geolocate.DefaultProbeNetworkName if s.location != nil { nn = s.location.NetworkName @@ -340,6 +460,8 @@ func (s *Session) ProbeNetworkName() string { // ProbeIP returns the probe IP. func (s *Session) ProbeIP() string { + defer s.mu.Unlock() + s.mu.Lock() ip := geolocate.DefaultProbeIP if s.location != nil { ip = s.location.ProbeIP @@ -359,6 +481,8 @@ func (s *Session) ResolverASNString() string { // ResolverASN returns the resolver ASN func (s *Session) ResolverASN() uint { + defer s.mu.Unlock() + s.mu.Lock() asn := geolocate.DefaultResolverASN if s.location != nil { asn = s.location.ResolverASN @@ -368,6 +492,8 @@ func (s *Session) ResolverASN() uint { // ResolverIP returns the resolver IP func (s *Session) ResolverIP() string { + defer s.mu.Unlock() + s.mu.Lock() ip := geolocate.DefaultResolverIP if s.location != nil { ip = s.location.ResolverIP @@ -377,6 +503,8 @@ func (s *Session) ResolverIP() string { // ResolverNetworkName returns the resolver network name. func (s *Session) ResolverNetworkName() string { + defer s.mu.Unlock() + s.mu.Lock() nn := geolocate.DefaultResolverNetworkName if s.location != nil { nn = s.location.ResolverNetworkName @@ -423,7 +551,10 @@ func (s *Session) MaybeUpdateResources(ctx context.Context) error { return (&resourcesmanager.CopyWorker{DestDir: s.assetsDir}).Ensure() } -func (s *Session) getAvailableProbeServices() []model.Service { +// getAvailableProbeServicesUnlocked returns the available probe +// services. This function WILL NOT acquire the mu mutex, therefore, +// you MUST ensure you are using it from a locked context. +func (s *Session) getAvailableProbeServicesUnlocked() []model.Service { if len(s.availableProbeServices) > 0 { return s.availableProbeServices } @@ -458,22 +589,27 @@ func (s *Session) initOrchestraClient( return clnt, nil } -// LookupASN maps an IP address to its ASN and network name. This method implements -// LocationLookupASNLookupper.LookupASN. -func (s *Session) LookupASN(dbPath, ip string) (uint, string, error) { - return geolocate.LookupASN(dbPath, ip) -} - // ErrAllProbeServicesFailed indicates all probe services failed. var ErrAllProbeServicesFailed = errors.New("all available probe services failed") -func (s *Session) maybeLookupBackends(ctx context.Context) error { - // TODO(bassosimone): do we need a mutex here? +// maybeLookupBackendsContext uses testMaybeLookupBackendsContext if +// not nil, otherwise it calls MaybeLookupBackendsContext. +func (s *Session) maybeLookupBackendsContext(ctx context.Context) error { + if s.testMaybeLookupBackendsContext != nil { + return s.testMaybeLookupBackendsContext(ctx) + } + return s.MaybeLookupBackendsContext(ctx) +} + +// MaybeLookupBackendsContext is like MaybeLookupBackends but with context. +func (s *Session) MaybeLookupBackendsContext(ctx context.Context) error { + defer s.mu.Unlock() + s.mu.Lock() if s.selectedProbeService != nil { return nil } s.queryProbeServicesCount.Add(1) - candidates := probeservices.TryAll(ctx, s, s.getAvailableProbeServices()) + candidates := probeservices.TryAll(ctx, s, s.getAvailableProbeServicesUnlocked()) selected := probeservices.SelectBest(candidates) if selected == nil { return ErrAllProbeServicesFailed @@ -499,11 +635,26 @@ func (s *Session) LookupLocationContext(ctx context.Context) (*geolocate.Results return task.Run(ctx) } +// lookupLocationContext calls testLookupLocationContext if set and +// otherwise calls LookupLocationContext. +func (s *Session) lookupLocationContext(ctx context.Context) (*geolocate.Results, error) { + if s.testLookupLocationContext != nil { + return s.testLookupLocationContext(ctx) + } + return s.LookupLocationContext(ctx) +} + // MaybeLookupLocationContext is like MaybeLookupLocation but with a context -// that can be used to interrupt this long running operation. +// that can be used to interrupt this long running operation. This function +// will fail IMMEDIATELY if given a cancelled context. func (s *Session) MaybeLookupLocationContext(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() // helps with testing + } + defer s.mu.Unlock() + s.mu.Lock() if s.location == nil { - location, err := s.LookupLocationContext(ctx) + location, err := s.lookupLocationContext(ctx) if err != nil { return err } diff --git a/internal/engine/session_integration_test.go b/internal/engine/session_integration_test.go index 2de6d9b..f007637 100644 --- a/internal/engine/session_integration_test.go +++ b/internal/engine/session_integration_test.go @@ -3,6 +3,7 @@ package engine import ( "context" "errors" + "io" "io/ioutil" "net/http" "net/http/httptest" @@ -10,17 +11,34 @@ import ( "os" "syscall" "testing" - "time" "github.com/apex/log" "github.com/google/go-cmp/cmp" "github.com/ooni/probe-cli/v3/internal/engine/geolocate" "github.com/ooni/probe-cli/v3/internal/engine/model" - "github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" "github.com/ooni/probe-cli/v3/internal/version" ) +func TestSessionByteCounter(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + s := newSessionForTesting(t) + client := s.DefaultHTTPClient() + resp, err := client.Get("https://www.google.com") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil { + t.Fatal(err) + } + if s.KibiBytesSent() <= 0 || s.KibiBytesReceived() <= 0 { + t.Fatal("byte counter is not working") + } +} + func TestNewSessionBuilderChecks(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") @@ -307,6 +325,21 @@ func TestSessionLocationLookup(t *testing.T) { } } +func TestSessionCheckInWithRealAPI(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + sess := newSessionForTesting(t) + defer sess.Close() + results, err := sess.CheckIn(context.Background(), &model.CheckInConfig{}) + if err != nil { + t.Fatal(err) + } + if results == nil { + t.Fatal("expected non nil results here") + } +} + func TestSessionCloseCancelsTempDir(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") @@ -584,50 +617,31 @@ func TestNewOrchestraClientMaybeLookupBackendsFailure(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } + errMocked := errors.New("mocked error") sess := newSessionForTestingNoLookups(t) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // fail immediately - client, err := sess.NewOrchestraClient(ctx) - if !errors.Is(err, ErrAllProbeServicesFailed) { - t.Fatal("not the error we expected") + sess.testMaybeLookupBackendsContext = func(ctx context.Context) error { + return errMocked + } + client, err := sess.NewOrchestraClient(context.Background()) + if !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) } if client != nil { t.Fatal("expected nil client here") } } -type httpTransportThatSleeps struct { - txp netx.HTTPRoundTripper - st time.Duration -} - -func (txp httpTransportThatSleeps) RoundTrip(req *http.Request) (*http.Response, error) { - resp, err := txp.txp.RoundTrip(req) - time.Sleep(txp.st) - return resp, err -} - -func (txp httpTransportThatSleeps) CloseIdleConnections() { - txp.txp.CloseIdleConnections() -} - func TestNewOrchestraClientMaybeLookupLocationFailure(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } + errMocked := errors.New("mocked error") sess := newSessionForTestingNoLookups(t) - sess.httpDefaultTransport = httpTransportThatSleeps{ - txp: sess.httpDefaultTransport, - st: 5 * time.Second, + sess.testMaybeLookupLocationContext = func(ctx context.Context) error { + return errMocked } - // The transport sleeps for five seconds, so the context should be expired by - // the time in which we attempt at looking up the location. Because the - // implementation performs the round-trip and _then_ sleeps, it means we'll - // see the context expired error when performing the location lookup. - ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) - defer cancel() - client, err := sess.NewOrchestraClient(ctx) - if !errors.Is(err, geolocate.ErrAllIPLookuppersFailed) { + client, err := sess.NewOrchestraClient(context.Background()) + if !errors.Is(err, errMocked) { t.Fatalf("not the error we expected: %+v", err) } if client != nil { @@ -651,3 +665,14 @@ func TestNewOrchestraClientProbeServicesNewClientFailure(t *testing.T) { t.Fatal("expected nil client here") } } + +func TestSessionNewSubmitterReturnsNonNilSubmitter(t *testing.T) { + sess := newSessionForTesting(t) + subm, err := sess.NewSubmitter(context.Background()) + if err != nil { + t.Fatal(err) + } + if subm == nil { + t.Fatal("expected non nil submitter here") + } +} diff --git a/internal/engine/session_internal_test.go b/internal/engine/session_internal_test.go index 416b3b1..b90b6fc 100644 --- a/internal/engine/session_internal_test.go +++ b/internal/engine/session_internal_test.go @@ -1,6 +1,13 @@ package engine import ( + "context" + "errors" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/engine/geolocate" "github.com/ooni/probe-cli/v3/internal/engine/model" ) @@ -9,7 +16,7 @@ func (s *Session) SetAssetsDir(assetsDir string) { } func (s *Session) GetAvailableProbeServices() []model.Service { - return s.getAvailableProbeServices() + return s.getAvailableProbeServicesUnlocked() } func (s *Session) AppendAvailableProbeService(svc model.Service) { @@ -19,3 +26,189 @@ func (s *Session) AppendAvailableProbeService(svc model.Service) { func (s *Session) QueryProbeServicesCount() int64 { return s.queryProbeServicesCount.Load() } + +// mockableProbeServicesClientForCheckIn allows us to mock the +// probeservices.Client used by Session.CheckIn. +type mockableProbeServicesClientForCheckIn struct { + // Config is the config passed to the call. + Config *model.CheckInConfig + + // Results contains the results of the call. This field MUST be + // non-nil if and only if Error is nil. + Results *model.CheckInInfo + + // Error indicates whether the call failed. This field MUST be + // non-nil if and only if Error is nil. + Error error + + // mu provides mutual exclusion. + mu sync.Mutex +} + +// CheckIn implements sessionProbeServicesClientForCheckIn.CheckIn. +func (c *mockableProbeServicesClientForCheckIn) CheckIn( + ctx context.Context, config model.CheckInConfig) (*model.CheckInInfo, error) { + defer c.mu.Unlock() + c.mu.Lock() + if c.Config != nil { + return nil, errors.New("called more than once") + } + c.Config = &config + if c.Results == nil && c.Error == nil { + return nil, errors.New("misconfigured mockableProbeServicesClientForCheckIn") + } + return c.Results, c.Error +} + +func TestSessionCheckInSuccessful(t *testing.T) { + results := &model.CheckInInfo{ + WebConnectivity: &model.CheckInInfoWebConnectivity{ + ReportID: "xxx-x-xx", + URLs: []model.URLInfo{{ + CategoryCode: "NEWS", + CountryCode: "IT", + URL: "https://www.repubblica.it/", + }, { + CategoryCode: "NEWS", + CountryCode: "IT", + URL: "https://www.unita.it/", + }}, + }, + } + mockedClnt := &mockableProbeServicesClientForCheckIn{ + Results: results, + } + s := &Session{ + location: &geolocate.Results{ + ASN: 137, + CountryCode: "IT", + }, + softwareName: "miniooni", + softwareVersion: "0.1.0-dev", + testMaybeLookupLocationContext: func(ctx context.Context) error { + return nil + }, + testNewProbeServicesClientForCheckIn: func( + ctx context.Context) (sessionProbeServicesClientForCheckIn, error) { + return mockedClnt, nil + }, + } + out, err := s.CheckIn(context.Background(), &model.CheckInConfig{}) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(results, out); diff != "" { + t.Fatal(diff) + } + if mockedClnt.Config.Platform != s.Platform() { + t.Fatal("invalid Config.Platform") + } + if mockedClnt.Config.ProbeASN != "AS137" { + t.Fatal("invalid Config.ProbeASN") + } + if mockedClnt.Config.ProbeCC != "IT" { + t.Fatal("invalid Config.ProbeCC") + } + if mockedClnt.Config.RunType != "timed" { + t.Fatal("invalid Config.RunType") + } + if mockedClnt.Config.SoftwareName != "miniooni" { + t.Fatal("invalid Config.SoftwareName") + } + if mockedClnt.Config.SoftwareVersion != "0.1.0-dev" { + t.Fatal("invalid Config.SoftwareVersion") + } + if mockedClnt.Config.WebConnectivity.CategoryCodes == nil { + t.Fatal("invalid ...CategoryCodes") + } +} + +func TestSessionCheckInCannotLookupLocation(t *testing.T) { + errMocked := errors.New("mocked error") + s := &Session{ + testMaybeLookupLocationContext: func(ctx context.Context) error { + return errMocked + }, + } + out, err := s.CheckIn(context.Background(), &model.CheckInConfig{}) + if !errors.Is(err, errMocked) { + t.Fatal("no the error we expected", err) + } + if out != nil { + t.Fatal("expected nil result here") + } +} + +func TestSessionCheckInCannotCreateProbeServicesClient(t *testing.T) { + errMocked := errors.New("mocked error") + s := &Session{ + location: &geolocate.Results{ + ASN: 137, + CountryCode: "IT", + }, + softwareName: "miniooni", + softwareVersion: "0.1.0-dev", + testMaybeLookupLocationContext: func(ctx context.Context) error { + return nil + }, + testNewProbeServicesClientForCheckIn: func( + ctx context.Context) (sessionProbeServicesClientForCheckIn, error) { + return nil, errMocked + }, + } + out, err := s.CheckIn(context.Background(), &model.CheckInConfig{}) + if !errors.Is(err, errMocked) { + t.Fatal("no the error we expected", err) + } + if out != nil { + t.Fatal("expected nil result here") + } +} + +func TestLowercaseMaybeLookupLocationContextWithCancelledContext(t *testing.T) { + s := &Session{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately kill the context + err := s.maybeLookupLocationContext(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatal("not the error we expected", err) + } +} + +func TestNewProbeServicesClientForCheckIn(t *testing.T) { + s := &Session{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately kill the context + clnt, err := s.newProbeServicesClientForCheckIn(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatal("not the error we expected", err) + } + if clnt != nil { + t.Fatal("expected nil client here") + } +} + +func TestSessionNewSubmitterWithCancelledContext(t *testing.T) { + sess := newSessionForTesting(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // fail immediately + subm, err := sess.NewSubmitter(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatal("not the error we expected", err) + } + if subm != nil { + t.Fatal("expected nil submitter here") + } +} + +func TestSessionMaybeLookupLocationContextLookupLocationContextFailure(t *testing.T) { + errMocked := errors.New("mocked error") + sess := newSessionForTestingNoLookups(t) + sess.testLookupLocationContext = func(ctx context.Context) (*geolocate.Results, error) { + return nil, errMocked + } + err := sess.MaybeLookupLocationContext(context.Background()) + if !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } +} diff --git a/pkg/oonimkall/session.go b/pkg/oonimkall/session.go index 3a8f624..6458946 100644 --- a/pkg/oonimkall/session.go +++ b/pkg/oonimkall/session.go @@ -479,17 +479,17 @@ func (sess *Session) FetchURLList(ctx *Context, config *URLListConfig) (*URLList if config.CountryCode == "" { config.CountryCode = "XX" info, err := sess.sessp.LookupLocationContext(ctx.ctx) + // TODO(bassosimone): this piece of code feels wrong to me. We don't + // want to continue if we cannot discover the country. if err == nil && info != nil { config.CountryCode = info.CountryCode } } - cfg := model.URLListConfig{ Categories: config.Categories, CountryCode: config.CountryCode, Limit: config.Limit, } - result, err := psc.FetchURLList(ctx.ctx, cfg) if err != nil { return nil, err diff --git a/pkg/oonimkall/session_integration_test.go b/pkg/oonimkall/session_integration_test.go index 26f1bb6..725c872 100644 --- a/pkg/oonimkall/session_integration_test.go +++ b/pkg/oonimkall/session_integration_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - engine "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/engine/geolocate" "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/pkg/oonimkall" @@ -362,7 +361,7 @@ func TestCheckInNewProbeServicesFailure(t *testing.T) { config.WebConnectivity.Add("NEWS") config.WebConnectivity.Add("CULTR") result, err := sess.CheckIn(ctx, &config) - if !errors.Is(err, engine.ErrAllProbeServicesFailed) { + if !errors.Is(err, context.Canceled) { t.Fatalf("not the error we expected: %+v", err) } if result != nil { @@ -440,11 +439,18 @@ func TestFetchURLListSuccess(t *testing.T) { if result == nil || result.Results == nil { t.Fatal("got nil result") } - for _, entry := range result.Results { + for idx := int64(0); idx < result.Size(); idx++ { + entry := result.At(idx) if entry.CategoryCode != "NEWS" && entry.CategoryCode != "CULTR" { t.Fatalf("unexpected category code: %+v", entry) } } + if result.At(-1) != nil { + t.Fatal("expected nil here") + } + if result.At(result.Size()) != nil { + t.Fatal("expected nil here") + } } func TestFetchURLListWithCC(t *testing.T) { diff --git a/pkg/oonimkall/task_integration_test.go b/pkg/oonimkall/task_integration_test.go index f8d1081..e8e81de 100644 --- a/pkg/oonimkall/task_integration_test.go +++ b/pkg/oonimkall/task_integration_test.go @@ -58,10 +58,10 @@ func TestGood(t *testing.T) { if err := json.Unmarshal([]byte(eventstr), &event); err != nil { t.Fatal(err) } - if event.Key != "task_terminated" { - t.Fatalf("unexpected event.Key: %s", event.Key) + if event.Key == "task_terminated" { + break } - break + t.Fatalf("unexpected event.Key: %s", event.Key) } } From 5973c88a0548ebed2f0e9cd8c3291cbfed6b25a9 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 29 Mar 2021 18:46:26 +0200 Subject: [PATCH 12/28] feat(inputloader): use check-in to fetch URLs (#267) * ongoing work * reduce diff with master * feat(inputloader): use the check-in API Part of https://github.com/ooni/probe/issues/1299 * fix: better naming for a variable * chore: add more tests * fix: add one more TODO --- cmd/ooniprobe/internal/cli/run/run.go | 10 -- .../internal/nettests/web_connectivity.go | 25 +++- internal/engine/inputloader.go | 58 ++++---- .../engine/inputloader_integration_test.go | 10 +- internal/engine/inputloader_test.go | 137 +++++++++++------- internal/libminiooni/libminiooni.go | 23 ++- 6 files changed, 156 insertions(+), 107 deletions(-) diff --git a/cmd/ooniprobe/internal/cli/run/run.go b/cmd/ooniprobe/internal/cli/run/run.go index 32d27cd..be23822 100644 --- a/cmd/ooniprobe/internal/cli/run/run.go +++ b/cmd/ooniprobe/internal/cli/run/run.go @@ -1,8 +1,6 @@ package run import ( - "runtime" - "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/fatih/color" @@ -77,14 +75,6 @@ func init() { unattendedCmd := cmd.Command("unattended", "") unattendedCmd.Action(func(_ *kingpin.ParseContext) error { - // Until we have enabled the check-in API we're called every - // hour on darwin and we need to self throttle. - // TODO(bassosimone): switch to check-in and remove this hack. - switch runtime.GOOS { - case "darwin", "windows": - const veryFew = 10 - probe.Config().Nettests.WebsitesURLLimit = veryFew - } return functionalRun(func(name string, gr nettests.Group) bool { return gr.UnattendedOK == true }) diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index 1286845..c4752ad 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -6,16 +6,29 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database" engine "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/model" ) +// TODO(bassosimone): we should remove the limit argument and +// we should also remove it from the config. + +// TODO(bassosimone): we should propagate the kind of run +// to here such that we get the right runType. + +// TODO(bassosimone): we are breaking the use case in which +// someone choose the number of URLs explicitly via the config. + func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) { inputloader := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOrQueryBackend, - Session: ctl.Session, - SourceFiles: ctl.InputFiles, - StaticInputs: ctl.Inputs, - URLCategories: categories, - URLLimit: limit, + CheckInConfig: &model.CheckInConfig{ + WebConnectivity: model.CheckInConfigWebConnectivity{ + CategoryCodes: categories, + }, + }, + InputPolicy: engine.InputOrQueryBackend, + Session: ctl.Session, + SourceFiles: ctl.InputFiles, + StaticInputs: ctl.Inputs, }) testlist, err := inputloader.Load(context.Background()) var urls []string diff --git a/internal/engine/inputloader.go b/internal/engine/inputloader.go index ccd1583..d96e06d 100644 --- a/internal/engine/inputloader.go +++ b/internal/engine/inputloader.go @@ -12,6 +12,7 @@ import ( // These errors are returned by the InputLoader. var ( + ErrNoURLsReturned = errors.New("no URLs returned") ErrDetectedEmptyFile = errors.New("file did not contain any input") ErrInputRequired = errors.New("no input provided") ErrNoInputExpected = errors.New("we did not expect any input") @@ -20,9 +21,8 @@ var ( // InputLoaderSession is the session according to an InputLoader. We // introduce this abstraction because it helps us with testing. type InputLoaderSession interface { - MaybeLookupLocationContext(ctx context.Context) error - NewOrchestraClient(ctx context.Context) (model.ExperimentOrchestraClient, error) - ProbeCC() string + CheckIn(ctx context.Context, + config *model.CheckInConfig) (*model.CheckInInfo, error) } // InputLoader loads input according to the specified policy @@ -59,6 +59,22 @@ type InputLoader interface { // InputLoaderConfig contains config for InputLoader. type InputLoaderConfig struct { + // CheckInConfig contains options for the CheckIn API. If + // not set, then we'll create a default config. If set but + // there are fields inside it that are not set, then we + // will set them to a default value. + CheckInConfig *model.CheckInConfig + + // InputPolicy specifies the input policy for the + // current experiment. We will not load any input if + // the policy says we should not. You MUST fill in + // this field. + InputPolicy InputPolicy + + // Session is the current measurement session. You + // MUST fill in this field. + Session InputLoaderSession + // StaticInputs contains optional input to be added // to the resulting input list if possible. StaticInputs []string @@ -68,22 +84,6 @@ type InputLoaderConfig struct { // per line. We will fail if any file is unreadable // as well as if any file is empty. SourceFiles []string - - // InputPolicy specifies the input policy for the - // current experiment. We will not load any input if - // the policy says we should not. - InputPolicy InputPolicy - - // Session is the current measurement session. - Session InputLoaderSession - - // URLLimit is the optional limit on the number of URLs - // that probe services should return to us. - URLLimit int64 - - // URLCategories limits the categories of URLs that - // probe services should return to us. - URLCategories []string } // NewInputLoader creates a new InputLoader. @@ -218,16 +218,20 @@ type inputLoaderLoadRemoteConfig struct { // loadRemote loads inputs from a remote source. func (il inputLoader) loadRemote(conf inputLoaderLoadRemoteConfig) ([]model.URLInfo, error) { - if err := conf.session.MaybeLookupLocationContext(conf.ctx); err != nil { - return nil, err + config := il.CheckInConfig + if config == nil { + // Note: Session.CheckIn documentation says it will fill in + // any field with a required value with a reasonable default + // if such value is missing. So, here we just need to be + // concerned about NOT passing it a NULL pointer. + config = &model.CheckInConfig{} } - client, err := conf.session.NewOrchestraClient(conf.ctx) + reply, err := conf.session.CheckIn(conf.ctx, config) if err != nil { return nil, err } - return client.FetchURLList(conf.ctx, model.URLListConfig{ - CountryCode: conf.session.ProbeCC(), - Limit: il.URLLimit, - Categories: il.URLCategories, - }) + if reply.WebConnectivity == nil || len(reply.WebConnectivity.URLs) <= 0 { + return nil, ErrNoURLsReturned + } + return reply.WebConnectivity.URLs, nil } diff --git a/internal/engine/inputloader_integration_test.go b/internal/engine/inputloader_integration_test.go index 744157a..1662746 100644 --- a/internal/engine/inputloader_integration_test.go +++ b/internal/engine/inputloader_integration_test.go @@ -204,7 +204,7 @@ func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { } } -func TestInputLoaderInputOrQueryTestListsWithInput(t *testing.T) { +func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) { il := engine.NewInputLoader(engine.InputLoaderConfig{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ @@ -233,7 +233,7 @@ func TestInputLoaderInputOrQueryTestListsWithInput(t *testing.T) { } } -func TestInputLoaderInputOrQueryTestListsWithNoInputAndCancelledContext(t *testing.T) { +func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing.T) { sess, err := engine.NewSession(engine.SessionConfig{ AssetsDir: "testdata", KVStore: kvstore.NewMemoryKeyValueStore(), @@ -261,7 +261,7 @@ func TestInputLoaderInputOrQueryTestListsWithNoInputAndCancelledContext(t *testi } } -func TestInputLoaderInputOrQueryTestListsWithNoInput(t *testing.T) { +func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } @@ -284,7 +284,6 @@ func TestInputLoaderInputOrQueryTestListsWithNoInput(t *testing.T) { il := engine.NewInputLoader(engine.InputLoaderConfig{ InputPolicy: engine.InputOrQueryBackend, Session: sess, - URLLimit: 30, }) ctx := context.Background() out, err := il.Load(ctx) @@ -292,11 +291,12 @@ func TestInputLoaderInputOrQueryTestListsWithNoInput(t *testing.T) { t.Fatal(err) } if len(out) < 10 { + // check-in SHOULD return AT LEAST 20 URLs at a time. t.Fatal("not the output length we expected") } } -func TestInputLoaderInputOrQueryTestListsWithEmptyFile(t *testing.T) { +func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { il := engine.NewInputLoader(engine.InputLoaderConfig{ InputPolicy: engine.InputOrQueryBackend, SourceFiles: []string{ diff --git a/internal/engine/inputloader_test.go b/internal/engine/inputloader_test.go index d40483e..43bfd47 100644 --- a/internal/engine/inputloader_test.go +++ b/internal/engine/inputloader_test.go @@ -8,6 +8,7 @@ import ( "syscall" "testing" + "github.com/google/go-cmp/cmp" "github.com/ooni/probe-cli/v3/internal/engine/internal/fsx" "github.com/ooni/probe-cli/v3/internal/engine/model" ) @@ -43,65 +44,33 @@ func TestInputLoaderReadfileScannerFailure(t *testing.T) { } } -type InputLoaderBrokenSession struct { - OrchestraClient model.ExperimentOrchestraClient - Error error +// InputLoaderMockableSession is a mockable session +// used by InputLoader tests. +type InputLoaderMockableSession struct { + // Output contains the output of CheckIn. It should + // be nil when Error is not-nil. + Output *model.CheckInInfo + + // Error is the error to be returned by CheckIn. It + // should be nil when Output is not-nil. + Error error } -func (InputLoaderBrokenSession) MaybeLookupLocationContext(ctx context.Context) error { - return nil -} - -func (ilbs InputLoaderBrokenSession) NewOrchestraClient(ctx context.Context) (model.ExperimentOrchestraClient, error) { - if ilbs.OrchestraClient != nil { - return ilbs.OrchestraClient, nil +// CheckIn implements InputLoaderSession.CheckIn. +func (sess *InputLoaderMockableSession) CheckIn( + ctx context.Context, config *model.CheckInConfig) (*model.CheckInInfo, error) { + if sess.Output == nil && sess.Error == nil { + return nil, errors.New("both Output and Error are nil") } - return nil, io.EOF + return sess.Output, sess.Error } -func (InputLoaderBrokenSession) ProbeCC() string { - return "IT" -} - -func TestInputLoaderNewOrchestraClientFailure(t *testing.T) { - il := inputLoader{} - lrc := inputLoaderLoadRemoteConfig{ - ctx: context.Background(), - session: InputLoaderBrokenSession{}, - } - out, err := il.loadRemote(lrc) - if !errors.Is(err, io.EOF) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("expected nil output here") - } -} - -type InputLoaderBrokenOrchestraClient struct{} - -func (InputLoaderBrokenOrchestraClient) CheckIn(ctx context.Context, config model.CheckInConfig) (*model.CheckInInfo, error) { - return nil, io.EOF -} - -func (InputLoaderBrokenOrchestraClient) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { - return nil, io.EOF -} - -func (InputLoaderBrokenOrchestraClient) FetchTorTargets(ctx context.Context, cc string) (map[string]model.TorTarget, error) { - return nil, io.EOF -} - -func (InputLoaderBrokenOrchestraClient) FetchURLList(ctx context.Context, config model.URLListConfig) ([]model.URLInfo, error) { - return nil, io.EOF -} - -func TestInputLoaderFetchURLListFailure(t *testing.T) { +func TestInputLoaderCheckInFailure(t *testing.T) { il := inputLoader{} lrc := inputLoaderLoadRemoteConfig{ ctx: context.Background(), - session: InputLoaderBrokenSession{ - OrchestraClient: InputLoaderBrokenOrchestraClient{}, + session: &InputLoaderMockableSession{ + Error: io.EOF, }, } out, err := il.loadRemote(lrc) @@ -112,3 +81,69 @@ func TestInputLoaderFetchURLListFailure(t *testing.T) { t.Fatal("expected nil output here") } } + +func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) { + il := inputLoader{} + lrc := inputLoaderLoadRemoteConfig{ + ctx: context.Background(), + session: &InputLoaderMockableSession{ + Output: &model.CheckInInfo{}, + }, + } + out, err := il.loadRemote(lrc) + if !errors.Is(err, ErrNoURLsReturned) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("expected nil output here") + } +} + +func TestInputLoaderCheckInSuccessWithNoURLs(t *testing.T) { + il := inputLoader{} + lrc := inputLoaderLoadRemoteConfig{ + ctx: context.Background(), + session: &InputLoaderMockableSession{ + Output: &model.CheckInInfo{ + WebConnectivity: &model.CheckInInfoWebConnectivity{}, + }, + }, + } + out, err := il.loadRemote(lrc) + if !errors.Is(err, ErrNoURLsReturned) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("expected nil output here") + } +} + +func TestInputLoaderCheckInSuccessWithSomeURLs(t *testing.T) { + expect := []model.URLInfo{{ + CategoryCode: "NEWS", + CountryCode: "IT", + URL: "https://repubblica.it", + }, { + CategoryCode: "NEWS", + CountryCode: "IT", + URL: "https://corriere.it", + }} + il := inputLoader{} + lrc := inputLoaderLoadRemoteConfig{ + ctx: context.Background(), + session: &InputLoaderMockableSession{ + Output: &model.CheckInInfo{ + WebConnectivity: &model.CheckInInfoWebConnectivity{ + URLs: expect, + }, + }, + }, + } + out, err := il.loadRemote(lrc) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expect, out); diff != "" { + t.Fatal(diff) + } +} diff --git a/internal/libminiooni/libminiooni.go b/internal/libminiooni/libminiooni.go index df6c540..d0daa6e 100644 --- a/internal/libminiooni/libminiooni.go +++ b/internal/libminiooni/libminiooni.go @@ -12,6 +12,13 @@ // code at github.com/bassosimone/aladdin). package libminiooni +// TODO(bassosimone): now that this code cannot be imported from +// external sources anymore, we should move it back into the +// internal/cmd/miniooni folder and reduce the number of packages +// that are unnecessarily exposed inside ./internal/engine. + +// TODO(bassosimone): we need to deprecate or remove --limit. + import ( "context" "errors" @@ -140,10 +147,6 @@ func init() { ) } -func fatalWithString(msg string) { - panic(msg) -} - func fatalIfFalse(cond bool, msg string) { if !cond { panic(msg) @@ -371,11 +374,15 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { fatalOnError(err, "cannot create experiment builder") inputLoader := engine.NewInputLoader(engine.InputLoaderConfig{ + CheckInConfig: &model.CheckInConfig{ + RunType: "manual", + OnWiFi: true, // meaning: not on 4G + Charging: true, + }, + InputPolicy: builder.InputPolicy(), StaticInputs: currentOptions.Inputs, SourceFiles: currentOptions.InputFilePaths, - InputPolicy: builder.InputPolicy(), Session: sess, - URLLimit: currentOptions.Limit, }) inputs, err := inputLoader.Load(context.Background()) fatalOnError(err, "cannot load inputs") @@ -399,14 +406,14 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { }() submitter, err := engine.NewSubmitter(ctx, engine.SubmitterConfig{ - Enabled: currentOptions.NoCollector == false, + Enabled: !currentOptions.NoCollector, Session: sess, Logger: log.Log, }) fatalOnError(err, "cannot create submitter") saver, err := engine.NewSaver(engine.SaverConfig{ - Enabled: currentOptions.NoJSON == false, + Enabled: !currentOptions.NoJSON, Experiment: experiment, FilePath: currentOptions.ReportFile, Logger: log.Log, From f5b5ac47b0a7364c4238dbaad03b6c1972cd8da4 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 29 Mar 2021 19:03:53 +0200 Subject: [PATCH 13/28] refactor: merge libminiooni into cmd/miniooni (#268) We used to have an external package called libminiooni so that third parties could use it. We wrote this such that we could support github.com/bassosimone/aladdin. That was actually a not-so-good idea because it added to the APIs we needed to maintain. Since the merge of engine into cli, such an API is not public anymore and aladdin has been deprecated and archived. Therefore, we can now cleanup the situation and merge libminiooni into miniooni again, thus making the codebase more local. This cleanup has been identified while working on https://github.com/ooni/probe/issues/1299. --- .../{libminiooni => cmd/miniooni}/.gitignore | 0 internal/cmd/miniooni/README.md | 7 ++++--- .../miniooni}/libminiooni.go | 19 +------------------ internal/cmd/miniooni/libminiooni_test.go | 12 ++++++++++++ internal/cmd/miniooni/main.go | 6 +----- internal/libminiooni/README.md | 13 ------------- .../libminiooni_integration_test.go | 16 ---------------- 7 files changed, 18 insertions(+), 55 deletions(-) rename internal/{libminiooni => cmd/miniooni}/.gitignore (100%) rename internal/{libminiooni => cmd/miniooni}/libminiooni.go (93%) create mode 100644 internal/cmd/miniooni/libminiooni_test.go delete mode 100644 internal/libminiooni/README.md delete mode 100644 internal/libminiooni/libminiooni_integration_test.go diff --git a/internal/libminiooni/.gitignore b/internal/cmd/miniooni/.gitignore similarity index 100% rename from internal/libminiooni/.gitignore rename to internal/cmd/miniooni/.gitignore diff --git a/internal/cmd/miniooni/README.md b/internal/cmd/miniooni/README.md index ae2b3fd..c4ab7b6 100644 --- a/internal/cmd/miniooni/README.md +++ b/internal/cmd/miniooni/README.md @@ -3,6 +3,7 @@ This directory contains the source code of a simple CLI client that we use for research as well as for running QA scripts. We designed this tool to have a CLI similar to MK and OONI Probe v2.x to ease running Jafar -scripts that check whether these tools behave similarly. - -See also libminiooni. +scripts that check whether these tools behave similarly. Perfect backwards +compatibility was not a design goal for miniooni. Rather, we aimed to +have as little conflict as possible, such that we can run side-by-side +QA checks. diff --git a/internal/libminiooni/libminiooni.go b/internal/cmd/miniooni/libminiooni.go similarity index 93% rename from internal/libminiooni/libminiooni.go rename to internal/cmd/miniooni/libminiooni.go index d0daa6e..51fc51c 100644 --- a/internal/libminiooni/libminiooni.go +++ b/internal/cmd/miniooni/libminiooni.go @@ -1,21 +1,4 @@ -// Package libminiooni implements the cmd/miniooni CLI. Miniooni is our -// experimental client used for research and QA testing. -// -// This CLI has CLI options that do not conflict with Measurement Kit -// v0.10.x CLI options. There are some options conflict with the legacy -// OONI Probe CLI options. Perfect backwards compatibility is not a -// design goal for miniooni. Rather, we aim to have as little conflict -// as possible such that we can run side by side QA checks. -// -// We extracted this package from cmd/miniooni to allow us to further -// integrate the miniooni CLI into other binaries (see for example the -// code at github.com/bassosimone/aladdin). -package libminiooni - -// TODO(bassosimone): now that this code cannot be imported from -// external sources anymore, we should move it back into the -// internal/cmd/miniooni folder and reduce the number of packages -// that are unnecessarily exposed inside ./internal/engine. +package main // TODO(bassosimone): we need to deprecate or remove --limit. diff --git a/internal/cmd/miniooni/libminiooni_test.go b/internal/cmd/miniooni/libminiooni_test.go new file mode 100644 index 0000000..01ec1a5 --- /dev/null +++ b/internal/cmd/miniooni/libminiooni_test.go @@ -0,0 +1,12 @@ +package main + +import "testing" + +func TestSimple(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + MainWithConfiguration("example", Options{ + Yes: true, + }) +} diff --git a/internal/cmd/miniooni/main.go b/internal/cmd/miniooni/main.go index 79546db..e46620f 100644 --- a/internal/cmd/miniooni/main.go +++ b/internal/cmd/miniooni/main.go @@ -1,14 +1,10 @@ // Command miniooni is a simple binary for research and QA purposes // with a CLI interface similar to MK and OONI Probe v2.x. -// -// See also libminiooni, which is where we implement this CLI. package main import ( "fmt" "os" - - "github.com/ooni/probe-cli/v3/internal/libminiooni" ) func main() { @@ -17,5 +13,5 @@ func main() { fmt.Fprintf(os.Stderr, "%s", s) } }() - libminiooni.Main() + Main() } diff --git a/internal/libminiooni/README.md b/internal/libminiooni/README.md deleted file mode 100644 index 693b629..0000000 --- a/internal/libminiooni/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Package github.com/ooni/probe-cli/internal/libminiooni - -Package libminiooni implements the cmd/miniooni CLI. Miniooni is our -experimental client used for research and QA testing. - -This CLI has CLI options that do not conflict with Measurement Kit -v0.10.x CLI options. There are some options conflict with the legacy -OONI Probe CLI options. Perfect backwards compatibility is not a -design goal for miniooni. Rather, we aim to have as little conflict -as possible such that we can run side by side QA checks. - -This package was split off from cmd/miniooni in ooni/probe-engine. For -now we are keeping this split, but we will merge them in the future. diff --git a/internal/libminiooni/libminiooni_integration_test.go b/internal/libminiooni/libminiooni_integration_test.go deleted file mode 100644 index 66870ca..0000000 --- a/internal/libminiooni/libminiooni_integration_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package libminiooni_test - -import ( - "testing" - - "github.com/ooni/probe-cli/v3/internal/libminiooni" -) - -func TestSimple(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - libminiooni.MainWithConfiguration("example", libminiooni.Options{ - Yes: true, - }) -} From 87e52345860532845e0b120965802778be8b1a65 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 29 Mar 2021 19:19:06 +0200 Subject: [PATCH 14/28] refactor(inputloader): all fast tests together (#269) This change allows us to have all fasts tests together. They are mostly unit tests or integration tests that do not require the network. The advantage of this strategy is the following. We can now run all these tests with a single click in VSCode. In turn, doing that tells us which lines of code we are not covering. The tests requiring the network are in a separate file, so we can easily see which lines of code are testing without using the network and which ones instead depend on that. (Currently, 100% of the inputloader.go file is tested without using the network.) While there, rename the other file such that is clear that it contains tests requiring the network. We now have some tests in inputloader_test.go that are not strictly unit tests. This refactoring was identified as useful while working on https://github.com/ooni/probe/issues/1299. --- .../engine/inputloader_integration_test.go | 316 ------------------ internal/engine/inputloader_network_test.go | 46 +++ internal/engine/inputloader_test.go | 269 +++++++++++++++ 3 files changed, 315 insertions(+), 316 deletions(-) delete mode 100644 internal/engine/inputloader_integration_test.go create mode 100644 internal/engine/inputloader_network_test.go diff --git a/internal/engine/inputloader_integration_test.go b/internal/engine/inputloader_integration_test.go deleted file mode 100644 index 1662746..0000000 --- a/internal/engine/inputloader_integration_test.go +++ /dev/null @@ -1,316 +0,0 @@ -package engine_test - -import ( - "context" - "errors" - "syscall" - "testing" - - "github.com/apex/log" - "github.com/google/go-cmp/cmp" - engine "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/kvstore" - "github.com/ooni/probe-cli/v3/internal/engine/model" -) - -func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - StaticInputs: []string{"https://www.google.com/"}, - InputPolicy: engine.InputNone, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if !errors.Is(err, engine.ErrNoInputExpected) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", - }, - InputPolicy: engine.InputNone, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if !errors.Is(err, engine.ErrNoInputExpected) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputNoneWithBothInputs(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - StaticInputs: []string{"https://www.google.com/"}, - SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", - }, - InputPolicy: engine.InputNone, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if !errors.Is(err, engine.ErrNoInputExpected) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputNoneWithNoInput(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputNone, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if err != nil { - t.Fatal(err) - } - if len(out) != 1 || out[0].URL != "" { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputOptionalWithNoInput(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOptional, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if err != nil { - t.Fatal(err) - } - if len(out) != 1 || out[0].URL != "" { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputOptionalWithInput(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - StaticInputs: []string{"https://www.google.com/"}, - SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", - }, - InputPolicy: engine.InputOptional, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if err != nil { - t.Fatal(err) - } - if len(out) != 5 { - t.Fatal("not the output length we expected") - } - expect := []model.URLInfo{ - {URL: "https://www.google.com/"}, - {URL: "https://www.x.org/"}, - {URL: "https://www.slashdot.org/"}, - {URL: "https://abc.xyz/"}, - {URL: "https://run.ooni.io/"}, - } - if diff := cmp.Diff(out, expect); diff != "" { - t.Fatal(diff) - } -} - -func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - StaticInputs: []string{"https://www.google.com/"}, - SourceFiles: []string{ - "testdata/inputloader1.txt", - "/nonexistent", - "testdata/inputloader2.txt", - }, - InputPolicy: engine.InputOptional, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if !errors.Is(err, syscall.ENOENT) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - StaticInputs: []string{"https://www.google.com/"}, - SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", - }, - InputPolicy: engine.InputStrictlyRequired, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if err != nil { - t.Fatal(err) - } - if len(out) != 5 { - t.Fatal("not the output length we expected") - } - expect := []model.URLInfo{ - {URL: "https://www.google.com/"}, - {URL: "https://www.x.org/"}, - {URL: "https://www.slashdot.org/"}, - {URL: "https://abc.xyz/"}, - {URL: "https://run.ooni.io/"}, - } - if diff := cmp.Diff(out, expect); diff != "" { - t.Fatal(diff) - } -} - -func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputStrictlyRequired, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if !errors.Is(err, engine.ErrInputRequired) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputStrictlyRequired, - SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader3.txt", // we want it before inputloader2.txt - "testdata/inputloader2.txt", - }, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if !errors.Is(err, engine.ErrDetectedEmptyFile) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - StaticInputs: []string{"https://www.google.com/"}, - SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", - }, - InputPolicy: engine.InputOrQueryBackend, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if err != nil { - t.Fatal(err) - } - if len(out) != 5 { - t.Fatal("not the output length we expected") - } - expect := []model.URLInfo{ - {URL: "https://www.google.com/"}, - {URL: "https://www.x.org/"}, - {URL: "https://www.slashdot.org/"}, - {URL: "https://abc.xyz/"}, - {URL: "https://run.ooni.io/"}, - } - if diff := cmp.Diff(out, expect); diff != "" { - t.Fatal(diff) - } -} - -func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing.T) { - sess, err := engine.NewSession(engine.SessionConfig{ - AssetsDir: "testdata", - KVStore: kvstore.NewMemoryKeyValueStore(), - Logger: log.Log, - SoftwareName: "miniooni", - SoftwareVersion: "0.1.0-dev", - TempDir: "testdata", - }) - if err != nil { - t.Fatal(err) - } - defer sess.Close() - il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOrQueryBackend, - Session: sess, - }) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // fail immediately - out, err := il.Load(ctx) - if !errors.Is(err, context.Canceled) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("not the output we expected") - } -} - -func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - sess, err := engine.NewSession(engine.SessionConfig{ - AvailableProbeServices: []model.Service{{ - Address: "https://ams-pg-test.ooni.org/", - Type: "https", - }}, - AssetsDir: "testdata", - KVStore: kvstore.NewMemoryKeyValueStore(), - Logger: log.Log, - SoftwareName: "miniooni", - SoftwareVersion: "0.1.0-dev", - TempDir: "testdata", - }) - if err != nil { - t.Fatal(err) - } - defer sess.Close() - il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOrQueryBackend, - Session: sess, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if err != nil { - t.Fatal(err) - } - if len(out) < 10 { - // check-in SHOULD return AT LEAST 20 URLs at a time. - t.Fatal("not the output length we expected") - } -} - -func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { - il := engine.NewInputLoader(engine.InputLoaderConfig{ - InputPolicy: engine.InputOrQueryBackend, - SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader3.txt", // we want it before inputloader2.txt - "testdata/inputloader2.txt", - }, - }) - ctx := context.Background() - out, err := il.Load(ctx) - if !errors.Is(err, engine.ErrDetectedEmptyFile) { - t.Fatalf("not the error we expected: %+v", err) - } - if out != nil { - t.Fatal("not the output we expected") - } -} diff --git a/internal/engine/inputloader_network_test.go b/internal/engine/inputloader_network_test.go new file mode 100644 index 0000000..7155d6d --- /dev/null +++ b/internal/engine/inputloader_network_test.go @@ -0,0 +1,46 @@ +package engine_test + +import ( + "context" + "testing" + + "github.com/apex/log" + engine "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/kvstore" + "github.com/ooni/probe-cli/v3/internal/engine/model" +) + +func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + sess, err := engine.NewSession(engine.SessionConfig{ + AvailableProbeServices: []model.Service{{ + Address: "https://ams-pg-test.ooni.org/", + Type: "https", + }}, + AssetsDir: "testdata", + KVStore: kvstore.NewMemoryKeyValueStore(), + Logger: log.Log, + SoftwareName: "miniooni", + SoftwareVersion: "0.1.0-dev", + TempDir: "testdata", + }) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + il := engine.NewInputLoader(engine.InputLoaderConfig{ + InputPolicy: engine.InputOrQueryBackend, + Session: sess, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if err != nil { + t.Fatal(err) + } + if len(out) < 10 { + // check-in SHOULD return AT LEAST 20 URLs at a time. + t.Fatal("not the output length we expected") + } +} diff --git a/internal/engine/inputloader_test.go b/internal/engine/inputloader_test.go index 43bfd47..7dbf9b1 100644 --- a/internal/engine/inputloader_test.go +++ b/internal/engine/inputloader_test.go @@ -8,11 +8,280 @@ import ( "syscall" "testing" + "github.com/apex/log" "github.com/google/go-cmp/cmp" "github.com/ooni/probe-cli/v3/internal/engine/internal/fsx" + "github.com/ooni/probe-cli/v3/internal/engine/kvstore" "github.com/ooni/probe-cli/v3/internal/engine/model" ) +func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + StaticInputs: []string{"https://www.google.com/"}, + InputPolicy: InputNone, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if !errors.Is(err, ErrNoInputExpected) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + SourceFiles: []string{ + "testdata/inputloader1.txt", + "testdata/inputloader2.txt", + }, + InputPolicy: InputNone, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if !errors.Is(err, ErrNoInputExpected) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputNoneWithBothInputs(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + StaticInputs: []string{"https://www.google.com/"}, + SourceFiles: []string{ + "testdata/inputloader1.txt", + "testdata/inputloader2.txt", + }, + InputPolicy: InputNone, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if !errors.Is(err, ErrNoInputExpected) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputNoneWithNoInput(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + InputPolicy: InputNone, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0].URL != "" { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputOptionalWithNoInput(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + InputPolicy: InputOptional, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0].URL != "" { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputOptionalWithInput(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + StaticInputs: []string{"https://www.google.com/"}, + SourceFiles: []string{ + "testdata/inputloader1.txt", + "testdata/inputloader2.txt", + }, + InputPolicy: InputOptional, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if err != nil { + t.Fatal(err) + } + if len(out) != 5 { + t.Fatal("not the output length we expected") + } + expect := []model.URLInfo{ + {URL: "https://www.google.com/"}, + {URL: "https://www.x.org/"}, + {URL: "https://www.slashdot.org/"}, + {URL: "https://abc.xyz/"}, + {URL: "https://run.ooni.io/"}, + } + if diff := cmp.Diff(out, expect); diff != "" { + t.Fatal(diff) + } +} + +func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + StaticInputs: []string{"https://www.google.com/"}, + SourceFiles: []string{ + "testdata/inputloader1.txt", + "/nonexistent", + "testdata/inputloader2.txt", + }, + InputPolicy: InputOptional, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if !errors.Is(err, syscall.ENOENT) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + StaticInputs: []string{"https://www.google.com/"}, + SourceFiles: []string{ + "testdata/inputloader1.txt", + "testdata/inputloader2.txt", + }, + InputPolicy: InputStrictlyRequired, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if err != nil { + t.Fatal(err) + } + if len(out) != 5 { + t.Fatal("not the output length we expected") + } + expect := []model.URLInfo{ + {URL: "https://www.google.com/"}, + {URL: "https://www.x.org/"}, + {URL: "https://www.slashdot.org/"}, + {URL: "https://abc.xyz/"}, + {URL: "https://run.ooni.io/"}, + } + if diff := cmp.Diff(out, expect); diff != "" { + t.Fatal(diff) + } +} + +func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + InputPolicy: InputStrictlyRequired, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if !errors.Is(err, ErrInputRequired) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + InputPolicy: InputStrictlyRequired, + SourceFiles: []string{ + "testdata/inputloader1.txt", + "testdata/inputloader3.txt", // we want it before inputloader2.txt + "testdata/inputloader2.txt", + }, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if !errors.Is(err, ErrDetectedEmptyFile) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + StaticInputs: []string{"https://www.google.com/"}, + SourceFiles: []string{ + "testdata/inputloader1.txt", + "testdata/inputloader2.txt", + }, + InputPolicy: InputOrQueryBackend, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if err != nil { + t.Fatal(err) + } + if len(out) != 5 { + t.Fatal("not the output length we expected") + } + expect := []model.URLInfo{ + {URL: "https://www.google.com/"}, + {URL: "https://www.x.org/"}, + {URL: "https://www.slashdot.org/"}, + {URL: "https://abc.xyz/"}, + {URL: "https://run.ooni.io/"}, + } + if diff := cmp.Diff(out, expect); diff != "" { + t.Fatal(diff) + } +} + +func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing.T) { + sess, err := NewSession(SessionConfig{ + AssetsDir: "testdata", + KVStore: kvstore.NewMemoryKeyValueStore(), + Logger: log.Log, + SoftwareName: "miniooni", + SoftwareVersion: "0.1.0-dev", + TempDir: "testdata", + }) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + il := NewInputLoader(InputLoaderConfig{ + InputPolicy: InputOrQueryBackend, + Session: sess, + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // fail immediately + out, err := il.Load(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("not the output we expected") + } +} + +func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { + il := NewInputLoader(InputLoaderConfig{ + InputPolicy: InputOrQueryBackend, + SourceFiles: []string{ + "testdata/inputloader1.txt", + "testdata/inputloader3.txt", // we want it before inputloader2.txt + "testdata/inputloader2.txt", + }, + }) + ctx := context.Background() + out, err := il.Load(ctx) + if !errors.Is(err, ErrDetectedEmptyFile) { + t.Fatalf("not the error we expected: %+v", err) + } + if out != nil { + t.Fatal("not the output we expected") + } +} + type InputLoaderBrokenFS struct{} func (InputLoaderBrokenFS) Open(filepath string) (fsx.File, error) { From 1da64f6d9f01cb0a7b53e7ae892b761efb66491c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 29 Mar 2021 19:37:32 +0200 Subject: [PATCH 15/28] fix(internal/fsx): remove pre Go 1.16 definitions (#270) Occurred to me while working on https://github.com/ooni/probe/issues/1299. --- internal/engine/inputloader.go | 3 ++- internal/engine/inputloader_test.go | 4 ++-- internal/engine/internal/fsx/fsx.go | 26 ++++++++---------------- internal/engine/internal/fsx/fsx_test.go | 5 +++-- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/internal/engine/inputloader.go b/internal/engine/inputloader.go index d96e06d..5561693 100644 --- a/internal/engine/inputloader.go +++ b/internal/engine/inputloader.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io/fs" "github.com/ooni/probe-cli/v3/internal/engine/internal/fsx" "github.com/ooni/probe-cli/v3/internal/engine/model" @@ -182,7 +183,7 @@ func (il inputLoader) loadLocal() ([]model.URLInfo, error) { } // inputLoaderOpenFn is the type of the function to open a file. -type inputLoaderOpenFn func(filepath string) (fsx.File, error) +type inputLoaderOpenFn func(filepath string) (fs.File, error) // readfile reads inputs from the specified file. The open argument should be // compatibile with stdlib's fs.Open and helps us with unit testing. diff --git a/internal/engine/inputloader_test.go b/internal/engine/inputloader_test.go index 7dbf9b1..e722944 100644 --- a/internal/engine/inputloader_test.go +++ b/internal/engine/inputloader_test.go @@ -4,13 +4,13 @@ import ( "context" "errors" "io" + "io/fs" "os" "syscall" "testing" "github.com/apex/log" "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-cli/v3/internal/engine/internal/fsx" "github.com/ooni/probe-cli/v3/internal/engine/kvstore" "github.com/ooni/probe-cli/v3/internal/engine/model" ) @@ -284,7 +284,7 @@ func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { type InputLoaderBrokenFS struct{} -func (InputLoaderBrokenFS) Open(filepath string) (fsx.File, error) { +func (InputLoaderBrokenFS) Open(filepath string) (fs.File, error) { return InputLoaderBrokenFile{}, nil } diff --git a/internal/engine/internal/fsx/fsx.go b/internal/engine/internal/fsx/fsx.go index 6805ed1..d5a8bf7 100644 --- a/internal/engine/internal/fsx/fsx.go +++ b/internal/engine/internal/fsx/fsx.go @@ -3,31 +3,18 @@ package fsx import ( "fmt" + "io/fs" "os" "syscall" ) -// File is a generic file. This interface is taken from the draft -// iofs golang design. We'll use fs.File when available. -type File interface { - Stat() (os.FileInfo, error) - Read([]byte) (int, error) - Close() error -} - -// FS is a generic file system. Like File, it's adapted from -// the draft iofs golang design document. -type FS interface { - Open(name string) (File, error) -} - // Open is a wrapper for os.Open that ensures that we're opening a file. -func Open(pathname string) (File, error) { +func Open(pathname string) (fs.File, error) { return OpenWithFS(filesystem{}, pathname) } // OpenWithFS is like Open but with explicit file system argument. -func OpenWithFS(fs FS, pathname string) (File, error) { +func OpenWithFS(fs fs.FS, pathname string) (fs.File, error) { file, err := fs.Open(pathname) if err != nil { return nil, err @@ -39,13 +26,16 @@ func OpenWithFS(fs FS, pathname string) (File, error) { } if info.IsDir() { file.Close() - return nil, fmt.Errorf("input path points to a directory: %w", syscall.EISDIR) + return nil, fmt.Errorf( + "input path points to a directory: %w", syscall.EISDIR) } return file, nil } +// filesystem is a private implementation of fs.FS. type filesystem struct{} -func (filesystem) Open(pathname string) (File, error) { +// Open implements fs.FS.Open. +func (filesystem) Open(pathname string) (fs.File, error) { return os.Open(pathname) } diff --git a/internal/engine/internal/fsx/fsx_test.go b/internal/engine/internal/fsx/fsx_test.go index 9555923..e38da71 100644 --- a/internal/engine/internal/fsx/fsx_test.go +++ b/internal/engine/internal/fsx/fsx_test.go @@ -2,6 +2,7 @@ package fsx_test import ( "errors" + "io/fs" "os" "sync/atomic" "syscall" @@ -26,8 +27,8 @@ func (FailingStatFile) Stat() (os.FileInfo, error) { return nil, errStatFailed } -func (fs FailingStatFS) Open(pathname string) (fsx.File, error) { - return FailingStatFile{CloseCount: fs.CloseCount}, nil +func (f FailingStatFS) Open(pathname string) (fs.File, error) { + return FailingStatFile(f), nil } func (fs FailingStatFile) Close() error { From b718335ee36361735ec3fdde18115a368754be35 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 29 Mar 2021 20:00:50 +0200 Subject: [PATCH 16/28] refactor(inputloader): remove unnecessary javisms (#271) Part of https://github.com/ooni/probe/issues/1299. --- .../internal/nettests/web_connectivity.go | 4 +- internal/cmd/miniooni/libminiooni.go | 4 +- internal/engine/inputloader.go | 62 +++---------- internal/engine/inputloader_network_test.go | 4 +- internal/engine/inputloader_test.go | 86 +++++++++---------- 5 files changed, 59 insertions(+), 101 deletions(-) diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index c4752ad..3e15472 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -19,7 +19,7 @@ import ( // someone choose the number of URLs explicitly via the config. func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) { - inputloader := engine.NewInputLoader(engine.InputLoaderConfig{ + inputloader := &engine.InputLoader{ CheckInConfig: &model.CheckInConfig{ WebConnectivity: model.CheckInConfigWebConnectivity{ CategoryCodes: categories, @@ -29,7 +29,7 @@ func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, ma Session: ctl.Session, SourceFiles: ctl.InputFiles, StaticInputs: ctl.Inputs, - }) + } testlist, err := inputloader.Load(context.Background()) var urls []string urlIDMap := make(map[int64]int64) diff --git a/internal/cmd/miniooni/libminiooni.go b/internal/cmd/miniooni/libminiooni.go index 51fc51c..765cc1a 100644 --- a/internal/cmd/miniooni/libminiooni.go +++ b/internal/cmd/miniooni/libminiooni.go @@ -356,7 +356,7 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { builder, err := sess.NewExperimentBuilder(experimentName) fatalOnError(err, "cannot create experiment builder") - inputLoader := engine.NewInputLoader(engine.InputLoaderConfig{ + inputLoader := &engine.InputLoader{ CheckInConfig: &model.CheckInConfig{ RunType: "manual", OnWiFi: true, // meaning: not on 4G @@ -366,7 +366,7 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { StaticInputs: currentOptions.Inputs, SourceFiles: currentOptions.InputFilePaths, Session: sess, - }) + } inputs, err := inputLoader.Load(context.Background()) fatalOnError(err, "cannot load inputs") diff --git a/internal/engine/inputloader.go b/internal/engine/inputloader.go index 5561693..2dc1f87 100644 --- a/internal/engine/inputloader.go +++ b/internal/engine/inputloader.go @@ -30,6 +30,9 @@ type InputLoaderSession interface { // either from command line and input files or from OONI services. The // behaviour depends on the input policy as described below. // +// You MUST NOT change any public field of this structure when +// in use, because that MAY lead to data races. +// // InputNone // // We fail if there is any StaticInput or any SourceFiles. If @@ -52,14 +55,7 @@ type InputLoaderSession interface { // // We gather input from StaticInput and SourceFiles. If there is // input, we return it. Otherwise, we return an error. -type InputLoader interface { - // Load attempts to load input using the specified input loader. We will - // return a list of URLs because this is the only input we support. - Load(ctx context.Context) ([]model.URLInfo, error) -} - -// InputLoaderConfig contains config for InputLoader. -type InputLoaderConfig struct { +type InputLoader struct { // CheckInConfig contains options for the CheckIn API. If // not set, then we'll create a default config. If set but // there are fields inside it that are not set, then we @@ -87,32 +83,9 @@ type InputLoaderConfig struct { SourceFiles []string } -// NewInputLoader creates a new InputLoader. -func NewInputLoader(config InputLoaderConfig) InputLoader { - // TODO(bassosimone): the current implementation stems from a - // simple refactoring from a previous implementation where - // we weren't using interfaces. Because now we're using interfaces, - // there is the opportunity to select behaviour here depending - // on the specified policy rather than later inside Load. - return inputLoader{InputLoaderConfig: config} -} - -// TODO(bassosimone): it seems there's no reason to return an -// interface from the constructor. Generally, "Effective Go" -// recommends that an interface is used by the receiver rather -// than by the sender. We should follow that rule of thumb. - -// inputLoader is the concrete implementation of InputLoader. -type inputLoader struct { - InputLoaderConfig -} - -// verifies that inputLoader is an InputLoader. -var _ InputLoader = inputLoader{} - // Load attempts to load input using the specified input loader. We will // return a list of URLs because this is the only input we support. -func (il inputLoader) Load(ctx context.Context) ([]model.URLInfo, error) { +func (il *InputLoader) Load(ctx context.Context) ([]model.URLInfo, error) { switch il.InputPolicy { case InputOptional: return il.loadOptional() @@ -126,7 +99,7 @@ func (il inputLoader) Load(ctx context.Context) ([]model.URLInfo, error) { } // loadNone implements the InputNone policy. -func (il inputLoader) loadNone() ([]model.URLInfo, error) { +func (il *InputLoader) loadNone() ([]model.URLInfo, error) { if len(il.StaticInputs) > 0 || len(il.SourceFiles) > 0 { return nil, ErrNoInputExpected } @@ -135,7 +108,7 @@ func (il inputLoader) loadNone() ([]model.URLInfo, error) { } // loadOptional implements the InputOptional policy. -func (il inputLoader) loadOptional() ([]model.URLInfo, error) { +func (il *InputLoader) loadOptional() ([]model.URLInfo, error) { inputs, err := il.loadLocal() if err == nil && len(inputs) <= 0 { // Note that we need to return a single empty entry. @@ -145,7 +118,7 @@ func (il inputLoader) loadOptional() ([]model.URLInfo, error) { } // loadStrictlyRequired implements the InputStrictlyRequired policy. -func (il inputLoader) loadStrictlyRequired(ctx context.Context) ([]model.URLInfo, error) { +func (il *InputLoader) loadStrictlyRequired(ctx context.Context) ([]model.URLInfo, error) { inputs, err := il.loadLocal() if err != nil || len(inputs) > 0 { return inputs, err @@ -154,16 +127,16 @@ func (il inputLoader) loadStrictlyRequired(ctx context.Context) ([]model.URLInfo } // loadOrQueryBackend implements the InputOrQueryBackend policy. -func (il inputLoader) loadOrQueryBackend(ctx context.Context) ([]model.URLInfo, error) { +func (il *InputLoader) loadOrQueryBackend(ctx context.Context) ([]model.URLInfo, error) { inputs, err := il.loadLocal() if err != nil || len(inputs) > 0 { return inputs, err } - return il.loadRemote(inputLoaderLoadRemoteConfig{ctx: ctx, session: il.Session}) + return il.loadRemote(ctx) } // loadLocal loads inputs from StaticInputs and SourceFiles. -func (il inputLoader) loadLocal() ([]model.URLInfo, error) { +func (il *InputLoader) loadLocal() ([]model.URLInfo, error) { inputs := []model.URLInfo{} for _, input := range il.StaticInputs { inputs = append(inputs, model.URLInfo{URL: input}) @@ -187,7 +160,7 @@ type inputLoaderOpenFn func(filepath string) (fs.File, error) // readfile reads inputs from the specified file. The open argument should be // compatibile with stdlib's fs.Open and helps us with unit testing. -func (il inputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]model.URLInfo, error) { +func (il *InputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]model.URLInfo, error) { inputs := []model.URLInfo{} filep, err := open(filepath) if err != nil { @@ -210,15 +183,8 @@ func (il inputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]model return inputs, nil } -// inputLoaderLoadRemoteConfig contains configuration for loading the input from -// a remote source (which currently is _only_ the OONI backend). -type inputLoaderLoadRemoteConfig struct { - ctx context.Context - session InputLoaderSession -} - // loadRemote loads inputs from a remote source. -func (il inputLoader) loadRemote(conf inputLoaderLoadRemoteConfig) ([]model.URLInfo, error) { +func (il *InputLoader) loadRemote(ctx context.Context) ([]model.URLInfo, error) { config := il.CheckInConfig if config == nil { // Note: Session.CheckIn documentation says it will fill in @@ -227,7 +193,7 @@ func (il inputLoader) loadRemote(conf inputLoaderLoadRemoteConfig) ([]model.URLI // concerned about NOT passing it a NULL pointer. config = &model.CheckInConfig{} } - reply, err := conf.session.CheckIn(conf.ctx, config) + reply, err := il.Session.CheckIn(ctx, config) if err != nil { return nil, err } diff --git a/internal/engine/inputloader_network_test.go b/internal/engine/inputloader_network_test.go index 7155d6d..3d10957 100644 --- a/internal/engine/inputloader_network_test.go +++ b/internal/engine/inputloader_network_test.go @@ -30,10 +30,10 @@ func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) { t.Fatal(err) } defer sess.Close() - il := engine.NewInputLoader(engine.InputLoaderConfig{ + il := &engine.InputLoader{ InputPolicy: engine.InputOrQueryBackend, Session: sess, - }) + } ctx := context.Background() out, err := il.Load(ctx) if err != nil { diff --git a/internal/engine/inputloader_test.go b/internal/engine/inputloader_test.go index e722944..7287c46 100644 --- a/internal/engine/inputloader_test.go +++ b/internal/engine/inputloader_test.go @@ -16,10 +16,10 @@ import ( ) func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ StaticInputs: []string{"https://www.google.com/"}, InputPolicy: InputNone, - }) + } ctx := context.Background() out, err := il.Load(ctx) if !errors.Is(err, ErrNoInputExpected) { @@ -31,13 +31,13 @@ func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) { } func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, InputPolicy: InputNone, - }) + } ctx := context.Background() out, err := il.Load(ctx) if !errors.Is(err, ErrNoInputExpected) { @@ -49,14 +49,14 @@ func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) { } func TestInputLoaderInputNoneWithBothInputs(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, InputPolicy: InputNone, - }) + } ctx := context.Background() out, err := il.Load(ctx) if !errors.Is(err, ErrNoInputExpected) { @@ -68,9 +68,9 @@ func TestInputLoaderInputNoneWithBothInputs(t *testing.T) { } func TestInputLoaderInputNoneWithNoInput(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ InputPolicy: InputNone, - }) + } ctx := context.Background() out, err := il.Load(ctx) if err != nil { @@ -82,9 +82,9 @@ func TestInputLoaderInputNoneWithNoInput(t *testing.T) { } func TestInputLoaderInputOptionalWithNoInput(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ InputPolicy: InputOptional, - }) + } ctx := context.Background() out, err := il.Load(ctx) if err != nil { @@ -96,14 +96,14 @@ func TestInputLoaderInputOptionalWithNoInput(t *testing.T) { } func TestInputLoaderInputOptionalWithInput(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, InputPolicy: InputOptional, - }) + } ctx := context.Background() out, err := il.Load(ctx) if err != nil { @@ -125,7 +125,7 @@ func TestInputLoaderInputOptionalWithInput(t *testing.T) { } func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ "testdata/inputloader1.txt", @@ -133,7 +133,7 @@ func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) { "testdata/inputloader2.txt", }, InputPolicy: InputOptional, - }) + } ctx := context.Background() out, err := il.Load(ctx) if !errors.Is(err, syscall.ENOENT) { @@ -145,14 +145,14 @@ func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) { } func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, InputPolicy: InputStrictlyRequired, - }) + } ctx := context.Background() out, err := il.Load(ctx) if err != nil { @@ -174,9 +174,9 @@ func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) { } func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ InputPolicy: InputStrictlyRequired, - }) + } ctx := context.Background() out, err := il.Load(ctx) if !errors.Is(err, ErrInputRequired) { @@ -188,14 +188,14 @@ func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { } func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ InputPolicy: InputStrictlyRequired, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader3.txt", // we want it before inputloader2.txt "testdata/inputloader2.txt", }, - }) + } ctx := context.Background() out, err := il.Load(ctx) if !errors.Is(err, ErrDetectedEmptyFile) { @@ -207,14 +207,14 @@ func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { } func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader2.txt", }, InputPolicy: InputOrQueryBackend, - }) + } ctx := context.Background() out, err := il.Load(ctx) if err != nil { @@ -248,10 +248,10 @@ func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing t.Fatal(err) } defer sess.Close() - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ InputPolicy: InputOrQueryBackend, Session: sess, - }) + } ctx, cancel := context.WithCancel(context.Background()) cancel() // fail immediately out, err := il.Load(ctx) @@ -264,14 +264,14 @@ func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing } func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { - il := NewInputLoader(InputLoaderConfig{ + il := &InputLoader{ InputPolicy: InputOrQueryBackend, SourceFiles: []string{ "testdata/inputloader1.txt", "testdata/inputloader3.txt", // we want it before inputloader2.txt "testdata/inputloader2.txt", }, - }) + } ctx := context.Background() out, err := il.Load(ctx) if !errors.Is(err, ErrDetectedEmptyFile) { @@ -303,7 +303,7 @@ func (InputLoaderBrokenFile) Close() error { } func TestInputLoaderReadfileScannerFailure(t *testing.T) { - il := inputLoader{} + il := &InputLoader{} out, err := il.readfile("", InputLoaderBrokenFS{}.Open) if !errors.Is(err, syscall.EFAULT) { t.Fatal("not the error we expected") @@ -335,14 +335,12 @@ func (sess *InputLoaderMockableSession) CheckIn( } func TestInputLoaderCheckInFailure(t *testing.T) { - il := inputLoader{} - lrc := inputLoaderLoadRemoteConfig{ - ctx: context.Background(), - session: &InputLoaderMockableSession{ + il := &InputLoader{ + Session: &InputLoaderMockableSession{ Error: io.EOF, }, } - out, err := il.loadRemote(lrc) + out, err := il.loadRemote(context.Background()) if !errors.Is(err, io.EOF) { t.Fatalf("not the error we expected: %+v", err) } @@ -352,14 +350,12 @@ func TestInputLoaderCheckInFailure(t *testing.T) { } func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) { - il := inputLoader{} - lrc := inputLoaderLoadRemoteConfig{ - ctx: context.Background(), - session: &InputLoaderMockableSession{ + il := &InputLoader{ + Session: &InputLoaderMockableSession{ Output: &model.CheckInInfo{}, }, } - out, err := il.loadRemote(lrc) + out, err := il.loadRemote(context.Background()) if !errors.Is(err, ErrNoURLsReturned) { t.Fatalf("not the error we expected: %+v", err) } @@ -369,16 +365,14 @@ func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) { } func TestInputLoaderCheckInSuccessWithNoURLs(t *testing.T) { - il := inputLoader{} - lrc := inputLoaderLoadRemoteConfig{ - ctx: context.Background(), - session: &InputLoaderMockableSession{ + il := &InputLoader{ + Session: &InputLoaderMockableSession{ Output: &model.CheckInInfo{ WebConnectivity: &model.CheckInInfoWebConnectivity{}, }, }, } - out, err := il.loadRemote(lrc) + out, err := il.loadRemote(context.Background()) if !errors.Is(err, ErrNoURLsReturned) { t.Fatalf("not the error we expected: %+v", err) } @@ -397,10 +391,8 @@ func TestInputLoaderCheckInSuccessWithSomeURLs(t *testing.T) { CountryCode: "IT", URL: "https://corriere.it", }} - il := inputLoader{} - lrc := inputLoaderLoadRemoteConfig{ - ctx: context.Background(), - session: &InputLoaderMockableSession{ + il := &InputLoader{ + Session: &InputLoaderMockableSession{ Output: &model.CheckInInfo{ WebConnectivity: &model.CheckInInfoWebConnectivity{ URLs: expect, @@ -408,7 +400,7 @@ func TestInputLoaderCheckInSuccessWithSomeURLs(t *testing.T) { }, }, } - out, err := il.loadRemote(lrc) + out, err := il.loadRemote(context.Background()) if err != nil { t.Fatal(err) } From a0763756b2bf990e16f9f243fab0950e7970b9e9 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 29 Mar 2021 20:38:23 +0200 Subject: [PATCH 17/28] fix(miniooni): replace --limit with --max-runtime (#272) Part of https://github.com/ooni/probe/issues/1299 --- internal/cmd/miniooni/libminiooni.go | 27 +++++++++++---- internal/engine/inputprocessor.go | 20 ++++++++++- internal/engine/inputprocessor_test.go | 47 ++++++++++++++++++++++---- 3 files changed, 81 insertions(+), 13 deletions(-) diff --git a/internal/cmd/miniooni/libminiooni.go b/internal/cmd/miniooni/libminiooni.go index 765cc1a..a126cf4 100644 --- a/internal/cmd/miniooni/libminiooni.go +++ b/internal/cmd/miniooni/libminiooni.go @@ -1,7 +1,5 @@ package main -// TODO(bassosimone): we need to deprecate or remove --limit. - import ( "context" "errors" @@ -34,6 +32,7 @@ type Options struct { Inputs []string InputFilePaths []string Limit int64 + MaxRuntime int64 NoJSON bool NoCollector bool ProbeServicesURL string @@ -83,6 +82,10 @@ func init() { &globalOptions.Limit, "limit", 0, "Limit the number of URLs tested by Web Connectivity", "N", ) + getopt.FlagLong( + &globalOptions.MaxRuntime, "max-runtime", 0, + "Maximum runtime in seconds when looping over a list of inputs (zero means infinite)", "N", + ) getopt.FlagLong( &globalOptions.NoJSON, "no-json", 'N', "Disable writing to disk", ) @@ -260,12 +263,23 @@ func maybeWriteConsentFile(yes bool, filepath string) (err error) { return } +// limitRemoved is the text printed when the user uses --limit +const limitRemoved = `USAGE CHANGE: The --limit option has been removed in favor of +the --max-runtime option. Please, update your script to use --max-runtime +instead of --limit. The argument to --max-runtime is the maximum number +of seconds after which to stop running Web Connectivity. + +This error message will be removed after 2021-11-01. +` + // MainWithConfiguration is the miniooni main with a specific configuration // represented by the experiment name and the current options. // // This function will panic in case of a fatal error. It is up to you that // integrate this function to either handle the panic of ignore it. func MainWithConfiguration(experimentName string, currentOptions Options) { + fatalIfFalse(currentOptions.Limit == 0, limitRemoved) + ctx := context.Background() extraOptions := mustMakeMap(currentOptions.ExtraOptions) @@ -403,15 +417,16 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { }) fatalOnError(err, "cannot create saver") - inputProcessor := engine.InputProcessor{ + inputProcessor := &engine.InputProcessor{ Annotations: annotations, Experiment: &experimentWrapper{ child: engine.NewInputProcessorExperimentWrapper(experiment), total: len(inputs), }, - Inputs: inputs, - Options: currentOptions.ExtraOptions, - Saver: engine.NewInputProcessorSaverWrapper(saver), + Inputs: inputs, + MaxRuntime: time.Duration(currentOptions.MaxRuntime) * time.Second, + Options: currentOptions.ExtraOptions, + Saver: engine.NewInputProcessorSaverWrapper(saver), Submitter: submitterWrapper{ child: engine.NewInputProcessorSubmitterWrapper(submitter), }, diff --git a/internal/engine/inputprocessor.go b/internal/engine/inputprocessor.go index 1bf3d3a..2e890c1 100644 --- a/internal/engine/inputprocessor.go +++ b/internal/engine/inputprocessor.go @@ -2,6 +2,8 @@ package engine import ( "context" + "sync/atomic" + "time" "github.com/ooni/probe-cli/v3/internal/engine/model" ) @@ -50,6 +52,12 @@ type InputProcessor struct { // Inputs is the list of inputs to measure. Inputs []model.URLInfo + // MaxRuntime is the optional maximum runtime + // when looping over a list of inputs (e.g. when + // running Web Connectivity). Zero means that + // there will be no MaxRuntime limit. + MaxRuntime time.Duration + // Options contains command line options for this experiment. Options []string @@ -60,6 +68,11 @@ type InputProcessor struct { // Submitter is the code that will submit measurements // to the OONI collector. Submitter InputProcessorSubmitterWrapper + + // terminatedByMaxRuntime is an internal atomic variabile + // incremented when we're terminated by MaxRuntime. We + // only use this variable when testing. + terminatedByMaxRuntime int32 } // InputProcessorSaverWrapper is InputProcessor's @@ -115,8 +128,13 @@ func (ipsw inputProcessorSubmitterWrapper) Submit( // is always causing us to break out of the loop. The user // though is free to choose different policies by configuring // the Experiment, Submitter, and Saver fields properly. -func (ip InputProcessor) Run(ctx context.Context) error { +func (ip *InputProcessor) Run(ctx context.Context) error { + start := time.Now() for idx, url := range ip.Inputs { + if ip.MaxRuntime > 0 && time.Since(start) > ip.MaxRuntime { + atomic.AddInt32(&ip.terminatedByMaxRuntime, 1) + return nil + } input := url.URL meas, err := ip.Experiment.MeasureWithContext(ctx, idx, input) if err != nil { diff --git a/internal/engine/inputprocessor_test.go b/internal/engine/inputprocessor_test.go index 4d604b1..461c3bb 100644 --- a/internal/engine/inputprocessor_test.go +++ b/internal/engine/inputprocessor_test.go @@ -4,13 +4,15 @@ import ( "context" "errors" "testing" + "time" "github.com/ooni/probe-cli/v3/internal/engine/model" ) type FakeInputProcessorExperiment struct { - Err error - M []*model.Measurement + SleepTime time.Duration + Err error + M []*model.Measurement } func (fipe *FakeInputProcessorExperiment) MeasureWithContext( @@ -18,6 +20,9 @@ func (fipe *FakeInputProcessorExperiment) MeasureWithContext( if fipe.Err != nil { return nil, fipe.Err } + if fipe.SleepTime > 0 { + time.Sleep(fipe.SleepTime) + } m := new(model.Measurement) // Here we add annotations to ensure that the input processor // is MERGING annotations as opposed to overwriting them. @@ -30,7 +35,7 @@ func (fipe *FakeInputProcessorExperiment) MeasureWithContext( func TestInputProcessorMeasurementFailed(t *testing.T) { expected := errors.New("mocked error") - ip := InputProcessor{ + ip := &InputProcessor{ Experiment: NewInputProcessorExperimentWrapper( &FakeInputProcessorExperiment{Err: expected}, ), @@ -58,7 +63,7 @@ func (fips *FakeInputProcessorSubmitter) Submit( func TestInputProcessorSubmissionFailed(t *testing.T) { fipe := &FakeInputProcessorExperiment{} expected := errors.New("mocked error") - ip := InputProcessor{ + ip := &InputProcessor{ Annotations: map[string]string{ "foo": "bar", }, @@ -108,7 +113,7 @@ func (fips *FakeInputProcessorSaver) SaveMeasurement(m *model.Measurement) error func TestInputProcessorSaveOnDiskFailed(t *testing.T) { expected := errors.New("mocked error") - ip := InputProcessor{ + ip := &InputProcessor{ Experiment: NewInputProcessorExperimentWrapper( &FakeInputProcessorExperiment{}, ), @@ -133,7 +138,7 @@ func TestInputProcessorGood(t *testing.T) { fipe := &FakeInputProcessorExperiment{} saver := &FakeInputProcessorSaver{Err: nil} submitter := &FakeInputProcessorSubmitter{Err: nil} - ip := InputProcessor{ + ip := &InputProcessor{ Experiment: NewInputProcessorExperimentWrapper(fipe), Inputs: []model.URLInfo{{ URL: "https://www.kernel.org/", @@ -148,6 +153,9 @@ func TestInputProcessorGood(t *testing.T) { if err := ip.Run(ctx); err != nil { t.Fatal(err) } + if ip.terminatedByMaxRuntime > 0 { + t.Fatal("terminated by max runtime!?") + } if len(fipe.M) != 2 || len(saver.M) != 2 || len(submitter.M) != 2 { t.Fatal("not all measurements saved") } @@ -164,3 +172,30 @@ func TestInputProcessorGood(t *testing.T) { t.Fatal("invalid saver.M[1].Input") } } + +func TestInputProcessorMaxRuntime(t *testing.T) { + fipe := &FakeInputProcessorExperiment{ + SleepTime: 50 * time.Millisecond, + } + saver := &FakeInputProcessorSaver{Err: nil} + submitter := &FakeInputProcessorSubmitter{Err: nil} + ip := &InputProcessor{ + Experiment: NewInputProcessorExperimentWrapper(fipe), + Inputs: []model.URLInfo{{ + URL: "https://www.kernel.org/", + }, { + URL: "https://www.slashdot.org/", + }}, + MaxRuntime: 1 * time.Nanosecond, + Options: []string{"fake=true"}, + Saver: NewInputProcessorSaverWrapper(saver), + Submitter: NewInputProcessorSubmitterWrapper(submitter), + } + ctx := context.Background() + if err := ip.Run(ctx); err != nil { + t.Fatal(err) + } + if ip.terminatedByMaxRuntime <= 0 { + t.Fatal("not terminated by max runtime") + } +} From c264f615362613b322b6f1c013e69bc81a1a188a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Mar 2021 11:16:12 +0200 Subject: [PATCH 18/28] feat(ooniprobe): introduce websites_max_runtime (#273) We cannot control anymore the maximum number of URLs using the API because now we are using check-in, that has no such limit. We could theoretically clamp the number of URLs to measure after the call to check-in, and still honour the setting. Yet, the right thing to do seems to introduce a max runtime variable because that is what desktop and mobile do. Thus, introduce code that warns the user about the change in the settings, should they have set the URL limit to nonzero. We are going to do a best effort conversion from the URL limit to the maximum runtime for the rest of 2021. Since then, we will silently ignore the URL limit. This work is part of https://github.com/ooni/probe/issues/1299. --- cmd/ooniprobe/internal/config/settings.go | 34 +------------------ .../config/testdata/valid-config.json | 2 +- cmd/ooniprobe/internal/nettests/nettests.go | 12 +++++-- cmd/ooniprobe/internal/nettests/run.go | 31 ++++++++++++++++- .../internal/nettests/web_connectivity.go | 13 ++----- .../internal/ooni/default-config.json | 2 +- cmd/ooniprobe/testdata/testing-config.json | 2 +- debian/ooniprobe.conf.disabled | 2 +- 8 files changed, 47 insertions(+), 51 deletions(-) diff --git a/cmd/ooniprobe/internal/config/settings.go b/cmd/ooniprobe/internal/config/settings.go index 4715eef..e0e9a8a 100644 --- a/cmd/ooniprobe/internal/config/settings.go +++ b/cmd/ooniprobe/internal/config/settings.go @@ -1,38 +1,5 @@ package config -var websiteCategories = []string{ - "ALDR", - "ANON", - "COMM", - "COMT", - "CTRL", - "CULTR", - "DATE", - "ECON", - "ENV", - "FILE", - "GAME", - "GMB", - "GOVT", - "GRP", - "HACK", - "HATE", - "HOST", - "HUMR", - "IGO", - "LGBT", - "MILX", - "MMED", - "NEWS", - "POLR", - "PORN", - "PROV", - "PUBH", - "REL", - "SRCH", - "XED", -} - // Sharing settings type Sharing struct { UploadResults bool `json:"upload_results"` @@ -45,6 +12,7 @@ type Advanced struct { // Nettests related settings type Nettests struct { + WebsitesMaxRuntime int64 `json:"websites_max_runtime"` WebsitesURLLimit int64 `json:"websites_url_limit"` WebsitesEnabledCategoryCodes []string `json:"websites_enabled_category_codes"` } diff --git a/cmd/ooniprobe/internal/config/testdata/valid-config.json b/cmd/ooniprobe/internal/config/testdata/valid-config.json index f609bdf..a331c8b 100644 --- a/cmd/ooniprobe/internal/config/testdata/valid-config.json +++ b/cmd/ooniprobe/internal/config/testdata/valid-config.json @@ -5,7 +5,7 @@ "upload_results": true }, "nettests": { - "websites_url_limit": 0 + "websites_max_runtime": 0 }, "advanced": { "send_crash_reports": true diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index 71ba5b0..14e788a 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -111,10 +111,16 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err } } - c.ntStartTime = time.Now() + maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second + start := time.Now() + c.ntStartTime = start for idx, input := range inputs { - if c.Probe.IsTerminated() == true { - log.Debug("isTerminated == true, breaking the input loop") + if c.Probe.IsTerminated() { + log.Info("user requested us to terminate using Ctrl-C") + break + } + if maxRuntime > 0 && time.Since(start) > maxRuntime { + log.Info("exceeded maximum runtime") break } c.curInputIdx = idx // allow for precise progress diff --git a/cmd/ooniprobe/internal/nettests/run.go b/cmd/ooniprobe/internal/nettests/run.go index ca92692..216fd41 100644 --- a/cmd/ooniprobe/internal/nettests/run.go +++ b/cmd/ooniprobe/internal/nettests/run.go @@ -1,6 +1,8 @@ package nettests import ( + "time" + "github.com/apex/log" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni" @@ -15,9 +17,36 @@ type RunGroupConfig struct { Inputs []string } +const websitesURLLimitRemoved = `WARNING: CONFIGURATION CHANGE REQUIRED: + +* Since ooniprobe 3.9.0, websites_url_limit has been replaced + by websites_max_runtime in the configuration + +* To silence this warning either set websites_url_limit to zero or + replace it with websites_max_runtime + +* For the rest of 2021, we will automatically convert websites_url_limit + to websites_max_runtime (if the latter is not already set) + +* We will consider that each URL in websites_url_limit takes five + seconds to run and thus calculate websites_max_runtime + +* Since 2022, we will start silently ignoring websites_url_limit +` + // RunGroup runs a group of nettests according to the specified config. func RunGroup(config RunGroupConfig) error { - if config.Probe.IsTerminated() == true { + if config.Probe.Config().Nettests.WebsitesURLLimit > 0 { + log.Warn(websitesURLLimitRemoved) + if config.Probe.Config().Nettests.WebsitesMaxRuntime <= 0 { + limit := config.Probe.Config().Nettests.WebsitesURLLimit + maxRuntime := 5 * limit + config.Probe.Config().Nettests.WebsitesMaxRuntime = maxRuntime + } + time.Sleep(30 * time.Second) + } + + if config.Probe.IsTerminated() { log.Debugf("context is terminated, stopping runNettestGroup early") return nil } diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index 3e15472..741572c 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -9,16 +9,10 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/model" ) -// TODO(bassosimone): we should remove the limit argument and -// we should also remove it from the config. - // TODO(bassosimone): we should propagate the kind of run // to here such that we get the right runType. -// TODO(bassosimone): we are breaking the use case in which -// someone choose the number of URLs explicitly via the config. - -func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) { +func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64, error) { inputloader := &engine.InputLoader{ CheckInConfig: &model.CheckInConfig{ WebConnectivity: model.CheckInConfigWebConnectivity{ @@ -53,13 +47,12 @@ func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, ma } // WebConnectivity test implementation -type WebConnectivity struct { -} +type WebConnectivity struct{} // Run starts the test func (n WebConnectivity) Run(ctl *Controller) error { log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) - urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesURLLimit, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) + urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) if err != nil { return err } diff --git a/cmd/ooniprobe/internal/ooni/default-config.json b/cmd/ooniprobe/internal/ooni/default-config.json index f609bdf..a331c8b 100644 --- a/cmd/ooniprobe/internal/ooni/default-config.json +++ b/cmd/ooniprobe/internal/ooni/default-config.json @@ -5,7 +5,7 @@ "upload_results": true }, "nettests": { - "websites_url_limit": 0 + "websites_max_runtime": 0 }, "advanced": { "send_crash_reports": true diff --git a/cmd/ooniprobe/testdata/testing-config.json b/cmd/ooniprobe/testdata/testing-config.json index 993f3b4..8de393b 100644 --- a/cmd/ooniprobe/testdata/testing-config.json +++ b/cmd/ooniprobe/testdata/testing-config.json @@ -5,7 +5,7 @@ "upload_results": true }, "nettests": { - "websites_url_limit": 10 + "websites_max_runtime": 15 }, "advanced": { "send_crash_reports": true diff --git a/debian/ooniprobe.conf.disabled b/debian/ooniprobe.conf.disabled index 7124af6..528b14f 100644 --- a/debian/ooniprobe.conf.disabled +++ b/debian/ooniprobe.conf.disabled @@ -6,7 +6,7 @@ "upload_results": true }, "nettests": { - "websites_url_limit": 0, + "websites_max_runtime": 0, "websites_enabled_category_codes": null }, "advanced": { From dae02ce5b647662638d2c846b8c501d19e70a7d3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Mar 2021 11:59:29 +0200 Subject: [PATCH 19/28] feat(ooniprobe): propagate the RunType CheckIn hint (#274) This diff propagates the RunType CheckIn hint such that we run using less URLs when running in the background. Part of https://github.com/ooni/probe/issues/1299. --- cmd/ooniprobe/internal/cli/run/run.go | 20 +++++++++++-------- cmd/ooniprobe/internal/nettests/nettests.go | 12 +++++++++-- cmd/ooniprobe/internal/nettests/run.go | 15 ++++++++++---- .../internal/nettests/web_connectivity.go | 10 +++++++--- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/cmd/ooniprobe/internal/cli/run/run.go b/cmd/ooniprobe/internal/cli/run/run.go index be23822..446e43e 100644 --- a/cmd/ooniprobe/internal/cli/run/run.go +++ b/cmd/ooniprobe/internal/cli/run/run.go @@ -26,19 +26,23 @@ func init() { log.WithError(err).Error("failed to perform onboarding") return err } - if *noCollector == true { + if *noCollector { probe.Config().Sharing.UploadResults = false } return nil }) - functionalRun := func(pred func(name string, gr nettests.Group) bool) error { + functionalRun := func(runType string, pred func(name string, gr nettests.Group) bool) error { for name, group := range nettests.All { - if pred(name, group) != true { + if !pred(name, group) { continue } log.Infof("Running %s tests", color.BlueString(name)) - conf := nettests.RunGroupConfig{GroupName: name, Probe: probe} + conf := nettests.RunGroupConfig{ + GroupName: name, + Probe: probe, + RunType: runType, + } if err := nettests.RunGroup(conf); err != nil { log.WithError(err).Errorf("failed to run %s", name) } @@ -48,7 +52,7 @@ func init() { genRunWithGroupName := func(targetName string) func(*kingpin.ParseContext) error { return func(*kingpin.ParseContext) error { - return functionalRun(func(groupName string, gr nettests.Group) bool { + return functionalRun("manual", func(groupName string, gr nettests.Group) bool { return groupName == targetName }) } @@ -75,14 +79,14 @@ func init() { unattendedCmd := cmd.Command("unattended", "") unattendedCmd.Action(func(_ *kingpin.ParseContext) error { - return functionalRun(func(name string, gr nettests.Group) bool { - return gr.UnattendedOK == true + return functionalRun("timed", func(name string, gr nettests.Group) bool { + return gr.UnattendedOK }) }) allCmd := cmd.Command("all", "").Default() allCmd.Action(func(_ *kingpin.ParseContext) error { - return functionalRun(func(name string, gr nettests.Group) bool { + return functionalRun("manual", func(name string, gr nettests.Group) bool { return true }) }) diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index 14e788a..9907194 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -53,6 +53,10 @@ type Controller struct { // using the command line using the --input flag. Inputs []string + // RunType contains the run_type hint for the CheckIn API. If + // not set, the underlying code defaults to "timed". + RunType string + // numInputs is the total number of inputs numInputs int @@ -115,6 +119,8 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err start := time.Now() c.ntStartTime = start for idx, input := range inputs { + // TODO(bassosimone): should we allow for interruption when running + // in unattended mode? Likewise, should we honor MaxRuntime? if c.Probe.IsTerminated() { log.Info("user requested us to terminate using Ctrl-C") break @@ -172,7 +178,7 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err } } // We only save the measurement to disk if we failed to upload the measurement - if saveToDisk == true { + if saveToDisk { if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil { return errors.Wrap(err, "failed to save measurement on disk") } @@ -204,6 +210,8 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err // OnProgress should be called when a new progress event is available. func (c *Controller) OnProgress(perc float64, msg string) { + // TODO(bassosimone): should we adjust this algorithm when we have a + // maximum runtime that we would like to honor? log.Debugf("OnProgress: %f - %s", perc, msg) var eta float64 eta = -1.0 @@ -213,7 +221,7 @@ func (c *Controller) OnProgress(perc float64, msg string) { step := 1.0 / float64(c.numInputs) perc = floor + perc*step if c.curInputIdx > 0 { - eta = (time.Now().Sub(c.ntStartTime).Seconds() / float64(c.curInputIdx)) * float64(c.numInputs-c.curInputIdx) + eta = (time.Since(c.ntStartTime).Seconds() / float64(c.curInputIdx)) * float64(c.numInputs-c.curInputIdx) } } if c.ntCount > 0 { diff --git a/cmd/ooniprobe/internal/nettests/run.go b/cmd/ooniprobe/internal/nettests/run.go index 216fd41..6450cdf 100644 --- a/cmd/ooniprobe/internal/nettests/run.go +++ b/cmd/ooniprobe/internal/nettests/run.go @@ -1,6 +1,7 @@ package nettests import ( + "sync" "time" "github.com/apex/log" @@ -12,9 +13,10 @@ import ( // RunGroupConfig contains the settings for running a nettest group. type RunGroupConfig struct { GroupName string - Probe *ooni.Probe InputFiles []string Inputs []string + Probe *ooni.Probe + RunType string // hint for check-in API } const websitesURLLimitRemoved = `WARNING: CONFIGURATION CHANGE REQUIRED: @@ -34,16 +36,20 @@ const websitesURLLimitRemoved = `WARNING: CONFIGURATION CHANGE REQUIRED: * Since 2022, we will start silently ignoring websites_url_limit ` +var deprecationWarningOnce sync.Once + // RunGroup runs a group of nettests according to the specified config. func RunGroup(config RunGroupConfig) error { if config.Probe.Config().Nettests.WebsitesURLLimit > 0 { - log.Warn(websitesURLLimitRemoved) if config.Probe.Config().Nettests.WebsitesMaxRuntime <= 0 { limit := config.Probe.Config().Nettests.WebsitesURLLimit maxRuntime := 5 * limit config.Probe.Config().Nettests.WebsitesMaxRuntime = maxRuntime } - time.Sleep(30 * time.Second) + deprecationWarningOnce.Do(func() { + log.Warn(websitesURLLimitRemoved) + time.Sleep(30 * time.Second) + }) } if config.Probe.IsTerminated() { @@ -90,7 +96,7 @@ func RunGroup(config RunGroupConfig) error { config.Probe.ListenForSignals() config.Probe.MaybeListenForStdinClosed() for i, nt := range group.Nettests { - if config.Probe.IsTerminated() == true { + if config.Probe.IsTerminated() { log.Debugf("context is terminated, stopping group.Nettests early") break } @@ -98,6 +104,7 @@ func RunGroup(config RunGroupConfig) error { ctl := NewController(nt, config.Probe, result, sess) ctl.InputFiles = config.InputFiles ctl.Inputs = config.Inputs + ctl.RunType = config.RunType ctl.SetNettestIndex(i, len(group.Nettests)) if err = nt.Run(ctl); err != nil { log.WithError(err).Errorf("Failed to run %s", group.Label) diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index 741572c..fc752b2 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -9,12 +9,15 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/model" ) -// TODO(bassosimone): we should propagate the kind of run -// to here such that we get the right runType. - func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64, error) { inputloader := &engine.InputLoader{ CheckInConfig: &model.CheckInConfig{ + // Setting Charging and OnWiFi to true causes the CheckIn + // API to return to us as much URL as possible with the + // given RunType hint. + Charging: true, + OnWiFi: true, + RunType: ctl.RunType, WebConnectivity: model.CheckInConfigWebConnectivity{ CategoryCodes: categories, }, @@ -24,6 +27,7 @@ func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64 SourceFiles: ctl.InputFiles, StaticInputs: ctl.Inputs, } + log.Infof("Calling CheckIn API with %s runType", ctl.RunType) testlist, err := inputloader.Load(context.Background()) var urls []string urlIDMap := make(map[int64]int64) From 991b0a6120f55fdb438ac895db7a9b32fb771bb4 Mon Sep 17 00:00:00 2001 From: cyBerta Date: Tue, 30 Mar 2021 12:02:51 +0200 Subject: [PATCH 20/28] riseupvpn: reduce false positives (#233) * fetch RiseupVPN CA cert with MultiGetter. It allows us to write better tests and ensures this test step is added in the logs * Implement TransportStatus for RiseupVPN tests. It indicates if a whole transport is blocked, which is considered as a test anomaly * Redesign unit tests for RiseupVPN. Instead of a real backend, mocked server responses are used. Tests for invalid CA certs and for TransportStatus are added. * Update internal/engine/experiment/riseupvpn/riseupvpn.go Co-authored-by: Simone Basso --- .../engine/experiment/riseupvpn/riseupvpn.go | 99 ++- .../experiment/riseupvpn/riseupvpn_test.go | 754 ++++++++++++------ 2 files changed, 589 insertions(+), 264 deletions(-) diff --git a/internal/engine/experiment/riseupvpn/riseupvpn.go b/internal/engine/experiment/riseupvpn/riseupvpn.go index 8295dbb..91f8b20 100644 --- a/internal/engine/experiment/riseupvpn/riseupvpn.go +++ b/internal/engine/experiment/riseupvpn/riseupvpn.go @@ -9,7 +9,6 @@ import ( "errors" "time" - "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/netx" @@ -18,7 +17,7 @@ import ( const ( testName = "riseupvpn" - testVersion = "0.1.0" + testVersion = "0.2.0" eipServiceURL = "https://api.black.riseup.net:443/3/config/eip-service.json" providerURL = "https://riseup.net/provider.json" geoServiceURL = "https://api.black.riseup.net:9001/json" @@ -66,6 +65,7 @@ type TestKeys struct { APIStatus string `json:"api_status"` CACertStatus bool `json:"ca_cert_status"` FailingGateways []GatewayConnection `json:"failing_gateways"` + TransportStatus map[string]string `json:"transport_status"` } // NewTestKeys creates new riseupvpn TestKeys. @@ -75,6 +75,7 @@ func NewTestKeys() *TestKeys { APIStatus: "ok", CACertStatus: true, FailingGateways: nil, + TransportStatus: nil, } } @@ -96,6 +97,7 @@ func (tk *TestKeys) UpdateProviderAPITestKeys(v urlgetter.MultiOutput) { } // AddGatewayConnectTestKeys updates the TestKeys using the given MultiOutput result of gateway connectivity testing. +// Sets TransportStatus to "ok" if any successful TCP connection could be made func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transportType string) { tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...) tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...) @@ -108,6 +110,29 @@ func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transport return } +func (tk *TestKeys) updateTransportStatus(openvpnGatewayCount int, obfs4GatewayCount int) { + failingOpenvpnGateways, failingObfs4Gateways := 0, 0 + for _, gw := range tk.FailingGateways { + if gw.TransportType == "openvpn" { + failingOpenvpnGateways++ + } else if gw.TransportType == "obfs4" { + failingObfs4Gateways++ + } + } + + if failingOpenvpnGateways < openvpnGatewayCount { + tk.TransportStatus["openvpn"] = "ok" + } else { + tk.TransportStatus["openvpn"] = "blocked" + } + + if failingObfs4Gateways < obfs4GatewayCount { + tk.TransportStatus["obfs4"] = "ok" + } else { + tk.TransportStatus["obfs4"] = "blocked" + } +} + func newGatewayConnection(tcpConnect archival.TCPConnectEntry, transportType string) *GatewayConnection { return &GatewayConnection{ IP: tcpConnect.IP, @@ -160,30 +185,32 @@ func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession, urlgetter.RegisterExtensions(measurement) caTarget := "https://black.riseup.net/ca.crt" - caGetter := urlgetter.Getter{ - Config: m.Config.Config, - Session: sess, - Target: caTarget, - } - log.Info("Getting CA certificate; please be patient...") - tk, err := caGetter.Get(ctx) - testkeys.AddCACertFetchTestKeys(tk) - - if err != nil { - log.Error("Getting CA certificate failed. Aborting test.") - return nil - } - certPool := netx.NewDefaultCertPool() - if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok { - testkeys.CACertStatus = false - testkeys.APIStatus = "blocked" - errorValue := "invalid_ca" - testkeys.APIFailure = &errorValue - return nil + + multi := urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess} + inputs := []urlgetter.MultiInput{ + {Target: caTarget, Config: urlgetter.Config{ + Method: "GET", + FailOnHTTPError: true, + }}, + } + for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) { + tk := entry.TestKeys + testkeys.AddCACertFetchTestKeys(tk) + if tk.Failure != nil { + return nil + } + + if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok { + testkeys.CACertStatus = false + testkeys.APIStatus = "blocked" + errorValue := "invalid_ca" + testkeys.APIFailure = &errorValue + return nil + } } - inputs := []urlgetter.MultiInput{ + inputs = []urlgetter.MultiInput{ // Here we need to provide the method explicitly. See // https://github.com/ooni/probe-engine/issues/827. @@ -203,30 +230,34 @@ func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession, FailOnHTTPError: true, }}, } - multi := urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess} + multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess} - for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) { + for entry := range multi.CollectOverall(ctx, inputs, 1, 50, "riseupvpn", callbacks) { testkeys.UpdateProviderAPITestKeys(entry) } // test gateways now + testkeys.TransportStatus = map[string]string{} gateways := parseGateways(testkeys) openvpnEndpoints := generateMultiInputs(gateways, "openvpn") obfs4Endpoints := generateMultiInputs(gateways, "obfs4") - overallCount := len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints) + overallCount := 1 + len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints) // measure openvpn in parallel multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess} - for entry := range multi.CollectOverall(ctx, openvpnEndpoints, len(inputs), overallCount, "riseupvpn", callbacks) { + for entry := range multi.CollectOverall(ctx, openvpnEndpoints, 1+len(inputs), overallCount, "riseupvpn", callbacks) { testkeys.AddGatewayConnectTestKeys(entry, "openvpn") } // measure obfs4 in parallel multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess} - for entry := range multi.CollectOverall(ctx, obfs4Endpoints, len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) { + for entry := range multi.CollectOverall(ctx, obfs4Endpoints, 1+len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) { testkeys.AddGatewayConnectTestKeys(entry, "obfs4") } + // set transport status based on gateway test results + testkeys.updateTransportStatus(len(openvpnEndpoints), len(obfs4Endpoints)) + return nil } @@ -287,10 +318,11 @@ func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { // Note that this structure is part of the ABI contract with probe-cli // therefore we should be careful when changing it. type SummaryKeys struct { - APIBlocked bool `json:"api_blocked"` - ValidCACert bool `json:"valid_ca_cert"` - FailingGateways int `json:"failing_gateways"` - IsAnomaly bool `json:"-"` + APIBlocked bool `json:"api_blocked"` + ValidCACert bool `json:"valid_ca_cert"` + FailingGateways int `json:"failing_gateways"` + TransportStatus map[string]string `json:"transport_status"` + IsAnomaly bool `json:"-"` } // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. @@ -303,7 +335,8 @@ func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, e sk.APIBlocked = tk.APIStatus != "ok" sk.ValidCACert = tk.CACertStatus sk.FailingGateways = len(tk.FailingGateways) + sk.TransportStatus = tk.TransportStatus sk.IsAnomaly = (sk.APIBlocked == true || tk.CACertStatus == false || - sk.FailingGateways != 0) + tk.TransportStatus["openvpn"] == "blocked" || tk.TransportStatus["obfs4"] == "blocked") return sk, nil } diff --git a/internal/engine/experiment/riseupvpn/riseupvpn_test.go b/internal/engine/experiment/riseupvpn/riseupvpn_test.go index 3798d5d..1be8aa4 100644 --- a/internal/engine/experiment/riseupvpn/riseupvpn_test.go +++ b/internal/engine/experiment/riseupvpn/riseupvpn_test.go @@ -2,15 +2,14 @@ package riseupvpn_test import ( "context" - "crypto/tls" - "crypto/x509" - "errors" + "encoding/json" "fmt" - "io/ioutil" - "math/rand" - "net/http" + "io" + "strconv" + "strings" "testing" - "time" + + "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" "github.com/apex/log" "github.com/google/go-cmp/cmp" @@ -19,36 +18,204 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/internal/mockable" "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor" ) +const ( + provider = `{ + "api_uri": "https://api.black.riseup.net:443", + "api_version": "3", + "ca_cert_fingerprint": "SHA256: a5244308a1374709a9afce95e3ae47c1b44bc2398c0a70ccbf8b3a8a97f29494", + "ca_cert_uri": "https://black.riseup.net/ca.crt", + "default_language": "en", + "description": { + "en": "Riseup is a non-profit collective in Seattle that provides online communication tools for people and groups working toward liberatory social change." + }, + "domain": "riseup.net", + "enrollment_policy": "closed", + "languages": [ + "en" + ], + "name": { + "en": "Riseup Networks" + }, + "service": { + "allow_anonymous": true, + "allow_free": true, + "allow_limited_bandwidth": false, + "allow_paid": false, + "allow_registration": false, + "allow_unlimited_bandwidth": true, + "bandwidth_limit": 102400, + "default_service_level": 1, + "levels": { + "1": { + "description": "Please donate.", + "name": "free" + } + } + }, + "services": [ + "openvpn" + ] + }` + eipservice = `{ + "gateways": [ + { + "capabilities": { + "adblock": false, + "filter_dns": false, + "limited": false, + "transport":[ + { + "type":"openvpn", + "protocols":[ + "tcp" + ], + "ports":[ + "443" + ] + } + ], + "user_ips": false + }, + "host": "test1.riseup.net", + "ip_address": "123.456.123.456", + "location": "paris" + }, + { + "capabilities": { + "adblock": false, + "filter_dns": false, + "limited": false, + "transport":[ + { + "type":"obfs4", + "protocols":[ + "tcp" + ], + "ports":[ + "23042" + ], + "options": { + "cert": "XXXXXXXXXXXXXXXXXXXXXXXXX", + "iatMode": "0" + } + }, + { + "type":"openvpn", + "protocols":[ + "tcp" + ], + "ports":[ + "443" + ] + } + ], + "user_ips": false + }, + "host": "test2.riseup.net", + "ip_address": "234.345.234.345", + "location": "seattle" + } + ], + "locations": { + "paris": { + "country_code": "FR", + "hemisphere": "N", + "name": "Paris", + "timezone": "+2" + }, + "seattle": { + "country_code": "US", + "hemisphere": "N", + "name": "Seattle", + "timezone": "-7" + } + }, + "openvpn_configuration": { + "auth": "SHA1", + "cipher": "AES-128-CBC", + "keepalive": "10 30", + "tls-cipher": "DHE-RSA-AES128-SHA", + "tun-ipv6": true + }, + "serial": 3, + "version": 3 + }` + geoservice = `{"ip":"51.15.0.88","cc":"NL","city":"Haarlem","lat":52.381,"lon":4.6275,"gateways":["test1.riseup.net","test2.riseup.net"]}` + cacert = `-----BEGIN CERTIFICATE----- +MIIFjTCCA3WgAwIBAgIBATANBgkqhkiG9w0BAQ0FADBZMRgwFgYDVQQKDA9SaXNl +dXAgTmV0d29ya3MxGzAZBgNVBAsMEmh0dHBzOi8vcmlzZXVwLm5ldDEgMB4GA1UE +AwwXUmlzZXVwIE5ldHdvcmtzIFJvb3QgQ0EwHhcNMTQwNDI4MDAwMDAwWhcNMjQw +NDI4MDAwMDAwWjBZMRgwFgYDVQQKDA9SaXNldXAgTmV0d29ya3MxGzAZBgNVBAsM +Emh0dHBzOi8vcmlzZXVwLm5ldDEgMB4GA1UEAwwXUmlzZXVwIE5ldHdvcmtzIFJv +b3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC76J4ciMJ8Sg0m +TP7DF2DT9zNe0Csk4myoMFC57rfJeqsAlJCv1XMzBmXrw8wq/9z7XHv6n/0sWU7a +7cF2hLR33ktjwODlx7vorU39/lXLndo492ZBhXQtG1INMShyv+nlmzO6GT7ESfNE +LliFitEzwIegpMqxCIHXFuobGSCWF4N0qLHkq/SYUMoOJ96O3hmPSl1kFDRMtWXY +iw1SEKjUvpyDJpVs3NGxeLCaA7bAWhDY5s5Yb2fA1o8ICAqhowurowJpW7n5ZuLK +5VNTlNy6nZpkjt1QycYvNycffyPOFm/Q/RKDlvnorJIrihPkyniV3YY5cGgP+Qkx +HUOT0uLA6LHtzfiyaOqkXwc4b0ZcQD5Vbf6Prd20Ppt6ei0zazkUPwxld3hgyw58 +m/4UIjG3PInWTNf293GngK2Bnz8Qx9e/6TueMSAn/3JBLem56E0WtmbLVjvko+LF +PM5xA+m0BmuSJtrD1MUCXMhqYTtiOvgLBlUm5zkNxALzG+cXB28k6XikXt6MRG7q +hzIPG38zwkooM55yy5i1YfcIi5NjMH6A+t4IJxxwb67MSb6UFOwg5kFokdONZcwj +shczHdG9gLKSBIvrKa03Nd3W2dF9hMbRu//STcQxOailDBQCnXXfAATj9pYzdY4k +ha8VCAREGAKTDAex9oXf1yRuktES4QIDAQABo2AwXjAdBgNVHQ4EFgQUC4tdmLVu +f9hwfK4AGliaet5KkcgwDgYDVR0PAQH/BAQDAgIEMAwGA1UdEwQFMAMBAf8wHwYD +VR0jBBgwFoAUC4tdmLVuf9hwfK4AGliaet5KkcgwDQYJKoZIhvcNAQENBQADggIB +AGzL+GRnYu99zFoy0bXJKOGCF5XUXP/3gIXPRDqQf5g7Cu/jYMID9dB3No4Zmf7v +qHjiSXiS8jx1j/6/Luk6PpFbT7QYm4QLs1f4BlfZOti2KE8r7KRDPIecUsUXW6P/ +3GJAVYH/+7OjA39za9AieM7+H5BELGccGrM5wfl7JeEz8in+V2ZWDzHQO4hMkiTQ +4ZckuaL201F68YpiItBNnJ9N5nHr1MRiGyApHmLXY/wvlrOpclh95qn+lG6/2jk7 +3AmihLOKYMlPwPakJg4PYczm3icFLgTpjV5sq2md9bRyAg3oPGfAuWHmKj2Ikqch +Td5CHKGxEEWbGUWEMP0s1A/JHWiCbDigc4Cfxhy56CWG4q0tYtnc2GMw8OAUO6Wf +Xu5pYKNkzKSEtT/MrNJt44tTZWbKV/Pi/N2Fx36my7TgTUj7g3xcE9eF4JV2H/sg +tsK3pwE0FEqGnT4qMFbixQmc8bGyuakr23wjMvfO7eZUxBuWYR2SkcP26sozF9PF +tGhbZHQVGZUTVPyvwahMUEhbPGVerOW0IYpxkm0x/eaWdTc4vPpf/rIlgbAjarnJ +UN9SaWRlWKSdP4haujnzCoJbM7dU9bjvlGZNyXEekgeT0W2qFeGGp+yyUWw8tNsp +0BuC1b7uW/bBn/xKm319wXVDvBgZgcktMolak39V7DVO +-----END CERTIFICATE-----` + + eipserviceurl = "https://api.black.riseup.net:443/3/config/eip-service.json" + providerurl = "https://riseup.net/provider.json" + geoserviceurl = "https://api.black.riseup.net:9001/json" + cacerturl = "https://black.riseup.net/ca.crt" + openvpnurl1 = "tcpconnect://234.345.234.345:443" + openvpnurl2 = "tcpconnect://123.456.123.456:443" + obfs4url1 = "tcpconnect://234.345.234.345:23042" +) + +var RequestResponse = map[string]string{ + eipserviceurl: eipservice, + providerurl: provider, + geoserviceurl: geoservice, + cacerturl: cacert, + openvpnurl1: "", + openvpnurl2: "", + obfs4url1: "", +} + func TestNewExperimentMeasurer(t *testing.T) { measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) if measurer.ExperimentName() != "riseupvpn" { t.Fatal("unexpected name") } - if measurer.ExperimentVersion() != "0.1.0" { + if measurer.ExperimentVersion() != "0.2.0" { t.Fatal("unexpected version") } } func TestGood(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) - measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - &mockable.Session{ - MockableLogger: log.Log, - }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) - if err != nil { - t.Fatal(err) - } + measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ + cacerturl: true, + eipserviceurl: true, + providerurl: true, + geoserviceurl: true, + openvpnurl1: true, + openvpnurl2: true, + obfs4url1: true, + })) + tk := measurement.TestKeys.(*riseupvpn.TestKeys) if tk.Agent != "" { t.Fatal("unexpected Agent: " + tk.Agent) @@ -59,21 +226,6 @@ func TestGood(t *testing.T) { if tk.Failure != nil { t.Fatal("unexpected Failure") } - if len(tk.NetworkEvents) <= 0 { - t.Fatal("no NetworkEvents?!") - } - if len(tk.Queries) <= 0 { - t.Fatal("no Queries?!") - } - if len(tk.Requests) <= 0 { - t.Fatal("no Requests?!") - } - if len(tk.TCPConnect) <= 0 { - t.Fatal("no TCPConnect?!") - } - if len(tk.TLSHandshakes) <= 0 { - t.Fatal("no TLSHandshakes?!") - } if tk.APIFailure != nil { t.Fatal("unexpected ApiFailure") } @@ -86,6 +238,12 @@ func TestGood(t *testing.T) { if tk.FailingGateways != nil { t.Fatal("unexpected FailingGateways value") } + if tk.TransportStatus == nil { + t.Fatal("unexpected nil TransportStatus struct ") + } + if tk.TransportStatus["openvpn"] != "ok" { + t.Fatal("unexpected openvpn transport status") + } } // TestUpdateWithMixedResults tests if one operation failed @@ -132,13 +290,39 @@ func TestUpdateWithMixedResults(t *testing.T) { if *tk.APIFailure != errorx.FailureEOFError { t.Fatal("invalid ApiFailure") } + if tk.FailingGateways != nil { + t.Fatal("invalid FailingGateways") + } + if tk.TransportStatus != nil { + t.Fatal("invalid TransportStatus") + } } -func TestFailureCaCertFetch(t *testing.T) { - measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) +func TestInvalidCaCert(t *testing.T) { + requestResponseMap := map[string]string{ + eipserviceurl: eipservice, + providerurl: provider, + geoserviceurl: geoservice, + cacerturl: "invalid", + openvpnurl1: "", + openvpnurl2: "", + obfs4url1: "", + } + measurer := riseupvpn.Measurer{ + Config: riseupvpn.Config{}, + Getter: generateMockGetter(requestResponseMap, map[string]bool{ + cacerturl: true, + eipserviceurl: true, + providerurl: true, + geoserviceurl: true, + openvpnurl1: false, + openvpnurl2: true, + obfs4url1: true, + }), + } + ctx, cancel := context.WithCancel(context.Background()) - // we're cancelling immediately so that the CA Cert fetch fails - cancel() + defer cancel() sess := &mockable.Session{MockableLogger: log.Log} measurement := new(model.Measurement) @@ -147,6 +331,26 @@ func TestFailureCaCertFetch(t *testing.T) { if err != nil { t.Fatal(err) } + tk := measurement.TestKeys.(*riseupvpn.TestKeys) + if tk.CACertStatus == true { + t.Fatal("unexpected CaCertStatus") + } + if tk.APIStatus != "blocked" { + t.Fatal("ApiStatus should be blocked") + } +} + +func TestFailureCaCertFetch(t *testing.T) { + measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ + cacerturl: false, + eipserviceurl: true, + providerurl: true, + geoserviceurl: true, + openvpnurl1: true, + openvpnurl2: true, + obfs4url1: true, + })) + tk := measurement.TestKeys.(*riseupvpn.TestKeys) if tk.CACertStatus != false { t.Fatal("invalid CACertStatus ") @@ -164,21 +368,15 @@ func TestFailureCaCertFetch(t *testing.T) { } func TestFailureEipServiceBlocked(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - selfcensor.Enable(`{"PoisonSystemDNS":{"api.black.riseup.net":["NXDOMAIN"]}}`) - - sess := &mockable.Session{MockableLogger: log.Log} - measurement := new(model.Measurement) - callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) - if err != nil { - t.Fatal(err) - } + measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ + cacerturl: true, + eipserviceurl: false, + providerurl: true, + geoserviceurl: true, + openvpnurl1: true, + openvpnurl2: true, + obfs4url1: true, + })) tk := measurement.TestKeys.(*riseupvpn.TestKeys) if tk.CACertStatus != true { t.Fatal("invalid CACertStatus ") @@ -202,21 +400,15 @@ func TestFailureEipServiceBlocked(t *testing.T) { } func TestFailureProviderUrlBlocked(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.70:443":"REJECT"}}`) - - sess := &mockable.Session{MockableLogger: log.Log} - measurement := new(model.Measurement) - callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) - if err != nil { - t.Fatal(err) - } + measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ + cacerturl: true, + eipserviceurl: true, + providerurl: false, + geoserviceurl: true, + openvpnurl1: true, + openvpnurl2: true, + obfs4url1: true, + })) tk := measurement.TestKeys.(*riseupvpn.TestKeys) for _, entry := range tk.Requests { @@ -240,21 +432,15 @@ func TestFailureProviderUrlBlocked(t *testing.T) { } func TestFailureGeoIpServiceBlocked(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.107:9001":"REJECT"}}`) - - sess := &mockable.Session{MockableLogger: log.Log} - measurement := new(model.Measurement) - callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) - if err != nil { - t.Fatal(err) - } + measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ + cacerturl: true, + eipserviceurl: true, + providerurl: true, + geoserviceurl: false, + openvpnurl1: true, + openvpnurl2: true, + obfs4url1: true, + })) tk := measurement.TestKeys.(*riseupvpn.TestKeys) if tk.CACertStatus != true { t.Fatal("invalid CACertStatus ") @@ -277,138 +463,16 @@ func TestFailureGeoIpServiceBlocked(t *testing.T) { } } -func TestFailureGateway(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - var testCases = [...]string{"openvpn", "obfs4"} - eipService, err := fetchEipService() - if err != nil { - t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error()) - t.SkipNow() - } - for _, tc := range testCases { - t.Run(fmt.Sprintf("testing censored transport %s", tc), func(t *testing.T) { - censoredGateway, err := selfCensorRandomGateway(eipService, tc) - if err == nil { - censorString := `{"BlockedEndpoints":{"` + censoredGateway.IP + `:` + censoredGateway.Port + `":"REJECT"}}` - selfcensor.Enable(censorString) - } else { - t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error()) - t.SkipNow() - } - - // - run measurement - runGatewayTest(t, censoredGateway) - }) - } -} - -type SelfCensoredGateway struct { - IP string - Port string -} - -func fetchEipService() (*riseupvpn.EipService, error) { - // - fetch client cert and add to certpool - caFetchClient := &http.Client{ - Timeout: time.Second * 30, - } - - caCertResponse, err := caFetchClient.Get("https://black.riseup.net/ca.crt") - if err != nil { - return nil, err - } - - var bodyString string - - if caCertResponse.StatusCode != http.StatusOK { - return nil, errors.New("unexpected HTTP response code") - } - bodyBytes, err := ioutil.ReadAll(caCertResponse.Body) - defer caCertResponse.Body.Close() - - if err != nil { - return nil, err - } - bodyString = string(bodyBytes) - - certs := x509.NewCertPool() - certs.AppendCertsFromPEM([]byte(bodyString)) - - // - fetch and parse eip-service.json - client := &http.Client{ - Timeout: time.Second * 30, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certs, - }, - }, - } - - eipResponse, err := client.Get("https://api.black.riseup.net/3/config/eip-service.json") - if err != nil { - return nil, err - } - if eipResponse.StatusCode != http.StatusOK { - return nil, errors.New("Unexpected HTTP response code") - } - - bodyBytes, err = ioutil.ReadAll(eipResponse.Body) - defer eipResponse.Body.Close() - if err != nil { - return nil, err - } - bodyString = string(bodyBytes) - - eipService, err := riseupvpn.DecodeEIP3(bodyString) - if err != nil { - return nil, err - } - return eipService, nil -} - -func selfCensorRandomGateway(eipService *riseupvpn.EipService, transportType string) (*SelfCensoredGateway, error) { - - // - self censor random gateway - gateways := eipService.Gateways - if gateways == nil || len(gateways) == 0 { - return nil, errors.New("No gateways found") - } - - var selfcensoredGateways []SelfCensoredGateway - for _, gateway := range gateways { - for _, transport := range gateway.Capabilities.Transport { - if transport.Type == transportType { - selfcensoredGateways = append(selfcensoredGateways, SelfCensoredGateway{IP: gateway.IPAddress, Port: transport.Ports[0]}) - } - } - } - - if len(selfcensoredGateways) == 0 { - return nil, errors.New("transport " + transportType + " doesn't seem to be supported.") - } - - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - min := 0 - max := len(selfcensoredGateways) - 1 - randomIndex := rnd.Intn(max-min+1) + min - return &selfcensoredGateways[randomIndex], nil - -} - -func runGatewayTest(t *testing.T, censoredGateway *SelfCensoredGateway) { - measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sess := &mockable.Session{MockableLogger: log.Log} - measurement := new(model.Measurement) - callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) - if err != nil { - t.Fatal(err) - } +func TestFailureGateway1(t *testing.T) { + measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ + cacerturl: true, + eipserviceurl: true, + providerurl: true, + geoserviceurl: true, + openvpnurl1: false, + openvpnurl2: true, + obfs4url1: true, + })) tk := measurement.TestKeys.(*riseupvpn.TestKeys) if tk.CACertStatus != true { t.Fatal("invalid CACertStatus ") @@ -418,18 +482,122 @@ func runGatewayTest(t *testing.T, censoredGateway *SelfCensoredGateway) { t.Fatal("unexpected amount of failing gateways") } - entry := tk.FailingGateways[0] - if entry.IP != censoredGateway.IP || fmt.Sprint(entry.Port) != censoredGateway.Port { - t.Fatal("unexpected failed gateway configuration") + gw := tk.FailingGateways[0] + if gw.IP != "234.345.234.345" { + t.Fatal("invalid failed gateway ip: " + fmt.Sprint(gw.IP)) + } + if gw.Port != 443 { + t.Fatal("invalid failed gateway port: " + fmt.Sprint(gw.Port)) + } + if gw.TransportType != "openvpn" { + t.Fatal("invalid failed transport type: " + fmt.Sprint(gw.TransportType)) } if tk.APIStatus == "blocked" { - t.Fatal("invalid ApiStatus", tk.APIStatus) + t.Fatal("invalid ApiStatus") } if tk.APIFailure != nil { t.Fatal("ApiFailure should be null") } + + if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] == "blocked" { + t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) + } + + if tk.TransportStatus == nil || tk.TransportStatus["obfs4"] == "blocked" { + t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) + } +} + +func TestFailureTransport(t *testing.T) { + measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ + cacerturl: true, + eipserviceurl: true, + providerurl: true, + geoserviceurl: true, + openvpnurl1: false, + openvpnurl2: false, + obfs4url1: false, + })) + tk := measurement.TestKeys.(*riseupvpn.TestKeys) + + if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] != "blocked" { + t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) + } + + if tk.TransportStatus == nil || tk.TransportStatus["obfs4"] != "blocked" { + t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) + } +} + +func TestMissingTransport(t *testing.T) { + eipService, err := riseupvpn.DecodeEIP3(eipservice) + if err != nil { + t.Fatal("Preconditions for the test are not met.") + } + + //remove obfs4 capability from 2. gateway so that our + //mock provider supports only openvpn + index := -1 + transports := eipService.Gateways[1].Capabilities.Transport + for i, transport := range transports { + if transport.Type == "obfs4" { + index = i + break + } + } + if index == -1 { + t.Fatal("Preconditions for the test are not met. Default eipservice string should contain obfs4 transport.") + } + + transports[index] = transports[len(transports)-1] + transports = transports[:len(transports)-1] + eipService.Gateways[1].Capabilities.Transport = transports + eipservicejson, err := json.Marshal(eipservice) + + requestResponseMap := map[string]string{ + eipserviceurl: string(eipservicejson), + providerurl: provider, + geoserviceurl: geoservice, + cacerturl: cacert, + openvpnurl1: "", + openvpnurl2: "", + obfs4url1: "", + } + + measurer := riseupvpn.Measurer{ + Config: riseupvpn.Config{}, + Getter: generateMockGetter(requestResponseMap, map[string]bool{ + cacerturl: true, + eipserviceurl: true, + providerurl: true, + geoserviceurl: true, + openvpnurl1: true, + openvpnurl2: true, + obfs4url1: false, + }), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sess := &mockable.Session{MockableLogger: log.Log} + measurement := new(model.Measurement) + callbacks := model.NewPrinterCallbacks(log.Log) + err = measurer.Run(ctx, sess, measurement, callbacks) + if err != nil { + t.Fatal(err) + } + tk := measurement.TestKeys.(*riseupvpn.TestKeys) + if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] != "blocked" { + t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) + } + + if _, found := tk.TransportStatus["obfs"]; found { + t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) + } + } func TestSummaryKeysInvalidType(t *testing.T) { @@ -450,21 +618,27 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) { APIStatus: "blocked", CACertStatus: true, FailingGateways: nil, + TransportStatus: nil, }, sk: riseupvpn.SummaryKeys{ - APIBlocked: true, - ValidCACert: true, - IsAnomaly: true, + APIBlocked: true, + ValidCACert: true, + IsAnomaly: true, + TransportStatus: nil, + FailingGateways: 0, }, }, { tk: riseupvpn.TestKeys{ APIStatus: "ok", CACertStatus: false, FailingGateways: nil, + TransportStatus: nil, }, sk: riseupvpn.SummaryKeys{ - ValidCACert: false, - IsAnomaly: true, + ValidCACert: false, + IsAnomaly: true, + FailingGateways: 0, + TransportStatus: nil, }, }, { tk: riseupvpn.TestKeys{ @@ -475,13 +649,39 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) { Port: 443, TransportType: "obfs4", }}, + TransportStatus: map[string]string{ + "obfs4": "blocked", + "openvpn": "ok", + }, }, sk: riseupvpn.SummaryKeys{ FailingGateways: 1, IsAnomaly: true, ValidCACert: true, + TransportStatus: map[string]string{ + "obfs4": "blocked", + "openvpn": "ok", + }, }, - }} + }, { + tk: riseupvpn.TestKeys{ + APIStatus: "ok", + CACertStatus: true, + FailingGateways: nil, + TransportStatus: map[string]string{ + "openvpn": "ok", + }, + }, + sk: riseupvpn.SummaryKeys{ + ValidCACert: true, + IsAnomaly: false, + FailingGateways: 0, + TransportStatus: map[string]string{ + "openvpn": "ok", + }, + }, + }, + } for idx, tt := range tests { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { m := &riseupvpn.Measurer{} @@ -498,3 +698,95 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) { }) } } + +func generateMockGetter(requestResponse map[string]string, responseStatus map[string]bool) urlgetter.MultiGetter { + return func(ctx context.Context, g urlgetter.Getter) (urlgetter.TestKeys, error) { + url := g.Target + responseBody, foundRequest := requestResponse[url] + isSuccessStatus, foundStatus := responseStatus[url] + if !foundRequest || !foundStatus { + return urlgetter.DefaultMultiGetter(ctx, g) + } + + var failure *string + var failedOperation *string + isBlocked := !isSuccessStatus + var responseStatus int64 = 200 + + if isBlocked { + responseBody = "" + eofError := io.EOF.Error() + failure = &eofError + connectOperation := errorx.ConnectOperation + failedOperation = &connectOperation + responseStatus = 0 + } + + tcpConnect := archival.TCPConnectEntry{ + // use some dummy IP/Port combination for URLs, we don't do DNS resolution + IP: "123.456.234.123", + Port: 443, + Status: archival.TCPConnectStatus{ + Success: isSuccessStatus, + Blocked: &isBlocked, + Failure: failure, + }, + } + if strings.Contains(url, "tcpconnect://") { + ipPort := strings.Split(strings.Split(url, "//")[1], ":") + port, err := strconv.ParseInt(ipPort[1], 10, 32) + if err == nil { + tcpConnect.IP = ipPort[0] + tcpConnect.Port = int(port) + } + } + + tk := urlgetter.TestKeys{ + Failure: failure, + FailedOperation: failedOperation, + HTTPResponseStatus: responseStatus, + HTTPResponseBody: responseBody, + Requests: []archival.RequestEntry{archival.RequestEntry{ + Failure: failure, + Request: archival.HTTPRequest{ + URL: url, + Body: archival.MaybeBinaryValue{}, + BodyIsTruncated: false, + }, + Response: archival.HTTPResponse{ + Body: archival.HTTPBody{ + Value: responseBody, + }, + BodyIsTruncated: false, + }}, + }, + TCPConnect: []archival.TCPConnectEntry{tcpConnect}, + } + return tk, nil + } +} +func generateDefaultMockGetter(responseStatuses map[string]bool) urlgetter.MultiGetter { + return generateMockGetter(RequestResponse, responseStatuses) +} + +func runDefaultMockTest(t *testing.T, multiGetter urlgetter.MultiGetter) *model.Measurement { + measurer := riseupvpn.Measurer{ + Config: riseupvpn.Config{}, + Getter: multiGetter, + } + + measurement := new(model.Measurement) + err := measurer.Run( + context.Background(), + &mockable.Session{ + MockableLogger: log.Log, + }, + measurement, + model.NewPrinterCallbacks(log.Log), + ) + + if err != nil { + t.Fatal(err) + } + return measurement +} From 4e344f1fcff05047530013e4597e028e15b63349 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 31 Mar 2021 12:19:03 +0200 Subject: [PATCH 21/28] fix(ooniprobe): disable maxRuntime when in the background (#276) Part of https://github.com/ooni/probe/issues/1299 --- cmd/ooniprobe/internal/nettests/nettests.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index 9907194..0712691 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -116,11 +116,13 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err } maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second + if c.RunType == "timed" && maxRuntime > 0 { + log.Debug("disabling maxRuntime when running in the background") + maxRuntime = 0 + } start := time.Now() c.ntStartTime = start for idx, input := range inputs { - // TODO(bassosimone): should we allow for interruption when running - // in unattended mode? Likewise, should we honor MaxRuntime? if c.Probe.IsTerminated() { log.Info("user requested us to terminate using Ctrl-C") break From 969d8b772fac01d6e21f4a3e17cbbd21727002e3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 31 Mar 2021 14:06:05 +0200 Subject: [PATCH 22/28] fix(ooniprobe): consistent progress with maxRuntime (#277) See https://github.com/ooni/probe/issues/1299 --- cmd/ooniprobe/internal/cli/run/run.go | 1 + cmd/ooniprobe/internal/nettests/nettests.go | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/ooniprobe/internal/cli/run/run.go b/cmd/ooniprobe/internal/cli/run/run.go index 446e43e..e0e827d 100644 --- a/cmd/ooniprobe/internal/cli/run/run.go +++ b/cmd/ooniprobe/internal/cli/run/run.go @@ -68,6 +68,7 @@ func init() { Probe: probe, InputFiles: *inputFile, Inputs: *input, + RunType: "manual", }) }) diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index 0712691..89dd005 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -212,8 +212,18 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err // OnProgress should be called when a new progress event is available. func (c *Controller) OnProgress(perc float64, msg string) { - // TODO(bassosimone): should we adjust this algorithm when we have a - // maximum runtime that we would like to honor? + // when we have maxRuntime, honor it + maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second + if c.RunType == "manual" && maxRuntime > 0 { + elapsed := time.Since(c.ntStartTime) + perc = float64(elapsed) / float64(maxRuntime) + eta := maxRuntime.Seconds() - elapsed.Seconds() + log.Debugf("OnProgress: %f - %s", perc, msg) + key := fmt.Sprintf("%T", c.nt) + output.Progress(key, perc, eta, msg) + return + } + // otherwise estimate the ETA log.Debugf("OnProgress: %f - %s", perc, msg) var eta float64 eta = -1.0 From 5c47a87af7b026b391a29a9dab9afcc5794a3078 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 31 Mar 2021 14:30:30 +0200 Subject: [PATCH 23/28] feat(ooniprobe): discard lists not in selected categories (#278) * feat(ooniprobe): discard lists not in selected categories One day we may make an integration mistake and for any reason we may end up with URLs that do not belong to the categories originally selected by the user. If that happens, it's nice to have a safety net where we remove URLs that do not belong to the right category before proceeding with testing. This diff was conceived while discussing the robustness of https://github.com/ooni/probe/issues/1299 with @hellais. * fix behavior and add unit test * more robust --- .../internal/nettests/web_connectivity.go | 29 ++++++- .../nettests/web_connectivity_test.go | 77 +++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 cmd/ooniprobe/internal/nettests/web_connectivity_test.go diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index fc752b2..c304701 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -9,6 +9,30 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/model" ) +// preventMistakes makes the code more robust with respect to any possible +// integration issue where the backend returns to us URLs that don't +// belong to the category codes we requested. +func preventMistakes(input []model.URLInfo, categories []string) (output []model.URLInfo) { + if len(categories) <= 0 { + return input + } + for _, entry := range input { + var found bool + for _, cat := range categories { + if entry.CategoryCode == cat { + found = true + break + } + } + if !found { + log.Warnf("URL %+v not in %+v; skipping", entry, categories) + continue + } + output = append(output, entry) + } + return +} + func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64, error) { inputloader := &engine.InputLoader{ CheckInConfig: &model.CheckInConfig{ @@ -29,11 +53,12 @@ func lookupURLs(ctl *Controller, categories []string) ([]string, map[int64]int64 } log.Infof("Calling CheckIn API with %s runType", ctl.RunType) testlist, err := inputloader.Load(context.Background()) - var urls []string - urlIDMap := make(map[int64]int64) if err != nil { return nil, nil, err } + testlist = preventMistakes(testlist, categories) + var urls []string + urlIDMap := make(map[int64]int64) for idx, url := range testlist { log.Debugf("Going over URL %d", idx) urlID, err := database.CreateOrUpdateURL( diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity_test.go b/cmd/ooniprobe/internal/nettests/web_connectivity_test.go new file mode 100644 index 0000000..5c17c64 --- /dev/null +++ b/cmd/ooniprobe/internal/nettests/web_connectivity_test.go @@ -0,0 +1,77 @@ +package nettests + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/engine/model" +) + +func TestPreventMistakesWithCategories(t *testing.T) { + input := []model.URLInfo{{ + CategoryCode: "NEWS", + URL: "https://repubblica.it/", + CountryCode: "IT", + }, { + CategoryCode: "HACK", + URL: "https://2600.com", + CountryCode: "XX", + }, { + CategoryCode: "FILE", + URL: "https://addons.mozilla.org/", + CountryCode: "XX", + }} + desired := []model.URLInfo{{ + CategoryCode: "NEWS", + URL: "https://repubblica.it/", + CountryCode: "IT", + }, { + CategoryCode: "FILE", + URL: "https://addons.mozilla.org/", + CountryCode: "XX", + }} + output := preventMistakes(input, []string{"NEWS", "FILE"}) + if diff := cmp.Diff(desired, output); diff != "" { + t.Fatal(diff) + } +} + +func TestPreventMistakesWithoutCategoriesAndNil(t *testing.T) { + input := []model.URLInfo{{ + CategoryCode: "NEWS", + URL: "https://repubblica.it/", + CountryCode: "IT", + }, { + CategoryCode: "HACK", + URL: "https://2600.com", + CountryCode: "XX", + }, { + CategoryCode: "FILE", + URL: "https://addons.mozilla.org/", + CountryCode: "XX", + }} + output := preventMistakes(input, nil) + if diff := cmp.Diff(input, output); diff != "" { + t.Fatal(diff) + } +} + +func TestPreventMistakesWithoutCategoriesAndEmpty(t *testing.T) { + input := []model.URLInfo{{ + CategoryCode: "NEWS", + URL: "https://repubblica.it/", + CountryCode: "IT", + }, { + CategoryCode: "HACK", + URL: "https://2600.com", + CountryCode: "XX", + }, { + CategoryCode: "FILE", + URL: "https://addons.mozilla.org/", + CountryCode: "XX", + }} + output := preventMistakes(input, []string{}) + if diff := cmp.Diff(input, output); diff != "" { + t.Fatal(diff) + } +} From bd451016f5467d8dc61918b0119aea217ce65e70 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 31 Mar 2021 15:59:19 +0200 Subject: [PATCH 24/28] release 3.9.0 process: reduce warnings (#279) * fix(riseupvpn): address gofmt warning Thanks to https://goreportcard.com/report/github.com/ooni/probe-cli. * fix(utils.go): correct the docu-comment Thanks to https://goreportcard.com/report/github.com/ooni/probe-cli * fix: improve spelling Thanks to https://goreportcard.com/report/github.com/ooni/probe-cli * fix(modelx_test.go): avoid inefassign warning Thanks to https://goreportcard.com/report/github.com/ooni/probe-cli * fix: reduce number of ineffective assignments Thanks to https://goreportcard.com/report/github.com/ooni/probe-cli --- cmd/ooniprobe/internal/utils/utils.go | 2 +- internal/engine/experiment/riseupvpn/riseupvpn_test.go | 5 ++++- internal/engine/experiment/tlstool/internal/splitter.go | 1 - internal/engine/inputloader.go | 2 +- internal/engine/legacy/netx/dialer_test.go | 3 +++ internal/engine/legacy/netx/modelx/modelx_test.go | 2 +- internal/engine/netx/errorx/errorx_test.go | 3 +++ pkg/oonimkall/session_integration_test.go | 2 +- 8 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cmd/ooniprobe/internal/utils/utils.go b/cmd/ooniprobe/internal/utils/utils.go index 328e4da..ff25e20 100644 --- a/cmd/ooniprobe/internal/utils/utils.go +++ b/cmd/ooniprobe/internal/utils/utils.go @@ -39,7 +39,7 @@ func EscapeAwareRuneCountInString(s string) int { return n } -// RightPadd adds right padding in from of a string +// RightPad adds right padding in from of a string func RightPad(str string, length int) string { c := length - EscapeAwareRuneCountInString(str) if c < 0 { diff --git a/internal/engine/experiment/riseupvpn/riseupvpn_test.go b/internal/engine/experiment/riseupvpn/riseupvpn_test.go index 1be8aa4..6206034 100644 --- a/internal/engine/experiment/riseupvpn/riseupvpn_test.go +++ b/internal/engine/experiment/riseupvpn/riseupvpn_test.go @@ -555,6 +555,9 @@ func TestMissingTransport(t *testing.T) { transports = transports[:len(transports)-1] eipService.Gateways[1].Capabilities.Transport = transports eipservicejson, err := json.Marshal(eipservice) + if err != nil { + t.Fatal(err) + } requestResponseMap := map[string]string{ eipserviceurl: string(eipservicejson), @@ -746,7 +749,7 @@ func generateMockGetter(requestResponse map[string]string, responseStatus map[st FailedOperation: failedOperation, HTTPResponseStatus: responseStatus, HTTPResponseBody: responseBody, - Requests: []archival.RequestEntry{archival.RequestEntry{ + Requests: []archival.RequestEntry{{ Failure: failure, Request: archival.HTTPRequest{ URL: url, diff --git a/internal/engine/experiment/tlstool/internal/splitter.go b/internal/engine/experiment/tlstool/internal/splitter.go index 795f9a6..0aecfb8 100644 --- a/internal/engine/experiment/tlstool/internal/splitter.go +++ b/internal/engine/experiment/tlstool/internal/splitter.go @@ -29,7 +29,6 @@ func SNISplitter(input []byte, sni []byte) (output [][]byte) { } if len(buf) > 0 { output = append(output, buf) - buf = nil } output = append(output, input[idx+len(sni):]) return diff --git a/internal/engine/inputloader.go b/internal/engine/inputloader.go index 2dc1f87..4f28d3e 100644 --- a/internal/engine/inputloader.go +++ b/internal/engine/inputloader.go @@ -159,7 +159,7 @@ func (il *InputLoader) loadLocal() ([]model.URLInfo, error) { type inputLoaderOpenFn func(filepath string) (fs.File, error) // readfile reads inputs from the specified file. The open argument should be -// compatibile with stdlib's fs.Open and helps us with unit testing. +// compatible with stdlib's fs.Open and helps us with unit testing. func (il *InputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]model.URLInfo, error) { inputs := []model.URLInfo{} filep, err := open(filepath) diff --git a/internal/engine/legacy/netx/dialer_test.go b/internal/engine/legacy/netx/dialer_test.go index 7f5e016..186fdd8 100644 --- a/internal/engine/legacy/netx/dialer_test.go +++ b/internal/engine/legacy/netx/dialer_test.go @@ -101,6 +101,9 @@ func TestDialerSetCABundleWAI(t *testing.T) { func TestDialerForceSpecificSNI(t *testing.T) { dialer := netx.NewDialer() err := dialer.ForceSpecificSNI("www.facebook.com") + if err != nil { + t.Fatal(err) + } conn, err := dialer.DialTLS("tcp", "www.google.com:443") if err == nil { t.Fatal("expected an error here") diff --git a/internal/engine/legacy/netx/modelx/modelx_test.go b/internal/engine/legacy/netx/modelx/modelx_test.go index 82deb2e..a0ec4b4 100644 --- a/internal/engine/legacy/netx/modelx/modelx_test.go +++ b/internal/engine/legacy/netx/modelx/modelx_test.go @@ -56,7 +56,7 @@ func TestMeasurementRootWithMeasurementRootPanic(t *testing.T) { } }() ctx := context.Background() - ctx = WithMeasurementRoot(ctx, nil) + _ = WithMeasurementRoot(ctx, nil) } func TestErrWrapperPublicAPI(t *testing.T) { diff --git a/internal/engine/netx/errorx/errorx_test.go b/internal/engine/netx/errorx/errorx_test.go index 9b69d2a..f37ff98 100644 --- a/internal/engine/netx/errorx/errorx_test.go +++ b/internal/engine/netx/errorx/errorx_test.go @@ -175,6 +175,9 @@ func TestToFailureString(t *testing.T) { defer cancel() // fail immediately udpAddr := &net.UDPAddr{IP: net.ParseIP("216.58.212.164"), Port: 80, Zone: ""} udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0}) + if err != nil { + t.Fatal(err) + } sess, err := quic.DialEarlyContext(ctx, udpConn, udpAddr, "google.com:80", &tls.Config{}, &quic.Config{}) if err == nil { t.Fatal("expected an error here") diff --git a/pkg/oonimkall/session_integration_test.go b/pkg/oonimkall/session_integration_test.go index 725c872..9d9b7ff 100644 --- a/pkg/oonimkall/session_integration_test.go +++ b/pkg/oonimkall/session_integration_test.go @@ -78,7 +78,7 @@ func TestMaybeUpdateResourcesWithCancelledContext(t *testing.T) { ctx := sess.NewContext() ctx.Cancel() // cause immediate failure err = sess.MaybeUpdateResources(ctx) - // Explaination: we embed resources. We should change the API + // Explanation: we embed resources. We should change the API // and remove the context. Until we do that, let us just assert // that we have embedding and the context does not matter. if err != nil { From 7ca32b5ce62d20a7c7abf460fa9e04f9a6be1a30 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 31 Mar 2021 16:40:58 +0200 Subject: [PATCH 25/28] release process: update dependencies (#280) Part of the check-list at https://github.com/ooni/probe/issues/1369 --- Readme.md | 10 +-- go.mod | 18 ++--- go.sum | 65 ++++++++----------- ...tionstate_go1.14.go => connectionstate.go} | 4 +- .../netx/quicdialer/connectionstate_go1.15.go | 14 ---- 5 files changed, 37 insertions(+), 74 deletions(-) rename internal/engine/netx/quicdialer/{connectionstate_go1.14.go => connectionstate.go} (81%) delete mode 100644 internal/engine/netx/quicdialer/connectionstate_go1.15.go diff --git a/Readme.md b/Readme.md index 7abcd38..8520de3 100644 --- a/Readme.md +++ b/Readme.md @@ -80,10 +80,6 @@ go get -u -v ./... && go mod tidy ## Releasing -1. update binary data as described above; - -2. update `internal/version/version.go`; - -3. make sure you have updated dependencies; - -4. run `./build.sh release` and follow instructions. +Create an issue according to [the routine release template]( +https://github.com/ooni/probe/blob/master/.github/ISSUE_TEMPLATE/routine-sprint-releases.md) +and perform any item inside the check-list. diff --git a/go.mod b/go.mod index b24c571..f8bd676 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/cretz/bine v0.1.0 github.com/dchest/siphash v1.2.2 // indirect github.com/fatih/color v1.10.0 - github.com/golang/protobuf v1.4.3 // indirect github.com/google/go-cmp v0.5.2 github.com/google/martian/v3 v3.1.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 @@ -19,29 +18,26 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/hexops/gotextdiff v1.0.3 github.com/iancoleman/strcase v0.1.3 - github.com/lucas-clemente/quic-go v0.19.3 - github.com/marten-seemann/qtls-go1-15 v0.1.2 // indirect + github.com/lucas-clemente/quic-go v0.20.0 github.com/mattn/go-colorable v0.1.8 github.com/mattn/go-sqlite3 v1.14.6 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/miekg/dns v1.1.40 + github.com/miekg/dns v1.1.41 github.com/montanaflynn/stats v0.6.5 - github.com/ooni/psiphon v0.5.0 + github.com/ooni/psiphon v0.6.0 github.com/oschwald/geoip2-golang v1.5.0 github.com/pborman/getopt/v2 v2.1.0 github.com/pion/stun v0.3.5 github.com/pkg/errors v0.9.1 - github.com/rogpeppe/go-internal v1.7.0 + github.com/rogpeppe/go-internal v1.8.0 github.com/rubenv/sql-migrate v0.0.0-20210215143335-f84234893558 github.com/sirupsen/logrus v1.7.0 // indirect gitlab.com/yawning/obfs4.git v0.0.0-20201217005658-f638c33f6c6f - golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 - golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect - golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/net v0.0.0-20210331060903-cb1fcc7394e5 + golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 golang.org/x/text v0.3.5 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/protobuf v1.25.0 // indirect gopkg.in/AlecAivazis/survey.v1 v1.8.8 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect upper.io/db.v3 v3.8.0+incompatible diff --git a/go.sum b/go.sum index 4547262..b601407 100644 --- a/go.sum +++ b/go.sum @@ -139,13 +139,12 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -154,10 +153,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -165,7 +161,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -262,19 +257,18 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4= -github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= +github.com/lucas-clemente/quic-go v0.20.0 h1:FSU3YN5VnLafHR27Ejs1r1CYMS7XMyIVDzRewkDLNBw= +github.com/lucas-clemente/quic-go v0.20.0/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc= -github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= -github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-15 v0.1.2 h1:KLXnVazsIS+EhrEqXqg0NyQZ2rwxkaSaNxMFc1krFIA= -github.com/marten-seemann/qtls-go1-15 v0.1.2/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.4 h1:RehYMOyRW8hPVEja1KBVsFVNSm35Jj9Mvs5yNoZZ28A= +github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJGYmwfwKQN2gU= +github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -298,8 +292,8 @@ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.40 h1:pyyPFfGMnciYUk/mXpKkVmeMQjfXqt3FAJ2hy7tPiLA= -github.com/miekg/dns v1.1.40/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -341,8 +335,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/ooni/psiphon v0.5.0 h1:DAKZ66tmh4r6uop4yfSiQDObv7yNGb0j1jY+xeft6h4= -github.com/ooni/psiphon v0.5.0/go.mod h1:i1v6JweJtxDKaI0i1aEw2/Fr/CUi5BoQ75GYz5KmKwU= +github.com/ooni/psiphon v0.6.0 h1:TWXFSlXBOFUwfGblLKaLXSoI7UL6dreoPXHW2uQTHlc= +github.com/ooni/psiphon v0.6.0/go.mod h1:i1v6JweJtxDKaI0i1aEw2/Fr/CUi5BoQ75GYz5KmKwU= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -368,6 +362,7 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg= github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -402,8 +397,8 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.7.0 h1:3qqXGV8nn7GJT65debw77Dzrx9sfWYgP0DDo7xcMFRk= -github.com/rogpeppe/go-internal v1.7.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rubenv/sql-migrate v0.0.0-20210215143335-f84234893558 h1:o8N+eY3HGAzZ+5sXNdcbCVOHW3NOksmKeEOuygusmr8= github.com/rubenv/sql-migrate v0.0.0-20210215143335-f84234893558/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -521,8 +516,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -532,6 +527,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -553,11 +549,11 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210331060903-cb1fcc7394e5 h1:zuP3axpB9rV3xH0EA7n3/gCrNPZm2SRl0l4mVH2BRj4= +golang.org/x/net v0.0.0-20210331060903-cb1fcc7394e5/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -569,8 +565,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -593,7 +589,6 @@ golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -604,12 +599,12 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 h1:cEhElsAv9LUt9ZUUocxzWe05oFLVd+AA2nstydTeI8g= +golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -634,7 +629,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -660,7 +655,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -672,17 +666,12 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/AlecAivazis/survey.v1 v1.8.8 h1:5UtTowJZTz1j7NxVzDGKTz6Lm9IWm8DDF6b7a2wq9VY= gopkg.in/AlecAivazis/survey.v1 v1.8.8/go.mod h1:CaHjv79TCgAvXMSFJSVgonHXYWxnhzI3eoHtnX5UgUo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -719,8 +708,6 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= diff --git a/internal/engine/netx/quicdialer/connectionstate_go1.14.go b/internal/engine/netx/quicdialer/connectionstate.go similarity index 81% rename from internal/engine/netx/quicdialer/connectionstate_go1.14.go rename to internal/engine/netx/quicdialer/connectionstate.go index 80d6f3f..eb69462 100644 --- a/internal/engine/netx/quicdialer/connectionstate_go1.14.go +++ b/internal/engine/netx/quicdialer/connectionstate.go @@ -1,5 +1,3 @@ -// +build !go1.15 - package quicdialer import ( @@ -10,5 +8,5 @@ import ( // ConnectionState returns the ConnectionState of a QUIC Session. func ConnectionState(sess quic.EarlySession) tls.ConnectionState { - return tls.ConnectionState{} + return sess.ConnectionState().TLS.ConnectionState } diff --git a/internal/engine/netx/quicdialer/connectionstate_go1.15.go b/internal/engine/netx/quicdialer/connectionstate_go1.15.go deleted file mode 100644 index c43ca00..0000000 --- a/internal/engine/netx/quicdialer/connectionstate_go1.15.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build go1.15 - -package quicdialer - -import ( - "crypto/tls" - - "github.com/lucas-clemente/quic-go" -) - -// ConnectionState returns the ConnectionState of a QUIC Session. -func ConnectionState(sess quic.EarlySession) tls.ConnectionState { - return sess.ConnectionState().ConnectionState -} From 31e478b04ef517741fdfd34ab47ff7e5131a92a1 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 1 Apr 2021 16:57:31 +0200 Subject: [PATCH 26/28] refactor: redesign how we import assets (#260) * fix(pkg.go.dev): import a subpackage containing the assets We're trying to fix this issue that pkg.go.dev does not build. Thanks to @hellais for this very neat idea! Let's keep our fingers crossed and see whether it fixes! * feat: use embedded geoip databases Closes https://github.com/ooni/probe/issues/1372. Work done as part of https://github.com/ooni/probe/issues/1369. * fix(assetsx): add tests * feat: simplify and just vendor uncompressed DBs * remove tests that seems not necessary anymore * fix: run go mod tidy * Address https://github.com/ooni/probe-cli/pull/260/files#r605181364 * rewrite a test in a better way * fix: gently cleanup the legacy assetsdir Do not remove the whole directory with brute force. Just zap the files whose name we know. Then attempt to delete the legacy directory as well. If not empty, just fail. This is fine because it means the user has stored other files inside the directory. * fix: create .miniooni if missing --- .github/workflows/alltests.yml | 1 - .github/workflows/coverage.yml | 1 - .github/workflows/shorttests.yml | 1 - E2E/miniooni.bash | 1 - QA/pyrun.sh | 1 - Readme.md | 10 +- build-android.bash | 1 - build-ios.bash | 1 - build-miniooni.sh | 3 - build.sh | 5 - cmd/ooniprobe/internal/ooni/ooni.go | 10 +- go.mod | 1 + go.sum | 2 + internal/cmd/getresources/getresources.go | 47 +----- internal/cmd/miniooni/libminiooni.go | 15 +- internal/engine/experiment.go | 3 - .../engine/experiment/dnscheck/dnscheck.go | 2 +- .../fbmessenger/fbmessenger_test.go | 1 - internal/engine/experiment/hhfm/hhfm_test.go | 1 - .../stunreachability/stunreachability.go | 2 +- .../engine/experiment/urlgetter/getter.go | 5 +- .../experiment/webconnectivity/control.go | 2 +- .../webconnectivity/control_test.go | 9 - .../webconnectivity/webconnectivity_test.go | 1 - internal/engine/geolocate/geolocate.go | 44 +---- internal/engine/geolocate/geolocate_test.go | 77 +-------- internal/engine/geolocate/mmdblookup.go | 13 +- internal/engine/geolocate/mmdblookup_test.go | 56 +------ internal/engine/inputloader_network_test.go | 1 - internal/engine/inputloader_test.go | 1 - internal/engine/internal/mockable/mockable.go | 6 - .../engine/internal/psiphonx/psiphonx_test.go | 2 - internal/engine/legacy/assetsdir/assetsdir.go | 62 +++++++ .../engine/legacy/assetsdir/assetsdir_test.go | 40 +++++ internal/engine/model/experiment.go | 1 - internal/engine/netx/archival/archival.go | 12 +- .../netx/archival/archival_internal_test.go | 41 +++++ .../engine/netx/archival/archival_test.go | 45 +---- .../netx/archival/archival_test_internal.go | 8 - internal/engine/resources/assets.go | 42 ----- internal/engine/resources/doc.go | 3 - internal/engine/resourcesmanager/.gitignore | 3 - .../resourcesmanager/resourcesmanager.go | 157 ------------------ .../resourcesmanager/resourcesmanager_test.go | 142 ---------------- internal/engine/session.go | 43 ++--- internal/engine/session_integration_test.go | 44 +---- internal/engine/session_internal_test.go | 4 - pkg/oonimkall/internal/tasks/runner.go | 1 - pkg/oonimkall/session.go | 25 ++- pkg/oonimkall/session_integration_test.go | 20 +-- pkg/oonimkall/task_integration_test.go | 32 ---- 51 files changed, 243 insertions(+), 808 deletions(-) create mode 100644 internal/engine/legacy/assetsdir/assetsdir.go create mode 100644 internal/engine/legacy/assetsdir/assetsdir_test.go create mode 100644 internal/engine/netx/archival/archival_internal_test.go delete mode 100644 internal/engine/netx/archival/archival_test_internal.go delete mode 100644 internal/engine/resources/assets.go delete mode 100644 internal/engine/resources/doc.go delete mode 100644 internal/engine/resourcesmanager/.gitignore delete mode 100644 internal/engine/resourcesmanager/resourcesmanager.go delete mode 100644 internal/engine/resourcesmanager/resourcesmanager_test.go diff --git a/.github/workflows/alltests.yml b/.github/workflows/alltests.yml index 34ddc2e..be88cdd 100644 --- a/.github/workflows/alltests.yml +++ b/.github/workflows/alltests.yml @@ -13,5 +13,4 @@ jobs: with: go-version: "1.16" - uses: actions/checkout@v2 - - run: go run ./internal/cmd/getresources - run: go test -race -tags shaping ./... diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 2399cb5..b242c76 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,7 +15,6 @@ jobs: with: go-version: "${{ matrix.go }}" - uses: actions/checkout@v2 - - run: go run ./internal/cmd/getresources - run: go test -short -race -tags shaping -coverprofile=probe-cli.cov ./... - uses: shogo82148/actions-goveralls@v1 with: diff --git a/.github/workflows/shorttests.yml b/.github/workflows/shorttests.yml index ffcdb09..3ade3d5 100644 --- a/.github/workflows/shorttests.yml +++ b/.github/workflows/shorttests.yml @@ -15,5 +15,4 @@ jobs: with: go-version: "${{ matrix.go }}" - uses: actions/checkout@v2 - - run: go run ./internal/cmd/getresources - run: go test -short -race -tags shaping ./... diff --git a/E2E/miniooni.bash b/E2E/miniooni.bash index 01fb975..36b7ce5 100755 --- a/E2E/miniooni.bash +++ b/E2E/miniooni.bash @@ -1,6 +1,5 @@ #!/bin/bash set -e -go run ./internal/cmd/getresources go build -v ./internal/cmd/miniooni probeservices=() probeservices+=( "https://ps1.ooni.io" ) diff --git a/QA/pyrun.sh b/QA/pyrun.sh index 2c8db8a..76f3f28 100755 --- a/QA/pyrun.sh +++ b/QA/pyrun.sh @@ -1,7 +1,6 @@ #!/bin/sh set -ex export GOPATH=/jafar/QA/GOPATH GOCACHE=/jafar/QA/GOCACHE GO111MODULE=on -go run ./internal/cmd/getresources go build -v ./internal/cmd/miniooni go build -v ./internal/cmd/jafar sudo ./QA/$1.py ./miniooni diff --git a/Readme.md b/Readme.md index 8520de3..acd0838 100644 --- a/Readme.md +++ b/Readme.md @@ -24,15 +24,7 @@ Every top-level directory contains an explanatory README file. ## Development setup Be sure you have golang >= 1.16 and a C compiler (when developing for Windows, you -need Mingw-w64 installed). - -You need to download assets first using: - -```bash -go run ./internal/cmd/getresources -``` - -Then you can build using: +need Mingw-w64 installed). You can build using: ```bash go build -v ./cmd/ooniprobe diff --git a/build-android.bash b/build-android.bash index 2b5e9ab..161edb7 100755 --- a/build-android.bash +++ b/build-android.bash @@ -28,5 +28,4 @@ export PATH=$(go env GOPATH)/bin:$PATH go get -u golang.org/x/mobile/cmd/gomobile gomobile init output=MOBILE/android/oonimkall.aar -go run ./internal/cmd/getresources gomobile bind -target=android -o $output -ldflags="-s -w" ./pkg/oonimkall diff --git a/build-ios.bash b/build-ios.bash index 70962a5..a2cc5a6 100755 --- a/build-ios.bash +++ b/build-ios.bash @@ -6,5 +6,4 @@ export PATH=$(go env GOPATH)/bin:$PATH go get -u golang.org/x/mobile/cmd/gomobile gomobile init output=MOBILE/ios/oonimkall.framework -go run ./internal/cmd/getresources gomobile bind -target=ios -o $output -ldflags="-s -w" ./pkg/oonimkall diff --git a/build-miniooni.sh b/build-miniooni.sh index 85d1610..18b2434 100755 --- a/build-miniooni.sh +++ b/build-miniooni.sh @@ -2,12 +2,10 @@ set -e case $1 in macos|darwin) - go run ./internal/cmd/getresources export GOOS=darwin GOARCH=amd64 go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./internal/cmd/miniooni echo "Binary ready at ./CLI/darwin/amd64/miniooni";; linux) - go run ./internal/cmd/getresources export GOOS=linux GOARCH=386 go build -o ./CLI/linux/386 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni echo "Binary ready at ./CLI/linux/386/miniooni" @@ -21,7 +19,6 @@ case $1 in go build -o ./CLI/linux/arm64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni echo "Binary ready at ./CLI/linux/arm64/miniooni";; windows) - go run ./internal/cmd/getresources export GOOS=windows GOARCH=386 go build -o ./CLI/windows/386 -ldflags="-s -w" ./internal/cmd/miniooni echo "Binary ready at ./CLI/windows/386/miniooni.exe" diff --git a/build.sh b/build.sh index a8196ac..456f28e 100755 --- a/build.sh +++ b/build.sh @@ -12,7 +12,6 @@ case $1 in ;; windows_amd64) - go run ./internal/cmd/getresources # Note! This assumes we've installed the mingw-w64 compiler. GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \ go build -ldflags='-s -w' ./cmd/ooniprobe @@ -23,7 +22,6 @@ case $1 in ;; windows_386) - go run ./internal/cmd/getresources # Note! This assumes we've installed the mingw-w64 compiler. GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc \ go build -ldflags='-s -w' ./cmd/ooniprobe @@ -40,7 +38,6 @@ case $1 in ;; linux_amd64) - go run ./internal/cmd/getresources docker pull --platform linux/amd64 golang:1.16-alpine docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine tar -cvzf ooniprobe_${v}_linux_amd64.tar.gz LICENSE.md Readme.md ooniprobe @@ -48,7 +45,6 @@ case $1 in ;; linux_386) - go run ./internal/cmd/getresources docker pull --platform linux/386 golang:1.16-alpine docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine tar -cvzf ooniprobe_${v}_linux_386.tar.gz LICENSE.md Readme.md ooniprobe @@ -64,7 +60,6 @@ case $1 in macos|darwin) set -x - go run ./internal/cmd/getresources # Note! The following line _assumes_ you have a working C compiler. If you # have Xcode command line tools installed, you are fine. go build -ldflags='-s -w' ./cmd/ooniprobe diff --git a/cmd/ooniprobe/internal/ooni/ooni.go b/cmd/ooniprobe/internal/ooni/ooni.go index ab9c065..7dfb079 100644 --- a/cmd/ooniprobe/internal/ooni/ooni.go +++ b/cmd/ooniprobe/internal/ooni/ooni.go @@ -14,6 +14,7 @@ import ( "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils" engine "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/legacy/assetsdir" "github.com/pkg/errors" "upper.io/db.v3/lib/sqlbuilder" ) @@ -177,6 +178,14 @@ func (p *Probe) Init(softwareName, softwareVersion string) error { } p.db = db + // We cleanup the assets files used by versions of ooniprobe + // older than v3.9.0, where we started embedding the assets + // into the binary and use that directly. This cleanup doesn't + // remove the whole directory but only known files inside it + // and then the directory itself, if empty. We explicitly discard + // the return value as it does not matter to us here. + _, _ = assetsdir.Cleanup(utils.AssetsDir(p.home)) + tempDir, err := ioutil.TempDir("", "ooni") if err != nil { return errors.Wrap(err, "creating TempDir") @@ -199,7 +208,6 @@ func (p *Probe) NewSession() (*engine.Session, error) { return nil, errors.Wrap(err, "creating engine's kvstore") } return engine.NewSession(engine.SessionConfig{ - AssetsDir: utils.AssetsDir(p.home), KVStore: kvstore, Logger: enginex.Logger, SoftwareName: p.softwareName, diff --git a/go.mod b/go.mod index f8bd676..77327f2 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/miekg/dns v1.1.41 github.com/montanaflynn/stats v0.6.5 + github.com/ooni/probe-assets v0.0.0-20210401100648-90ed7b6dff90 github.com/ooni/psiphon v0.6.0 github.com/oschwald/geoip2-golang v1.5.0 github.com/pborman/getopt/v2 v2.1.0 diff --git a/go.sum b/go.sum index b601407..1fe72ff 100644 --- a/go.sum +++ b/go.sum @@ -335,6 +335,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/ooni/probe-assets v0.0.0-20210401100648-90ed7b6dff90 h1:G7urihGBD88p5iU1bg7cFAqX/38qIPZH03wCTmyUAXI= +github.com/ooni/probe-assets v0.0.0-20210401100648-90ed7b6dff90/go.mod h1:N0PyNM3aadlYDDCFXAPzs54HC54+MZA/4/xnCtd9EAo= github.com/ooni/psiphon v0.6.0 h1:TWXFSlXBOFUwfGblLKaLXSoI7UL6dreoPXHW2uQTHlc= github.com/ooni/psiphon v0.6.0/go.mod h1:i1v6JweJtxDKaI0i1aEw2/Fr/CUi5BoQ75GYz5KmKwU= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= diff --git a/internal/cmd/getresources/getresources.go b/internal/cmd/getresources/getresources.go index 3f6ba7d..c8eebb1 100644 --- a/internal/cmd/getresources/getresources.go +++ b/internal/cmd/getresources/getresources.go @@ -2,50 +2,13 @@ package main import ( - "crypto/sha256" - "errors" - "fmt" - "io/ioutil" "log" - "net/http" - "net/url" - "path/filepath" - - "github.com/ooni/probe-cli/v3/internal/engine/resources" + "time" ) func main() { - for name, ri := range resources.All { - if err := getit(name, &ri); err != nil { - log.Fatal(err) - } - } -} - -func getit(name string, ri *resources.ResourceInfo) error { - workDir := filepath.Join("internal", "engine", "resourcesmanager") - URL, err := url.Parse(resources.BaseURL) - if err != nil { - return err - } - URL.Path = ri.URLPath - log.Println("fetching", URL.String()) - resp, err := http.Get(URL.String()) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return errors.New("http request failed") - } - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - checksum := fmt.Sprintf("%x", sha256.Sum256(data)) - if checksum != ri.GzSHA256 { - return errors.New("sha256 mismatch") - } - fullpath := filepath.Join(workDir, name+".gz") - return ioutil.WriteFile(fullpath, data, 0644) + log.Printf("This command is no longer needed. We will keep it into") + log.Printf("the repository until the end of 2021, so you have\n") + log.Printf("time to adjust your build workflow.\n") + time.Sleep(5 * time.Second) } diff --git a/internal/cmd/miniooni/libminiooni.go b/internal/cmd/miniooni/libminiooni.go index a126cf4..25b3a65 100644 --- a/internal/cmd/miniooni/libminiooni.go +++ b/internal/cmd/miniooni/libminiooni.go @@ -18,6 +18,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/engine/humanizex" + "github.com/ooni/probe-cli/v3/internal/engine/legacy/assetsdir" "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor" "github.com/ooni/probe-cli/v3/internal/version" @@ -303,9 +304,18 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { homeDir := gethomedir(currentOptions.HomeDir) fatalIfFalse(homeDir != "", "home directory is empty") miniooniDir := path.Join(homeDir, ".miniooni") + err = os.MkdirAll(miniooniDir, 0700) + fatalOnError(err, "cannot create $HOME/.miniooni directory") + + // We cleanup the assets files used by versions of ooniprobe + // older than v3.9.0, where we started embedding the assets + // into the binary and use that directly. This cleanup doesn't + // remove the whole directory but only known files inside it + // and then the directory itself, if empty. We explicitly discard + // the return value as it does not matter to us here. assetsDir := path.Join(miniooniDir, "assets") - err = os.MkdirAll(assetsDir, 0700) - fatalOnError(err, "cannot create assets directory") + _, _ = assetsdir.Cleanup(assetsDir) + log.Debugf("miniooni state directory: %s", miniooniDir) consentFile := path.Join(miniooniDir, "informed") @@ -324,7 +334,6 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { fatalOnError(err, "cannot create kvstore2 directory") config := engine.SessionConfig{ - AssetsDir: assetsDir, KVStore: kvstore, Logger: logger, ProxyURL: proxyURL, diff --git a/internal/engine/experiment.go b/internal/engine/experiment.go index 0ba160c..cd6ad89 100644 --- a/internal/engine/experiment.go +++ b/internal/engine/experiment.go @@ -7,7 +7,6 @@ import ( "errors" "net/http" "os" - "strconv" "time" "github.com/ooni/probe-cli/v3/internal/engine/geolocate" @@ -17,7 +16,6 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/dialer" "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" - "github.com/ooni/probe-cli/v3/internal/engine/resources" "github.com/ooni/probe-cli/v3/internal/version" ) @@ -168,7 +166,6 @@ func (e *Experiment) newMeasurement(input string) *model.Measurement { TestStartTime: e.testStartTime, TestVersion: e.testVersion, } - m.AddAnnotation("assets_version", strconv.FormatInt(resources.Version, 10)) m.AddAnnotation("engine_name", "ooniprobe-engine") m.AddAnnotation("engine_version", version.Version) m.AddAnnotation("platform", platform.Name()) diff --git a/internal/engine/experiment/dnscheck/dnscheck.go b/internal/engine/experiment/dnscheck/dnscheck.go index 1a17b3e..6e8906c 100644 --- a/internal/engine/experiment/dnscheck/dnscheck.go +++ b/internal/engine/experiment/dnscheck/dnscheck.go @@ -169,7 +169,7 @@ func (m *Measurer) Run( ResolveSaver: evsaver, }) addrs, err := m.lookupHost(ctx, URL.Hostname(), resolver) - queries := archival.NewDNSQueriesList(begin, evsaver.Read(), sess.ASNDatabasePath()) + queries := archival.NewDNSQueriesList(begin, evsaver.Read()) tk.BootstrapFailure = archival.NewFailure(err) if len(queries) > 0 { // We get no queries in case we are resolving an IP address, since diff --git a/internal/engine/experiment/fbmessenger/fbmessenger_test.go b/internal/engine/experiment/fbmessenger/fbmessenger_test.go index e6640bc..ab6438e 100644 --- a/internal/engine/experiment/fbmessenger/fbmessenger_test.go +++ b/internal/engine/experiment/fbmessenger/fbmessenger_test.go @@ -222,7 +222,6 @@ func TestComputeEndpointStatsDNSIsLying(t *testing.T) { func newsession(t *testing.T) model.ExperimentSession { sess, err := engine.NewSession(engine.SessionConfig{ - AssetsDir: "../../testdata", AvailableProbeServices: []model.Service{{ Address: "https://ams-pg-test.ooni.org", Type: "https", diff --git a/internal/engine/experiment/hhfm/hhfm_test.go b/internal/engine/experiment/hhfm/hhfm_test.go index 63ab715..1cd1504 100644 --- a/internal/engine/experiment/hhfm/hhfm_test.go +++ b/internal/engine/experiment/hhfm/hhfm_test.go @@ -559,7 +559,6 @@ func TestTransactCannotReadBody(t *testing.T) { func newsession(t *testing.T) model.ExperimentSession { sess, err := engine.NewSession(engine.SessionConfig{ - AssetsDir: "../../testdata", AvailableProbeServices: []model.Service{{ Address: "https://ams-pg-test.ooni.org", Type: "https", diff --git a/internal/engine/experiment/stunreachability/stunreachability.go b/internal/engine/experiment/stunreachability/stunreachability.go index 4d5ba5b..b4b2211 100644 --- a/internal/engine/experiment/stunreachability/stunreachability.go +++ b/internal/engine/experiment/stunreachability/stunreachability.go @@ -107,7 +107,7 @@ func (tk *TestKeys) run( tk.NetworkEvents, archival.NewNetworkEventsList(begin, events)..., ) tk.Queries = append( - tk.Queries, archival.NewDNSQueriesList(begin, events, sess.ASNDatabasePath())..., + tk.Queries, archival.NewDNSQueriesList(begin, events)..., ) return err } diff --git a/internal/engine/experiment/urlgetter/getter.go b/internal/engine/experiment/urlgetter/getter.go index 6ec1dd6..9957efa 100644 --- a/internal/engine/experiment/urlgetter/getter.go +++ b/internal/engine/experiment/urlgetter/getter.go @@ -58,10 +58,7 @@ func (g Getter) Get(ctx context.Context) (TestKeys, error) { tk.FailedOperation = archival.NewFailedOperation(err) tk.Failure = archival.NewFailure(err) events := saver.Read() - tk.Queries = append( - tk.Queries, archival.NewDNSQueriesList( - g.Begin, events, g.Session.ASNDatabasePath())..., - ) + tk.Queries = append(tk.Queries, archival.NewDNSQueriesList(g.Begin, events)...) tk.NetworkEvents = append( tk.NetworkEvents, archival.NewNetworkEventsList(g.Begin, events)..., ) diff --git a/internal/engine/experiment/webconnectivity/control.go b/internal/engine/experiment/webconnectivity/control.go index 4d189b0..f3cd6b3 100644 --- a/internal/engine/experiment/webconnectivity/control.go +++ b/internal/engine/experiment/webconnectivity/control.go @@ -78,7 +78,7 @@ func (dns *ControlDNSResult) FillASNs(sess model.ExperimentSession) { for _, ip := range dns.Addrs { // TODO(bassosimone): this would be more efficient if we'd open just // once the database and then reuse it for every address. - asn, _, _ := geolocate.LookupASN(sess.ASNDatabasePath(), ip) + asn, _, _ := geolocate.LookupASN(ip) dns.ASNs = append(dns.ASNs, int64(asn)) } } diff --git a/internal/engine/experiment/webconnectivity/control_test.go b/internal/engine/experiment/webconnectivity/control_test.go index dc704ad..e33ff61 100644 --- a/internal/engine/experiment/webconnectivity/control_test.go +++ b/internal/engine/experiment/webconnectivity/control_test.go @@ -16,15 +16,6 @@ func TestFillASNsEmpty(t *testing.T) { } } -func TestFillASNsNoDatabase(t *testing.T) { - dns := new(webconnectivity.ControlDNSResult) - dns.Addrs = []string{"8.8.8.8", "1.1.1.1"} - dns.FillASNs(new(mockable.Session)) - if diff := cmp.Diff(dns.ASNs, []int64{0, 0}); diff != "" { - t.Fatal(diff) - } -} - func TestFillASNsSuccess(t *testing.T) { sess := newsession(t, false) dns := new(webconnectivity.ControlDNSResult) diff --git a/internal/engine/experiment/webconnectivity/webconnectivity_test.go b/internal/engine/experiment/webconnectivity/webconnectivity_test.go index f61ba0b..13a6bfa 100644 --- a/internal/engine/experiment/webconnectivity/webconnectivity_test.go +++ b/internal/engine/experiment/webconnectivity/webconnectivity_test.go @@ -202,7 +202,6 @@ func TestMeasureWithNoAvailableTestHelpers(t *testing.T) { func newsession(t *testing.T, lookupBackends bool) model.ExperimentSession { sess, err := engine.NewSession(engine.SessionConfig{ - AssetsDir: "../../testdata", AvailableProbeServices: []model.Service{{ Address: "https://ams-pg-test.ooni.org", Type: "https", diff --git a/internal/engine/geolocate/geolocate.go b/internal/engine/geolocate/geolocate.go index 15378cb..ec75df6 100644 --- a/internal/engine/geolocate/geolocate.go +++ b/internal/engine/geolocate/geolocate.go @@ -3,7 +3,6 @@ package geolocate import ( "context" - "errors" "fmt" "github.com/ooni/probe-cli/v3/internal/engine/model" @@ -43,12 +42,6 @@ var ( DefaultResolverASNString = fmt.Sprintf("AS%d", DefaultResolverASN) ) -var ( - // ErrMissingResourcesManager indicates that no resources - // manager has been configured inside of Config. - ErrMissingResourcesManager = errors.New("geolocate: ResourcesManager is nil") -) - // Logger is the definition of Logger used by this package. type Logger interface { Debug(msg string) @@ -96,30 +89,17 @@ type probeIPLookupper interface { } type asnLookupper interface { - LookupASN(path string, ip string) (asn uint, network string, err error) + LookupASN(ip string) (asn uint, network string, err error) } type countryLookupper interface { - LookupCC(path string, ip string) (cc string, err error) + LookupCC(ip string) (cc string, err error) } type resolverIPLookupper interface { LookupResolverIP(ctx context.Context) (addr string, err error) } -// ResourcesManager manages the required resources. -type ResourcesManager interface { - // ASNDatabasePath returns the path of the ASN database. - ASNDatabasePath() string - - // CountryDatabasePath returns the path of the country database. - CountryDatabasePath() string - - // MaybeUpdateResources ensures that the required resources - // have been downloaded and are current. - MaybeUpdateResources(ctx context.Context) error -} - // Resolver is a DNS resolver. type Resolver interface { LookupHost(ctx context.Context, domain string) ([]string, error) @@ -142,10 +122,6 @@ type Config struct { // use a logger that discards all messages. Logger Logger - // ResourcesManager is the mandatory resources manager. If not - // set, we will not be able to perform any lookup. - ResourcesManager ResourcesManager - // UserAgent is the user agent to use. If not set, then // we will use a default user agent. UserAgent string @@ -162,9 +138,6 @@ func NewTask(config Config) (*Task, error) { if config.Logger == nil { config.Logger = model.DiscardLogger } - if config.ResourcesManager == nil { - return nil, ErrMissingResourcesManager - } if config.UserAgent == "" { config.UserAgent = fmt.Sprintf("ooniprobe-engine/%s", version.Version) } @@ -183,7 +156,6 @@ func NewTask(config Config) (*Task, error) { probeASNLookupper: mmdbLookupper{}, resolverASNLookupper: mmdbLookupper{}, resolverIPLookupper: resolverLookupClient{}, - resourcesManager: config.ResourcesManager, }, nil } @@ -196,7 +168,6 @@ type Task struct { probeASNLookupper asnLookupper resolverASNLookupper asnLookupper resolverIPLookupper resolverIPLookupper - resourcesManager ResourcesManager } // Run runs the task. @@ -211,23 +182,18 @@ func (op Task) Run(ctx context.Context) (*Results, error) { ResolverIP: DefaultResolverIP, ResolverNetworkName: DefaultResolverNetworkName, } - if err := op.resourcesManager.MaybeUpdateResources(ctx); err != nil { - return out, fmt.Errorf("MaybeUpdateResource failed: %w", err) - } ip, err := op.probeIPLookupper.LookupProbeIP(ctx) if err != nil { return out, fmt.Errorf("lookupProbeIP failed: %w", err) } out.ProbeIP = ip - asn, networkName, err := op.probeASNLookupper.LookupASN( - op.resourcesManager.ASNDatabasePath(), out.ProbeIP) + asn, networkName, err := op.probeASNLookupper.LookupASN(out.ProbeIP) if err != nil { return out, fmt.Errorf("lookupASN failed: %w", err) } out.ASN = asn out.NetworkName = networkName - cc, err := op.countryLookupper.LookupCC( - op.resourcesManager.CountryDatabasePath(), out.ProbeIP) + cc, err := op.countryLookupper.LookupCC(out.ProbeIP) if err != nil { return out, fmt.Errorf("lookupProbeCC failed: %w", err) } @@ -244,7 +210,7 @@ func (op Task) Run(ctx context.Context) (*Results, error) { } out.ResolverIP = resolverIP resolverASN, resolverNetworkName, err := op.resolverASNLookupper.LookupASN( - op.resourcesManager.ASNDatabasePath(), out.ResolverIP, + out.ResolverIP, ) if err != nil { return out, nil diff --git a/internal/engine/geolocate/geolocate_test.go b/internal/engine/geolocate/geolocate_test.go index cabf2e2..b7d8fbf 100644 --- a/internal/engine/geolocate/geolocate_test.go +++ b/internal/engine/geolocate/geolocate_test.go @@ -6,57 +6,6 @@ import ( "testing" ) -type taskResourcesManager struct { - asnDatabasePath string - countryDatabasePath string - err error -} - -func (c taskResourcesManager) ASNDatabasePath() string { - return c.asnDatabasePath -} - -func (c taskResourcesManager) CountryDatabasePath() string { - return c.countryDatabasePath -} - -func (c taskResourcesManager) MaybeUpdateResources(ctx context.Context) error { - return c.err -} - -func TestLocationLookupCannotUpdateResources(t *testing.T) { - expected := errors.New("mocked error") - op := Task{ - resourcesManager: taskResourcesManager{err: expected}, - } - ctx := context.Background() - out, err := op.Run(ctx) - if !errors.Is(err, expected) { - t.Fatalf("not the error we expected: %+v", err) - } - if out.ASN != DefaultProbeASN { - t.Fatal("invalid ASN value") - } - if out.CountryCode != DefaultProbeCC { - t.Fatal("invalid CountryCode value") - } - if out.NetworkName != DefaultProbeNetworkName { - t.Fatal("invalid NetworkName value") - } - if out.ProbeIP != DefaultProbeIP { - t.Fatal("invalid ProbeIP value") - } - if out.ResolverASN != DefaultResolverASN { - t.Fatal("invalid ResolverASN value") - } - if out.ResolverIP != DefaultResolverIP { - t.Fatal("invalid ResolverIP value") - } - if out.ResolverNetworkName != DefaultResolverNetworkName { - t.Fatal("invalid ResolverNetworkName value") - } -} - type taskProbeIPLookupper struct { ip string err error @@ -69,7 +18,6 @@ func (c taskProbeIPLookupper) LookupProbeIP(ctx context.Context) (string, error) func TestLocationLookupCannotLookupProbeIP(t *testing.T) { expected := errors.New("mocked error") op := Task{ - resourcesManager: taskResourcesManager{}, probeIPLookupper: taskProbeIPLookupper{err: expected}, } ctx := context.Background() @@ -106,14 +54,13 @@ type taskASNLookupper struct { name string } -func (c taskASNLookupper) LookupASN(path string, ip string) (uint, string, error) { +func (c taskASNLookupper) LookupASN(ip string) (uint, string, error) { return c.asn, c.name, c.err } func TestLocationLookupCannotLookupProbeASN(t *testing.T) { expected := errors.New("mocked error") op := Task{ - resourcesManager: taskResourcesManager{}, probeIPLookupper: taskProbeIPLookupper{ip: "1.2.3.4"}, probeASNLookupper: taskASNLookupper{err: expected}, } @@ -150,14 +97,13 @@ type taskCCLookupper struct { cc string } -func (c taskCCLookupper) LookupCC(path string, ip string) (string, error) { +func (c taskCCLookupper) LookupCC(ip string) (string, error) { return c.cc, c.err } func TestLocationLookupCannotLookupProbeCC(t *testing.T) { expected := errors.New("mocked error") op := Task{ - resourcesManager: taskResourcesManager{}, probeIPLookupper: taskProbeIPLookupper{ip: "1.2.3.4"}, probeASNLookupper: taskASNLookupper{asn: 1234, name: "1234.com"}, countryLookupper: taskCCLookupper{cc: "US", err: expected}, @@ -202,7 +148,6 @@ func (c taskResolverIPLookupper) LookupResolverIP(ctx context.Context) (string, func TestLocationLookupCannotLookupResolverIP(t *testing.T) { expected := errors.New("mocked error") op := Task{ - resourcesManager: taskResourcesManager{}, probeIPLookupper: taskProbeIPLookupper{ip: "1.2.3.4"}, probeASNLookupper: taskASNLookupper{asn: 1234, name: "1234.com"}, countryLookupper: taskCCLookupper{cc: "IT"}, @@ -243,7 +188,6 @@ func TestLocationLookupCannotLookupResolverIP(t *testing.T) { func TestLocationLookupCannotLookupResolverNetworkName(t *testing.T) { expected := errors.New("mocked error") op := Task{ - resourcesManager: taskResourcesManager{}, probeIPLookupper: taskProbeIPLookupper{ip: "1.2.3.4"}, probeASNLookupper: taskASNLookupper{asn: 1234, name: "1234.com"}, countryLookupper: taskCCLookupper{cc: "IT"}, @@ -284,7 +228,6 @@ func TestLocationLookupCannotLookupResolverNetworkName(t *testing.T) { func TestLocationLookupSuccessWithResolverLookup(t *testing.T) { op := Task{ - resourcesManager: taskResourcesManager{}, probeIPLookupper: taskProbeIPLookupper{ip: "1.2.3.4"}, probeASNLookupper: taskASNLookupper{asn: 1234, name: "1234.com"}, countryLookupper: taskCCLookupper{cc: "IT"}, @@ -325,7 +268,6 @@ func TestLocationLookupSuccessWithResolverLookup(t *testing.T) { func TestLocationLookupSuccessWithoutResolverLookup(t *testing.T) { op := Task{ - resourcesManager: taskResourcesManager{}, probeIPLookupper: taskProbeIPLookupper{ip: "1.2.3.4"}, probeASNLookupper: taskASNLookupper{asn: 1234, name: "1234.com"}, countryLookupper: taskCCLookupper{cc: "IT"}, @@ -364,13 +306,8 @@ func TestLocationLookupSuccessWithoutResolverLookup(t *testing.T) { } func TestSmoke(t *testing.T) { - maybeFetchResources(t) config := Config{ EnableResolverLookup: true, - ResourcesManager: taskResourcesManager{ - asnDatabasePath: asnDBPath, - countryDatabasePath: countryDBPath, - }, } task := Must(NewTask(config)) result, err := task.Run(context.Background()) @@ -384,16 +321,6 @@ func TestSmoke(t *testing.T) { // value is okay for all codepaths. } -func TestNewTaskWithNoResourcesManager(t *testing.T) { - task, err := NewTask(Config{}) - if !errors.Is(err, ErrMissingResourcesManager) { - t.Fatal("not the error we expected") - } - if task != nil { - t.Fatal("expected nil task here") - } -} - func TestASNStringWorks(t *testing.T) { r := Results{ASN: 1234} if r.ASNString() != "AS1234" { diff --git a/internal/engine/geolocate/mmdblookup.go b/internal/engine/geolocate/mmdblookup.go index dc33158..bb02acf 100644 --- a/internal/engine/geolocate/mmdblookup.go +++ b/internal/engine/geolocate/mmdblookup.go @@ -3,14 +3,15 @@ package geolocate import ( "net" + "github.com/ooni/probe-assets/assets" "github.com/oschwald/geoip2-golang" ) type mmdbLookupper struct{} -func (mmdbLookupper) LookupASN(path, ip string) (asn uint, org string, err error) { +func (mmdbLookupper) LookupASN(ip string) (asn uint, org string, err error) { asn, org = DefaultProbeASN, DefaultProbeNetworkName - db, err := geoip2.Open(path) + db, err := geoip2.FromBytes(assets.ASNDatabaseData()) if err != nil { return } @@ -28,13 +29,13 @@ func (mmdbLookupper) LookupASN(path, ip string) (asn uint, org string, err error // LookupASN returns the ASN and the organization associated with the // given ip using the ASN database at path. -func LookupASN(path, ip string) (asn uint, org string, err error) { - return (mmdbLookupper{}).LookupASN(path, ip) +func LookupASN(ip string) (asn uint, org string, err error) { + return (mmdbLookupper{}).LookupASN(ip) } -func (mmdbLookupper) LookupCC(path, ip string) (cc string, err error) { +func (mmdbLookupper) LookupCC(ip string) (cc string, err error) { cc = DefaultProbeCC - db, err := geoip2.Open(path) + db, err := geoip2.FromBytes(assets.CountryDatabaseData()) if err != nil { return } diff --git a/internal/engine/geolocate/mmdblookup_test.go b/internal/engine/geolocate/mmdblookup_test.go index e66fbe8..7b9c055 100644 --- a/internal/engine/geolocate/mmdblookup_test.go +++ b/internal/engine/geolocate/mmdblookup_test.go @@ -1,27 +1,11 @@ package geolocate -import ( - "testing" +import "testing" - "github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager" -) - -const ( - asnDBPath = "../testdata/asn.mmdb" - countryDBPath = "../testdata/country.mmdb" - ipAddr = "35.204.49.125" -) - -func maybeFetchResources(t *testing.T) { - c := &resourcesmanager.CopyWorker{DestDir: "../testdata/"} - if err := c.Ensure(); err != nil { - t.Fatal(err) - } -} +const ipAddr = "35.204.49.125" func TestLookupASN(t *testing.T) { - maybeFetchResources(t) - asn, org, err := LookupASN(asnDBPath, ipAddr) + asn, org, err := LookupASN(ipAddr) if err != nil { t.Fatal(err) } @@ -33,23 +17,8 @@ func TestLookupASN(t *testing.T) { } } -func TestLookupASNInvalidFile(t *testing.T) { - maybeFetchResources(t) - asn, org, err := LookupASN("/nonexistent", ipAddr) - if err == nil { - t.Fatal("expected an error here") - } - if asn != DefaultProbeASN { - t.Fatal("expected a zero ASN") - } - if org != DefaultProbeNetworkName { - t.Fatal("expected an empty org") - } -} - func TestLookupASNInvalidIP(t *testing.T) { - maybeFetchResources(t) - asn, org, err := LookupASN(asnDBPath, "xxx") + asn, org, err := LookupASN("xxx") if err == nil { t.Fatal("expected an error here") } @@ -62,8 +31,7 @@ func TestLookupASNInvalidIP(t *testing.T) { } func TestLookupCC(t *testing.T) { - maybeFetchResources(t) - cc, err := (mmdbLookupper{}).LookupCC(countryDBPath, ipAddr) + cc, err := (mmdbLookupper{}).LookupCC(ipAddr) if err != nil { t.Fatal(err) } @@ -72,20 +40,8 @@ func TestLookupCC(t *testing.T) { } } -func TestLookupCCInvalidFile(t *testing.T) { - maybeFetchResources(t) - cc, err := (mmdbLookupper{}).LookupCC("/nonexistent", ipAddr) - if err == nil { - t.Fatal("expected an error here") - } - if cc != DefaultProbeCC { - t.Fatal("expected an empty cc") - } -} - func TestLookupCCInvalidIP(t *testing.T) { - maybeFetchResources(t) - cc, err := (mmdbLookupper{}).LookupCC(asnDBPath, "xxx") + cc, err := (mmdbLookupper{}).LookupCC("xxx") if err == nil { t.Fatal("expected an error here") } diff --git a/internal/engine/inputloader_network_test.go b/internal/engine/inputloader_network_test.go index 3d10957..97c5e24 100644 --- a/internal/engine/inputloader_network_test.go +++ b/internal/engine/inputloader_network_test.go @@ -19,7 +19,6 @@ func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) { Address: "https://ams-pg-test.ooni.org/", Type: "https", }}, - AssetsDir: "testdata", KVStore: kvstore.NewMemoryKeyValueStore(), Logger: log.Log, SoftwareName: "miniooni", diff --git a/internal/engine/inputloader_test.go b/internal/engine/inputloader_test.go index 7287c46..764796c 100644 --- a/internal/engine/inputloader_test.go +++ b/internal/engine/inputloader_test.go @@ -237,7 +237,6 @@ func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) { func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing.T) { sess, err := NewSession(SessionConfig{ - AssetsDir: "testdata", KVStore: kvstore.NewMemoryKeyValueStore(), Logger: log.Log, SoftwareName: "miniooni", diff --git a/internal/engine/internal/mockable/mockable.go b/internal/engine/internal/mockable/mockable.go index 032e24d..7c606b2 100644 --- a/internal/engine/internal/mockable/mockable.go +++ b/internal/engine/internal/mockable/mockable.go @@ -18,7 +18,6 @@ import ( // Session allows to mock sessions. type Session struct { - MockableASNDatabasePath string MockableTestHelpers map[string][]model.Service MockableHTTPClient *http.Client MockableLogger model.Logger @@ -39,11 +38,6 @@ type Session struct { MockableUserAgent string } -// ASNDatabasePath implements ExperimentSession.ASNDatabasePath -func (sess *Session) ASNDatabasePath() string { - return sess.MockableASNDatabasePath -} - // GetTestHelpersByName implements ExperimentSession.GetTestHelpersByName func (sess *Session) GetTestHelpersByName(name string) ([]model.Service, bool) { services, okay := sess.MockableTestHelpers[name] diff --git a/internal/engine/internal/psiphonx/psiphonx_test.go b/internal/engine/internal/psiphonx/psiphonx_test.go index b5fc1be..427820b 100644 --- a/internal/engine/internal/psiphonx/psiphonx_test.go +++ b/internal/engine/internal/psiphonx/psiphonx_test.go @@ -17,7 +17,6 @@ func TestStartWithCancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() sess, err := engine.NewSession(engine.SessionConfig{ - AssetsDir: "../../testdata", Logger: log.Log, SoftwareName: "ooniprobe-engine", SoftwareVersion: "0.0.1", @@ -39,7 +38,6 @@ func TestStartStop(t *testing.T) { t.Skip("skip test in short mode") } sess, err := engine.NewSession(engine.SessionConfig{ - AssetsDir: "../../testdata", Logger: log.Log, SoftwareName: "ooniprobe-engine", SoftwareVersion: "0.0.1", diff --git a/internal/engine/legacy/assetsdir/assetsdir.go b/internal/engine/legacy/assetsdir/assetsdir.go new file mode 100644 index 0000000..9a517a6 --- /dev/null +++ b/internal/engine/legacy/assetsdir/assetsdir.go @@ -0,0 +1,62 @@ +// Package assetsdir contains code to cleanup the assets dir. We removed +// the assetsdir in the 3.9.0 development cycle. +package assetsdir + +import ( + "errors" + "os" + "path/filepath" +) + +// ErrEmptyDir indicates that you passed to Cleanup an empty dir. +var ErrEmptyDir = errors.New("empty assets directory") + +// Result is the result of a Cleanup run. +type Result struct { + // ASNDatabaseErr is the error of deleting the + // file containing the old ASN database. + ASNDatabaseErr error + + // CABundleErr is the error of deleting the file + // containing the old CA bundle. + CABundleErr error + + // CountryDatabaseErr is the error of deleting the + // file containing the old country database. + CountryDatabaseErr error + + // RmdirErr is the error of deleting the supposedly + // empty directory that contained assets. + RmdirErr error +} + +// Cleanup removes data from the assetsdir. This function will +// try to delete the known assets inside of dir. It then also +// tries to delete the directory. If the directory is not empty, +// this operation will fail. That means the user has put some +// extra data in there and we don't want to remove it. +// +// Returns the Result of cleaning up the assets on success and +// an error on failure. The only cause of error is passing to +// this function an empty directory. The Result data structure +// contains the result of each individual remove operation. +func Cleanup(dir string) (*Result, error) { + return fcleanup(dir, os.Remove) +} + +// fcleanup is a version of Cleanup where we can mock the real function +// used for removing files and dirs, so we can write unit tests. +func fcleanup(dir string, remove func(name string) error) (*Result, error) { + if dir == "" { + return nil, ErrEmptyDir + } + r := &Result{} + asndb := filepath.Join(dir, "asn.mmdb") + r.ASNDatabaseErr = os.Remove(asndb) + cabundle := filepath.Join(dir, "ca-bundle.pem") + r.CABundleErr = os.Remove(cabundle) + countrydb := filepath.Join(dir, "country.mmdb") + r.CountryDatabaseErr = os.Remove(countrydb) + r.RmdirErr = os.Remove(dir) + return r, nil +} diff --git a/internal/engine/legacy/assetsdir/assetsdir_test.go b/internal/engine/legacy/assetsdir/assetsdir_test.go new file mode 100644 index 0000000..3fb8d0a --- /dev/null +++ b/internal/engine/legacy/assetsdir/assetsdir_test.go @@ -0,0 +1,40 @@ +package assetsdir + +import ( + "errors" + "strings" + "testing" +) + +func TestCleanupNormalUsage(t *testing.T) { + result, err := Cleanup("testdata") + if err != nil { + t.Fatal(err) + } + // we expect a bunch of ENOENT because the directory does not exist. + isExpectedErr := func(err error) bool { + return err != nil && strings.HasSuffix(err.Error(), "no such file or directory") + } + if !isExpectedErr(result.ASNDatabaseErr) { + t.Fatal("unexpected error", result.ASNDatabaseErr) + } + if !isExpectedErr(result.CABundleErr) { + t.Fatal("unexpected error", result.CABundleErr) + } + if !isExpectedErr(result.CountryDatabaseErr) { + t.Fatal("unexpected error", result.CountryDatabaseErr) + } + if !isExpectedErr(result.RmdirErr) { + t.Fatal("unexpected error", result.RmdirErr) + } +} + +func TestCleanupWithEmptyInput(t *testing.T) { + result, err := Cleanup("") + if !errors.Is(err, ErrEmptyDir) { + t.Fatal("unexpected error", err) + } + if result != nil { + t.Fatal("expected nil result") + } +} diff --git a/internal/engine/model/experiment.go b/internal/engine/model/experiment.go index 18e5f32..25e39e1 100644 --- a/internal/engine/model/experiment.go +++ b/internal/engine/model/experiment.go @@ -24,7 +24,6 @@ type ExperimentOrchestraClient interface { // ExperimentSession is the experiment's view of a session. type ExperimentSession interface { - ASNDatabasePath() string GetTestHelpersByName(name string) ([]Service, bool) DefaultHTTPClient() *http.Client Logger() Logger diff --git a/internal/engine/netx/archival/archival.go b/internal/engine/netx/archival/archival.go index 67ef5f8..11fd776 100644 --- a/internal/engine/netx/archival/archival.go +++ b/internal/engine/netx/archival/archival.go @@ -397,7 +397,7 @@ type DNSQueryEntry struct { type dnsQueryType string // NewDNSQueriesList returns a list of DNS queries. -func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []DNSQueryEntry { +func NewDNSQueriesList(begin time.Time, events []trace.Event) []DNSQueryEntry { // TODO(bassosimone): add support for CNAME lookups. var out []DNSQueryEntry for _, ev := range events { @@ -409,7 +409,7 @@ func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []D for _, addr := range ev.Addresses { if qtype.ipoftype(addr) { entry.Answers = append( - entry.Answers, qtype.makeanswerentry(addr, dbpath)) + entry.Answers, qtype.makeanswerentry(addr)) } } if len(entry.Answers) <= 0 && ev.Err == nil { @@ -431,16 +431,16 @@ func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []D func (qtype dnsQueryType) ipoftype(addr string) bool { switch qtype { case "A": - return strings.Contains(addr, ":") == false + return !strings.Contains(addr, ":") case "AAAA": - return strings.Contains(addr, ":") == true + return strings.Contains(addr, ":") } return false } -func (qtype dnsQueryType) makeanswerentry(addr string, dbpath string) DNSAnswerEntry { +func (qtype dnsQueryType) makeanswerentry(addr string) DNSAnswerEntry { answer := DNSAnswerEntry{AnswerType: string(qtype)} - asn, org, _ := geolocate.LookupASN(dbpath, addr) + asn, org, _ := geolocate.LookupASN(addr) answer.ASN = int64(asn) answer.ASOrgName = org switch qtype { diff --git a/internal/engine/netx/archival/archival_internal_test.go b/internal/engine/netx/archival/archival_internal_test.go new file mode 100644 index 0000000..e927f9f --- /dev/null +++ b/internal/engine/netx/archival/archival_internal_test.go @@ -0,0 +1,41 @@ +package archival + +import "testing" + +func TestDNSQueryIPOfType(t *testing.T) { + type expectation struct { + qtype dnsQueryType + ip string + output bool + } + var expectations = []expectation{{ + qtype: "A", + ip: "8.8.8.8", + output: true, + }, { + qtype: "A", + ip: "2a00:1450:4002:801::2004", + output: false, + }, { + qtype: "AAAA", + ip: "8.8.8.8", + output: false, + }, { + qtype: "AAAA", + ip: "2a00:1450:4002:801::2004", + output: true, + }, { + qtype: "ANTANI", + ip: "2a00:1450:4002:801::2004", + output: false, + }, { + qtype: "ANTANI", + ip: "8.8.8.8", + output: false, + }} + for _, exp := range expectations { + if exp.qtype.ipoftype(exp.ip) != exp.output { + t.Fatalf("failure for %+v", exp) + } + } +} diff --git a/internal/engine/netx/archival/archival_test.go b/internal/engine/netx/archival/archival_test.go index df38a3e..5172ed6 100644 --- a/internal/engine/netx/archival/archival_test.go +++ b/internal/engine/netx/archival/archival_test.go @@ -16,7 +16,6 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" - "github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager" ) func TestNewTCPConnectList(t *testing.T) { @@ -285,15 +284,10 @@ func TestNewRequestList(t *testing.T) { } func TestNewDNSQueriesList(t *testing.T) { - err := (&resourcesmanager.CopyWorker{DestDir: "../../testdata"}).Ensure() - if err != nil { - t.Fatal(err) - } begin := time.Now() type args struct { begin time.Time events []trace.Event - dbpath string } tests := []struct { name string @@ -334,9 +328,13 @@ func TestNewDNSQueriesList(t *testing.T) { }, want: []archival.DNSQueryEntry{{ Answers: []archival.DNSAnswerEntry{{ + ASN: 15169, + ASOrgName: "Google LLC", AnswerType: "A", IPv4: "8.8.8.8", }, { + ASN: 15169, + ASOrgName: "Google LLC", AnswerType: "A", IPv4: "8.8.4.4", }}, @@ -357,27 +355,6 @@ func TestNewDNSQueriesList(t *testing.T) { Time: begin.Add(200 * time.Millisecond), }}, }, - want: []archival.DNSQueryEntry{{ - Answers: []archival.DNSAnswerEntry{{ - AnswerType: "AAAA", - IPv6: "2001:4860:4860::8888", - }}, - Hostname: "dns.google.com", - QueryType: "AAAA", - T: 0.2, - }}, - }, { - name: "run with ASN DB", - args: args{ - begin: begin, - events: []trace.Event{{ - Addresses: []string{"2001:4860:4860::8888"}, - Hostname: "dns.google.com", - Name: "resolve_done", - Time: begin.Add(200 * time.Millisecond), - }}, - dbpath: "../../testdata/asn.mmdb", - }, want: []archival.DNSQueryEntry{{ Answers: []archival.DNSAnswerEntry{{ ASN: 15169, @@ -399,7 +376,6 @@ func TestNewDNSQueriesList(t *testing.T) { Name: "resolve_done", Time: begin.Add(200 * time.Millisecond), }}, - dbpath: "../../testdata/asn.mmdb", }, want: []archival.DNSQueryEntry{{ Answers: nil, @@ -419,9 +395,9 @@ func TestNewDNSQueriesList(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := archival.NewDNSQueriesList( - tt.args.begin, tt.args.events, tt.args.dbpath); !reflect.DeepEqual(got, tt.want) { - t.Error(cmp.Diff(got, tt.want)) + got := archival.NewDNSQueriesList(tt.args.begin, tt.args.events) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) } }) } @@ -1009,13 +985,6 @@ func TestNewFailure(t *testing.T) { } } -func TestDNSQueryTypeInvalidIPOfType(t *testing.T) { - qtype := archival.DNSQueryType("ANTANI") - if qtype.IPOfType("8.8.8.8") != false { - t.Fatal("unexpected return value") - } -} - func TestNewFailedOperation(t *testing.T) { type args struct { err error diff --git a/internal/engine/netx/archival/archival_test_internal.go b/internal/engine/netx/archival/archival_test_internal.go deleted file mode 100644 index 976ba3f..0000000 --- a/internal/engine/netx/archival/archival_test_internal.go +++ /dev/null @@ -1,8 +0,0 @@ -package archival - -// DNSQueryType allows to access dnsQueryType from unit tests -type DNSQueryType = dnsQueryType - -func (qtype dnsQueryType) IPOfType(addr string) bool { - return qtype.ipoftype(addr) -} diff --git a/internal/engine/resources/assets.go b/internal/engine/resources/assets.go deleted file mode 100644 index 6e6d6ed..0000000 --- a/internal/engine/resources/assets.go +++ /dev/null @@ -1,42 +0,0 @@ -package resources - -const ( - // Version contains the assets version. - Version = 20210303114512 - - // ASNDatabaseName is the ASN-DB file name - ASNDatabaseName = "asn.mmdb" - - // CountryDatabaseName is country-DB file name - CountryDatabaseName = "country.mmdb" - - // BaseURL is the asset's repository base URL - BaseURL = "https://github.com/" -) - -// ResourceInfo contains information on a resource. -type ResourceInfo struct { - // URLPath is the resource's URL path. - URLPath string - - // GzSHA256 is used to validate the downloaded file. - GzSHA256 string - - // SHA256 is used to check whether the assets file - // stored locally is still up-to-date. - SHA256 string -} - -// All contains info on all known assets. -var All = map[string]ResourceInfo{ - "asn.mmdb": { - URLPath: "/ooni/probe-assets/releases/download/20210303114512/asn.mmdb.gz", - GzSHA256: "efafd5a165c5a4e6bf6258d87ed685254a2660669eb4557e25c5ed72e48d039a", - SHA256: "675dbaec3fa1e6f12957c4e4ddee03f50f5192507b5095ccb9ed057468c2441b", - }, - "country.mmdb": { - URLPath: "/ooni/probe-assets/releases/download/20210303114512/country.mmdb.gz", - GzSHA256: "7f1db0e2903271258319834f26bbcdedd2d0641457a8c0a63b048a985b7d6e7b", - SHA256: "19e4d2c5cd31789da1a67baf883995f2ea03c4b8ba7342b69ef8ae2c2aa8409c", - }, -} diff --git a/internal/engine/resources/doc.go b/internal/engine/resources/doc.go deleted file mode 100644 index a694698..0000000 --- a/internal/engine/resources/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package resources contains info on resources. See also -// the resourcesmanager package. -package resources diff --git a/internal/engine/resourcesmanager/.gitignore b/internal/engine/resourcesmanager/.gitignore deleted file mode 100644 index de9444a..0000000 --- a/internal/engine/resourcesmanager/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/asn.mmdb.gz -/country.mmdb.gz -/testdata diff --git a/internal/engine/resourcesmanager/resourcesmanager.go b/internal/engine/resourcesmanager/resourcesmanager.go deleted file mode 100644 index d7a891e..0000000 --- a/internal/engine/resourcesmanager/resourcesmanager.go +++ /dev/null @@ -1,157 +0,0 @@ -// Package resourcesmanager contains the resources manager. -package resourcesmanager - -import ( - "compress/gzip" - "crypto/sha256" - "embed" - "errors" - "fmt" - "io" - "io/fs" - "io/ioutil" - "os" - "path/filepath" - - "github.com/ooni/probe-cli/v3/internal/engine/resources" -) - -// Errors returned by this package. -var ( - ErrDestDirEmpty = errors.New("resources: DestDir is empty") - ErrSHA256Mismatch = errors.New("resources: sha256 mismatch") -) - -// CopyWorker ensures that resources are current. You always need to set -// the DestDir attribute. All the rest is optional. -type CopyWorker struct { - DestDir string // mandatory - Different func(left, right string) bool // optional - Equal func(left, right string) bool // optional - MkdirAll func(path string, perm os.FileMode) error // optional - NewReader func(r io.Reader) (io.ReadCloser, error) // optional - Open func(path string) (fs.File, error) // optional - ReadAll func(r io.Reader) ([]byte, error) // optional - ReadFile func(filename string) ([]byte, error) // optional - WriteFile func(filename string, data []byte, perm fs.FileMode) error // optional -} - -// If you arrive here because of this error: -// -// internal/engine/resourcesmanager/resourcesmanager.go:39:12: pattern *.mmdb.gz: no matching files found -// internal/engine/resourcesmanager/resourcesmanager.go:39:12: pattern *.mmdb.gz: no matching files found -// -// then your problem is that you need to fetch resources _before_ compiling -// ooniprobe. See Readme.md for instructions on how to do that. - -//go:embed *.mmdb.gz -var efs embed.FS - -func (cw *CopyWorker) mkdirAll(path string, perm os.FileMode) error { - if cw.MkdirAll != nil { - return cw.MkdirAll(path, perm) - } - return os.MkdirAll(path, perm) -} - -// Ensure ensures that the resources on disk are current. -func (cw *CopyWorker) Ensure() error { - if cw.DestDir == "" { - return ErrDestDirEmpty - } - if err := cw.mkdirAll(cw.DestDir, 0700); err != nil { - return err - } - for name, resource := range resources.All { - if err := cw.ensureFor(name, &resource); err != nil { - return err - } - } - return nil -} - -func (cw *CopyWorker) readFile(path string) ([]byte, error) { - if cw.ReadFile != nil { - return cw.ReadFile(path) - } - return ioutil.ReadFile(path) -} - -func (cw *CopyWorker) equal(left, right string) bool { - if cw.Equal != nil { - return cw.Equal(left, right) - } - return left == right -} - -func (cw *CopyWorker) different(left, right string) bool { - if cw.Different != nil { - return cw.Different(left, right) - } - return left != right -} - -func (cw *CopyWorker) open(path string) (fs.File, error) { - if cw.Open != nil { - return cw.Open(path) - } - return efs.Open(path) -} - -func (cw *CopyWorker) newReader(r io.Reader) (io.ReadCloser, error) { - if cw.NewReader != nil { - return cw.NewReader(r) - } - return gzip.NewReader(r) -} - -func (cw *CopyWorker) readAll(r io.Reader) ([]byte, error) { - if cw.ReadAll != nil { - return cw.ReadAll(r) - } - return ioutil.ReadAll(r) -} - -func (cw *CopyWorker) writeFile(filename string, data []byte, perm fs.FileMode) error { - if cw.WriteFile != nil { - return cw.WriteFile(filename, data, perm) - } - return ioutil.WriteFile(filename, data, perm) -} - -func (cw *CopyWorker) sha256sum(data []byte) string { - return fmt.Sprintf("%x", sha256.Sum256(data)) -} - -func (cw *CopyWorker) allGood(rpath string, resource *resources.ResourceInfo) bool { - data, err := cw.readFile(rpath) - if err != nil { - return false - } - return cw.equal(cw.sha256sum(data), resource.SHA256) -} - -func (cw *CopyWorker) ensureFor(name string, resource *resources.ResourceInfo) error { - rpath := filepath.Join(cw.DestDir, name) - if cw.allGood(rpath, resource) { - return nil - } - filep, err := cw.open(name + ".gz") - if err != nil { - return err - } - defer filep.Close() - gzfilep, err := cw.newReader(filep) - if err != nil { - return err - } - defer gzfilep.Close() - data, err := cw.readAll(gzfilep) - if err != nil { - return err - } - if cw.different(cw.sha256sum(data), resource.SHA256) { - return ErrSHA256Mismatch - } - return cw.writeFile(rpath, data, 0600) -} diff --git a/internal/engine/resourcesmanager/resourcesmanager_test.go b/internal/engine/resourcesmanager/resourcesmanager_test.go deleted file mode 100644 index 5101e5f..0000000 --- a/internal/engine/resourcesmanager/resourcesmanager_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package resourcesmanager - -import ( - "errors" - "io" - "io/fs" - "os" - "testing" -) - -func TestAllGood(t *testing.T) { - // make sure we start from scratch - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - // first iteration should copy the resources - cw := &CopyWorker{DestDir: "testdata"} - if err := cw.Ensure(); err != nil { - t.Fatal(err) - } - // second iteration should just ensure they're there - if err := cw.Ensure(); err != nil { - t.Fatal(err) - } -} - -func TestEmptyDestDir(t *testing.T) { - cw := &CopyWorker{DestDir: ""} - if err := cw.Ensure(); !errors.Is(err, ErrDestDirEmpty) { - t.Fatal("not the error we expected", err) - } -} - -func TestMkdirAllFailure(t *testing.T) { - errMocked := errors.New("mocked error") - cw := &CopyWorker{ - DestDir: "testdata", - MkdirAll: func(path string, perm os.FileMode) error { - return errMocked - }, - } - if err := cw.Ensure(); !errors.Is(err, errMocked) { - t.Fatal("not the error we expected", err) - } -} - -func TestOpenFailure(t *testing.T) { - errMocked := errors.New("mocked error") - cw := &CopyWorker{ - DestDir: "testdata", - MkdirAll: func(path string, perm os.FileMode) error { - return nil - }, - ReadFile: func(path string) ([]byte, error) { - return []byte(`fake`), nil - }, - Equal: func(left, right string) bool { - return false - }, - Open: func(path string) (fs.File, error) { - return nil, errMocked - }, - } - if err := cw.Ensure(); !errors.Is(err, errMocked) { - t.Fatal("not the error we expected", err) - } -} - -func TestNewReaderFailure(t *testing.T) { - errMocked := errors.New("mocked error") - cw := &CopyWorker{ - DestDir: "testdata", - MkdirAll: func(path string, perm os.FileMode) error { - return nil - }, - Equal: func(left, right string) bool { - return false - }, - NewReader: func(r io.Reader) (io.ReadCloser, error) { - return nil, errMocked - }, - } - if err := cw.Ensure(); !errors.Is(err, errMocked) { - t.Fatal("not the error we expected", err) - } -} - -func TestReadAllFailure(t *testing.T) { - errMocked := errors.New("mocked error") - cw := &CopyWorker{ - DestDir: "testdata", - MkdirAll: func(path string, perm os.FileMode) error { - return nil - }, - Equal: func(left, right string) bool { - return false - }, - ReadAll: func(r io.Reader) ([]byte, error) { - return nil, errMocked - }, - } - if err := cw.Ensure(); !errors.Is(err, errMocked) { - t.Fatal("not the error we expected", err) - } -} - -func TestSHA256Mismatch(t *testing.T) { - cw := &CopyWorker{ - DestDir: "testdata", - MkdirAll: func(path string, perm os.FileMode) error { - return nil - }, - Equal: func(left, right string) bool { - return false - }, - Different: func(left, right string) bool { - return true - }, - } - if err := cw.Ensure(); !errors.Is(err, ErrSHA256Mismatch) { - t.Fatal("not the error we expected", err) - } -} - -func TestWriteFileFailure(t *testing.T) { - errMocked := errors.New("mocked error") - cw := &CopyWorker{ - DestDir: "testdata", - MkdirAll: func(path string, perm os.FileMode) error { - return nil - }, - Equal: func(left, right string) bool { - return false - }, - WriteFile: func(filename string, data []byte, perm fs.FileMode) error { - return errMocked - }, - } - if err := cw.Ensure(); !errors.Is(err, errMocked) { - t.Fatal("not the error we expected", err) - } -} diff --git a/internal/engine/session.go b/internal/engine/session.go index 862e14a..af165e1 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -8,7 +8,6 @@ import ( "net/http" "net/url" "os" - "path/filepath" "sync" "github.com/ooni/probe-cli/v3/internal/engine/atomicx" @@ -21,14 +20,11 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" - "github.com/ooni/probe-cli/v3/internal/engine/resources" - "github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager" "github.com/ooni/probe-cli/v3/internal/version" ) // SessionConfig contains the Session config type SessionConfig struct { - AssetsDir string AvailableProbeServices []model.Service KVStore KVStore Logger model.Logger @@ -40,9 +36,11 @@ type SessionConfig struct { TorBinary string } -// Session is a measurement session. +// Session is a measurement session. It contains shared information +// required to run a measurement session, and it controls the lifecycle +// of such resources. It is not possible to reuse a Session. You MUST +// NOT attempt to use a Session again after Session.Close. type Session struct { - assetsDir string availableProbeServices []model.Service availableTestHelpers map[string][]model.Service byteCounter *bytecounter.Counter @@ -64,6 +62,9 @@ type Session struct { tunnelName string tunnel tunnel.Tunnel + // closeOnce allows us to call Close just once. + closeOnce sync.Once + // mu provides mutual exclusion. mu sync.Mutex @@ -93,9 +94,6 @@ type sessionProbeServicesClientForCheckIn interface { // NewSession creates a new session or returns an error func NewSession(config SessionConfig) (*Session, error) { - if config.AssetsDir == "" { - return nil, errors.New("AssetsDir is empty") - } if config.Logger == nil { return nil, errors.New("Logger is empty") } @@ -117,7 +115,6 @@ func NewSession(config SessionConfig) (*Session, error) { return nil, err } sess := &Session{ - assetsDir: config.AssetsDir, availableProbeServices: config.AvailableProbeServices, byteCounter: bytecounter.New(), kvStore: config.KVStore, @@ -147,12 +144,6 @@ func NewSession(config SessionConfig) (*Session, error) { return sess, nil } -// ASNDatabasePath returns the path where the ASN database path should -// be if you have called s.FetchResourcesIdempotent. -func (s *Session) ASNDatabasePath() string { - return filepath.Join(s.assetsDir, resources.ASNDatabaseName) -} - // KibiBytesReceived accounts for the KibiBytes received by the HTTP clients // managed by this session so far, including experiments. func (s *Session) KibiBytesReceived() float64 { @@ -255,19 +246,19 @@ func (s *Session) newProbeServicesClientForCheckIn( // cause memory leaks in your application because of open idle connections, // as well as excessive usage of disk space. func (s *Session) Close() error { - // TODO(bassosimone): introduce a sync.Once to make this method idempotent. + s.closeOnce.Do(s.doClose) + return nil +} + +// doClose implements Close. This function is called just once. +func (s *Session) doClose() { s.httpDefaultTransport.CloseIdleConnections() s.resolver.CloseIdleConnections() s.logger.Infof("%s", s.resolver.Stats()) if s.tunnel != nil { s.tunnel.Stop() } - return os.RemoveAll(s.tempDir) -} - -// CountryDatabasePath is like ASNDatabasePath but for the country DB path. -func (s *Session) CountryDatabasePath() string { - return filepath.Join(s.assetsDir, resources.CountryDatabaseName) + _ = os.RemoveAll(s.tempDir) } // GetTestHelpersByName returns the available test helpers that @@ -546,11 +537,6 @@ func (s *Session) UserAgent() (useragent string) { return } -// MaybeUpdateResources updates the resources if needed. -func (s *Session) MaybeUpdateResources(ctx context.Context) error { - return (&resourcesmanager.CopyWorker{DestDir: s.assetsDir}).Ensure() -} - // getAvailableProbeServicesUnlocked returns the available probe // services. This function WILL NOT acquire the mu mutex, therefore, // you MUST ensure you are using it from a locked context. @@ -629,7 +615,6 @@ func (s *Session) LookupLocationContext(ctx context.Context) (*geolocate.Results EnableResolverLookup: s.proxyURL == nil, Logger: s.Logger(), Resolver: s.resolver, - ResourcesManager: s, UserAgent: s.UserAgent(), })) return task.Run(ctx) diff --git a/internal/engine/session_integration_test.go b/internal/engine/session_integration_test.go index f007637..08aa198 100644 --- a/internal/engine/session_integration_test.go +++ b/internal/engine/session_integration_test.go @@ -46,27 +46,19 @@ func TestNewSessionBuilderChecks(t *testing.T) { t.Run("with no settings", func(t *testing.T) { newSessionMustFail(t, SessionConfig{}) }) - t.Run("with only assets dir", func(t *testing.T) { - newSessionMustFail(t, SessionConfig{ - AssetsDir: "testdata", - }) - }) t.Run("with also logger", func(t *testing.T) { newSessionMustFail(t, SessionConfig{ - AssetsDir: "testdata", - Logger: model.DiscardLogger, + Logger: model.DiscardLogger, }) }) t.Run("with also software name", func(t *testing.T) { newSessionMustFail(t, SessionConfig{ - AssetsDir: "testdata", Logger: model.DiscardLogger, SoftwareName: "ooniprobe-engine", }) }) t.Run("with software version and wrong tempdir", func(t *testing.T) { newSessionMustFail(t, SessionConfig{ - AssetsDir: "testdata", Logger: model.DiscardLogger, SoftwareName: "ooniprobe-engine", SoftwareVersion: "0.0.1", @@ -97,7 +89,6 @@ func TestSessionTorArgsTorBinary(t *testing.T) { t.Skip("skip test in short mode") } sess, err := NewSession(SessionConfig{ - AssetsDir: "testdata", AvailableProbeServices: []model.Service{{ Address: "https://ams-pg-test.ooni.org", Type: "https", @@ -130,7 +121,6 @@ func TestSessionTorArgsTorBinary(t *testing.T) { func newSessionForTestingNoLookupsWithProxyURL(t *testing.T, URL *url.URL) *Session { sess, err := NewSession(SessionConfig{ - AssetsDir: "testdata", AvailableProbeServices: []model.Service{{ Address: "https://ams-pg-test.ooni.org", Type: "https", @@ -357,40 +347,11 @@ func TestSessionCloseCancelsTempDir(t *testing.T) { } } -func TestSessionDownloadResources(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - tmpdir, err := ioutil.TempDir("", "test-download-resources-idempotent") - if err != nil { - t.Fatal(err) - } - ctx := context.Background() - sess := newSessionForTestingNoLookups(t) - defer sess.Close() - sess.SetAssetsDir(tmpdir) - err = sess.MaybeUpdateResources(ctx) - if err != nil { - t.Fatal(err) - } - readfile := func(path string) (err error) { - _, err = ioutil.ReadFile(path) - return - } - if err := readfile(sess.ASNDatabasePath()); err != nil { - t.Fatal(err) - } - if err := readfile(sess.CountryDatabasePath()); err != nil { - t.Fatal(err) - } -} - func TestGetAvailableProbeServices(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } sess, err := NewSession(SessionConfig{ - AssetsDir: "testdata", Logger: model.DiscardLogger, SoftwareName: "ooniprobe-engine", SoftwareVersion: "0.0.1", @@ -411,7 +372,6 @@ func TestMaybeLookupBackendsFailure(t *testing.T) { t.Skip("skip test in short mode") } sess, err := NewSession(SessionConfig{ - AssetsDir: "testdata", Logger: model.DiscardLogger, SoftwareName: "ooniprobe-engine", SoftwareVersion: "0.0.1", @@ -433,7 +393,6 @@ func TestMaybeLookupTestHelpersIdempotent(t *testing.T) { t.Skip("skip test in short mode") } sess, err := NewSession(SessionConfig{ - AssetsDir: "testdata", Logger: model.DiscardLogger, SoftwareName: "ooniprobe-engine", SoftwareVersion: "0.0.1", @@ -459,7 +418,6 @@ func TestAllProbeServicesUnsupported(t *testing.T) { t.Skip("skip test in short mode") } sess, err := NewSession(SessionConfig{ - AssetsDir: "testdata", Logger: model.DiscardLogger, SoftwareName: "ooniprobe-engine", SoftwareVersion: "0.0.1", diff --git a/internal/engine/session_internal_test.go b/internal/engine/session_internal_test.go index b90b6fc..e13e60b 100644 --- a/internal/engine/session_internal_test.go +++ b/internal/engine/session_internal_test.go @@ -11,10 +11,6 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/model" ) -func (s *Session) SetAssetsDir(assetsDir string) { - s.assetsDir = assetsDir -} - func (s *Session) GetAvailableProbeServices() []model.Service { return s.getAvailableProbeServicesUnlocked() } diff --git a/pkg/oonimkall/internal/tasks/runner.go b/pkg/oonimkall/internal/tasks/runner.go index 34345e2..ccdc9a9 100644 --- a/pkg/oonimkall/internal/tasks/runner.go +++ b/pkg/oonimkall/internal/tasks/runner.go @@ -75,7 +75,6 @@ func (r *Runner) newsession(logger *ChanLogger) (*engine.Session, error) { return nil, err } config := engine.SessionConfig{ - AssetsDir: r.settings.AssetsDir, KVStore: kvstore, Logger: logger, SoftwareName: r.settings.Options.SoftwareName, diff --git a/pkg/oonimkall/session.go b/pkg/oonimkall/session.go index 6458946..537339b 100644 --- a/pkg/oonimkall/session.go +++ b/pkg/oonimkall/session.go @@ -9,6 +9,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/engine/atomicx" + "github.com/ooni/probe-cli/v3/internal/engine/legacy/assetsdir" "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" "github.com/ooni/probe-cli/v3/internal/engine/runtimex" @@ -54,6 +55,9 @@ type ExperimentCallbacks interface { type SessionConfig struct { // AssetsDir is the mandatory directory where to store assets // required by a Session, e.g. MaxMind DB files. + // + // This field is currently deprecated and unused. We will + // remove it when we'll bump the major number. AssetsDir string // Logger is the optional logger that will receive all the @@ -116,6 +120,15 @@ func NewSession(config *SessionConfig) (*Session, error) { if err != nil { return nil, err } + + // We cleanup the assets files used by versions of ooniprobe + // older than v3.9.0, where we started embedding the assets + // into the binary and use that directly. This cleanup doesn't + // remove the whole directory but only known files inside it + // and then the directory itself, if empty. We explicitly discard + // the return value as it does not matter to us here. + _, _ = assetsdir.Cleanup(config.AssetsDir) + var availableps []model.Service if config.ProbeServicesURL != "" { availableps = append(availableps, model.Service{ @@ -124,7 +137,6 @@ func NewSession(config *SessionConfig) (*Session, error) { }) } engineConfig := engine.SessionConfig{ - AssetsDir: config.AssetsDir, AvailableProbeServices: availableps, KVStore: kvstore, Logger: newLogger(config.Logger, config.Verbose), @@ -212,15 +224,10 @@ type GeolocateResults struct { Org string } -// MaybeUpdateResources ensures that resources are up to date. This function -// could perform network activity when we need to update resources. -// -// This function locks the session until it's done. That is, no other operation -// can be performed as long as this function is pending. +// MaybeUpdateResources is a legacy stub. It does nothing. We will +// remove it when we're ready to bump the major number. func (sess *Session) MaybeUpdateResources(ctx *Context) error { - sess.mtx.Lock() - defer sess.mtx.Unlock() - return sess.sessp.MaybeUpdateResources(ctx.ctx) + return nil } // Geolocate performs a geolocate operation and returns the results. diff --git a/pkg/oonimkall/session_integration_test.go b/pkg/oonimkall/session_integration_test.go index 9d9b7ff..2569047 100644 --- a/pkg/oonimkall/session_integration_test.go +++ b/pkg/oonimkall/session_integration_test.go @@ -47,25 +47,9 @@ func TestNewSessionWithInvalidStateDir(t *testing.T) { } } -func TestNewSessionWithMissingSoftwareName(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - sess, err := oonimkall.NewSession(&oonimkall.SessionConfig{ - StateDir: "../testdata/oonimkall/state", - }) - if err == nil || err.Error() != "AssetsDir is empty" { - t.Fatal("not the error we expected") - } - if sess != nil { - t.Fatal("expected a nil Session here") - } -} - func TestMaybeUpdateResourcesWithCancelledContext(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } + // Note that MaybeUpdateResources is now a deprecated stub that + // does nothing. We will remove it when we bump major. dir, err := ioutil.TempDir("", "xx") if err != nil { t.Fatal(err) diff --git a/pkg/oonimkall/task_integration_test.go b/pkg/oonimkall/task_integration_test.go index e8e81de..bfb149f 100644 --- a/pkg/oonimkall/task_integration_test.go +++ b/pkg/oonimkall/task_integration_test.go @@ -172,38 +172,6 @@ func TestEmptyStateDir(t *testing.T) { } } -func TestEmptyAssetsDir(t *testing.T) { - task, err := oonimkall.StartTask(`{ - "log_level": "DEBUG", - "name": "Example", - "options": { - "software_name": "oonimkall-test", - "software_version": "0.1.0" - }, - "state_dir": "../testdata/oonimkall/state", - "version": 1 - }`) - if err != nil { - t.Fatal(err) - } - var seen bool - for !task.IsDone() { - eventstr := task.WaitForNextEvent() - var event eventlike - if err := json.Unmarshal([]byte(eventstr), &event); err != nil { - t.Fatal(err) - } - if event.Key == "failure.startup" { - if strings.Contains(eventstr, "AssetsDir is empty") { - seen = true - } - } - } - if !seen { - t.Fatal("did not see failure.startup") - } -} - func TestUnknownExperiment(t *testing.T) { task, err := oonimkall.StartTask(`{ "assets_dir": "../testdata/oonimkall/assets", From 2ca9496c04715fa3500781ba13b860f0eee9a034 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 1 Apr 2021 18:40:30 +0200 Subject: [PATCH 27/28] Release: update user-agent, bundled CA, version number (#281) * chore: update the user-agent we use Part of the check-list at https://github.com/ooni/probe/issues/1369. * chore: set version to 3.9.0 See https://github.com/ooni/probe/issues/1369 * chore: run go generate ./... This is meant to update the bundled CA. We have heard of issues with our bundled CA, but it seems there have been no changes upstream. The website https://curl.se/docs/caextract.html still lists as the last change the one done on Jan 19, 2021, which is the version of the CA that we're currently bundling. For the sake of continuing with the release process, I am going to further investigate the CA once the release is done. This chore is part of https://github.com/ooni/probe/issues/1369. --- internal/engine/httpheader/useragent.go | 4 ++-- internal/engine/netx/gocertifi/certifi.go | 4 ++-- internal/engine/ooapi/apis.go | 2 +- internal/engine/ooapi/apis_test.go | 2 +- internal/engine/ooapi/caching.go | 2 +- internal/engine/ooapi/caching_test.go | 2 +- internal/engine/ooapi/callers.go | 2 +- internal/engine/ooapi/clientcall.go | 2 +- internal/engine/ooapi/clientcall_test.go | 2 +- internal/engine/ooapi/cloners.go | 2 +- internal/engine/ooapi/fakeapi_test.go | 2 +- internal/engine/ooapi/login.go | 2 +- internal/engine/ooapi/login_test.go | 2 +- internal/engine/ooapi/requests.go | 2 +- internal/engine/ooapi/responses.go | 2 +- internal/engine/ooapi/swagger_test.go | 4 ++-- internal/version/version.go | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/internal/engine/httpheader/useragent.go b/internal/engine/httpheader/useragent.go index 6b2811b..2ba4e5f 100644 --- a/internal/engine/httpheader/useragent.go +++ b/internal/engine/httpheader/useragent.go @@ -3,8 +3,8 @@ package httpheader // UserAgent returns the User-Agent header used for measuring. func UserAgent() string { - // 8.0% as of Mar 3, 2021 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ - const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36" + // 11.4% as of Mar 31, 2021 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ + const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" return ua } diff --git a/internal/engine/netx/gocertifi/certifi.go b/internal/engine/netx/gocertifi/certifi.go index 7968d85..4857759 100644 --- a/internal/engine/netx/gocertifi/certifi.go +++ b/internal/engine/netx/gocertifi/certifi.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 12:20:35.042573349 +0100 CET m=+0.416591177 +// 2021-03-31 16:50:03.221493757 +0200 CEST m=+1.318280953 // https://curl.haxx.se/ca/cacert.pem package gocertifi @@ -3241,7 +3241,7 @@ kpzNNIaRkPpkUZ3+/uul9XXeifdy ` // CACerts builds an X.509 certificate pool containing the -// certificate bundle from https://curl.haxx.se/ca/cacert.pem fetch on 2021-03-10 12:20:35.042573349 +0100 CET m=+0.416591177. +// certificate bundle from https://curl.haxx.se/ca/cacert.pem fetch on 2021-03-31 16:50:03.221493757 +0200 CEST m=+1.318280953. // Returns nil on error along with an appropriate error code. func CACerts() (*x509.CertPool, error) { pool := x509.NewCertPool() diff --git a/internal/engine/ooapi/apis.go b/internal/engine/ooapi/apis.go index 0bf2126..9fdf343 100644 --- a/internal/engine/ooapi/apis.go +++ b/internal/engine/ooapi/apis.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:31.904167818 +0100 CET m=+0.000077877 +// 2021-03-31 16:50:03.708459546 +0200 CEST m=+0.000228783 package ooapi diff --git a/internal/engine/ooapi/apis_test.go b/internal/engine/ooapi/apis_test.go index f613123..3516771 100644 --- a/internal/engine/ooapi/apis_test.go +++ b/internal/engine/ooapi/apis_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:32.206416809 +0100 CET m=+0.000074280 +// 2021-03-31 16:50:04.057051721 +0200 CEST m=+0.000093878 package ooapi diff --git a/internal/engine/ooapi/caching.go b/internal/engine/ooapi/caching.go index 725d2ab..f0eb820 100644 --- a/internal/engine/ooapi/caching.go +++ b/internal/engine/ooapi/caching.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:32.455000394 +0100 CET m=+0.000119352 +// 2021-03-31 16:50:04.317557242 +0200 CEST m=+0.000161626 package ooapi diff --git a/internal/engine/ooapi/caching_test.go b/internal/engine/ooapi/caching_test.go index b75e46f..b278884 100644 --- a/internal/engine/ooapi/caching_test.go +++ b/internal/engine/ooapi/caching_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:32.727159555 +0100 CET m=+0.000080664 +// 2021-03-31 16:50:04.580341787 +0200 CEST m=+0.000148802 package ooapi diff --git a/internal/engine/ooapi/callers.go b/internal/engine/ooapi/callers.go index d539634..469ce03 100644 --- a/internal/engine/ooapi/callers.go +++ b/internal/engine/ooapi/callers.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:32.990745473 +0100 CET m=+0.000093232 +// 2021-03-31 16:50:04.79291318 +0200 CEST m=+0.000077728 package ooapi diff --git a/internal/engine/ooapi/clientcall.go b/internal/engine/ooapi/clientcall.go index e6658aa..98a589b 100644 --- a/internal/engine/ooapi/clientcall.go +++ b/internal/engine/ooapi/clientcall.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:33.307415377 +0100 CET m=+0.000150212 +// 2021-03-31 16:50:05.02947443 +0200 CEST m=+0.000143471 package ooapi diff --git a/internal/engine/ooapi/clientcall_test.go b/internal/engine/ooapi/clientcall_test.go index 82096b4..51557cc 100644 --- a/internal/engine/ooapi/clientcall_test.go +++ b/internal/engine/ooapi/clientcall_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:33.590560357 +0100 CET m=+0.000145982 +// 2021-03-31 16:50:05.248051533 +0200 CEST m=+0.000086484 package ooapi diff --git a/internal/engine/ooapi/cloners.go b/internal/engine/ooapi/cloners.go index 903318b..16c9584 100644 --- a/internal/engine/ooapi/cloners.go +++ b/internal/engine/ooapi/cloners.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:33.849313529 +0100 CET m=+0.000136772 +// 2021-03-31 16:50:05.487854795 +0200 CEST m=+0.000140777 package ooapi diff --git a/internal/engine/ooapi/fakeapi_test.go b/internal/engine/ooapi/fakeapi_test.go index 81d096c..f03c3cb 100644 --- a/internal/engine/ooapi/fakeapi_test.go +++ b/internal/engine/ooapi/fakeapi_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:34.086220417 +0100 CET m=+0.000080714 +// 2021-03-31 16:50:05.702711633 +0200 CEST m=+0.000209246 package ooapi diff --git a/internal/engine/ooapi/login.go b/internal/engine/ooapi/login.go index 53a1a72..7296e74 100644 --- a/internal/engine/ooapi/login.go +++ b/internal/engine/ooapi/login.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:34.342751918 +0100 CET m=+0.000196026 +// 2021-03-31 16:50:05.963781069 +0200 CEST m=+0.000149834 package ooapi diff --git a/internal/engine/ooapi/login_test.go b/internal/engine/ooapi/login_test.go index 209d1ec..d510d25 100644 --- a/internal/engine/ooapi/login_test.go +++ b/internal/engine/ooapi/login_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:34.605701732 +0100 CET m=+0.000131680 +// 2021-03-31 16:50:06.223242139 +0200 CEST m=+0.000161967 package ooapi diff --git a/internal/engine/ooapi/requests.go b/internal/engine/ooapi/requests.go index 78a920f..0e05a0a 100644 --- a/internal/engine/ooapi/requests.go +++ b/internal/engine/ooapi/requests.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:35.068399906 +0100 CET m=+0.000095438 +// 2021-03-31 16:50:06.482208133 +0200 CEST m=+0.000110650 package ooapi diff --git a/internal/engine/ooapi/responses.go b/internal/engine/ooapi/responses.go index 01d682c..a4a9732 100644 --- a/internal/engine/ooapi/responses.go +++ b/internal/engine/ooapi/responses.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:35.43791782 +0100 CET m=+0.000136842 +// 2021-03-31 16:50:06.785000911 +0200 CEST m=+0.000179961 package ooapi diff --git a/internal/engine/ooapi/swagger_test.go b/internal/engine/ooapi/swagger_test.go index e775b2f..bc2dc53 100644 --- a/internal/engine/ooapi/swagger_test.go +++ b/internal/engine/ooapi/swagger_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-03-10 13:17:35.72281221 +0100 CET m=+0.000577472 +// 2021-03-31 16:50:07.052341608 +0200 CEST m=+0.000586077 package ooapi @@ -9,7 +9,7 @@ const swagger = `{ "swagger": "2.0", "info": { "title": "OONI API specification", - "version": "0.20210310.3121735" + "version": "0.20210331.3145007" }, "host": "api.ooni.io", "basePath": "/", diff --git a/internal/version/version.go b/internal/version/version.go index 848e907..c01bb74 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -3,5 +3,5 @@ package version const ( // Version is the software version - Version = "3.9.0-alpha" + Version = "3.9.0" ) From 51459e23b25c5753ee518cf2cabaf4ff17a79154 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 1 Apr 2021 20:11:13 +0200 Subject: [PATCH 28/28] fix(oohelper): make sure the CI is green (#282) * chore(oohelper): increase tests verbosity Hopefully this helps with https://github.com/ooni/probe/issues/1409. * fix(oohelper): use a nonstandard resolver * fix previous * make the diff pleasant/committable/correct --- internal/cmd/oohelper/oohelper.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/cmd/oohelper/oohelper.go b/internal/cmd/oohelper/oohelper.go index df5482c..914ea8e 100644 --- a/internal/cmd/oohelper/oohelper.go +++ b/internal/cmd/oohelper/oohelper.go @@ -24,9 +24,21 @@ var ( target = flag.String("target", "", "Target URL for the test helper") ) +func newhttpclient() *http.Client { + // Use a nonstandard resolver, which is enough to work around the + // puzzling https://github.com/ooni/probe/issues/1409 issue. + childResolver, err := netx.NewDNSClient( + netx.Config{Logger: log.Log}, "dot://8.8.8.8:853") + runtimex.PanicOnError(err, "netx.NewDNSClient should not fail here") + txp := netx.NewHTTPTransport(netx.Config{ + BaseResolver: childResolver, + Logger: log.Log, + }) + return &http.Client{Transport: txp} +} + func init() { - txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log}) - httpClient = &http.Client{Transport: txp} + httpClient = newhttpclient() resolver = netx.NewResolver(netx.Config{Logger: log.Log}) }