c/ooni-probe-cli
Watch
1
0
Fork
You've already forked ooni-probe-cli
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

chore: merge probe-engine into probe-cli (#201)

Browse source 
This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso 2021-02-02 12:05:47 +01:00 • committed by GitHub
commit d57c78bc71
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
535 changed files with 66182 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

8
internal/engine/probeservices/README.md Normal file
View file

@ -0,0 +1,8 @@
# Package github.com/ooni/probe-engine/probeservices
This package contains code to contact OONI probe services.
The probe services are HTTPS endpoints distributed across a bunch of data
centres implementing a bunch of OONI APIs. When started, OONI will benchmark
the available probe services and select the fastest one. Eventually all the
possible OONI APIs will run as probe services.

148
internal/engine/probeservices/benchselect.go Normal file
View file

@ -0,0 +1,148 @@
package probeservices
import (
"context"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// Default returns the default probe services
func Default() []model.Service {
return []model.Service{{
Address: "https://ps1.ooni.io",
Type: "https",
}, {
Address: "https://ps2.ooni.io",
Type: "https",
}, {
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}}
}
// SortEndpoints gives priority to https, then cloudfronted, then onion.
func SortEndpoints(in []model.Service) (out []model.Service) {
for _, entry := range in {
if entry.Type == "https" {
out = append(out, entry)
}
}
for _, entry := range in {
if entry.Type == "cloudfront" {
out = append(out, entry)
}
}
for _, entry := range in {
if entry.Type == "onion" {
out = append(out, entry)
}
}
return
}
// OnlyHTTPS returns the HTTPS endpoints only.
func OnlyHTTPS(in []model.Service) (out []model.Service) {
for _, entry := range in {
if entry.Type == "https" {
out = append(out, entry)
}
}
return
}
// OnlyFallbacks returns the fallback endpoints only.
func OnlyFallbacks(in []model.Service) (out []model.Service) {
for _, entry := range SortEndpoints(in) {
if entry.Type != "https" {
out = append(out, entry)
}
}
return
}
// Candidate is a candidate probe service.
type Candidate struct {
// Duration is the time it took to access the service.
Duration time.Duration
// Err indicates whether the service works.
Err error
// Endpoint is the service endpoint.
Endpoint model.Service
// TestHelpers contains the data returned by the endpoint.
TestHelpers map[string][]model.Service
}
func (c *Candidate) try(ctx context.Context, sess Session) {
client, err := NewClient(sess, c.Endpoint)
if err != nil {
c.Err = err
return
}
start := time.Now()
testhelpers, err := client.GetTestHelpers(ctx)
c.Duration = time.Now().Sub(start)
c.Err = err
c.TestHelpers = testhelpers
sess.Logger().Debugf("probe services: %+v: %+v %s", c.Endpoint, err, c.Duration)
}
func try(ctx context.Context, sess Session, svc model.Service) *Candidate {
candidate := &Candidate{Endpoint: svc}
candidate.try(ctx, sess)
return candidate
}
// TryAll tries all the input services using the provided context and session. It
// returns a list containing information on each candidate that was tried. We will
// try all the HTTPS candidates first. So, the beginning of the list will contain
// all of them, and for each of them you will know whether it worked (by checking the
// Err field) and how fast it was (by checking the Duration field). You should pick
// the fastest one that worked. If none of them works, then TryAll will subsequently
// attempt with all the available fallbacks, and return at the first success. In
// such case, you will see a list of N failing HTTPS candidates, followed by a single
// successful fallback candidate (e.g. cloudfronted). If all candidates fail, you
// see in output a list containing all entries where Err is not nil.
func TryAll(ctx context.Context, sess Session, in []model.Service) (out []*Candidate) {
var found bool
for _, svc := range OnlyHTTPS(in) {
candidate := try(ctx, sess, svc)
out = append(out, candidate)
if candidate.Err == nil {
found = true
}
}
if !found {
for _, svc := range OnlyFallbacks(in) {
candidate := try(ctx, sess, svc)
out = append(out, candidate)
if candidate.Err == nil {
return
}
}
}
return
}
// SelectBest selects the best among the candidates. If there is no
// suitable candidate, then this function returns nil.
func SelectBest(candidates []*Candidate) (selected *Candidate) {
for _, e := range candidates {
if e.Err != nil {
continue
}
if selected == nil {
selected = e
continue
}
if selected.Duration > e.Duration {
selected = e
continue
}
}
return
}

14
internal/engine/probeservices/bouncer.go Normal file
View file

@ -0,0 +1,14 @@
package probeservices
import (
"context"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// GetTestHelpers is like GetCollectors but for test helpers.
func (c Client) GetTestHelpers(
ctx context.Context) (output map[string][]model.Service, err error) {
err = c.Client.GetJSON(ctx, "/api/v1/test-helpers", &output)
return
}

16
internal/engine/probeservices/bouncer_test.go Normal file
View file

@ -0,0 +1,16 @@
package probeservices_test
import (
"context"
"testing"
)
func TestGetTestHelpers(t *testing.T) {
testhelpers, err := newclient().GetTestHelpers(context.Background())
if err != nil {
t.Fatal(err)
}
if len(testhelpers) <= 1 {
t.Fatal("no returned test helpers?!")
}
}

23
internal/engine/probeservices/checkin.go Normal file
View file

@ -0,0 +1,23 @@
package probeservices
import (
"context"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
type checkInResult struct {
Tests model.CheckInInfo `json:"tests"`
V int `json:"v"`
}
// CheckIn function is called by probes asking if there are tests to be run
// The config argument contains the mandatory settings.
// Returns the list of tests to run and the URLs, on success, or an explanatory error, in case of failure.
func (c Client) CheckIn(ctx context.Context, config model.CheckInConfig) (*model.CheckInInfo, error) {
var response checkInResult
if err := c.Client.PostJSON(ctx, "/api/v1/check-in", config, &response); err != nil {
return nil, err
}
return &response.Tests, nil
}

72
internal/engine/probeservices/checkin_test.go Normal file
View file

@ -0,0 +1,72 @@
package probeservices_test
import (
"context"
"strings"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
func TestCheckInSuccess(t *testing.T) {
client := newclient()
client.BaseURL = "https://ams-pg-test.ooni.org"
config := model.CheckInConfig{
Charging: true,
OnWiFi: true,
Platform: "android",
ProbeASN: "AS12353",
ProbeCC: "PT",
RunType: "timed",
SoftwareName: "ooniprobe-android",
SoftwareVersion: "2.7.1",
WebConnectivity: model.CheckInConfigWebConnectivity{
CategoryCodes: []string{"NEWS", "CULTR"},
},
}
ctx := context.Background()
result, err := client.CheckIn(ctx, config)
if err != nil {
t.Fatal(err)
}
if result == nil || result.WebConnectivity == nil {
t.Fatal("got nil result or WebConnectivity")
}
if result.WebConnectivity.ReportID == "" {
t.Fatal("ReportID is empty")
}
if len(result.WebConnectivity.URLs) < 1 {
t.Fatal("unexpected number of URLs")
}
for _, entry := range result.WebConnectivity.URLs {
if entry.CategoryCode != "NEWS" && entry.CategoryCode != "CULTR" {
t.Fatalf("unexpected category code: %+v", entry)
}
}
}
func TestCheckInFailure(t *testing.T) {
client := newclient()
client.BaseURL = "https://\t\t\t/" // cause test to fail
config := model.CheckInConfig{
Charging: true,
OnWiFi: true,
Platform: "android",
ProbeASN: "AS12353",
ProbeCC: "PT",
RunType: "timed",
SoftwareName: "ooniprobe-android",
SoftwareVersion: "2.7.1",
WebConnectivity: model.CheckInConfigWebConnectivity{
CategoryCodes: []string{"NEWS", "CULTR"},
},
}
ctx := context.Background()
result, err := client.CheckIn(ctx, config)
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("not the error we expected")
}
if result != nil {
t.Fatal("results?!")
}
}

29
internal/engine/probeservices/checkreportid.go Normal file
View file

@ -0,0 +1,29 @@
package probeservices
import (
"context"
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
)
type checkReportIDResponse struct {
Found bool `json:"found"`
}
// CheckReportID checks whether the given ReportID exists.
func (c Client) CheckReportID(ctx context.Context, reportID string) (bool, error) {
query := url.Values{}
query.Add("report_id", reportID)
var response checkReportIDResponse
err := (httpx.Client{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
Logger: c.Logger,
UserAgent: c.UserAgent,
}).GetJSONWithQuery(ctx, "/api/_/check_report_id", query, &response)
if err != nil {
return false, err
}
return response.Found, nil
}

61
internal/engine/probeservices/checkreportid_test.go Normal file
View file

@ -0,0 +1,61 @@
package probeservices_test
import (
"context"
"errors"
"net/http"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
)
func TestCheckReportIDWorkingAsIntended(t *testing.T) {
client := probeservices.Client{
Client: httpx.Client{
BaseURL: "https://ams-pg.ooni.org/",
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "miniooni/0.1.0-dev",
},
LoginCalls: atomicx.NewInt64(),
RegisterCalls: atomicx.NewInt64(),
StateFile: probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
}
reportID := `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`
ctx := context.Background()
found, err := client.CheckReportID(ctx, reportID)
if err != nil {
t.Fatal(err)
}
if found != true {
t.Fatal("unexpected found value")
}
}
func TestCheckReportIDWorkingWithCancelledContext(t *testing.T) {
client := probeservices.Client{
Client: httpx.Client{
BaseURL: "https://ams-pg.ooni.org/",
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "miniooni/0.1.0-dev",
},
LoginCalls: atomicx.NewInt64(),
RegisterCalls: atomicx.NewInt64(),
StateFile: probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
}
reportID := `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
found, err := client.CheckReportID(ctx, reportID)
if !errors.Is(err, context.Canceled) {
t.Fatalf("not the error we expected: %+v", err)
}
if found != false {
t.Fatal("unexpected found value")
}
}

214
internal/engine/probeservices/collector.go Normal file
View file

@ -0,0 +1,214 @@
package probeservices
import (
"context"
"errors"
"fmt"
"reflect"
"sync"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
const (
// DefaultDataFormatVersion is the default data format version.
//
// See https://github.com/ooni/spec/tree/master/data-formats#history.
DefaultDataFormatVersion = "0.2.0"
// DefaultFormat is the default format
DefaultFormat = "json"
)
var (
// ErrUnsupportedDataFormatVersion indicates that the user provided
// in input a data format version that we do not support.
ErrUnsupportedDataFormatVersion = errors.New("Unsupported data format version")
// ErrUnsupportedFormat indicates that the format is not supported.
ErrUnsupportedFormat = errors.New("Unsupported format")
// ErrJSONFormatNotSupported indicates that the collector we're using
// does not support the JSON report format.
ErrJSONFormatNotSupported = errors.New("JSON format not supported")
)
// ReportTemplate is the template for opening a report
type ReportTemplate struct {
// DataFormatVersion is unconditionally set to DefaultDataFormatVersion
// and you don't need to be concerned about it.
DataFormatVersion string `json:"data_format_version"`
// Format is unconditionally set to `json` and you don't need
// to be concerned about it.
Format string `json:"format"`
// ProbeASN is the probe's autonomous system number (e.g. `AS1234`)
ProbeASN string `json:"probe_asn"`
// ProbeCC is the probe's country code (e.g. `IT`)
ProbeCC string `json:"probe_cc"`
// SoftwareName is the app name (e.g. `measurement-kit`)
SoftwareName string `json:"software_name"`
// SoftwareVersion is the app version (e.g. `0.9.1`)
SoftwareVersion string `json:"software_version"`
// TestName is the test name (e.g. `ndt`)
TestName string `json:"test_name"`
// TestStartTime contains the test start time
TestStartTime string `json:"test_start_time"`
// TestVersion is the test version (e.g. `1.0.1`)
TestVersion string `json:"test_version"`
}
// NewReportTemplate creates a new ReportTemplate from a Measurement.
func NewReportTemplate(m *model.Measurement) ReportTemplate {
return ReportTemplate{
DataFormatVersion: DefaultDataFormatVersion,
Format: DefaultFormat,
ProbeASN: m.ProbeASN,
ProbeCC: m.ProbeCC,
SoftwareName: m.SoftwareName,
SoftwareVersion: m.SoftwareVersion,
TestName: m.TestName,
TestStartTime: m.TestStartTime,
TestVersion: m.TestVersion,
}
}
type collectorOpenResponse struct {
ID string `json:"report_id"`
SupportedFormats []string `json:"supported_formats"`
}
type reportChan struct {
// ID is the report ID
ID string
// client is the client that was used.
client Client
// tmpl is the template used when opening this report.
tmpl ReportTemplate
}
// OpenReport opens a new report.
func (c Client) OpenReport(ctx context.Context, rt ReportTemplate) (ReportChannel, error) {
if rt.DataFormatVersion != DefaultDataFormatVersion {
return nil, ErrUnsupportedDataFormatVersion
}
if rt.Format != DefaultFormat {
return nil, ErrUnsupportedFormat
}
var cor collectorOpenResponse
if err := c.Client.PostJSON(ctx, "/report", rt, &cor); err != nil {
return nil, err
}
for _, format := range cor.SupportedFormats {
if format == "json" {
return &reportChan{ID: cor.ID, client: c, tmpl: rt}, nil
}
}
return nil, ErrJSONFormatNotSupported
}
type collectorUpdateRequest struct {
// Format is the data format
Format string `json:"format"`
// Content is the actual report
Content interface{} `json:"content"`
}
type collectorUpdateResponse struct {
// ID is the measurement ID
ID string `json:"measurement_id"`
}
// CanSubmit returns true whether the provided measurement belongs to
// this report, false otherwise. We say that a given measurement belongs
// to this report if its report template matches the report's one.
func (r reportChan) CanSubmit(m *model.Measurement) bool {
return reflect.DeepEqual(NewReportTemplate(m), r.tmpl)
}
// SubmitMeasurement submits a measurement belonging to the report
// to the OONI collector. On success, we will modify the measurement
// such that it contains the report ID for which it has been
// submitted. Otherwise, we'll set the report ID to the empty
// string, so that you know which measurements weren't submitted.
func (r reportChan) SubmitMeasurement(ctx context.Context, m *model.Measurement) error {
var updateResponse collectorUpdateResponse
m.ReportID = r.ID
err := r.client.Client.PostJSON(
ctx, fmt.Sprintf("/report/%s", r.ID), collectorUpdateRequest{
Format: "json",
Content: m,
}, &updateResponse,
)
if err != nil {
m.ReportID = ""
return err
}
return nil
}
// ReportID returns the report ID.
func (r reportChan) ReportID() string {
return r.ID
}
// ReportChannel is a channel through which one could submit measurements
// belonging to the same report. The Report struct belongs to this interface.
type ReportChannel interface {
CanSubmit(m *model.Measurement) bool
ReportID() string
SubmitMeasurement(ctx context.Context, m *model.Measurement) error
}
var _ ReportChannel = &reportChan{}
// ReportOpener is any struct that is able to open a new ReportChannel. The
// Client struct belongs to this interface.
type ReportOpener interface {
OpenReport(ctx context.Context, rt ReportTemplate) (ReportChannel, error)
}
var _ ReportOpener = Client{}
// Submitter is an abstraction allowing you to submit arbitrary measurements
// to a given OONI backend. This implementation will take care of opening
// reports when needed as well as of closing reports when needed. Nonetheless
// you need to remember to call its Close method when done, because there is
// likely an open report that has not been closed yet.
type Submitter struct {
channel ReportChannel
logger model.Logger
mu sync.Mutex
opener ReportOpener
}
// NewSubmitter creates a new Submitter instance.
func NewSubmitter(opener ReportOpener, logger model.Logger) *Submitter {
return &Submitter{opener: opener, logger: logger}
}
// Submit submits the current measurement to the OONI backend created using
// the ReportOpener passed to the constructor.
func (sub *Submitter) Submit(ctx context.Context, m *model.Measurement) error {
var err error
sub.mu.Lock()
defer sub.mu.Unlock()
if sub.channel == nil || !sub.channel.CanSubmit(m) {
sub.channel, err = sub.opener.OpenReport(ctx, NewReportTemplate(m))
if err != nil {
return err
}
sub.logger.Infof("New reportID: %s", sub.channel.ReportID())
}
return sub.channel.SubmitMeasurement(ctx, m)
}

459
internal/engine/probeservices/collector_test.go Normal file
View file

@ -0,0 +1,459 @@
package probeservices_test
import (
"context"
"errors"
"io/ioutil"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync"
"testing"
"github.com/apex/log"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
)
type fakeTestKeys struct {
Failure *string `json:"failure"`
}
func makeMeasurement(rt probeservices.ReportTemplate, ID string) model.Measurement {
return model.Measurement{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
ID: "bdd20d7a-bba5-40dd-a111-9863d7908572",
MeasurementRuntime: 5.0565230846405,
MeasurementStartTime: "2018-11-01 15:33:20",
ProbeIP: "1.2.3.4",
ProbeASN: rt.ProbeASN,
ProbeCC: rt.ProbeCC,
ReportID: ID,
ResolverASN: "AS15169",
ResolverIP: "8.8.8.8",
ResolverNetworkName: "Google LLC",
SoftwareName: rt.SoftwareName,
SoftwareVersion: rt.SoftwareVersion,
TestKeys: fakeTestKeys{Failure: nil},
TestName: rt.TestName,
TestStartTime: rt.TestStartTime,
TestVersion: rt.TestVersion,
}
}
func TestNewReportTemplate(t *testing.T) {
m := &model.Measurement{
ProbeASN: "AS117",
ProbeCC: "IT",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
rt := probeservices.NewReportTemplate(m)
expect := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: probeservices.DefaultFormat,
ProbeASN: "AS117",
ProbeCC: "IT",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
if diff := cmp.Diff(expect, rt); diff != "" {
t.Fatal(diff)
}
}
func TestReportLifecycle(t *testing.T) {
ctx := context.Background()
template := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: probeservices.DefaultFormat,
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
client := newclient()
report, err := client.OpenReport(ctx, template)
if err != nil {
t.Fatal(err)
}
measurement := makeMeasurement(template, report.ReportID())
if report.CanSubmit(&measurement) != true {
t.Fatal("report should be able to submit this measurement")
}
if err = report.SubmitMeasurement(ctx, &measurement); err != nil {
t.Fatal(err)
}
if measurement.ReportID != report.ReportID() {
t.Fatal("report ID mismatch")
}
}
func TestReportLifecycleWrongExperiment(t *testing.T) {
ctx := context.Background()
template := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: probeservices.DefaultFormat,
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
client := newclient()
report, err := client.OpenReport(ctx, template)
if err != nil {
t.Fatal(err)
}
measurement := makeMeasurement(template, report.ReportID())
measurement.TestName = "antani"
if report.CanSubmit(&measurement) != false {
t.Fatal("report should not be able to submit this measurement")
}
}
func TestOpenReportInvalidDataFormatVersion(t *testing.T) {
ctx := context.Background()
template := probeservices.ReportTemplate{
DataFormatVersion: "0.1.0",
Format: probeservices.DefaultFormat,
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
client := newclient()
report, err := client.OpenReport(ctx, template)
if !errors.Is(err, probeservices.ErrUnsupportedDataFormatVersion) {
t.Fatal("not the error we expected")
}
if report != nil {
t.Fatal("expected a nil report here")
}
}
func TestOpenReportInvalidFormat(t *testing.T) {
ctx := context.Background()
template := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: "yaml",
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
client := newclient()
report, err := client.OpenReport(ctx, template)
if !errors.Is(err, probeservices.ErrUnsupportedFormat) {
t.Fatal("not the error we expected")
}
if report != nil {
t.Fatal("expected a nil report here")
}
}
func TestJSONAPIClientCreateFailure(t *testing.T) {
ctx := context.Background()
template := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: probeservices.DefaultFormat,
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
client := newclient()
client.BaseURL = "\t" // breaks the URL parser
report, err := client.OpenReport(ctx, template)
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("not the error we expected")
}
if report != nil {
t.Fatal("expected a nil report here")
}
}
func TestOpenResponseNoJSONSupport(t *testing.T) {
server := httptest.NewServer(
http.HandlerFunc(func(writer http.ResponseWriter, req *http.Request) {
writer.Write([]byte(`{"ID":"abc","supported_formats":["yaml"]}`))
}),
)
defer server.Close()
ctx := context.Background()
template := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: probeservices.DefaultFormat,
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
client := newclient()
client.BaseURL = server.URL
report, err := client.OpenReport(ctx, template)
if !errors.Is(err, probeservices.ErrJSONFormatNotSupported) {
t.Fatal("expected an error here")
}
if report != nil {
t.Fatal("expected a nil report here")
}
}
func TestEndToEnd(t *testing.T) {
server := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.RequestURI == "/report" {
w.Write([]byte(`{"report_id":"_id","supported_formats":["json"]}`))
return
}
if r.RequestURI == "/report/_id" {
data, err := ioutil.ReadAll(r.Body)
if err != nil {
panic(err)
}
sdata, err := ioutil.ReadFile("../testdata/collector-expected.jsonl")
if err != nil {
panic(err)
}
if diff := cmp.Diff(data, sdata); diff != "" {
panic(diff)
}
w.Write([]byte(`{"measurement_id":"e00c584e6e9e5326"}`))
return
}
if r.RequestURI == "/report/_id/close" {
w.Write([]byte(`{}`))
return
}
panic(r.RequestURI)
}),
)
defer server.Close()
ctx := context.Background()
template := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: probeservices.DefaultFormat,
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2018-11-01 15:33:17",
TestVersion: "0.1.0",
}
client := newclient()
client.BaseURL = server.URL
report, err := client.OpenReport(ctx, template)
if err != nil {
t.Fatal(err)
}
measurement := makeMeasurement(template, report.ReportID())
if err = report.SubmitMeasurement(ctx, &measurement); err != nil {
t.Fatal(err)
}
}
type RecordingReportChannel struct {
tmpl probeservices.ReportTemplate
m []*model.Measurement
mu sync.Mutex
}
func (rrc *RecordingReportChannel) CanSubmit(m *model.Measurement) bool {
return reflect.DeepEqual(probeservices.NewReportTemplate(m), rrc.tmpl)
}
func (rrc *RecordingReportChannel) SubmitMeasurement(ctx context.Context, m *model.Measurement) error {
if ctx.Err() != nil {
return ctx.Err()
}
rrc.mu.Lock()
defer rrc.mu.Unlock()
rrc.m = append(rrc.m, m)
return nil
}
func (rrc *RecordingReportChannel) Close(ctx context.Context) error {
if ctx.Err() != nil {
return ctx.Err()
}
rrc.mu.Lock()
defer rrc.mu.Unlock()
return nil
}
func (rrc *RecordingReportChannel) ReportID() string {
return ""
}
type RecordingReportOpener struct {
channels []*RecordingReportChannel
mu sync.Mutex
}
func (rro *RecordingReportOpener) OpenReport(
ctx context.Context, rt probeservices.ReportTemplate,
) (probeservices.ReportChannel, error) {
if ctx.Err() != nil {
return nil, ctx.Err()
}
rrc := &RecordingReportChannel{tmpl: rt}
rro.mu.Lock()
defer rro.mu.Unlock()
rro.channels = append(rro.channels, rrc)
return rrc, nil
}
func TestOpenReportCancelledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // immediately abort
template := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: probeservices.DefaultFormat,
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
client := newclient()
report, err := client.OpenReport(ctx, template)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if report != nil {
t.Fatal("expected nil report here")
}
}
func TestSubmitMeasurementCancelledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
template := probeservices.ReportTemplate{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
Format: probeservices.DefaultFormat,
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.1.0",
TestName: "dummy",
TestStartTime: "2019-10-28 12:51:06",
TestVersion: "0.1.0",
}
client := newclient()
report, err := client.OpenReport(ctx, template)
if err != nil {
t.Fatal(err)
}
measurement := makeMeasurement(template, report.ReportID())
if report.CanSubmit(&measurement) != true {
t.Fatal("report should be able to submit this measurement")
}
cancel() // cause submission to fail
err = report.SubmitMeasurement(ctx, &measurement)
if !errors.Is(err, context.Canceled) {
t.Fatalf("not the error we expected: %+v", err)
}
if measurement.ReportID != "" {
t.Fatal("report ID should be empty here")
}
}
func makeMeasurementWithoutTemplate(failure, testName string) *model.Measurement {
return &model.Measurement{
DataFormatVersion: probeservices.DefaultDataFormatVersion,
ID: "bdd20d7a-bba5-40dd-a111-9863d7908572",
MeasurementRuntime: 5.0565230846405,
MeasurementStartTime: "2018-11-01 15:33:20",
ProbeIP: "1.2.3.4",
ProbeASN: "AS123",
ProbeCC: "IT",
ReportID: "",
ResolverASN: "AS15169",
ResolverIP: "8.8.8.8",
ResolverNetworkName: "Google LLC",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
TestKeys: fakeTestKeys{Failure: &failure},
TestName: testName,
TestStartTime: "2018-11-01 15:33:17",
TestVersion: "0.1.0",
}
}
func TestSubmitterLifecyle(t *testing.T) {
rro := &RecordingReportOpener{}
submitter := probeservices.NewSubmitter(rro, log.Log)
ctx := context.Background()
m1 := makeMeasurementWithoutTemplate("antani", "example")
if err := submitter.Submit(ctx, m1); err != nil {
t.Fatal(err)
}
m2 := makeMeasurementWithoutTemplate("mascetti", "example")
if err := submitter.Submit(ctx, m2); err != nil {
t.Fatal(err)
}
m3 := makeMeasurementWithoutTemplate("antani", "example_extended")
if err := submitter.Submit(ctx, m3); err != nil {
t.Fatal(err)
}
if len(rro.channels) != 2 {
t.Fatal("unexpected number of channels")
}
if len(rro.channels[0].m) != 2 {
t.Fatal("unexpected number of measurements in first channel")
}
if len(rro.channels[1].m) != 1 {
t.Fatal("unexpected number of measurements in second channel")
}
}
func TestSubmitterCannotOpenNewChannel(t *testing.T) {
rro := &RecordingReportOpener{}
submitter := probeservices.NewSubmitter(rro, log.Log)
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
m1 := makeMeasurementWithoutTemplate("antani", "example")
if err := submitter.Submit(ctx, m1); !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
m2 := makeMeasurementWithoutTemplate("mascetti", "example")
if err := submitter.Submit(ctx, m2); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
m3 := makeMeasurementWithoutTemplate("antani", "example_extended")
if err := submitter.Submit(ctx, m3); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if len(rro.channels) != 0 {
t.Fatal("unexpected number of channels")
}
}

38
internal/engine/probeservices/login.go Normal file
View file

@ -0,0 +1,38 @@
package probeservices
import (
"context"
"time"
)
// LoginCredentials contains the login credentials
type LoginCredentials struct {
ClientID string `json:"username"`
Password string `json:"password"`
}
// LoginAuth contains authentication info
type LoginAuth struct {
Expire time.Time `json:"expire"`
Token string `json:"token"`
}
// MaybeLogin performs login if necessary
func (c Client) MaybeLogin(ctx context.Context) error {
state := c.StateFile.Get()
if state.Auth() != nil {
return nil // we're already good
}
creds := state.Credentials()
if creds == nil {
return ErrNotRegistered
}
c.LoginCalls.Add(1)
var auth LoginAuth
if err := c.Client.PostJSON(ctx, "/api/v1/login", *creds, &auth); err != nil {
return err
}
state.Expire = auth.Expire
state.Token = auth.Token
return c.StateFile.Set(state)
}

73
internal/engine/probeservices/login_test.go Normal file
View file

@ -0,0 +1,73 @@
package probeservices_test
import (
"context"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
)
func TestMaybeLogin(t *testing.T) {
t.Run("when we already have a token", func(t *testing.T) {
clnt := newclient()
state := probeservices.State{
Expire: time.Now().Add(time.Hour),
Token: "xx-xxx-x-xxxx",
}
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
ctx := context.Background()
if err := clnt.MaybeLogin(ctx); err != nil {
t.Fatal(err)
}
})
t.Run("when we have already registered", func(t *testing.T) {
clnt := newclient()
state := probeservices.State{
// Explicitly empty to clarify what this test does
}
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
ctx := context.Background()
if err := clnt.MaybeLogin(ctx); err == nil {
t.Fatal("expected an error here")
}
})
t.Run("when the API call fails", func(t *testing.T) {
clnt := newclient()
clnt.BaseURL = "\t\t\t" // causes the code to fail
state := probeservices.State{
ClientID: "xx-xxx-x-xxxx",
Password: "xx",
}
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
ctx := context.Background()
if err := clnt.MaybeLogin(ctx); err == nil {
t.Fatal("expected an error here")
}
})
}
func TestMaybeLoginIdempotent(t *testing.T) {
clnt := newclient()
ctx := context.Background()
metadata := testorchestra.MetadataFixture()
if err := clnt.MaybeRegister(ctx, metadata); err != nil {
t.Fatal(err)
}
if err := clnt.MaybeLogin(ctx); err != nil {
t.Fatal(err)
}
if err := clnt.MaybeLogin(ctx); err != nil {
t.Fatal(err)
}
if clnt.LoginCalls.Load() != 1 {
t.Fatal("called login API too many times")
}
}

67
internal/engine/probeservices/measurementmeta.go Normal file
View file

@ -0,0 +1,67 @@
package probeservices
import (
"context"
"net/url"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
)
// MeasurementMetaConfig contains configuration for GetMeasurementMeta.
type MeasurementMetaConfig struct {
// ReportID is the mandatory report ID.
ReportID string
// Full indicates whether we also want the full measurement body.
Full bool
// Input is the optional input.
Input string
}
// MeasurementMeta contains measurement metadata.
type MeasurementMeta struct {
// Fields returned by the API server whenever we are
// calling /api/v1/measurement_meta.
Anomaly bool `json:"anomaly"`
CategoryCode string `json:"category_code"`
Confirmed bool `json:"confirmed"`
Failure bool `json:"failure"`
Input *string `json:"input"`
MeasurementStartTime time.Time `json:"measurement_start_time"`
ProbeASN int64 `json:"probe_asn"`
ProbeCC string `json:"probe_cc"`
ReportID string `json:"report_id"`
Scores string `json:"scores"`
TestName string `json:"test_name"`
TestStartTime time.Time `json:"test_start_time"`
// This field is only included if the user has specified
// the config.Full option, otherwise it's empty.
RawMeasurement string `json:"raw_measurement"`
}
// GetMeasurementMeta returns meta information about a measurement.
func (c Client) GetMeasurementMeta(
ctx context.Context, config MeasurementMetaConfig) (*MeasurementMeta, error) {
query := url.Values{}
query.Add("report_id", config.ReportID)
if config.Input != "" {
query.Add("input", config.Input)
}
if config.Full {
query.Add("full", "true")
}
var response MeasurementMeta
err := (httpx.Client{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
Logger: c.Logger,
UserAgent: c.UserAgent,
}).GetJSONWithQuery(ctx, "/api/v1/measurement_meta", query, &response)
if err != nil {
return nil, err
}
return &response, nil
}

111
internal/engine/probeservices/measurementmeta_test.go Normal file
View file

@ -0,0 +1,111 @@
package probeservices_test
import (
"context"
"encoding/json"
"errors"
"net/http"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
)
func TestGetMeasurementMetaWorkingAsIntended(t *testing.T) {
client := probeservices.Client{
Client: httpx.Client{
BaseURL: "https://ams-pg.ooni.org/",
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "miniooni/0.1.0-dev",
},
LoginCalls: atomicx.NewInt64(),
RegisterCalls: atomicx.NewInt64(),
StateFile: probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
}
config := probeservices.MeasurementMetaConfig{
ReportID: `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`,
Full: true,
Input: `https://www.example.org`,
}
ctx := context.Background()
mmeta, err := client.GetMeasurementMeta(ctx, config)
if err != nil {
t.Fatal(err)
}
if mmeta.Anomaly != false {
t.Fatal("unexpected anomaly value")
}
if mmeta.CategoryCode != "" {
t.Fatal("unexpected category code value")
}
if mmeta.Confirmed != false {
t.Fatal("unexpected confirmed value")
}
if mmeta.Failure != true {
// TODO(bassosimone): this field seems wrong
t.Fatal("unexpected failure value")
}
if mmeta.Input == nil || *mmeta.Input != config.Input {
t.Fatal("unexpected input value")
}
if mmeta.MeasurementStartTime.String() != "2020-12-09 05:22:25 +0000 UTC" {
t.Fatal("unexpected measurement start time value")
}
if mmeta.ProbeASN != 30722 {
t.Fatal("unexpected probe asn value")
}
if mmeta.ProbeCC != "IT" {
t.Fatal("unexpected probe cc value")
}
if mmeta.ReportID != config.ReportID {
t.Fatal("unexpected report id value")
}
// TODO(bassosimone): we could better this check
var scores interface{}
if err := json.Unmarshal([]byte(mmeta.Scores), &scores); err != nil {
t.Fatalf("cannot parse scores value: %+v", err)
}
if mmeta.TestName != "urlgetter" {
t.Fatal("unexpected test name value")
}
if mmeta.TestStartTime.String() != "2020-12-09 05:22:25 +0000 UTC" {
t.Fatal("unexpected test start time value")
}
// TODO(bassosimone): we could better this check
var rawmeas interface{}
if err := json.Unmarshal([]byte(mmeta.RawMeasurement), &rawmeas); err != nil {
t.Fatalf("cannot parse raw measurement: %+v", err)
}
}
func TestGetMeasurementMetaWorkingWithCancelledContext(t *testing.T) {
client := probeservices.Client{
Client: httpx.Client{
BaseURL: "https://ams-pg.ooni.org/",
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "miniooni/0.1.0-dev",
},
LoginCalls: atomicx.NewInt64(),
RegisterCalls: atomicx.NewInt64(),
StateFile: probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
}
config := probeservices.MeasurementMetaConfig{
ReportID: `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`,
Full: true,
Input: `https://www.example.org`,
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
mmeta, err := client.GetMeasurementMeta(ctx, config)
if !errors.Is(err, context.Canceled) {
t.Fatalf("not the error we expected: %+v", err)
}
if mmeta != nil {
t.Fatal("we expected a nil mmeta here")
}
}

52
internal/engine/probeservices/metadata.go Normal file
View file

@ -0,0 +1,52 @@
package probeservices
// Metadata contains metadata about a probe. This message is
// included into a bunch of messages sent to orchestra.
type Metadata struct {
AvailableBandwidth string `json:"available_bandwidth,omitempty"`
DeviceToken string `json:"device_token,omitempty"`
Language string `json:"language,omitempty"`
NetworkType string `json:"network_type,omitempty"`
Platform string `json:"platform"`
ProbeASN string `json:"probe_asn"`
ProbeCC string `json:"probe_cc"`
ProbeFamily string `json:"probe_family,omitempty"`
ProbeTimezone string `json:"probe_timezone,omitempty"`
SoftwareName string `json:"software_name"`
SoftwareVersion string `json:"software_version"`
SupportedTests []string `json:"supported_tests"`
}
// Valid returns true if metadata is valid, false otherwise. Metadata is
// considered valid if all the mandatory fields are not empty. If a field
// is marked `json:",omitempty"` in the structure definition, then it's
// for sure mandatory. The "device_token" field is mandatory only if the
// platform is "ios" or "android", because there's currently no device
// token that we know of for desktop devices.
func (m Metadata) Valid() bool {
if m.ProbeCC == "" {
return false
}
if m.ProbeASN == "" {
return false
}
if m.Platform == "" {
return false
}
if m.SoftwareName == "" {
return false
}
if m.SoftwareVersion == "" {
return false
}
if len(m.SupportedTests) < 1 {
return false
}
switch m.Platform {
case "ios", "android":
if m.DeviceToken == "" {
return false
}
}
return true
}

106
internal/engine/probeservices/metadata_test.go Normal file
View file

@ -0,0 +1,106 @@
package probeservices_test
import (
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
)
func TestValid(t *testing.T) {
t.Run("fail on probe_cc", func(t *testing.T) {
var m probeservices.Metadata
if m.Valid() != false {
t.Fatal("expected false here")
}
})
t.Run("fail on probe_asn", func(t *testing.T) {
m := probeservices.Metadata{
ProbeCC: "IT",
}
if m.Valid() != false {
t.Fatal("expected false here")
}
})
t.Run("fail on platform", func(t *testing.T) {
m := probeservices.Metadata{
ProbeCC: "IT",
ProbeASN: "AS1234",
}
if m.Valid() != false {
t.Fatal("expected false here")
}
})
t.Run("fail on software_name", func(t *testing.T) {
m := probeservices.Metadata{
ProbeCC: "IT",
ProbeASN: "AS1234",
Platform: "linux",
}
if m.Valid() != false {
t.Fatal("expected false here")
}
})
t.Run("fail on software_version", func(t *testing.T) {
m := probeservices.Metadata{
ProbeCC: "IT",
ProbeASN: "AS1234",
Platform: "linux",
SoftwareName: "miniooni",
}
if m.Valid() != false {
t.Fatal("expected false here")
}
})
t.Run("fail on supported_tests", func(t *testing.T) {
m := probeservices.Metadata{
ProbeCC: "IT",
ProbeASN: "AS1234",
Platform: "linux",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
}
if m.Valid() != false {
t.Fatal("expected false here")
}
})
t.Run("fail on missing device_token", func(t *testing.T) {
m := probeservices.Metadata{
ProbeCC: "IT",
ProbeASN: "AS1234",
Platform: "ios",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
SupportedTests: []string{"web_connectivity"},
}
if m.Valid() != false {
t.Fatal("expected false here")
}
})
t.Run("success for desktop", func(t *testing.T) {
m := probeservices.Metadata{
ProbeCC: "IT",
ProbeASN: "AS1234",
Platform: "linux",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
SupportedTests: []string{"web_connectivity"},
}
if m.Valid() != true {
t.Fatal("expected true here")
}
})
t.Run("success for mobile", func(t *testing.T) {
m := probeservices.Metadata{
DeviceToken: "xx-xxx-xx-xxxx",
ProbeCC: "IT",
ProbeASN: "AS1234",
Platform: "android",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
SupportedTests: []string{"web_connectivity"},
}
if m.Valid() != true {
t.Fatal("expected true here")
}
})
}

129
internal/engine/probeservices/probeservices.go Normal file
View file

@ -0,0 +1,129 @@
// Package probeservices contains code to contact OONI probe services.
//
// The probe services are HTTPS endpoints distributed across a bunch of data
// centres implementing a bunch of OONI APIs. When started, OONI will benchmark
// the available probe services and select the fastest one. Eventually all the
// possible OONI APIs will run as probe services.
//
// This package implements the following APIs:
//
// 1. v2.0.0 of the OONI bouncer specification defined
// in https://github.com/ooni/spec/blob/master/backends/bk-004-bouncer;
//
// 2. v2.0.0 of the OONI collector specification defined
// in https://github.com/ooni/spec/blob/master/backends/bk-003-collector.md;
//
// 3. most of the OONI orchestra API: login, register, fetch URLs for
// the Web Connectivity experiment, input for Tor and Psiphon.
//
// Orchestra is a set of OONI APIs for probe orchestration. We currently mainly
// using it for fetching inputs for the tor, psiphon, and web experiments.
//
// In addition, this package also contains code to benchmark the available
// probe services, discard non working ones, select the fastest.
package probeservices
import (
"errors"
"net/http"
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
var (
// ErrUnsupportedEndpoint indicates that we don't support this endpoint type.
ErrUnsupportedEndpoint = errors.New("probe services: unsupported endpoint type")
// ErrUnsupportedCloudFrontAddress indicates that we don't support this
// cloudfront address (e.g. wrong scheme, presence of port).
ErrUnsupportedCloudFrontAddress = errors.New(
"probe services: unsupported cloud front address",
)
// ErrNotRegistered indicates that the probe is not registered
// with the OONI orchestra backend.
ErrNotRegistered = errors.New("not registered")
// ErrNotLoggedIn indicates that we are not logged in
ErrNotLoggedIn = errors.New("not logged in")
// ErrInvalidMetadata indicates that the metadata is not valid
ErrInvalidMetadata = errors.New("invalid metadata")
)
// Session is how this package sees a Session.
type Session interface {
DefaultHTTPClient() *http.Client
KeyValueStore() model.KeyValueStore
Logger() model.Logger
ProxyURL() *url.URL
UserAgent() string
}
// Client is a client for the OONI probe services API.
type Client struct {
httpx.Client
LoginCalls *atomicx.Int64
RegisterCalls *atomicx.Int64
StateFile StateFile
}
// GetCredsAndAuth is an utility function that returns the credentials with
// which we are registered and the token with which we're logged in. If we're
// not registered or not logged in, an error is returned instead.
func (c Client) GetCredsAndAuth() (*LoginCredentials, *LoginAuth, error) {
state := c.StateFile.Get()
creds := state.Credentials()
if creds == nil {
return nil, nil, ErrNotRegistered
}
auth := state.Auth()
if auth == nil {
return nil, nil, ErrNotLoggedIn
}
return creds, auth, nil
}
// NewClient creates a new client for the specified probe services endpoint. This
// function fails, e.g., we don't support the specified endpoint.
func NewClient(sess Session, endpoint model.Service) (*Client, error) {
client := &Client{
Client: httpx.Client{
BaseURL: endpoint.Address,
HTTPClient: sess.DefaultHTTPClient(),
Logger: sess.Logger(),
ProxyURL: sess.ProxyURL(),
UserAgent: sess.UserAgent(),
},
LoginCalls: atomicx.NewInt64(),
RegisterCalls: atomicx.NewInt64(),
StateFile: NewStateFile(sess.KeyValueStore()),
}
switch endpoint.Type {
case "https":
return client, nil
case "cloudfront":
// Do the cloudfronting dance. The front must appear inside of the
// URL, so that we use it for DNS resolution and SNI. The real domain
// must instead appear inside of the Host header.
URL, err := url.Parse(client.BaseURL)
if err != nil {
return nil, err
}
if URL.Scheme != "https" || URL.Host != URL.Hostname() {
return nil, ErrUnsupportedCloudFrontAddress
}
client.Client.Host = URL.Hostname()
URL.Host = endpoint.Front
client.BaseURL = URL.String()
if _, err := url.Parse(client.BaseURL); err != nil {
return nil, err
}
return client, nil
default:
return nil, ErrUnsupportedEndpoint
}
}

627
internal/engine/probeservices/probeservices_test.go Normal file
View file

@ -0,0 +1,627 @@
package probeservices_test
import (
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"regexp"
"strings"
"testing"
"time"
"github.com/apex/log"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
)
func newclient() *probeservices.Client {
client, err := probeservices.NewClient(
&mockable.Session{
MockableHTTPClient: http.DefaultClient,
MockableLogger: log.Log,
},
model.Service{
Address: "https://ams-pg-test.ooni.org/",
Type: "https",
},
)
if err != nil {
panic(err) // so fail the test
}
return client
}
func TestNewClientHTTPS(t *testing.T) {
client, err := probeservices.NewClient(
&mockable.Session{}, model.Service{
Address: "https://x.org",
Type: "https",
})
if err != nil {
t.Fatal(err)
}
if client.BaseURL != "https://x.org" {
t.Fatal("not the URL we expected")
}
}
func TestNewClientUnsupportedEndpoint(t *testing.T) {
client, err := probeservices.NewClient(
&mockable.Session{}, model.Service{
Address: "https://x.org",
Type: "onion",
})
if !errors.Is(err, probeservices.ErrUnsupportedEndpoint) {
t.Fatal("not the error we expected")
}
if client != nil {
t.Fatal("expected nil client here")
}
}
func TestNewClientCloudfrontInvalidURL(t *testing.T) {
client, err := probeservices.NewClient(
&mockable.Session{}, model.Service{
Address: "\t\t\t",
Type: "cloudfront",
})
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("not the error we expected")
}
if client != nil {
t.Fatal("expected nil client here")
}
}
func TestNewClientCloudfrontInvalidURLScheme(t *testing.T) {
client, err := probeservices.NewClient(
&mockable.Session{}, model.Service{
Address: "http://x.org",
Type: "cloudfront",
})
if !errors.Is(err, probeservices.ErrUnsupportedCloudFrontAddress) {
t.Fatal("not the error we expected")
}
if client != nil {
t.Fatal("expected nil client here")
}
}
func TestNewClientCloudfrontInvalidURLWithPort(t *testing.T) {
client, err := probeservices.NewClient(
&mockable.Session{}, model.Service{
Address: "https://x.org:54321",
Type: "cloudfront",
})
if !errors.Is(err, probeservices.ErrUnsupportedCloudFrontAddress) {
t.Fatal("not the error we expected")
}
if client != nil {
t.Fatal("expected nil client here")
}
}
func TestNewClientCloudfrontInvalidFront(t *testing.T) {
client, err := probeservices.NewClient(
&mockable.Session{}, model.Service{
Address: "https://x.org",
Type: "cloudfront",
Front: "\t\t\t",
})
if err == nil || !strings.HasSuffix(err.Error(), `invalid URL escape "%09"`) {
t.Fatal("not the error we expected")
}
if client != nil {
t.Fatal("expected nil client here")
}
}
func TestNewClientCloudfrontGood(t *testing.T) {
client, err := probeservices.NewClient(
&mockable.Session{}, model.Service{
Address: "https://x.org",
Type: "cloudfront",
Front: "google.com",
})
if err != nil {
t.Fatal(err)
}
if client.BaseURL != "https://google.com" {
t.Fatal("not the BaseURL we expected")
}
if client.Host != "x.org" {
t.Fatal("not the Host we expected")
}
}
func TestCloudfront(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
client, err := probeservices.NewClient(
&mockable.Session{}, model.Service{
Address: "https://meek.azureedge.net",
Type: "cloudfront",
Front: "ajax.aspnetcdn.com",
})
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("GET", client.BaseURL, nil)
if err != nil {
t.Fatal(err)
}
req.Host = client.Host
resp, err := http.DefaultTransport.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatal("unexpected status code")
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if string(data) != "I’m just a happy little web server.\n" {
t.Fatal("unexpected response body")
}
}
func TestDefaultProbeServicesWorkAsIntended(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
for _, e := range probeservices.Default() {
client, err := probeservices.NewClient(&mockable.Session{
MockableHTTPClient: http.DefaultClient,
MockableLogger: log.Log,
}, e)
if err != nil {
t.Fatal(err)
}
testhelpers, err := client.GetTestHelpers(context.Background())
if err != nil {
t.Fatal(err)
}
if len(testhelpers) < 1 {
t.Fatal("no test helpers?!")
}
}
}
func TestSortEndpoints(t *testing.T) {
in := []model.Service{{
Type: "onion",
Address: "httpo://jehhrikjjqrlpufu.onion",
}, {
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}, {
Type: "https",
Address: "https://ams-ps2.ooni.nu:443",
}}
expect := []model.Service{{
Type: "https",
Address: "https://ams-ps2.ooni.nu:443",
}, {
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}, {
Type: "onion",
Address: "httpo://jehhrikjjqrlpufu.onion",
}}
out := probeservices.SortEndpoints(in)
diff := cmp.Diff(out, expect)
if diff != "" {
t.Fatal(diff)
}
}
func TestOnlyHTTPS(t *testing.T) {
in := []model.Service{{
Type: "onion",
Address: "httpo://jehhrikjjqrlpufu.onion",
}, {
Type: "https",
Address: "https://ams-ps-nonexistent.ooni.io",
}, {
Type: "https",
Address: "https://hkg-ps-nonexistent.ooni.io",
}, {
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}, {
Type: "https",
Address: "https://mia-ps-nonexistent.ooni.io",
}}
expect := []model.Service{{
Type: "https",
Address: "https://ams-ps-nonexistent.ooni.io",
}, {
Type: "https",
Address: "https://hkg-ps-nonexistent.ooni.io",
}, {
Type: "https",
Address: "https://mia-ps-nonexistent.ooni.io",
}}
out := probeservices.OnlyHTTPS(in)
diff := cmp.Diff(out, expect)
if diff != "" {
t.Fatal(diff)
}
}
func TestOnlyFallbacks(t *testing.T) {
// put onion first so we also verify that we sort the endpoints
in := []model.Service{{
Type: "onion",
Address: "httpo://jehhrikjjqrlpufu.onion",
}, {
Type: "https",
Address: "https://ams-ps-nonexistent.ooni.io",
}, {
Type: "https",
Address: "https://hkg-ps-nonexistent.ooni.io",
}, {
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}, {
Type: "https",
Address: "https://mia-ps-nonexistent.ooni.io",
}}
expect := []model.Service{{
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}, {
Type: "onion",
Address: "httpo://jehhrikjjqrlpufu.onion",
}}
out := probeservices.OnlyFallbacks(in)
diff := cmp.Diff(out, expect)
if diff != "" {
t.Fatal(diff)
}
}
func TestTryAllCanceledContext(t *testing.T) {
// put onion first so we also verify that we sort the endpoints
in := []model.Service{{
Type: "onion",
Address: "httpo://jehhrikjjqrlpufu.onion",
}, {
Type: "https",
Address: "https://ams-ps-nonexistent.ooni.io",
}, {
Type: "https",
Address: "https://hkg-ps-nonexistent.ooni.io",
}, {
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}, {
Type: "https",
Address: "https://mia-ps-nonexistent.ooni.io",
}}
ctx, cancel := context.WithCancel(context.Background())
cancel() // immediately cancel and cause every attempt to fail
sess := &mockable.Session{
MockableHTTPClient: http.DefaultClient,
MockableLogger: log.Log,
}
out := probeservices.TryAll(ctx, sess, in)
if len(out) != 5 {
t.Fatal("invalid number of entries")
}
//
if out[0].Duration <= 0 {
t.Fatal("invalid duration")
}
if !errors.Is(out[0].Err, context.Canceled) {
t.Fatal("invalid error")
}
if out[0].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if out[0].Endpoint.Address != "https://ams-ps-nonexistent.ooni.io" {
t.Fatal("invalid endpoint type")
}
//
if out[1].Duration <= 0 {
t.Fatal("invalid duration")
}
if !errors.Is(out[1].Err, context.Canceled) {
t.Fatal("invalid error")
}
if out[1].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if out[1].Endpoint.Address != "https://hkg-ps-nonexistent.ooni.io" {
t.Fatal("invalid endpoint type")
}
//
if out[2].Duration <= 0 {
t.Fatal("invalid duration")
}
if !errors.Is(out[2].Err, context.Canceled) {
t.Fatal("invalid error")
}
if out[2].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if out[2].Endpoint.Address != "https://mia-ps-nonexistent.ooni.io" {
t.Fatal("invalid endpoint type")
}
//
if out[3].Duration <= 0 {
t.Fatal("invalid duration")
}
if !errors.Is(out[3].Err, context.Canceled) {
t.Fatal("invalid error")
}
if out[3].Endpoint.Type != "cloudfront" {
t.Fatal("invalid endpoint type")
}
if out[3].Endpoint.Front != "dkyhjv0wpi2dk.cloudfront.net" {
t.Fatal("invalid endpoint type")
}
if out[3].Endpoint.Address != "https://dkyhjv0wpi2dk.cloudfront.net" {
t.Fatal("invalid endpoint type")
}
//
// Note: here duration may be zero because the endpoint is not supported
// and so we don't basically do anything. But it also may be nonzero since
// we also run tests in the cloud, which is slower than my desktop. So, I
// have not written a specific test concerning out[4].Duration.
if !errors.Is(out[4].Err, probeservices.ErrUnsupportedEndpoint) {
t.Fatal("invalid error")
}
if out[4].Endpoint.Type != "onion" {
t.Fatal("invalid endpoint type")
}
if out[4].Endpoint.Address != "httpo://jehhrikjjqrlpufu.onion" {
t.Fatal("invalid endpoint type")
}
}
func TestTryAllIntegrationWeRaceForFastestHTTPS(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
const pattern = "^https://ps[1-4].ooni.io$"
// put onion first so we also verify that we sort the endpoints
in := []model.Service{{
Type: "onion",
Address: "httpo://jehhrikjjqrlpufu.onion",
}, {
Type: "https",
Address: "https://ps1.ooni.io",
}, {
Type: "https",
Address: "https://ps2.ooni.io",
}, {
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}, {
Type: "https",
Address: "https://ps3.ooni.io",
}}
sess := &mockable.Session{
MockableHTTPClient: http.DefaultClient,
MockableLogger: log.Log,
}
out := probeservices.TryAll(context.Background(), sess, in)
if len(out) != 3 {
t.Fatal("invalid number of entries")
}
//
if out[0].Duration <= 0 {
t.Fatal("invalid duration")
}
if out[0].Err != nil {
t.Fatal("invalid error")
}
if out[0].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if ok, _ := regexp.MatchString(pattern, out[0].Endpoint.Address); !ok {
t.Fatal("invalid endpoint type")
}
//
if out[1].Duration <= 0 {
t.Fatal("invalid duration")
}
if out[1].Err != nil {
t.Fatal("invalid error")
}
if out[1].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if ok, _ := regexp.MatchString(pattern, out[1].Endpoint.Address); !ok {
t.Fatal("invalid endpoint type")
}
//
if out[2].Duration <= 0 {
t.Fatal("invalid duration")
}
if out[2].Err != nil {
t.Fatal("invalid error")
}
if out[2].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if ok, _ := regexp.MatchString(pattern, out[2].Endpoint.Address); !ok {
t.Fatal("invalid endpoint type")
}
}
func TestTryAllIntegrationWeFallback(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
// put onion first so we also verify that we sort the endpoints
in := []model.Service{{
Type: "onion",
Address: "httpo://jehhrikjjqrlpufu.onion",
}, {
Type: "https",
Address: "https://ps-nonexistent.ooni.io",
}, {
Type: "https",
Address: "https://hkg-ps-nonexistent.ooni.nu",
}, {
Front: "dkyhjv0wpi2dk.cloudfront.net",
Type: "cloudfront",
Address: "https://dkyhjv0wpi2dk.cloudfront.net",
}, {
Type: "https",
Address: "https://mia-ps2-nonexistent.ooni.nu",
}}
sess := &mockable.Session{
MockableHTTPClient: http.DefaultClient,
MockableLogger: log.Log,
}
out := probeservices.TryAll(context.Background(), sess, in)
if len(out) != 4 {
t.Fatal("invalid number of entries")
}
//
if out[0].Duration <= 0 {
t.Fatal("invalid duration")
}
if !strings.HasSuffix(out[0].Err.Error(), "no such host") {
t.Fatal("invalid error")
}
if out[0].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if out[0].Endpoint.Address != "https://ps-nonexistent.ooni.io" {
t.Fatal("invalid endpoint type")
}
//
if out[1].Duration <= 0 {
t.Fatal("invalid duration")
}
if !strings.HasSuffix(out[1].Err.Error(), "no such host") {
t.Fatal("invalid error")
}
if out[1].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if out[1].Endpoint.Address != "https://hkg-ps-nonexistent.ooni.nu" {
t.Fatal("invalid endpoint type")
}
//
if out[2].Duration <= 0 {
t.Fatal("invalid duration")
}
if !strings.HasSuffix(out[2].Err.Error(), "no such host") {
t.Fatal("invalid error")
}
if out[2].Endpoint.Type != "https" {
t.Fatal("invalid endpoint type")
}
if out[2].Endpoint.Address != "https://mia-ps2-nonexistent.ooni.nu" {
t.Fatal("invalid endpoint type")
}
//
if out[3].Duration <= 0 {
t.Fatal("invalid duration")
}
if out[3].Err != nil {
t.Fatal("invalid error")
}
if out[3].Endpoint.Type != "cloudfront" {
t.Fatal("invalid endpoint type")
}
if out[3].Endpoint.Address != "https://dkyhjv0wpi2dk.cloudfront.net" {
t.Fatal("invalid endpoint type")
}
if out[3].Endpoint.Front != "dkyhjv0wpi2dk.cloudfront.net" {
t.Fatal("invalid front")
}
}
func TestSelectBestEmptyInput(t *testing.T) {
if out := probeservices.SelectBest(nil); out != nil {
t.Fatal("expected nil output here")
}
}
func TestSelectBestOnlyFailures(t *testing.T) {
in := []*probeservices.Candidate{{
Duration: 10 * time.Millisecond,
Err: io.EOF,
}}
if out := probeservices.SelectBest(in); out != nil {
t.Fatal("expected nil output here")
}
}
func TestSelectBestSelectsTheFastest(t *testing.T) {
in := []*probeservices.Candidate{{
Duration: 10 * time.Millisecond,
Endpoint: model.Service{
Address: "https://ps1.ooni.io",
Type: "https",
},
}, {
Duration: 4 * time.Millisecond,
Endpoint: model.Service{
Address: "https://ps2.ooni.io",
Type: "https",
},
}, {
Duration: 7 * time.Millisecond,
Endpoint: model.Service{
Address: "https://ps3.ooni.io",
Type: "https",
},
}, {
Duration: 11 * time.Millisecond,
Endpoint: model.Service{
Address: "https://ps4.ooni.io",
Type: "https",
},
}}
expected := &probeservices.Candidate{
Duration: 4 * time.Millisecond,
Endpoint: model.Service{
Address: "https://ps2.ooni.io",
Type: "https",
},
}
out := probeservices.SelectBest(in)
if diff := cmp.Diff(out, expected); diff != "" {
t.Fatal(diff)
}
}
func TestGetCredsAndAuthNotLoggedIn(t *testing.T) {
clnt := newclient()
if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
t.Fatal(err)
}
creds, auth, err := clnt.GetCredsAndAuth()
if !errors.Is(err, probeservices.ErrNotLoggedIn) {
t.Fatal("not the error we expected")
}
if creds != nil {
t.Fatal("expected nil creds here")
}
if auth != nil {
t.Fatal("expected nil auth here")
}
}

17
internal/engine/probeservices/psiphon.go Normal file
View file

@ -0,0 +1,17 @@
package probeservices
import (
"context"
"fmt"
)
// FetchPsiphonConfig fetches psiphon config from authenticated OONI orchestra.
func (c Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) {
_, auth, err := c.GetCredsAndAuth()
if err != nil {
return nil, err
}
client := c.Client
client.Authorization = fmt.Sprintf("Bearer %s", auth.Token)
return client.FetchResource(ctx, "/api/v1/test-list/psiphon-config")
}

46
internal/engine/probeservices/psiphon_test.go Normal file
View file

@ -0,0 +1,46 @@
package probeservices_test
import (
"context"
"encoding/json"
"errors"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
)
func TestFetchPsiphonConfig(t *testing.T) {
clnt := newclient()
if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
t.Fatal(err)
}
if err := clnt.MaybeLogin(context.Background()); err != nil {
t.Fatal(err)
}
data, err := clnt.FetchPsiphonConfig(context.Background())
if err != nil {
t.Fatal(err)
}
var config interface{}
if err := json.Unmarshal(data, &config); err != nil {
t.Fatal(err)
}
}
func TestFetchPsiphonConfigNotRegistered(t *testing.T) {
clnt := newclient()
state := probeservices.State{
// Explicitly empty so the test is more clear
}
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
data, err := clnt.FetchPsiphonConfig(context.Background())
if !errors.Is(err, probeservices.ErrNotRegistered) {
t.Fatal("expected an error here")
}
if data != nil {
t.Fatal("expected nil data here")
}
}

40
internal/engine/probeservices/register.go Normal file
View file

@ -0,0 +1,40 @@
package probeservices
import (
"context"
"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
)
type registerRequest struct {
Metadata
Password string `json:"password"`
}
type registerResult struct {
ClientID string `json:"client_id"`
}
// MaybeRegister registers this client if not already registered
func (c Client) MaybeRegister(ctx context.Context, metadata Metadata) error {
if !metadata.Valid() {
return ErrInvalidMetadata
}
state := c.StateFile.Get()
if state.Credentials() != nil {
return nil // we're already good
}
c.RegisterCalls.Add(1)
pwd := randx.Letters(64)
req := &registerRequest{
Metadata: metadata,
Password: pwd,
}
var resp registerResult
if err := c.Client.PostJSON(ctx, "/api/v1/register", req, &resp); err != nil {
return err
}
state.ClientID = resp.ClientID
state.Password = pwd
return c.StateFile.Set(state)
}

63
internal/engine/probeservices/register_test.go Normal file
View file

@ -0,0 +1,63 @@
package probeservices_test
import (
"context"
"errors"
"strings"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
)
func TestMaybeRegister(t *testing.T) {
t.Run("when metadata is not valid", func(t *testing.T) {
clnt := newclient()
ctx := context.Background()
var metadata probeservices.Metadata
err := clnt.MaybeRegister(ctx, metadata)
if !errors.Is(err, probeservices.ErrInvalidMetadata) {
t.Fatal("expected an error here")
}
})
t.Run("when we have already registered", func(t *testing.T) {
clnt := newclient()
state := probeservices.State{
ClientID: "xx-xxx-x-xxxx",
Password: "xx",
}
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
ctx := context.Background()
metadata := testorchestra.MetadataFixture()
if err := clnt.MaybeRegister(ctx, metadata); err != nil {
t.Fatal(err)
}
})
t.Run("when the API call fails", func(t *testing.T) {
clnt := newclient()
clnt.BaseURL = "\t\t\t" // makes it fail
ctx := context.Background()
metadata := testorchestra.MetadataFixture()
err := clnt.MaybeRegister(ctx, metadata)
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("expected an error here")
}
})
}
func TestMaybeRegisterIdempotent(t *testing.T) {
clnt := newclient()
ctx := context.Background()
metadata := testorchestra.MetadataFixture()
if err := clnt.MaybeRegister(ctx, metadata); err != nil {
t.Fatal(err)
}
if err := clnt.MaybeRegister(ctx, metadata); err != nil {
t.Fatal(err)
}
if clnt.RegisterCalls.Load() != 1 {
t.Fatal("called register API too many times")
}
}

87
internal/engine/probeservices/statefile.go Normal file
View file

@ -0,0 +1,87 @@
package probeservices
import (
"encoding/json"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// State is the state stored inside the state file
type State struct {
ClientID string
Expire time.Time
Password string
Token string
}
// Auth returns an authentication structure, if possible, otherwise
// it returns nil, meaning that you should login again.
func (s State) Auth() *LoginAuth {
if s.Token == "" {
return nil
}
if time.Now().Add(30 * time.Second).After(s.Expire) {
return nil
}
return &LoginAuth{Expire: s.Expire, Token: s.Token}
}
// Credentials returns login credentials, if possible, otherwise it
// returns nil, meaning that you should create an account.
func (s State) Credentials() *LoginCredentials {
if s.ClientID == "" {
return nil
}
if s.Password == "" {
return nil
}
return &LoginCredentials{ClientID: s.ClientID, Password: s.Password}
}
// StateFile is the orchestra state file. It is backed by
// a generic key-value store configured by the user.
type StateFile struct {
Store model.KeyValueStore
key string
}
// NewStateFile creates a new state file backed by a key-value store
func NewStateFile(kvstore model.KeyValueStore) StateFile {
return StateFile{key: "orchestra.state", Store: kvstore}
}
// SetMockable is a mockable version of Set
func (sf StateFile) SetMockable(s State, mf func(interface{}) ([]byte, error)) error {
data, err := mf(s)
if err != nil {
return err
}
return sf.Store.Set(sf.key, data)
}
// Set saves the current state on the key-value store.
func (sf StateFile) Set(s State) error {
return sf.SetMockable(s, json.Marshal)
}
// GetMockable is a mockable version of Get
func (sf StateFile) GetMockable(sfget func(string) ([]byte, error),
unmarshal func([]byte, interface{}) error) (State, error) {
value, err := sfget(sf.key)
if err != nil {
return State{}, err
}
var state State
if err := unmarshal(value, &state); err != nil {
return State{}, err
}
return state, nil
}
// Get returns the current state. In case of any error with the
// underlying key-value store, we return an empty state.
func (sf StateFile) Get() (state State) {
state, _ = sf.GetMockable(sf.Store.Get, json.Unmarshal)
return
}

153
internal/engine/probeservices/statefile_test.go Normal file
View file

@ -0,0 +1,153 @@
package probeservices_test
import (
"encoding/json"
"errors"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
)
func TestStateAuth(t *testing.T) {
t.Run("with no Token", func(t *testing.T) {
state := probeservices.State{Expire: time.Now().Add(10 * time.Hour)}
if state.Auth() != nil {
t.Fatal("expected nil here")
}
})
t.Run("with expired Token", func(t *testing.T) {
state := probeservices.State{
Expire: time.Now().Add(-1 * time.Hour),
Token: "xx-x-xxx-xx",
}
if state.Auth() != nil {
t.Fatal("expected nil here")
}
})
t.Run("with good Token", func(t *testing.T) {
state := probeservices.State{
Expire: time.Now().Add(10 * time.Hour),
Token: "xx-x-xxx-xx",
}
if state.Auth() == nil {
t.Fatal("expected valid auth here")
}
})
}
func TestStateCredentials(t *testing.T) {
t.Run("with no ClientID", func(t *testing.T) {
state := probeservices.State{}
if state.Credentials() != nil {
t.Fatal("expected nil here")
}
})
t.Run("with no Password", func(t *testing.T) {
state := probeservices.State{
ClientID: "xx-x-xxx-xx",
}
if state.Credentials() != nil {
t.Fatal("expected nil here")
}
})
t.Run("with all good", func(t *testing.T) {
state := probeservices.State{
ClientID: "xx-x-xxx-xx",
Password: "xx",
}
if state.Credentials() == nil {
t.Fatal("expected valid auth here")
}
})
}
func TestStateFileMemoryIntegration(t *testing.T) {
// Does the StateFile have the property that we can write
// values into it and then read again the same files?
sf := probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore())
s := probeservices.State{
Expire: time.Now(),
Password: "xy",
Token: "abc",
ClientID: "xx",
}
if err := sf.Set(s); err != nil {
t.Fatal(err)
}
os := sf.Get()
diff := cmp.Diff(s, os)
if diff != "" {
t.Fatal(diff)
}
}
func TestStateFileSetMarshalError(t *testing.T) {
sf := probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore())
s := probeservices.State{
Expire: time.Now(),
Password: "xy",
Token: "abc",
ClientID: "xx",
}
expected := errors.New("mocked error")
failingfunc := func(v interface{}) ([]byte, error) {
return nil, expected
}
if err := sf.SetMockable(s, failingfunc); !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
}
func TestStateFileGetKVStoreGetError(t *testing.T) {
sf := probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore())
expected := errors.New("mocked error")
failingfunc := func(string) ([]byte, error) {
return nil, expected
}
s, err := sf.GetMockable(failingfunc, json.Unmarshal)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if s.ClientID != "" {
t.Fatal("unexpected ClientID field")
}
if !s.Expire.IsZero() {
t.Fatal("unexpected Expire field")
}
if s.Password != "" {
t.Fatal("unexpected Password field")
}
if s.Token != "" {
t.Fatal("unexpected Token field")
}
}
func TestStateFileGetUnmarshalError(t *testing.T) {
sf := probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore())
if err := sf.Set(probeservices.State{}); err != nil {
t.Fatal(err)
}
expected := errors.New("mocked error")
failingfunc := func([]byte, interface{}) error {
return expected
}
s, err := sf.GetMockable(sf.Store.Get, failingfunc)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if s.ClientID != "" {
t.Fatal("unexpected ClientID field")
}
if !s.Expire.IsZero() {
t.Fatal("unexpected Expire field")
}
if s.Password != "" {
t.Fatal("unexpected Password field")
}
if s.Token != "" {
t.Fatal("unexpected Token field")
}
}

19
internal/engine/probeservices/testorchestra/testorchestra.go Normal file
View file

@ -0,0 +1,19 @@
// Package testorchestra helps with testing the OONI orchestra API.
package testorchestra
import "github.com/ooni/probe-cli/v3/internal/engine/probeservices"
// MetadataFixture returns a valid metadata struct. This is mostly
// useful for testing. (We should see if we can make this private.)
func MetadataFixture() probeservices.Metadata {
return probeservices.Metadata{
Platform: "linux",
ProbeASN: "AS15169",
ProbeCC: "US",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
SupportedTests: []string{
"web_connectivity",
},
}
}

24
internal/engine/probeservices/tor.go Normal file
View file

@ -0,0 +1,24 @@
package probeservices
import (
"context"
"fmt"
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// FetchTorTargets returns the targets for the tor experiment.
func (c Client) FetchTorTargets(ctx context.Context, cc string) (result map[string]model.TorTarget, err error) {
_, auth, err := c.GetCredsAndAuth()
if err != nil {
return nil, err
}
client := c.Client
client.Authorization = fmt.Sprintf("Bearer %s", auth.Token)
query := url.Values{}
query.Add("country_code", cc)
err = client.GetJSONWithQuery(
ctx, "/api/v1/test-list/tor-targets", query, &result)
return
}

82
internal/engine/probeservices/tor_test.go Normal file
View file

@ -0,0 +1,82 @@
package probeservices_test
import (
"context"
"net/http"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices/testorchestra"
)
func TestFetchTorTargets(t *testing.T) {
clnt := newclient()
if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
t.Fatal(err)
}
if err := clnt.MaybeLogin(context.Background()); err != nil {
t.Fatal(err)
}
data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
if err != nil {
t.Fatal(err)
}
if data == nil || len(data) <= 0 {
t.Fatal("invalid data")
}
}
func TestFetchTorTargetsNotRegistered(t *testing.T) {
clnt := newclient()
state := probeservices.State{
// Explicitly empty so the test is more clear
}
if err := clnt.StateFile.Set(state); err != nil {
t.Fatal(err)
}
data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
if err == nil {
t.Fatal("expected an error here")
}
if data != nil {
t.Fatal("expected nil data here")
}
}
type FetchTorTargetsHTTPTransport struct {
Response *http.Response
}
func (clnt *FetchTorTargetsHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := http.DefaultTransport.RoundTrip(req)
if err != nil {
return nil, err
}
if req.URL.Path == "/api/v1/test-list/tor-targets" {
clnt.Response = resp
}
return resp, err
}
func TestFetchTorTargetsSetsQueryString(t *testing.T) {
clnt := newclient()
txp := new(FetchTorTargetsHTTPTransport)
clnt.HTTPClient.Transport = txp
if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
t.Fatal(err)
}
if err := clnt.MaybeLogin(context.Background()); err != nil {
t.Fatal(err)
}
data, err := clnt.FetchTorTargets(context.Background(), "ZZ")
if err != nil {
t.Fatal(err)
}
if data == nil || len(data) <= 0 {
t.Fatal("invalid data")
}
requestURL := txp.Response.Request.URL
if requestURL.Query().Get("country_code") != "ZZ" {
t.Fatal("invalid country code")
}
}

36
internal/engine/probeservices/urls.go Normal file
View file

@ -0,0 +1,36 @@
package probeservices
import (
"context"
"fmt"
"net/url"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
type urlListResult struct {
Results []model.URLInfo `json:"results"`
}
// FetchURLList fetches the list of URLs used by WebConnectivity. The config
// argument contains the optional settings. Returns the list of URLs, on success,
// or an explanatory error, in case of failure.
func (c Client) FetchURLList(ctx context.Context, config model.URLListConfig) ([]model.URLInfo, error) {
query := url.Values{}
if config.CountryCode != "" {
query.Set("country_code", config.CountryCode)
}
if config.Limit > 0 {
query.Set("limit", fmt.Sprintf("%d", config.Limit))
}
if len(config.Categories) > 0 {
query.Set("category_codes", strings.Join(config.Categories, ","))
}
var response urlListResult
err := c.Client.GetJSONWithQuery(ctx, "/api/v1/test-list/urls", query, &response)
if err != nil {
return nil, err
}
return response.Results, nil
}

50
internal/engine/probeservices/urls_test.go Normal file
View file

@ -0,0 +1,50 @@
package probeservices_test
import (
"context"
"strings"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
func TestFetchURLListSuccess(t *testing.T) {
client := newclient()
client.BaseURL = "https://ams-pg-test.ooni.org"
config := model.URLListConfig{
Categories: []string{"NEWS", "CULTR"},
CountryCode: "IT",
Limit: 17,
}
ctx := context.Background()
result, err := client.FetchURLList(ctx, config)
if err != nil {
t.Fatal(err)
}
if len(result) != 17 {
t.Fatal("unexpected number of results")
}
for _, entry := range result {
if entry.CategoryCode != "NEWS" && entry.CategoryCode != "CULTR" {
t.Fatalf("unexpected category code: %+v", entry)
}
}
}
func TestFetchURLListFailure(t *testing.T) {
client := newclient()
client.BaseURL = "https://\t\t\t/" // cause test to fail
config := model.URLListConfig{
Categories: []string{"NEWS", "CULTR"},
CountryCode: "IT",
Limit: 17,
}
ctx := context.Background()
result, err := client.FetchURLList(ctx, config)
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("not the error we expected")
}
if len(result) != 0 {
t.Fatal("results?!")
}
}