engine/ooapi: autogenerated API with login and caching (#234)

* internal/engine/ooapi: auto-generated API client

* feat: introduce the callers abstraction

* feat: implement API caching on disk

* feat: implement cloneWithToken when we require login

* feat: implement login

* fix: do not cache all APIs

* feat: start making space for more tests

* feat: implement caching policy

* feat: write tests for caching layer

* feat: add integration tests and fix some minor issues

* feat: write much more unit tests

* feat: add some more easy unit tests

* feat: add tests that use a local server

While there, make sure many fields we care about are OK.

* doc: write basic documentation

* fix: tweak sentence

* doc: improve ooapi documentation

* doc(ooapi): other documentation improvements

* fix(ooapi): remove caching for most APIs

We discussed this topic yesterday with @FedericoCeratto. The only
place where we want LRU caching is MeasurementMeta.

* feat(ooapi): improve handling of errors during login

This was also discussed yesterday with @FedericoCeratto

* fix(swaggerdiff_test.go): temporarily disable

Before I work on this, I need to tend onto other tasks.

* fix(ootest): add one more test case

We're going towards 100% coverage of this package, as it ought to be.

* feat(ooapi): test cases for when the probe clock is off

* fix(ooapi): change test to have 100% unittest coverage

* feat: sync server and client APIs definition

Companion PR: https://github.com/ooni/api/pull/218

* fix(ooapi): start testing again against API

* fix(ooapi): only generate each file once

* chore: set version to 3.7.0-alpha

While there, make sure we don't always skip a currently failing
riseupvpn test, and slightly clarify the readme.

* fix(kvstore): less scoped error message

internal/engine/ooapi/apimodel/checkin.go

@@ -0,0 +1,47 @@
+package apimodel
+
+// CheckInRequestWebConnectivity contains WebConnectivity
+// specific parameters to include into CheckInRequest
+type CheckInRequestWebConnectivity struct {
+	CategoryCodes []string `json:"category_codes"`
+}
+
+// CheckInRequest is the check-in API request
+type CheckInRequest struct {
+	Charging        bool                          `json:"charging"`
+	OnWiFi          bool                          `json:"on_wifi"`
+	Platform        string                        `json:"platform"`
+	ProbeASN        string                        `json:"probe_asn"`
+	ProbeCC         string                        `json:"probe_cc"`
+	RunType         string                        `json:"run_type"`
+	SoftwareName    string                        `json:"software_name"`
+	SoftwareVersion string                        `json:"software_version"`
+	WebConnectivity CheckInRequestWebConnectivity `json:"web_connectivity"`
+}
+
+// CheckInResponseURLInfo contains information about an URL.
+type CheckInResponseURLInfo struct {
+	CategoryCode string `json:"category_code"`
+	CountryCode  string `json:"country_code"`
+	URL          string `json:"url"`
+}
+
+// CheckInResponseWebConnectivity contains WebConnectivity
+// specific information of a CheckInResponse
+type CheckInResponseWebConnectivity struct {
+	ReportID string                   `json:"report_id"`
+	URLs     []CheckInResponseURLInfo `json:"urls"`
+}
+
+// CheckInResponse is the check-in API response
+type CheckInResponse struct {
+	ProbeASN string               `json:"probe_asn"`
+	ProbeCC  string               `json:"probe_cc"`
+	Tests    CheckInResponseTests `json:"tests"`
+	V        int64                `json:"v"`
+}
+
+// CheckInResponseTests contains configuration for tests
+type CheckInResponseTests struct {
+	WebConnectivity CheckInResponseWebConnectivity `json:"web_connectivity"`
+}

internal/engine/ooapi/apimodel/checkreportid.go

@@ -0,0 +1,13 @@
+package apimodel
+
+// CheckReportIDRequest is the CheckReportID request.
+type CheckReportIDRequest struct {
+	ReportID string `query:"report_id" required:"true"`
+}
+
+// CheckReportIDResponse is the CheckReportID response.
+type CheckReportIDResponse struct {
+	Error string `json:"error"`
+	Found bool   `json:"found"`
+	V     int64  `json:"v"`
+}

internal/engine/ooapi/apimodel/doc.go

@@ -0,0 +1,22 @@
+// Package apimodel describes the data types used by OONI's API.
+//
+// If you edit this package to integrate the data model, remember to
+// run `go generate ./...`.
+//
+// We annotate fields with tagging. When a field should be sent
+// over as JSON, use the usual `json` tag.
+//
+// When a field needs to be sent using the query string, use
+// the `query` tag instead. We limit what can be sent using the
+// query string to int64, string, and bool.
+//
+// The `path` tag indicates that the URL path contains a
+// template. We will replace the value of this field with
+// the template. Note that the template should use the
+// Go name of the field (e.g. `{{ .ReportID }}`) as opposed
+// to the name in the tag, which is only used when we
+// generate the API Swagger.
+//
+// The `required` tag indicates required fields. A required
+// field cannot be empty (for the Go definition of empty).
+package apimodel

internal/engine/ooapi/apimodel/login.go

@@ -0,0 +1,15 @@
+package apimodel
+
+import "time"
+
+// LoginRequest is the login API request
+type LoginRequest struct {
+	ClientID string `json:"username"`
+	Password string `json:"password"`
+}
+
+// LoginResponse is the login API response
+type LoginResponse struct {
+	Expire time.Time `json:"expire"`
+	Token  string    `json:"token"`
+}

internal/engine/ooapi/apimodel/measurementmeta.go

@@ -0,0 +1,25 @@
+package apimodel
+
+// MeasurementMetaRequest is the MeasurementMeta Request.
+type MeasurementMetaRequest struct {
+	ReportID string `query:"report_id" required:"true"`
+	Full     bool   `query:"full"`
+	Input    string `query:"input"`
+}
+
+// MeasurementMetaResponse is the MeasurementMeta Response.
+type MeasurementMetaResponse struct {
+	Anomaly              bool   `json:"anomaly"`
+	CategoryCode         string `json:"category_code"`
+	Confirmed            bool   `json:"confirmed"`
+	Failure              bool   `json:"failure"`
+	Input                string `json:"input"`
+	MeasurementStartTime string `json:"measurement_start_time"`
+	ProbeASN             int64  `json:"probe_asn"`
+	ProbeCC              string `json:"probe_cc"`
+	RawMeasurement       string `json:"raw_measurement"`
+	ReportID             string `json:"report_id"`
+	Scores               string `json:"scores"`
+	TestName             string `json:"test_name"`
+	TestStartTime        string `json:"test_start_time"`
+}

internal/engine/ooapi/apimodel/openreport.go

@@ -0,0 +1,21 @@
+package apimodel
+
+// OpenReportRequest is the OpenReport request.
+type OpenReportRequest struct {
+	DataFormatVersion string `json:"data_format_version"`
+	Format            string `json:"format"`
+	ProbeASN          string `json:"probe_asn"`
+	ProbeCC           string `json:"probe_cc"`
+	SoftwareName      string `json:"software_name"`
+	SoftwareVersion   string `json:"software_version"`
+	TestName          string `json:"test_name"`
+	TestStartTime     string `json:"test_start_time"`
+	TestVersion       string `json:"test_version"`
+}
+
+// OpenReportResponse is the OpenReport response.
+type OpenReportResponse struct {
+	BackendVersion   string   `json:"backend_version"`
+	ReportID         string   `json:"report_id"`
+	SupportedFormats []string `json:"supported_formats"`
+}

internal/engine/ooapi/apimodel/psiphonconfig.go

@@ -0,0 +1,7 @@
+package apimodel
+
+// PsiphonConfigRequest is the request for the PsiphonConfig API
+type PsiphonConfigRequest struct{}
+
+// PsiphonConfigResponse is the response from the PsiphonConfig API
+type PsiphonConfigResponse map[string]interface{}

internal/engine/ooapi/apimodel/register.go

@@ -0,0 +1,26 @@
+package apimodel
+
+// RegisterRequest is the request for the Register API.
+type RegisterRequest struct {
+	// just password
+	Password string `json:"password"`
+
+	// metadata
+	AvailableBandwidth string   `json:"available_bandwidth,omitempty"`
+	DeviceToken        string   `json:"device_token,omitempty"`
+	Language           string   `json:"language,omitempty"`
+	NetworkType        string   `json:"network_type,omitempty"`
+	Platform           string   `json:"platform"`
+	ProbeASN           string   `json:"probe_asn"`
+	ProbeCC            string   `json:"probe_cc"`
+	ProbeFamily        string   `json:"probe_family,omitempty"`
+	ProbeTimezone      string   `json:"probe_timezone,omitempty"`
+	SoftwareName       string   `json:"software_name"`
+	SoftwareVersion    string   `json:"software_version"`
+	SupportedTests     []string `json:"supported_tests"`
+}
+
+// RegisterResponse is the response from the Register API.
+type RegisterResponse struct {
+	ClientID string `json:"client_id"`
+}

internal/engine/ooapi/apimodel/submitmeasurement.go

@@ -0,0 +1,13 @@
+package apimodel
+
+// SubmitMeasurementRequest is the SubmitMeasurement request.
+type SubmitMeasurementRequest struct {
+	ReportID string      `path:"report_id"`
+	Format   string      `json:"format"`
+	Content  interface{} `json:"content"`
+}
+
+// SubmitMeasurementResponse is the SubmitMeasurement response.
+type SubmitMeasurementResponse struct {
+	MeasurementUID string `json:"measurement_uid"`
+}

internal/engine/ooapi/apimodel/testhelpers.go

@@ -0,0 +1,15 @@
+package apimodel
+
+// TestHelpersRequest is the TestHelpers request.
+type TestHelpersRequest struct{}
+
+// TestHelpersResponse is the TestHelpers response.
+type TestHelpersResponse map[string][]TestHelpersHelperInfo
+
+// TestHelpersHelperInfo is a single helper within the
+// response returned by the TestHelpers API.
+type TestHelpersHelperInfo struct {
+	Address string `json:"address"`
+	Type    string `json:"type"`
+	Front   string `json:"front,omitempty"`
+}

internal/engine/ooapi/apimodel/tortargets.go

@@ -0,0 +1,16 @@
+package apimodel
+
+// TorTargetsRequest is a request for the TorTargets API.
+type TorTargetsRequest struct{}
+
+// TorTargetsResponse is the response from the TorTargets API.
+type TorTargetsResponse map[string]TorTargetsTarget
+
+// TorTargetsTarget is a target for the tor experiment.
+type TorTargetsTarget struct {
+	Address  string              `json:"address"`
+	Name     string              `json:"name"`
+	Params   map[string][]string `json:"params"`
+	Protocol string              `json:"protocol"`
+	Source   string              `json:"source"`
+}

internal/engine/ooapi/apimodel/urls.go

@@ -0,0 +1,26 @@
+package apimodel
+
+// URLsRequest is the URLs request.
+type URLsRequest struct {
+	CategoryCodes string `query:"category_codes"`
+	CountryCode   string `query:"country_code"`
+	Limit         int64  `query:"limit"`
+}
+
+// URLsResponse is the URLs response.
+type URLsResponse struct {
+	Metadata URLsMetadata      `json:"metadata"`
+	Results  []URLsResponseURL `json:"results"`
+}
+
+// URLsMetadata contains metadata in the URLs response.
+type URLsMetadata struct {
+	Count int64 `json:"count"`
+}
+
+// URLsResponseURL is a single URL in the URLs response.
+type URLsResponseURL struct {
+	CategoryCode string `json:"category_code"`
+	CountryCode  string `json:"country_code"`
+	URL          string `json:"url"`
+}