feat(oonirun): add support for OONIRun v2 links (#844)
This diff adds support for OONIRun v2 links. Part of https://github.com/ooni/probe/issues/2184.
This commit is contained in:
parent
ebb78c2848
commit
9a0153a349
1
go.mod
1
go.mod
|
@ -17,6 +17,7 @@ require (
|
||||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
|
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
|
||||||
github.com/google/uuid v1.3.0
|
github.com/google/uuid v1.3.0
|
||||||
github.com/gorilla/websocket v1.5.0
|
github.com/gorilla/websocket v1.5.0
|
||||||
|
github.com/hexops/gotextdiff v1.0.3
|
||||||
github.com/iancoleman/strcase v0.2.0
|
github.com/iancoleman/strcase v0.2.0
|
||||||
github.com/lucas-clemente/quic-go v0.27.0
|
github.com/lucas-clemente/quic-go v0.27.0
|
||||||
github.com/mattn/go-colorable v0.1.12
|
github.com/mattn/go-colorable v0.1.12
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -378,6 +378,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
|
||||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||||
|
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||||
|
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||||
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
|
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
|
||||||
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
|
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
|
||||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||||
|
|
|
@ -421,6 +421,11 @@ func ooniRunMain(ctx context.Context,
|
||||||
for _, URL := range currentOptions.Inputs {
|
for _, URL := range currentOptions.Inputs {
|
||||||
r := oonirun.NewLinkRunner(cfg, URL)
|
r := oonirun.NewLinkRunner(cfg, URL)
|
||||||
if err := r.Run(ctx); err != nil {
|
if err := r.Run(ctx); err != nil {
|
||||||
|
if errors.Is(err, oonirun.ErrNeedToAcceptChanges) {
|
||||||
|
logger.Warnf("oonirun: to accept these changes, rerun adding `-y` to the command line")
|
||||||
|
logger.Warnf("oonirun: we'll show this error every time the upstream link changes")
|
||||||
|
panic("oonirun: need to accept changes using `-y`")
|
||||||
|
}
|
||||||
logger.Warnf("oonirun: running link failed: %s", err.Error())
|
logger.Warnf("oonirun: running link failed: %s", err.Error())
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,6 +27,8 @@ type Session struct {
|
||||||
MockableFetchTorTargetsErr error
|
MockableFetchTorTargetsErr error
|
||||||
MockableFetchURLListResult []model.OOAPIURLInfo
|
MockableFetchURLListResult []model.OOAPIURLInfo
|
||||||
MockableFetchURLListErr error
|
MockableFetchURLListErr error
|
||||||
|
MockableCheckInInfo *model.OOAPICheckInInfo
|
||||||
|
MockableCheckInErr error
|
||||||
MockableResolverIP string
|
MockableResolverIP string
|
||||||
MockableSoftwareName string
|
MockableSoftwareName string
|
||||||
MockableSoftwareVersion string
|
MockableSoftwareVersion string
|
||||||
|
|
|
@ -148,21 +148,19 @@ func (c *apiClient) newRequestWithJSONBody(
|
||||||
return request, nil
|
return request, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// joinURLPath appends the path of resource URL to the baseURL taking
|
// joinURLPath appends resourcePath to the urlPath.
|
||||||
// care of multiple forward slashes gracefully.
|
func (c *apiClient) joinURLPath(urlPath, resourcePath string) string {
|
||||||
func (c *apiClient) joinURLPath(origPath string, newPath string) string {
|
if resourcePath == "" {
|
||||||
|
if urlPath == "" {
|
||||||
// If the BaseURL path doesn't end with a slash, added one
|
return "/"
|
||||||
if !strings.HasSuffix(origPath, "/") {
|
|
||||||
origPath += "/"
|
|
||||||
}
|
}
|
||||||
|
return urlPath
|
||||||
// If the resourceURL path has a leading slash, it is removed
|
|
||||||
if strings.HasPrefix(newPath, "/") {
|
|
||||||
newPath = newPath[1:]
|
|
||||||
}
|
}
|
||||||
|
if !strings.HasSuffix(urlPath, "/") {
|
||||||
return origPath + newPath
|
urlPath += "/"
|
||||||
|
}
|
||||||
|
resourcePath = strings.TrimPrefix(resourcePath, "/")
|
||||||
|
return urlPath + resourcePath
|
||||||
}
|
}
|
||||||
|
|
||||||
// newRequest creates a new request.
|
// newRequest creates a new request.
|
||||||
|
|
|
@ -94,12 +94,27 @@ func newAPIClient() *apiClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestJoinURLPath(t *testing.T) {
|
func TestJoinURLPath(t *testing.T) {
|
||||||
|
t.Run("the whole path is inside basePath and there's no resource path", func(t *testing.T) {
|
||||||
|
ac := newAPIClient()
|
||||||
|
ac.BaseURL = "https://example.com/robots.txt"
|
||||||
|
req, err := ac.newRequest(context.Background(), "GET", "", nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if req.URL.String() != "https://example.com/robots.txt" {
|
||||||
|
t.Fatal("unexpected result", req.URL.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("empty baseURL path and slash-prefixed resource path", func(t *testing.T) {
|
t.Run("empty baseURL path and slash-prefixed resource path", func(t *testing.T) {
|
||||||
ac := newAPIClient()
|
ac := newAPIClient()
|
||||||
ac.BaseURL = "https://example.com"
|
ac.BaseURL = "https://example.com"
|
||||||
req, err := ac.newRequest(context.Background(), "GET", "/foo", nil, nil)
|
req, err := ac.newRequest(context.Background(), "GET", "/foo", nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if req.URL.String() != "https://example.com/foo" {
|
if req.URL.String() != "https://example.com/foo" {
|
||||||
t.Fatal("unexpected result", err)
|
t.Fatal("unexpected result", req.URL.String())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -107,8 +122,11 @@ func TestJoinURLPath(t *testing.T) {
|
||||||
ac := newAPIClient()
|
ac := newAPIClient()
|
||||||
ac.BaseURL = "https://example.com/"
|
ac.BaseURL = "https://example.com/"
|
||||||
req, err := ac.newRequest(context.Background(), "GET", "/foo", nil, nil)
|
req, err := ac.newRequest(context.Background(), "GET", "/foo", nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if req.URL.String() != "https://example.com/foo" {
|
if req.URL.String() != "https://example.com/foo" {
|
||||||
t.Fatal("unexpected result", err)
|
t.Fatal("unexpected result", req.URL.String())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -116,8 +134,11 @@ func TestJoinURLPath(t *testing.T) {
|
||||||
ac := newAPIClient()
|
ac := newAPIClient()
|
||||||
ac.BaseURL = "https://example.com"
|
ac.BaseURL = "https://example.com"
|
||||||
req, err := ac.newRequest(context.Background(), "GET", "", nil, nil)
|
req, err := ac.newRequest(context.Background(), "GET", "", nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if req.URL.String() != "https://example.com/" {
|
if req.URL.String() != "https://example.com/" {
|
||||||
t.Fatal("unexpected result", err)
|
t.Fatal("unexpected result", req.URL.String())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -125,8 +146,11 @@ func TestJoinURLPath(t *testing.T) {
|
||||||
ac := newAPIClient()
|
ac := newAPIClient()
|
||||||
ac.BaseURL = "http://example.com/foo"
|
ac.BaseURL = "http://example.com/foo"
|
||||||
req, err := ac.newRequest(context.Background(), "GET", "/bar", nil, nil)
|
req, err := ac.newRequest(context.Background(), "GET", "/bar", nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if req.URL.String() != "http://example.com/foo/bar" {
|
if req.URL.String() != "http://example.com/foo/bar" {
|
||||||
t.Fatal("unexpected result", err)
|
t.Fatal("unexpected result", req.URL.String())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -134,8 +158,11 @@ func TestJoinURLPath(t *testing.T) {
|
||||||
ac := newAPIClient()
|
ac := newAPIClient()
|
||||||
ac.BaseURL = "http://example.com/foo/"
|
ac.BaseURL = "http://example.com/foo/"
|
||||||
req, err := ac.newRequest(context.Background(), "GET", "/bar", nil, nil)
|
req, err := ac.newRequest(context.Background(), "GET", "/bar", nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if req.URL.String() != "http://example.com/foo/bar" {
|
if req.URL.String() != "http://example.com/foo/bar" {
|
||||||
t.Fatal("unexpected result", err)
|
t.Fatal("unexpected result", req.URL.String())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -143,8 +170,11 @@ func TestJoinURLPath(t *testing.T) {
|
||||||
ac := newAPIClient()
|
ac := newAPIClient()
|
||||||
ac.BaseURL = "http://example.com/foo/"
|
ac.BaseURL = "http://example.com/foo/"
|
||||||
req, err := ac.newRequest(context.Background(), "GET", "bar", nil, nil)
|
req, err := ac.newRequest(context.Background(), "GET", "bar", nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if req.URL.String() != "http://example.com/foo/bar" {
|
if req.URL.String() != "http://example.com/foo/bar" {
|
||||||
t.Fatal("unexpected result", err)
|
t.Fatal("unexpected result", req.URL.String())
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
20
internal/model/mocks/keyvaluestore.go
Normal file
20
internal/model/mocks/keyvaluestore.go
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
package mocks
|
||||||
|
|
||||||
|
import "github.com/ooni/probe-cli/v3/internal/model"
|
||||||
|
|
||||||
|
// KeyValueStore allows mocking model.KeyValueStore.
|
||||||
|
type KeyValueStore struct {
|
||||||
|
MockGet func(key string) (value []byte, err error)
|
||||||
|
|
||||||
|
MockSet func(key string, value []byte) (err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ model.KeyValueStore = &KeyValueStore{}
|
||||||
|
|
||||||
|
func (kvs *KeyValueStore) Get(key string) (value []byte, err error) {
|
||||||
|
return kvs.MockGet(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (kvs *KeyValueStore) Set(key string, value []byte) (err error) {
|
||||||
|
return kvs.MockSet(key, value)
|
||||||
|
}
|
37
internal/model/mocks/keyvaluestore_test.go
Normal file
37
internal/model/mocks/keyvaluestore_test.go
Normal file
|
@ -0,0 +1,37 @@
|
||||||
|
package mocks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestKeyValueStore(t *testing.T) {
|
||||||
|
t.Run("Get", func(t *testing.T) {
|
||||||
|
expect := errors.New("mocked error")
|
||||||
|
kvs := &KeyValueStore{
|
||||||
|
MockGet: func(key string) (value []byte, err error) {
|
||||||
|
return nil, expect
|
||||||
|
},
|
||||||
|
}
|
||||||
|
out, err := kvs.Get("antani")
|
||||||
|
if !errors.Is(err, expect) {
|
||||||
|
t.Fatal("unexpected err", err)
|
||||||
|
}
|
||||||
|
if out != nil {
|
||||||
|
t.Fatal("unexpected out")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("Set", func(t *testing.T) {
|
||||||
|
expect := errors.New("mocked error")
|
||||||
|
kvs := &KeyValueStore{
|
||||||
|
MockSet: func(key string, value []byte) (err error) {
|
||||||
|
return expect
|
||||||
|
},
|
||||||
|
}
|
||||||
|
err := kvs.Set("antani", nil)
|
||||||
|
if !errors.Is(err, expect) {
|
||||||
|
t.Fatal("unexpected err", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
|
@ -84,9 +84,7 @@ func NewLinkRunner(c *LinkConfig, URL string) LinkRunner {
|
||||||
case strings.HasPrefix(URL, "ooni://nettest"):
|
case strings.HasPrefix(URL, "ooni://nettest"):
|
||||||
out.f = v1Measure
|
out.f = v1Measure
|
||||||
default:
|
default:
|
||||||
// TODO(bassosimone): this panic will go away when we merge
|
out.f = v2MeasureHTTPS
|
||||||
// the next patch which will implement v2.
|
|
||||||
panic("unsupported OONI Run link")
|
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
244
internal/oonirun/v2.go
Normal file
244
internal/oonirun/v2.go
Normal file
|
@ -0,0 +1,244 @@
|
||||||
|
package oonirun
|
||||||
|
|
||||||
|
//
|
||||||
|
// OONI Run v2 implementation
|
||||||
|
//
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/hexops/gotextdiff"
|
||||||
|
"github.com/hexops/gotextdiff/myers"
|
||||||
|
"github.com/hexops/gotextdiff/span"
|
||||||
|
"github.com/ooni/probe-cli/v3/internal/atomicx"
|
||||||
|
"github.com/ooni/probe-cli/v3/internal/httpx"
|
||||||
|
"github.com/ooni/probe-cli/v3/internal/kvstore"
|
||||||
|
"github.com/ooni/probe-cli/v3/internal/model"
|
||||||
|
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// v2CountEmptyNettestNames counts the number of cases in which we have been
|
||||||
|
// given an empty nettest name, which is useful for testing.
|
||||||
|
v2CountEmptyNettestNames = &atomicx.Int64{}
|
||||||
|
)
|
||||||
|
|
||||||
|
// v2Descriptor describes a single nettest to run.
|
||||||
|
type v2Descriptor struct {
|
||||||
|
// Name is the name of this descriptor.
|
||||||
|
Name string `json:"name"`
|
||||||
|
|
||||||
|
// Description contains a long description.
|
||||||
|
Description string `json:"description"`
|
||||||
|
|
||||||
|
// Author contains the author's name.
|
||||||
|
Author string `json:"author"`
|
||||||
|
|
||||||
|
// Nettests contains the list of nettests to run.
|
||||||
|
Nettests []v2Nettest `json:"nettests"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// v2Nettest specifies how a nettest should run.
|
||||||
|
type v2Nettest struct {
|
||||||
|
// Inputs contains inputs for the experiment.
|
||||||
|
Inputs []string `json:"inputs"`
|
||||||
|
|
||||||
|
// Options contains the experiment options.
|
||||||
|
Options map[string]any `json:"options"`
|
||||||
|
|
||||||
|
// TestName contains the nettest name.
|
||||||
|
TestName string `json:"test_name"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrHTTPRequestFailed indicates that an HTTP request failed.
|
||||||
|
var ErrHTTPRequestFailed = errors.New("oonirun: HTTP request failed")
|
||||||
|
|
||||||
|
// getV2DescriptorFromHTTPSURL GETs a v2Descriptor instance from
|
||||||
|
// a static URL (e.g., from a GitHub repo or from a Gist).
|
||||||
|
func getV2DescriptorFromHTTPSURL(ctx context.Context, client model.HTTPClient,
|
||||||
|
logger model.Logger, URL string) (*v2Descriptor, error) {
|
||||||
|
template := httpx.APIClientTemplate{
|
||||||
|
Accept: "",
|
||||||
|
Authorization: "",
|
||||||
|
BaseURL: URL,
|
||||||
|
HTTPClient: client,
|
||||||
|
Host: "",
|
||||||
|
LogBody: true,
|
||||||
|
Logger: logger,
|
||||||
|
UserAgent: model.HTTPHeaderUserAgent,
|
||||||
|
}
|
||||||
|
var desc v2Descriptor
|
||||||
|
if err := template.Build().GetJSON(ctx, "", &desc); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &desc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// v2DescriptorCache contains all the known v2Descriptor entries.
|
||||||
|
type v2DescriptorCache struct {
|
||||||
|
// Entries contains all the cached descriptors.
|
||||||
|
Entries map[string]*v2Descriptor
|
||||||
|
}
|
||||||
|
|
||||||
|
// v2DescriptorCacheKey is the name of the kvstore2 entry keeping
|
||||||
|
// information about already known v2Descriptor instances.
|
||||||
|
const v2DescriptorCacheKey = "oonirun-v2.state"
|
||||||
|
|
||||||
|
// v2DescriptorCacheLoad loads the v2DescriptorCache.
|
||||||
|
func v2DescriptorCacheLoad(fsstore model.KeyValueStore) (*v2DescriptorCache, error) {
|
||||||
|
data, err := fsstore.Get(v2DescriptorCacheKey)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, kvstore.ErrNoSuchKey) {
|
||||||
|
cache := &v2DescriptorCache{
|
||||||
|
Entries: make(map[string]*v2Descriptor),
|
||||||
|
}
|
||||||
|
return cache, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var cache v2DescriptorCache
|
||||||
|
if err := json.Unmarshal(data, &cache); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if cache.Entries == nil {
|
||||||
|
cache.Entries = make(map[string]*v2Descriptor)
|
||||||
|
}
|
||||||
|
return &cache, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PullChangesWithoutSideEffects fetches v2Descriptor changes.
|
||||||
|
//
|
||||||
|
// This function DOES NOT change the state of the cache. It just returns to
|
||||||
|
// the caller what changed for a given entry. It is up-to-the-caller to choose
|
||||||
|
// what to do in case there are changes depending on the CLI flags.
|
||||||
|
//
|
||||||
|
// Arguments:
|
||||||
|
//
|
||||||
|
// - ctx is the context for deadline/cancellation;
|
||||||
|
//
|
||||||
|
// - client is the HTTPClient to use;
|
||||||
|
//
|
||||||
|
// - URL is the URL from which to download/update the OONIRun v2Descriptor.
|
||||||
|
//
|
||||||
|
// Return values:
|
||||||
|
//
|
||||||
|
// - oldValue is the old v2Descriptor, which may be nil;
|
||||||
|
//
|
||||||
|
// - newValue is the new v2Descriptor, which may be nil;
|
||||||
|
//
|
||||||
|
// - err is the error that occurred, or nil in case of success.
|
||||||
|
func (cache *v2DescriptorCache) PullChangesWithoutSideEffects(
|
||||||
|
ctx context.Context, client model.HTTPClient, logger model.Logger,
|
||||||
|
URL string) (oldValue, newValue *v2Descriptor, err error) {
|
||||||
|
oldValue = cache.Entries[URL]
|
||||||
|
newValue, err = getV2DescriptorFromHTTPSURL(ctx, client, logger, URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update updates the given cache entry and writes back onto the disk.
|
||||||
|
//
|
||||||
|
// Note: this method modifies cache and is not safe for concurrent usage.
|
||||||
|
func (cache *v2DescriptorCache) Update(
|
||||||
|
fsstore model.KeyValueStore, URL string, entry *v2Descriptor) error {
|
||||||
|
cache.Entries[URL] = entry
|
||||||
|
data, err := json.Marshal(cache)
|
||||||
|
runtimex.PanicOnError(err, "json.Marshal failed")
|
||||||
|
return fsstore.Set(v2DescriptorCacheKey, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrNilDescriptor indicates that we have been passed a descriptor that is nil.
|
||||||
|
var ErrNilDescriptor = errors.New("oonirun: descriptor is nil")
|
||||||
|
|
||||||
|
// v2MeasureDescriptor performs the measurement or measurements
|
||||||
|
// described by the given list of v2Descriptor.
|
||||||
|
func v2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *v2Descriptor) error {
|
||||||
|
if desc == nil {
|
||||||
|
// Note: we have a test checking that we can handle a nil
|
||||||
|
// descriptor, yet adding also this extra safety net feels
|
||||||
|
// more robust in terms of the implementation.
|
||||||
|
return ErrNilDescriptor
|
||||||
|
}
|
||||||
|
logger := config.Session.Logger()
|
||||||
|
for _, nettest := range desc.Nettests {
|
||||||
|
if nettest.TestName == "" {
|
||||||
|
logger.Warn("oonirun: nettest name cannot be empty")
|
||||||
|
v2CountEmptyNettestNames.Add(1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
exp := &Experiment{
|
||||||
|
Annotations: config.Annotations,
|
||||||
|
ExtraOptions: nettest.Options,
|
||||||
|
Inputs: nettest.Inputs,
|
||||||
|
InputFilePaths: nil,
|
||||||
|
MaxRuntime: config.MaxRuntime,
|
||||||
|
Name: nettest.TestName,
|
||||||
|
NoCollector: config.NoCollector,
|
||||||
|
NoJSON: config.NoJSON,
|
||||||
|
Random: config.Random,
|
||||||
|
ReportFile: config.ReportFile,
|
||||||
|
Session: config.Session,
|
||||||
|
}
|
||||||
|
if err := exp.Run(ctx); err != nil {
|
||||||
|
logger.Warnf("cannot run experiment: %s", err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrNeedToAcceptChanges indicates that the user needs to accept
|
||||||
|
// changes (i.e., a new or modified set of descriptors) before
|
||||||
|
// we can actually run this set of descriptors.
|
||||||
|
var ErrNeedToAcceptChanges = errors.New("oonirun: need to accept changes")
|
||||||
|
|
||||||
|
// v2DescriptorDiff shows what changed between the old and the new descriptors.
|
||||||
|
func v2DescriptorDiff(oldValue, newValue *v2Descriptor, URL string) string {
|
||||||
|
oldData, err := json.MarshalIndent(oldValue, "", " ")
|
||||||
|
runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly")
|
||||||
|
newData, err := json.MarshalIndent(newValue, "", " ")
|
||||||
|
runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly")
|
||||||
|
oldString, newString := string(oldData)+"\n", string(newData)+"\n"
|
||||||
|
oldFile := "OLD " + URL
|
||||||
|
newFile := "NEW " + URL
|
||||||
|
edits := myers.ComputeEdits(span.URIFromPath(oldFile), oldString, newString)
|
||||||
|
return fmt.Sprint(gotextdiff.ToUnified(oldFile, newFile, oldString, edits))
|
||||||
|
}
|
||||||
|
|
||||||
|
// v2MeasureHTTPS performs a measurement using an HTTPS v2 OONI Run URL
|
||||||
|
// and returns whether performing this measurement failed.
|
||||||
|
//
|
||||||
|
// This function maintains an on-disk cache that tracks the status of
|
||||||
|
// OONI Run v2 links. If there are any changes and the user has not
|
||||||
|
// provided config.AcceptChanges, this function will log what has changed
|
||||||
|
// and will return with an ErrNeedToAcceptChanges error.
|
||||||
|
//
|
||||||
|
// In such a case, the caller SHOULD print additional information
|
||||||
|
// explaining how to accept changes and then SHOULD exit 1 or similar.
|
||||||
|
func v2MeasureHTTPS(ctx context.Context, config *LinkConfig, URL string) error {
|
||||||
|
logger := config.Session.Logger()
|
||||||
|
logger.Infof("oonirun/v2: running %s", URL)
|
||||||
|
cache, err := v2DescriptorCacheLoad(config.KVStore)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
clnt := config.Session.DefaultHTTPClient()
|
||||||
|
oldValue, newValue, err := cache.PullChangesWithoutSideEffects(ctx, clnt, logger, URL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
diff := v2DescriptorDiff(oldValue, newValue, URL)
|
||||||
|
if !config.AcceptChanges && diff != "" {
|
||||||
|
logger.Warnf("oonirun: %s changed as follows:\n\n%s", URL, diff)
|
||||||
|
logger.Warnf("oonirun: we are not going to run this link until you accept changes")
|
||||||
|
return ErrNeedToAcceptChanges
|
||||||
|
}
|
||||||
|
if diff != "" {
|
||||||
|
if err := cache.Update(config.KVStore, URL, newValue); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return v2MeasureDescriptor(ctx, config, newValue) // handles nil newValue gracefully
|
||||||
|
}
|
285
internal/oonirun/v2_test.go
Normal file
285
internal/oonirun/v2_test.go
Normal file
|
@ -0,0 +1,285 @@
|
||||||
|
package oonirun
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/ooni/probe-cli/v3/internal/kvstore"
|
||||||
|
"github.com/ooni/probe-cli/v3/internal/model/mocks"
|
||||||
|
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TODO(bassosimone): it would be cool to write unit tests. However, to do that
|
||||||
|
// we need to ~redesign the engine package for unit-testability.
|
||||||
|
|
||||||
|
func TestOONIRunV2LinkCommonCase(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
descriptor := &v2Descriptor{
|
||||||
|
Name: "",
|
||||||
|
Description: "",
|
||||||
|
Author: "",
|
||||||
|
Nettests: []v2Nettest{{
|
||||||
|
Inputs: []string{},
|
||||||
|
Options: map[string]any{
|
||||||
|
"SleepTime": int64(10 * time.Millisecond),
|
||||||
|
},
|
||||||
|
TestName: "example",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(descriptor)
|
||||||
|
runtimex.PanicOnError(err, "json.Marshal failed")
|
||||||
|
w.Write(data)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
ctx := context.Background()
|
||||||
|
config := &LinkConfig{
|
||||||
|
AcceptChanges: true, // avoid "oonirun: need to accept changes" error
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"platform": "linux",
|
||||||
|
},
|
||||||
|
KVStore: &kvstore.Memory{},
|
||||||
|
MaxRuntime: 0,
|
||||||
|
NoCollector: true,
|
||||||
|
NoJSON: true,
|
||||||
|
Random: false,
|
||||||
|
ReportFile: "",
|
||||||
|
Session: newSession(ctx, t),
|
||||||
|
}
|
||||||
|
r := NewLinkRunner(config, server.URL)
|
||||||
|
if err := r.Run(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOONIRunV2LinkCannotUpdateCache(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
descriptor := &v2Descriptor{
|
||||||
|
Name: "",
|
||||||
|
Description: "",
|
||||||
|
Author: "",
|
||||||
|
Nettests: []v2Nettest{{
|
||||||
|
Inputs: []string{},
|
||||||
|
Options: map[string]any{
|
||||||
|
"SleepTime": int64(10 * time.Millisecond),
|
||||||
|
},
|
||||||
|
TestName: "example",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(descriptor)
|
||||||
|
runtimex.PanicOnError(err, "json.Marshal failed")
|
||||||
|
w.Write(data)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
ctx := context.Background()
|
||||||
|
expected := errors.New("mocked")
|
||||||
|
config := &LinkConfig{
|
||||||
|
AcceptChanges: true, // avoid "oonirun: need to accept changes" error
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"platform": "linux",
|
||||||
|
},
|
||||||
|
KVStore: &mocks.KeyValueStore{
|
||||||
|
MockGet: func(key string) ([]byte, error) {
|
||||||
|
return []byte("{}"), nil
|
||||||
|
},
|
||||||
|
MockSet: func(key string, value []byte) error {
|
||||||
|
return expected
|
||||||
|
},
|
||||||
|
},
|
||||||
|
MaxRuntime: 0,
|
||||||
|
NoCollector: true,
|
||||||
|
NoJSON: true,
|
||||||
|
Random: false,
|
||||||
|
ReportFile: "",
|
||||||
|
Session: newSession(ctx, t),
|
||||||
|
}
|
||||||
|
r := NewLinkRunner(config, server.URL)
|
||||||
|
err := r.Run(ctx)
|
||||||
|
if !errors.Is(err, expected) {
|
||||||
|
t.Fatal("unexpected err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOONIRunV2LinkWithoutAcceptChanges(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
descriptor := &v2Descriptor{
|
||||||
|
Name: "",
|
||||||
|
Description: "",
|
||||||
|
Author: "",
|
||||||
|
Nettests: []v2Nettest{{
|
||||||
|
Inputs: []string{},
|
||||||
|
Options: map[string]any{
|
||||||
|
"SleepTime": int64(10 * time.Millisecond),
|
||||||
|
},
|
||||||
|
TestName: "example",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(descriptor)
|
||||||
|
runtimex.PanicOnError(err, "json.Marshal failed")
|
||||||
|
w.Write(data)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
ctx := context.Background()
|
||||||
|
config := &LinkConfig{
|
||||||
|
AcceptChanges: false, // should see "oonirun: need to accept changes" error
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"platform": "linux",
|
||||||
|
},
|
||||||
|
KVStore: &kvstore.Memory{},
|
||||||
|
MaxRuntime: 0,
|
||||||
|
NoCollector: true,
|
||||||
|
NoJSON: true,
|
||||||
|
Random: false,
|
||||||
|
ReportFile: "",
|
||||||
|
Session: newSession(ctx, t),
|
||||||
|
}
|
||||||
|
r := NewLinkRunner(config, server.URL)
|
||||||
|
err := r.Run(ctx)
|
||||||
|
if !errors.Is(err, ErrNeedToAcceptChanges) {
|
||||||
|
t.Fatal("unexpected err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOONIRunV2LinkNilDescriptor(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Write([]byte("null"))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
ctx := context.Background()
|
||||||
|
config := &LinkConfig{
|
||||||
|
AcceptChanges: true, // avoid "oonirun: need to accept changes" error
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"platform": "linux",
|
||||||
|
},
|
||||||
|
KVStore: &kvstore.Memory{},
|
||||||
|
MaxRuntime: 0,
|
||||||
|
NoCollector: true,
|
||||||
|
NoJSON: true,
|
||||||
|
Random: false,
|
||||||
|
ReportFile: "",
|
||||||
|
Session: newSession(ctx, t),
|
||||||
|
}
|
||||||
|
r := NewLinkRunner(config, server.URL)
|
||||||
|
if err := r.Run(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOONIRunV2LinkEmptyTestName(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
descriptor := &v2Descriptor{
|
||||||
|
Name: "",
|
||||||
|
Description: "",
|
||||||
|
Author: "",
|
||||||
|
Nettests: []v2Nettest{{
|
||||||
|
Inputs: []string{},
|
||||||
|
Options: map[string]any{
|
||||||
|
"SleepTime": int64(10 * time.Millisecond),
|
||||||
|
},
|
||||||
|
TestName: "", // empty!
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(descriptor)
|
||||||
|
runtimex.PanicOnError(err, "json.Marshal failed")
|
||||||
|
w.Write(data)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
ctx := context.Background()
|
||||||
|
config := &LinkConfig{
|
||||||
|
AcceptChanges: true, // avoid "oonirun: need to accept changes" error
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"platform": "linux",
|
||||||
|
},
|
||||||
|
KVStore: &kvstore.Memory{},
|
||||||
|
MaxRuntime: 0,
|
||||||
|
NoCollector: true,
|
||||||
|
NoJSON: true,
|
||||||
|
Random: false,
|
||||||
|
ReportFile: "",
|
||||||
|
Session: newSession(ctx, t),
|
||||||
|
}
|
||||||
|
r := NewLinkRunner(config, server.URL)
|
||||||
|
if err := r.Run(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v2CountEmptyNettestNames.Load() != 1 {
|
||||||
|
t.Fatal("expected to see 1 instance of empty nettest names")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestV2MeasureDescriptor(t *testing.T) {
|
||||||
|
t.Run("with nil descriptor", func(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
config := &LinkConfig{}
|
||||||
|
err := v2MeasureDescriptor(ctx, config, nil)
|
||||||
|
if !errors.Is(err, ErrNilDescriptor) {
|
||||||
|
t.Fatal("unexpected err", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestV2MeasureHTTPS(t *testing.T) {
|
||||||
|
t.Run("when we cannot load from cache", func(t *testing.T) {
|
||||||
|
expected := errors.New("mocked error")
|
||||||
|
ctx := context.Background()
|
||||||
|
config := &LinkConfig{
|
||||||
|
AcceptChanges: false,
|
||||||
|
Annotations: map[string]string{},
|
||||||
|
KVStore: &mocks.KeyValueStore{
|
||||||
|
MockGet: func(key string) (value []byte, err error) {
|
||||||
|
return nil, expected
|
||||||
|
},
|
||||||
|
},
|
||||||
|
MaxRuntime: 0,
|
||||||
|
NoCollector: false,
|
||||||
|
NoJSON: false,
|
||||||
|
Random: false,
|
||||||
|
ReportFile: "",
|
||||||
|
Session: newSession(ctx, t),
|
||||||
|
}
|
||||||
|
err := v2MeasureHTTPS(ctx, config, "")
|
||||||
|
if !errors.Is(err, expected) {
|
||||||
|
t.Fatal("unexpected err", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("when we cannot pull changes", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel() // fail immediately
|
||||||
|
config := &LinkConfig{
|
||||||
|
AcceptChanges: false,
|
||||||
|
Annotations: map[string]string{},
|
||||||
|
KVStore: &kvstore.Memory{},
|
||||||
|
MaxRuntime: 0,
|
||||||
|
NoCollector: false,
|
||||||
|
NoJSON: false,
|
||||||
|
Random: false,
|
||||||
|
ReportFile: "",
|
||||||
|
Session: newSession(ctx, t),
|
||||||
|
}
|
||||||
|
err := v2MeasureHTTPS(ctx, config, "https://example.com") // should not use URL
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal("unexpected err", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestV2DescriptorCacheLoad(t *testing.T) {
|
||||||
|
t.Run("cannot unmarshal cache content", func(t *testing.T) {
|
||||||
|
fsstore := &kvstore.Memory{}
|
||||||
|
if err := fsstore.Set(v2DescriptorCacheKey, []byte("{")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cache, err := v2DescriptorCacheLoad(fsstore)
|
||||||
|
if err == nil || err.Error() != "unexpected end of JSON input" {
|
||||||
|
t.Fatal("unexpected err", err)
|
||||||
|
}
|
||||||
|
if cache != nil {
|
||||||
|
t.Fatal("expected nil cache")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user