ooni-probe-cli/internal/oonirun/v2.go

package oonirun

//
// OONI Run v2 implementation
//

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"

	"github.com/hexops/gotextdiff"
	"github.com/hexops/gotextdiff/myers"
	"github.com/hexops/gotextdiff/span"
	"github.com/ooni/probe-cli/v3/internal/atomicx"
	"github.com/ooni/probe-cli/v3/internal/httpx"
	"github.com/ooni/probe-cli/v3/internal/kvstore"
	"github.com/ooni/probe-cli/v3/internal/model"
	"github.com/ooni/probe-cli/v3/internal/runtimex"
)

var (
	// v2CountEmptyNettestNames counts the number of cases in which we have been
	// given an empty nettest name, which is useful for testing.
	v2CountEmptyNettestNames = &atomicx.Int64{}

	// v2CountFailedExperiments countes the number of failed experiments
	// and is useful when testing this package
	v2CountFailedExperiments = &atomicx.Int64{}
)

// V2Descriptor describes a list of nettests to run together.
type V2Descriptor struct {
	// Name is the name of this descriptor.
	Name string `json:"name"`

	// Description contains a long description.
	Description string `json:"description"`

	// Author contains the author's name.
	Author string `json:"author"`

	// Nettests contains the list of nettests to run.
	Nettests []V2Nettest `json:"nettests"`
}

// V2Nettest specifies how a nettest should run.
type V2Nettest struct {
	// Inputs contains inputs for the experiment.
	Inputs []string `json:"inputs"`

	// Options contains the experiment options. Any option name starting with
	// `Safe` will be available for the experiment run, but omitted from
	// the serialized Measurement that the experiment builder will submit
	// to the OONI backend.
	Options map[string]any `json:"options"`

	// TestName contains the nettest name.
	TestName string `json:"test_name"`
}

// ErrHTTPRequestFailed indicates that an HTTP request failed.
var ErrHTTPRequestFailed = errors.New("oonirun: HTTP request failed")

// getV2DescriptorFromHTTPSURL GETs a v2Descriptor instance from
// a static URL (e.g., from a GitHub repo or from a Gist).
func getV2DescriptorFromHTTPSURL(ctx context.Context, client model.HTTPClient,
	logger model.Logger, URL string) (*V2Descriptor, error) {
	template := httpx.APIClientTemplate{
		Accept:        "",
		Authorization: "",
		BaseURL:       URL,
		HTTPClient:    client,
		Host:          "",
		LogBody:       true,
		Logger:        logger,
		UserAgent:     model.HTTPHeaderUserAgent,
	}
	var desc V2Descriptor
	if err := template.Build().GetJSON(ctx, "", &desc); err != nil {
		return nil, err
	}
	return &desc, nil
}

// v2DescriptorCache contains all the known v2Descriptor entries.
type v2DescriptorCache struct {
	// Entries contains all the cached descriptors.
	Entries map[string]*V2Descriptor
}

// v2DescriptorCacheKey is the name of the kvstore2 entry keeping
// information about already known v2Descriptor instances.
const v2DescriptorCacheKey = "oonirun-v2.state"

// v2DescriptorCacheLoad loads the v2DescriptorCache.
func v2DescriptorCacheLoad(fsstore model.KeyValueStore) (*v2DescriptorCache, error) {
	data, err := fsstore.Get(v2DescriptorCacheKey)
	if err != nil {
		if errors.Is(err, kvstore.ErrNoSuchKey) {
			cache := &v2DescriptorCache{
				Entries: make(map[string]*V2Descriptor),
			}
			return cache, nil
		}
		return nil, err
	}
	var cache v2DescriptorCache
	if err := json.Unmarshal(data, &cache); err != nil {
		return nil, err
	}
	if cache.Entries == nil {
		cache.Entries = make(map[string]*V2Descriptor)
	}
	return &cache, nil
}

// PullChangesWithoutSideEffects fetches v2Descriptor changes.
//
// This function DOES NOT change the state of the cache. It just returns to
// the caller what changed for a given entry. It is up-to-the-caller to choose
// what to do in case there are changes depending on the CLI flags.
//
// Arguments:
//
// - ctx is the context for deadline/cancellation;
//
// - client is the HTTPClient to use;
//
// - URL is the URL from which to download/update the OONIRun v2Descriptor.
//
// Return values:
//
// - oldValue is the old v2Descriptor, which may be nil;
//
// - newValue is the new v2Descriptor, which may be nil;
//
// - err is the error that occurred, or nil in case of success.
func (cache *v2DescriptorCache) PullChangesWithoutSideEffects(
	ctx context.Context, client model.HTTPClient, logger model.Logger,
	URL string) (oldValue, newValue *V2Descriptor, err error) {
	oldValue = cache.Entries[URL]
	newValue, err = getV2DescriptorFromHTTPSURL(ctx, client, logger, URL)
	return
}

// Update updates the given cache entry and writes back onto the disk.
//
// Note: this method modifies cache and is not safe for concurrent usage.
func (cache *v2DescriptorCache) Update(
	fsstore model.KeyValueStore, URL string, entry *V2Descriptor) error {
	cache.Entries[URL] = entry
	data, err := json.Marshal(cache)
	runtimex.PanicOnError(err, "json.Marshal failed")
	return fsstore.Set(v2DescriptorCacheKey, data)
}

// ErrNilDescriptor indicates that we have been passed a descriptor that is nil.
var ErrNilDescriptor = errors.New("oonirun: descriptor is nil")

// V2MeasureDescriptor performs the measurement or measurements
// described by the given list of v2Descriptor.
func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descriptor) error {
	if desc == nil {
		// Note: we have a test checking that we can handle a nil
		// descriptor, yet adding also this extra safety net feels
		// more robust in terms of the implementation.
		return ErrNilDescriptor
	}
	logger := config.Session.Logger()
	for _, nettest := range desc.Nettests {
		if nettest.TestName == "" {
			logger.Warn("oonirun: nettest name cannot be empty")
			v2CountEmptyNettestNames.Add(1)
			continue
		}
		exp := &Experiment{
			Annotations:            config.Annotations,
			ExtraOptions:           nettest.Options,
			Inputs:                 nettest.Inputs,
			InputFilePaths:         nil,
			MaxRuntime:             config.MaxRuntime,
			Name:                   nettest.TestName,
			NoCollector:            config.NoCollector,
			NoJSON:                 config.NoJSON,
			Random:                 config.Random,
			ReportFile:             config.ReportFile,
			Session:                config.Session,
			newExperimentBuilderFn: nil,
			newInputLoaderFn:       nil,
			newSubmitterFn:         nil,
			newSaverFn:             nil,
			newInputProcessorFn:    nil,
		}
		if err := exp.Run(ctx); err != nil {
			logger.Warnf("cannot run experiment: %s", err.Error())
			v2CountFailedExperiments.Add(1)
			continue
		}
	}
	return nil
}

// ErrNeedToAcceptChanges indicates that the user needs to accept
// changes (i.e., a new or modified set of descriptors) before
// we can actually run this set of descriptors.
var ErrNeedToAcceptChanges = errors.New("oonirun: need to accept changes")

// v2DescriptorDiff shows what changed between the old and the new descriptors.
func v2DescriptorDiff(oldValue, newValue *V2Descriptor, URL string) string {
	oldData, err := json.MarshalIndent(oldValue, "", "  ")
	runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly")
	newData, err := json.MarshalIndent(newValue, "", "  ")
	runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly")
	oldString, newString := string(oldData)+"\n", string(newData)+"\n"
	oldFile := "OLD " + URL
	newFile := "NEW " + URL
	edits := myers.ComputeEdits(span.URIFromPath(oldFile), oldString, newString)
	return fmt.Sprint(gotextdiff.ToUnified(oldFile, newFile, oldString, edits))
}

// v2MeasureHTTPS performs a measurement using an HTTPS v2 OONI Run URL
// and returns whether performing this measurement failed.
//
// This function maintains an on-disk cache that tracks the status of
// OONI Run v2 links. If there are any changes and the user has not
// provided config.AcceptChanges, this function will log what has changed
// and will return with an ErrNeedToAcceptChanges error.
//
// In such a case, the caller SHOULD print additional information
// explaining how to accept changes and then SHOULD exit 1 or similar.
func v2MeasureHTTPS(ctx context.Context, config *LinkConfig, URL string) error {
	logger := config.Session.Logger()
	logger.Infof("oonirun/v2: running %s", URL)
	cache, err := v2DescriptorCacheLoad(config.KVStore)
	if err != nil {
		return err
	}
	clnt := config.Session.DefaultHTTPClient()
	oldValue, newValue, err := cache.PullChangesWithoutSideEffects(ctx, clnt, logger, URL)
	if err != nil {
		return err
	}
	diff := v2DescriptorDiff(oldValue, newValue, URL)
	if !config.AcceptChanges && diff != "" {
		logger.Warnf("oonirun: %s changed as follows:\n\n%s", URL, diff)
		logger.Warnf("oonirun: we are not going to run this link until you accept changes")
		return ErrNeedToAcceptChanges
	}
	if diff != "" {
		if err := cache.Update(config.KVStore, URL, newValue); err != nil {
			return err
		}
	}
	return V2MeasureDescriptor(ctx, config, newValue) // handles nil newValue gracefully
}