ooni-probe-cli/internal/model/measurement.go
Simone Basso 110a11828b
refactor: spin geoipx off geolocate (#893)
A bunch of packages (including oohelperd) just need the ability to
use MaxMind-like databases. They don't need the additional functionality
implemented by the geolocate package. Such a package, in fact, is
mostly (if not only) needed by the engine package.

Therefore, move code to query MaxMind-like databases to a separate
package, and avoid depending on geolocate in all the packages for
which it's sufficient to use geoipx.

Part of https://github.com/ooni/probe/issues/2240
2022-08-28 20:00:25 +02:00

207 lines
6.5 KiB
Go

package model
//
// Definition of the result of a network measurement.
//
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net"
"time"
)
const (
// DefaultProbeASN is the default probe ASN as a number.
DefaultProbeASN uint = 0
// DefaultProbeCC is the default probe CC.
DefaultProbeCC = "ZZ"
// DefaultProbeIP is the default probe IP.
DefaultProbeIP = "127.0.0.1"
// DefaultProbeNetworkName is the default probe network name.
DefaultProbeNetworkName = ""
// DefaultResolverASN is the default resolver ASN.
DefaultResolverASN uint = 0
// DefaultResolverIP is the default resolver IP.
DefaultResolverIP = "127.0.0.2"
// DefaultResolverNetworkName is the default resolver network name.
DefaultResolverNetworkName = ""
)
var (
// DefaultProbeASNString is the default probe ASN as a string.
DefaultProbeASNString = fmt.Sprintf("AS%d", DefaultProbeASN)
// DefaultResolverASNString is the default resolver ASN as a string.
DefaultResolverASNString = fmt.Sprintf("AS%d", DefaultResolverASN)
)
// MeasurementTarget is the target of a OONI measurement.
type MeasurementTarget string
// MarshalJSON serializes the MeasurementTarget.
func (t MeasurementTarget) MarshalJSON() ([]byte, error) {
if t == "" {
return json.Marshal(nil)
}
return json.Marshal(string(t))
}
// Measurement is a OONI measurement.
//
// This structure is compatible with the definition of the base data format in
// https://github.com/ooni/spec/blob/master/data-formats/df-000-base.md.
type Measurement struct {
// Annotations contains results annotations
Annotations map[string]string `json:"annotations,omitempty"`
// DataFormatVersion is the version of the data format
DataFormatVersion string `json:"data_format_version"`
// Extensions contains information about the extensions included
// into the test_keys of this measurement.
Extensions map[string]int64 `json:"extensions,omitempty"`
// ID is the locally generated measurement ID
ID string `json:"id,omitempty"`
// Input is the measurement input
Input MeasurementTarget `json:"input"`
// InputHashes contains input hashes
InputHashes []string `json:"input_hashes,omitempty"`
// MeasurementStartTime is the time when the measurement started
MeasurementStartTime string `json:"measurement_start_time"`
// MeasurementStartTimeSaved is the moment in time when we
// started the measurement. This is not included into the JSON
// and is only used within the ./internal pkg as a "zero" time.
MeasurementStartTimeSaved time.Time `json:"-"`
// Options contains command line options
Options []string `json:"options,omitempty"`
// ProbeASN contains the probe autonomous system number
ProbeASN string `json:"probe_asn"`
// ProbeCC contains the probe country code
ProbeCC string `json:"probe_cc"`
// ProbeCity contains the probe city
ProbeCity string `json:"probe_city,omitempty"`
// ProbeIP contains the probe IP
ProbeIP string `json:"probe_ip,omitempty"`
// ProbeNetworkName contains the probe network name
ProbeNetworkName string `json:"probe_network_name"`
// ReportID contains the report ID
ReportID string `json:"report_id"`
// ResolverASN is the ASN of the resolver
ResolverASN string `json:"resolver_asn"`
// ResolverIP is the resolver IP
ResolverIP string `json:"resolver_ip"`
// ResolverNetworkName is the network name of the resolver.
ResolverNetworkName string `json:"resolver_network_name"`
// SoftwareName contains the software name
SoftwareName string `json:"software_name"`
// SoftwareVersion contains the software version
SoftwareVersion string `json:"software_version"`
// TestHelpers contains the test helpers. It seems this structure is more
// complex than we would like. In particular, using a map from string to
// string does not fit into the web_connectivity use case. Hence, for now
// we're going to represent this using interface{}. In going forward we
// may probably want to have more uniform test helpers.
TestHelpers map[string]interface{} `json:"test_helpers,omitempty"`
// TestKeys contains the real test result. This field is opaque because
// each experiment will insert here a different structure.
TestKeys interface{} `json:"test_keys"`
// TestName contains the test name
TestName string `json:"test_name"`
// MeasurementRuntime contains the measurement runtime. The JSON name
// is test_runtime because this is the name expected by the OONI backend
// even though that name is clearly a misleading one.
MeasurementRuntime float64 `json:"test_runtime"`
// TestStartTime contains the test start time
TestStartTime string `json:"test_start_time"`
// TestVersion contains the test version
TestVersion string `json:"test_version"`
}
// AddAnnotations adds the annotations from input to m.Annotations.
func (m *Measurement) AddAnnotations(input map[string]string) {
for key, value := range input {
m.AddAnnotation(key, value)
}
}
// AddAnnotation adds a single annotations to m.Annotations.
func (m *Measurement) AddAnnotation(key, value string) {
if m.Annotations == nil {
m.Annotations = make(map[string]string)
}
m.Annotations[key] = value
}
// ErrInvalidProbeIP indicates that we're dealing with a string that
// is not the valid serialization of an IP address.
var ErrInvalidProbeIP = errors.New("model: invalid probe IP")
// Scrub scrubs the probeIP out of the measurement.
func (m *Measurement) Scrub(probeIP string) (err error) {
// We now behave like we can share everything except the
// probe IP, which we instead cannot ever share
m.ProbeIP = DefaultProbeIP
return m.MaybeRewriteTestKeys(probeIP, json.Marshal)
}
// Scrubbed is the string that replaces IP addresses.
const Scrubbed = `[scrubbed]`
// MaybeRewriteTestKeys is the function called by Scrub that
// ensures that m's serialization doesn't include the IP
func (m *Measurement) MaybeRewriteTestKeys(
currentIP string, marshal func(interface{}) ([]byte, error)) error {
if net.ParseIP(currentIP) == nil {
return ErrInvalidProbeIP
}
data, err := marshal(m.TestKeys)
if err != nil {
return err
}
// The check using Count is to save an unnecessary copy performed by
// ReplaceAll when there are no matches into the body. This is what
// we would like the common case to be, meaning that the code has done
// its job correctly and has not leaked the IP.
bpip := []byte(currentIP)
if bytes.Count(data, bpip) <= 0 {
return nil
}
data = bytes.ReplaceAll(data, bpip, []byte(Scrubbed))
// We add an annotation such that hopefully later we can measure the
// number of cases where we failed to sanitize properly.
m.AddAnnotation("_probe_engine_sanitize_test_keys", "true")
return json.Unmarshal(data, &m.TestKeys)
}