chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
2021-02-02 12:05:47 +01:00
committed by GitHub
parent b1ce300c8d
commit d57c78bc71
535 changed files with 66182 additions and 23 deletions
@@ -0,0 +1,306 @@
// Package sniblocking contains the SNI blocking network experiment.
//
// See https://github.com/ooni/spec/blob/master/nettests/ts-024-sni-blocking.md.
package sniblocking
import (
"context"
"errors"
"fmt"
"math/rand"
"net"
"net/url"
"sync"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
const (
testName = "sni_blocking"
testVersion = "0.3.0"
)
// Config contains the experiment config.
type Config struct {
// ControlSNI is the SNI to be used for the control.
ControlSNI string
// TestHelperAddress is the address of the test helper.
TestHelperAddress string
}
// Subresult contains the keys of a single measurement
// that targets either the target or the control.
type Subresult struct {
urlgetter.TestKeys
Cached bool `json:"-"`
SNI string `json:"sni"`
THAddress string `json:"th_address"`
}
// TestKeys contains sniblocking test keys.
type TestKeys struct {
Control Subresult `json:"control"`
Result string `json:"result"`
Target Subresult `json:"target"`
}
const (
classAnomalyTestHelperUnreachable = "anomaly.test_helper_unreachable"
classAnomalyTimeout = "anomaly.timeout"
classAnomalyUnexpectedFailure = "anomaly.unexpected_failure"
classInterferenceClosed = "interference.closed"
classInterferenceInvalidCertificate = "interference.invalid_certificate"
classInterferenceReset = "interference.reset"
classInterferenceUnknownAuthority = "interference.unknown_authority"
classSuccessGotServerHello = "success.got_server_hello"
)
func (tk *TestKeys) classify() string {
if tk.Target.Failure == nil {
return classSuccessGotServerHello
}
switch *tk.Target.Failure {
case errorx.FailureConnectionRefused:
return classAnomalyTestHelperUnreachable
case errorx.FailureConnectionReset:
return classInterferenceReset
case errorx.FailureDNSNXDOMAINError:
return classAnomalyTestHelperUnreachable
case errorx.FailureEOFError:
return classInterferenceClosed
case errorx.FailureGenericTimeoutError:
if tk.Control.Failure != nil {
return classAnomalyTestHelperUnreachable
}
return classAnomalyTimeout
case errorx.FailureSSLInvalidCertificate:
return classInterferenceInvalidCertificate
case errorx.FailureSSLInvalidHostname:
return classSuccessGotServerHello
case errorx.FailureSSLUnknownAuthority:
return classInterferenceUnknownAuthority
}
return classAnomalyUnexpectedFailure
}
// Measurer performs the measurement.
type Measurer struct {
cache map[string]Subresult
config Config
mu sync.Mutex
}
// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
func (m *Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
func (m *Measurer) ExperimentVersion() string {
return testVersion
}
func (m *Measurer) measureone(
ctx context.Context,
sess model.ExperimentSession,
beginning time.Time,
sni string,
thaddr string,
) Subresult {
// slightly delay the measurement
gen := rand.New(rand.NewSource(time.Now().UnixNano()))
sleeptime := time.Duration(gen.Intn(250)) * time.Millisecond
select {
case <-time.After(sleeptime):
case <-ctx.Done():
s := errorx.FailureInterrupted
failedop := errorx.TopLevelOperation
return Subresult{
TestKeys: urlgetter.TestKeys{
FailedOperation: &failedop,
Failure: &s,
},
THAddress: thaddr,
SNI: sni,
}
}
// perform the measurement
g := urlgetter.Getter{
Begin: beginning,
Config: urlgetter.Config{TLSServerName: sni},
Session: sess,
Target: fmt.Sprintf("tlshandshake://%s", thaddr),
}
// Ignoring the error because g.Get() sets the tk.Failure field
// to be the OONI equivalent of the error that occurred.
tk, _ := g.Get(ctx)
// assemble and publish the results
smk := Subresult{
SNI: sni,
THAddress: thaddr,
TestKeys: tk,
}
return smk
}
func (m *Measurer) measureonewithcache(
ctx context.Context,
output chan<- Subresult,
sess model.ExperimentSession,
beginning time.Time,
sni string,
thaddr string,
) {
cachekey := sni + thaddr
m.mu.Lock()
smk, okay := m.cache[cachekey]
m.mu.Unlock()
if okay {
output <- smk
return
}
smk = m.measureone(ctx, sess, beginning, sni, thaddr)
output <- smk
smk.Cached = true
m.mu.Lock()
m.cache[cachekey] = smk
m.mu.Unlock()
}
func (m *Measurer) startall(
ctx context.Context, sess model.ExperimentSession,
measurement *model.Measurement, inputs []string,
) <-chan Subresult {
outputs := make(chan Subresult, len(inputs))
for _, input := range inputs {
go m.measureonewithcache(
ctx, outputs, sess,
measurement.MeasurementStartTimeSaved,
input, m.config.TestHelperAddress,
)
}
return outputs
}
func processall(
outputs <-chan Subresult,
measurement *model.Measurement,
callbacks model.ExperimentCallbacks,
inputs []string,
sess model.ExperimentSession,
controlSNI string,
) *TestKeys {
var (
current int
testkeys = new(TestKeys)
)
for smk := range outputs {
if smk.SNI == controlSNI {
testkeys.Control = smk
} else if smk.SNI == string(measurement.Input) {
testkeys.Target = smk
} else {
panic("unexpected smk.SNI")
}
current++
sess.Logger().Debugf(
"sni_blocking: %s: %s [cached: %+v]", smk.SNI,
asString(smk.Failure), smk.Cached)
if current >= len(inputs) {
break
}
}
testkeys.Result = testkeys.classify()
sess.Logger().Infof("sni_blocking: result: %s", testkeys.Result)
return testkeys
}
// maybeURLToSNI handles the case where the input is from the test-lists
// and hence every input is a URL rather than a domain.
func maybeURLToSNI(input model.MeasurementTarget) (model.MeasurementTarget, error) {
parsed, err := url.Parse(string(input))
if err != nil {
return "", err
}
if parsed.Path == string(input) {
return input, nil
}
return model.MeasurementTarget(parsed.Hostname()), nil
}
// Run implements ExperimentMeasurer.Run.
func (m *Measurer) Run(
ctx context.Context,
sess model.ExperimentSession,
measurement *model.Measurement,
callbacks model.ExperimentCallbacks,
) error {
m.mu.Lock()
if m.cache == nil {
m.cache = make(map[string]Subresult)
}
m.mu.Unlock()
if m.config.ControlSNI == "" {
m.config.ControlSNI = "example.org"
}
if measurement.Input == "" {
return errors.New("Experiment requires measurement.Input")
}
if m.config.TestHelperAddress == "" {
m.config.TestHelperAddress = net.JoinHostPort(
m.config.ControlSNI, "443",
)
}
urlgetter.RegisterExtensions(measurement)
// TODO(bassosimone): if the user has configured DoT or DoH, here we
// probably want to perform the name resolution before the measurements
// or to make sure that the classify logic is robust to that.
//
// See https://github.com/ooni/probe-cli/v3/internal/engine/issues/392.
maybeParsed, err := maybeURLToSNI(measurement.Input)
if err != nil {
return err
}
measurement.Input = maybeParsed
inputs := []string{m.config.ControlSNI}
if string(measurement.Input) != m.config.ControlSNI {
inputs = append(inputs, string(measurement.Input))
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second*time.Duration(len(inputs)))
defer cancel()
outputs := m.startall(ctx, sess, measurement, inputs)
measurement.TestKeys = processall(
outputs, measurement, callbacks, inputs, sess, m.config.ControlSNI,
)
return nil
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return &Measurer{config: config}
}
func asString(failure *string) (result string) {
result = "success"
if failure != nil {
result = *failure
}
return
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with probe-cli
// therefore we should be careful when changing it.
type SummaryKeys struct {
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
return SummaryKeys{IsAnomaly: false}, nil
}
@@ -0,0 +1,434 @@
package sniblocking
import (
"context"
"strings"
"testing"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
const (
softwareName = "ooniprobe-example"
softwareVersion = "0.0.1"
)
func TestTestKeysClassify(t *testing.T) {
asStringPtr := func(s string) *string {
return &s
}
t.Run("with tk.Target.Failure == nil", func(t *testing.T) {
tk := new(TestKeys)
if tk.classify() != classSuccessGotServerHello {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == connection_refused", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureConnectionRefused)
if tk.classify() != classAnomalyTestHelperUnreachable {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == dns_nxdomain_error", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureDNSNXDOMAINError)
if tk.classify() != classAnomalyTestHelperUnreachable {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == connection_reset", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureConnectionReset)
if tk.classify() != classInterferenceReset {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == eof_error", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureEOFError)
if tk.classify() != classInterferenceClosed {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == ssl_invalid_hostname", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureSSLInvalidHostname)
if tk.classify() != classSuccessGotServerHello {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == ssl_unknown_authority", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureSSLUnknownAuthority)
if tk.classify() != classInterferenceUnknownAuthority {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == ssl_invalid_certificate", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureSSLInvalidCertificate)
if tk.classify() != classInterferenceInvalidCertificate {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == generic_timeout_error #1", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureGenericTimeoutError)
if tk.classify() != classAnomalyTimeout {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == generic_timeout_error #2", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr(errorx.FailureGenericTimeoutError)
tk.Control.Failure = asStringPtr(errorx.FailureGenericTimeoutError)
if tk.classify() != classAnomalyTestHelperUnreachable {
t.Fatal("unexpected result")
}
})
t.Run("with tk.Target.Failure == unknown_failure", func(t *testing.T) {
tk := new(TestKeys)
tk.Target.Failure = asStringPtr("unknown_failure")
if tk.classify() != classAnomalyUnexpectedFailure {
t.Fatal("unexpected result")
}
})
}
func TestNewExperimentMeasurer(t *testing.T) {
measurer := NewExperimentMeasurer(Config{})
if measurer.ExperimentName() != "sni_blocking" {
t.Fatal("unexpected name")
}
if measurer.ExperimentVersion() != "0.3.0" {
t.Fatal("unexpected version")
}
}
func TestMeasurerMeasureNoMeasurementInput(t *testing.T) {
measurer := NewExperimentMeasurer(Config{
ControlSNI: "example.com",
})
err := measurer.Run(
context.Background(),
newsession(),
new(model.Measurement),
model.NewPrinterCallbacks(log.Log),
)
if err.Error() != "Experiment requires measurement.Input" {
t.Fatal("not the error we expected")
}
}
func TestMeasurerMeasureWithInvalidInput(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // immediately cancel the context
measurer := NewExperimentMeasurer(Config{
ControlSNI: "example.com",
})
measurement := &model.Measurement{
Input: "\t",
}
err := measurer.Run(
ctx,
newsession(),
measurement,
model.NewPrinterCallbacks(log.Log),
)
if err == nil {
t.Fatal("expected an error here")
}
}
func TestMeasurerMeasureWithCancelledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // immediately cancel the context
measurer := NewExperimentMeasurer(Config{
ControlSNI: "example.com",
})
measurement := &model.Measurement{
Input: "kernel.org",
}
err := measurer.Run(
ctx,
newsession(),
measurement,
model.NewPrinterCallbacks(log.Log),
)
if err != nil {
t.Fatal(err)
}
sk, err := measurer.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
if _, ok := sk.(SummaryKeys); !ok {
t.Fatal("invalid type for summary keys")
}
}
func TestMeasureoneCancelledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // immediately cancel the context
result := new(Measurer).measureone(
ctx,
&mockable.Session{MockableLogger: log.Log},
time.Now(),
"kernel.org",
"example.com:443",
)
if result.Agent != "" {
t.Fatal("not the expected Agent")
}
if result.BootstrapTime != 0.0 {
t.Fatal("not the expected BootstrapTime")
}
if result.DNSCache != nil {
t.Fatal("not the expected DNSCache")
}
if result.FailedOperation == nil || *result.FailedOperation != errorx.TopLevelOperation {
t.Fatal("not the expected FailedOperation")
}
if result.Failure == nil || *result.Failure != errorx.FailureInterrupted {
t.Fatal("not the expected failure")
}
if result.NetworkEvents != nil {
t.Fatal("not the expected NetworkEvents")
}
if result.Queries != nil {
t.Fatal("not the expected Queries")
}
if result.Requests != nil {
t.Fatal("not the expected Requests")
}
if result.SOCKSProxy != "" {
t.Fatal("not the expected SOCKSProxy")
}
if result.TCPConnect != nil {
t.Fatal("not the expected TCPConnect")
}
if result.TLSHandshakes != nil {
t.Fatal("not the expected TLSHandshakes")
}
if result.Tunnel != "" {
t.Fatal("not the expected Tunnel")
}
if result.SNI != "kernel.org" {
t.Fatal("unexpected SNI")
}
if result.THAddress != "example.com:443" {
t.Fatal("unexpected THAddress")
}
}
func TestMeasureoneWithPreMeasurementFailure(t *testing.T) {
result := new(Measurer).measureone(
context.Background(),
&mockable.Session{MockableLogger: log.Log},
time.Now(),
"kernel.org",
"example.com:443\t\t\t", // cause URL parse error
)
if result.Agent != "redirect" {
t.Fatal("not the expected Agent")
}
if result.BootstrapTime != 0.0 {
t.Fatal("not the expected BootstrapTime")
}
if result.DNSCache != nil {
t.Fatal("not the expected DNSCache")
}
if result.FailedOperation == nil || *result.FailedOperation != "top_level" {
t.Fatal("not the expected FailedOperation")
}
if result.Failure == nil || !strings.Contains(*result.Failure, "invalid target URL") {
t.Fatal("not the expected failure")
}
if result.NetworkEvents != nil {
t.Fatal("not the expected NetworkEvents")
}
if result.Queries != nil {
t.Fatal("not the expected Queries")
}
if result.Requests != nil {
t.Fatal("not the expected Requests")
}
if result.SOCKSProxy != "" {
t.Fatal("not the expected SOCKSProxy")
}
if result.TCPConnect != nil {
t.Fatal("not the expected TCPConnect")
}
if result.TLSHandshakes != nil {
t.Fatal("not the expected TLSHandshakes")
}
if result.Tunnel != "" {
t.Fatal("not the expected Tunnel")
}
if result.SNI != "kernel.org" {
t.Fatal("unexpected SNI")
}
if result.THAddress != "example.com:443\t\t\t" {
t.Fatal("unexpected THAddress")
}
}
func TestMeasureoneSuccess(t *testing.T) {
result := new(Measurer).measureone(
context.Background(),
&mockable.Session{MockableLogger: log.Log},
time.Now(),
"kernel.org",
"example.com:443",
)
if result.Agent != "redirect" {
t.Fatal("not the expected Agent")
}
if result.BootstrapTime != 0.0 {
t.Fatal("not the expected BootstrapTime")
}
if result.DNSCache != nil {
t.Fatal("not the expected DNSCache")
}
if result.FailedOperation == nil || *result.FailedOperation != errorx.TLSHandshakeOperation {
t.Fatal("not the expected FailedOperation")
}
if result.Failure == nil || *result.Failure != errorx.FailureSSLInvalidHostname {
t.Fatal("unexpected failure")
}
if len(result.NetworkEvents) < 1 {
t.Fatal("not the expected NetworkEvents")
}
if len(result.Queries) < 1 {
t.Fatal("not the expected Queries")
}
if result.Requests != nil {
t.Fatal("not the expected Requests")
}
if result.SOCKSProxy != "" {
t.Fatal("not the expected SOCKSProxy")
}
if len(result.TCPConnect) < 1 {
t.Fatal("not the expected TCPConnect")
}
if len(result.TLSHandshakes) < 1 {
t.Fatal("not the expected TLSHandshakes")
}
if result.Tunnel != "" {
t.Fatal("not the expected Tunnel")
}
if result.SNI != "kernel.org" {
t.Fatal("unexpected SNI")
}
if result.THAddress != "example.com:443" {
t.Fatal("unexpected THAddress")
}
}
func TestMeasureonewithcacheWorks(t *testing.T) {
measurer := &Measurer{cache: make(map[string]Subresult)}
output := make(chan Subresult, 2)
for i := 0; i < 2; i++ {
measurer.measureonewithcache(
context.Background(),
output,
&mockable.Session{MockableLogger: log.Log},
time.Now(),
"kernel.org",
"example.com:443",
)
}
for _, expected := range []bool{false, true} {
result := <-output
if result.Cached != expected {
t.Fatal("unexpected cached")
}
if *result.Failure != errorx.FailureSSLInvalidHostname {
t.Fatal("unexpected failure")
}
if result.SNI != "kernel.org" {
t.Fatal("unexpected SNI")
}
}
}
func TestProcessallPanicsIfInvalidSNI(t *testing.T) {
defer func() {
panicdata := recover()
if panicdata == nil {
t.Fatal("expected to see panic here")
}
if panicdata.(string) != "unexpected smk.SNI" {
t.Fatal("not the panic we expected")
}
}()
outputs := make(chan Subresult, 1)
measurement := &model.Measurement{
Input: "kernel.org",
}
go func() {
outputs <- Subresult{
SNI: "antani.io",
}
}()
processall(
outputs,
measurement,
model.NewPrinterCallbacks(log.Log),
[]string{"kernel.org", "example.com"},
newsession(),
"example.com",
)
}
func TestMaybeURLToSNI(t *testing.T) {
t.Run("for invalid URL", func(t *testing.T) {
parsed, err := maybeURLToSNI("\t")
if err == nil {
t.Fatal("expected an error here")
}
if parsed != "" {
t.Fatal("expected empty parsed here")
}
})
t.Run("for domain name", func(t *testing.T) {
parsed, err := maybeURLToSNI("kernel.org")
if err != nil {
t.Fatal(err)
}
if parsed != "kernel.org" {
t.Fatal("expected different domain here")
}
})
t.Run("for valid URL", func(t *testing.T) {
parsed, err := maybeURLToSNI("https://kernel.org/robots.txt")
if err != nil {
t.Fatal(err)
}
if parsed != "kernel.org" {
t.Fatal("expected different domain here")
}
})
}
func newsession() model.ExperimentSession {
return &mockable.Session{MockableLogger: log.Log}
}
func TestSummaryKeysGeneric(t *testing.T) {
measurement := &model.Measurement{TestKeys: &TestKeys{}}
m := &Measurer{}
osk, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
sk := osk.(SummaryKeys)
if sk.IsAnomaly {
t.Fatal("invalid isAnomaly")
}
}