chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
2021-02-02 12:05:47 +01:00
committed by GitHub
parent b1ce300c8d
commit d57c78bc71
535 changed files with 66182 additions and 23 deletions
@@ -0,0 +1,174 @@
// Package telegram contains the Telegram network experiment.
//
// See https://github.com/ooni/spec/blob/master/nettests/ts-020-telegram.md.
package telegram
import (
"context"
"errors"
"strings"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
const (
testName = "telegram"
testVersion = "0.2.0"
)
// Config contains the telegram experiment config.
type Config struct{}
// TestKeys contains telegram test keys.
type TestKeys struct {
urlgetter.TestKeys
TelegramHTTPBlocking bool `json:"telegram_http_blocking"`
TelegramTCPBlocking bool `json:"telegram_tcp_blocking"`
TelegramWebFailure *string `json:"telegram_web_failure"`
TelegramWebStatus string `json:"telegram_web_status"`
}
// NewTestKeys creates new telegram TestKeys.
func NewTestKeys() *TestKeys {
return &TestKeys{
TelegramHTTPBlocking: true,
TelegramTCPBlocking: true,
TelegramWebFailure: nil,
TelegramWebStatus: "ok",
}
}
// Update updates the TestKeys using the given MultiOutput result.
func (tk *TestKeys) Update(v urlgetter.MultiOutput) {
// update the easy to update entries first
tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
tk.Queries = append(tk.Queries, v.TestKeys.Queries...)
tk.Requests = append(tk.Requests, v.TestKeys.Requests...)
tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
tk.TLSHandshakes = append(tk.TLSHandshakes, v.TestKeys.TLSHandshakes...)
// then process access points
if v.Input.Config.Method != "GET" {
if v.TestKeys.Failure == nil {
tk.TelegramHTTPBlocking = false
tk.TelegramTCPBlocking = false
return // found successful access point connection
}
if v.TestKeys.FailedOperation == nil || *v.TestKeys.FailedOperation != errorx.ConnectOperation {
tk.TelegramTCPBlocking = false
}
return
}
// now take care of web
if tk.TelegramWebStatus != "ok" {
return // we already flipped the state
}
if v.TestKeys.Failure != nil {
tk.TelegramWebStatus = "blocked"
tk.TelegramWebFailure = v.TestKeys.Failure
return
}
title := `<title>Telegram Web</title>`
if strings.Contains(v.TestKeys.HTTPResponseBody, title) == false {
failureString := "telegram_missing_title_error"
tk.TelegramWebFailure = &failureString
tk.TelegramWebStatus = "blocked"
return
}
return
}
// Measurer performs the measurement
type Measurer struct {
// Config contains the experiment settings. If empty we
// will be using default settings.
Config Config
// Getter is an optional getter to be used for testing.
Getter urlgetter.MultiGetter
}
// ExperimentName implements ExperimentMeasurer.ExperimentName
func (m Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
func (m Measurer) ExperimentVersion() string {
return testVersion
}
// Run implements ExperimentMeasurer.Run
func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
urlgetter.RegisterExtensions(measurement)
inputs := []urlgetter.MultiInput{
{Target: "http://149.154.175.50/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.167.51/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.175.100/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.167.91/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.171.5/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.175.50:443/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.167.51:443/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.175.100:443/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.167.91:443/", Config: urlgetter.Config{Method: "POST"}},
{Target: "http://149.154.171.5:443/", Config: urlgetter.Config{Method: "POST"}},
// Here we need to provide the method explicitly. See
// https://github.com/ooni/probe-cli/v3/internal/engine/issues/827.
{Target: "http://web.telegram.org/", Config: urlgetter.Config{
Method: "GET",
FailOnHTTPError: true,
}},
{Target: "https://web.telegram.org/", Config: urlgetter.Config{
Method: "GET",
FailOnHTTPError: true,
}},
}
multi := urlgetter.Multi{Begin: time.Now(), Getter: m.Getter, Session: sess}
testkeys := NewTestKeys()
testkeys.Agent = "redirect"
measurement.TestKeys = testkeys
for entry := range multi.Collect(ctx, inputs, "telegram", callbacks) {
testkeys.Update(entry)
}
return nil
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return Measurer{Config: config}
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with probe-cli
// therefore we should be careful when changing it.
type SummaryKeys struct {
HTTPBlocking bool `json:"telegram_http_blocking"`
TCPBlocking bool `json:"telegram_tcp_blocking"`
WebBlocking bool `json:"telegram_web_blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{IsAnomaly: false}
tk, ok := measurement.TestKeys.(*TestKeys)
if !ok {
return sk, errors.New("invalid test keys type")
}
tcpBlocking := tk.TelegramTCPBlocking
httpBlocking := tk.TelegramHTTPBlocking
webBlocking := tk.TelegramWebFailure != nil
sk.TCPBlocking = tcpBlocking
sk.HTTPBlocking = httpBlocking
sk.WebBlocking = webBlocking
sk.IsAnomaly = webBlocking || httpBlocking || tcpBlocking
return sk, nil
}
@@ -0,0 +1,416 @@
package telegram_test
import (
"context"
"fmt"
"io"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
func TestNewExperimentMeasurer(t *testing.T) {
measurer := telegram.NewExperimentMeasurer(telegram.Config{})
if measurer.ExperimentName() != "telegram" {
t.Fatal("unexpected name")
}
if measurer.ExperimentVersion() != "0.2.0" {
t.Fatal("unexpected version")
}
}
func TestGood(t *testing.T) {
measurer := telegram.NewExperimentMeasurer(telegram.Config{})
measurement := new(model.Measurement)
err := measurer.Run(
context.Background(),
&mockable.Session{
MockableLogger: log.Log,
},
measurement,
model.NewPrinterCallbacks(log.Log),
)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*telegram.TestKeys)
if tk.Agent != "redirect" {
t.Fatal("unexpected Agent")
}
if tk.FailedOperation != nil {
t.Fatal("unexpected FailedOperation")
}
if tk.Failure != nil {
t.Fatal("unexpected Failure")
}
if len(tk.NetworkEvents) <= 0 {
t.Fatal("no NetworkEvents?!")
}
if len(tk.Queries) <= 0 {
t.Fatal("no Queries?!")
}
if len(tk.Requests) <= 0 {
t.Fatal("no Requests?!")
}
if len(tk.TCPConnect) <= 0 {
t.Fatal("no TCPConnect?!")
}
if len(tk.TLSHandshakes) <= 0 {
t.Fatal("no TLSHandshakes?!")
}
if tk.TelegramHTTPBlocking != false {
t.Fatal("unexpected TelegramHTTPBlocking")
}
if tk.TelegramTCPBlocking != false {
t.Fatal("unexpected TelegramTCPBlocking")
}
if tk.TelegramWebFailure != nil {
t.Fatal("unexpected TelegramWebFailure")
}
if tk.TelegramWebStatus != "ok" {
t.Fatal("unexpected TelegramWebStatus")
}
sk, err := measurer.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
if _, ok := sk.(telegram.SummaryKeys); !ok {
t.Fatal("invalid type for summary keys")
}
}
func TestUpdateWithNoAccessPointsBlocking(t *testing.T) {
tk := telegram.NewTestKeys()
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "POST"},
Target: "http://149.154.175.50/",
},
TestKeys: urlgetter.TestKeys{
Failure: (func() *string {
s := errorx.FailureEOFError
return &s
})(),
},
})
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "POST"},
Target: "http://149.154.175.50:443/",
},
TestKeys: urlgetter.TestKeys{
Failure: nil, // this should be enough to declare success
},
})
if tk.TelegramHTTPBlocking == true {
t.Fatal("there should be no TelegramHTTPBlocking")
}
if tk.TelegramTCPBlocking == true {
t.Fatal("there should be no TelegramTCPBlocking")
}
}
func TestUpdateWithNilFailedOperation(t *testing.T) {
tk := telegram.NewTestKeys()
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "POST"},
Target: "http://149.154.175.50/",
},
TestKeys: urlgetter.TestKeys{
Failure: (func() *string {
s := errorx.FailureEOFError
return &s
})(),
},
})
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "POST"},
Target: "http://149.154.175.50:443/",
},
TestKeys: urlgetter.TestKeys{
Failure: (func() *string {
s := errorx.FailureEOFError
return &s
})(),
},
})
if tk.TelegramHTTPBlocking == false {
t.Fatal("there should be TelegramHTTPBlocking")
}
if tk.TelegramTCPBlocking == true {
t.Fatal("there should be no TelegramTCPBlocking")
}
}
func TestUpdateWithNonConnectFailedOperation(t *testing.T) {
tk := telegram.NewTestKeys()
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "POST"},
Target: "http://149.154.175.50/",
},
TestKeys: urlgetter.TestKeys{
FailedOperation: (func() *string {
s := errorx.ConnectOperation
return &s
})(),
Failure: (func() *string {
s := errorx.FailureEOFError
return &s
})(),
},
})
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "POST"},
Target: "http://149.154.175.50:443/",
},
TestKeys: urlgetter.TestKeys{
FailedOperation: (func() *string {
s := errorx.HTTPRoundTripOperation
return &s
})(),
Failure: (func() *string {
s := errorx.FailureEOFError
return &s
})(),
},
})
if tk.TelegramHTTPBlocking == false {
t.Fatal("there should be TelegramHTTPBlocking")
}
if tk.TelegramTCPBlocking == true {
t.Fatal("there should be no TelegramTCPBlocking")
}
}
func TestUpdateWithAllConnectsFailed(t *testing.T) {
tk := telegram.NewTestKeys()
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "POST"},
Target: "http://149.154.175.50/",
},
TestKeys: urlgetter.TestKeys{
FailedOperation: (func() *string {
s := errorx.ConnectOperation
return &s
})(),
Failure: (func() *string {
s := errorx.FailureEOFError
return &s
})(),
},
})
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "POST"},
Target: "http://149.154.175.50:443/",
},
TestKeys: urlgetter.TestKeys{
FailedOperation: (func() *string {
s := errorx.ConnectOperation
return &s
})(),
Failure: (func() *string {
s := errorx.FailureEOFError
return &s
})(),
},
})
if tk.TelegramHTTPBlocking == false {
t.Fatal("there should be TelegramHTTPBlocking")
}
if tk.TelegramTCPBlocking == false {
t.Fatal("there should be TelegramTCPBlocking")
}
}
func TestUpdateWebWithMixedResults(t *testing.T) {
tk := telegram.NewTestKeys()
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "GET"},
Target: "http://web.telegram.org/",
},
TestKeys: urlgetter.TestKeys{
FailedOperation: (func() *string {
s := errorx.HTTPRoundTripOperation
return &s
})(),
Failure: (func() *string {
s := errorx.FailureEOFError
return &s
})(),
},
})
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "GET"},
Target: "https://web.telegram.org/",
},
TestKeys: urlgetter.TestKeys{
HTTPResponseBody: `<title>Telegram Web</title>`,
HTTPResponseStatus: 200,
},
})
if tk.TelegramWebStatus != "blocked" {
t.Fatal("TelegramWebStatus should be blocked")
}
if *tk.TelegramWebFailure != errorx.FailureEOFError {
t.Fatal("invalid TelegramWebFailure")
}
}
func TestWeConfigureWebChecksToFailOnHTTPError(t *testing.T) {
called := atomicx.NewInt64()
failOnErrorHTTPS := atomicx.NewInt64()
failOnErrorHTTP := atomicx.NewInt64()
measurer := telegram.Measurer{
Config: telegram.Config{},
Getter: func(ctx context.Context, g urlgetter.Getter) (urlgetter.TestKeys, error) {
called.Add(1)
switch g.Target {
case "https://web.telegram.org/":
if g.Config.FailOnHTTPError {
failOnErrorHTTPS.Add(1)
}
case "http://web.telegram.org/":
if g.Config.FailOnHTTPError {
failOnErrorHTTP.Add(1)
}
}
return urlgetter.DefaultMultiGetter(ctx, g)
},
}
ctx := context.Background()
sess := &mockable.Session{
MockableLogger: log.Log,
}
measurement := new(model.Measurement)
callbacks := model.NewPrinterCallbacks(log.Log)
if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil {
t.Fatal(err)
}
if called.Load() < 1 {
t.Fatal("not called")
}
if failOnErrorHTTPS.Load() != 1 {
t.Fatal("not configured fail on error for HTTPS")
}
if failOnErrorHTTP.Load() != 1 {
t.Fatal("not configured fail on error for HTTP")
}
}
func TestUpdateWithMissingTitle(t *testing.T) {
tk := telegram.NewTestKeys()
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "GET"},
Target: "http://web.telegram.org/",
},
TestKeys: urlgetter.TestKeys{
HTTPResponseStatus: 200,
HTTPResponseBody: "<HTML><title>Telegram Web</title></HTML>",
},
})
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "GET"},
Target: "http://web.telegram.org/",
},
TestKeys: urlgetter.TestKeys{
HTTPResponseStatus: 200,
HTTPResponseBody: "<HTML><title>Antani Web</title></HTML>",
},
})
if tk.TelegramWebStatus != "blocked" {
t.Fatal("TelegramWebStatus should be blocked")
}
if *tk.TelegramWebFailure != "telegram_missing_title_error" {
t.Fatal("invalid TelegramWebFailure")
}
}
func TestUpdateWithAllGood(t *testing.T) {
tk := telegram.NewTestKeys()
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "GET"},
Target: "http://web.telegram.org/",
},
TestKeys: urlgetter.TestKeys{
HTTPResponseStatus: 200,
HTTPResponseBody: "<HTML><title>Telegram Web</title></HTML>",
},
})
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{
Config: urlgetter.Config{Method: "GET"},
Target: "http://web.telegram.org/",
},
TestKeys: urlgetter.TestKeys{
HTTPResponseStatus: 200,
HTTPResponseBody: "<HTML><title>Telegram Web</title></HTML>",
},
})
if tk.TelegramWebStatus != "ok" {
t.Fatal("TelegramWebStatus should be ok")
}
if tk.TelegramWebFailure != nil {
t.Fatal("invalid TelegramWebFailure")
}
}
func TestSummaryKeysInvalidType(t *testing.T) {
measurement := new(model.Measurement)
m := &telegram.Measurer{}
_, err := m.GetSummaryKeys(measurement)
if err.Error() != "invalid test keys type" {
t.Fatal("not the error we expected")
}
}
func TestSummaryKeysWorksAsIntended(t *testing.T) {
failure := io.EOF.Error()
tests := []struct {
tk telegram.TestKeys
isAnomaly bool
}{{
tk: telegram.TestKeys{},
isAnomaly: false,
}, {
tk: telegram.TestKeys{TelegramTCPBlocking: true},
isAnomaly: true,
}, {
tk: telegram.TestKeys{TelegramHTTPBlocking: true},
isAnomaly: true,
}, {
tk: telegram.TestKeys{TelegramWebFailure: &failure},
isAnomaly: true,
}}
for idx, tt := range tests {
t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
m := &telegram.Measurer{}
measurement := &model.Measurement{TestKeys: &tt.tk}
got, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
return
}
sk := got.(telegram.SummaryKeys)
if sk.IsAnomaly != tt.isAnomaly {
t.Fatal("unexpected isAnomaly value")
}
})
}
}