chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
2021-02-02 12:05:47 +01:00
committed by GitHub
parent b1ce300c8d
commit d57c78bc71
535 changed files with 66182 additions and 23 deletions
@@ -0,0 +1,227 @@
// Package fbmessenger contains the Facebook Messenger network experiment.
//
// See https://github.com/ooni/spec/blob/master/nettests/ts-019-facebook-messenger.md
package fbmessenger
import (
"context"
"errors"
"math/rand"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
const (
// FacebookASN is Facebook's ASN
FacebookASN = 32934
// ServiceSTUN is the STUN service
ServiceSTUN = "dnslookup://stun.fbsbx.com"
// ServiceBAPI is the b-api service
ServiceBAPI = "tcpconnect://b-api.facebook.com:443"
// ServiceBGraph is the b-graph service
ServiceBGraph = "tcpconnect://b-graph.facebook.com:443"
// ServiceEdge is the edge service
ServiceEdge = "tcpconnect://edge-mqtt.facebook.com:443"
// ServiceExternalCDN is the external CDN service
ServiceExternalCDN = "tcpconnect://external.xx.fbcdn.net:443"
// ServiceScontentCDN is the scontent CDN service
ServiceScontentCDN = "tcpconnect://scontent.xx.fbcdn.net:443"
// ServiceStar is the star service
ServiceStar = "tcpconnect://star.c10r.facebook.com:443"
testName = "facebook_messenger"
testVersion = "0.2.0"
)
// Config contains the experiment config.
type Config struct{}
// TestKeys contains the experiment results
type TestKeys struct {
urlgetter.TestKeys
FacebookBAPIDNSConsistent *bool `json:"facebook_b_api_dns_consistent"`
FacebookBAPIReachable *bool `json:"facebook_b_api_reachable"`
FacebookBGraphDNSConsistent *bool `json:"facebook_b_graph_dns_consistent"`
FacebookBGraphReachable *bool `json:"facebook_b_graph_reachable"`
FacebookEdgeDNSConsistent *bool `json:"facebook_edge_dns_consistent"`
FacebookEdgeReachable *bool `json:"facebook_edge_reachable"`
FacebookExternalCDNDNSConsistent *bool `json:"facebook_external_cdn_dns_consistent"`
FacebookExternalCDNReachable *bool `json:"facebook_external_cdn_reachable"`
FacebookScontentCDNDNSConsistent *bool `json:"facebook_scontent_cdn_dns_consistent"`
FacebookScontentCDNReachable *bool `json:"facebook_scontent_cdn_reachable"`
FacebookStarDNSConsistent *bool `json:"facebook_star_dns_consistent"`
FacebookStarReachable *bool `json:"facebook_star_reachable"`
FacebookSTUNDNSConsistent *bool `json:"facebook_stun_dns_consistent"`
FacebookSTUNReachable *bool `json:"facebook_stun_reachable"`
FacebookDNSBlocking *bool `json:"facebook_dns_blocking"`
FacebookTCPBlocking *bool `json:"facebook_tcp_blocking"`
}
// Update updates the TestKeys using the given MultiOutput result.
func (tk *TestKeys) Update(v urlgetter.MultiOutput) {
// Update the easy to update entries first
tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
tk.Queries = append(tk.Queries, v.TestKeys.Queries...)
tk.Requests = append(tk.Requests, v.TestKeys.Requests...)
tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
tk.TLSHandshakes = append(tk.TLSHandshakes, v.TestKeys.TLSHandshakes...)
// Set the status of endpoints
switch v.Input.Target {
case ServiceSTUN:
var ignored *bool
tk.ComputeEndpointStatus(v, &tk.FacebookSTUNDNSConsistent, &ignored)
case ServiceBAPI:
tk.ComputeEndpointStatus(
v, &tk.FacebookBAPIDNSConsistent, &tk.FacebookBAPIReachable)
case ServiceBGraph:
tk.ComputeEndpointStatus(
v, &tk.FacebookBGraphDNSConsistent, &tk.FacebookBGraphReachable)
case ServiceEdge:
tk.ComputeEndpointStatus(
v, &tk.FacebookEdgeDNSConsistent, &tk.FacebookEdgeReachable)
case ServiceExternalCDN:
tk.ComputeEndpointStatus(
v, &tk.FacebookExternalCDNDNSConsistent, &tk.FacebookExternalCDNReachable)
case ServiceScontentCDN:
tk.ComputeEndpointStatus(
v, &tk.FacebookScontentCDNDNSConsistent, &tk.FacebookScontentCDNReachable)
case ServiceStar:
tk.ComputeEndpointStatus(
v, &tk.FacebookStarDNSConsistent, &tk.FacebookStarReachable)
}
}
var (
trueValue = true
falseValue = false
)
// ComputeEndpointStatus computes the DNS and TCP status of a specific endpoint.
func (tk *TestKeys) ComputeEndpointStatus(v urlgetter.MultiOutput, dns, tcp **bool) {
// start where all is unknown
*dns, *tcp = nil, nil
// process DNS first
if v.TestKeys.FailedOperation != nil && *v.TestKeys.FailedOperation == errorx.ResolveOperation {
tk.FacebookDNSBlocking = &trueValue
*dns = &falseValue
return // we know that the DNS has failed
}
for _, query := range v.TestKeys.Queries {
for _, ans := range query.Answers {
if ans.ASN != FacebookASN {
tk.FacebookDNSBlocking = &trueValue
*dns = &falseValue
return // because DNS is lying
}
}
}
*dns = &trueValue
// now process connect
if v.TestKeys.FailedOperation != nil && *v.TestKeys.FailedOperation == errorx.ConnectOperation {
tk.FacebookTCPBlocking = &trueValue
*tcp = &falseValue
return // because connect failed
}
// all good
*tcp = &trueValue
}
// Measurer performs the measurement
type Measurer struct {
// Config contains the experiment settings. If empty we
// will be using default settings.
Config Config
// Getter is an optional getter to be used for testing.
Getter urlgetter.MultiGetter
}
// ExperimentName implements ExperimentMeasurer.ExperimentName
func (m Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
func (m Measurer) ExperimentVersion() string {
return testVersion
}
// Run implements ExperimentMeasurer.Run
func (m Measurer) Run(
ctx context.Context, sess model.ExperimentSession,
measurement *model.Measurement, callbacks model.ExperimentCallbacks,
) error {
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
urlgetter.RegisterExtensions(measurement)
// generate targets
services := []string{
ServiceSTUN, ServiceBAPI, ServiceBGraph, ServiceEdge, ServiceExternalCDN,
ServiceScontentCDN, ServiceStar,
}
var inputs []urlgetter.MultiInput
for _, service := range services {
inputs = append(inputs, urlgetter.MultiInput{Target: service})
}
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
rnd.Shuffle(len(inputs), func(i, j int) {
inputs[i], inputs[j] = inputs[j], inputs[i]
})
// measure in parallel
multi := urlgetter.Multi{Begin: time.Now(), Getter: m.Getter, Session: sess}
testkeys := new(TestKeys)
testkeys.Agent = "redirect"
measurement.TestKeys = testkeys
for entry := range multi.Collect(ctx, inputs, "facebook_messenger", callbacks) {
testkeys.Update(entry)
}
// if we haven't yet determined the status of DNS blocking and TCP blocking
// then no blocking has been detected and we can set them
if testkeys.FacebookDNSBlocking == nil {
testkeys.FacebookDNSBlocking = &falseValue
}
if testkeys.FacebookTCPBlocking == nil {
testkeys.FacebookTCPBlocking = &falseValue
}
return nil
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return Measurer{Config: config}
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with probe-cli
// therefore we should be careful when changing it.
type SummaryKeys struct {
DNSBlocking bool `json:"facebook_dns_blocking"`
TCPBlocking bool `json:"facebook_tcp_blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{IsAnomaly: false}
tk, ok := measurement.TestKeys.(*TestKeys)
if !ok {
return sk, errors.New("invalid test keys type")
}
dnsBlocking := tk.FacebookDNSBlocking != nil && *tk.FacebookDNSBlocking
tcpBlocking := tk.FacebookTCPBlocking != nil && *tk.FacebookTCPBlocking
sk.DNSBlocking = dnsBlocking
sk.TCPBlocking = tcpBlocking
sk.IsAnomaly = dnsBlocking || tcpBlocking
return sk, nil
}
@@ -0,0 +1,363 @@
package fbmessenger_test
import (
"context"
"io"
"testing"
"github.com/apex/log"
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
func TestNewExperimentMeasurer(t *testing.T) {
measurer := fbmessenger.NewExperimentMeasurer(fbmessenger.Config{})
if measurer.ExperimentName() != "facebook_messenger" {
t.Fatal("unexpected name")
}
if measurer.ExperimentVersion() != "0.2.0" {
t.Fatal("unexpected version")
}
}
func TestSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
measurer := fbmessenger.NewExperimentMeasurer(fbmessenger.Config{})
ctx := context.Background()
// we need a real session because we need the ASN database
sess := newsession(t)
measurement := new(model.Measurement)
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*fbmessenger.TestKeys)
if *tk.FacebookBAPIDNSConsistent != true {
t.Fatal("invalid FacebookBAPIDNSConsistent")
}
if *tk.FacebookBAPIReachable != true {
t.Fatal("invalid FacebookBAPIReachable")
}
if *tk.FacebookBGraphDNSConsistent != true {
t.Fatal("invalid FacebookBGraphDNSConsistent")
}
if *tk.FacebookBGraphReachable != true {
t.Fatal("invalid FacebookBGraphReachable")
}
if *tk.FacebookEdgeDNSConsistent != true {
t.Fatal("invalid FacebookEdgeDNSConsistent")
}
if *tk.FacebookEdgeReachable != true {
t.Fatal("invalid FacebookEdgeReachable")
}
if *tk.FacebookExternalCDNDNSConsistent != true {
t.Fatal("invalid FacebookExternalCDNDNSConsistent")
}
if *tk.FacebookExternalCDNReachable != true {
t.Fatal("invalid FacebookExternalCDNReachable")
}
if *tk.FacebookScontentCDNDNSConsistent != true {
t.Fatal("invalid FacebookScontentCDNDNSConsistent")
}
if *tk.FacebookScontentCDNReachable != true {
t.Fatal("invalid FacebookScontentCDNReachable")
}
if *tk.FacebookStarDNSConsistent != true {
t.Fatal("invalid FacebookStarDNSConsistent")
}
if *tk.FacebookStarReachable != true {
t.Fatal("invalid FacebookStarReachable")
}
if *tk.FacebookSTUNDNSConsistent != true {
t.Fatal("invalid FacebookSTUNDNSConsistent")
}
if tk.FacebookSTUNReachable != nil {
t.Fatal("invalid FacebookSTUNReachable")
}
if *tk.FacebookDNSBlocking != false {
t.Fatal("invalid FacebookDNSBlocking")
}
if *tk.FacebookTCPBlocking != false {
t.Fatal("invalid FacebookTCPBlocking")
}
}
func TestWithCancelledContext(t *testing.T) {
measurer := fbmessenger.NewExperimentMeasurer(fbmessenger.Config{})
ctx, cancel := context.WithCancel(context.Background())
cancel() // so we fail immediately
sess := &mockable.Session{MockableLogger: log.Log}
measurement := new(model.Measurement)
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*fbmessenger.TestKeys)
if *tk.FacebookBAPIDNSConsistent != false {
t.Fatal("invalid FacebookBAPIDNSConsistent")
}
if tk.FacebookBAPIReachable != nil {
t.Fatal("invalid FacebookBAPIReachable")
}
if *tk.FacebookBGraphDNSConsistent != false {
t.Fatal("invalid FacebookBGraphDNSConsistent")
}
if tk.FacebookBGraphReachable != nil {
t.Fatal("invalid FacebookBGraphReachable")
}
if *tk.FacebookEdgeDNSConsistent != false {
t.Fatal("invalid FacebookEdgeDNSConsistent")
}
if tk.FacebookEdgeReachable != nil {
t.Fatal("invalid FacebookEdgeReachable")
}
if *tk.FacebookExternalCDNDNSConsistent != false {
t.Fatal("invalid FacebookExternalCDNDNSConsistent")
}
if tk.FacebookExternalCDNReachable != nil {
t.Fatal("invalid FacebookExternalCDNReachable")
}
if *tk.FacebookScontentCDNDNSConsistent != false {
t.Fatal("invalid FacebookScontentCDNDNSConsistent")
}
if tk.FacebookScontentCDNReachable != nil {
t.Fatal("invalid FacebookScontentCDNReachable")
}
if *tk.FacebookStarDNSConsistent != false {
t.Fatal("invalid FacebookStarDNSConsistent")
}
if tk.FacebookStarReachable != nil {
t.Fatal("invalid FacebookStarReachable")
}
if *tk.FacebookSTUNDNSConsistent != false {
t.Fatal("invalid FacebookSTUNDNSConsistent")
}
if tk.FacebookSTUNReachable != nil {
t.Fatal("invalid FacebookSTUNReachable")
}
if *tk.FacebookDNSBlocking != true {
t.Fatal("invalid FacebookDNSBlocking")
}
// no TCP blocking because we didn't ever reach TCP connect
if *tk.FacebookTCPBlocking != false {
t.Fatal("invalid FacebookTCPBlocking")
}
sk, err := measurer.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
if _, ok := sk.(fbmessenger.SummaryKeys); !ok {
t.Fatal("invalid type for summary keys")
}
}
func TestComputeEndpointStatsTCPBlocking(t *testing.T) {
failure := io.EOF.Error()
operation := errorx.ConnectOperation
tk := fbmessenger.TestKeys{}
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{Target: fbmessenger.ServiceEdge},
TestKeys: urlgetter.TestKeys{
Failure: &failure,
FailedOperation: &operation,
Queries: []archival.DNSQueryEntry{{
Answers: []archival.DNSAnswerEntry{{
ASN: fbmessenger.FacebookASN,
}},
}},
},
})
if *tk.FacebookEdgeDNSConsistent != true {
t.Fatal("invalid FacebookEdgeDNSConsistent")
}
if *tk.FacebookEdgeReachable != false {
t.Fatal("invalid FacebookEdgeReachable")
}
if tk.FacebookDNSBlocking != nil { // meaning: not determined yet
t.Fatal("invalid FacebookDNSBlocking")
}
if *tk.FacebookTCPBlocking != true {
t.Fatal("invalid FacebookTCPBlocking")
}
}
func TestComputeEndpointStatsDNSIsLying(t *testing.T) {
failure := io.EOF.Error()
operation := errorx.ConnectOperation
tk := fbmessenger.TestKeys{}
tk.Update(urlgetter.MultiOutput{
Input: urlgetter.MultiInput{Target: fbmessenger.ServiceEdge},
TestKeys: urlgetter.TestKeys{
Failure: &failure,
FailedOperation: &operation,
Queries: []archival.DNSQueryEntry{{
Answers: []archival.DNSAnswerEntry{{
ASN: 0,
}},
}},
},
})
if *tk.FacebookEdgeDNSConsistent != false {
t.Fatal("invalid FacebookEdgeDNSConsistent")
}
if tk.FacebookEdgeReachable != nil {
t.Fatal("invalid FacebookEdgeReachable")
}
if *tk.FacebookDNSBlocking != true {
t.Fatal("invalid FacebookDNSBlocking")
}
if tk.FacebookTCPBlocking != nil { // meaning: not determined yet
t.Fatal("invalid FacebookTCPBlocking")
}
}
func newsession(t *testing.T) model.ExperimentSession {
sess, err := engine.NewSession(engine.SessionConfig{
AssetsDir: "../../testdata",
AvailableProbeServices: []model.Service{{
Address: "https://ams-pg-test.ooni.org",
Type: "https",
}},
Logger: log.Log,
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.0.1",
})
if err != nil {
t.Fatal(err)
}
if err := sess.MaybeLookupLocation(); err != nil {
t.Fatal(err)
}
return sess
}
func TestSummaryKeysInvalidType(t *testing.T) {
measurement := new(model.Measurement)
m := &fbmessenger.Measurer{}
_, err := m.GetSummaryKeys(measurement)
if err.Error() != "invalid test keys type" {
t.Fatal("not the error we expected")
}
}
func TestSummaryKeysWithNils(t *testing.T) {
measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{}}
m := &fbmessenger.Measurer{}
osk, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
sk := osk.(fbmessenger.SummaryKeys)
if sk.DNSBlocking {
t.Fatal("invalid dnsBlocking")
}
if sk.TCPBlocking {
t.Fatal("invalid tcpBlocking")
}
if sk.IsAnomaly {
t.Fatal("invalid isAnomaly")
}
}
func TestSummaryKeysWithFalseFalse(t *testing.T) {
falsy := false
measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{
FacebookTCPBlocking: &falsy,
FacebookDNSBlocking: &falsy,
}}
m := &fbmessenger.Measurer{}
osk, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
sk := osk.(fbmessenger.SummaryKeys)
if sk.DNSBlocking {
t.Fatal("invalid dnsBlocking")
}
if sk.TCPBlocking {
t.Fatal("invalid tcpBlocking")
}
if sk.IsAnomaly {
t.Fatal("invalid isAnomaly")
}
}
func TestSummaryKeysWithFalseTrue(t *testing.T) {
falsy := false
truy := true
measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{
FacebookTCPBlocking: &falsy,
FacebookDNSBlocking: &truy,
}}
m := &fbmessenger.Measurer{}
osk, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
sk := osk.(fbmessenger.SummaryKeys)
if sk.DNSBlocking == false {
t.Fatal("invalid dnsBlocking")
}
if sk.TCPBlocking {
t.Fatal("invalid tcpBlocking")
}
if sk.IsAnomaly == false {
t.Fatal("invalid isAnomaly")
}
}
func TestSummaryKeysWithTrueFalse(t *testing.T) {
falsy := false
truy := true
measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{
FacebookTCPBlocking: &truy,
FacebookDNSBlocking: &falsy,
}}
m := &fbmessenger.Measurer{}
osk, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
sk := osk.(fbmessenger.SummaryKeys)
if sk.DNSBlocking {
t.Fatal("invalid dnsBlocking")
}
if sk.TCPBlocking == false {
t.Fatal("invalid tcpBlocking")
}
if sk.IsAnomaly == false {
t.Fatal("invalid isAnomaly")
}
}
func TestSummaryKeysWithTrueTrue(t *testing.T) {
truy := true
measurement := &model.Measurement{TestKeys: &fbmessenger.TestKeys{
FacebookTCPBlocking: &truy,
FacebookDNSBlocking: &truy,
}}
m := &fbmessenger.Measurer{}
osk, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
sk := osk.(fbmessenger.SummaryKeys)
if sk.DNSBlocking == false {
t.Fatal("invalid dnsBlocking")
}
if sk.TCPBlocking == false {
t.Fatal("invalid tcpBlocking")
}
if sk.IsAnomaly == false {
t.Fatal("invalid isAnomaly")
}
}