chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/experiment/riseupvpn/riseupvpn.go

@@ -0,0 +1,309 @@
+// Package riseupvpn contains the RiseupVPN network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-026-riseupvpn.md
+package riseupvpn
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
+)
+
+const (
+	testName      = "riseupvpn"
+	testVersion   = "0.1.0"
+	eipServiceURL = "https://api.black.riseup.net:443/3/config/eip-service.json"
+	providerURL   = "https://riseup.net/provider.json"
+	geoServiceURL = "https://api.black.riseup.net:9001/json"
+	tcpConnect    = "tcpconnect://"
+)
+
+// EipService main json object of eip-service.json
+type EipService struct {
+	Gateways []GatewayV3
+}
+
+// GatewayV3 json obj Version 3
+type GatewayV3 struct {
+	Capabilities struct {
+		Transport []TransportV3
+	}
+	Host      string
+	IPAddress string `json:"ip_address"`
+}
+
+// TransportV3 json obj Version 3
+type TransportV3 struct {
+	Type      string
+	Protocols []string
+	Ports     []string
+	Options   map[string]string
+}
+
+// GatewayConnection describes the connection to a riseupvpn gateway
+type GatewayConnection struct {
+	IP            string `json:"ip"`
+	Port          int    `json:"port"`
+	TransportType string `json:"transport_type"`
+}
+
+// Config contains the riseupvpn experiment config.
+type Config struct {
+	urlgetter.Config
+}
+
+// TestKeys contains riseupvpn test keys.
+type TestKeys struct {
+	urlgetter.TestKeys
+	APIFailure      *string             `json:"api_failure"`
+	APIStatus       string              `json:"api_status"`
+	CACertStatus    bool                `json:"ca_cert_status"`
+	FailingGateways []GatewayConnection `json:"failing_gateways"`
+}
+
+// NewTestKeys creates new riseupvpn TestKeys.
+func NewTestKeys() *TestKeys {
+	return &TestKeys{
+		APIFailure:      nil,
+		APIStatus:       "ok",
+		CACertStatus:    true,
+		FailingGateways: nil,
+	}
+}
+
+// UpdateProviderAPITestKeys updates the TestKeys using the given MultiOutput result.
+func (tk *TestKeys) UpdateProviderAPITestKeys(v urlgetter.MultiOutput) {
+	tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
+	tk.Queries = append(tk.Queries, v.TestKeys.Queries...)
+	tk.Requests = append(tk.Requests, v.TestKeys.Requests...)
+	tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
+	tk.TLSHandshakes = append(tk.TLSHandshakes, v.TestKeys.TLSHandshakes...)
+	if tk.APIStatus != "ok" {
+		return // we already flipped the state
+	}
+	if v.TestKeys.Failure != nil {
+		tk.APIStatus = "blocked"
+		tk.APIFailure = v.TestKeys.Failure
+		return
+	}
+}
+
+// AddGatewayConnectTestKeys updates the TestKeys using the given MultiOutput result of gateway connectivity testing.
+func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transportType string) {
+	tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
+	tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
+	for _, tcpConnect := range v.TestKeys.TCPConnect {
+		if !tcpConnect.Status.Success {
+			gatewayConnection := newGatewayConnection(tcpConnect, transportType)
+			tk.FailingGateways = append(tk.FailingGateways, *gatewayConnection)
+		}
+	}
+	return
+}
+
+func newGatewayConnection(tcpConnect archival.TCPConnectEntry, transportType string) *GatewayConnection {
+	return &GatewayConnection{
+		IP:            tcpConnect.IP,
+		Port:          tcpConnect.Port,
+		TransportType: transportType,
+	}
+}
+
+// AddCACertFetchTestKeys Adding generic urlgetter.Get() testKeys to riseupvpn specific test keys
+func (tk *TestKeys) AddCACertFetchTestKeys(testKeys urlgetter.TestKeys) {
+	tk.NetworkEvents = append(tk.NetworkEvents, testKeys.NetworkEvents...)
+	tk.Queries = append(tk.Queries, testKeys.Queries...)
+	tk.Requests = append(tk.Requests, testKeys.Requests...)
+	tk.TCPConnect = append(tk.TCPConnect, testKeys.TCPConnect...)
+	tk.TLSHandshakes = append(tk.TLSHandshakes, testKeys.TLSHandshakes...)
+	if testKeys.Failure != nil {
+		tk.APIStatus = "blocked"
+		tk.APIFailure = tk.Failure
+		tk.CACertStatus = false
+	}
+}
+
+// Measurer performs the measurement
+type Measurer struct {
+	// Config contains the experiment settings. If empty we
+	// will be using default settings.
+	Config Config
+
+	// Getter is an optional getter to be used for testing.
+	Getter urlgetter.MultiGetter
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName
+func (m Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
+func (m Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run
+func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
+	measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
+	ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
+	defer cancel()
+	testkeys := NewTestKeys()
+	measurement.TestKeys = testkeys
+	urlgetter.RegisterExtensions(measurement)
+
+	caTarget := "https://black.riseup.net/ca.crt"
+	caGetter := urlgetter.Getter{
+		Config:  m.Config.Config,
+		Session: sess,
+		Target:  caTarget,
+	}
+	log.Info("Getting CA certificate; please be patient...")
+	tk, err := caGetter.Get(ctx)
+	testkeys.AddCACertFetchTestKeys(tk)
+
+	if err != nil {
+		log.Error("Getting CA certificate failed. Aborting test.")
+		return nil
+	}
+
+	certPool := netx.NewDefaultCertPool()
+	if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok {
+		testkeys.CACertStatus = false
+		testkeys.APIStatus = "blocked"
+		errorValue := "invalid_ca"
+		testkeys.APIFailure = &errorValue
+		return nil
+	}
+
+	inputs := []urlgetter.MultiInput{
+
+		// Here we need to provide the method explicitly. See
+		// https://github.com/ooni/probe-cli/v3/internal/engine/issues/827.
+		{Target: providerURL, Config: urlgetter.Config{
+			CertPool:        certPool,
+			Method:          "GET",
+			FailOnHTTPError: true,
+		}},
+		{Target: eipServiceURL, Config: urlgetter.Config{
+			CertPool:        certPool,
+			Method:          "GET",
+			FailOnHTTPError: true,
+		}},
+		{Target: geoServiceURL, Config: urlgetter.Config{
+			CertPool:        certPool,
+			Method:          "GET",
+			FailOnHTTPError: true,
+		}},
+	}
+	multi := urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
+
+	for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) {
+		testkeys.UpdateProviderAPITestKeys(entry)
+	}
+
+	// test gateways now
+	gateways := parseGateways(testkeys)
+	openvpnEndpoints := generateMultiInputs(gateways, "openvpn")
+	obfs4Endpoints := generateMultiInputs(gateways, "obfs4")
+	overallCount := len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints)
+
+	// measure openvpn in parallel
+	multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
+	for entry := range multi.CollectOverall(ctx, openvpnEndpoints, len(inputs), overallCount, "riseupvpn", callbacks) {
+		testkeys.AddGatewayConnectTestKeys(entry, "openvpn")
+	}
+
+	// measure obfs4 in parallel
+	multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
+	for entry := range multi.CollectOverall(ctx, obfs4Endpoints, len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) {
+		testkeys.AddGatewayConnectTestKeys(entry, "obfs4")
+	}
+
+	return nil
+}
+
+func generateMultiInputs(gateways []GatewayV3, transportType string) []urlgetter.MultiInput {
+	var gatewayInputs []urlgetter.MultiInput
+	for _, gateway := range gateways {
+		for _, transport := range gateway.Capabilities.Transport {
+			if transport.Type != transportType {
+				continue
+			}
+			supportsTCP := false
+			for _, protocol := range transport.Protocols {
+				if protocol == "tcp" {
+					supportsTCP = true
+				}
+			}
+			if !supportsTCP {
+				continue
+			}
+			for _, port := range transport.Ports {
+				tcpConnection := tcpConnect + gateway.IPAddress + ":" + port
+				gatewayInputs = append(gatewayInputs, urlgetter.MultiInput{Target: tcpConnection})
+			}
+		}
+	}
+	return gatewayInputs
+}
+
+func parseGateways(testKeys *TestKeys) []GatewayV3 {
+	for _, requestEntry := range testKeys.Requests {
+		if requestEntry.Request.URL == eipServiceURL && requestEntry.Failure == nil {
+			eipService, err := DecodeEIP3(requestEntry.Response.Body.Value)
+			if err == nil {
+				return eipService.Gateways
+			}
+		}
+	}
+	return nil
+}
+
+// DecodeEIP3 decodes eip-service.json version 3
+func DecodeEIP3(body string) (*EipService, error) {
+	var eip EipService
+	err := json.Unmarshal([]byte(body), &eip)
+	if err != nil {
+		return nil, err
+	}
+	return &eip, nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with probe-cli
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	APIBlocked      bool `json:"api_blocked"`
+	ValidCACert     bool `json:"valid_ca_cert"`
+	FailingGateways int  `json:"failing_gateways"`
+	IsAnomaly       bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	sk := SummaryKeys{IsAnomaly: false}
+	tk, ok := measurement.TestKeys.(*TestKeys)
+	if !ok {
+		return sk, errors.New("invalid test keys type")
+	}
+	sk.APIBlocked = tk.APIStatus != "ok"
+	sk.ValidCACert = tk.CACertStatus
+	sk.FailingGateways = len(tk.FailingGateways)
+	sk.IsAnomaly = (sk.APIBlocked == true || tk.CACertStatus == false ||
+		sk.FailingGateways != 0)
+	return sk, nil
+}

internal/engine/experiment/riseupvpn/riseupvpn_test.go

@@ -0,0 +1,497 @@
+package riseupvpn_test
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
+)
+
+func TestNewExperimentMeasurer(t *testing.T) {
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	if measurer.ExperimentName() != "riseupvpn" {
+		t.Fatal("unexpected name")
+	}
+	if measurer.ExperimentVersion() != "0.1.0" {
+		t.Fatal("unexpected version")
+	}
+}
+
+func TestGood(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	measurement := new(model.Measurement)
+	err := measurer.Run(
+		context.Background(),
+		&mockable.Session{
+			MockableLogger: log.Log,
+		},
+		measurement,
+		model.NewPrinterCallbacks(log.Log),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.Agent != "" {
+		t.Fatal("unexpected Agent: " + tk.Agent)
+	}
+	if tk.FailedOperation != nil {
+		t.Fatal("unexpected FailedOperation")
+	}
+	if tk.Failure != nil {
+		t.Fatal("unexpected Failure")
+	}
+	if len(tk.NetworkEvents) <= 0 {
+		t.Fatal("no NetworkEvents?!")
+	}
+	if len(tk.Queries) <= 0 {
+		t.Fatal("no Queries?!")
+	}
+	if len(tk.Requests) <= 0 {
+		t.Fatal("no Requests?!")
+	}
+	if len(tk.TCPConnect) <= 0 {
+		t.Fatal("no TCPConnect?!")
+	}
+	if len(tk.TLSHandshakes) <= 0 {
+		t.Fatal("no TLSHandshakes?!")
+	}
+	if tk.APIFailure != nil {
+		t.Fatal("unexpected ApiFailure")
+	}
+	if tk.APIStatus != "ok" {
+		t.Fatal("unexpected ApiStatus")
+	}
+	if tk.CACertStatus != true {
+		t.Fatal("unexpected CaCertStatus")
+	}
+	if tk.FailingGateways != nil {
+		t.Fatal("unexpected FailingGateways value")
+	}
+}
+
+// TestUpdateWithMixedResults tests if one operation failed
+// ApiStatus is considered as blocked
+func TestUpdateWithMixedResults(t *testing.T) {
+	tk := riseupvpn.NewTestKeys()
+	tk.UpdateProviderAPITestKeys(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "https://api.black.riseup.net:443/3/config/eip-service.json",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseStatus: 200,
+		},
+	})
+	tk.UpdateProviderAPITestKeys(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "https://riseup.net/provider.json",
+		},
+		TestKeys: urlgetter.TestKeys{
+			FailedOperation: (func() *string {
+				s := errorx.HTTPRoundTripOperation
+				return &s
+			})(),
+			Failure: (func() *string {
+				s := errorx.FailureEOFError
+				return &s
+			})(),
+		},
+	})
+	tk.UpdateProviderAPITestKeys(urlgetter.MultiOutput{
+		Input: urlgetter.MultiInput{
+			Config: urlgetter.Config{Method: "GET"},
+			Target: "https://api.black.riseup.net:9001/json",
+		},
+		TestKeys: urlgetter.TestKeys{
+			HTTPResponseStatus: 200,
+		},
+	})
+	if tk.APIStatus != "blocked" {
+		t.Fatal("ApiStatus should be blocked")
+	}
+	if *tk.APIFailure != errorx.FailureEOFError {
+		t.Fatal("invalid ApiFailure")
+	}
+}
+
+func TestFailureCaCertFetch(t *testing.T) {
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	// we're cancelling immediately so that the CA Cert fetch fails
+	cancel()
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus != false {
+		t.Fatal("invalid CACertStatus ")
+	}
+	if tk.APIStatus != "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure != nil {
+		t.Fatal("ApiFailure should be null")
+	}
+	if len(tk.Requests) > 1 {
+		t.Fatal("Unexpected requests")
+	}
+}
+
+func TestFailureEipServiceBlocked(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	selfcensor.Enable(`{"PoisonSystemDNS":{"api.black.riseup.net":["NXDOMAIN"]}}`)
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus != true {
+		t.Fatal("invalid CACertStatus ")
+	}
+
+	for _, entry := range tk.Requests {
+		if entry.Request.URL == "https://api.black.riseup.net:443/3/config/eip-service.json" {
+			if entry.Failure == nil {
+				t.Fatal("Failure for " + entry.Request.URL + " should not be null")
+			}
+		}
+	}
+
+	if tk.APIStatus != "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure == nil {
+		t.Fatal("ApiFailure should not be null")
+	}
+}
+
+func TestFailureProviderUrlBlocked(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.70:443":"REJECT"}}`)
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+
+	for _, entry := range tk.Requests {
+		if entry.Request.URL == "https://riseup.net/provider.json" {
+			if entry.Failure == nil {
+				t.Fatal("Failure for " + entry.Request.URL + " should not be null")
+			}
+		}
+	}
+
+	if tk.CACertStatus != true {
+		t.Fatal("invalid CACertStatus ")
+	}
+	if tk.APIStatus != "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure == nil {
+		t.Fatal("ApiFailure should not be null")
+	}
+}
+
+func TestFailureGeoIpServiceBlocked(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.107:9001":"REJECT"}}`)
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus != true {
+		t.Fatal("invalid CACertStatus ")
+	}
+
+	for _, entry := range tk.Requests {
+		if entry.Request.URL == "https://api.black.riseup.net:9001/json" {
+			if entry.Failure == nil {
+				t.Fatal("Failure for " + entry.Request.URL + " should not be null")
+			}
+		}
+	}
+
+	if tk.APIStatus != "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure == nil {
+		t.Fatal("ApiFailure should not be null")
+	}
+}
+
+func TestFailureGateway(t *testing.T) {
+	var testCases = [...]string{"openvpn", "obfs4"}
+	eipService, err := fetchEipService()
+	if err != nil {
+		t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error())
+		t.SkipNow()
+	}
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("testing censored transport %s", tc), func(t *testing.T) {
+			censoredGateway, err := selfCensorRandomGateway(eipService, tc)
+			if err == nil {
+				censorString := `{"BlockedEndpoints":{"` + censoredGateway.IP + `:` + censoredGateway.Port + `":"REJECT"}}`
+				selfcensor.Enable(censorString)
+			} else {
+				t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error())
+				t.SkipNow()
+			}
+
+			// - run measurement
+			runGatewayTest(t, censoredGateway)
+		})
+	}
+}
+
+type SelfCensoredGateway struct {
+	IP   string
+	Port string
+}
+
+func fetchEipService() (*riseupvpn.EipService, error) {
+	// - fetch client cert and add to certpool
+	caFetchClient := &http.Client{
+		Timeout: time.Second * 30,
+	}
+
+	caCertResponse, err := caFetchClient.Get("https://black.riseup.net/ca.crt")
+	if err != nil {
+		return nil, err
+	}
+
+	var bodyString string
+
+	if caCertResponse.StatusCode != http.StatusOK {
+		return nil, errors.New("unexpected HTTP response code")
+	}
+	bodyBytes, err := ioutil.ReadAll(caCertResponse.Body)
+	defer caCertResponse.Body.Close()
+
+	if err != nil {
+		return nil, err
+	}
+	bodyString = string(bodyBytes)
+
+	certs := x509.NewCertPool()
+	certs.AppendCertsFromPEM([]byte(bodyString))
+
+	// - fetch and parse eip-service.json
+	client := &http.Client{
+		Timeout: time.Second * 30,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: certs,
+			},
+		},
+	}
+
+	eipResponse, err := client.Get("https://api.black.riseup.net/3/config/eip-service.json")
+	if err != nil {
+		return nil, err
+	}
+	if eipResponse.StatusCode != http.StatusOK {
+		return nil, errors.New("Unexpected HTTP response code")
+	}
+
+	bodyBytes, err = ioutil.ReadAll(eipResponse.Body)
+	defer eipResponse.Body.Close()
+	if err != nil {
+		return nil, err
+	}
+	bodyString = string(bodyBytes)
+
+	eipService, err := riseupvpn.DecodeEIP3(bodyString)
+	if err != nil {
+		return nil, err
+	}
+	return eipService, nil
+}
+
+func selfCensorRandomGateway(eipService *riseupvpn.EipService, transportType string) (*SelfCensoredGateway, error) {
+
+	// - self censor random gateway
+	gateways := eipService.Gateways
+	if gateways == nil || len(gateways) == 0 {
+		return nil, errors.New("No gateways found")
+	}
+
+	var selfcensoredGateways []SelfCensoredGateway
+	for _, gateway := range gateways {
+		for _, transport := range gateway.Capabilities.Transport {
+			if transport.Type == transportType {
+				selfcensoredGateways = append(selfcensoredGateways, SelfCensoredGateway{IP: gateway.IPAddress, Port: transport.Ports[0]})
+			}
+		}
+	}
+
+	if len(selfcensoredGateways) == 0 {
+		return nil, errors.New("transport " + transportType + " doesn't seem to be supported.")
+	}
+
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+	min := 0
+	max := len(selfcensoredGateways) - 1
+	randomIndex := rnd.Intn(max-min+1) + min
+	return &selfcensoredGateways[randomIndex], nil
+
+}
+
+func runGatewayTest(t *testing.T, censoredGateway *SelfCensoredGateway) {
+	measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	sess := &mockable.Session{MockableLogger: log.Log}
+	measurement := new(model.Measurement)
+	callbacks := model.NewPrinterCallbacks(log.Log)
+	err := measurer.Run(ctx, sess, measurement, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := measurement.TestKeys.(*riseupvpn.TestKeys)
+	if tk.CACertStatus != true {
+		t.Fatal("invalid CACertStatus ")
+	}
+
+	if tk.FailingGateways == nil || len(tk.FailingGateways) != 1 {
+		t.Fatal("unexpected amount of failing gateways")
+	}
+
+	entry := tk.FailingGateways[0]
+	if entry.IP != censoredGateway.IP || fmt.Sprint(entry.Port) != censoredGateway.Port {
+		t.Fatal("unexpected failed gateway configuration")
+	}
+
+	if tk.APIStatus == "blocked" {
+		t.Fatal("invalid ApiStatus")
+	}
+
+	if tk.APIFailure != nil {
+		t.Fatal("ApiFailure should be null")
+	}
+}
+
+func TestSummaryKeysInvalidType(t *testing.T) {
+	measurement := new(model.Measurement)
+	m := &riseupvpn.Measurer{}
+	_, err := m.GetSummaryKeys(measurement)
+	if err.Error() != "invalid test keys type" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestSummaryKeysWorksAsIntended(t *testing.T) {
+	tests := []struct {
+		tk riseupvpn.TestKeys
+		sk riseupvpn.SummaryKeys
+	}{{
+		tk: riseupvpn.TestKeys{
+			APIStatus:       "blocked",
+			CACertStatus:    true,
+			FailingGateways: nil,
+		},
+		sk: riseupvpn.SummaryKeys{
+			APIBlocked:  true,
+			ValidCACert: true,
+			IsAnomaly:   true,
+		},
+	}, {
+		tk: riseupvpn.TestKeys{
+			APIStatus:       "ok",
+			CACertStatus:    false,
+			FailingGateways: nil,
+		},
+		sk: riseupvpn.SummaryKeys{
+			ValidCACert: false,
+			IsAnomaly:   true,
+		},
+	}, {
+		tk: riseupvpn.TestKeys{
+			APIStatus:    "ok",
+			CACertStatus: true,
+			FailingGateways: []riseupvpn.GatewayConnection{{
+				IP:            "1.1.1.1",
+				Port:          443,
+				TransportType: "obfs4",
+			}},
+		},
+		sk: riseupvpn.SummaryKeys{
+			FailingGateways: 1,
+			IsAnomaly:       true,
+			ValidCACert:     true,
+		},
+	}}
+	for idx, tt := range tests {
+		t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
+			m := &riseupvpn.Measurer{}
+			measurement := &model.Measurement{TestKeys: &tt.tk}
+			got, err := m.GetSummaryKeys(measurement)
+			if err != nil {
+				t.Fatal(err)
+				return
+			}
+			sk := got.(riseupvpn.SummaryKeys)
+			if diff := cmp.Diff(tt.sk, sk); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}