c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

riseupvpn: reduce false positives (#233)

Browse Source 
* fetch RiseupVPN CA cert with MultiGetter. It allows us to write better tests and ensures this test step is added in the logs

* Implement TransportStatus for RiseupVPN tests. It indicates if a whole transport is blocked, which is considered as a test anomaly

* Redesign unit tests for RiseupVPN. Instead of a real backend, mocked server responses are used. Tests for invalid CA certs and for TransportStatus are added.

* Update internal/engine/experiment/riseupvpn/riseupvpn.go

Co-authored-by: Simone Basso <bassosimone@gmail.com>
This commit is contained in:
cyBerta 2021-03-30 12:02:51 +02:00 committed by GitHub
parent dae02ce5b6
commit 991b0a6120
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 589 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
internal/engine/experiment/riseupvpn/riseupvpn.go
View File

@ -9,7 +9,6 @@ import (
"errors" "errors"
"time" "time"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx"
@ -18,7 +17,7 @@ import (
const ( const (
testName = "riseupvpn" testName = "riseupvpn"
testVersion = "0.1.0" testVersion = "0.2.0"
eipServiceURL = "https://api.black.riseup.net:443/3/config/eip-service.json" eipServiceURL = "https://api.black.riseup.net:443/3/config/eip-service.json"
providerURL = "https://riseup.net/provider.json" providerURL = "https://riseup.net/provider.json"
geoServiceURL = "https://api.black.riseup.net:9001/json" geoServiceURL = "https://api.black.riseup.net:9001/json"
@ -66,6 +65,7 @@ type TestKeys struct {
APIStatus string `json:"api_status"` APIStatus string `json:"api_status"`
CACertStatus bool `json:"ca_cert_status"` CACertStatus bool `json:"ca_cert_status"`
FailingGateways []GatewayConnection `json:"failing_gateways"` FailingGateways []GatewayConnection `json:"failing_gateways"`
TransportStatus map[string]string `json:"transport_status"`
} }
// NewTestKeys creates new riseupvpn TestKeys. // NewTestKeys creates new riseupvpn TestKeys.
@ -75,6 +75,7 @@ func NewTestKeys() *TestKeys {
APIStatus: "ok", APIStatus: "ok",
CACertStatus: true, CACertStatus: true,
FailingGateways: nil, FailingGateways: nil,
TransportStatus: nil,
} }
} }
@ -96,6 +97,7 @@ func (tk *TestKeys) UpdateProviderAPITestKeys(v urlgetter.MultiOutput) {
} }
// AddGatewayConnectTestKeys updates the TestKeys using the given MultiOutput result of gateway connectivity testing. // AddGatewayConnectTestKeys updates the TestKeys using the given MultiOutput result of gateway connectivity testing.
// Sets TransportStatus to "ok" if any successful TCP connection could be made
func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transportType string) { func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transportType string) {
tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...) tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...)
tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...) tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...)
@ -108,6 +110,29 @@ func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transport
return return
} }
func (tk *TestKeys) updateTransportStatus(openvpnGatewayCount int, obfs4GatewayCount int) {
failingOpenvpnGateways, failingObfs4Gateways := 0, 0
for _, gw := range tk.FailingGateways {
if gw.TransportType == "openvpn" {
failingOpenvpnGateways++
} else if gw.TransportType == "obfs4" {
failingObfs4Gateways++
}
}
if failingOpenvpnGateways < openvpnGatewayCount {
tk.TransportStatus["openvpn"] = "ok"
} else {
tk.TransportStatus["openvpn"] = "blocked"
}
if failingObfs4Gateways < obfs4GatewayCount {
tk.TransportStatus["obfs4"] = "ok"
} else {
tk.TransportStatus["obfs4"] = "blocked"
}
}
func newGatewayConnection(tcpConnect archival.TCPConnectEntry, transportType string) *GatewayConnection { func newGatewayConnection(tcpConnect archival.TCPConnectEntry, transportType string) *GatewayConnection {
return &GatewayConnection{ return &GatewayConnection{
IP: tcpConnect.IP, IP: tcpConnect.IP,
@ -160,21 +185,22 @@ func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
urlgetter.RegisterExtensions(measurement) urlgetter.RegisterExtensions(measurement)
caTarget := "https://black.riseup.net/ca.crt" caTarget := "https://black.riseup.net/ca.crt"
caGetter := urlgetter.Getter{ certPool := netx.NewDefaultCertPool()
Config: m.Config.Config,
Session: sess,
Target: caTarget,
}
log.Info("Getting CA certificate; please be patient...")
tk, err := caGetter.Get(ctx)
testkeys.AddCACertFetchTestKeys(tk)
if err != nil { multi := urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
log.Error("Getting CA certificate failed. Aborting test.") inputs := []urlgetter.MultiInput{
{Target: caTarget, Config: urlgetter.Config{
Method: "GET",
FailOnHTTPError: true,
}},
}
for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) {
tk := entry.TestKeys
testkeys.AddCACertFetchTestKeys(tk)
if tk.Failure != nil {
return nil return nil
} }
certPool := netx.NewDefaultCertPool()
if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok { if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok {
testkeys.CACertStatus = false testkeys.CACertStatus = false
testkeys.APIStatus = "blocked" testkeys.APIStatus = "blocked"
@ -182,8 +208,9 @@ func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
testkeys.APIFailure = &errorValue testkeys.APIFailure = &errorValue
return nil return nil
} }
}
inputs := []urlgetter.MultiInput{ inputs = []urlgetter.MultiInput{
// Here we need to provide the method explicitly. See // Here we need to provide the method explicitly. See
// https://github.com/ooni/probe-engine/issues/827. // https://github.com/ooni/probe-engine/issues/827.
@ -203,30 +230,34 @@ func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession,
FailOnHTTPError: true, FailOnHTTPError: true,
}}, }},
} }
multi := urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess} multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) { for entry := range multi.CollectOverall(ctx, inputs, 1, 50, "riseupvpn", callbacks) {
testkeys.UpdateProviderAPITestKeys(entry) testkeys.UpdateProviderAPITestKeys(entry)
} }
// test gateways now // test gateways now
testkeys.TransportStatus = map[string]string{}
gateways := parseGateways(testkeys) gateways := parseGateways(testkeys)
openvpnEndpoints := generateMultiInputs(gateways, "openvpn") openvpnEndpoints := generateMultiInputs(gateways, "openvpn")
obfs4Endpoints := generateMultiInputs(gateways, "obfs4") obfs4Endpoints := generateMultiInputs(gateways, "obfs4")
overallCount := len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints) overallCount := 1 + len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints)
// measure openvpn in parallel // measure openvpn in parallel
multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess} multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
for entry := range multi.CollectOverall(ctx, openvpnEndpoints, len(inputs), overallCount, "riseupvpn", callbacks) { for entry := range multi.CollectOverall(ctx, openvpnEndpoints, 1+len(inputs), overallCount, "riseupvpn", callbacks) {
testkeys.AddGatewayConnectTestKeys(entry, "openvpn") testkeys.AddGatewayConnectTestKeys(entry, "openvpn")
} }
// measure obfs4 in parallel // measure obfs4 in parallel
multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess} multi = urlgetter.Multi{Begin: measurement.MeasurementStartTimeSaved, Getter: m.Getter, Session: sess}
for entry := range multi.CollectOverall(ctx, obfs4Endpoints, len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) { for entry := range multi.CollectOverall(ctx, obfs4Endpoints, 1+len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) {
testkeys.AddGatewayConnectTestKeys(entry, "obfs4") testkeys.AddGatewayConnectTestKeys(entry, "obfs4")
} }
// set transport status based on gateway test results
testkeys.updateTransportStatus(len(openvpnEndpoints), len(obfs4Endpoints))
return nil return nil
} }
@ -290,6 +321,7 @@ type SummaryKeys struct {
APIBlocked bool `json:"api_blocked"` APIBlocked bool `json:"api_blocked"`
ValidCACert bool `json:"valid_ca_cert"` ValidCACert bool `json:"valid_ca_cert"`
FailingGateways int `json:"failing_gateways"` FailingGateways int `json:"failing_gateways"`
TransportStatus map[string]string `json:"transport_status"`
IsAnomaly bool `json:"-"` IsAnomaly bool `json:"-"`
} }
@ -303,7 +335,8 @@ func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, e
sk.APIBlocked = tk.APIStatus != "ok" sk.APIBlocked = tk.APIStatus != "ok"
sk.ValidCACert = tk.CACertStatus sk.ValidCACert = tk.CACertStatus
sk.FailingGateways = len(tk.FailingGateways) sk.FailingGateways = len(tk.FailingGateways)
sk.TransportStatus = tk.TransportStatus
sk.IsAnomaly = (sk.APIBlocked == true || tk.CACertStatus == false || sk.IsAnomaly = (sk.APIBlocked == true || tk.CACertStatus == false ||
sk.FailingGateways != 0) tk.TransportStatus["openvpn"] == "blocked" || tk.TransportStatus["obfs4"] == "blocked")
return sk, nil return sk, nil
} }

744
internal/engine/experiment/riseupvpn/riseupvpn_test.go
View File

@ -2,15 +2,14 @@ package riseupvpn_test
import ( import (
"context" "context"
"crypto/tls" "encoding/json"
"crypto/x509"
"errors"
"fmt" "fmt"
"io/ioutil" "io"
"math/rand" "strconv"
"net/http" "strings"
"testing" "testing"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/apex/log" "github.com/apex/log"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
@ -19,36 +18,204 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable" "github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
) )
const (
provider = `{
"api_uri": "https://api.black.riseup.net:443",
"api_version": "3",
"ca_cert_fingerprint": "SHA256: a5244308a1374709a9afce95e3ae47c1b44bc2398c0a70ccbf8b3a8a97f29494",
"ca_cert_uri": "https://black.riseup.net/ca.crt",
"default_language": "en",
"description": {
"en": "Riseup is a non-profit collective in Seattle that provides online communication tools for people and groups working toward liberatory social change."
},
"domain": "riseup.net",
"enrollment_policy": "closed",
"languages": [
"en"
],
"name": {
"en": "Riseup Networks"
},
"service": {
"allow_anonymous": true,
"allow_free": true,
"allow_limited_bandwidth": false,
"allow_paid": false,
"allow_registration": false,
"allow_unlimited_bandwidth": true,
"bandwidth_limit": 102400,
"default_service_level": 1,
"levels": {
"1": {
"description": "Please donate.",
"name": "free"
}
}
},
"services": [
"openvpn"
]
}`
eipservice = `{
"gateways": [
{
"capabilities": {
"adblock": false,
"filter_dns": false,
"limited": false,
"transport":[
{
"type":"openvpn",
"protocols":[
"tcp"
],
"ports":[
"443"
]
}
],
"user_ips": false
},
"host": "test1.riseup.net",
"ip_address": "123.456.123.456",
"location": "paris"
},
{
"capabilities": {
"adblock": false,
"filter_dns": false,
"limited": false,
"transport":[
{
"type":"obfs4",
"protocols":[
"tcp"
],
"ports":[
"23042"
],
"options": {
"cert": "XXXXXXXXXXXXXXXXXXXXXXXXX",
"iatMode": "0"
}
},
{
"type":"openvpn",
"protocols":[
"tcp"
],
"ports":[
"443"
]
}
],
"user_ips": false
},
"host": "test2.riseup.net",
"ip_address": "234.345.234.345",
"location": "seattle"
}
],
"locations": {
"paris": {
"country_code": "FR",
"hemisphere": "N",
"name": "Paris",
"timezone": "+2"
},
"seattle": {
"country_code": "US",
"hemisphere": "N",
"name": "Seattle",
"timezone": "-7"
}
},
"openvpn_configuration": {
"auth": "SHA1",
"cipher": "AES-128-CBC",
"keepalive": "10 30",
"tls-cipher": "DHE-RSA-AES128-SHA",
"tun-ipv6": true
},
"serial": 3,
"version": 3
}`
geoservice = `{"ip":"51.15.0.88","cc":"NL","city":"Haarlem","lat":52.381,"lon":4.6275,"gateways":["test1.riseup.net","test2.riseup.net"]}`
cacert = `-----BEGIN CERTIFICATE-----
MIIFjTCCA3WgAwIBAgIBATANBgkqhkiG9w0BAQ0FADBZMRgwFgYDVQQKDA9SaXNl
dXAgTmV0d29ya3MxGzAZBgNVBAsMEmh0dHBzOi8vcmlzZXVwLm5ldDEgMB4GA1UE
AwwXUmlzZXVwIE5ldHdvcmtzIFJvb3QgQ0EwHhcNMTQwNDI4MDAwMDAwWhcNMjQw
NDI4MDAwMDAwWjBZMRgwFgYDVQQKDA9SaXNldXAgTmV0d29ya3MxGzAZBgNVBAsM
Emh0dHBzOi8vcmlzZXVwLm5ldDEgMB4GA1UEAwwXUmlzZXVwIE5ldHdvcmtzIFJv
b3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC76J4ciMJ8Sg0m
TP7DF2DT9zNe0Csk4myoMFC57rfJeqsAlJCv1XMzBmXrw8wq/9z7XHv6n/0sWU7a
7cF2hLR33ktjwODlx7vorU39/lXLndo492ZBhXQtG1INMShyv+nlmzO6GT7ESfNE
LliFitEzwIegpMqxCIHXFuobGSCWF4N0qLHkq/SYUMoOJ96O3hmPSl1kFDRMtWXY
iw1SEKjUvpyDJpVs3NGxeLCaA7bAWhDY5s5Yb2fA1o8ICAqhowurowJpW7n5ZuLK
5VNTlNy6nZpkjt1QycYvNycffyPOFm/Q/RKDlvnorJIrihPkyniV3YY5cGgP+Qkx
HUOT0uLA6LHtzfiyaOqkXwc4b0ZcQD5Vbf6Prd20Ppt6ei0zazkUPwxld3hgyw58
m/4UIjG3PInWTNf293GngK2Bnz8Qx9e/6TueMSAn/3JBLem56E0WtmbLVjvko+LF
PM5xA+m0BmuSJtrD1MUCXMhqYTtiOvgLBlUm5zkNxALzG+cXB28k6XikXt6MRG7q
hzIPG38zwkooM55yy5i1YfcIi5NjMH6A+t4IJxxwb67MSb6UFOwg5kFokdONZcwj
shczHdG9gLKSBIvrKa03Nd3W2dF9hMbRu//STcQxOailDBQCnXXfAATj9pYzdY4k
ha8VCAREGAKTDAex9oXf1yRuktES4QIDAQABo2AwXjAdBgNVHQ4EFgQUC4tdmLVu
f9hwfK4AGliaet5KkcgwDgYDVR0PAQH/BAQDAgIEMAwGA1UdEwQFMAMBAf8wHwYD
VR0jBBgwFoAUC4tdmLVuf9hwfK4AGliaet5KkcgwDQYJKoZIhvcNAQENBQADggIB
AGzL+GRnYu99zFoy0bXJKOGCF5XUXP/3gIXPRDqQf5g7Cu/jYMID9dB3No4Zmf7v
qHjiSXiS8jx1j/6/Luk6PpFbT7QYm4QLs1f4BlfZOti2KE8r7KRDPIecUsUXW6P/
3GJAVYH/+7OjA39za9AieM7+H5BELGccGrM5wfl7JeEz8in+V2ZWDzHQO4hMkiTQ
4ZckuaL201F68YpiItBNnJ9N5nHr1MRiGyApHmLXY/wvlrOpclh95qn+lG6/2jk7
3AmihLOKYMlPwPakJg4PYczm3icFLgTpjV5sq2md9bRyAg3oPGfAuWHmKj2Ikqch
Td5CHKGxEEWbGUWEMP0s1A/JHWiCbDigc4Cfxhy56CWG4q0tYtnc2GMw8OAUO6Wf
Xu5pYKNkzKSEtT/MrNJt44tTZWbKV/Pi/N2Fx36my7TgTUj7g3xcE9eF4JV2H/sg
tsK3pwE0FEqGnT4qMFbixQmc8bGyuakr23wjMvfO7eZUxBuWYR2SkcP26sozF9PF
tGhbZHQVGZUTVPyvwahMUEhbPGVerOW0IYpxkm0x/eaWdTc4vPpf/rIlgbAjarnJ
UN9SaWRlWKSdP4haujnzCoJbM7dU9bjvlGZNyXEekgeT0W2qFeGGp+yyUWw8tNsp
0BuC1b7uW/bBn/xKm319wXVDvBgZgcktMolak39V7DVO
-----END CERTIFICATE-----`
eipserviceurl = "https://api.black.riseup.net:443/3/config/eip-service.json"
providerurl = "https://riseup.net/provider.json"
geoserviceurl = "https://api.black.riseup.net:9001/json"
cacerturl = "https://black.riseup.net/ca.crt"
openvpnurl1 = "tcpconnect://234.345.234.345:443"
openvpnurl2 = "tcpconnect://123.456.123.456:443"
obfs4url1 = "tcpconnect://234.345.234.345:23042"
)
var RequestResponse = map[string]string{
eipserviceurl: eipservice,
providerurl: provider,
geoserviceurl: geoservice,
cacerturl: cacert,
openvpnurl1: "",
openvpnurl2: "",
obfs4url1: "",
}
func TestNewExperimentMeasurer(t *testing.T) { func TestNewExperimentMeasurer(t *testing.T) {
measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
if measurer.ExperimentName() != "riseupvpn" { if measurer.ExperimentName() != "riseupvpn" {
t.Fatal("unexpected name") t.Fatal("unexpected name")
} }
if measurer.ExperimentVersion() != "0.1.0" { if measurer.ExperimentVersion() != "0.2.0" {
t.Fatal("unexpected version") t.Fatal("unexpected version")
} }
} }
func TestGood(t *testing.T) { func TestGood(t *testing.T) {
if testing.Short() { measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
t.Skip("skip test in short mode") cacerturl: true,
} eipserviceurl: true,
measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) providerurl: true,
measurement := new(model.Measurement) geoserviceurl: true,
err := measurer.Run( openvpnurl1: true,
context.Background(), openvpnurl2: true,
&mockable.Session{ obfs4url1: true,
MockableLogger: log.Log, }))
},
measurement,
model.NewPrinterCallbacks(log.Log),
)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*riseupvpn.TestKeys) tk := measurement.TestKeys.(*riseupvpn.TestKeys)
if tk.Agent != "" { if tk.Agent != "" {
t.Fatal("unexpected Agent: " + tk.Agent) t.Fatal("unexpected Agent: " + tk.Agent)
@ -59,21 +226,6 @@ func TestGood(t *testing.T) {
if tk.Failure != nil { if tk.Failure != nil {
t.Fatal("unexpected Failure") t.Fatal("unexpected Failure")
} }
if len(tk.NetworkEvents) <= 0 {
t.Fatal("no NetworkEvents?!")
}
if len(tk.Queries) <= 0 {
t.Fatal("no Queries?!")
}
if len(tk.Requests) <= 0 {
t.Fatal("no Requests?!")
}
if len(tk.TCPConnect) <= 0 {
t.Fatal("no TCPConnect?!")
}
if len(tk.TLSHandshakes) <= 0 {
t.Fatal("no TLSHandshakes?!")
}
if tk.APIFailure != nil { if tk.APIFailure != nil {
t.Fatal("unexpected ApiFailure") t.Fatal("unexpected ApiFailure")
} }
@ -86,6 +238,12 @@ func TestGood(t *testing.T) {
if tk.FailingGateways != nil { if tk.FailingGateways != nil {
t.Fatal("unexpected FailingGateways value") t.Fatal("unexpected FailingGateways value")
} }
if tk.TransportStatus == nil {
t.Fatal("unexpected nil TransportStatus struct ")
}
if tk.TransportStatus["openvpn"] != "ok" {
t.Fatal("unexpected openvpn transport status")
}
} }
// TestUpdateWithMixedResults tests if one operation failed // TestUpdateWithMixedResults tests if one operation failed
@ -132,13 +290,39 @@ func TestUpdateWithMixedResults(t *testing.T) {
if *tk.APIFailure != errorx.FailureEOFError { if *tk.APIFailure != errorx.FailureEOFError {
t.Fatal("invalid ApiFailure") t.Fatal("invalid ApiFailure")
} }
if tk.FailingGateways != nil {
t.Fatal("invalid FailingGateways")
}
if tk.TransportStatus != nil {
t.Fatal("invalid TransportStatus")
}
}
func TestInvalidCaCert(t *testing.T) {
requestResponseMap := map[string]string{
eipserviceurl: eipservice,
providerurl: provider,
geoserviceurl: geoservice,
cacerturl: "invalid",
openvpnurl1: "",
openvpnurl2: "",
obfs4url1: "",
}
measurer := riseupvpn.Measurer{
Config: riseupvpn.Config{},
Getter: generateMockGetter(requestResponseMap, map[string]bool{
cacerturl: true,
eipserviceurl: true,
providerurl: true,
geoserviceurl: true,
openvpnurl1: false,
openvpnurl2: true,
obfs4url1: true,
}),
} }
func TestFailureCaCertFetch(t *testing.T) {
measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
// we're cancelling immediately so that the CA Cert fetch fails defer cancel()
cancel()
sess := &mockable.Session{MockableLogger: log.Log} sess := &mockable.Session{MockableLogger: log.Log}
measurement := new(model.Measurement) measurement := new(model.Measurement)
@ -147,6 +331,26 @@ func TestFailureCaCertFetch(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
tk := measurement.TestKeys.(*riseupvpn.TestKeys)
if tk.CACertStatus == true {
t.Fatal("unexpected CaCertStatus")
}
if tk.APIStatus != "blocked" {
t.Fatal("ApiStatus should be blocked")
}
}
func TestFailureCaCertFetch(t *testing.T) {
measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
cacerturl: false,
eipserviceurl: true,
providerurl: true,
geoserviceurl: true,
openvpnurl1: true,
openvpnurl2: true,
obfs4url1: true,
}))
tk := measurement.TestKeys.(*riseupvpn.TestKeys) tk := measurement.TestKeys.(*riseupvpn.TestKeys)
if tk.CACertStatus != false { if tk.CACertStatus != false {
t.Fatal("invalid CACertStatus ") t.Fatal("invalid CACertStatus ")
@ -164,21 +368,15 @@ func TestFailureCaCertFetch(t *testing.T) {
} }
func TestFailureEipServiceBlocked(t *testing.T) { func TestFailureEipServiceBlocked(t *testing.T) {
if testing.Short() { measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
t.Skip("skip test in short mode") cacerturl: true,
} eipserviceurl: false,
measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) providerurl: true,
ctx, cancel := context.WithCancel(context.Background()) geoserviceurl: true,
defer cancel() openvpnurl1: true,
selfcensor.Enable(`{"PoisonSystemDNS":{"api.black.riseup.net":["NXDOMAIN"]}}`) openvpnurl2: true,
obfs4url1: true,
sess := &mockable.Session{MockableLogger: log.Log} }))
measurement := new(model.Measurement)
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*riseupvpn.TestKeys) tk := measurement.TestKeys.(*riseupvpn.TestKeys)
if tk.CACertStatus != true { if tk.CACertStatus != true {
t.Fatal("invalid CACertStatus ") t.Fatal("invalid CACertStatus ")
@ -202,21 +400,15 @@ func TestFailureEipServiceBlocked(t *testing.T) {
} }
func TestFailureProviderUrlBlocked(t *testing.T) { func TestFailureProviderUrlBlocked(t *testing.T) {
if testing.Short() { measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
t.Skip("skip test in short mode") cacerturl: true,
} eipserviceurl: true,
measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) providerurl: false,
ctx, cancel := context.WithCancel(context.Background()) geoserviceurl: true,
defer cancel() openvpnurl1: true,
selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.70:443":"REJECT"}}`) openvpnurl2: true,
obfs4url1: true,
sess := &mockable.Session{MockableLogger: log.Log} }))
measurement := new(model.Measurement)
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*riseupvpn.TestKeys) tk := measurement.TestKeys.(*riseupvpn.TestKeys)
for _, entry := range tk.Requests { for _, entry := range tk.Requests {
@ -240,21 +432,15 @@ func TestFailureProviderUrlBlocked(t *testing.T) {
} }
func TestFailureGeoIpServiceBlocked(t *testing.T) { func TestFailureGeoIpServiceBlocked(t *testing.T) {
if testing.Short() { measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
t.Skip("skip test in short mode") cacerturl: true,
} eipserviceurl: true,
measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{}) providerurl: true,
ctx, cancel := context.WithCancel(context.Background()) geoserviceurl: false,
defer cancel() openvpnurl1: true,
selfcensor.Enable(`{"BlockedEndpoints":{"198.252.153.107:9001":"REJECT"}}`) openvpnurl2: true,
obfs4url1: true,
sess := &mockable.Session{MockableLogger: log.Log} }))
measurement := new(model.Measurement)
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*riseupvpn.TestKeys) tk := measurement.TestKeys.(*riseupvpn.TestKeys)
if tk.CACertStatus != true { if tk.CACertStatus != true {
t.Fatal("invalid CACertStatus ") t.Fatal("invalid CACertStatus ")
@ -277,138 +463,16 @@ func TestFailureGeoIpServiceBlocked(t *testing.T) {
} }
} }
func TestFailureGateway(t *testing.T) { func TestFailureGateway1(t *testing.T) {
if testing.Short() { measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
t.Skip("skip test in short mode") cacerturl: true,
} eipserviceurl: true,
var testCases = [...]string{"openvpn", "obfs4"} providerurl: true,
eipService, err := fetchEipService() geoserviceurl: true,
if err != nil { openvpnurl1: false,
t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error()) openvpnurl2: true,
t.SkipNow() obfs4url1: true,
} }))
for _, tc := range testCases {
t.Run(fmt.Sprintf("testing censored transport %s", tc), func(t *testing.T) {
censoredGateway, err := selfCensorRandomGateway(eipService, tc)
if err == nil {
censorString := `{"BlockedEndpoints":{"` + censoredGateway.IP + `:` + censoredGateway.Port + `":"REJECT"}}`
selfcensor.Enable(censorString)
} else {
t.Log("Preconditions for the test are not met. Skipping due to: " + err.Error())
t.SkipNow()
}
// - run measurement
runGatewayTest(t, censoredGateway)
})
}
}
type SelfCensoredGateway struct {
IP string
Port string
}
func fetchEipService() (*riseupvpn.EipService, error) {
// - fetch client cert and add to certpool
caFetchClient := &http.Client{
Timeout: time.Second * 30,
}
caCertResponse, err := caFetchClient.Get("https://black.riseup.net/ca.crt")
if err != nil {
return nil, err
}
var bodyString string
if caCertResponse.StatusCode != http.StatusOK {
return nil, errors.New("unexpected HTTP response code")
}
bodyBytes, err := ioutil.ReadAll(caCertResponse.Body)
defer caCertResponse.Body.Close()
if err != nil {
return nil, err
}
bodyString = string(bodyBytes)
certs := x509.NewCertPool()
certs.AppendCertsFromPEM([]byte(bodyString))
// - fetch and parse eip-service.json
client := &http.Client{
Timeout: time.Second * 30,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: certs,
},
},
}
eipResponse, err := client.Get("https://api.black.riseup.net/3/config/eip-service.json")
if err != nil {
return nil, err
}
if eipResponse.StatusCode != http.StatusOK {
return nil, errors.New("Unexpected HTTP response code")
}
bodyBytes, err = ioutil.ReadAll(eipResponse.Body)
defer eipResponse.Body.Close()
if err != nil {
return nil, err
}
bodyString = string(bodyBytes)
eipService, err := riseupvpn.DecodeEIP3(bodyString)
if err != nil {
return nil, err
}
return eipService, nil
}
func selfCensorRandomGateway(eipService *riseupvpn.EipService, transportType string) (*SelfCensoredGateway, error) {
// - self censor random gateway
gateways := eipService.Gateways
if gateways == nil || len(gateways) == 0 {
return nil, errors.New("No gateways found")
}
var selfcensoredGateways []SelfCensoredGateway
for _, gateway := range gateways {
for _, transport := range gateway.Capabilities.Transport {
if transport.Type == transportType {
selfcensoredGateways = append(selfcensoredGateways, SelfCensoredGateway{IP: gateway.IPAddress, Port: transport.Ports[0]})
}
}
}
if len(selfcensoredGateways) == 0 {
return nil, errors.New("transport " + transportType + " doesn't seem to be supported.")
}
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
min := 0
max := len(selfcensoredGateways) - 1
randomIndex := rnd.Intn(max-min+1) + min
return &selfcensoredGateways[randomIndex], nil
}
func runGatewayTest(t *testing.T, censoredGateway *SelfCensoredGateway) {
measurer := riseupvpn.NewExperimentMeasurer(riseupvpn.Config{})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sess := &mockable.Session{MockableLogger: log.Log}
measurement := new(model.Measurement)
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*riseupvpn.TestKeys) tk := measurement.TestKeys.(*riseupvpn.TestKeys)
if tk.CACertStatus != true { if tk.CACertStatus != true {
t.Fatal("invalid CACertStatus ") t.Fatal("invalid CACertStatus ")
@ -418,18 +482,122 @@ func runGatewayTest(t *testing.T, censoredGateway *SelfCensoredGateway) {
t.Fatal("unexpected amount of failing gateways") t.Fatal("unexpected amount of failing gateways")
} }
entry := tk.FailingGateways[0] gw := tk.FailingGateways[0]
if entry.IP != censoredGateway.IP || fmt.Sprint(entry.Port) != censoredGateway.Port { if gw.IP != "234.345.234.345" {
t.Fatal("unexpected failed gateway configuration") t.Fatal("invalid failed gateway ip: " + fmt.Sprint(gw.IP))
}
if gw.Port != 443 {
t.Fatal("invalid failed gateway port: " + fmt.Sprint(gw.Port))
}
if gw.TransportType != "openvpn" {
t.Fatal("invalid failed transport type: " + fmt.Sprint(gw.TransportType))
} }
if tk.APIStatus == "blocked" { if tk.APIStatus == "blocked" {
t.Fatal("invalid ApiStatus", tk.APIStatus) t.Fatal("invalid ApiStatus")
} }
if tk.APIFailure != nil { if tk.APIFailure != nil {
t.Fatal("ApiFailure should be null") t.Fatal("ApiFailure should be null")
} }
if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] == "blocked" {
t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
}
if tk.TransportStatus == nil || tk.TransportStatus["obfs4"] == "blocked" {
t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
}
}
func TestFailureTransport(t *testing.T) {
measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{
cacerturl: true,
eipserviceurl: true,
providerurl: true,
geoserviceurl: true,
openvpnurl1: false,
openvpnurl2: false,
obfs4url1: false,
}))
tk := measurement.TestKeys.(*riseupvpn.TestKeys)
if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] != "blocked" {
t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
}
if tk.TransportStatus == nil || tk.TransportStatus["obfs4"] != "blocked" {
t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
}
}
func TestMissingTransport(t *testing.T) {
eipService, err := riseupvpn.DecodeEIP3(eipservice)
if err != nil {
t.Fatal("Preconditions for the test are not met.")
}
//remove obfs4 capability from 2. gateway so that our
//mock provider supports only openvpn
index := -1
transports := eipService.Gateways[1].Capabilities.Transport
for i, transport := range transports {
if transport.Type == "obfs4" {
index = i
break
}
}
if index == -1 {
t.Fatal("Preconditions for the test are not met. Default eipservice string should contain obfs4 transport.")
}
transports[index] = transports[len(transports)-1]
transports = transports[:len(transports)-1]
eipService.Gateways[1].Capabilities.Transport = transports
eipservicejson, err := json.Marshal(eipservice)
requestResponseMap := map[string]string{
eipserviceurl: string(eipservicejson),
providerurl: provider,
geoserviceurl: geoservice,
cacerturl: cacert,
openvpnurl1: "",
openvpnurl2: "",
obfs4url1: "",
}
measurer := riseupvpn.Measurer{
Config: riseupvpn.Config{},
Getter: generateMockGetter(requestResponseMap, map[string]bool{
cacerturl: true,
eipserviceurl: true,
providerurl: true,
geoserviceurl: true,
openvpnurl1: true,
openvpnurl2: true,
obfs4url1: false,
}),
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sess := &mockable.Session{MockableLogger: log.Log}
measurement := new(model.Measurement)
callbacks := model.NewPrinterCallbacks(log.Log)
err = measurer.Run(ctx, sess, measurement, callbacks)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*riseupvpn.TestKeys)
if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] != "blocked" {
t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
}
if _, found := tk.TransportStatus["obfs"]; found {
t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus))
}
} }
func TestSummaryKeysInvalidType(t *testing.T) { func TestSummaryKeysInvalidType(t *testing.T) {
@ -450,21 +618,27 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) {
APIStatus: "blocked", APIStatus: "blocked",
CACertStatus: true, CACertStatus: true,
FailingGateways: nil, FailingGateways: nil,
TransportStatus: nil,
}, },
sk: riseupvpn.SummaryKeys{ sk: riseupvpn.SummaryKeys{
APIBlocked: true, APIBlocked: true,
ValidCACert: true, ValidCACert: true,
IsAnomaly: true, IsAnomaly: true,
TransportStatus: nil,
FailingGateways: 0,
}, },
}, { }, {
tk: riseupvpn.TestKeys{ tk: riseupvpn.TestKeys{
APIStatus: "ok", APIStatus: "ok",
CACertStatus: false, CACertStatus: false,
FailingGateways: nil, FailingGateways: nil,
TransportStatus: nil,
}, },
sk: riseupvpn.SummaryKeys{ sk: riseupvpn.SummaryKeys{
ValidCACert: false, ValidCACert: false,
IsAnomaly: true, IsAnomaly: true,
FailingGateways: 0,
TransportStatus: nil,
}, },
}, { }, {
tk: riseupvpn.TestKeys{ tk: riseupvpn.TestKeys{
@ -475,13 +649,39 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) {
Port: 443, Port: 443,
TransportType: "obfs4", TransportType: "obfs4",
}}, }},
TransportStatus: map[string]string{
"obfs4": "blocked",
"openvpn": "ok",
},
}, },
sk: riseupvpn.SummaryKeys{ sk: riseupvpn.SummaryKeys{
FailingGateways: 1, FailingGateways: 1,
IsAnomaly: true, IsAnomaly: true,
ValidCACert: true, ValidCACert: true,
TransportStatus: map[string]string{
"obfs4": "blocked",
"openvpn": "ok",
}, },
}} },
}, {
tk: riseupvpn.TestKeys{
APIStatus: "ok",
CACertStatus: true,
FailingGateways: nil,
TransportStatus: map[string]string{
"openvpn": "ok",
},
},
sk: riseupvpn.SummaryKeys{
ValidCACert: true,
IsAnomaly: false,
FailingGateways: 0,
TransportStatus: map[string]string{
"openvpn": "ok",
},
},
},
}
for idx, tt := range tests { for idx, tt := range tests {
t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
m := &riseupvpn.Measurer{} m := &riseupvpn.Measurer{}
@ -498,3 +698,95 @@ func TestSummaryKeysWorksAsIntended(t *testing.T) {
}) })
} }
} }
func generateMockGetter(requestResponse map[string]string, responseStatus map[string]bool) urlgetter.MultiGetter {
return func(ctx context.Context, g urlgetter.Getter) (urlgetter.TestKeys, error) {
url := g.Target
responseBody, foundRequest := requestResponse[url]
isSuccessStatus, foundStatus := responseStatus[url]
if !foundRequest || !foundStatus {
return urlgetter.DefaultMultiGetter(ctx, g)
}
var failure *string
var failedOperation *string
isBlocked := !isSuccessStatus
var responseStatus int64 = 200
if isBlocked {
responseBody = ""
eofError := io.EOF.Error()
failure = &eofError
connectOperation := errorx.ConnectOperation
failedOperation = &connectOperation
responseStatus = 0
}
tcpConnect := archival.TCPConnectEntry{
// use some dummy IP/Port combination for URLs, we don't do DNS resolution
IP: "123.456.234.123",
Port: 443,
Status: archival.TCPConnectStatus{
Success: isSuccessStatus,
Blocked: &isBlocked,
Failure: failure,
},
}
if strings.Contains(url, "tcpconnect://") {
ipPort := strings.Split(strings.Split(url, "//")[1], ":")
port, err := strconv.ParseInt(ipPort[1], 10, 32)
if err == nil {
tcpConnect.IP = ipPort[0]
tcpConnect.Port = int(port)
}
}
tk := urlgetter.TestKeys{
Failure: failure,
FailedOperation: failedOperation,
HTTPResponseStatus: responseStatus,
HTTPResponseBody: responseBody,
Requests: []archival.RequestEntry{archival.RequestEntry{
Failure: failure,
Request: archival.HTTPRequest{
URL: url,
Body: archival.MaybeBinaryValue{},
BodyIsTruncated: false,
},
Response: archival.HTTPResponse{
Body: archival.HTTPBody{
Value: responseBody,
},
BodyIsTruncated: false,
}},
},
TCPConnect: []archival.TCPConnectEntry{tcpConnect},
}
return tk, nil
}
}
func generateDefaultMockGetter(responseStatuses map[string]bool) urlgetter.MultiGetter {
return generateMockGetter(RequestResponse, responseStatuses)
}
func runDefaultMockTest(t *testing.T, multiGetter urlgetter.MultiGetter) *model.Measurement {
measurer := riseupvpn.Measurer{
Config: riseupvpn.Config{},
Getter: multiGetter,
}
measurement := new(model.Measurement)
err := measurer.Run(
context.Background(),
&mockable.Session{
MockableLogger: log.Log,
},
measurement,
model.NewPrinterCallbacks(log.Log),
)
if err != nil {
t.Fatal(err)
}
return measurement
}