c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: merge probe-engine into probe-cli (#201)

Browse Source 
This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
2021-02-02 12:05:47 +01:00
committed by GitHub
parent b1ce300c8d
commit d57c78bc71
535 changed files with 66182 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/experiment/webconnectivity/connects.go
+58
View File
@@ -0,0 +1,58 @@
package webconnectivity
import (
"context"
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// ConnectsConfig contains the config for Connects
type ConnectsConfig struct {
Session model.ExperimentSession
TargetURL *url.URL
URLGetterURLs []string
}
// TODO(bassosimone): we should normalize the timings
// ConnectsResult contains the results of Connects
type ConnectsResult struct {
AllKeys []urlgetter.TestKeys
Successes int
Total int
}
// Connects performs 0..N connects (either using TCP or TLS) to
// check whether the resolved endpoints are reachable.
func Connects(ctx context.Context, config ConnectsConfig) (out ConnectsResult) {
out.AllKeys = []urlgetter.TestKeys{}
multi := urlgetter.Multi{Session: config.Session}
inputs := []urlgetter.MultiInput{}
for _, url := range config.URLGetterURLs {
inputs = append(inputs, urlgetter.MultiInput{
Config: urlgetter.Config{
TLSServerName: config.TargetURL.Hostname(),
},
Target: url,
})
}
outputs := multi.Collect(ctx, inputs, "check", ConnectsNoCallbacks{})
for multiout := range outputs {
out.AllKeys = append(out.AllKeys, multiout.TestKeys)
for _, entry := range multiout.TestKeys.TCPConnect {
if entry.Status.Success {
out.Successes++
}
out.Total++
}
}
return
}
// ConnectsNoCallbacks suppresses the callbacks
type ConnectsNoCallbacks struct{}
// OnProgress implements ExperimentCallbacks.OnProgress
func (ConnectsNoCallbacks) OnProgress(percentage float64, message string) {}
internal/engine/experiment/webconnectivity/connects_test.go
+55
View File
@@ -0,0 +1,55 @@
package webconnectivity_test
import (
"context"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
)
func TestConnectsSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
ctx := context.Background()
r := webconnectivity.Connects(ctx, webconnectivity.ConnectsConfig{
Session: newsession(t, false),
TargetURL: &url.URL{Scheme: "https", Host: "cloudflare-dns.com", Path: "/"},
URLGetterURLs: []string{
"tlshandshake://104.16.249.249:443", "tlshandshake://104.16.248.249:443",
"tlshandshake://[2606:4700::6810:f9f9]:443",
"tlshandshake://[2606:4700::6810:f8f9]:443",
},
})
if len(r.AllKeys) != 4 {
t.Fatal("unexpected number of TestKeys lists")
}
if r.Successes < 1 {
t.Fatal("no successes?!")
}
if r.Total != 4 {
t.Fatal("unexpected number of attempts")
}
}
func TestConnectsNoInput(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
ctx := context.Background()
r := webconnectivity.Connects(ctx, webconnectivity.ConnectsConfig{
Session: newsession(t, false),
TargetURL: &url.URL{Scheme: "https", Host: "cloudflare-dns.com", Path: "/"},
URLGetterURLs: []string{},
})
if len(r.AllKeys) != 0 {
t.Fatal("unexpected number of TestKeys lists")
}
if r.Successes != 0 {
t.Fatal("successes?!")
}
if r.Total != 0 {
t.Fatal("unexpected number of attempts")
}
}
internal/engine/experiment/webconnectivity/control.go
+84
View File
@@ -0,0 +1,84 @@
package webconnectivity
import (
"context"
"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
// ControlRequest is the request that we send to the control
type ControlRequest struct {
HTTPRequest string `json:"http_request"`
HTTPRequestHeaders map[string][]string `json:"http_request_headers"`
TCPConnect []string `json:"tcp_connect"`
}
// ControlTCPConnectResult is the result of the TCP connect
// attempt performed by the control vantage point.
type ControlTCPConnectResult struct {
Status bool `json:"status"`
Failure *string `json:"failure"`
}
// ControlHTTPRequestResult is the result of the HTTP request
// performed by the control vantage point.
type ControlHTTPRequestResult struct {
BodyLength int64 `json:"body_length"`
Failure *string `json:"failure"`
Title string `json:"title"`
Headers map[string]string `json:"headers"`
StatusCode int64 `json:"status_code"`
}
// ControlDNSResult is the result of the DNS lookup
// performed by the control vantage point.
type ControlDNSResult struct {
Failure *string `json:"failure"`
Addrs []string `json:"addrs"`
ASNs []int64 `json:"-"` // not visible from the JSON
}
// ControlResponse is the response from the control service.
type ControlResponse struct {
TCPConnect map[string]ControlTCPConnectResult `json:"tcp_connect"`
HTTPRequest ControlHTTPRequestResult `json:"http_request"`
DNS ControlDNSResult `json:"dns"`
}
// Control performs the control request and returns the response.
func Control(
ctx context.Context, sess model.ExperimentSession,
thAddr string, creq ControlRequest) (out ControlResponse, err error) {
clnt := httpx.Client{
BaseURL: thAddr,
HTTPClient: sess.DefaultHTTPClient(),
Logger: sess.Logger(),
}
sess.Logger().Infof("control %s...", creq.HTTPRequest)
// make sure error is wrapped
err = errorx.SafeErrWrapperBuilder{
Error: clnt.PostJSON(ctx, "/", creq, &out),
Operation: errorx.TopLevelOperation,
}.MaybeBuild()
sess.Logger().Infof("control %s... %+v", creq.HTTPRequest, err)
(&out.DNS).FillASNs(sess)
return
}
// FillASNs fills the ASNs array of ControlDNSResult. For each Addr inside
// of the ControlDNSResult structure, we obtain the corresponding ASN.
//
// This is very useful to know what ASNs were the IP addresses returned by
// the control according to the probe's ASN database.
func (dns *ControlDNSResult) FillASNs(sess model.ExperimentSession) {
dns.ASNs = []int64{}
for _, ip := range dns.Addrs {
// TODO(bassosimone): this would be more efficient if we'd open just
// once the database and then reuse it for every address.
asn, _, _ := geolocate.LookupASN(sess.ASNDatabasePath(), ip)
dns.ASNs = append(dns.ASNs, int64(asn))
}
}
internal/engine/experiment/webconnectivity/control_test.go
+36
View File
@@ -0,0 +1,36 @@
package webconnectivity_test
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
)
func TestFillASNsEmpty(t *testing.T) {
dns := new(webconnectivity.ControlDNSResult)
dns.FillASNs(new(mockable.Session))
if diff := cmp.Diff(dns.ASNs, []int64{}); diff != "" {
t.Fatal(diff)
}
}
func TestFillASNsNoDatabase(t *testing.T) {
dns := new(webconnectivity.ControlDNSResult)
dns.Addrs = []string{"8.8.8.8", "1.1.1.1"}
dns.FillASNs(new(mockable.Session))
if diff := cmp.Diff(dns.ASNs, []int64{0, 0}); diff != "" {
t.Fatal(diff)
}
}
func TestFillASNsSuccess(t *testing.T) {
sess := newsession(t, false)
dns := new(webconnectivity.ControlDNSResult)
dns.Addrs = []string{"8.8.8.8", "1.1.1.1"}
dns.FillASNs(sess)
if diff := cmp.Diff(dns.ASNs, []int64{15169, 13335}); diff != "" {
t.Fatal(diff)
}
}
internal/engine/experiment/webconnectivity/dnsanalysis.go
+99
View File
@@ -0,0 +1,99 @@
package webconnectivity
import (
"net"
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
// DNSAnalysisResult contains the results of analysing comparing
// the measurement and the control DNS results.
type DNSAnalysisResult struct {
DNSConsistency *string `json:"dns_consistency"`
}
// DNSNameError is the error returned by the control on NXDOMAIN
const DNSNameError = "dns_name_error"
var (
// DNSConsistent indicates that the measurement and the
// control have consistent DNS results.
DNSConsistent = "consistent"
// DNSInconsistent indicates that the measurement and the
// control have inconsistent DNS results.
DNSInconsistent = "inconsistent"
)
// DNSAnalysis compares the measurement and the control DNS results. This
// implementation is a simplified version of the implementation of the same
// check implemented in Measurement Kit v0.10.11.
func DNSAnalysis(URL *url.URL, measurement DNSLookupResult,
control ControlResponse) (out DNSAnalysisResult) {
// 0. start assuming it's not consistent
out.DNSConsistency = &DNSInconsistent
// 1. flip to consistent if we're targeting an IP address because the
// control will actually return dns_name_error in this case.
if net.ParseIP(URL.Hostname()) != nil {
out.DNSConsistency = &DNSConsistent
return
}
// 2. flip to consistent if the failures are compatible
if measurement.Failure != nil && control.DNS.Failure != nil {
switch *control.DNS.Failure {
case DNSNameError: // the control returns this on NXDOMAIN error
switch *measurement.Failure {
case errorx.FailureDNSNXDOMAINError:
out.DNSConsistency = &DNSConsistent
}
}
return
}
// 3. flip to consistent if measurement and control returned IP addresses
// that belong to the same Autonomous System(s).
//
// This specific check is present in MK's implementation.
//
// Note that this covers also the cases where the measurement contains only
// bogons while the control does not contain bogons.
//
// Note that this also covers the cases where results are equal.
const (
inMeasurement = 1 << 0
inControl = 1 << 1
inBoth = inMeasurement | inControl
)
asnmap := make(map[int64]int)
for _, asn := range measurement.Addrs {
asnmap[asn] |= inMeasurement
}
for _, asn := range control.DNS.ASNs {
asnmap[asn] |= inControl
}
for key, value := range asnmap {
// zero means that ASN lookup failed
if key != 0 && (value&inBoth) == inBoth {
out.DNSConsistency = &DNSConsistent
return
}
}
// 4. when ASN lookup failed (unlikely), check whether
// there is overlap in the returned IP addresses
ipmap := make(map[string]int)
for ip := range measurement.Addrs {
ipmap[ip] |= inMeasurement
}
for _, ip := range control.DNS.Addrs {
ipmap[ip] |= inControl
}
for key, value := range ipmap {
// just in case an empty string slipped through
if key != "" && (value&inBoth) == inBoth {
out.DNSConsistency = &DNSConsistent
return
}
}
// 5. conclude that measurement and control are inconsistent
return
}
internal/engine/experiment/webconnectivity/dnsanalysis_test.go
+193
View File
@@ -0,0 +1,193 @@
package webconnectivity_test
import (
"io"
"net/url"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
func TestDNSAnalysis(t *testing.T) {
measurementFailure := errorx.FailureDNSNXDOMAINError
controlFailure := webconnectivity.DNSNameError
eofFailure := io.EOF.Error()
type args struct {
URL *url.URL
measurement webconnectivity.DNSLookupResult
control webconnectivity.ControlResponse
}
tests := []struct {
name string
args args
wantOut webconnectivity.DNSAnalysisResult
}{{
name: "when the URL contains an IP address",
args: args{
URL: &url.URL{
Host: "10.0.0.1",
},
control: webconnectivity.ControlResponse{
DNS: webconnectivity.ControlDNSResult{
Failure: &controlFailure,
},
},
},
wantOut: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSConsistent,
},
}, {
name: "when the failures are not compatible",
args: args{
URL: &url.URL{
Host: "www.kerneltrap.org",
},
measurement: webconnectivity.DNSLookupResult{
Failure: &eofFailure,
},
control: webconnectivity.ControlResponse{
DNS: webconnectivity.ControlDNSResult{
Failure: &controlFailure,
},
},
},
wantOut: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSInconsistent,
},
}, {
name: "when the failures are compatible",
args: args{
URL: &url.URL{
Host: "www.kerneltrap.org",
},
measurement: webconnectivity.DNSLookupResult{
Failure: &measurementFailure,
},
control: webconnectivity.ControlResponse{
DNS: webconnectivity.ControlDNSResult{
Failure: &controlFailure,
},
},
},
wantOut: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSConsistent,
},
}, {
name: "when the ASNs are equal",
args: args{
URL: &url.URL{
Host: "fancy.dns",
},
measurement: webconnectivity.DNSLookupResult{
Addrs: map[string]int64{
"1.1.1.1": 15169,
"8.8.8.8": 13335,
},
},
control: webconnectivity.ControlResponse{
DNS: webconnectivity.ControlDNSResult{
ASNs: []int64{13335, 15169},
},
},
},
wantOut: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSConsistent,
},
}, {
name: "when the ASNs overlap",
args: args{
URL: &url.URL{
Host: "fancy.dns",
},
measurement: webconnectivity.DNSLookupResult{
Addrs: map[string]int64{
"1.1.1.1": 15169,
"8.8.8.8": 13335,
},
},
control: webconnectivity.ControlResponse{
DNS: webconnectivity.ControlDNSResult{
ASNs: []int64{13335, 13335},
},
},
},
wantOut: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSConsistent,
},
}, {
name: "when the ASNs do not overlap",
args: args{
URL: &url.URL{
Host: "fancy.dns",
},
measurement: webconnectivity.DNSLookupResult{
Addrs: map[string]int64{
"1.1.1.1": 15169,
"8.8.8.8": 15169,
},
},
control: webconnectivity.ControlResponse{
DNS: webconnectivity.ControlDNSResult{
ASNs: []int64{13335, 13335},
},
},
},
wantOut: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSInconsistent,
},
}, {
name: "when ASNs lookup fails but IPs overlap",
args: args{
URL: &url.URL{
Host: "fancy.dns",
},
measurement: webconnectivity.DNSLookupResult{
Addrs: map[string]int64{
"2001:4860:4860::8844": 0,
"8.8.4.4": 0,
},
},
control: webconnectivity.ControlResponse{
DNS: webconnectivity.ControlDNSResult{
Addrs: []string{"8.8.8.8", "2001:4860:4860::8844"},
ASNs: []int64{0, 0},
},
},
},
wantOut: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSConsistent,
},
}, {
name: "when ASNs lookup fails and IPs do not overlap",
args: args{
URL: &url.URL{
Host: "fancy.dns",
},
measurement: webconnectivity.DNSLookupResult{
Addrs: map[string]int64{
"2001:4860:4860::8888": 0,
"8.8.8.8": 0,
},
},
control: webconnectivity.ControlResponse{
DNS: webconnectivity.ControlDNSResult{
Addrs: []string{"8.8.4.4", "2001:4860:4860::8844"},
ASNs: []int64{0, 0},
},
},
},
wantOut: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSInconsistent,
},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := webconnectivity.DNSAnalysis(tt.args.URL, tt.args.measurement, tt.args.control)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
internal/engine/experiment/webconnectivity/dnslookup.go
+59
View File
@@ -0,0 +1,59 @@
package webconnectivity
import (
"context"
"fmt"
"net/url"
"sort"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// DNSLookupConfig contains settings for the DNS lookup.
type DNSLookupConfig struct {
Session model.ExperimentSession
URL *url.URL
}
// DNSLookupResult contains the result of the DNS lookup.
type DNSLookupResult struct {
Addrs map[string]int64
Failure *string
TestKeys urlgetter.TestKeys
}
// DNSLookup performs the DNS lookup part of Web Connectivity.
func DNSLookup(ctx context.Context, config DNSLookupConfig) (out DNSLookupResult) {
target := fmt.Sprintf("dnslookup://%s", config.URL.Hostname())
config.Session.Logger().Infof("%s...", target)
result, err := urlgetter.Getter{Session: config.Session, Target: target}.Get(ctx)
out.Addrs = make(map[string]int64)
for _, query := range result.Queries {
for _, answer := range query.Answers {
if answer.IPv4 != "" {
out.Addrs[answer.IPv4] = answer.ASN
continue
}
if answer.IPv6 != "" {
out.Addrs[answer.IPv6] = answer.ASN
}
}
}
config.Session.Logger().Infof("%s... %+v", target, err)
out.Failure = result.Failure
out.TestKeys = result
return
}
// Addresses returns the IP addresses in the DNSLookupResult
func (r DNSLookupResult) Addresses() (out []string) {
out = []string{}
for addr := range r.Addrs {
out = append(out, addr)
}
sort.Slice(out, func(i, j int) bool {
return out[i] < out[j]
})
return
}
internal/engine/experiment/webconnectivity/dnslookup_test.go
+79
View File
@@ -0,0 +1,79 @@
package webconnectivity_test
import (
"context"
"net"
"net/url"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
)
func TestDNSLookup(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
config := webconnectivity.DNSLookupConfig{
Session: newsession(t, true),
URL: &url.URL{Host: "dns.google"},
}
out := webconnectivity.DNSLookup(context.Background(), config)
if out.Failure != nil {
t.Fatal(*out.Failure)
}
if len(out.Addrs) < 1 {
t.Fatal("no addresses?!")
}
for addr, asn := range out.Addrs {
if net.ParseIP(addr) == nil {
t.Fatal("invalid addr")
}
if asn != 15169 {
t.Fatal("invalid asn")
}
}
if len(out.TestKeys.NetworkEvents) < 1 {
t.Fatal("no network events?!")
}
if len(out.TestKeys.Queries) < 1 {
t.Fatal("no queries?!")
}
}
func TestDNSLookupResult_Addresses(t *testing.T) {
type fields struct {
Addrs map[string]int64
Failure *string
TestKeys urlgetter.TestKeys
}
tests := []struct {
name string
fields fields
wantOut []string
}{{
name: "with no entries",
fields: fields{},
wantOut: []string{},
}, {
name: "with some entries",
fields: fields{
Addrs: map[string]int64{"1.1.1.1": 1, "2001:4860:4860::8844": 2},
},
wantOut: []string{"1.1.1.1", "2001:4860:4860::8844"},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := webconnectivity.DNSLookupResult{
Addrs: tt.fields.Addrs,
Failure: tt.fields.Failure,
TestKeys: tt.fields.TestKeys,
}
gotOut := r.Addresses()
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
internal/engine/experiment/webconnectivity/endpoints.go
+77
View File
@@ -0,0 +1,77 @@
package webconnectivity
import (
"net"
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
)
// EndpointInfo describes a TCP/TLS endpoint.
type EndpointInfo struct {
String string // String representation
URLGetterURL string // URL for urlgetter
}
// EndpointsList is a list of EndpointInfo
type EndpointsList []EndpointInfo
// Endpoints returns a list of endpoints for TCP connect
func (el EndpointsList) Endpoints() (out []string) {
out = []string{}
for _, ei := range el {
out = append(out, ei.String)
}
return
}
// URLs returns a list of URLs for TCP urlgetter
func (el EndpointsList) URLs() (out []string) {
out = []string{}
for _, ei := range el {
out = append(out, ei.URLGetterURL)
}
return
}
// NewEndpoints creates a list of TCP/TLS endpoints to test from the
// target URL and the list of resolved IP addresses.
func NewEndpoints(URL *url.URL, addrs []string) (out EndpointsList) {
out = EndpointsList{}
port := NewEndpointPort(URL)
for _, addr := range addrs {
endpoint := net.JoinHostPort(addr, port.Port)
out = append(out, EndpointInfo{
String: endpoint,
URLGetterURL: (&url.URL{Scheme: port.URLGetterScheme, Host: endpoint}).String(),
})
}
return
}
// EndpointPort is the port to be used by a TCP/TLS endpoint.
type EndpointPort struct {
URLGetterScheme string
Port string
}
// NewEndpointPort creates an EndpointPort from the given URL. This function
// panic if the scheme is not `http` or `https` as well as if the host is not
// valid. The latter should not happen if you used url.Parse.
func NewEndpointPort(URL *url.URL) (out EndpointPort) {
if URL.Scheme != "http" && URL.Scheme != "https" {
panic("passed an unexpected scheme")
}
switch URL.Scheme {
case "http":
out.URLGetterScheme, out.Port = "tcpconnect", "80"
case "https":
out.URLGetterScheme, out.Port = "tlshandshake", "443"
}
if URL.Host != URL.Hostname() {
_, port, err := net.SplitHostPort(URL.Host)
runtimex.PanicOnError(err, "SplitHostPort should not fail here")
out.Port = port
}
return
}
internal/engine/experiment/webconnectivity/endpoints_test.go
+224
View File
@@ -0,0 +1,224 @@
package webconnectivity_test
import (
"net/url"
"sync"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
)
func TestNewEndpointPortPanicsWithInvalidScheme(t *testing.T) {
counter := atomicx.NewInt64()
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer func() {
if recover() != nil {
counter.Add(1)
}
wg.Done()
}()
webconnectivity.NewEndpointPort(&url.URL{Scheme: "antani"})
}()
wg.Wait()
if counter.Load() != 1 {
t.Fatal("did not panic")
}
}
func TestNewEndpointPortPanicsWithInvalidHost(t *testing.T) {
counter := atomicx.NewInt64()
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer func() {
if recover() != nil {
counter.Add(1)
}
wg.Done()
}()
webconnectivity.NewEndpointPort(&url.URL{Scheme: "http", Host: "[::1"})
}()
wg.Wait()
if counter.Load() != 1 {
t.Fatal("did not panic")
}
}
func TestNewEndpointPortCommonCase(t *testing.T) {
type args struct {
URL *url.URL
}
tests := []struct {
name string
args args
wantOut webconnectivity.EndpointPort
}{{
name: "with http and no default port",
args: args{URL: &url.URL{
Scheme: "http",
Host: "www.example.com",
Path: "/",
}},
wantOut: webconnectivity.EndpointPort{
URLGetterScheme: "tcpconnect",
Port: "80",
},
}, {
name: "with https and no default port",
args: args{URL: &url.URL{
Scheme: "https",
Host: "www.example.com",
Path: "/",
}},
wantOut: webconnectivity.EndpointPort{
URLGetterScheme: "tlshandshake",
Port: "443",
},
}, {
name: "with http and custom port",
args: args{URL: &url.URL{
Scheme: "http",
Host: "www.example.com:11",
Path: "/",
}},
wantOut: webconnectivity.EndpointPort{
URLGetterScheme: "tcpconnect",
Port: "11",
},
}, {
name: "with https and custom port",
args: args{URL: &url.URL{
Scheme: "https",
Host: "www.example.com:11",
Path: "/",
}},
wantOut: webconnectivity.EndpointPort{
URLGetterScheme: "tlshandshake",
Port: "11",
},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := webconnectivity.NewEndpointPort(tt.args.URL)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestNewEndpoints(t *testing.T) {
type args struct {
URL *url.URL
addrs []string
}
tests := []struct {
name string
args args
wantOut webconnectivity.EndpointsList
}{{
name: "with all empty",
args: args{
URL: &url.URL{
Scheme: "http",
},
},
wantOut: webconnectivity.EndpointsList{},
}, {
name: "with some https endpoints",
args: args{
URL: &url.URL{
Scheme: "https",
},
addrs: []string{"1.1.1.1", "8.8.8.8"},
},
wantOut: webconnectivity.EndpointsList{{
URLGetterURL: "tlshandshake://1.1.1.1:443",
String: "1.1.1.1:443",
}, {
URLGetterURL: "tlshandshake://8.8.8.8:443",
String: "8.8.8.8:443",
}},
}, {
name: "with some http endpoints",
args: args{
URL: &url.URL{
Scheme: "http",
},
addrs: []string{"2001:4860:4860::8888", "2001:4860:4860::8844"},
},
wantOut: webconnectivity.EndpointsList{{
URLGetterURL: "tcpconnect://[2001:4860:4860::8888]:80",
String: "[2001:4860:4860::8888]:80",
}, {
URLGetterURL: "tcpconnect://[2001:4860:4860::8844]:80",
String: "[2001:4860:4860::8844]:80",
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := webconnectivity.NewEndpoints(tt.args.URL, tt.args.addrs)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestEndpointsList_Endpoints(t *testing.T) {
tests := []struct {
name string
el webconnectivity.EndpointsList
wantOut []string
}{{
name: "when empty",
wantOut: []string{},
}, {
name: "common case",
el: webconnectivity.EndpointsList{{
String: "1.1.1.1:443",
}, {
String: "8.8.8.8:80",
}},
wantOut: []string{"1.1.1.1:443", "8.8.8.8:80"},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := tt.el.Endpoints()
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestEndpointsList_URLs(t *testing.T) {
tests := []struct {
name string
el webconnectivity.EndpointsList
wantOut []string
}{{
name: "when empty",
wantOut: []string{},
}, {
name: "common case",
el: webconnectivity.EndpointsList{{
URLGetterURL: "tlshandshake://1.1.1.1:443",
}, {
URLGetterURL: "tcpconnect://8.8.8.8:80",
}},
wantOut: []string{"tlshandshake://1.1.1.1:443", "tcpconnect://8.8.8.8:80"},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := tt.el.URLs()
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
internal/engine/experiment/webconnectivity/httpanalysis.go
+250
View File
@@ -0,0 +1,250 @@
package webconnectivity
import (
"reflect"
"regexp"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// HTTPAnalysisResult contains the results of the analysis performed on the
// client. We obtain it by comparing the measurement and the control.
type HTTPAnalysisResult struct {
BodyLengthMatch *bool `json:"body_length_match"`
BodyProportion float64 `json:"body_proportion"`
StatusCodeMatch *bool `json:"status_code_match"`
HeadersMatch *bool `json:"headers_match"`
TitleMatch *bool `json:"title_match"`
}
// Log logs the results of the analysis
func (har HTTPAnalysisResult) Log(logger model.Logger) {
logger.Infof("BodyLengthMatch: %+v", internal.BoolPointerToString(har.BodyLengthMatch))
logger.Infof("BodyProportion: %+v", har.BodyProportion)
logger.Infof("StatusCodeMatch: %+v", internal.BoolPointerToString(har.StatusCodeMatch))
logger.Infof("HeadersMatch: %+v", internal.BoolPointerToString(har.HeadersMatch))
logger.Infof("TitleMatch: %+v", internal.BoolPointerToString(har.TitleMatch))
}
// HTTPAnalysis performs follow-up analysis on the webconnectivity measurement by
// comparing the measurement test keys and the control.
func HTTPAnalysis(tk urlgetter.TestKeys, ctrl ControlResponse) (out HTTPAnalysisResult) {
out.BodyLengthMatch, out.BodyProportion = HTTPBodyLengthChecks(tk, ctrl)
out.StatusCodeMatch = HTTPStatusCodeMatch(tk, ctrl)
out.HeadersMatch = HTTPHeadersMatch(tk, ctrl)
out.TitleMatch = HTTPTitleMatch(tk, ctrl)
return
}
// HTTPBodyLengthChecks returns whether the measured body is reasonably
// long as much as the control body as well as the proportion between
// the two bodies. This check may return nil, nil when such a
// comparison would actually not be applicable.
func HTTPBodyLengthChecks(
tk urlgetter.TestKeys, ctrl ControlResponse) (match *bool, proportion float64) {
control := ctrl.HTTPRequest.BodyLength
if control <= 0 {
return
}
if len(tk.Requests) <= 0 {
return
}
response := tk.Requests[0].Response
if response.BodyIsTruncated {
return
}
measurement := int64(len(response.Body.Value))
if measurement <= 0 {
return
}
const bodyProportionFactor = 0.7
if measurement >= control {
proportion = float64(control) / float64(measurement)
} else {
proportion = float64(measurement) / float64(control)
}
v := proportion > bodyProportionFactor
match = &v
return
}
// HTTPStatusCodeMatch returns whether the status code of the measurement
// matches the status code of the control, or nil if such comparison
// is actually not applicable.
func HTTPStatusCodeMatch(tk urlgetter.TestKeys, ctrl ControlResponse) (out *bool) {
control := ctrl.HTTPRequest.StatusCode
if len(tk.Requests) < 1 {
return // no real status code
}
measurement := tk.Requests[0].Response.Code
if control == 0 {
return // no real status code
}
if measurement == 0 {
return // no real status code
}
value := control == measurement
if value == true {
// if the status codes are equal, they clearly match
out = &value
return
}
// This fix is part of Web Connectivity in MK and in Python since
// basically forever; my recollection is that we want to work around
// cases where the test helper is failing(?!). Unlike previous
// implementations, this implementation avoids a false positive
// when both measurement and control statuses are 500.
if control/100 == 5 {
return
}
out = &value
return
}
// HTTPHeadersMatch returns whether uncommon headers match between control and
// measurement, or nil if check is not applicable.
func HTTPHeadersMatch(tk urlgetter.TestKeys, ctrl ControlResponse) *bool {
if len(tk.Requests) <= 0 {
return nil
}
if tk.Requests[0].Response.Code == 0 {
return nil
}
if ctrl.HTTPRequest.StatusCode == 0 {
return nil
}
control := ctrl.HTTPRequest.Headers
// Implementation note: using map because we only care about the
// keys being different and we ignore the values.
measurement := tk.Requests[0].Response.Headers
const (
inMeasurement = 1 << 0
inControl = 1 << 1
inBoth = inMeasurement | inControl
)
commonHeaders := map[string]bool{
"date": true,
"content-type": true,
"server": true,
"cache-control": true,
"vary": true,
"set-cookie": true,
"location": true,
"expires": true,
"x-powered-by": true,
"content-encoding": true,
"last-modified": true,
"accept-ranges": true,
"pragma": true,
"x-frame-options": true,
"etag": true,
"x-content-type-options": true,
"age": true,
"via": true,
"p3p": true,
"x-xss-protection": true,
"content-language": true,
"cf-ray": true,
"strict-transport-security": true,
"link": true,
"x-varnish": true,
}
matching := make(map[string]int)
ours := make(map[string]bool)
for key := range measurement {
key = strings.ToLower(key)
if _, ok := commonHeaders[key]; !ok {
matching[key] |= inMeasurement
}
ours[key] = true
}
theirs := make(map[string]bool)
for key := range control {
key = strings.ToLower(key)
if _, ok := commonHeaders[key]; !ok {
matching[key] |= inControl
}
theirs[key] = true
}
// if they are equal we're done
if good := reflect.DeepEqual(ours, theirs); good {
return &good
}
// compute the intersection of uncommon headers
var intersection int
for _, value := range matching {
if (value & inBoth) == inBoth {
intersection++
}
}
good := intersection > 0
return &good
}
// GetTitle returns the title or an empty string.
func GetTitle(measurementBody string) string {
re := regexp.MustCompile(`(?i)<title>([^<]{1,128})</title>`) // like MK
v := re.FindStringSubmatch(measurementBody)
if len(v) < 2 {
return ""
}
return v[1]
}
// HTTPTitleMatch returns whether the measurement and the control titles
// reasonably match, or nil if not applicable.
func HTTPTitleMatch(tk urlgetter.TestKeys, ctrl ControlResponse) (out *bool) {
if len(tk.Requests) <= 0 {
return
}
response := tk.Requests[0].Response
if response.Code == 0 {
return
}
if response.BodyIsTruncated {
return
}
if ctrl.HTTPRequest.StatusCode == 0 {
return
}
control := ctrl.HTTPRequest.Title
measurementBody := response.Body.Value
measurement := GetTitle(measurementBody)
if measurement == "" {
return
}
const (
inMeasurement = 1 << 0
inControl = 1 << 1
inBoth = inMeasurement | inControl
)
words := make(map[string]int)
// We don't consider to match words that are shorter than 5
// characters (5 is the average word length for english)
//
// The original implementation considered the word order but
// considering different languages it seems we could have less
// false positives by ignoring the word order.
const minWordLength = 5
for _, word := range strings.Split(measurement, " ") {
if len(word) >= minWordLength {
words[strings.ToLower(word)] |= inMeasurement
}
}
for _, word := range strings.Split(control, " ") {
if len(word) >= minWordLength {
words[strings.ToLower(word)] |= inControl
}
}
good := true
for _, score := range words {
if (score & inBoth) != inBoth {
good = false
break
}
}
return &good
}
internal/engine/experiment/webconnectivity/httpanalysis_test.go
+760
View File
@@ -0,0 +1,760 @@
package webconnectivity_test
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
)
func TestHTTPBodyLengthChecks(t *testing.T) {
var (
trueValue = true
falseValue = false
)
type args struct {
tk urlgetter.TestKeys
ctrl webconnectivity.ControlResponse
}
tests := []struct {
name string
args args
lengthMatch *bool
proportion float64
}{{
name: "nothing",
args: args{},
lengthMatch: nil,
}, {
name: "control length is nonzero",
args: args{
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
BodyLength: 1024,
},
},
},
lengthMatch: nil,
}, {
name: "response body is truncated",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
BodyIsTruncated: true,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
BodyLength: 1024,
},
},
},
lengthMatch: nil,
}, {
name: "response body length is zero",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
BodyLength: 1024,
},
},
},
lengthMatch: nil,
}, {
name: "match with bigger control",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Body: archival.MaybeBinaryValue{
Value: randx.Letters(768),
},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
BodyLength: 1024,
},
},
},
lengthMatch: &trueValue,
proportion: 0.75,
}, {
name: "match with bigger measurement",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Body: archival.MaybeBinaryValue{
Value: randx.Letters(1024),
},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
BodyLength: 768,
},
},
},
lengthMatch: &trueValue,
proportion: 0.75,
}, {
name: "not match with bigger control",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Body: archival.MaybeBinaryValue{
Value: randx.Letters(8),
},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
BodyLength: 16,
},
},
},
lengthMatch: &falseValue,
proportion: 0.5,
}, {
name: "match with bigger measurement",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Body: archival.MaybeBinaryValue{
Value: randx.Letters(16),
},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
BodyLength: 8,
},
},
},
lengthMatch: &falseValue,
proportion: 0.5,
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
match, proportion := webconnectivity.HTTPBodyLengthChecks(tt.args.tk, tt.args.ctrl)
if diff := cmp.Diff(tt.lengthMatch, match); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(tt.proportion, proportion); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestStatusCodeMatch(t *testing.T) {
var (
trueValue = true
falseValue = false
)
type args struct {
tk urlgetter.TestKeys
ctrl webconnectivity.ControlResponse
}
tests := []struct {
name string
args args
wantOut *bool
}{{
name: "with all zero",
args: args{},
}, {
name: "with a request but zero status codes",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{}},
},
},
}, {
name: "with equal status codes including 5xx",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 501,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 501,
},
},
},
wantOut: &trueValue,
}, {
name: "with different status codes and the control being 5xx",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 407,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 501,
},
},
},
wantOut: nil,
}, {
name: "with different status codes and the control being not 5xx",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 407,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 200,
},
},
},
wantOut: &falseValue,
}, {
name: "with only response status code and no control status code",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 200,
},
}},
},
},
}, {
name: "with only control status code and no response status code",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 0,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 200,
},
},
},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := webconnectivity.HTTPStatusCodeMatch(tt.args.tk, tt.args.ctrl)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestHeadersMatch(t *testing.T) {
var (
trueValue = true
falseValue = false
)
type args struct {
tk urlgetter.TestKeys
ctrl webconnectivity.ControlResponse
}
tests := []struct {
name string
args args
want *bool
}{{
name: "with no requests",
args: args{
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
"Date": "Mon Jul 13 21:05:43 CEST 2020",
"Antani": "Mascetti",
},
StatusCode: 200,
},
},
},
want: nil,
}, {
name: "with basically nothing",
args: args{},
want: nil,
}, {
name: "with request and no response status code",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
"Date": "Mon Jul 13 21:05:43 CEST 2020",
"Antani": "Mascetti",
},
StatusCode: 200,
},
},
},
want: nil,
}, {
name: "with no control status code",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Headers: map[string]archival.MaybeBinaryValue{
"Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"},
},
Code: 200,
},
}},
},
ctrl: webconnectivity.ControlResponse{},
},
want: nil,
}, {
name: "with no uncommon headers",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Headers: map[string]archival.MaybeBinaryValue{
"Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"},
},
Code: 200,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
"Date": "Mon Jul 13 21:05:43 CEST 2020",
},
StatusCode: 200,
},
},
},
want: &trueValue,
}, {
name: "with equal uncommon headers",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Headers: map[string]archival.MaybeBinaryValue{
"Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"},
"Antani": {Value: "MASCETTI"},
},
Code: 200,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
"Date": "Mon Jul 13 21:05:43 CEST 2020",
"Antani": "MELANDRI",
},
StatusCode: 200,
},
},
},
want: &trueValue,
}, {
name: "with different uncommon headers",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Headers: map[string]archival.MaybeBinaryValue{
"Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"},
"Antani": {Value: "MASCETTI"},
},
Code: 200,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
"Date": "Mon Jul 13 21:05:43 CEST 2020",
"Melandri": "MASCETTI",
},
StatusCode: 200,
},
},
},
want: &falseValue,
}, {
name: "with small uncommon intersection (X-Cache)",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Headers: map[string]archival.MaybeBinaryValue{
"Accept-Ranges": {Value: "bytes"},
"Age": {Value: "404727"},
"Cache-Control": {Value: "max-age=604800"},
"Content-Length": {Value: "1256"},
"Content-Type": {Value: "text/html; charset=UTF-8"},
"Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"},
"Etag": {Value: "\"3147526947\""},
"Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"},
"Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"},
"Server": {Value: "ECS (dcb/7F3C)"},
"Vary": {Value: "Accept-Encoding"},
"X-Cache": {Value: "HIT"},
},
Code: 200,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
// Note: the test helper was probably requesting the
// resource in a different way. There is content-length
// in this response, maybe it's using HTTP/1.0?
"Accept-Ranges": "bytes",
"Age": "469182",
"Cache-Control": "max-age=604800",
"Content-Type": "text/html; charset=UTF-8",
"Date": "Tue, 14 Jul 2020 22:26:08 GMT",
"Etag": "\"3147526947\"",
"Expires": "Tue, 21 Jul 2020 22:26:08 GMT",
"Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT",
"Server": "ECS (nyb/1D07)",
"Vary": "Accept-Encoding",
"X-Cache": "HIT",
},
StatusCode: 200,
},
},
},
want: &trueValue,
}, {
name: "with no uncommon intersection",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Headers: map[string]archival.MaybeBinaryValue{
"Accept-Ranges": {Value: "bytes"},
"Age": {Value: "404727"},
"Cache-Control": {Value: "max-age=604800"},
"Content-Length": {Value: "1256"},
"Content-Type": {Value: "text/html; charset=UTF-8"},
"Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"},
"Etag": {Value: "\"3147526947\""},
"Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"},
"Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"},
"Server": {Value: "ECS (dcb/7F3C)"},
"Vary": {Value: "Accept-Encoding"},
},
Code: 200,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
// Note: the test helper was probably requesting the
// resource in a different way. There is content-length
// in this response, maybe it's using HTTP/1.0?
"Accept-Ranges": "bytes",
"Age": "469182",
"Cache-Control": "max-age=604800",
"Content-Type": "text/html; charset=UTF-8",
"Date": "Tue, 14 Jul 2020 22:26:08 GMT",
"Etag": "\"3147526947\"",
"Expires": "Tue, 21 Jul 2020 22:26:08 GMT",
"Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT",
"Server": "ECS (nyb/1D07)",
"Vary": "Accept-Encoding",
},
StatusCode: 200,
},
},
},
want: &falseValue,
}, {
name: "with exactly equal headers",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Headers: map[string]archival.MaybeBinaryValue{
"Accept-Ranges": {Value: "bytes"},
"Age": {Value: "404727"},
"Cache-Control": {Value: "max-age=604800"},
"Content-Type": {Value: "text/html; charset=UTF-8"},
"Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"},
"Etag": {Value: "\"3147526947\""},
"Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"},
"Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"},
"Server": {Value: "ECS (dcb/7F3C)"},
"Vary": {Value: "Accept-Encoding"},
},
Code: 200,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
"Accept-Ranges": "bytes",
"Age": "469182",
"Cache-Control": "max-age=604800",
"Content-Type": "text/html; charset=UTF-8",
"Date": "Tue, 14 Jul 2020 22:26:08 GMT",
"Etag": "\"3147526947\"",
"Expires": "Tue, 21 Jul 2020 22:26:08 GMT",
"Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT",
"Server": "ECS (nyb/1D07)",
"Vary": "Accept-Encoding",
},
StatusCode: 200,
},
},
},
want: &trueValue,
}, {
name: "with equal headers except for the case",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Headers: map[string]archival.MaybeBinaryValue{
"accept-ranges": {Value: "bytes"},
"AGE": {Value: "404727"},
"cache-Control": {Value: "max-age=604800"},
"Content-TyPe": {Value: "text/html; charset=UTF-8"},
"DatE": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"},
"etag": {Value: "\"3147526947\""},
"expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"},
"Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"},
"SerVer": {Value: "ECS (dcb/7F3C)"},
"Vary": {Value: "Accept-Encoding"},
},
Code: 200,
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Headers: map[string]string{
"Accept-Ranges": "bytes",
"Age": "469182",
"Cache-Control": "max-age=604800",
"Content-Type": "text/html; charset=UTF-8",
"Date": "Tue, 14 Jul 2020 22:26:08 GMT",
"Etag": "\"3147526947\"",
"Expires": "Tue, 21 Jul 2020 22:26:08 GMT",
"Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT",
"Server": "ECS (nyb/1D07)",
"Vary": "Accept-Encoding",
},
StatusCode: 200,
},
},
},
want: &trueValue,
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := webconnectivity.HTTPHeadersMatch(tt.args.tk, tt.args.ctrl)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestTitleMatch(t *testing.T) {
var (
trueValue = true
falseValue = false
)
type args struct {
tk urlgetter.TestKeys
ctrl webconnectivity.ControlResponse
}
tests := []struct {
name string
args args
wantOut *bool
}{{
name: "with all empty",
args: args{},
wantOut: nil,
}, {
name: "with a request and no response",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{}},
},
},
wantOut: nil,
}, {
name: "with a response with truncated body",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 200,
BodyIsTruncated: true,
},
}},
},
},
wantOut: nil,
}, {
name: "with a response with good body",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 200,
Body: archival.MaybeBinaryValue{Value: "<HTML/>"},
},
}},
},
},
wantOut: nil,
}, {
name: "with all good but no titles",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 200,
Body: archival.MaybeBinaryValue{Value: "<HTML/>"},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 200,
Title: "",
},
},
},
wantOut: nil,
}, {
name: "reasonably common case where it succeeds",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 200,
Body: archival.MaybeBinaryValue{
Value: "<HTML><TITLE>La community di MSN</TITLE></HTML>"},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 200,
Title: "MSN Community",
},
},
},
wantOut: &trueValue,
}, {
name: "reasonably common case where it fails",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 200,
Body: archival.MaybeBinaryValue{
Value: "<HTML><TITLE>La communità di MSN</TITLE></HTML>"},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 200,
Title: "MSN Community",
},
},
},
wantOut: &falseValue,
}, {
name: "when the title is too long",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 200,
Body: archival.MaybeBinaryValue{
Value: "<HTML><TITLE>" + randx.Letters(1024) + "</TITLE></HTML>"},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 200,
Title: "MSN Community",
},
},
},
wantOut: nil,
}, {
name: "reasonably common case where it succeeds with case variations",
args: args{
tk: urlgetter.TestKeys{
Requests: []archival.RequestEntry{{
Response: archival.HTTPResponse{
Code: 200,
Body: archival.MaybeBinaryValue{
Value: "<HTML><TiTLe>La commUNity di MSN</tITLE></HTML>"},
},
}},
},
ctrl: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
StatusCode: 200,
Title: "MSN COmmunity",
},
},
},
wantOut: &trueValue,
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := webconnectivity.HTTPTitleMatch(tt.args.tk, tt.args.ctrl)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
internal/engine/experiment/webconnectivity/httpget.go
+50
View File
@@ -0,0 +1,50 @@
package webconnectivity
import (
"context"
"fmt"
"net/url"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// HTTPGetConfig contains the config for HTTPGet
type HTTPGetConfig struct {
Addresses []string
Session model.ExperimentSession
TargetURL *url.URL
}
// TODO(bassosimone): we should normalize the timings
// HTTPGetResult contains the results of HTTPGet
type HTTPGetResult struct {
TestKeys urlgetter.TestKeys
Failure *string
}
// HTTPGet performs the HTTP/HTTPS part of Web Connectivity.
func HTTPGet(ctx context.Context, config HTTPGetConfig) (out HTTPGetResult) {
addresses := strings.Join(config.Addresses, " ")
if addresses == "" {
// TODO(bassosimone): what to do in this case? We clearly
// cannot fill the DNS cache...
return
}
target := config.TargetURL.String()
config.Session.Logger().Infof("GET %s...", target)
domain := config.TargetURL.Hostname()
result, err := urlgetter.Getter{
Config: urlgetter.Config{
DNSCache: fmt.Sprintf("%s %s", domain, addresses),
},
Session: config.Session,
Target: target,
}.Get(ctx)
config.Session.Logger().Infof("GET %s... %+v", target, err)
out.Failure = result.Failure
out.TestKeys = result
return
}
internal/engine/experiment/webconnectivity/httpget_test.go
+27
View File
@@ -0,0 +1,27 @@
package webconnectivity_test
import (
"context"
"net/url"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
)
func TestHTTPGet(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
ctx := context.Background()
r := webconnectivity.HTTPGet(ctx, webconnectivity.HTTPGetConfig{
Addresses: []string{"104.16.249.249", "104.16.248.249"},
Session: newsession(t, false),
TargetURL: &url.URL{Scheme: "https", Host: "cloudflare-dns.com", Path: "/"},
})
if r.TestKeys.Failure != nil {
t.Fatal(*r.TestKeys.Failure)
}
if r.Failure != nil {
t.Fatal(*r.Failure)
}
}
internal/engine/experiment/webconnectivity/internal/internal.go
+23
View File
@@ -0,0 +1,23 @@
// Package internal contains internal code.
package internal
import "fmt"
// StringPointerToString converts a string pointer to a string. When the
// pointer is null, we return the "nil" string.
func StringPointerToString(v *string) (out string) {
out = "nil"
if v != nil {
out = fmt.Sprintf("%+v", *v)
}
return
}
// BoolPointerToString is like StringPointerToString but for bool.
func BoolPointerToString(v *bool) (out string) {
out = "nil"
if v != nil {
out = fmt.Sprintf("%+v", *v)
}
return
}
internal/engine/experiment/webconnectivity/internal/internal_test.go
+31
View File
@@ -0,0 +1,31 @@
package internal_test
import (
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
)
func TestStringPointerToString(t *testing.T) {
s := "ANTANI"
if internal.StringPointerToString(&s) != s {
t.Fatal("unexpected result")
}
if internal.StringPointerToString(nil) != "nil" {
t.Fatal("unexpected result")
}
}
func TestBoolPointerToString(t *testing.T) {
v := true
if internal.BoolPointerToString(&v) != "true" {
t.Fatal("unexpected result")
}
v = false
if internal.BoolPointerToString(&v) != "false" {
t.Fatal("unexpected result")
}
if internal.BoolPointerToString(nil) != "nil" {
t.Fatal("unexpected result")
}
}
internal/engine/experiment/webconnectivity/summary.go
+280
View File
@@ -0,0 +1,280 @@
package webconnectivity
import (
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
// The following set of status flags identifies in a more nuanced way the
// reason why we say something is blocked, accessible, etc.
//
// This is an experimental implementation. The objective is to start using
// it and learning from it, to eventually understand in which direction
// the Web Connectivity experiment should evolve. For example, there are
// a bunch of flags where our understandind is fuzzy or unclear.
//
// It also helps to write more precise unit and integration tests.
const (
StatusSuccessSecure = 1 << iota // success when using HTTPS
StatusSuccessCleartext // success when using HTTP
StatusSuccessNXDOMAIN // probe and control agree on NXDOMAIN
StatusAnomalyControlUnreachable // cannot access the control
StatusAnomalyControlFailure // control failed for HTTP
StatusAnomalyDNS // probe seems blocked by their DNS
StatusAnomalyHTTPDiff // probe and control do not agree on HTTP features
StatusAnomalyConnect // we saw an error when connecting
StatusAnomalyReadWrite // we saw an error when doing I/O
StatusAnomalyUnknown // we don't know when the error happened
StatusAnomalyTLSHandshake // we think error was during TLS handshake
StatusExperimentDNS // we noticed something in the DNS experiment
StatusExperimentConnect // ... in the connect experiment
StatusExperimentHTTP // ... in the HTTP experiment
StatusBugNoRequests // this should never happen
)
// Summary contains the Web Connectivity summary.
type Summary struct {
// Accessible is nil when the measurement failed, true if we do
// not think there was blocking, false in case of blocking.
Accessible *bool `json:"accessible"`
// BlockingReason indicates the cause of blocking when the Accessible
// variable is false. BlockingReason is meaningless otherwise.
//
// This is an intermediate variable used to compute Blocking, which
// is what OONI data consumers expect to see.
BlockingReason *string `json:"-"`
// Blocking implements the blocking variable as expected by OONI
// data consumers. See DetermineBlocking's docs.
Blocking interface{} `json:"blocking"`
// Status contains zero or more status flags. This is currently
// an experimental interface subject to change at any time.
Status int64 `json:"x_status"`
}
// DetermineBlocking returns the value of Summary.Blocking according to
// the expectations of OONI data consumers (nil|false|string).
//
// Measurement Kit sets blocking to false when accessible is true. The spec
// doesn't mention this possibility, as of 2019-08-20-001. Yet we implemented
// it back in 2016, with little explanation <https://git.io/JJHOl>.
//
// We eventually managed to link such a change with the 0.3.4 release of
// Measurement Kit <https://git.io/JJHOS>. This led us to find out the
// related issue #867 <https://git.io/JJHOH>. From this issue it become
// clear that the change on Measurement Kit was applied to mirror a change
// implemented in OONI Probe Legacy <https://git.io/JJH3T>. In such a
// change, determine_blocking() was modified to return False in case no
// blocking was detected, to distinguish this case from the case where
// there was an early failure in the experiment.
//
// Indeed, the OONI Android app uses the case where `blocking` is `null`
// to flag failed tests. Instead, success is identified by `blocking` being
// false and all other cases indicate anomaly <https://git.io/JJH3C>.
//
// Because of that, we must preserve the original behaviour.
func DetermineBlocking(s Summary) interface{} {
if s.Accessible != nil && *s.Accessible == true {
return false
}
return s.BlockingReason
}
// Log logs the summary using the provided logger.
func (s Summary) Log(logger model.Logger) {
logger.Infof("Blocking: %+v", internal.StringPointerToString(s.BlockingReason))
logger.Infof("Accessible: %+v", internal.BoolPointerToString(s.Accessible))
}
// Summarize computes the summary from the TestKeys.
func Summarize(tk *TestKeys) (out Summary) {
// Make sure we correctly set out.Blocking's value.
defer func() {
out.Blocking = DetermineBlocking(out)
}()
var (
accessible = true
inaccessible = false
dns = "dns"
httpDiff = "http-diff"
httpFailure = "http-failure"
tcpIP = "tcp_ip"
)
// If the measurement was for an HTTPS website and the HTTP experiment
// succeded, then either there is a compromised CA in our pool (which is
// certifi-go), or there is transparent proxying, or we are actually
// speaking with the legit server. We assume the latter. This applies
// also to cases in which we are redirected to HTTPS.
if len(tk.Requests) > 0 && tk.Requests[0].Failure == nil &&
strings.HasPrefix(tk.Requests[0].Request.URL, "https://") {
out.Accessible = &accessible
out.Status |= StatusSuccessSecure
return
}
// If we couldn't contact the control, we cannot do much more here.
if tk.ControlFailure != nil {
out.Status |= StatusAnomalyControlUnreachable
return
}
// If DNS failed with NXDOMAIN and the control DNS is consistent, then it
// means this website does not exist anymore.
if tk.DNSExperimentFailure != nil &&
*tk.DNSExperimentFailure == errorx.FailureDNSNXDOMAINError &&
tk.DNSConsistency != nil && *tk.DNSConsistency == DNSConsistent {
// TODO(bassosimone): MK flags this as accessible. This result is debateable. We
// are doing what MK does. But we most likely want to make it better later.
//
// See <https://github.com/ooni/probe-cli/v3/internal/engine/issues/579>.
out.Accessible = &accessible
out.Status |= StatusSuccessNXDOMAIN | StatusExperimentDNS
return
}
// Otherwise, if DNS failed with NXDOMAIN, it's DNS based blocking.
// TODO(bassosimone): do we wanna include other errors here? Like timeout?
if tk.DNSExperimentFailure != nil &&
*tk.DNSExperimentFailure == errorx.FailureDNSNXDOMAINError {
out.Accessible = &inaccessible
out.BlockingReason = &dns
out.Status |= StatusAnomalyDNS | StatusExperimentDNS
return
}
// If we tried to connect more than once and never succeded and we were
// able to measure DNS consistency, then we can conclude something.
if tk.TCPConnectAttempts > 0 && tk.TCPConnectSuccesses <= 0 && tk.DNSConsistency != nil {
out.Status |= StatusAnomalyConnect | StatusExperimentConnect
switch *tk.DNSConsistency {
case DNSConsistent:
// If the DNS is consistent, then it's TCP/IP blocking.
out.BlockingReason = &tcpIP
out.Accessible = &inaccessible
case DNSInconsistent:
// Otherwise, the culprit is the DNS.
out.BlockingReason = &dns
out.Accessible = &inaccessible
out.Status |= StatusAnomalyDNS
default:
// this case should not happen with this implementation
// so it's fine to leave this as unknown
out.Status |= StatusAnomalyUnknown
}
return
}
// If the control failed for HTTP it's not immediate for us to
// say anything specific on this measurement.
if tk.Control.HTTPRequest.Failure != nil {
out.Status |= StatusAnomalyControlFailure
return
}
// Likewise, if we don't have requests to examine, leave it.
if len(tk.Requests) < 1 {
out.Status |= StatusBugNoRequests
return
}
// If the HTTP measurement failed there could be a bunch of reasons
// why this occurred, because of HTTP redirects. Try to guess what
// could have been wrong by inspecting the error code.
if tk.Requests[0].Failure != nil {
out.Status |= StatusExperimentHTTP
switch *tk.Requests[0].Failure {
case errorx.FailureConnectionRefused:
// This is possibly because a subsequent connection to some
// other endpoint has been blocked. We call this http-failure
// because this is what MK would actually do.
out.BlockingReason = &httpFailure
out.Accessible = &inaccessible
out.Status |= StatusAnomalyConnect
case errorx.FailureConnectionReset:
// We don't currently support TLS failures and we don't have a
// way to know if it was during TLS or later. So, for now we are
// going to call this error condition an http-failure.
out.BlockingReason = &httpFailure
out.Accessible = &inaccessible
out.Status |= StatusAnomalyReadWrite
case errorx.FailureDNSNXDOMAINError:
// This is possibly because a subsequent resolution to
// some other domain name has been blocked.
out.BlockingReason = &dns
out.Accessible = &inaccessible
out.Status |= StatusAnomalyDNS
case errorx.FailureEOFError:
// We have seen this happening with TLS handshakes as well as
// sometimes with HTTP blocking. So http-failure.
out.BlockingReason = &httpFailure
out.Accessible = &inaccessible
out.Status |= StatusAnomalyReadWrite
case errorx.FailureGenericTimeoutError:
// Alas, here we don't know whether it's connect or whether it's
// perhaps the TLS handshake. So use the same classification used by MK.
out.BlockingReason = &httpFailure
out.Accessible = &inaccessible
out.Status |= StatusAnomalyUnknown
case errorx.FailureSSLInvalidHostname,
errorx.FailureSSLInvalidCertificate,
errorx.FailureSSLUnknownAuthority:
// We treat these three cases equally. Misconfiguration is a bit
// less likely since we also checked with the control. Since there
// is no TLS, for now we're going to call this http-failure.
out.BlockingReason = &httpFailure
out.Accessible = &inaccessible
out.Status |= StatusAnomalyTLSHandshake
default:
// We have not been able to classify the error. Could this perhaps be
// caused by a programmer's error? Let us be conservative.
}
// So, good that we have classified the error. Yet, how long is the
// redirect chain? If it's exactly one and we have determined that we
// should not trust the resolver, then let's bet on the DNS. If the
// chain is longer, for now better to be conservative. (I would argue
// that with a lying DNS that's likely the culprit, honestly.)
if out.BlockingReason != nil && len(tk.Requests) == 1 &&
tk.DNSConsistency != nil && *tk.DNSConsistency == DNSInconsistent {
out.BlockingReason = &dns
out.Status |= StatusAnomalyDNS
}
return
}
// So the HTTP request did not fail in the measurement and did not
// fail in the control as well, didn't it? Then, let us try to guess
// whether we've got the expected webpage after all. This set of
// conditions is adapted from MK v0.10.11.
if tk.StatusCodeMatch != nil && *tk.StatusCodeMatch {
if tk.BodyLengthMatch != nil && *tk.BodyLengthMatch {
out.Accessible = &accessible
out.Status |= StatusSuccessCleartext
return
}
if tk.HeadersMatch != nil && *tk.HeadersMatch {
out.Accessible = &accessible
out.Status |= StatusSuccessCleartext
return
}
if tk.TitleMatch != nil && *tk.TitleMatch {
out.Accessible = &accessible
out.Status |= StatusSuccessCleartext
return
}
}
// Set the status flag first
out.Status |= StatusAnomalyHTTPDiff
// It seems we didn't get the expected web page. What now? Well, if
// the DNS does not seem trustworthy, let us blame it.
if tk.DNSConsistency != nil && *tk.DNSConsistency == DNSInconsistent {
out.BlockingReason = &dns
out.Accessible = &inaccessible
out.Status |= StatusAnomalyDNS
return
}
// The only remaining conclusion seems that the web page we have got
// doesn't match what we were expecting.
out.BlockingReason = &httpDiff
out.Accessible = &inaccessible
return
}
internal/engine/experiment/webconnectivity/summary_test.go
+459
View File
@@ -0,0 +1,459 @@
package webconnectivity_test
import (
"io"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
func TestSummarize(t *testing.T) {
var (
genericFailure = io.EOF.Error()
dns = "dns"
falseValue = false
httpDiff = "http-diff"
httpFailure = "http-failure"
nilstring *string
probeConnectionRefused = errorx.FailureConnectionRefused
probeConnectionReset = errorx.FailureConnectionReset
probeEOFError = errorx.FailureEOFError
probeNXDOMAIN = errorx.FailureDNSNXDOMAINError
probeTimeout = errorx.FailureGenericTimeoutError
probeSSLInvalidHost = errorx.FailureSSLInvalidHostname
probeSSLInvalidCert = errorx.FailureSSLInvalidCertificate
probeSSLUnknownAuth = errorx.FailureSSLUnknownAuthority
tcpIP = "tcp_ip"
trueValue = true
)
type args struct {
tk *webconnectivity.TestKeys
}
tests := []struct {
name string
args args
wantOut webconnectivity.Summary
}{{
name: "with an HTTPS request with no failure",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Request: archival.HTTPRequest{
URL: "https://www.kernel.org/",
},
Failure: nil,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: false,
Accessible: &trueValue,
Status: webconnectivity.StatusSuccessSecure,
},
}, {
name: "with failure in contacting the control",
args: args{
tk: &webconnectivity.TestKeys{
ControlFailure: &genericFailure,
},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: nilstring,
Accessible: nil,
Status: webconnectivity.StatusAnomalyControlUnreachable,
},
}, {
name: "with non-existing website",
args: args{
tk: &webconnectivity.TestKeys{
DNSExperimentFailure: &probeNXDOMAIN,
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSConsistent,
},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: false,
Accessible: &trueValue,
Status: webconnectivity.StatusSuccessNXDOMAIN |
webconnectivity.StatusExperimentDNS,
},
}, {
name: "with NXDOMAIN measured only by the probe",
args: args{
tk: &webconnectivity.TestKeys{
DNSExperimentFailure: &probeNXDOMAIN,
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSInconsistent,
},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &dns,
Blocking: &dns,
Accessible: &falseValue,
Status: webconnectivity.StatusAnomalyDNS |
webconnectivity.StatusExperimentDNS,
},
}, {
name: "with TCP total failure and consistent DNS",
args: args{
tk: &webconnectivity.TestKeys{
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSConsistent,
},
TCPConnectAttempts: 7,
TCPConnectSuccesses: 0,
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &tcpIP,
Blocking: &tcpIP,
Accessible: &falseValue,
Status: webconnectivity.StatusAnomalyConnect |
webconnectivity.StatusExperimentConnect,
},
}, {
name: "with TCP total failure and inconsistent DNS",
args: args{
tk: &webconnectivity.TestKeys{
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSInconsistent,
},
TCPConnectAttempts: 7,
TCPConnectSuccesses: 0,
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &dns,
Blocking: &dns,
Accessible: &falseValue,
Status: webconnectivity.StatusAnomalyConnect |
webconnectivity.StatusExperimentConnect |
webconnectivity.StatusAnomalyDNS,
},
}, {
name: "with TCP total failure and unexpected DNS consistency",
args: args{
tk: &webconnectivity.TestKeys{
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: func() *string {
s := "ANTANI"
return &s
}(),
},
TCPConnectAttempts: 7,
TCPConnectSuccesses: 0,
},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: nilstring,
Accessible: nil,
Status: webconnectivity.StatusAnomalyConnect |
webconnectivity.StatusExperimentConnect |
webconnectivity.StatusAnomalyUnknown,
},
}, {
name: "with failed control HTTP request",
args: args{
tk: &webconnectivity.TestKeys{
Control: webconnectivity.ControlResponse{
HTTPRequest: webconnectivity.ControlHTTPRequestResult{
Failure: &genericFailure,
},
},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: nilstring,
Accessible: nil,
Status: webconnectivity.StatusAnomalyControlFailure,
},
}, {
name: "with less that one request entry",
args: args{
tk: &webconnectivity.TestKeys{},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: nilstring,
Accessible: nil,
Status: webconnectivity.StatusBugNoRequests,
},
}, {
name: "with connection refused",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Failure: &probeConnectionRefused,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpFailure,
Blocking: &httpFailure,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyConnect,
},
}, {
name: "with connection reset",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Failure: &probeConnectionReset,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpFailure,
Blocking: &httpFailure,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyReadWrite,
},
}, {
name: "with NXDOMAIN",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Failure: &probeNXDOMAIN,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &dns,
Blocking: &dns,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyDNS,
},
}, {
name: "with EOF",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Failure: &probeEOFError,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpFailure,
Blocking: &httpFailure,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyReadWrite,
},
}, {
name: "with timeout",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Failure: &probeTimeout,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpFailure,
Blocking: &httpFailure,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyUnknown,
},
}, {
name: "with SSL invalid hostname",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Failure: &probeSSLInvalidHost,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpFailure,
Blocking: &httpFailure,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyTLSHandshake,
},
}, {
name: "with SSL invalid cert",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Failure: &probeSSLInvalidCert,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpFailure,
Blocking: &httpFailure,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyTLSHandshake,
},
}, {
name: "with SSL unknown auth",
args: args{
tk: &webconnectivity.TestKeys{
Requests: []archival.RequestEntry{{
Failure: &probeSSLUnknownAuth,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpFailure,
Blocking: &httpFailure,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyTLSHandshake,
},
}, {
name: "with SSL unknown auth _and_ untrustworthy DNS",
args: args{
tk: &webconnectivity.TestKeys{
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSInconsistent,
},
Requests: []archival.RequestEntry{{
Failure: &probeSSLUnknownAuth,
}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &dns,
Blocking: &dns,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyTLSHandshake |
webconnectivity.StatusAnomalyDNS,
},
}, {
name: "with SSL unknown auth _and_ untrustworthy DNS _and_ a longer chain",
args: args{
tk: &webconnectivity.TestKeys{
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSInconsistent,
},
Requests: []archival.RequestEntry{{
Failure: &probeSSLUnknownAuth,
}, {}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpFailure,
Blocking: &httpFailure,
Accessible: &falseValue,
Status: webconnectivity.StatusExperimentHTTP |
webconnectivity.StatusAnomalyTLSHandshake,
},
}, {
name: "with status code and body length matching",
args: args{
tk: &webconnectivity.TestKeys{
HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
StatusCodeMatch: &trueValue,
BodyLengthMatch: &trueValue,
},
Requests: []archival.RequestEntry{{}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: falseValue,
Accessible: &trueValue,
Status: webconnectivity.StatusSuccessCleartext,
},
}, {
name: "with status code and headers matching",
args: args{
tk: &webconnectivity.TestKeys{
HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
StatusCodeMatch: &trueValue,
HeadersMatch: &trueValue,
},
Requests: []archival.RequestEntry{{}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: falseValue,
Accessible: &trueValue,
Status: webconnectivity.StatusSuccessCleartext,
},
}, {
name: "with status code and title matching",
args: args{
tk: &webconnectivity.TestKeys{
HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
StatusCodeMatch: &trueValue,
TitleMatch: &trueValue,
},
Requests: []archival.RequestEntry{{}},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: falseValue,
Accessible: &trueValue,
Status: webconnectivity.StatusSuccessCleartext,
},
}, {
name: "with suspect http-diff and inconsistent DNS",
args: args{
tk: &webconnectivity.TestKeys{
HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
StatusCodeMatch: &falseValue,
TitleMatch: &trueValue,
},
Requests: []archival.RequestEntry{{}},
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSInconsistent,
},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &dns,
Blocking: &dns,
Accessible: &falseValue,
Status: webconnectivity.StatusAnomalyHTTPDiff |
webconnectivity.StatusAnomalyDNS,
},
}, {
name: "with suspect http-diff and consistent DNS",
args: args{
tk: &webconnectivity.TestKeys{
HTTPAnalysisResult: webconnectivity.HTTPAnalysisResult{
StatusCodeMatch: &falseValue,
TitleMatch: &trueValue,
},
Requests: []archival.RequestEntry{{}},
DNSAnalysisResult: webconnectivity.DNSAnalysisResult{
DNSConsistency: &webconnectivity.DNSConsistent,
},
},
},
wantOut: webconnectivity.Summary{
BlockingReason: &httpDiff,
Blocking: &httpDiff,
Accessible: &falseValue,
Status: webconnectivity.StatusAnomalyHTTPDiff,
},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotOut := webconnectivity.Summarize(tt.args.tk)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
internal/engine/experiment/webconnectivity/webconnectivity.go
+225
View File
@@ -0,0 +1,225 @@
// Package webconnectivity implements OONI's Web Connectivity experiment.
//
// See https://github.com/ooni/spec/blob/master/nettests/ts-017-web-connectivity.md
package webconnectivity
import (
"context"
"errors"
"net"
"net/url"
"strconv"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
)
const (
testName = "web_connectivity"
testVersion = "0.2.0"
)
// Config contains the experiment config.
type Config struct{}
// TestKeys contains webconnectivity test keys.
type TestKeys struct {
Agent string `json:"agent"`
ClientResolver string `json:"client_resolver"`
Retries *int64 `json:"retries"` // unused
SOCKSProxy *string `json:"socksproxy"` // unused
// DNS experiment
Queries []archival.DNSQueryEntry `json:"queries"`
DNSExperimentFailure *string `json:"dns_experiment_failure"`
DNSAnalysisResult
// Control experiment
ControlFailure *string `json:"control_failure"`
ControlRequest ControlRequest `json:"-"`
Control ControlResponse `json:"control"`
// TCP connect experiment
TCPConnect []archival.TCPConnectEntry `json:"tcp_connect"`
TCPConnectSuccesses int `json:"-"`
TCPConnectAttempts int `json:"-"`
// HTTP experiment
Requests []archival.RequestEntry `json:"requests"`
HTTPExperimentFailure *string `json:"http_experiment_failure"`
HTTPAnalysisResult
// Top-level analysis
Summary
}
// Measurer performs the measurement.
type Measurer struct {
Config Config
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return Measurer{Config: config}
}
// ExperimentName implements ExperimentMeasurer.ExperExperimentName.
func (m Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion.
func (m Measurer) ExperimentVersion() string {
return testVersion
}
var (
// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
ErrNoAvailableTestHelpers = errors.New("no available helpers")
// ErrNoInput indicates that no input was provided
ErrNoInput = errors.New("no input provided")
// ErrInputIsNotAnURL indicates that the input is not an URL.
ErrInputIsNotAnURL = errors.New("input is not an URL")
// ErrUnsupportedInput indicates that the input URL scheme is unsupported.
ErrUnsupportedInput = errors.New("unsupported input scheme")
)
// Run implements ExperimentMeasurer.Run.
func (m Measurer) Run(
ctx context.Context,
sess model.ExperimentSession,
measurement *model.Measurement,
callbacks model.ExperimentCallbacks,
) error {
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
tk := new(TestKeys)
measurement.TestKeys = tk
tk.Agent = "redirect"
tk.ClientResolver = sess.ResolverIP()
if measurement.Input == "" {
return ErrNoInput
}
URL, err := url.Parse(string(measurement.Input))
if err != nil {
return ErrInputIsNotAnURL
}
if URL.Scheme != "http" && URL.Scheme != "https" {
return ErrUnsupportedInput
}
// 1. find test helper
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
var testhelper *model.Service
for _, th := range testhelpers {
if th.Type == "https" {
testhelper = &th
break
}
}
if testhelper == nil {
return ErrNoAvailableTestHelpers
}
measurement.TestHelpers = map[string]interface{}{
"backend": testhelper,
}
// 2. perform the DNS lookup step
dnsResult := DNSLookup(ctx, DNSLookupConfig{Session: sess, URL: URL})
tk.Queries = append(tk.Queries, dnsResult.TestKeys.Queries...)
tk.DNSExperimentFailure = dnsResult.Failure
epnts := NewEndpoints(URL, dnsResult.Addresses())
sess.Logger().Infof("using control: %s", testhelper.Address)
// 3. perform the control measurement
tk.Control, err = Control(ctx, sess, testhelper.Address, ControlRequest{
HTTPRequest: URL.String(),
HTTPRequestHeaders: map[string][]string{
"Accept": {httpheader.Accept()},
"Accept-Language": {httpheader.AcceptLanguage()},
"User-Agent": {httpheader.UserAgent()},
},
TCPConnect: epnts.Endpoints(),
})
tk.ControlFailure = archival.NewFailure(err)
// 4. analyze DNS results
if tk.ControlFailure == nil {
tk.DNSAnalysisResult = DNSAnalysis(URL, dnsResult, tk.Control)
}
sess.Logger().Infof("DNS analysis result: %+v", internal.StringPointerToString(
tk.DNSAnalysisResult.DNSConsistency))
// 5. perform TCP/TLS connects
connectsResult := Connects(ctx, ConnectsConfig{
Session: sess,
TargetURL: URL,
URLGetterURLs: epnts.URLs(),
})
sess.Logger().Infof(
"TCP/TLS endpoints: %d/%d reachable", connectsResult.Successes, connectsResult.Total)
for _, tcpkeys := range connectsResult.AllKeys {
// rewrite TCPConnect to include blocking information - it is very
// sad that we're storing analysis result inside the measurement
tk.TCPConnect = append(tk.TCPConnect, ComputeTCPBlocking(
tcpkeys.TCPConnect, tk.Control.TCPConnect)...)
}
tk.TCPConnectAttempts = connectsResult.Total
tk.TCPConnectSuccesses = connectsResult.Successes
// 6. perform HTTP/HTTPS measurement
httpResult := HTTPGet(ctx, HTTPGetConfig{
Addresses: dnsResult.Addresses(),
Session: sess,
TargetURL: URL,
})
tk.HTTPExperimentFailure = httpResult.Failure
tk.Requests = append(tk.Requests, httpResult.TestKeys.Requests...)
// 7. compare HTTP measurement to control
tk.HTTPAnalysisResult = HTTPAnalysis(httpResult.TestKeys, tk.Control)
tk.HTTPAnalysisResult.Log(sess.Logger())
tk.Summary = Summarize(tk)
tk.Summary.Log(sess.Logger())
return nil
}
// ComputeTCPBlocking will return a copy of the input TCPConnect structure
// where we set the Blocking value depending on the control results.
func ComputeTCPBlocking(measurement []archival.TCPConnectEntry,
control map[string]ControlTCPConnectResult) (out []archival.TCPConnectEntry) {
out = []archival.TCPConnectEntry{}
for _, me := range measurement {
epnt := net.JoinHostPort(me.IP, strconv.Itoa(me.Port))
if ce, ok := control[epnt]; ok {
v := ce.Failure == nil && me.Status.Failure != nil
me.Status.Blocked = &v
}
out = append(out, me)
}
return
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with probe-cli
// therefore we should be careful when changing it.
type SummaryKeys struct {
Accessible bool `json:"accessible"`
Blocking string `json:"blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{IsAnomaly: false}
tk, ok := measurement.TestKeys.(*TestKeys)
if !ok {
return sk, errors.New("invalid test keys type")
}
sk.IsAnomaly = tk.BlockingReason != nil
if tk.BlockingReason != nil {
sk.Blocking = *tk.BlockingReason
}
sk.Accessible = tk.Accessible != nil && *tk.Accessible
return sk, nil
}
internal/engine/experiment/webconnectivity/webconnectivity_test.go
+417
View File
@@ -0,0 +1,417 @@
package webconnectivity_test
import (
"context"
"errors"
"fmt"
"io"
"testing"
"github.com/apex/log"
"github.com/google/go-cmp/cmp"
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
func TestNewExperimentMeasurer(t *testing.T) {
measurer := webconnectivity.NewExperimentMeasurer(webconnectivity.Config{})
if measurer.ExperimentName() != "web_connectivity" {
t.Fatal("unexpected name")
}
if measurer.ExperimentVersion() != "0.2.0" {
t.Fatal("unexpected version")
}
}
func TestSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
measurer := webconnectivity.NewExperimentMeasurer(webconnectivity.Config{})
ctx := context.Background()
// we need a real session because we need the web-connectivity helper
// as well as the ASN database
sess := newsession(t, true)
measurement := &model.Measurement{Input: "http://www.example.com"}
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*webconnectivity.TestKeys)
if tk.ControlFailure != nil {
t.Fatal("unexpected control_failure")
}
if tk.DNSExperimentFailure != nil {
t.Fatal("unexpected dns_experiment_failure")
}
if tk.HTTPExperimentFailure != nil {
t.Fatal("unexpected http_experiment_failure")
}
// TODO(bassosimone): write further checks here?
}
func TestMeasureWithCancelledContext(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
measurer := webconnectivity.NewExperimentMeasurer(webconnectivity.Config{})
ctx, cancel := context.WithCancel(context.Background())
cancel() // immediately fail
// we need a real session because we need the web-connectivity helper
sess := newsession(t, true)
measurement := &model.Measurement{Input: "http://www.example.com"}
callbacks := model.NewPrinterCallbacks(log.Log)
if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil {
t.Fatal(err)
}
tk := measurement.TestKeys.(*webconnectivity.TestKeys)
if *tk.ControlFailure != errorx.FailureInterrupted {
t.Fatal("unexpected control_failure")
}
if *tk.DNSExperimentFailure != errorx.FailureInterrupted {
t.Fatal("unexpected dns_experiment_failure")
}
if tk.HTTPExperimentFailure != nil {
t.Fatal("unexpected http_experiment_failure")
}
// TODO(bassosimone): write further checks here?
sk, err := measurer.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
if _, ok := sk.(webconnectivity.SummaryKeys); !ok {
t.Fatal("invalid type for summary keys")
}
}
func TestMeasureWithNoInput(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
measurer := webconnectivity.NewExperimentMeasurer(webconnectivity.Config{})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// we need a real session because we need the web-connectivity helper
sess := newsession(t, true)
measurement := &model.Measurement{Input: ""}
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if !errors.Is(err, webconnectivity.ErrNoInput) {
t.Fatal(err)
}
tk := measurement.TestKeys.(*webconnectivity.TestKeys)
if tk.ControlFailure != nil {
t.Fatal("unexpected control_failure")
}
if tk.DNSExperimentFailure != nil {
t.Fatal("unexpected dns_experiment_failure")
}
if tk.HTTPExperimentFailure != nil {
t.Fatal("unexpected http_experiment_failure")
}
// TODO(bassosimone): write further checks here?
}
func TestMeasureWithInputNotBeingAnURL(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
measurer := webconnectivity.NewExperimentMeasurer(webconnectivity.Config{})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// we need a real session because we need the web-connectivity helper
sess := newsession(t, true)
measurement := &model.Measurement{Input: "\t\t\t\t\t\t"}
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if !errors.Is(err, webconnectivity.ErrInputIsNotAnURL) {
t.Fatal(err)
}
tk := measurement.TestKeys.(*webconnectivity.TestKeys)
if tk.ControlFailure != nil {
t.Fatal("unexpected control_failure")
}
if tk.DNSExperimentFailure != nil {
t.Fatal("unexpected dns_experiment_failure")
}
if tk.HTTPExperimentFailure != nil {
t.Fatal("unexpected http_experiment_failure")
}
// TODO(bassosimone): write further checks here?
}
func TestMeasureWithUnsupportedInput(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
measurer := webconnectivity.NewExperimentMeasurer(webconnectivity.Config{})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// we need a real session because we need the web-connectivity helper
sess := newsession(t, true)
measurement := &model.Measurement{Input: "dnslookup://example.com"}
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if !errors.Is(err, webconnectivity.ErrUnsupportedInput) {
t.Fatal(err)
}
tk := measurement.TestKeys.(*webconnectivity.TestKeys)
if tk.ControlFailure != nil {
t.Fatal("unexpected control_failure")
}
if tk.DNSExperimentFailure != nil {
t.Fatal("unexpected dns_experiment_failure")
}
if tk.HTTPExperimentFailure != nil {
t.Fatal("unexpected http_experiment_failure")
}
// TODO(bassosimone): write further checks here?
}
func TestMeasureWithNoAvailableTestHelpers(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
measurer := webconnectivity.NewExperimentMeasurer(webconnectivity.Config{})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// we need a real session because we need the web-connectivity helper
sess := newsession(t, false)
measurement := &model.Measurement{Input: "https://www.example.com"}
callbacks := model.NewPrinterCallbacks(log.Log)
err := measurer.Run(ctx, sess, measurement, callbacks)
if !errors.Is(err, webconnectivity.ErrNoAvailableTestHelpers) {
t.Fatal(err)
}
tk := measurement.TestKeys.(*webconnectivity.TestKeys)
if tk.ControlFailure != nil {
t.Fatal("unexpected control_failure")
}
if tk.DNSExperimentFailure != nil {
t.Fatal("unexpected dns_experiment_failure")
}
if tk.HTTPExperimentFailure != nil {
t.Fatal("unexpected http_experiment_failure")
}
// TODO(bassosimone): write further checks here?
}
func newsession(t *testing.T, lookupBackends bool) model.ExperimentSession {
sess, err := engine.NewSession(engine.SessionConfig{
AssetsDir: "../../testdata",
AvailableProbeServices: []model.Service{{
Address: "https://ams-pg-test.ooni.org",
Type: "https",
}},
Logger: log.Log,
SoftwareName: "ooniprobe-engine",
SoftwareVersion: "0.0.1",
})
if err != nil {
t.Fatal(err)
}
if lookupBackends {
if err := sess.MaybeLookupBackends(); err != nil {
t.Fatal(err)
}
}
if err := sess.MaybeLookupLocation(); err != nil {
t.Fatal(err)
}
return sess
}
func TestComputeTCPBlocking(t *testing.T) {
var (
falseValue = false
trueValue = true
)
failure := io.EOF.Error()
anotherFailure := "unknown_error"
type args struct {
measurement []archival.TCPConnectEntry
control map[string]webconnectivity.ControlTCPConnectResult
}
tests := []struct {
name string
args args
want []archival.TCPConnectEntry
}{{
name: "with all empty",
args: args{},
want: []archival.TCPConnectEntry{},
}, {
name: "with control failure",
args: args{
measurement: []archival.TCPConnectEntry{{
IP: "1.1.1.1",
Port: 853,
Status: archival.TCPConnectStatus{
Failure: &failure,
Success: false,
},
}},
},
want: []archival.TCPConnectEntry{{
IP: "1.1.1.1",
Port: 853,
Status: archival.TCPConnectStatus{
Failure: &failure,
Success: false,
},
}},
}, {
name: "with failures on both ends",
args: args{
measurement: []archival.TCPConnectEntry{{
IP: "1.1.1.1",
Port: 853,
Status: archival.TCPConnectStatus{
Failure: &failure,
Success: false,
},
}},
control: map[string]webconnectivity.ControlTCPConnectResult{
"1.1.1.1:853": {
Failure: &anotherFailure,
Status: false,
},
},
},
want: []archival.TCPConnectEntry{{
IP: "1.1.1.1",
Port: 853,
Status: archival.TCPConnectStatus{
Blocked: &falseValue,
Failure: &failure,
Success: false,
},
}},
}, {
name: "with failure on the probe side",
args: args{
measurement: []archival.TCPConnectEntry{{
IP: "1.1.1.1",
Port: 853,
Status: archival.TCPConnectStatus{
Failure: &failure,
Success: false,
},
}},
control: map[string]webconnectivity.ControlTCPConnectResult{
"1.1.1.1:853": {
Failure: nil,
Status: true,
},
},
},
want: []archival.TCPConnectEntry{{
IP: "1.1.1.1",
Port: 853,
Status: archival.TCPConnectStatus{
Blocked: &trueValue,
Failure: &failure,
Success: false,
},
}},
}, {
name: "with failure on the control side",
args: args{
measurement: []archival.TCPConnectEntry{{
IP: "1.1.1.1",
Port: 853,
Status: archival.TCPConnectStatus{
Failure: nil,
Success: true,
},
}},
control: map[string]webconnectivity.ControlTCPConnectResult{
"1.1.1.1:853": {
Failure: &failure,
Status: false,
},
},
},
want: []archival.TCPConnectEntry{{
IP: "1.1.1.1",
Port: 853,
Status: archival.TCPConnectStatus{
Blocked: &falseValue,
Failure: nil,
Success: true,
},
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := webconnectivity.ComputeTCPBlocking(tt.args.measurement, tt.args.control)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestSummaryKeysInvalidType(t *testing.T) {
measurement := new(model.Measurement)
m := &webconnectivity.Measurer{}
_, err := m.GetSummaryKeys(measurement)
if err.Error() != "invalid test keys type" {
t.Fatal("not the error we expected")
}
}
func TestSummaryKeysWorksAsIntended(t *testing.T) {
failure := io.EOF.Error()
truy := true
tests := []struct {
tk webconnectivity.TestKeys
Accessible bool
Blocking string
isAnomaly bool
}{{
tk: webconnectivity.TestKeys{},
Accessible: false,
Blocking: "",
isAnomaly: false,
}, {
tk: webconnectivity.TestKeys{Summary: webconnectivity.Summary{
BlockingReason: &failure,
}},
Accessible: false,
Blocking: failure,
isAnomaly: true,
}, {
tk: webconnectivity.TestKeys{Summary: webconnectivity.Summary{
Accessible: &truy,
}},
Accessible: true,
Blocking: "",
isAnomaly: false,
}}
for idx, tt := range tests {
t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
m := &webconnectivity.Measurer{}
measurement := &model.Measurement{TestKeys: &tt.tk}
got, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
return
}
sk := got.(webconnectivity.SummaryKeys)
if sk.IsAnomaly != tt.isAnomaly {
t.Fatal("unexpected isAnomaly value")
}
if sk.Accessible != tt.Accessible {
t.Fatal("unexpected Accessible value")
}
if sk.Blocking != tt.Blocking {
t.Fatal("unexpected Accessible value")
}
})
}
}