c/ooni-probe-cli
Watch
1
0
Fork
You've already forked ooni-probe-cli
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

refactor(netx): merge archival, trace, and the savers (#772)

Browse source 
This diff creates a new package under netx called tracex that
contains everything we need to perform measurements using events
tracing and postprocessing (which is the technique with which
we implement most network experiments).

The general idea here is to (1) create a unique package out of
all of these packages; (2) clean up the code a bit (improve tests,
docs, apply more recent code patterns); (3) move the resulting
code as a toplevel package inside of internal.

Once this is done, netx can be further refactored to avoid
subpackages and we can search for more code to salvage/refactor.

See https://github.com/ooni/probe/issues/2121
This commit is contained in:
Simone Basso 2022-05-31 21:53:01 +02:00 • committed by GitHub
commit bbcd2e2280
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
53 changed files with 869 additions and 865 deletions
Download patch file Download diff file Expand all files Collapse all files

332
internal/engine/netx/tracex/archival.go Normal file
View file

@ -0,0 +1,332 @@
package tracex
import (
"crypto/x509"
"errors"
"net"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// Compatibility types
type (
ExtSpec = model.ArchivalExtSpec
TCPConnectEntry = model.ArchivalTCPConnectResult
TCPConnectStatus = model.ArchivalTCPConnectStatus
MaybeBinaryValue = model.ArchivalMaybeBinaryData
DNSQueryEntry = model.ArchivalDNSLookupResult
DNSAnswerEntry = model.ArchivalDNSAnswer
TLSHandshake = model.ArchivalTLSOrQUICHandshakeResult
HTTPBody = model.ArchivalHTTPBody
HTTPHeader = model.ArchivalHTTPHeader
RequestEntry = model.ArchivalHTTPRequestResult
HTTPRequest = model.ArchivalHTTPRequest
HTTPResponse = model.ArchivalHTTPResponse
NetworkEvent = model.ArchivalNetworkEvent
)
// Compatibility variables
var (
ExtDNS = model.ArchivalExtDNS
ExtNetevents = model.ArchivalExtNetevents
ExtHTTP = model.ArchivalExtHTTP
ExtTCPConnect = model.ArchivalExtTCPConnect
ExtTLSHandshake = model.ArchivalExtTLSHandshake
ExtTunnel = model.ArchivalExtTunnel
)
// NewTCPConnectList creates a new TCPConnectList
func NewTCPConnectList(begin time.Time, events []Event) []TCPConnectEntry {
var out []TCPConnectEntry
for _, event := range events {
if event.Name != netxlite.ConnectOperation {
continue
}
if event.Proto != "tcp" {
continue
}
// We assume Go is passing us legit data structures
ip, sport, _ := net.SplitHostPort(event.Address)
iport, _ := strconv.Atoi(sport)
out = append(out, TCPConnectEntry{
IP: ip,
Port: iport,
Status: TCPConnectStatus{
Failure: NewFailure(event.Err),
Success: event.Err == nil,
},
T: event.Time.Sub(begin).Seconds(),
})
}
return out
}
// NewFailure creates a failure nullable string from the given error
func NewFailure(err error) *string {
if err == nil {
return nil
}
// The following code guarantees that the error is always wrapped even
// when we could not actually hit our code that does the wrapping. A case
// in which this happen is with context deadline for HTTP.
err = netxlite.NewTopLevelGenericErrWrapper(err)
errWrapper := err.(*netxlite.ErrWrapper)
s := errWrapper.Failure
if s == "" {
s = "unknown_failure: errWrapper.Failure is empty"
}
return &s
}
// NewFailedOperation creates a failed operation string from the given error.
func NewFailedOperation(err error) *string {
if err == nil {
return nil
}
var (
errWrapper *netxlite.ErrWrapper
s = netxlite.UnknownOperation
)
if errors.As(err, &errWrapper) && errWrapper.Operation != "" {
s = errWrapper.Operation
}
return &s
}
func addheaders(
source http.Header,
destList *[]HTTPHeader,
destMap *map[string]MaybeBinaryValue,
) {
for key, values := range source {
for index, value := range values {
value := MaybeBinaryValue{Value: value}
// With the map representation we can only represent a single
// value for every key. Hence the list representation.
if index == 0 {
(*destMap)[key] = value
}
*destList = append(*destList, HTTPHeader{
Key: key,
Value: value,
})
}
}
sort.Slice(*destList, func(i, j int) bool {
return (*destList)[i].Key < (*destList)[j].Key
})
}
// NewRequestList returns the list for "requests"
func NewRequestList(begin time.Time, events []Event) []RequestEntry {
// OONI wants the last request to appear first
var out []RequestEntry
tmp := newRequestList(begin, events)
for i := len(tmp) - 1; i >= 0; i-- {
out = append(out, tmp[i])
}
return out
}
func newRequestList(begin time.Time, events []Event) []RequestEntry {
var (
out []RequestEntry
entry RequestEntry
)
for _, ev := range events {
switch ev.Name {
case "http_transaction_start":
entry = RequestEntry{}
entry.T = ev.Time.Sub(begin).Seconds()
case "http_request_body_snapshot":
entry.Request.Body.Value = string(ev.Data)
entry.Request.BodyIsTruncated = ev.DataIsTruncated
case "http_request_metadata":
entry.Request.Headers = make(map[string]MaybeBinaryValue)
addheaders(
ev.HTTPHeaders, &entry.Request.HeadersList, &entry.Request.Headers)
entry.Request.Method = ev.HTTPMethod
entry.Request.URL = ev.HTTPURL
entry.Request.Transport = ev.Transport
case "http_response_metadata":
entry.Response.Headers = make(map[string]MaybeBinaryValue)
addheaders(
ev.HTTPHeaders, &entry.Response.HeadersList, &entry.Response.Headers)
entry.Response.Code = int64(ev.HTTPStatusCode)
entry.Response.Locations = ev.HTTPHeaders.Values("Location")
case "http_response_body_snapshot":
entry.Response.Body.Value = string(ev.Data)
entry.Response.BodyIsTruncated = ev.DataIsTruncated
case "http_transaction_done":
entry.Failure = NewFailure(ev.Err)
out = append(out, entry)
}
}
return out
}
type dnsQueryType string
// NewDNSQueriesList returns a list of DNS queries.
func NewDNSQueriesList(begin time.Time, events []Event) []DNSQueryEntry {
// TODO(bassosimone): add support for CNAME lookups.
var out []DNSQueryEntry
for _, ev := range events {
if ev.Name != "resolve_done" {
continue
}
for _, qtype := range []dnsQueryType{"A", "AAAA"} {
entry := qtype.makequeryentry(begin, ev)
for _, addr := range ev.Addresses {
if qtype.ipoftype(addr) {
entry.Answers = append(
entry.Answers, qtype.makeanswerentry(addr))
}
}
if len(entry.Answers) <= 0 && ev.Err == nil {
// This allows us to skip cases where the server does not have
// an IPv6 address but has an IPv4 address. Instead, when we
// receive an error, we want to track its existence. The main
// issue here is that we are cheating, because we are creating
// entries representing queries, but we don't know what the
// resolver actually did, especially the system resolver. So,
// this output is just our best guess.
continue
}
out = append(out, entry)
}
}
return out
}
func (qtype dnsQueryType) ipoftype(addr string) bool {
switch qtype {
case "A":
return !strings.Contains(addr, ":")
case "AAAA":
return strings.Contains(addr, ":")
}
return false
}
func (qtype dnsQueryType) makeanswerentry(addr string) DNSAnswerEntry {
answer := DNSAnswerEntry{AnswerType: string(qtype)}
asn, org, _ := geolocate.LookupASN(addr)
answer.ASN = int64(asn)
answer.ASOrgName = org
switch qtype {
case "A":
answer.IPv4 = addr
case "AAAA":
answer.IPv6 = addr
}
return answer
}
func (qtype dnsQueryType) makequeryentry(begin time.Time, ev Event) DNSQueryEntry {
return DNSQueryEntry{
Engine: ev.Proto,
Failure: NewFailure(ev.Err),
Hostname: ev.Hostname,
QueryType: string(qtype),
ResolverAddress: ev.Address,
T: ev.Time.Sub(begin).Seconds(),
}
}
// NewNetworkEventsList returns a list of DNS queries.
func NewNetworkEventsList(begin time.Time, events []Event) []NetworkEvent {
var out []NetworkEvent
for _, ev := range events {
if ev.Name == netxlite.ConnectOperation {
out = append(out, NetworkEvent{
Address: ev.Address,
Failure: NewFailure(ev.Err),
Operation: ev.Name,
Proto: ev.Proto,
T: ev.Time.Sub(begin).Seconds(),
})
continue
}
if ev.Name == netxlite.ReadOperation {
out = append(out, NetworkEvent{
Failure: NewFailure(ev.Err),
Operation: ev.Name,
NumBytes: int64(ev.NumBytes),
T: ev.Time.Sub(begin).Seconds(),
})
continue
}
if ev.Name == netxlite.WriteOperation {
out = append(out, NetworkEvent{
Failure: NewFailure(ev.Err),
Operation: ev.Name,
NumBytes: int64(ev.NumBytes),
T: ev.Time.Sub(begin).Seconds(),
})
continue
}
if ev.Name == netxlite.ReadFromOperation {
out = append(out, NetworkEvent{
Address: ev.Address,
Failure: NewFailure(ev.Err),
Operation: ev.Name,
NumBytes: int64(ev.NumBytes),
T: ev.Time.Sub(begin).Seconds(),
})
continue
}
if ev.Name == netxlite.WriteToOperation {
out = append(out, NetworkEvent{
Address: ev.Address,
Failure: NewFailure(ev.Err),
Operation: ev.Name,
NumBytes: int64(ev.NumBytes),
T: ev.Time.Sub(begin).Seconds(),
})
continue
}
out = append(out, NetworkEvent{
Failure: NewFailure(ev.Err),
Operation: ev.Name,
T: ev.Time.Sub(begin).Seconds(),
})
}
return out
}
// NewTLSHandshakesList creates a new TLSHandshakesList
func NewTLSHandshakesList(begin time.Time, events []Event) []TLSHandshake {
var out []TLSHandshake
for _, ev := range events {
if !strings.Contains(ev.Name, "_handshake_done") {
continue
}
out = append(out, TLSHandshake{
Address: ev.Address,
CipherSuite: ev.TLSCipherSuite,
Failure: NewFailure(ev.Err),
NegotiatedProtocol: ev.TLSNegotiatedProto,
NoTLSVerify: ev.NoTLSVerify,
PeerCertificates: makePeerCerts(ev.TLSPeerCerts),
ServerName: ev.TLSServerName,
T: ev.Time.Sub(begin).Seconds(),
TLSVersion: ev.TLSVersion,
})
}
return out
}
func makePeerCerts(in []*x509.Certificate) (out []MaybeBinaryValue) {
for _, e := range in {
out = append(out, MaybeBinaryValue{Value: string(e.Raw)})
}
return
}

746
internal/engine/netx/tracex/archival_test.go Normal file
View file

@ -0,0 +1,746 @@
package tracex
import (
"context"
"crypto/x509"
"errors"
"io"
"net/http"
"reflect"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/gorilla/websocket"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestDNSQueryIPOfType(t *testing.T) {
type expectation struct {
qtype dnsQueryType
ip string
output bool
}
var expectations = []expectation{{
qtype: "A",
ip: "8.8.8.8",
output: true,
}, {
qtype: "A",
ip: "2a00:1450:4002:801::2004",
output: false,
}, {
qtype: "AAAA",
ip: "8.8.8.8",
output: false,
}, {
qtype: "AAAA",
ip: "2a00:1450:4002:801::2004",
output: true,
}, {
qtype: "ANTANI",
ip: "2a00:1450:4002:801::2004",
output: false,
}, {
qtype: "ANTANI",
ip: "8.8.8.8",
output: false,
}}
for _, exp := range expectations {
if exp.qtype.ipoftype(exp.ip) != exp.output {
t.Fatalf("failure for %+v", exp)
}
}
}
func TestNewTCPConnectList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []TCPConnectEntry
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
events: []Event{{
Addresses: []string{"8.8.8.8", "8.8.4.4"},
Hostname: "dns.google.com",
Name: "resolve_done",
Time: begin.Add(100 * time.Millisecond),
}, {
Address: "8.8.8.8:853",
Duration: 30 * time.Millisecond,
Name: netxlite.ConnectOperation,
Proto: "tcp",
Time: begin.Add(130 * time.Millisecond),
}, {
Address: "8.8.8.8:853",
Duration: 55 * time.Millisecond,
Name: netxlite.ConnectOperation,
Proto: "udp",
Time: begin.Add(130 * time.Millisecond),
}, {
Address: "8.8.4.4:53",
Duration: 50 * time.Millisecond,
Err: io.EOF,
Name: netxlite.ConnectOperation,
Proto: "tcp",
Time: begin.Add(180 * time.Millisecond),
}},
},
want: []TCPConnectEntry{{
IP: "8.8.8.8",
Port: 853,
Status: TCPConnectStatus{
Success: true,
},
T: 0.13,
}, {
IP: "8.8.4.4",
Port: 53,
Status: TCPConnectStatus{
Failure: NewFailure(io.EOF),
Success: false,
},
T: 0.18,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := NewTCPConnectList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) {
t.Error(cmp.Diff(got, tt.want))
}
})
}
}
func TestNewRequestList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []RequestEntry
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
events: []Event{{
Name: "http_transaction_start",
Time: begin.Add(10 * time.Millisecond),
}, {
Name: "http_request_body_snapshot",
Data: []byte("deadbeef"),
DataIsTruncated: false,
}, {
Name: "http_request_metadata",
HTTPHeaders: http.Header{
"User-Agent": []string{"miniooni/0.1.0-dev"},
},
HTTPMethod: "POST",
HTTPURL: "https://www.example.com/submit",
}, {
Name: "http_response_metadata",
HTTPHeaders: http.Header{
"Server": []string{"miniooni/0.1.0-dev"},
},
HTTPStatusCode: 200,
}, {
Name: "http_response_body_snapshot",
Data: []byte("{}"),
DataIsTruncated: false,
}, {
Name: "http_transaction_done",
}, {
Name: "http_transaction_start",
Time: begin.Add(20 * time.Millisecond),
}, {
Name: "http_request_metadata",
HTTPHeaders: http.Header{
"User-Agent": []string{"miniooni/0.1.0-dev"},
},
HTTPMethod: "GET",
HTTPURL: "https://www.example.com/result",
}, {
Name: "http_transaction_done",
Err: io.EOF,
}},
},
want: []RequestEntry{{
Failure: NewFailure(io.EOF),
Request: HTTPRequest{
HeadersList: []HTTPHeader{{
Key: "User-Agent",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"User-Agent": {Value: "miniooni/0.1.0-dev"},
},
Method: "GET",
URL: "https://www.example.com/result",
},
T: 0.02,
}, {
Request: HTTPRequest{
Body: MaybeBinaryValue{
Value: "deadbeef",
},
HeadersList: []HTTPHeader{{
Key: "User-Agent",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"User-Agent": {Value: "miniooni/0.1.0-dev"},
},
Method: "POST",
URL: "https://www.example.com/submit",
},
Response: HTTPResponse{
Body: MaybeBinaryValue{
Value: "{}",
},
Code: 200,
HeadersList: []HTTPHeader{{
Key: "Server",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"Server": {Value: "miniooni/0.1.0-dev"},
},
Locations: nil,
},
T: 0.01,
}},
}, {
// for an example of why we need to sort headers, see
// https://github.com/ooni/probe-engine/pull/751/checks?check_run_id=853562310
name: "run with redirect and headers to sort",
args: args{
begin: begin,
events: []Event{{
Name: "http_transaction_start",
Time: begin.Add(10 * time.Millisecond),
}, {
Name: "http_request_metadata",
HTTPHeaders: http.Header{
"User-Agent": []string{"miniooni/0.1.0-dev"},
},
HTTPMethod: "GET",
HTTPURL: "https://www.example.com/",
}, {
Name: "http_response_metadata",
HTTPHeaders: http.Header{
"Server": []string{"miniooni/0.1.0-dev"},
"Location": []string{"https://x.example.com", "https://y.example.com"},
},
HTTPStatusCode: 302,
}, {
Name: "http_transaction_done",
}},
},
want: []RequestEntry{{
Request: HTTPRequest{
HeadersList: []HTTPHeader{{
Key: "User-Agent",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"User-Agent": {Value: "miniooni/0.1.0-dev"},
},
Method: "GET",
URL: "https://www.example.com/",
},
Response: HTTPResponse{
Code: 302,
HeadersList: []HTTPHeader{{
Key: "Location",
Value: MaybeBinaryValue{
Value: "https://x.example.com",
},
}, {
Key: "Location",
Value: MaybeBinaryValue{
Value: "https://y.example.com",
},
}, {
Key: "Server",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"Server": {Value: "miniooni/0.1.0-dev"},
"Location": {Value: "https://x.example.com"},
},
Locations: []string{
"https://x.example.com", "https://y.example.com",
},
},
T: 0.01,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := NewRequestList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) {
t.Error(cmp.Diff(got, tt.want))
}
})
}
}
func TestNewDNSQueriesList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []DNSQueryEntry
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
events: []Event{{
Address: "1.1.1.1:853",
Addresses: []string{"8.8.8.8", "8.8.4.4"},
Hostname: "dns.google.com",
Name: "resolve_done",
Proto: "dot",
Time: begin.Add(100 * time.Millisecond),
}, {
Address: "8.8.8.8:853",
Duration: 30 * time.Millisecond,
Name: netxlite.ConnectOperation,
Proto: "tcp",
Time: begin.Add(130 * time.Millisecond),
}, {
Address: "8.8.4.4:53",
Duration: 50 * time.Millisecond,
Err: io.EOF,
Name: netxlite.ConnectOperation,
Proto: "tcp",
Time: begin.Add(180 * time.Millisecond),
}},
},
want: []DNSQueryEntry{{
Answers: []DNSAnswerEntry{{
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
IPv4: "8.8.8.8",
}, {
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
IPv4: "8.8.4.4",
}},
Engine: "dot",
Hostname: "dns.google.com",
QueryType: "A",
ResolverAddress: "1.1.1.1:853",
T: 0.1,
}},
}, {
name: "run with IPv6 results",
args: args{
begin: begin,
events: []Event{{
Addresses: []string{"2001:4860:4860::8888"},
Hostname: "dns.google.com",
Name: "resolve_done",
Time: begin.Add(200 * time.Millisecond),
}},
},
want: []DNSQueryEntry{{
Answers: []DNSAnswerEntry{{
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "AAAA",
IPv6: "2001:4860:4860::8888",
}},
Hostname: "dns.google.com",
QueryType: "AAAA",
T: 0.2,
}},
}, {
name: "run with errors",
args: args{
begin: begin,
events: []Event{{
Err: &netxlite.ErrWrapper{Failure: netxlite.FailureDNSNXDOMAINError},
Hostname: "dns.google.com",
Name: "resolve_done",
Time: begin.Add(200 * time.Millisecond),
}},
},
want: []DNSQueryEntry{{
Answers: nil,
Failure: NewFailure(
&netxlite.ErrWrapper{Failure: netxlite.FailureDNSNXDOMAINError}),
Hostname: "dns.google.com",
QueryType: "A",
T: 0.2,
}, {
Answers: nil,
Failure: NewFailure(
&netxlite.ErrWrapper{Failure: netxlite.FailureDNSNXDOMAINError}),
Hostname: "dns.google.com",
QueryType: "AAAA",
T: 0.2,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewDNSQueriesList(tt.args.begin, tt.args.events)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestNewNetworkEventsList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []NetworkEvent
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
events: []Event{{
Name: netxlite.ConnectOperation,
Address: "8.8.8.8:853",
Err: io.EOF,
Proto: "tcp",
Time: begin.Add(7 * time.Millisecond),
}, {
Name: netxlite.ReadOperation,
Err: context.Canceled,
NumBytes: 7117,
Time: begin.Add(11 * time.Millisecond),
}, {
Address: "8.8.8.8:853",
Name: netxlite.ReadFromOperation,
Err: context.Canceled,
NumBytes: 7117,
Time: begin.Add(11 * time.Millisecond),
}, {
Name: netxlite.WriteOperation,
Err: websocket.ErrBadHandshake,
NumBytes: 4114,
Time: begin.Add(14 * time.Millisecond),
}, {
Address: "8.8.8.8:853",
Name: netxlite.WriteToOperation,
Err: websocket.ErrBadHandshake,
NumBytes: 4114,
Time: begin.Add(14 * time.Millisecond),
}, {
Name: netxlite.CloseOperation,
Err: websocket.ErrReadLimit,
Time: begin.Add(17 * time.Millisecond),
}},
},
want: []NetworkEvent{{
Address: "8.8.8.8:853",
Failure: NewFailure(io.EOF),
Operation: netxlite.ConnectOperation,
Proto: "tcp",
T: 0.007,
}, {
Failure: NewFailure(context.Canceled),
NumBytes: 7117,
Operation: netxlite.ReadOperation,
T: 0.011,
}, {
Address: "8.8.8.8:853",
Failure: NewFailure(context.Canceled),
NumBytes: 7117,
Operation: netxlite.ReadFromOperation,
T: 0.011,
}, {
Failure: NewFailure(websocket.ErrBadHandshake),
NumBytes: 4114,
Operation: netxlite.WriteOperation,
T: 0.014,
}, {
Address: "8.8.8.8:853",
Failure: NewFailure(websocket.ErrBadHandshake),
NumBytes: 4114,
Operation: netxlite.WriteToOperation,
T: 0.014,
}, {
Failure: NewFailure(websocket.ErrReadLimit),
Operation: netxlite.CloseOperation,
T: 0.017,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := NewNetworkEventsList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) {
t.Error(cmp.Diff(got, tt.want))
}
})
}
}
func TestNewTLSHandshakesList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []TLSHandshake
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
events: []Event{{
Name: netxlite.CloseOperation,
Err: websocket.ErrReadLimit,
Time: begin.Add(17 * time.Millisecond),
}, {
Address: "131.252.210.176:443",
Name: "tls_handshake_done",
Err: io.EOF,
NoTLSVerify: false,
TLSCipherSuite: "SUITE",
TLSNegotiatedProto: "h2",
TLSPeerCerts: []*x509.Certificate{{
Raw: []byte("deadbeef"),
}, {
Raw: []byte("abad1dea"),
}},
TLSServerName: "x.org",
TLSVersion: "TLSv1.3",
Time: begin.Add(55 * time.Millisecond),
}},
},
want: []TLSHandshake{{
Address: "131.252.210.176:443",
CipherSuite: "SUITE",
Failure: NewFailure(io.EOF),
NegotiatedProtocol: "h2",
NoTLSVerify: false,
PeerCertificates: []MaybeBinaryValue{{
Value: "deadbeef",
}, {
Value: "abad1dea",
}},
ServerName: "x.org",
T: 0.055,
TLSVersion: "TLSv1.3",
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := NewTLSHandshakesList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) {
t.Error(cmp.Diff(got, tt.want))
}
})
}
}
func TestNewFailure(t *testing.T) {
type args struct {
err error
}
tests := []struct {
name string
args args
want *string
}{{
name: "when error is nil",
args: args{
err: nil,
},
want: nil,
}, {
name: "when error is wrapped and failure meaningful",
args: args{
err: &netxlite.ErrWrapper{
Failure: netxlite.FailureConnectionRefused,
},
},
want: func() *string {
s := netxlite.FailureConnectionRefused
return &s
}(),
}, {
name: "when error is wrapped and failure is not meaningful",
args: args{
err: &netxlite.ErrWrapper{},
},
want: func() *string {
s := "unknown_failure: errWrapper.Failure is empty"
return &s
}(),
}, {
name: "when error is not wrapped but wrappable",
args: args{err: io.EOF},
want: func() *string {
s := "eof_error"
return &s
}(),
}, {
name: "when the error is not wrapped and not wrappable",
args: args{
err: errors.New("use of closed socket 127.0.0.1:8080->10.0.0.1:22"),
},
want: func() *string {
s := "unknown_failure: use of closed socket [scrubbed]->[scrubbed]"
return &s
}(),
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewFailure(tt.args.err)
if tt.want == nil && got == nil {
return
}
if tt.want == nil && got != nil {
t.Errorf("NewFailure: want %+v, got %s", tt.want, *got)
return
}
if tt.want != nil && got == nil {
t.Errorf("NewFailure: want %s, got %+v", *tt.want, got)
return
}
if *tt.want != *got {
t.Errorf("NewFailure: want %s, got %s", *tt.want, *got)
return
}
})
}
}
func TestNewFailedOperation(t *testing.T) {
type args struct {
err error
}
tests := []struct {
name string
args args
want *string
}{{
name: "With no error",
args: args{
err: nil, // explicit
},
want: nil, // explicit
}, {
name: "With wrapped error and non-empty operation",
args: args{
err: &netxlite.ErrWrapper{
Failure: netxlite.FailureConnectionRefused,
Operation: netxlite.ConnectOperation,
},
},
want: (func() *string {
s := netxlite.ConnectOperation
return &s
})(),
}, {
name: "With wrapped error and empty operation",
args: args{
err: &netxlite.ErrWrapper{
Failure: netxlite.FailureConnectionRefused,
},
},
want: (func() *string {
s := netxlite.UnknownOperation
return &s
})(),
}, {
name: "With non wrapped error",
args: args{
err: io.EOF,
},
want: (func() *string {
s := netxlite.UnknownOperation
return &s
})(),
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewFailedOperation(tt.args.err)
if got == nil && tt.want == nil {
return
}
if got == nil && tt.want != nil {
t.Errorf("NewFailedOperation() = %v, want %v", got, tt.want)
return
}
if got != nil && tt.want == nil {
t.Errorf("NewFailedOperation() = %v, want %v", got, tt.want)
return
}
if got != nil && tt.want != nil && *got != *tt.want {
t.Errorf("NewFailedOperation() = %v, want %v", got, tt.want)
return
}
})
}
}

87
internal/engine/netx/tracex/dialer.go Normal file
View file

@ -0,0 +1,87 @@
package tracex
import (
"context"
"net"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// SaverDialer saves events occurring during the dial
type SaverDialer struct {
model.Dialer
Saver *Saver
}
// DialContext implements Dialer.DialContext
func (d *SaverDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
start := time.Now()
conn, err := d.Dialer.DialContext(ctx, network, address)
stop := time.Now()
d.Saver.Write(Event{
Address: address,
Duration: stop.Sub(start),
Err: err,
Name: netxlite.ConnectOperation,
Proto: network,
Time: stop,
})
return conn, err
}
// SaverConnDialer wraps the returned connection such that we
// collect all the read/write events that occur.
type SaverConnDialer struct {
model.Dialer
Saver *Saver
}
// DialContext implements Dialer.DialContext
func (d *SaverConnDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
conn, err := d.Dialer.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
return &saverConn{saver: d.Saver, Conn: conn}, nil
}
type saverConn struct {
net.Conn
saver *Saver
}
func (c *saverConn) Read(p []byte) (int, error) {
start := time.Now()
count, err := c.Conn.Read(p)
stop := time.Now()
c.saver.Write(Event{
Data: p[:count],
Duration: stop.Sub(start),
Err: err,
NumBytes: count,
Name: netxlite.ReadOperation,
Time: stop,
})
return count, err
}
func (c *saverConn) Write(p []byte) (int, error) {
start := time.Now()
count, err := c.Conn.Write(p)
stop := time.Now()
c.saver.Write(Event{
Data: p[:count],
Duration: stop.Sub(start),
Err: err,
NumBytes: count,
Name: netxlite.WriteOperation,
Time: stop,
})
return count, err
}
var _ model.Dialer = &SaverDialer{}
var _ model.Dialer = &SaverConnDialer{}
var _ net.Conn = &saverConn{}

138
internal/engine/netx/tracex/dialer_test.go Normal file
View file

@ -0,0 +1,138 @@
package tracex
import (
"context"
"errors"
"io"
"net"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestSaverDialerFailure(t *testing.T) {
expected := errors.New("mocked error")
saver := &Saver{}
dlr := &SaverDialer{
Dialer: &mocks.Dialer{
MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
return nil, expected
},
},
Saver: saver,
}
conn, err := dlr.DialContext(context.Background(), "tcp", "www.google.com:443")
if !errors.Is(err, expected) {
t.Fatal("expected another error here")
}
if conn != nil {
t.Fatal("expected nil conn here")
}
ev := saver.Read()
if len(ev) != 1 {
t.Fatal("expected a single event here")
}
if ev[0].Address != "www.google.com:443" {
t.Fatal("unexpected Address")
}
if ev[0].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if !errors.Is(ev[0].Err, expected) {
t.Fatal("unexpected Err")
}
if ev[0].Name != netxlite.ConnectOperation {
t.Fatal("unexpected Name")
}
if ev[0].Proto != "tcp" {
t.Fatal("unexpected Proto")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("unexpected Time")
}
}
func TestSaverConnDialerFailure(t *testing.T) {
expected := errors.New("mocked error")
saver := &Saver{}
dlr := &SaverConnDialer{
Dialer: &mocks.Dialer{
MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
return nil, expected
},
},
Saver: saver,
}
conn, err := dlr.DialContext(context.Background(), "tcp", "www.google.com:443")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if conn != nil {
t.Fatal("expected nil conn here")
}
}
func TestSaverConnDialerSuccess(t *testing.T) {
saver := &Saver{}
dlr := &SaverConnDialer{
Dialer: &SaverDialer{
Dialer: &mocks.Dialer{
MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
return &mocks.Conn{
MockRead: func(b []byte) (int, error) {
return 0, io.EOF
},
MockWrite: func(b []byte) (int, error) {
return 0, io.EOF
},
MockClose: func() error {
return io.EOF
},
MockLocalAddr: func() net.Addr {
return &net.TCPAddr{Port: 12345}
},
}, nil
},
},
Saver: saver,
},
Saver: saver,
}
conn, err := dlr.DialContext(context.Background(), "tcp", "www.google.com:443")
if err != nil {
t.Fatal("not the error we expected", err)
}
conn.Read(nil)
conn.Write(nil)
conn.Close()
events := saver.Read()
if len(events) != 3 {
t.Fatal("unexpected number of events saved", len(events))
}
if events[0].Name != "connect" {
t.Fatal("expected a connect event")
}
saverCheckConnectEvent(t, &events[0])
if events[1].Name != "read" {
t.Fatal("expected a read event")
}
saverCheckReadEvent(t, &events[1])
if events[2].Name != "write" {
t.Fatal("expected a write event")
}
saverCheckWriteEvent(t, &events[2])
}
func saverCheckConnectEvent(t *testing.T, ev *Event) {
// TODO(bassosimone): implement
}
func saverCheckReadEvent(t *testing.T, ev *Event) {
// TODO(bassosimone): implement
}
func saverCheckWriteEvent(t *testing.T, ev *Event) {
// TODO(bassosimone): implement
}

2
internal/engine/netx/tracex/doc.go Normal file
View file

@ -0,0 +1,2 @@
// Package tracex contains code to perform measurements using tracing.
package tracex

60
internal/engine/netx/tracex/event.go Normal file
View file

@ -0,0 +1,60 @@
package tracex
import (
"crypto/tls"
"crypto/x509"
"errors"
"net/http"
"time"
)
// Event is one of the events within a trace
type Event struct {
Addresses []string `json:",omitempty"`
Address string `json:",omitempty"`
DNSQuery []byte `json:",omitempty"`
DNSReply []byte `json:",omitempty"`
DataIsTruncated bool `json:",omitempty"`
Data []byte `json:",omitempty"`
Duration time.Duration `json:",omitempty"`
Err error `json:",omitempty"`
HTTPHeaders http.Header `json:",omitempty"`
HTTPMethod string `json:",omitempty"`
HTTPStatusCode int `json:",omitempty"`
HTTPURL string `json:",omitempty"`
Hostname string `json:",omitempty"`
Name string `json:",omitempty"`
NoTLSVerify bool `json:",omitempty"`
NumBytes int `json:",omitempty"`
Proto string `json:",omitempty"`
TLSServerName string `json:",omitempty"`
TLSCipherSuite string `json:",omitempty"`
TLSNegotiatedProto string `json:",omitempty"`
TLSNextProtos []string `json:",omitempty"`
TLSPeerCerts []*x509.Certificate `json:",omitempty"`
TLSVersion string `json:",omitempty"`
Time time.Time `json:",omitempty"`
Transport string `json:",omitempty"`
}
// PeerCerts returns the certificates presented by the peer regardless
// of whether the TLS handshake was successful
func PeerCerts(state tls.ConnectionState, err error) []*x509.Certificate {
var x509HostnameError x509.HostnameError
if errors.As(err, &x509HostnameError) {
// Test case: https://wrong.host.badssl.com/
return []*x509.Certificate{x509HostnameError.Certificate}
}
var x509UnknownAuthorityError x509.UnknownAuthorityError
if errors.As(err, &x509UnknownAuthorityError) {
// Test case: https://self-signed.badssl.com/. This error has
// never been among the ones returned by MK.
return []*x509.Certificate{x509UnknownAuthorityError.Cert}
}
var x509CertificateInvalidError x509.CertificateInvalidError
if errors.As(err, &x509CertificateInvalidError) {
// Test case: https://expired.badssl.com/
return []*x509.Certificate{x509CertificateInvalidError.Cert}
}
return state.PeerCertificates
}

141
internal/engine/netx/tracex/http.go Normal file
View file

@ -0,0 +1,141 @@
package tracex
import (
"bytes"
"context"
"io"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// SaverMetadataHTTPTransport is a RoundTripper that saves
// events related to HTTP request and response metadata
type SaverMetadataHTTPTransport struct {
model.HTTPTransport
Saver *Saver
}
// RoundTrip implements RoundTripper.RoundTrip
func (txp SaverMetadataHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
txp.Saver.Write(Event{
HTTPHeaders: txp.CloneHeaders(req),
HTTPMethod: req.Method,
HTTPURL: req.URL.String(),
Transport: txp.HTTPTransport.Network(),
Name: "http_request_metadata",
Time: time.Now(),
})
resp, err := txp.HTTPTransport.RoundTrip(req)
if err != nil {
return nil, err
}
txp.Saver.Write(Event{
HTTPHeaders: resp.Header,
HTTPStatusCode: resp.StatusCode,
Name: "http_response_metadata",
Time: time.Now(),
})
return resp, err
}
// CloneHeaders returns a clone of the headers where we have
// also set the host header, which normally is not set by
// golang until it serializes the request itself.
func (txp SaverMetadataHTTPTransport) CloneHeaders(req *http.Request) http.Header {
header := req.Header.Clone()
if req.Host != "" {
header.Set("Host", req.Host)
} else {
header.Set("Host", req.URL.Host)
}
return header
}
// SaverTransactionHTTPTransport is a RoundTripper that saves
// events related to the HTTP transaction
type SaverTransactionHTTPTransport struct {
model.HTTPTransport
Saver *Saver
}
// RoundTrip implements RoundTripper.RoundTrip
func (txp SaverTransactionHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
txp.Saver.Write(Event{
Name: "http_transaction_start",
Time: time.Now(),
})
resp, err := txp.HTTPTransport.RoundTrip(req)
txp.Saver.Write(Event{
Err: err,
Name: "http_transaction_done",
Time: time.Now(),
})
return resp, err
}
// SaverBodyHTTPTransport is a RoundTripper that saves
// body events occurring during the round trip
type SaverBodyHTTPTransport struct {
model.HTTPTransport
Saver *Saver
SnapshotSize int
}
// RoundTrip implements RoundTripper.RoundTrip
func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
const defaultSnapSize = 1 << 17
snapsize := defaultSnapSize
if txp.SnapshotSize != 0 {
snapsize = txp.SnapshotSize
}
if req.Body != nil {
data, err := saverSnapRead(req.Context(), req.Body, snapsize)
if err != nil {
return nil, err
}
req.Body = saverCompose(data, req.Body)
txp.Saver.Write(Event{
DataIsTruncated: len(data) >= snapsize,
Data: data,
Name: "http_request_body_snapshot",
Time: time.Now(),
})
}
resp, err := txp.HTTPTransport.RoundTrip(req)
if err != nil {
return nil, err
}
data, err := saverSnapRead(req.Context(), resp.Body, snapsize)
if err != nil {
resp.Body.Close()
return nil, err
}
resp.Body = saverCompose(data, resp.Body)
txp.Saver.Write(Event{
DataIsTruncated: len(data) >= snapsize,
Data: data,
Name: "http_response_body_snapshot",
Time: time.Now(),
})
return resp, nil
}
func saverSnapRead(ctx context.Context, r io.ReadCloser, snapsize int) ([]byte, error) {
return netxlite.ReadAllContext(ctx, io.LimitReader(r, int64(snapsize)))
}
func saverCompose(data []byte, r io.ReadCloser) io.ReadCloser {
return saverReadCloser{Closer: r, Reader: io.MultiReader(bytes.NewReader(data), r)}
}
type saverReadCloser struct {
io.Closer
io.Reader
}
var _ model.HTTPTransport = SaverMetadataHTTPTransport{}
var _ model.HTTPTransport = SaverBodyHTTPTransport{}
var _ model.HTTPTransport = SaverTransactionHTTPTransport{}

470
internal/engine/netx/tracex/http_test.go Normal file
View file

@ -0,0 +1,470 @@
package tracex
import (
"context"
"errors"
"io"
"net"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestSaverMetadataSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
saver := &Saver{}
txp := SaverMetadataHTTPTransport{
HTTPTransport: netxlite.NewHTTPTransportStdlib(model.DiscardLogger),
Saver: saver,
}
req, err := http.NewRequest("GET", "https://www.google.com", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Add("User-Agent", "miniooni/0.1.0-dev")
resp, err := txp.RoundTrip(req)
if err != nil {
t.Fatal("not the error we expected")
}
if resp == nil {
t.Fatal("expected non nil response here")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("expected two events")
}
//
if ev[0].HTTPMethod != "GET" {
t.Fatal("unexpected Method")
}
if len(ev[0].HTTPHeaders) <= 0 {
t.Fatal("unexpected Headers")
}
if ev[0].HTTPURL != "https://www.google.com" {
t.Fatal("unexpected URL")
}
if ev[0].Name != "http_request_metadata" {
t.Fatal("unexpected Name")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("unexpected Time")
}
//
if ev[1].HTTPStatusCode != 200 {
t.Fatal("unexpected StatusCode")
}
if len(ev[1].HTTPHeaders) <= 0 {
t.Fatal("unexpected Headers")
}
if ev[1].Name != "http_response_metadata" {
t.Fatal("unexpected Name")
}
if !ev[1].Time.After(ev[0].Time) {
t.Fatal("unexpected Time")
}
}
func TestSaverMetadataFailure(t *testing.T) {
expected := errors.New("mocked error")
saver := &Saver{}
txp := SaverMetadataHTTPTransport{
HTTPTransport: FakeTransport{
Err: expected,
},
Saver: saver,
}
req, err := http.NewRequest("GET", "http://www.google.com", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Add("User-Agent", "miniooni/0.1.0-dev")
resp, err := txp.RoundTrip(req)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if resp != nil {
t.Fatal("expected nil response here")
}
ev := saver.Read()
if len(ev) != 1 {
t.Fatal("expected one event")
}
if ev[0].HTTPMethod != "GET" {
t.Fatal("unexpected Method")
}
if len(ev[0].HTTPHeaders) <= 0 {
t.Fatal("unexpected Headers")
}
if ev[0].HTTPURL != "http://www.google.com" {
t.Fatal("unexpected URL")
}
if ev[0].Name != "http_request_metadata" {
t.Fatal("unexpected Name")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("unexpected Time")
}
}
func TestSaverTransactionSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
saver := &Saver{}
txp := SaverTransactionHTTPTransport{
HTTPTransport: netxlite.NewHTTPTransportStdlib(model.DiscardLogger),
Saver: saver,
}
req, err := http.NewRequest("GET", "https://www.google.com", nil)
if err != nil {
t.Fatal(err)
}
resp, err := txp.RoundTrip(req)
if err != nil {
t.Fatal("not the error we expected")
}
if resp == nil {
t.Fatal("expected non nil response here")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("expected two events")
}
//
if ev[0].Name != "http_transaction_start" {
t.Fatal("unexpected Name")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("unexpected Time")
}
//
if ev[1].Err != nil {
t.Fatal("unexpected Err")
}
if ev[1].Name != "http_transaction_done" {
t.Fatal("unexpected Name")
}
if !ev[1].Time.After(ev[0].Time) {
t.Fatal("unexpected Time")
}
}
func TestSaverTransactionFailure(t *testing.T) {
expected := errors.New("mocked error")
saver := &Saver{}
txp := SaverTransactionHTTPTransport{
HTTPTransport: FakeTransport{
Err: expected,
},
Saver: saver,
}
req, err := http.NewRequest("GET", "http://www.google.com", nil)
if err != nil {
t.Fatal(err)
}
resp, err := txp.RoundTrip(req)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if resp != nil {
t.Fatal("expected nil response here")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("expected two events")
}
if ev[0].Name != "http_transaction_start" {
t.Fatal("unexpected Name")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("unexpected Time")
}
if ev[1].Name != "http_transaction_done" {
t.Fatal("unexpected Name")
}
if !errors.Is(ev[1].Err, expected) {
t.Fatal("unexpected Err")
}
if !ev[1].Time.After(ev[0].Time) {
t.Fatal("unexpected Time")
}
}
func TestSaverBodySuccess(t *testing.T) {
saver := new(Saver)
txp := SaverBodyHTTPTransport{
HTTPTransport: FakeTransport{
Func: func(req *http.Request) (*http.Response, error) {
data, err := netxlite.ReadAllContext(context.Background(), req.Body)
if err != nil {
t.Fatal(err)
}
if string(data) != "deadbeef" {
t.Fatal("invalid data")
}
return &http.Response{
StatusCode: 501,
Body: io.NopCloser(strings.NewReader("abad1dea")),
}, nil
},
},
SnapshotSize: 4,
Saver: saver,
}
body := strings.NewReader("deadbeef")
req, err := http.NewRequest("POST", "http://x.org/y", body)
if err != nil {
t.Fatal(err)
}
resp, err := txp.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != 501 {
t.Fatal("unexpected status code")
}
defer resp.Body.Close()
data, err := netxlite.ReadAllContext(context.Background(), resp.Body)
if err != nil {
t.Fatal(err)
}
if string(data) != "abad1dea" {
t.Fatal("unexpected body")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("unexpected number of events")
}
if string(ev[0].Data) != "dead" {
t.Fatal("invalid Data")
}
if ev[0].DataIsTruncated != true {
t.Fatal("invalid DataIsTruncated")
}
if ev[0].Name != "http_request_body_snapshot" {
t.Fatal("invalid Name")
}
if ev[0].Time.After(time.Now()) {
t.Fatal("invalid Time")
}
if string(ev[1].Data) != "abad" {
t.Fatal("invalid Data")
}
if ev[1].DataIsTruncated != true {
t.Fatal("invalid DataIsTruncated")
}
if ev[1].Name != "http_response_body_snapshot" {
t.Fatal("invalid Name")
}
if ev[1].Time.Before(ev[0].Time) {
t.Fatal("invalid Time")
}
}
func TestSaverBodyRequestReadError(t *testing.T) {
saver := new(Saver)
txp := SaverBodyHTTPTransport{
HTTPTransport: FakeTransport{
Func: func(req *http.Request) (*http.Response, error) {
panic("should not be called")
},
},
SnapshotSize: 4,
Saver: saver,
}
expected := errors.New("mocked error")
body := FakeBody{Err: expected}
req, err := http.NewRequest("POST", "http://x.org/y", body)
if err != nil {
t.Fatal(err)
}
resp, err := txp.RoundTrip(req)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if resp != nil {
t.Fatal("expected nil response")
}
ev := saver.Read()
if len(ev) != 0 {
t.Fatal("unexpected number of events")
}
}
func TestSaverBodyRoundTripError(t *testing.T) {
saver := new(Saver)
expected := errors.New("mocked error")
txp := SaverBodyHTTPTransport{
HTTPTransport: FakeTransport{
Err: expected,
},
SnapshotSize: 4,
Saver: saver,
}
body := strings.NewReader("deadbeef")
req, err := http.NewRequest("POST", "http://x.org/y", body)
if err != nil {
t.Fatal(err)
}
resp, err := txp.RoundTrip(req)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if resp != nil {
t.Fatal("expected nil response")
}
ev := saver.Read()
if len(ev) != 1 {
t.Fatal("unexpected number of events")
}
if string(ev[0].Data) != "dead" {
t.Fatal("invalid Data")
}
if ev[0].DataIsTruncated != true {
t.Fatal("invalid DataIsTruncated")
}
if ev[0].Name != "http_request_body_snapshot" {
t.Fatal("invalid Name")
}
if ev[0].Time.After(time.Now()) {
t.Fatal("invalid Time")
}
}
func TestSaverBodyResponseReadError(t *testing.T) {
saver := new(Saver)
expected := errors.New("mocked error")
txp := SaverBodyHTTPTransport{
HTTPTransport: FakeTransport{
Func: func(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: 200,
Body: FakeBody{
Err: expected,
},
}, nil
},
},
SnapshotSize: 4,
Saver: saver,
}
body := strings.NewReader("deadbeef")
req, err := http.NewRequest("POST", "http://x.org/y", body)
if err != nil {
t.Fatal(err)
}
resp, err := txp.RoundTrip(req)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if resp != nil {
t.Fatal("expected nil response")
}
ev := saver.Read()
if len(ev) != 1 {
t.Fatal("unexpected number of events")
}
if string(ev[0].Data) != "dead" {
t.Fatal("invalid Data")
}
if ev[0].DataIsTruncated != true {
t.Fatal("invalid DataIsTruncated")
}
if ev[0].Name != "http_request_body_snapshot" {
t.Fatal("invalid Name")
}
if ev[0].Time.After(time.Now()) {
t.Fatal("invalid Time")
}
}
func TestCloneHeaders(t *testing.T) {
t.Run("with req.Host set", func(t *testing.T) {
req := &http.Request{
Host: "www.example.com",
URL: &url.URL{
Host: "www.kernel.org",
},
Header: http.Header{},
}
txp := SaverMetadataHTTPTransport{}
header := txp.CloneHeaders(req)
if header.Get("Host") != "www.example.com" {
t.Fatal("did not set Host header correctly")
}
})
t.Run("with only req.URL.Host set", func(t *testing.T) {
req := &http.Request{
Host: "",
URL: &url.URL{
Host: "www.kernel.org",
},
Header: http.Header{},
}
txp := SaverMetadataHTTPTransport{}
header := txp.CloneHeaders(req)
if header.Get("Host") != "www.kernel.org" {
t.Fatal("did not set Host header correctly")
}
})
}
type FakeDialer struct {
Conn net.Conn
Err error
}
func (d FakeDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
time.Sleep(10 * time.Microsecond)
return d.Conn, d.Err
}
type FakeTransport struct {
Name string
Err error
Func func(*http.Request) (*http.Response, error)
Resp *http.Response
}
func (txp FakeTransport) Network() string {
return txp.Name
}
func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond)
if txp.Func != nil {
return txp.Func(req)
}
if req.Body != nil {
netxlite.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {
return nil, txp.Err
}
txp.Resp.Request = req // non thread safe but it doesn't matter
return txp.Resp, nil
}
func (txp FakeTransport) CloseIdleConnections() {}
type FakeBody struct {
Err error
}
func (fb FakeBody) Read(p []byte) (int, error) {
time.Sleep(10 * time.Microsecond)
return 0, fb.Err
}
func (fb FakeBody) Close() error {
return nil
}

139
internal/engine/netx/tracex/quic.go Normal file
View file

@ -0,0 +1,139 @@
package tracex
import (
"context"
"crypto/tls"
"net"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// QUICHandshakeSaver saves events occurring during the handshake
type QUICHandshakeSaver struct {
Saver *Saver
model.QUICDialer
}
// DialContext implements ContextDialer.DialContext
func (h QUICHandshakeSaver) DialContext(ctx context.Context, network string,
host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) {
start := time.Now()
// TODO(bassosimone): in the future we probably want to also save
// information about what versions we're willing to accept.
h.Saver.Write(Event{
Address: host,
Name: "quic_handshake_start",
NoTLSVerify: tlsCfg.InsecureSkipVerify,
Proto: network,
TLSNextProtos: tlsCfg.NextProtos,
TLSServerName: tlsCfg.ServerName,
Time: start,
})
sess, err := h.QUICDialer.DialContext(ctx, network, host, tlsCfg, cfg)
stop := time.Now()
if err != nil {
h.Saver.Write(Event{
Duration: stop.Sub(start),
Err: err,
Name: "quic_handshake_done",
NoTLSVerify: tlsCfg.InsecureSkipVerify,
TLSNextProtos: tlsCfg.NextProtos,
TLSServerName: tlsCfg.ServerName,
Time: stop,
})
return nil, err
}
state := quicConnectionState(sess)
h.Saver.Write(Event{
Duration: stop.Sub(start),
Name: "quic_handshake_done",
NoTLSVerify: tlsCfg.InsecureSkipVerify,
TLSCipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite),
TLSNegotiatedProto: state.NegotiatedProtocol,
TLSNextProtos: tlsCfg.NextProtos,
TLSPeerCerts: PeerCerts(state, err),
TLSServerName: tlsCfg.ServerName,
TLSVersion: netxlite.TLSVersionString(state.Version),
Time: stop,
})
return sess, nil
}
// quicConnectionState returns the ConnectionState of a QUIC Session.
func quicConnectionState(sess quic.EarlyConnection) tls.ConnectionState {
return sess.ConnectionState().TLS.ConnectionState
}
// QUICListenerSaver is a QUICListener that also implements saving events.
type QUICListenerSaver struct {
// QUICListener is the underlying QUICListener.
model.QUICListener
// Saver is the underlying Saver.
Saver *Saver
}
// Listen implements QUICListener.Listen.
func (qls *QUICListenerSaver) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) {
pconn, err := qls.QUICListener.Listen(addr)
if err != nil {
return nil, err
}
return &saverUDPConn{
UDPLikeConn: pconn,
saver: qls.Saver,
}, nil
}
type saverUDPConn struct {
model.UDPLikeConn
saver *Saver
}
var _ model.UDPLikeConn = &saverUDPConn{}
func (c *saverUDPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
start := time.Now()
count, err := c.UDPLikeConn.WriteTo(p, addr)
stop := time.Now()
c.saver.Write(Event{
Address: addr.String(),
Data: p[:count],
Duration: stop.Sub(start),
Err: err,
NumBytes: count,
Name: netxlite.WriteToOperation,
Time: stop,
})
return count, err
}
func (c *saverUDPConn) ReadFrom(b []byte) (int, net.Addr, error) {
start := time.Now()
n, addr, err := c.UDPLikeConn.ReadFrom(b)
stop := time.Now()
var data []byte
if n > 0 {
data = b[:n]
}
c.saver.Write(Event{
Address: c.safeAddrString(addr),
Data: data,
Duration: stop.Sub(start),
Err: err,
NumBytes: n,
Name: netxlite.ReadFromOperation,
Time: stop,
})
return n, addr, err
}
func (c *saverUDPConn) safeAddrString(addr net.Addr) (out string) {
if addr != nil {
out = addr.String()
}
return
}

195
internal/engine/netx/tracex/quic_test.go Normal file
View file

@ -0,0 +1,195 @@
package tracex
import (
"context"
"crypto/tls"
"errors"
"net"
"reflect"
"strings"
"testing"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/netxlite/quictesting"
)
type MockDialer struct {
Dialer model.QUICDialer
Sess quic.EarlyConnection
Err error
}
func (d MockDialer) DialContext(ctx context.Context, network, host string,
tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) {
if d.Dialer != nil {
return d.Dialer.DialContext(ctx, network, host, tlsCfg, cfg)
}
return d.Sess, d.Err
}
func TestHandshakeSaverSuccess(t *testing.T) {
nextprotos := []string{"h3"}
servername := quictesting.Domain
tlsConf := &tls.Config{
NextProtos: nextprotos,
ServerName: servername,
}
saver := &Saver{}
dlr := QUICHandshakeSaver{
QUICDialer: &netxlite.QUICDialerQUICGo{
QUICListener: &netxlite.QUICListenerStdlib{},
},
Saver: saver,
}
sess, err := dlr.DialContext(context.Background(), "udp",
quictesting.Endpoint("443"), tlsConf, &quic.Config{})
if err != nil {
t.Fatal("unexpected error", err)
}
if sess == nil {
t.Fatal("unexpected nil sess")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("unexpected number of events")
}
if ev[0].Name != "quic_handshake_start" {
t.Fatal("unexpected Name")
}
if ev[0].TLSServerName != quictesting.Domain {
t.Fatal("unexpected TLSServerName")
}
if !reflect.DeepEqual(ev[0].TLSNextProtos, nextprotos) {
t.Fatal("unexpected TLSNextProtos")
}
if ev[0].Time.After(time.Now()) {
t.Fatal("unexpected Time")
}
if ev[1].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[1].Err != nil {
t.Fatal("unexpected Err", ev[1].Err)
}
if ev[1].Name != "quic_handshake_done" {
t.Fatal("unexpected Name")
}
if !reflect.DeepEqual(ev[1].TLSNextProtos, nextprotos) {
t.Fatal("unexpected TLSNextProtos")
}
if ev[1].TLSServerName != quictesting.Domain {
t.Fatal("unexpected TLSServerName")
}
if ev[1].Time.Before(ev[0].Time) {
t.Fatal("unexpected Time")
}
}
func TestHandshakeSaverHostNameError(t *testing.T) {
nextprotos := []string{"h3"}
servername := "example.com"
tlsConf := &tls.Config{
NextProtos: nextprotos,
ServerName: servername,
}
saver := &Saver{}
dlr := QUICHandshakeSaver{
QUICDialer: &netxlite.QUICDialerQUICGo{
QUICListener: &netxlite.QUICListenerStdlib{},
},
Saver: saver,
}
sess, err := dlr.DialContext(context.Background(), "udp",
quictesting.Endpoint("443"), tlsConf, &quic.Config{})
if err == nil {
t.Fatal("expected an error here")
}
if sess != nil {
t.Fatal("expected nil sess here")
}
for _, ev := range saver.Read() {
if ev.Name != "quic_handshake_done" {
continue
}
if ev.NoTLSVerify == true {
t.Fatal("expected NoTLSVerify to be false")
}
if !strings.HasSuffix(ev.Err.Error(), "tls: handshake failure") {
t.Fatal("unexpected error", ev.Err)
}
}
}
func TestQUICListenerSaverCannotListen(t *testing.T) {
expected := errors.New("mocked error")
qls := &QUICListenerSaver{
QUICListener: &mocks.QUICListener{
MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) {
return nil, expected
},
},
Saver: &Saver{},
}
pconn, err := qls.Listen(&net.UDPAddr{
IP: []byte{},
Port: 8080,
Zone: "",
})
if !errors.Is(err, expected) {
t.Fatal("unexpected error", err)
}
if pconn != nil {
t.Fatal("expected nil pconn here")
}
}
func TestSystemDialerSuccessWithReadWrite(t *testing.T) {
// This is the most common use case for collecting reads, writes
tlsConf := &tls.Config{
NextProtos: []string{"h3"},
ServerName: quictesting.Domain,
}
saver := &Saver{}
systemdialer := &netxlite.QUICDialerQUICGo{
QUICListener: &QUICListenerSaver{
QUICListener: &netxlite.QUICListenerStdlib{},
Saver: saver,
},
}
_, err := systemdialer.DialContext(context.Background(), "udp",
quictesting.Endpoint("443"), tlsConf, &quic.Config{})
if err != nil {
t.Fatal(err)
}
ev := saver.Read()
if len(ev) < 2 {
t.Fatal("unexpected number of events")
}
last := len(ev) - 1
for idx := 1; idx < last; idx++ {
if ev[idx].Data == nil {
t.Fatal("unexpected Data")
}
if ev[idx].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[idx].Err != nil {
t.Fatal("unexpected Err")
}
if ev[idx].NumBytes <= 0 {
t.Fatal("unexpected NumBytes")
}
switch ev[idx].Name {
case netxlite.ReadFromOperation, netxlite.WriteToOperation:
default:
t.Fatal("unexpected Name")
}
if ev[idx].Time.Before(ev[idx-1].Time) {
t.Fatal("unexpected Time", ev[idx].Time, ev[idx-1].Time)
}
}
}

86
internal/engine/netx/tracex/resolver.go Normal file
View file

@ -0,0 +1,86 @@
package tracex
import (
"context"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
)
// SaverResolver is a resolver that saves events
type SaverResolver struct {
model.Resolver
Saver *Saver
}
// LookupHost implements Resolver.LookupHost
func (r SaverResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
start := time.Now()
r.Saver.Write(Event{
Address: r.Resolver.Address(),
Hostname: hostname,
Name: "resolve_start",
Proto: r.Resolver.Network(),
Time: start,
})
addrs, err := r.Resolver.LookupHost(ctx, hostname)
stop := time.Now()
r.Saver.Write(Event{
Addresses: addrs,
Address: r.Resolver.Address(),
Duration: stop.Sub(start),
Err: err,
Hostname: hostname,
Name: "resolve_done",
Proto: r.Resolver.Network(),
Time: stop,
})
return addrs, err
}
// SaverDNSTransport is a DNS transport that saves events
type SaverDNSTransport struct {
model.DNSTransport
Saver *Saver
}
// RoundTrip implements RoundTripper.RoundTrip
func (txp SaverDNSTransport) RoundTrip(
ctx context.Context, query model.DNSQuery) (model.DNSResponse, error) {
start := time.Now()
txp.Saver.Write(Event{
Address: txp.Address(),
DNSQuery: txp.maybeQueryBytes(query),
Name: "dns_round_trip_start",
Proto: txp.Network(),
Time: start,
})
response, err := txp.DNSTransport.RoundTrip(ctx, query)
stop := time.Now()
txp.Saver.Write(Event{
Address: txp.Address(),
DNSQuery: txp.maybeQueryBytes(query),
DNSReply: txp.maybeResponseBytes(response),
Duration: stop.Sub(start),
Err: err,
Name: "dns_round_trip_done",
Proto: txp.Network(),
Time: stop,
})
return response, err
}
func (txp SaverDNSTransport) maybeQueryBytes(query model.DNSQuery) []byte {
data, _ := query.Bytes()
return data
}
func (txp SaverDNSTransport) maybeResponseBytes(response model.DNSResponse) []byte {
if response == nil {
return nil
}
return response.Bytes()
}
var _ model.Resolver = SaverResolver{}
var _ model.DNSTransport = SaverDNSTransport{}

287
internal/engine/netx/tracex/resolver_test.go Normal file
View file

@ -0,0 +1,287 @@
package tracex
import (
"bytes"
"context"
"errors"
"net"
"reflect"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func TestSaverResolverFailure(t *testing.T) {
expected := errors.New("no such host")
saver := &Saver{}
reso := SaverResolver{
Resolver: NewFakeResolverWithExplicitError(expected),
Saver: saver,
}
addrs, err := reso.LookupHost(context.Background(), "www.google.com")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if addrs != nil {
t.Fatal("expected nil address here")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("expected number of events")
}
if ev[0].Hostname != "www.google.com" {
t.Fatal("unexpected Hostname")
}
if ev[0].Name != "resolve_start" {
t.Fatal("unexpected name")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("the saved time is wrong")
}
if ev[1].Addresses != nil {
t.Fatal("unexpected Addresses")
}
if ev[1].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if !errors.Is(ev[1].Err, expected) {
t.Fatal("unexpected Err")
}
if ev[1].Hostname != "www.google.com" {
t.Fatal("unexpected Hostname")
}
if ev[1].Name != "resolve_done" {
t.Fatal("unexpected name")
}
if !ev[1].Time.After(ev[0].Time) {
t.Fatal("the saved time is wrong")
}
}
func TestSaverResolverSuccess(t *testing.T) {
expected := []string{"8.8.8.8", "8.8.4.4"}
saver := &Saver{}
reso := SaverResolver{
Resolver: NewFakeResolverWithResult(expected),
Saver: saver,
}
addrs, err := reso.LookupHost(context.Background(), "www.google.com")
if err != nil {
t.Fatal("expected nil error here")
}
if !reflect.DeepEqual(addrs, expected) {
t.Fatal("not the result we expected")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("expected number of events")
}
if ev[0].Hostname != "www.google.com" {
t.Fatal("unexpected Hostname")
}
if ev[0].Name != "resolve_start" {
t.Fatal("unexpected name")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("the saved time is wrong")
}
if !reflect.DeepEqual(ev[1].Addresses, expected) {
t.Fatal("unexpected Addresses")
}
if ev[1].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[1].Err != nil {
t.Fatal("unexpected Err")
}
if ev[1].Hostname != "www.google.com" {
t.Fatal("unexpected Hostname")
}
if ev[1].Name != "resolve_done" {
t.Fatal("unexpected name")
}
if !ev[1].Time.After(ev[0].Time) {
t.Fatal("the saved time is wrong")
}
}
func TestSaverDNSTransportFailure(t *testing.T) {
expected := errors.New("no such host")
saver := &Saver{}
txp := SaverDNSTransport{
DNSTransport: &mocks.DNSTransport{
MockRoundTrip: func(ctx context.Context, query model.DNSQuery) (model.DNSResponse, error) {
return nil, expected
},
MockNetwork: func() string {
return "fake"
},
MockAddress: func() string {
return ""
},
},
Saver: saver,
}
rawQuery := []byte{0xde, 0xad, 0xbe, 0xef}
query := &mocks.DNSQuery{
MockBytes: func() ([]byte, error) {
return rawQuery, nil
},
}
reply, err := txp.RoundTrip(context.Background(), query)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if reply != nil {
t.Fatal("expected nil reply here")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("expected number of events")
}
if !bytes.Equal(ev[0].DNSQuery, rawQuery) {
t.Fatal("unexpected DNSQuery")
}
if ev[0].Name != "dns_round_trip_start" {
t.Fatal("unexpected name")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("the saved time is wrong")
}
if !bytes.Equal(ev[1].DNSQuery, rawQuery) {
t.Fatal("unexpected DNSQuery")
}
if ev[1].DNSReply != nil {
t.Fatal("unexpected DNSReply")
}
if ev[1].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if !errors.Is(ev[1].Err, expected) {
t.Fatal("unexpected Err")
}
if ev[1].Name != "dns_round_trip_done" {
t.Fatal("unexpected name")
}
if !ev[1].Time.After(ev[0].Time) {
t.Fatal("the saved time is wrong")
}
}
func TestSaverDNSTransportSuccess(t *testing.T) {
expected := []byte{0xef, 0xbe, 0xad, 0xde}
saver := &Saver{}
response := &mocks.DNSResponse{
MockBytes: func() []byte {
return expected
},
}
txp := SaverDNSTransport{
DNSTransport: &mocks.DNSTransport{
MockRoundTrip: func(ctx context.Context, query model.DNSQuery) (model.DNSResponse, error) {
return response, nil
},
MockNetwork: func() string {
return "fake"
},
MockAddress: func() string {
return ""
},
},
Saver: saver,
}
rawQuery := []byte{0xde, 0xad, 0xbe, 0xef}
query := &mocks.DNSQuery{
MockBytes: func() ([]byte, error) {
return rawQuery, nil
},
}
reply, err := txp.RoundTrip(context.Background(), query)
if err != nil {
t.Fatal("we expected nil error here")
}
if !bytes.Equal(reply.Bytes(), expected) {
t.Fatal("expected another reply here")
}
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("expected number of events")
}
if !bytes.Equal(ev[0].DNSQuery, rawQuery) {
t.Fatal("unexpected DNSQuery")
}
if ev[0].Name != "dns_round_trip_start" {
t.Fatal("unexpected name")
}
if !ev[0].Time.Before(time.Now()) {
t.Fatal("the saved time is wrong")
}
if !bytes.Equal(ev[1].DNSQuery, rawQuery) {
t.Fatal("unexpected DNSQuery")
}
if !bytes.Equal(ev[1].DNSReply, expected) {
t.Fatal("unexpected DNSReply")
}
if ev[1].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[1].Err != nil {
t.Fatal("unexpected Err")
}
if ev[1].Name != "dns_round_trip_done" {
t.Fatal("unexpected name")
}
if !ev[1].Time.After(ev[0].Time) {
t.Fatal("the saved time is wrong")
}
}
func NewFakeResolverWithExplicitError(err error) model.Resolver {
runtimex.PanicIfNil(err, "passed nil error")
return &mocks.Resolver{
MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
return nil, err
},
MockNetwork: func() string {
return "fake"
},
MockAddress: func() string {
return ""
},
MockCloseIdleConnections: func() {
// nothing
},
MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) {
return nil, errors.New("not implemented")
},
MockLookupNS: func(ctx context.Context, domain string) ([]*net.NS, error) {
return nil, errors.New("not implemented")
},
}
}
func NewFakeResolverWithResult(r []string) model.Resolver {
return &mocks.Resolver{
MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
return r, nil
},
MockNetwork: func() string {
return "fake"
},
MockAddress: func() string {
return ""
},
MockCloseIdleConnections: func() {
// nothing
},
MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) {
return nil, errors.New("not implemented")
},
MockLookupNS: func(ctx context.Context, domain string) ([]*net.NS, error) {
return nil, errors.New("not implemented")
},
}
}

27
internal/engine/netx/tracex/saver.go Normal file
View file

@ -0,0 +1,27 @@
package tracex
import "sync"
// The Saver saves a trace
type Saver struct {
ops []Event
mu sync.Mutex
}
// Read reads and returns events inside the trace. It advances
// the read pointer so you won't see such events again.
func (s *Saver) Read() []Event {
s.mu.Lock()
defer s.mu.Unlock()
v := s.ops
s.ops = nil
return v
}
// Write adds the given event to the trace. A subsequent call
// to Read will read this event.
func (s *Saver) Write(ev Event) {
s.mu.Lock()
defer s.mu.Unlock()
s.ops = append(s.ops, ev)
}

24
internal/engine/netx/tracex/saver_test.go Normal file
View file

@ -0,0 +1,24 @@
package tracex
import (
"sync"
"testing"
)
func TestGood(t *testing.T) {
saver := Saver{}
var wg sync.WaitGroup
const parallel = 10
wg.Add(parallel)
for idx := 0; idx < parallel; idx++ {
go func() {
saver.Write(Event{})
wg.Done()
}()
}
wg.Wait()
ev := saver.Read()
if len(ev) != parallel {
t.Fatal("unexpected number of events read")
}
}

51
internal/engine/netx/tracex/tls.go Normal file
View file

@ -0,0 +1,51 @@
package tracex
import (
"context"
"crypto/tls"
"net"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// SaverTLSHandshaker saves events occurring during the handshake
type SaverTLSHandshaker struct {
model.TLSHandshaker
Saver *Saver
}
// Handshake implements TLSHandshaker.Handshake
func (h SaverTLSHandshaker) Handshake(
ctx context.Context, conn net.Conn, config *tls.Config,
) (net.Conn, tls.ConnectionState, error) {
start := time.Now()
h.Saver.Write(Event{
Name: "tls_handshake_start",
NoTLSVerify: config.InsecureSkipVerify,
TLSNextProtos: config.NextProtos,
TLSServerName: config.ServerName,
Time: start,
})
remoteAddr := conn.RemoteAddr().String()
tlsconn, state, err := h.TLSHandshaker.Handshake(ctx, conn, config)
stop := time.Now()
h.Saver.Write(Event{
Address: remoteAddr,
Duration: stop.Sub(start),
Err: err,
Name: "tls_handshake_done",
NoTLSVerify: config.InsecureSkipVerify,
TLSCipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite),
TLSNegotiatedProto: state.NegotiatedProtocol,
TLSNextProtos: config.NextProtos,
TLSPeerCerts: PeerCerts(state, err),
TLSServerName: config.ServerName,
TLSVersion: netxlite.TLSVersionString(state.Version),
Time: stop,
})
return tlsconn, state, err
}
var _ model.TLSHandshaker = SaverTLSHandshaker{}

316
internal/engine/netx/tracex/tls_test.go Normal file
View file

@ -0,0 +1,316 @@
package tracex
import (
"context"
"crypto/tls"
"reflect"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestSaverTLSHandshakerSuccessWithReadWrite(t *testing.T) {
// This is the most common use case for collecting reads, writes
if testing.Short() {
t.Skip("skip test in short mode")
}
nextprotos := []string{"h2"}
saver := &Saver{}
tlsdlr := &netxlite.TLSDialerLegacy{
Config: &tls.Config{NextProtos: nextprotos},
Dialer: netxlite.NewDialerWithResolver(
model.DiscardLogger,
netxlite.NewResolverStdlib(model.DiscardLogger),
func(dialer model.Dialer) model.Dialer {
return &SaverConnDialer{
Dialer: dialer,
Saver: saver,
}
},
),
TLSHandshaker: SaverTLSHandshaker{
TLSHandshaker: &netxlite.TLSHandshakerConfigurable{},
Saver: saver,
},
}
// Implementation note: we don't close the connection here because it is
// very handy to have the last event being the end of the handshake
_, err := tlsdlr.DialTLSContext(context.Background(), "tcp", "www.google.com:443")
if err != nil {
t.Fatal(err)
}
ev := saver.Read()
if len(ev) < 4 {
// it's a bit tricky to be sure about the right number of
// events because network conditions may influence that
t.Fatal("unexpected number of events")
}
if ev[0].Name != "tls_handshake_start" {
t.Fatal("unexpected Name")
}
if ev[0].TLSServerName != "www.google.com" {
t.Fatal("unexpected TLSServerName")
}
if !reflect.DeepEqual(ev[0].TLSNextProtos, nextprotos) {
t.Fatal("unexpected TLSNextProtos")
}
if ev[0].Time.After(time.Now()) {
t.Fatal("unexpected Time")
}
last := len(ev) - 1
for idx := 1; idx < last; idx++ {
if ev[idx].Data == nil {
t.Fatal("unexpected Data")
}
if ev[idx].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[idx].Err != nil {
t.Fatal("unexpected Err")
}
if ev[idx].NumBytes <= 0 {
t.Fatal("unexpected NumBytes")
}
switch ev[idx].Name {
case netxlite.ReadOperation, netxlite.WriteOperation:
default:
t.Fatal("unexpected Name")
}
if ev[idx].Time.Before(ev[idx-1].Time) {
t.Fatal("unexpected Time")
}
}
if ev[last].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[last].Err != nil {
t.Fatal("unexpected Err")
}
if ev[last].Name != "tls_handshake_done" {
t.Fatal("unexpected Name")
}
if ev[last].TLSCipherSuite == "" {
t.Fatal("unexpected TLSCipherSuite")
}
if ev[last].TLSNegotiatedProto != "h2" {
t.Fatal("unexpected TLSNegotiatedProto")
}
if !reflect.DeepEqual(ev[last].TLSNextProtos, nextprotos) {
t.Fatal("unexpected TLSNextProtos")
}
if ev[last].TLSPeerCerts == nil {
t.Fatal("unexpected TLSPeerCerts")
}
if ev[last].TLSServerName != "www.google.com" {
t.Fatal("unexpected TLSServerName")
}
if ev[last].TLSVersion == "" {
t.Fatal("unexpected TLSVersion")
}
if ev[last].Time.Before(ev[last-1].Time) {
t.Fatal("unexpected Time")
}
}
func TestSaverTLSHandshakerSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
nextprotos := []string{"h2"}
saver := &Saver{}
tlsdlr := &netxlite.TLSDialerLegacy{
Config: &tls.Config{NextProtos: nextprotos},
Dialer: netxlite.DefaultDialer,
TLSHandshaker: SaverTLSHandshaker{
TLSHandshaker: &netxlite.TLSHandshakerConfigurable{},
Saver: saver,
},
}
conn, err := tlsdlr.DialTLSContext(context.Background(), "tcp", "www.google.com:443")
if err != nil {
t.Fatal(err)
}
conn.Close()
ev := saver.Read()
if len(ev) != 2 {
t.Fatal("unexpected number of events")
}
if ev[0].Name != "tls_handshake_start" {
t.Fatal("unexpected Name")
}
if ev[0].TLSServerName != "www.google.com" {
t.Fatal("unexpected TLSServerName")
}
if !reflect.DeepEqual(ev[0].TLSNextProtos, nextprotos) {
t.Fatal("unexpected TLSNextProtos")
}
if ev[0].Time.After(time.Now()) {
t.Fatal("unexpected Time")
}
if ev[1].Duration <= 0 {
t.Fatal("unexpected Duration")
}
if ev[1].Err != nil {
t.Fatal("unexpected Err")
}
if ev[1].Name != "tls_handshake_done" {
t.Fatal("unexpected Name")
}
if ev[1].TLSCipherSuite == "" {
t.Fatal("unexpected TLSCipherSuite")
}
if ev[1].TLSNegotiatedProto != "h2" {
t.Fatal("unexpected TLSNegotiatedProto")
}
if !reflect.DeepEqual(ev[1].TLSNextProtos, nextprotos) {
t.Fatal("unexpected TLSNextProtos")
}
if ev[1].TLSPeerCerts == nil {
t.Fatal("unexpected TLSPeerCerts")
}
if ev[1].TLSServerName != "www.google.com" {
t.Fatal("unexpected TLSServerName")
}
if ev[1].TLSVersion == "" {
t.Fatal("unexpected TLSVersion")
}
if ev[1].Time.Before(ev[0].Time) {
t.Fatal("unexpected Time")
}
}
func TestSaverTLSHandshakerHostnameError(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
saver := &Saver{}
tlsdlr := &netxlite.TLSDialerLegacy{
Dialer: netxlite.DefaultDialer,
TLSHandshaker: SaverTLSHandshaker{
TLSHandshaker: &netxlite.TLSHandshakerConfigurable{},
Saver: saver,
},
}
conn, err := tlsdlr.DialTLSContext(
context.Background(), "tcp", "wrong.host.badssl.com:443")
if err == nil {
t.Fatal("expected an error here")
}
if conn != nil {
t.Fatal("expected nil conn here")
}
for _, ev := range saver.Read() {
if ev.Name != "tls_handshake_done" {
continue
}
if ev.NoTLSVerify == true {
t.Fatal("expected NoTLSVerify to be false")
}
if len(ev.TLSPeerCerts) < 1 {
t.Fatal("expected at least a certificate here")
}
}
}
func TestSaverTLSHandshakerInvalidCertError(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
saver := &Saver{}
tlsdlr := &netxlite.TLSDialerLegacy{
Dialer: netxlite.DefaultDialer,
TLSHandshaker: SaverTLSHandshaker{
TLSHandshaker: &netxlite.TLSHandshakerConfigurable{},
Saver: saver,
},
}
conn, err := tlsdlr.DialTLSContext(
context.Background(), "tcp", "expired.badssl.com:443")
if err == nil {
t.Fatal("expected an error here")
}
if conn != nil {
t.Fatal("expected nil conn here")
}
for _, ev := range saver.Read() {
if ev.Name != "tls_handshake_done" {
continue
}
if ev.NoTLSVerify == true {
t.Fatal("expected NoTLSVerify to be false")
}
if len(ev.TLSPeerCerts) < 1 {
t.Fatal("expected at least a certificate here")
}
}
}
func TestSaverTLSHandshakerAuthorityError(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
saver := &Saver{}
tlsdlr := &netxlite.TLSDialerLegacy{
Dialer: netxlite.DefaultDialer,
TLSHandshaker: SaverTLSHandshaker{
TLSHandshaker: &netxlite.TLSHandshakerConfigurable{},
Saver: saver,
},
}
conn, err := tlsdlr.DialTLSContext(
context.Background(), "tcp", "self-signed.badssl.com:443")
if err == nil {
t.Fatal("expected an error here")
}
if conn != nil {
t.Fatal("expected nil conn here")
}
for _, ev := range saver.Read() {
if ev.Name != "tls_handshake_done" {
continue
}
if ev.NoTLSVerify == true {
t.Fatal("expected NoTLSVerify to be false")
}
if len(ev.TLSPeerCerts) < 1 {
t.Fatal("expected at least a certificate here")
}
}
}
func TestSaverTLSHandshakerNoTLSVerify(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
saver := &Saver{}
tlsdlr := &netxlite.TLSDialerLegacy{
Config: &tls.Config{InsecureSkipVerify: true},
Dialer: netxlite.DefaultDialer,
TLSHandshaker: SaverTLSHandshaker{
TLSHandshaker: &netxlite.TLSHandshakerConfigurable{},
Saver: saver,
},
}
conn, err := tlsdlr.DialTLSContext(
context.Background(), "tcp", "self-signed.badssl.com:443")
if err != nil {
t.Fatal(err)
}
if conn == nil {
t.Fatal("expected non-nil conn here")
}
conn.Close()
for _, ev := range saver.Read() {
if ev.Name != "tls_handshake_done" {
continue
}
if ev.NoTLSVerify != true {
t.Fatal("expected NoTLSVerify to be true")
}
if len(ev.TLSPeerCerts) < 1 {
t.Fatal("expected at least a certificate here")
}
}
}