ooni-probe-cli/internal/tracex/archival_test.go
Simone Basso b78b9aca51
refactor(datafmt): use "udp" instead of "quic" (#946)
This diff changes the data format to prefer "udp" to "quic" everywhere we were previously using "quic".

Previously, the code inconsistently used "quic" for operations where we knew we were using "quic" and "udp" otherwise (e.g., for generic operations like ReadFrom).

While it would be more correct to say that a specific HTTP request used "quic" rather than "udp", using "udp" consistently allows one to see how distinct events such as ReadFrom and an handshake all refer to the same address, port, and protocol triple. Therefore, this change makes it easier to programmatically unpack a single measurement and create endpoint stats.

Before implementing this change, I discussed the problem with @hellais who mentioned that ooni/data is not currently using the "quic" string anywhere. I know that ooni/pipeline also doesn't rely on this string. The only users of this feature have been research-oriented experiments such as urlgetter, for which such a change would actually be acceptable.

See https://github.com/ooni/probe/issues/2238 and https://github.com/ooni/spec/pull/262.
2022-09-08 17:19:59 +02:00

755 lines
18 KiB
Go

package tracex
import (
"context"
"errors"
"io"
"net/http"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/gorilla/websocket"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestDNSQueryType(t *testing.T) {
t.Run("ipOfType", func(t *testing.T) {
type expectation struct {
qtype dnsQueryType
ip string
output bool
}
var expectations = []expectation{{
qtype: "A",
ip: "8.8.8.8",
output: true,
}, {
qtype: "A",
ip: "2a00:1450:4002:801::2004",
output: false,
}, {
qtype: "AAAA",
ip: "8.8.8.8",
output: false,
}, {
qtype: "AAAA",
ip: "2a00:1450:4002:801::2004",
output: true,
}, {
qtype: "ANTANI",
ip: "2a00:1450:4002:801::2004",
output: false,
}, {
qtype: "ANTANI",
ip: "8.8.8.8",
output: false,
}}
for _, exp := range expectations {
if exp.qtype.ipOfType(exp.ip) != exp.output {
t.Fatalf("failure for %+v", exp)
}
}
})
}
func TestNewTCPConnectList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []TCPConnectEntry
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
events: []Event{&EventResolveDone{&EventValue{ // skipped because not relevant
Addresses: []string{"8.8.8.8", "8.8.4.4"},
Hostname: "dns.google.com",
Time: begin.Add(100 * time.Millisecond),
}}, &EventConnectOperation{&EventValue{
Address: "8.8.8.8:853",
Duration: 30 * time.Millisecond,
Proto: "tcp",
Time: begin.Add(130 * time.Millisecond),
}}, &EventConnectOperation{&EventValue{
Address: "8.8.8.8:853",
Duration: 55 * time.Millisecond,
Proto: "udp", // this one should be skipped because it's UDP
Time: begin.Add(130 * time.Millisecond),
}}, &EventConnectOperation{&EventValue{
Address: "8.8.4.4:53",
Duration: 50 * time.Millisecond,
Err: netxlite.FailureEOFError,
Proto: "tcp",
Time: begin.Add(180 * time.Millisecond),
}}},
},
want: []TCPConnectEntry{{
IP: "8.8.8.8",
Port: 853,
Status: TCPConnectStatus{
Success: true,
},
T: 0.13,
}, {
IP: "8.8.4.4",
Port: 53,
Status: TCPConnectStatus{
Failure: NewFailure(io.EOF),
Success: false,
},
T: 0.18,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewTCPConnectList(tt.args.begin, tt.args.events)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestNewRequestList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []RequestEntry
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
// Two round trips so we can test the sorting expected by OONI
events: []Event{&EventHTTPTransactionDone{&EventValue{
HTTPRequestHeaders: http.Header{
"User-Agent": []string{"miniooni/0.1.0-dev"},
},
HTTPMethod: "POST",
HTTPURL: "https://www.example.com/submit",
HTTPResponseHeaders: http.Header{
"Server": []string{"miniooni/0.1.0-dev"},
},
HTTPStatusCode: 200,
HTTPResponseBody: []byte("{}"),
HTTPResponseBodyIsTruncated: false,
Time: begin.Add(10 * time.Millisecond),
}}, &EventHTTPTransactionDone{&EventValue{
HTTPRequestHeaders: http.Header{
"User-Agent": []string{"miniooni/0.1.0-dev"},
},
HTTPMethod: "GET",
HTTPURL: "https://www.example.com/result",
Err: netxlite.FailureEOFError,
Time: begin.Add(20 * time.Millisecond),
}}},
},
want: []RequestEntry{{
Failure: NewFailure(io.EOF),
Request: HTTPRequest{
HeadersList: []HTTPHeader{{
Key: "User-Agent",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"User-Agent": {Value: "miniooni/0.1.0-dev"},
},
Method: "GET",
URL: "https://www.example.com/result",
},
Response: HTTPResponse{
HeadersList: []HTTPHeader{},
Headers: make(map[string]MaybeBinaryValue),
},
T: 0.02,
}, {
Request: HTTPRequest{
Body: MaybeBinaryValue{
Value: "",
},
HeadersList: []HTTPHeader{{
Key: "User-Agent",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"User-Agent": {Value: "miniooni/0.1.0-dev"},
},
Method: "POST",
URL: "https://www.example.com/submit",
},
Response: HTTPResponse{
Body: MaybeBinaryValue{
Value: "{}",
},
Code: 200,
HeadersList: []HTTPHeader{{
Key: "Server",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"Server": {Value: "miniooni/0.1.0-dev"},
},
Locations: nil,
},
T: 0.01,
}},
}, {
// for an example of why we need to sort headers, see
// https://github.com/ooni/probe-engine/pull/751/checks?check_run_id=853562310
name: "run with redirect and headers to sort",
args: args{
begin: begin,
events: []Event{&EventHTTPTransactionDone{&EventValue{
HTTPRequestHeaders: http.Header{
"User-Agent": []string{"miniooni/0.1.0-dev"},
},
HTTPMethod: "GET",
HTTPURL: "https://www.example.com/",
HTTPResponseHeaders: http.Header{
"Server": []string{"miniooni/0.1.0-dev"},
"Location": []string{"https://x.example.com", "https://y.example.com"},
},
HTTPStatusCode: 302,
Time: begin.Add(10 * time.Millisecond),
}}},
},
want: []RequestEntry{{
Request: HTTPRequest{
HeadersList: []HTTPHeader{{
Key: "User-Agent",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"User-Agent": {Value: "miniooni/0.1.0-dev"},
},
Method: "GET",
URL: "https://www.example.com/",
},
Response: HTTPResponse{
Code: 302,
HeadersList: []HTTPHeader{{
Key: "Location",
Value: MaybeBinaryValue{
Value: "https://x.example.com",
},
}, {
Key: "Location",
Value: MaybeBinaryValue{
Value: "https://y.example.com",
},
}, {
Key: "Server",
Value: MaybeBinaryValue{
Value: "miniooni/0.1.0-dev",
},
}},
Headers: map[string]MaybeBinaryValue{
"Server": {Value: "miniooni/0.1.0-dev"},
"Location": {Value: "https://x.example.com"},
},
Locations: []string{
"https://x.example.com", "https://y.example.com",
},
},
T: 0.01,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewRequestList(tt.args.begin, tt.args.events)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestNewDNSQueriesList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []DNSQueryEntry
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
events: []Event{&EventResolveDone{&EventValue{
Address: "1.1.1.1:853",
Addresses: []string{"8.8.8.8", "8.8.4.4"},
Hostname: "dns.google.com",
Proto: "dot",
Time: begin.Add(100 * time.Millisecond),
}}, &EventConnectOperation{&EventValue{ // skipped because not relevant
Address: "8.8.8.8:853",
Duration: 30 * time.Millisecond,
Proto: "tcp",
Time: begin.Add(130 * time.Millisecond),
}}, &EventConnectOperation{&EventValue{ // skipped because not relevant
Address: "8.8.4.4:53",
Duration: 50 * time.Millisecond,
Err: netxlite.FailureEOFError,
Proto: "tcp",
Time: begin.Add(180 * time.Millisecond),
}}},
},
want: []DNSQueryEntry{{
Answers: []DNSAnswerEntry{{
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
IPv4: "8.8.8.8",
}, {
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
IPv4: "8.8.4.4",
}},
Engine: "dot",
Hostname: "dns.google.com",
QueryType: "A",
ResolverAddress: "1.1.1.1:853",
T: 0.1,
}},
}, {
name: "run with IPv6 results",
args: args{
begin: begin,
events: []Event{&EventResolveDone{&EventValue{
Addresses: []string{"2001:4860:4860::8888"},
Hostname: "dns.google.com",
Time: begin.Add(200 * time.Millisecond),
}}},
},
want: []DNSQueryEntry{{
Answers: []DNSAnswerEntry{{
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "AAAA",
IPv6: "2001:4860:4860::8888",
}},
Hostname: "dns.google.com",
QueryType: "AAAA",
T: 0.2,
}},
}, {
name: "run with errors",
args: args{
begin: begin,
events: []Event{&EventResolveDone{&EventValue{
Err: netxlite.FailureDNSNXDOMAINError,
Hostname: "dns.google.com",
Time: begin.Add(200 * time.Millisecond),
}}},
},
want: []DNSQueryEntry{{
Answers: nil,
Failure: NewFailure(
&netxlite.ErrWrapper{Failure: netxlite.FailureDNSNXDOMAINError}),
Hostname: "dns.google.com",
QueryType: "A",
T: 0.2,
}, {
Answers: nil,
Failure: NewFailure(
&netxlite.ErrWrapper{Failure: netxlite.FailureDNSNXDOMAINError}),
Hostname: "dns.google.com",
QueryType: "AAAA",
T: 0.2,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewDNSQueriesList(tt.args.begin, tt.args.events)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestNewNetworkEventsList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []NetworkEvent
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run",
args: args{
begin: begin,
events: []Event{&EventConnectOperation{&EventValue{
Address: "8.8.8.8:853",
Err: netxlite.FailureEOFError,
Proto: "tcp",
Time: begin.Add(7 * time.Millisecond),
}}, &EventReadOperation{&EventValue{
Err: netxlite.FailureInterrupted,
NumBytes: 7117,
Time: begin.Add(11 * time.Millisecond),
}}, &EventReadFromOperation{&EventValue{
Address: "8.8.8.8:853",
Err: netxlite.FailureInterrupted,
NumBytes: 7117,
Time: begin.Add(11 * time.Millisecond),
}}, &EventWriteOperation{&EventValue{
Err: NewFailureStr(websocket.ErrBadHandshake),
NumBytes: 4114,
Time: begin.Add(14 * time.Millisecond),
}}, &EventWriteToOperation{&EventValue{
Address: "8.8.8.8:853",
Err: NewFailureStr(websocket.ErrBadHandshake),
NumBytes: 4114,
Time: begin.Add(14 * time.Millisecond),
}}, &EventResolveStart{&EventValue{
// We expect this event to be logged event though it's not a typical I/O
// event (it seems these extra events are useful for debugging)
Time: begin.Add(15 * time.Millisecond),
}}},
},
want: []NetworkEvent{{
Address: "8.8.8.8:853",
Failure: NewFailure(io.EOF),
Operation: netxlite.ConnectOperation,
Proto: "tcp",
T: 0.007,
}, {
Failure: NewFailure(context.Canceled),
NumBytes: 7117,
Operation: netxlite.ReadOperation,
T: 0.011,
}, {
Address: "8.8.8.8:853",
Failure: NewFailure(context.Canceled),
NumBytes: 7117,
Operation: netxlite.ReadFromOperation,
T: 0.011,
}, {
Failure: NewFailure(websocket.ErrBadHandshake),
NumBytes: 4114,
Operation: netxlite.WriteOperation,
T: 0.014,
}, {
Address: "8.8.8.8:853",
Failure: NewFailure(websocket.ErrBadHandshake),
NumBytes: 4114,
Operation: netxlite.WriteToOperation,
T: 0.014,
}, {
Operation: "resolve_start",
T: 0.015,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewNetworkEventsList(tt.args.begin, tt.args.events)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestNewTLSHandshakesList(t *testing.T) {
begin := time.Now()
type args struct {
begin time.Time
events []Event
}
tests := []struct {
name string
args args
want []TLSHandshake
}{{
name: "empty run",
args: args{
begin: begin,
events: nil,
},
want: nil,
}, {
name: "realistic run with TLS",
args: args{
begin: begin,
events: []Event{&EventTLSHandshakeDone{&EventValue{
Address: "131.252.210.176:443",
Err: netxlite.FailureEOFError,
NoTLSVerify: false,
Proto: "tcp",
TLSCipherSuite: "SUITE",
TLSNegotiatedProto: "h2",
TLSPeerCerts: [][]byte{
[]byte("deadbeef"),
[]byte("abad1dea"),
},
TLSServerName: "x.org",
TLSVersion: "TLSv1.3",
Time: begin.Add(55 * time.Millisecond),
}}},
},
want: []TLSHandshake{{
Address: "131.252.210.176:443",
CipherSuite: "SUITE",
Failure: NewFailure(io.EOF),
NegotiatedProtocol: "h2",
NoTLSVerify: false,
PeerCertificates: []MaybeBinaryValue{{
Value: "deadbeef",
}, {
Value: "abad1dea",
}},
ServerName: "x.org",
T: 0.055,
TLSVersion: "TLSv1.3",
}},
}, {
name: "realistic run with QUIC",
args: args{
begin: begin,
events: []Event{&EventQUICHandshakeDone{&EventValue{
Address: "131.252.210.176:443",
Err: netxlite.FailureEOFError,
NoTLSVerify: false,
Proto: "udp",
TLSCipherSuite: "SUITE",
TLSNegotiatedProto: "h3",
TLSPeerCerts: [][]byte{
[]byte("deadbeef"),
[]byte("abad1dea"),
},
TLSServerName: "x.org",
TLSVersion: "TLSv1.3",
Time: begin.Add(55 * time.Millisecond),
}}},
},
want: []TLSHandshake{{
Address: "131.252.210.176:443",
CipherSuite: "SUITE",
Failure: NewFailure(io.EOF),
NegotiatedProtocol: "h3",
NoTLSVerify: false,
PeerCertificates: []MaybeBinaryValue{{
Value: "deadbeef",
}, {
Value: "abad1dea",
}},
ServerName: "x.org",
T: 0.055,
TLSVersion: "TLSv1.3",
}},
}, {
name: "realistic run with no suitable events",
args: args{
begin: begin,
events: []Event{&EventResolveStart{&EventValue{
Time: begin.Add(55 * time.Millisecond),
}}},
},
want: nil,
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewTLSHandshakesList(tt.args.begin, tt.args.events)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestNewFailure(t *testing.T) {
type args struct {
err error
}
tests := []struct {
name string
args args
want *string
}{{
name: "when error is nil",
args: args{
err: nil,
},
want: nil,
}, {
name: "when error is wrapped and failure meaningful",
args: args{
err: &netxlite.ErrWrapper{
Failure: netxlite.FailureConnectionRefused,
},
},
want: func() *string {
s := netxlite.FailureConnectionRefused
return &s
}(),
}, {
name: "when error is wrapped and failure is not meaningful",
args: args{
err: &netxlite.ErrWrapper{},
},
want: func() *string {
s := "unknown_failure: errWrapper.Failure is empty"
return &s
}(),
}, {
name: "when error is not wrapped but wrappable",
args: args{err: io.EOF},
want: func() *string {
s := "eof_error"
return &s
}(),
}, {
name: "when the error is not wrapped and not wrappable",
args: args{
err: errors.New("use of closed socket 127.0.0.1:8080->10.0.0.1:22"),
},
want: func() *string {
s := "unknown_failure: use of closed socket [scrubbed]->[scrubbed]"
return &s
}(),
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewFailure(tt.args.err)
if tt.want == nil && got == nil {
return
}
if tt.want == nil && got != nil {
t.Errorf("NewFailure: want %+v, got %s", tt.want, *got)
return
}
if tt.want != nil && got == nil {
t.Errorf("NewFailure: want %s, got %+v", *tt.want, got)
return
}
if *tt.want != *got {
t.Errorf("NewFailure: want %s, got %s", *tt.want, *got)
return
}
})
}
}
func TestNewFailedOperation(t *testing.T) {
type args struct {
err error
}
tests := []struct {
name string
args args
want *string
}{{
name: "With no error",
args: args{
err: nil, // explicit
},
want: nil, // explicit
}, {
name: "With wrapped error and non-empty operation",
args: args{
err: &netxlite.ErrWrapper{
Failure: netxlite.FailureConnectionRefused,
Operation: netxlite.ConnectOperation,
},
},
want: (func() *string {
s := netxlite.ConnectOperation
return &s
})(),
}, {
name: "With wrapped error and empty operation",
args: args{
err: &netxlite.ErrWrapper{
Failure: netxlite.FailureConnectionRefused,
},
},
want: (func() *string {
s := netxlite.UnknownOperation
return &s
})(),
}, {
name: "With non wrapped error",
args: args{
err: io.EOF,
},
want: (func() *string {
s := netxlite.UnknownOperation
return &s
})(),
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewFailedOperation(tt.args.err)
if got == nil && tt.want == nil {
return
}
if got == nil && tt.want != nil {
t.Errorf("NewFailedOperation() = %v, want %v", got, tt.want)
return
}
if got != nil && tt.want == nil {
t.Errorf("NewFailedOperation() = %v, want %v", got, tt.want)
return
}
if got != nil && tt.want != nil && *got != *tt.want {
t.Errorf("NewFailedOperation() = %v, want %v", got, tt.want)
return
}
})
}
}