diff --git a/internal/cmd/oohelperd/internal/webconnectivity/dns.go b/internal/cmd/oohelperd/internal/webconnectivity/dns.go index ebe7864..0327cac 100644 --- a/internal/cmd/oohelperd/internal/webconnectivity/dns.go +++ b/internal/cmd/oohelperd/internal/webconnectivity/dns.go @@ -5,13 +5,13 @@ import ( "sync" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) // newfailure is a convenience shortcut to save typing -var newfailure = archival.NewFailure +var newfailure = tracex.NewFailure // CtrlDNSResult is the result of the DNS check performed by // the Web Connectivity test helper. diff --git a/internal/cmd/oohelperd/internal/webconnectivity/http.go b/internal/cmd/oohelperd/internal/webconnectivity/http.go index 6487f00..95149cb 100644 --- a/internal/cmd/oohelperd/internal/webconnectivity/http.go +++ b/internal/cmd/oohelperd/internal/webconnectivity/http.go @@ -8,7 +8,7 @@ import ( "sync" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -81,7 +81,7 @@ func HTTPDo(ctx context.Context, config *HTTPConfig) { // See https://github.com/ooni/backend/blob/6ec4fda5b18/oonib/testhelpers/http_helpers.py#L361 func httpMapFailure(err error) *string { failure := newfailure(err) - failedOperation := archival.NewFailedOperation(err) + failedOperation := tracex.NewFailedOperation(err) switch failure { case nil: return nil diff --git a/internal/engine/experiment/dash/dash.go b/internal/engine/experiment/dash/dash.go index 2b86e67..e0a09d8 100644 --- a/internal/engine/experiment/dash/dash.go +++ b/internal/engine/experiment/dash/dash.go @@ -15,7 +15,7 @@ import ( "github.com/montanaflynn/stats" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/humanize" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" @@ -64,7 +64,7 @@ type TestKeys struct { type runner struct { callbacks model.ExperimentCallbacks httpClient *http.Client - saver *trace.Saver + saver *tracex.Saver sess model.ExperimentSession tk *TestKeys } @@ -255,7 +255,7 @@ func (m Measurer) Run( ) error { tk := new(TestKeys) measurement.TestKeys = tk - saver := &trace.Saver{} + saver := &tracex.Saver{} httpClient := &http.Client{ Transport: netx.NewHTTPTransport(netx.Config{ ContextByteCounting: true, diff --git a/internal/engine/experiment/dash/dash_test.go b/internal/engine/experiment/dash/dash_test.go index 3961c62..050f281 100644 --- a/internal/engine/experiment/dash/dash_test.go +++ b/internal/engine/experiment/dash/dash_test.go @@ -12,7 +12,7 @@ import ( "github.com/apex/log" "github.com/montanaflynn/stats" "github.com/ooni/probe-cli/v3/internal/engine/mockable" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -26,7 +26,7 @@ func TestRunnerLoopLocateFailure(t *testing.T) { err: expected, }, }, - saver: new(trace.Saver), + saver: new(tracex.Saver), sess: &mockable.Session{ MockableLogger: log.Log, }, @@ -56,7 +56,7 @@ func TestRunnerLoopNegotiateFailure(t *testing.T) { }, }, }, - saver: new(trace.Saver), + saver: new(tracex.Saver), sess: &mockable.Session{ MockableLogger: log.Log, }, @@ -93,7 +93,7 @@ func TestRunnerLoopMeasureFailure(t *testing.T) { }, }, }, - saver: new(trace.Saver), + saver: new(tracex.Saver), sess: &mockable.Session{ MockableLogger: log.Log, }, @@ -107,8 +107,8 @@ func TestRunnerLoopMeasureFailure(t *testing.T) { func TestRunnerLoopCollectFailure(t *testing.T) { expected := errors.New("mocked error") - saver := new(trace.Saver) - saver.Write(trace.Event{Name: netxlite.ConnectOperation, Duration: 150 * time.Millisecond}) + saver := new(tracex.Saver) + saver.Write(tracex.Event{Name: netxlite.ConnectOperation, Duration: 150 * time.Millisecond}) r := runner{ callbacks: model.NewPrinterCallbacks(log.Log), httpClient: &http.Client{ @@ -151,8 +151,8 @@ func TestRunnerLoopCollectFailure(t *testing.T) { } func TestRunnerLoopSuccess(t *testing.T) { - saver := new(trace.Saver) - saver.Write(trace.Event{Name: netxlite.ConnectOperation, Duration: 150 * time.Millisecond}) + saver := new(tracex.Saver) + saver.Write(tracex.Event{Name: netxlite.ConnectOperation, Duration: 150 * time.Millisecond}) r := runner{ callbacks: model.NewPrinterCallbacks(log.Log), httpClient: &http.Client{ diff --git a/internal/engine/experiment/dnscheck/dnscheck.go b/internal/engine/experiment/dnscheck/dnscheck.go index f920648..b0e41d6 100644 --- a/internal/engine/experiment/dnscheck/dnscheck.go +++ b/internal/engine/experiment/dnscheck/dnscheck.go @@ -16,8 +16,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" ) @@ -168,15 +167,15 @@ func (m *Measurer) Run( // with IP addresses successfully, we just get back the IPs when we are // passing as input an IP address rather than a domain name. begin := measurement.MeasurementStartTimeSaved - evsaver := new(trace.Saver) + evsaver := new(tracex.Saver) resolver := netx.NewResolver(netx.Config{ BogonIsError: true, Logger: sess.Logger(), ResolveSaver: evsaver, }) addrs, err := m.lookupHost(ctx, URL.Hostname(), resolver) - queries := archival.NewDNSQueriesList(begin, evsaver.Read()) - tk.BootstrapFailure = archival.NewFailure(err) + queries := tracex.NewDNSQueriesList(begin, evsaver.Read()) + tk.BootstrapFailure = tracex.NewFailure(err) if len(queries) > 0 { // We get no queries in case we are resolving an IP address, since // the address resolver doesn't generate events diff --git a/internal/engine/experiment/fbmessenger/fbmessenger_test.go b/internal/engine/experiment/fbmessenger/fbmessenger_test.go index 72cf194..a2d1df5 100644 --- a/internal/engine/experiment/fbmessenger/fbmessenger_test.go +++ b/internal/engine/experiment/fbmessenger/fbmessenger_test.go @@ -10,7 +10,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/mockable" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -169,8 +169,8 @@ func TestComputeEndpointStatsTCPBlocking(t *testing.T) { TestKeys: urlgetter.TestKeys{ Failure: &failure, FailedOperation: &operation, - Queries: []archival.DNSQueryEntry{{ - Answers: []archival.DNSAnswerEntry{{ + Queries: []tracex.DNSQueryEntry{{ + Answers: []tracex.DNSAnswerEntry{{ ASN: fbmessenger.FacebookASN, }}, }}, @@ -199,8 +199,8 @@ func TestComputeEndpointStatsDNSIsLying(t *testing.T) { TestKeys: urlgetter.TestKeys{ Failure: &failure, FailedOperation: &operation, - Queries: []archival.DNSQueryEntry{{ - Answers: []archival.DNSAnswerEntry{{ + Queries: []tracex.DNSQueryEntry{{ + Answers: []tracex.DNSAnswerEntry{{ ASN: 0, }}, }}, diff --git a/internal/engine/experiment/hhfm/hhfm.go b/internal/engine/experiment/hhfm/hhfm.go index de6eb96..bd7808c 100644 --- a/internal/engine/experiment/hhfm/hhfm.go +++ b/internal/engine/experiment/hhfm/hhfm.go @@ -15,7 +15,7 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/ooni/probe-cli/v3/internal/randx" @@ -34,11 +34,11 @@ type Config struct{} // Here we are emitting for the same set of test keys that are // produced by the MK implementation. type TestKeys struct { - Agent string `json:"agent"` - Failure *string `json:"failure"` - Requests []archival.RequestEntry `json:"requests"` - SOCKSProxy *string `json:"socksproxy"` - Tampering Tampering `json:"tampering"` + Agent string `json:"agent"` + Failure *string `json:"failure"` + Requests []tracex.RequestEntry `json:"requests"` + SOCKSProxy *string `json:"socksproxy"` + Tampering Tampering `json:"tampering"` } // Tampering describes the detected forms of tampering. @@ -151,7 +151,7 @@ func (m Measurer) Run( // from that and then see to improve the robustness in the future. resp, data, err := Transact(txp, req.WithContext(ctx), callbacks) if err != nil { - tk.Failure = archival.NewFailure(err) + tk.Failure = tracex.NewFailure(err) tk.Requests[0].Failure = tk.Failure tk.Tampering.Total = true return nil // measurement did not fail, we measured tampering @@ -246,23 +246,23 @@ func (tk *TestKeys) FillTampering( } } -// NewRequestEntryList creates a new []archival.RequestEntry given a +// NewRequestEntryList creates a new []tracex.RequestEntry given a // specific *http.Request and headers with random case. -func NewRequestEntryList(req *http.Request, headers map[string]string) (out []archival.RequestEntry) { - out = []archival.RequestEntry{{ - Request: archival.HTTPRequest{ - Headers: make(map[string]archival.MaybeBinaryValue), - HeadersList: []archival.HTTPHeader{}, +func NewRequestEntryList(req *http.Request, headers map[string]string) (out []tracex.RequestEntry) { + out = []tracex.RequestEntry{{ + Request: tracex.HTTPRequest{ + Headers: make(map[string]tracex.MaybeBinaryValue), + HeadersList: []tracex.HTTPHeader{}, Method: req.Method, URL: req.URL.String(), }, }} for key, value := range headers { // Using the random capitalization headers here - mbv := archival.MaybeBinaryValue{Value: value} + mbv := tracex.MaybeBinaryValue{Value: value} out[0].Request.Headers[key] = mbv out[0].Request.HeadersList = append(out[0].Request.HeadersList, - archival.HTTPHeader{Key: key, Value: mbv}) + tracex.HTTPHeader{Key: key, Value: mbv}) } sort.Slice(out[0].Request.HeadersList, func(i, j int) bool { return out[0].Request.HeadersList[i].Key < out[0].Request.HeadersList[j].Key @@ -270,19 +270,19 @@ func NewRequestEntryList(req *http.Request, headers map[string]string) (out []ar return } -// NewHTTPResponse creates a new archival.HTTPResponse given a +// NewHTTPResponse creates a new tracex.HTTPResponse given a // specific *http.Response instance and its body. -func NewHTTPResponse(resp *http.Response, data []byte) (out archival.HTTPResponse) { - out = archival.HTTPResponse{ - Body: archival.HTTPBody{Value: string(data)}, +func NewHTTPResponse(resp *http.Response, data []byte) (out tracex.HTTPResponse) { + out = tracex.HTTPResponse{ + Body: tracex.HTTPBody{Value: string(data)}, Code: int64(resp.StatusCode), - Headers: make(map[string]archival.MaybeBinaryValue), - HeadersList: []archival.HTTPHeader{}, + Headers: make(map[string]tracex.MaybeBinaryValue), + HeadersList: []tracex.HTTPHeader{}, } for key := range resp.Header { - mbv := archival.MaybeBinaryValue{Value: resp.Header.Get(key)} + mbv := tracex.MaybeBinaryValue{Value: resp.Header.Get(key)} out.Headers[key] = mbv - out.HeadersList = append(out.HeadersList, archival.HTTPHeader{Key: key, Value: mbv}) + out.HeadersList = append(out.HeadersList, tracex.HTTPHeader{Key: key, Value: mbv}) } sort.Slice(out.HeadersList, func(i, j int) bool { return out.HeadersList[i].Key < out.HeadersList[j].Key diff --git a/internal/engine/experiment/hhfm/hhfm_test.go b/internal/engine/experiment/hhfm/hhfm_test.go index 827fa3a..7dd9e24 100644 --- a/internal/engine/experiment/hhfm/hhfm_test.go +++ b/internal/engine/experiment/hhfm/hhfm_test.go @@ -16,7 +16,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/mockable" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -554,7 +554,7 @@ func TestTestKeys_FillTampering(t *testing.T) { type fields struct { Agent string Failure *string - Requests []archival.RequestEntry + Requests []tracex.RequestEntry SOCKSProxy *string Tampering hhfm.Tampering } @@ -689,7 +689,7 @@ func TestNewRequestEntryList(t *testing.T) { tests := []struct { name string args args - wantOut []archival.RequestEntry + wantOut []tracex.RequestEntry }{{ name: "common case", args: args{ @@ -706,16 +706,16 @@ func TestNewRequestEntryList(t *testing.T) { "User-aGENT": "foo/1.0", }, }, - wantOut: []archival.RequestEntry{{ - Request: archival.HTTPRequest{ - HeadersList: []archival.HTTPHeader{{ + wantOut: []tracex.RequestEntry{{ + Request: tracex.HTTPRequest{ + HeadersList: []tracex.HTTPHeader{{ Key: "ContENt-tYPE", - Value: archival.MaybeBinaryValue{Value: "text/plain"}, + Value: tracex.MaybeBinaryValue{Value: "text/plain"}, }, { Key: "User-aGENT", - Value: archival.MaybeBinaryValue{Value: "foo/1.0"}, + Value: tracex.MaybeBinaryValue{Value: "foo/1.0"}, }}, - Headers: map[string]archival.MaybeBinaryValue{ + Headers: map[string]tracex.MaybeBinaryValue{ "ContENt-tYPE": {Value: "text/plain"}, "User-aGENT": {Value: "foo/1.0"}, }, @@ -735,11 +735,11 @@ func TestNewRequestEntryList(t *testing.T) { }, }, }, - wantOut: []archival.RequestEntry{{ - Request: archival.HTTPRequest{ + wantOut: []tracex.RequestEntry{{ + Request: tracex.HTTPRequest{ Method: "GeT", - Headers: make(map[string]archival.MaybeBinaryValue), - HeadersList: []archival.HTTPHeader{}, + Headers: make(map[string]tracex.MaybeBinaryValue), + HeadersList: []tracex.HTTPHeader{}, URL: "http://10.0.0.1/", }, }}, @@ -762,7 +762,7 @@ func TestNewHTTPResponse(t *testing.T) { tests := []struct { name string args args - wantOut archival.HTTPResponse + wantOut tracex.HTTPResponse }{{ name: "common case", args: args{ @@ -775,17 +775,17 @@ func TestNewHTTPResponse(t *testing.T) { }, data: []byte("deadbeef"), }, - wantOut: archival.HTTPResponse{ - Body: archival.MaybeBinaryValue{Value: "deadbeef"}, + wantOut: tracex.HTTPResponse{ + Body: tracex.MaybeBinaryValue{Value: "deadbeef"}, Code: 200, - HeadersList: []archival.HTTPHeader{{ + HeadersList: []tracex.HTTPHeader{{ Key: "Content-Type", - Value: archival.MaybeBinaryValue{Value: "text/plain"}, + Value: tracex.MaybeBinaryValue{Value: "text/plain"}, }, { Key: "User-Agent", - Value: archival.MaybeBinaryValue{Value: "foo/1.0"}, + Value: tracex.MaybeBinaryValue{Value: "foo/1.0"}, }}, - Headers: map[string]archival.MaybeBinaryValue{ + Headers: map[string]tracex.MaybeBinaryValue{ "Content-Type": {Value: "text/plain"}, "User-Agent": {Value: "foo/1.0"}, }, @@ -795,11 +795,11 @@ func TestNewHTTPResponse(t *testing.T) { args: args{ resp: &http.Response{StatusCode: 200}, }, - wantOut: archival.HTTPResponse{ - Body: archival.MaybeBinaryValue{Value: ""}, + wantOut: tracex.HTTPResponse{ + Body: tracex.MaybeBinaryValue{Value: ""}, Code: 200, - HeadersList: []archival.HTTPHeader{}, - Headers: map[string]archival.MaybeBinaryValue{}, + HeadersList: []tracex.HTTPHeader{}, + Headers: map[string]tracex.MaybeBinaryValue{}, }, }} for _, tt := range tests { diff --git a/internal/engine/experiment/hirl/hirl.go b/internal/engine/experiment/hirl/hirl.go index b76ea9d..60d8233 100644 --- a/internal/engine/experiment/hirl/hirl.go +++ b/internal/engine/experiment/hirl/hirl.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/ooni/probe-cli/v3/internal/randx" @@ -29,11 +29,11 @@ type Config struct{} // TestKeys contains the experiment test keys. type TestKeys struct { - FailureList []*string `json:"failure_list"` - Received []archival.MaybeBinaryValue `json:"received"` - Sent []string `json:"sent"` - TamperingList []bool `json:"tampering_list"` - Tampering bool `json:"tampering"` + FailureList []*string `json:"failure_list"` + Received []tracex.MaybeBinaryValue `json:"received"` + Sent []string `json:"sent"` + TamperingList []bool `json:"tampering_list"` + Tampering bool `json:"tampering"` } // NewExperimentMeasurer creates a new ExperimentMeasurer. @@ -123,7 +123,7 @@ func (m Measurer) Run( } continue } - failure := archival.NewFailure(result.Err) + failure := tracex.NewFailure(result.Err) tk.FailureList = append(tk.FailureList, failure) tk.Received = append(tk.Received, result.Received) tk.Sent = append(tk.Sent, result.Sent) @@ -150,7 +150,7 @@ type MethodConfig struct { type MethodResult struct { Err error Name string - Received archival.MaybeBinaryValue + Received tracex.MaybeBinaryValue Sent string Tampering bool } diff --git a/internal/engine/experiment/hirl/hirl_test.go b/internal/engine/experiment/hirl/hirl_test.go index 517dc2f..f71207a 100644 --- a/internal/engine/experiment/hirl/hirl_test.go +++ b/internal/engine/experiment/hirl/hirl_test.go @@ -10,7 +10,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl" "github.com/ooni/probe-cli/v3/internal/engine/mockable" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -149,7 +149,7 @@ func (FakeMethodSuccessful) Name() string { func (meth FakeMethodSuccessful) Run(ctx context.Context, config hirl.MethodConfig) { config.Out <- hirl.MethodResult{ Name: meth.Name(), - Received: archival.MaybeBinaryValue{Value: "antani"}, + Received: tracex.MaybeBinaryValue{Value: "antani"}, Sent: "antani", Tampering: false, } @@ -164,7 +164,7 @@ func (FakeMethodFailure) Name() string { func (meth FakeMethodFailure) Run(ctx context.Context, config hirl.MethodConfig) { config.Out <- hirl.MethodResult{ Name: meth.Name(), - Received: archival.MaybeBinaryValue{Value: "antani"}, + Received: tracex.MaybeBinaryValue{Value: "antani"}, Sent: "melandri", Tampering: true, } diff --git a/internal/engine/experiment/quicping/quicping.go b/internal/engine/experiment/quicping/quicping.go index 4b210c1..6859877 100644 --- a/internal/engine/experiment/quicping/quicping.go +++ b/internal/engine/experiment/quicping/quicping.go @@ -17,7 +17,7 @@ import ( _ "crypto/sha256" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -102,7 +102,7 @@ func makeResponse(resp *responseInfo) *SinglePingResponse { } return &SinglePingResponse{ Data: data, - Failure: archival.NewFailure(resp.err), + Failure: tracex.NewFailure(resp.err), T: resp.t, SupportedVersions: resp.versions, } @@ -273,7 +273,7 @@ L: tk.Pings = append(tk.Pings, &SinglePing{ ConnIdDst: req.dstID, ConnIdSrc: req.srcID, - Failure: archival.NewFailure(req.err), + Failure: tracex.NewFailure(req.err), Request: &model.ArchivalMaybeBinaryData{Value: string(req.raw)}, T: req.t, }) @@ -313,7 +313,7 @@ L: tk.Pings = append(tk.Pings, &SinglePing{ ConnIdDst: ping.request.dstID, ConnIdSrc: ping.request.srcID, - Failure: archival.NewFailure(timeoutErr), + Failure: tracex.NewFailure(timeoutErr), Request: &model.ArchivalMaybeBinaryData{Value: string(ping.request.raw)}, T: ping.request.t, }) diff --git a/internal/engine/experiment/riseupvpn/riseupvpn.go b/internal/engine/experiment/riseupvpn/riseupvpn.go index e9dd7bc..5de2240 100644 --- a/internal/engine/experiment/riseupvpn/riseupvpn.go +++ b/internal/engine/experiment/riseupvpn/riseupvpn.go @@ -10,7 +10,7 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -132,7 +132,7 @@ func (tk *TestKeys) updateTransportStatus(openvpnGatewayCount, obfs4GatewayCount } func newGatewayConnection( - tcpConnect archival.TCPConnectEntry, transportType string) *GatewayConnection { + tcpConnect tracex.TCPConnectEntry, transportType string) *GatewayConnection { return &GatewayConnection{ IP: tcpConnect.IP, Port: tcpConnect.Port, diff --git a/internal/engine/experiment/riseupvpn/riseupvpn_test.go b/internal/engine/experiment/riseupvpn/riseupvpn_test.go index 5da3efe..793cb55 100644 --- a/internal/engine/experiment/riseupvpn/riseupvpn_test.go +++ b/internal/engine/experiment/riseupvpn/riseupvpn_test.go @@ -15,7 +15,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/mockable" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -735,11 +735,11 @@ func generateMockGetter(requestResponse map[string]string, responseStatus map[st responseStatus = 0 } - tcpConnect := archival.TCPConnectEntry{ + tcpConnect := tracex.TCPConnectEntry{ // use some dummy IP/Port combination for URLs, we don't do DNS resolution IP: "123.456.234.123", Port: 443, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Success: isSuccessStatus, Blocked: &isBlocked, Failure: failure, @@ -759,21 +759,21 @@ func generateMockGetter(requestResponse map[string]string, responseStatus map[st FailedOperation: failedOperation, HTTPResponseStatus: responseStatus, HTTPResponseBody: responseBody, - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: failure, - Request: archival.HTTPRequest{ + Request: tracex.HTTPRequest{ URL: url, - Body: archival.MaybeBinaryValue{}, + Body: tracex.MaybeBinaryValue{}, BodyIsTruncated: false, }, - Response: archival.HTTPResponse{ - Body: archival.HTTPBody{ + Response: tracex.HTTPResponse{ + Body: tracex.HTTPBody{ Value: responseBody, }, BodyIsTruncated: false, }}, }, - TCPConnect: []archival.TCPConnectEntry{tcpConnect}, + TCPConnect: []tracex.TCPConnectEntry{tcpConnect}, } return tk, nil } diff --git a/internal/engine/experiment/stunreachability/stunreachability.go b/internal/engine/experiment/stunreachability/stunreachability.go index e205ce3..e4c18d0 100644 --- a/internal/engine/experiment/stunreachability/stunreachability.go +++ b/internal/engine/experiment/stunreachability/stunreachability.go @@ -12,8 +12,7 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/pion/stun" @@ -32,15 +31,15 @@ type Config struct { // TestKeys contains the experiment's result. type TestKeys struct { - Endpoint string `json:"endpoint"` - Failure *string `json:"failure"` - NetworkEvents []archival.NetworkEvent `json:"network_events"` - Queries []archival.DNSQueryEntry `json:"queries"` + Endpoint string `json:"endpoint"` + Failure *string `json:"failure"` + NetworkEvents []tracex.NetworkEvent `json:"network_events"` + Queries []tracex.DNSQueryEntry `json:"queries"` } func registerExtensions(m *model.Measurement) { - archival.ExtDNS.AddTo(m) - archival.ExtNetevents.AddTo(m) + tracex.ExtDNS.AddTo(m) + tracex.ExtNetevents.AddTo(m) } // Measurer performs the measurement. @@ -113,7 +112,7 @@ func (tk *TestKeys) run( defer callbacks.OnProgress( 1, fmt.Sprintf("stunreachability: measuring: %s... done", endpoint)) tk.Endpoint = endpoint - saver := new(trace.Saver) + saver := new(tracex.Saver) begin := time.Now() err := tk.do(ctx, config, netx.NewDialer(netx.Config{ ContextByteCounting: true, @@ -124,10 +123,10 @@ func (tk *TestKeys) run( }), endpoint) events := saver.Read() tk.NetworkEvents = append( - tk.NetworkEvents, archival.NewNetworkEventsList(begin, events)..., + tk.NetworkEvents, tracex.NewNetworkEventsList(begin, events)..., ) tk.Queries = append( - tk.Queries, archival.NewDNSQueriesList(begin, events)..., + tk.Queries, tracex.NewDNSQueriesList(begin, events)..., ) return err } diff --git a/internal/engine/experiment/tlstool/tlstool.go b/internal/engine/experiment/tlstool/tlstool.go index 6a371fe..6188db1 100644 --- a/internal/engine/experiment/tlstool/tlstool.go +++ b/internal/engine/experiment/tlstool/tlstool.go @@ -17,7 +17,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool/internal" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" ) @@ -101,7 +101,7 @@ func (m Measurer) Run( percent := float64(idx) / float64(len(allMethods)) callbacks.OnProgress(percent, fmt.Sprintf("%s: %+v", meth.name, err)) tk.Experiment[meth.name] = &ExperimentKeys{ - Failure: archival.NewFailure(err), + Failure: tracex.NewFailure(err), } } return nil // return nil so we always submit the measurement diff --git a/internal/engine/experiment/tor/tor.go b/internal/engine/experiment/tor/tor.go index 0c771a3..a60be9d 100644 --- a/internal/engine/experiment/tor/tor.go +++ b/internal/engine/experiment/tor/tor.go @@ -13,7 +13,7 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/atomicx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/measurex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" @@ -62,11 +62,11 @@ type TargetResults struct { } func registerExtensions(m *model.Measurement) { - archival.ExtHTTP.AddTo(m) - archival.ExtNetevents.AddTo(m) - archival.ExtDNS.AddTo(m) - archival.ExtTCPConnect.AddTo(m) - archival.ExtTLSHandshake.AddTo(m) + tracex.ExtHTTP.AddTo(m) + tracex.ExtNetevents.AddTo(m) + tracex.ExtDNS.AddTo(m) + tracex.ExtTCPConnect.AddTo(m) + tracex.ExtTLSHandshake.AddTo(m) } // fillSummary fills the Summary field used by the UI. diff --git a/internal/engine/experiment/torsf/torsf.go b/internal/engine/experiment/torsf/torsf.go index d46307e..7675eb1 100644 --- a/internal/engine/experiment/torsf/torsf.go +++ b/internal/engine/experiment/torsf/torsf.go @@ -11,7 +11,7 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/bytecounter" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/ptx" "github.com/ooni/probe-cli/v3/internal/runtimex" @@ -232,8 +232,8 @@ func (m *Measurer) bootstrap(ctx context.Context, timeout time.Duration, sess mo tk.TorVersion = debugInfo.Version m.readTorLogs(sess.Logger(), tk, debugInfo.LogFilePath) if err != nil { - // Note: archival.NewFailure scrubs IP addresses - tk.Failure = archival.NewFailure(err) + // Note: tracex.NewFailure scrubs IP addresses + tk.Failure = tracex.NewFailure(err) if errors.Is(err, context.DeadlineExceeded) { tk.Error = &timeoutReachedError } else { diff --git a/internal/engine/experiment/urlgetter/configurer.go b/internal/engine/experiment/urlgetter/configurer.go index 4a22e3b..a1bf443 100644 --- a/internal/engine/experiment/urlgetter/configurer.go +++ b/internal/engine/experiment/urlgetter/configurer.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -20,7 +20,7 @@ type Configurer struct { Config Config Logger model.Logger ProxyURL *url.URL - Saver *trace.Saver + Saver *tracex.Saver } // The Configuration is the configuration for running a measurement. diff --git a/internal/engine/experiment/urlgetter/configurer_test.go b/internal/engine/experiment/urlgetter/configurer_test.go index 07dfd95..2c27c5d 100644 --- a/internal/engine/experiment/urlgetter/configurer_test.go +++ b/internal/engine/experiment/urlgetter/configurer_test.go @@ -9,13 +9,12 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/netxlite" ) func TestConfigurerNewConfigurationVanilla(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Logger: log.Log, Saver: saver, @@ -73,7 +72,7 @@ func TestConfigurerNewConfigurationVanilla(t *testing.T) { } func TestConfigurerNewConfigurationResolverDNSOverHTTPSPowerdns(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ ResolverURL: "doh://google", @@ -120,7 +119,7 @@ func TestConfigurerNewConfigurationResolverDNSOverHTTPSPowerdns(t *testing.T) { if !ok { t.Fatal("not the resolver we expected") } - stxp, ok := sr.Txp.(resolver.SaverDNSTransport) + stxp, ok := sr.Txp.(tracex.SaverDNSTransport) if !ok { t.Fatal("not the DNS transport we expected") } @@ -149,7 +148,7 @@ func TestConfigurerNewConfigurationResolverDNSOverHTTPSPowerdns(t *testing.T) { } func TestConfigurerNewConfigurationResolverDNSOverHTTPSGoogle(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ ResolverURL: "doh://google", @@ -196,7 +195,7 @@ func TestConfigurerNewConfigurationResolverDNSOverHTTPSGoogle(t *testing.T) { if !ok { t.Fatal("not the resolver we expected") } - stxp, ok := sr.Txp.(resolver.SaverDNSTransport) + stxp, ok := sr.Txp.(tracex.SaverDNSTransport) if !ok { t.Fatal("not the DNS transport we expected") } @@ -225,7 +224,7 @@ func TestConfigurerNewConfigurationResolverDNSOverHTTPSGoogle(t *testing.T) { } func TestConfigurerNewConfigurationResolverDNSOverHTTPSCloudflare(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ ResolverURL: "doh://cloudflare", @@ -272,7 +271,7 @@ func TestConfigurerNewConfigurationResolverDNSOverHTTPSCloudflare(t *testing.T) if !ok { t.Fatal("not the resolver we expected") } - stxp, ok := sr.Txp.(resolver.SaverDNSTransport) + stxp, ok := sr.Txp.(tracex.SaverDNSTransport) if !ok { t.Fatal("not the DNS transport we expected") } @@ -301,7 +300,7 @@ func TestConfigurerNewConfigurationResolverDNSOverHTTPSCloudflare(t *testing.T) } func TestConfigurerNewConfigurationResolverUDP(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ ResolverURL: "udp://8.8.8.8:53", @@ -348,7 +347,7 @@ func TestConfigurerNewConfigurationResolverUDP(t *testing.T) { if !ok { t.Fatal("not the resolver we expected") } - stxp, ok := sr.Txp.(resolver.SaverDNSTransport) + stxp, ok := sr.Txp.(tracex.SaverDNSTransport) if !ok { t.Fatal("not the DNS transport we expected") } @@ -377,7 +376,7 @@ func TestConfigurerNewConfigurationResolverUDP(t *testing.T) { } func TestConfigurerNewConfigurationDNSCacheInvalidString(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ DNSCache: "a", @@ -392,7 +391,7 @@ func TestConfigurerNewConfigurationDNSCacheInvalidString(t *testing.T) { } func TestConfigurerNewConfigurationDNSCacheNotDomain(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ DNSCache: "b b", @@ -407,7 +406,7 @@ func TestConfigurerNewConfigurationDNSCacheNotDomain(t *testing.T) { } func TestConfigurerNewConfigurationDNSCacheNotIP(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ DNSCache: "x.org b", @@ -422,7 +421,7 @@ func TestConfigurerNewConfigurationDNSCacheNotIP(t *testing.T) { } func TestConfigurerNewConfigurationDNSCacheGood(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ DNSCache: "dns.google.com 8.8.8.8 8.8.4.4", @@ -449,7 +448,7 @@ func TestConfigurerNewConfigurationDNSCacheGood(t *testing.T) { } func TestConfigurerNewConfigurationResolverInvalidURL(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ ResolverURL: "\t", @@ -464,7 +463,7 @@ func TestConfigurerNewConfigurationResolverInvalidURL(t *testing.T) { } func TestConfigurerNewConfigurationResolverInvalidURLScheme(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ ResolverURL: "antani://8.8.8.8:53", @@ -479,7 +478,7 @@ func TestConfigurerNewConfigurationResolverInvalidURLScheme(t *testing.T) { } func TestConfigurerNewConfigurationTLSServerName(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ TLSServerName: "www.x.org", @@ -506,7 +505,7 @@ func TestConfigurerNewConfigurationTLSServerName(t *testing.T) { } func TestConfigurerNewConfigurationNoTLSVerify(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ NoTLSVerify: true, @@ -524,7 +523,7 @@ func TestConfigurerNewConfigurationNoTLSVerify(t *testing.T) { } func TestConfigurerNewConfigurationTLSv1(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ TLSVersion: "TLSv1", @@ -554,7 +553,7 @@ func TestConfigurerNewConfigurationTLSv1(t *testing.T) { } func TestConfigurerNewConfigurationTLSv1dot0(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ TLSVersion: "TLSv1.0", @@ -584,7 +583,7 @@ func TestConfigurerNewConfigurationTLSv1dot0(t *testing.T) { } func TestConfigurerNewConfigurationTLSv1dot1(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ TLSVersion: "TLSv1.1", @@ -614,7 +613,7 @@ func TestConfigurerNewConfigurationTLSv1dot1(t *testing.T) { } func TestConfigurerNewConfigurationTLSv1dot2(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ TLSVersion: "TLSv1.2", @@ -644,7 +643,7 @@ func TestConfigurerNewConfigurationTLSv1dot2(t *testing.T) { } func TestConfigurerNewConfigurationTLSv1dot3(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ TLSVersion: "TLSv1.3", @@ -674,7 +673,7 @@ func TestConfigurerNewConfigurationTLSv1dot3(t *testing.T) { } func TestConfigurerNewConfigurationTLSvDefault(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{}, Logger: log.Log, @@ -702,7 +701,7 @@ func TestConfigurerNewConfigurationTLSvDefault(t *testing.T) { } func TestConfigurerNewConfigurationTLSvInvalid(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Config: urlgetter.Config{ TLSVersion: "SSLv3", @@ -718,7 +717,7 @@ func TestConfigurerNewConfigurationTLSvInvalid(t *testing.T) { func TestConfigurerNewConfigurationProxyURL(t *testing.T) { URL, _ := url.Parse("socks5://127.0.0.1:9050") - saver := new(trace.Saver) + saver := new(tracex.Saver) configurer := urlgetter.Configurer{ Logger: log.Log, Saver: saver, diff --git a/internal/engine/experiment/urlgetter/getter.go b/internal/engine/experiment/urlgetter/getter.go index 4e6ca5b..84de6fa 100644 --- a/internal/engine/experiment/urlgetter/getter.go +++ b/internal/engine/experiment/urlgetter/getter.go @@ -6,8 +6,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/ooni/probe-cli/v3/internal/tunnel" @@ -50,22 +49,22 @@ func (g Getter) Get(ctx context.Context) (TestKeys, error) { if g.Begin.IsZero() { g.Begin = time.Now() } - saver := new(trace.Saver) + saver := new(tracex.Saver) tk, err := g.get(ctx, saver) // Make sure we have an operation in cases where we fail before // hitting our httptransport that does error wrapping. if err != nil { err = netxlite.NewTopLevelGenericErrWrapper(err) } - tk.FailedOperation = archival.NewFailedOperation(err) - tk.Failure = archival.NewFailure(err) + tk.FailedOperation = tracex.NewFailedOperation(err) + tk.Failure = tracex.NewFailure(err) events := saver.Read() - tk.Queries = append(tk.Queries, archival.NewDNSQueriesList(g.Begin, events)...) + tk.Queries = append(tk.Queries, tracex.NewDNSQueriesList(g.Begin, events)...) tk.NetworkEvents = append( - tk.NetworkEvents, archival.NewNetworkEventsList(g.Begin, events)..., + tk.NetworkEvents, tracex.NewNetworkEventsList(g.Begin, events)..., ) tk.Requests = append( - tk.Requests, archival.NewRequestList(g.Begin, events)..., + tk.Requests, tracex.NewRequestList(g.Begin, events)..., ) if len(tk.Requests) > 0 { // OONI's convention is that the last request appears first @@ -74,10 +73,10 @@ func (g Getter) Get(ctx context.Context) (TestKeys, error) { tk.HTTPResponseLocations = tk.Requests[0].Response.Locations } tk.TCPConnect = append( - tk.TCPConnect, archival.NewTCPConnectList(g.Begin, events)..., + tk.TCPConnect, tracex.NewTCPConnectList(g.Begin, events)..., ) tk.TLSHandshakes = append( - tk.TLSHandshakes, archival.NewTLSHandshakesList(g.Begin, events)..., + tk.TLSHandshakes, tracex.NewTLSHandshakesList(g.Begin, events)..., ) return tk, err } @@ -90,7 +89,7 @@ func (g Getter) ioutilTempDir(dir, pattern string) (string, error) { return ioutil.TempDir(dir, pattern) } -func (g Getter) get(ctx context.Context, saver *trace.Saver) (TestKeys, error) { +func (g Getter) get(ctx context.Context, saver *tracex.Saver) (TestKeys, error) { tk := TestKeys{ Agent: "redirect", Tunnel: g.Config.Tunnel, diff --git a/internal/engine/experiment/urlgetter/urlgetter.go b/internal/engine/experiment/urlgetter/urlgetter.go index 2a0c06f..c9ffbe6 100644 --- a/internal/engine/experiment/urlgetter/urlgetter.go +++ b/internal/engine/experiment/urlgetter/urlgetter.go @@ -8,7 +8,7 @@ import ( "crypto/x509" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -45,18 +45,18 @@ type Config struct { // TestKeys contains the experiment's result. type TestKeys struct { // The following fields are part of the typical JSON emitted by OONI. - Agent string `json:"agent"` - BootstrapTime float64 `json:"bootstrap_time,omitempty"` - DNSCache []string `json:"dns_cache,omitempty"` - FailedOperation *string `json:"failed_operation"` - Failure *string `json:"failure"` - NetworkEvents []archival.NetworkEvent `json:"network_events"` - Queries []archival.DNSQueryEntry `json:"queries"` - Requests []archival.RequestEntry `json:"requests"` - SOCKSProxy string `json:"socksproxy,omitempty"` - TCPConnect []archival.TCPConnectEntry `json:"tcp_connect"` - TLSHandshakes []archival.TLSHandshake `json:"tls_handshakes"` - Tunnel string `json:"tunnel,omitempty"` + Agent string `json:"agent"` + BootstrapTime float64 `json:"bootstrap_time,omitempty"` + DNSCache []string `json:"dns_cache,omitempty"` + FailedOperation *string `json:"failed_operation"` + Failure *string `json:"failure"` + NetworkEvents []tracex.NetworkEvent `json:"network_events"` + Queries []tracex.DNSQueryEntry `json:"queries"` + Requests []tracex.RequestEntry `json:"requests"` + SOCKSProxy string `json:"socksproxy,omitempty"` + TCPConnect []tracex.TCPConnectEntry `json:"tcp_connect"` + TLSHandshakes []tracex.TLSHandshake `json:"tls_handshakes"` + Tunnel string `json:"tunnel,omitempty"` // The following fields are not serialised but are useful to simplify // analysing the measurements in telegram, whatsapp, etc. @@ -68,12 +68,12 @@ type TestKeys struct { // RegisterExtensions registers the extensions used by the urlgetter // experiment into the provided measurement. func RegisterExtensions(m *model.Measurement) { - archival.ExtHTTP.AddTo(m) - archival.ExtDNS.AddTo(m) - archival.ExtNetevents.AddTo(m) - archival.ExtTCPConnect.AddTo(m) - archival.ExtTLSHandshake.AddTo(m) - archival.ExtTunnel.AddTo(m) + tracex.ExtHTTP.AddTo(m) + tracex.ExtDNS.AddTo(m) + tracex.ExtNetevents.AddTo(m) + tracex.ExtTCPConnect.AddTo(m) + tracex.ExtTLSHandshake.AddTo(m) + tracex.ExtTunnel.AddTo(m) } // Measurer performs the measurement. diff --git a/internal/engine/experiment/vanillator/vanillator.go b/internal/engine/experiment/vanillator/vanillator.go index 625a1f0..d16c9b0 100644 --- a/internal/engine/experiment/vanillator/vanillator.go +++ b/internal/engine/experiment/vanillator/vanillator.go @@ -10,7 +10,7 @@ import ( "path" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/torlogs" @@ -172,8 +172,8 @@ func (m *Measurer) bootstrap(ctx context.Context, timeout time.Duration, tk.TorVersion = debugInfo.Version m.readTorLogs(sess.Logger(), tk, debugInfo.LogFilePath) if err != nil { - // Note: archival.NewFailure scrubs IP addresses - tk.Failure = archival.NewFailure(err) + // Note: tracex.NewFailure scrubs IP addresses + tk.Failure = tracex.NewFailure(err) if errors.Is(err, context.DeadlineExceeded) { tk.Error = &timeoutReachedError } else { diff --git a/internal/engine/experiment/webconnectivity/httpanalysis_test.go b/internal/engine/experiment/webconnectivity/httpanalysis_test.go index 2b6ee9f..62b630f 100644 --- a/internal/engine/experiment/webconnectivity/httpanalysis_test.go +++ b/internal/engine/experiment/webconnectivity/httpanalysis_test.go @@ -6,7 +6,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/randx" ) @@ -42,8 +42,8 @@ func TestHTTPBodyLengthChecks(t *testing.T) { name: "response body is truncated", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ BodyIsTruncated: true, }, }}, @@ -59,8 +59,8 @@ func TestHTTPBodyLengthChecks(t *testing.T) { name: "response body length is zero", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{}, + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{}, }}, }, ctrl: webconnectivity.ControlResponse{ @@ -74,9 +74,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { name: "control length is negative", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Body: archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Body: tracex.MaybeBinaryValue{ Value: randx.Letters(768), }, }, @@ -93,9 +93,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { name: "match with bigger control", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Body: archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Body: tracex.MaybeBinaryValue{ Value: randx.Letters(768), }, }, @@ -113,9 +113,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { name: "match with bigger measurement", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Body: archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Body: tracex.MaybeBinaryValue{ Value: randx.Letters(1024), }, }, @@ -133,9 +133,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { name: "not match with bigger control", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Body: archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Body: tracex.MaybeBinaryValue{ Value: randx.Letters(8), }, }, @@ -153,9 +153,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { name: "match with bigger measurement", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Body: archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Body: tracex.MaybeBinaryValue{ Value: randx.Letters(16), }, }, @@ -203,15 +203,15 @@ func TestStatusCodeMatch(t *testing.T) { name: "with a request but zero status codes", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{}}, + Requests: []tracex.RequestEntry{{}}, }, }, }, { name: "with equal status codes including 5xx", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 501, }, }}, @@ -227,8 +227,8 @@ func TestStatusCodeMatch(t *testing.T) { name: "with different status codes and the control being 5xx", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 407, }, }}, @@ -244,8 +244,8 @@ func TestStatusCodeMatch(t *testing.T) { name: "with different status codes and the control being not 5xx", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 407, }, }}, @@ -261,8 +261,8 @@ func TestStatusCodeMatch(t *testing.T) { name: "with only response status code and no control status code", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, }, }}, @@ -272,8 +272,8 @@ func TestStatusCodeMatch(t *testing.T) { name: "with response status code and -1 as control status code", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, }, }}, @@ -288,8 +288,8 @@ func TestStatusCodeMatch(t *testing.T) { name: "with only control status code and no response status code", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 0, }, }}, @@ -346,7 +346,7 @@ func TestHeadersMatch(t *testing.T) { name: "with request and no response status code", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{}}, + Requests: []tracex.RequestEntry{{}}, }, ctrl: webconnectivity.ControlResponse{ HTTPRequest: webconnectivity.ControlHTTPRequestResult{ @@ -363,9 +363,9 @@ func TestHeadersMatch(t *testing.T) { name: "with no control status code", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, }, Code: 200, @@ -379,9 +379,9 @@ func TestHeadersMatch(t *testing.T) { name: "with negative control status code", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, }, Code: 200, @@ -399,9 +399,9 @@ func TestHeadersMatch(t *testing.T) { name: "with no uncommon headers", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, }, Code: 200, @@ -422,9 +422,9 @@ func TestHeadersMatch(t *testing.T) { name: "with equal uncommon headers", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, "Antani": {Value: "MASCETTI"}, }, @@ -447,9 +447,9 @@ func TestHeadersMatch(t *testing.T) { name: "with different uncommon headers", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, "Antani": {Value: "MASCETTI"}, }, @@ -472,9 +472,9 @@ func TestHeadersMatch(t *testing.T) { name: "with small uncommon intersection (X-Cache)", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "Accept-Ranges": {Value: "bytes"}, "Age": {Value: "404727"}, "Cache-Control": {Value: "max-age=604800"}, @@ -519,9 +519,9 @@ func TestHeadersMatch(t *testing.T) { name: "with no uncommon intersection", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "Accept-Ranges": {Value: "bytes"}, "Age": {Value: "404727"}, "Cache-Control": {Value: "max-age=604800"}, @@ -564,9 +564,9 @@ func TestHeadersMatch(t *testing.T) { name: "with exactly equal headers", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "Accept-Ranges": {Value: "bytes"}, "Age": {Value: "404727"}, "Cache-Control": {Value: "max-age=604800"}, @@ -605,9 +605,9 @@ func TestHeadersMatch(t *testing.T) { name: "with equal headers except for the case", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ - Headers: map[string]archival.MaybeBinaryValue{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ + Headers: map[string]tracex.MaybeBinaryValue{ "accept-ranges": {Value: "bytes"}, "AGE": {Value: "404727"}, "cache-Control": {Value: "max-age=604800"}, @@ -674,7 +674,7 @@ func TestTitleMatch(t *testing.T) { name: "with a request and no response", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{}}, + Requests: []tracex.RequestEntry{{}}, }, }, wantOut: nil, @@ -682,8 +682,8 @@ func TestTitleMatch(t *testing.T) { name: "with a response with truncated body", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, BodyIsTruncated: true, }, @@ -695,10 +695,10 @@ func TestTitleMatch(t *testing.T) { name: "with a response with good body", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, - Body: archival.MaybeBinaryValue{Value: ""}, + Body: tracex.MaybeBinaryValue{Value: ""}, }, }}, }, @@ -708,10 +708,10 @@ func TestTitleMatch(t *testing.T) { name: "with all good but no titles", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, - Body: archival.MaybeBinaryValue{Value: ""}, + Body: tracex.MaybeBinaryValue{Value: ""}, }, }}, }, @@ -727,10 +727,10 @@ func TestTitleMatch(t *testing.T) { name: "reasonably common case where it succeeds", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, - Body: archival.MaybeBinaryValue{ + Body: tracex.MaybeBinaryValue{ Value: "La community di MSN"}, }, }}, @@ -747,10 +747,10 @@ func TestTitleMatch(t *testing.T) { name: "reasonably common case where it fails", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, - Body: archival.MaybeBinaryValue{ + Body: tracex.MaybeBinaryValue{ Value: "La communità di MSN"}, }, }}, @@ -767,10 +767,10 @@ func TestTitleMatch(t *testing.T) { name: "when the title is too long", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, - Body: archival.MaybeBinaryValue{ + Body: tracex.MaybeBinaryValue{ Value: "" + randx.Letters(1024) + ""}, }, }}, @@ -787,10 +787,10 @@ func TestTitleMatch(t *testing.T) { name: "reasonably common case where it succeeds with case variations", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, - Body: archival.MaybeBinaryValue{ + Body: tracex.MaybeBinaryValue{ Value: "La commUNity di MSN"}, }, }}, @@ -807,10 +807,10 @@ func TestTitleMatch(t *testing.T) { name: "when the control status code is negative", args: args{ tk: urlgetter.TestKeys{ - Requests: []archival.RequestEntry{{ - Response: archival.HTTPResponse{ + Requests: []tracex.RequestEntry{{ + Response: tracex.HTTPResponse{ Code: 200, - Body: archival.MaybeBinaryValue{ + Body: tracex.MaybeBinaryValue{ Value: "La commUNity di MSN"}, }, }}, diff --git a/internal/engine/experiment/webconnectivity/summary_test.go b/internal/engine/experiment/webconnectivity/summary_test.go index 5808e9f..f1f6487 100644 --- a/internal/engine/experiment/webconnectivity/summary_test.go +++ b/internal/engine/experiment/webconnectivity/summary_test.go @@ -6,7 +6,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -41,8 +41,8 @@ func TestSummarize(t *testing.T) { name: "with an HTTPS request with no failure", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ - Request: archival.HTTPRequest{ + Requests: []tracex.RequestEntry{{ + Request: tracex.HTTPRequest{ URL: "https://www.kernel.org/", }, Failure: nil, @@ -210,7 +210,7 @@ func TestSummarize(t *testing.T) { name: "with connection refused", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeConnectionRefused, }}, }, @@ -226,7 +226,7 @@ func TestSummarize(t *testing.T) { name: "with connection reset", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeConnectionReset, }}, }, @@ -242,7 +242,7 @@ func TestSummarize(t *testing.T) { name: "with NXDOMAIN", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeNXDOMAIN, }}, }, @@ -258,7 +258,7 @@ func TestSummarize(t *testing.T) { name: "with EOF", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeEOFError, }}, }, @@ -274,7 +274,7 @@ func TestSummarize(t *testing.T) { name: "with timeout", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeTimeout, }}, }, @@ -290,7 +290,7 @@ func TestSummarize(t *testing.T) { name: "with SSL invalid hostname", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeSSLInvalidHost, }}, }, @@ -306,7 +306,7 @@ func TestSummarize(t *testing.T) { name: "with SSL invalid cert", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeSSLInvalidCert, }}, }, @@ -322,7 +322,7 @@ func TestSummarize(t *testing.T) { name: "with SSL unknown auth", args: args{ tk: &webconnectivity.TestKeys{ - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeSSLUnknownAuth, }}, }, @@ -341,7 +341,7 @@ func TestSummarize(t *testing.T) { DNSAnalysisResult: webconnectivity.DNSAnalysisResult{ DNSConsistency: &webconnectivity.DNSInconsistent, }, - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeSSLUnknownAuth, }}, }, @@ -361,7 +361,7 @@ func TestSummarize(t *testing.T) { DNSAnalysisResult: webconnectivity.DNSAnalysisResult{ DNSConsistency: &webconnectivity.DNSInconsistent, }, - Requests: []archival.RequestEntry{{ + Requests: []tracex.RequestEntry{{ Failure: &probeSSLUnknownAuth, }, {}}, }, @@ -381,7 +381,7 @@ func TestSummarize(t *testing.T) { StatusCodeMatch: &trueValue, BodyLengthMatch: &trueValue, }, - Requests: []archival.RequestEntry{{}}, + Requests: []tracex.RequestEntry{{}}, }, }, wantOut: webconnectivity.Summary{ @@ -398,7 +398,7 @@ func TestSummarize(t *testing.T) { StatusCodeMatch: &trueValue, HeadersMatch: &trueValue, }, - Requests: []archival.RequestEntry{{}}, + Requests: []tracex.RequestEntry{{}}, }, }, wantOut: webconnectivity.Summary{ @@ -415,7 +415,7 @@ func TestSummarize(t *testing.T) { StatusCodeMatch: &trueValue, TitleMatch: &trueValue, }, - Requests: []archival.RequestEntry{{}}, + Requests: []tracex.RequestEntry{{}}, }, }, wantOut: webconnectivity.Summary{ @@ -432,7 +432,7 @@ func TestSummarize(t *testing.T) { StatusCodeMatch: &falseValue, TitleMatch: &trueValue, }, - Requests: []archival.RequestEntry{{}}, + Requests: []tracex.RequestEntry{{}}, DNSAnalysisResult: webconnectivity.DNSAnalysisResult{ DNSConsistency: &webconnectivity.DNSInconsistent, }, @@ -453,7 +453,7 @@ func TestSummarize(t *testing.T) { StatusCodeMatch: &falseValue, TitleMatch: &trueValue, }, - Requests: []archival.RequestEntry{{}}, + Requests: []tracex.RequestEntry{{}}, DNSAnalysisResult: webconnectivity.DNSAnalysisResult{ DNSConsistency: &webconnectivity.DNSConsistent, }, diff --git a/internal/engine/experiment/webconnectivity/webconnectivity.go b/internal/engine/experiment/webconnectivity/webconnectivity.go index ad43525..f0a2556 100644 --- a/internal/engine/experiment/webconnectivity/webconnectivity.go +++ b/internal/engine/experiment/webconnectivity/webconnectivity.go @@ -9,7 +9,7 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -34,12 +34,12 @@ type TestKeys struct { // is rather obvious where they come from. // // See https://github.com/ooni/probe/issues/1413. - NetworkEvents []archival.NetworkEvent `json:"network_events"` - TLSHandshakes []archival.TLSHandshake `json:"tls_handshakes"` + NetworkEvents []tracex.NetworkEvent `json:"network_events"` + TLSHandshakes []tracex.TLSHandshake `json:"tls_handshakes"` // DNS experiment - Queries []archival.DNSQueryEntry `json:"queries"` - DNSExperimentFailure *string `json:"dns_experiment_failure"` + Queries []tracex.DNSQueryEntry `json:"queries"` + DNSExperimentFailure *string `json:"dns_experiment_failure"` DNSAnalysisResult // Control experiment @@ -48,13 +48,13 @@ type TestKeys struct { Control ControlResponse `json:"control"` // TCP/TLS "connect" experiment - TCPConnect []archival.TCPConnectEntry `json:"tcp_connect"` - TCPConnectSuccesses int `json:"-"` - TCPConnectAttempts int `json:"-"` + TCPConnect []tracex.TCPConnectEntry `json:"tcp_connect"` + TCPConnectSuccesses int `json:"-"` + TCPConnectAttempts int `json:"-"` // HTTP experiment - Requests []archival.RequestEntry `json:"requests"` - HTTPExperimentFailure *string `json:"http_experiment_failure"` + Requests []tracex.RequestEntry `json:"requests"` + HTTPExperimentFailure *string `json:"http_experiment_failure"` HTTPAnalysisResult // Top-level analysis @@ -180,7 +180,7 @@ func (m Measurer) Run( TCPConnect: epnts.Endpoints(), }) tk.THRuntime = time.Since(thBegin) - tk.ControlFailure = archival.NewFailure(err) + tk.ControlFailure = tracex.NewFailure(err) // 4. analyze DNS results if tk.ControlFailure == nil { tk.DNSAnalysisResult = DNSAnalysis(URL, dnsResult, tk.Control) @@ -240,9 +240,9 @@ func (m Measurer) Run( // ComputeTCPBlocking will return a copy of the input TCPConnect structure // where we set the Blocking value depending on the control results. -func ComputeTCPBlocking(measurement []archival.TCPConnectEntry, - control map[string]ControlTCPConnectResult) (out []archival.TCPConnectEntry) { - out = []archival.TCPConnectEntry{} +func ComputeTCPBlocking(measurement []tracex.TCPConnectEntry, + control map[string]ControlTCPConnectResult) (out []tracex.TCPConnectEntry) { + out = []tracex.TCPConnectEntry{} for _, me := range measurement { epnt := net.JoinHostPort(me.IP, strconv.Itoa(me.Port)) if ce, ok := control[epnt]; ok { diff --git a/internal/engine/experiment/webconnectivity/webconnectivity_test.go b/internal/engine/experiment/webconnectivity/webconnectivity_test.go index 4ce01dd..fc47da8 100644 --- a/internal/engine/experiment/webconnectivity/webconnectivity_test.go +++ b/internal/engine/experiment/webconnectivity/webconnectivity_test.go @@ -11,7 +11,7 @@ import ( "github.com/google/go-cmp/cmp" engine "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -232,33 +232,33 @@ func TestComputeTCPBlocking(t *testing.T) { failure := io.EOF.Error() anotherFailure := "unknown_error" type args struct { - measurement []archival.TCPConnectEntry + measurement []tracex.TCPConnectEntry control map[string]webconnectivity.ControlTCPConnectResult } tests := []struct { name string args args - want []archival.TCPConnectEntry + want []tracex.TCPConnectEntry }{{ name: "with all empty", args: args{}, - want: []archival.TCPConnectEntry{}, + want: []tracex.TCPConnectEntry{}, }, { name: "with control failure", args: args{ - measurement: []archival.TCPConnectEntry{{ + measurement: []tracex.TCPConnectEntry{{ IP: "1.1.1.1", Port: 853, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Failure: &failure, Success: false, }, }}, }, - want: []archival.TCPConnectEntry{{ + want: []tracex.TCPConnectEntry{{ IP: "1.1.1.1", Port: 853, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Failure: &failure, Success: false, }, @@ -266,10 +266,10 @@ func TestComputeTCPBlocking(t *testing.T) { }, { name: "with failures on both ends", args: args{ - measurement: []archival.TCPConnectEntry{{ + measurement: []tracex.TCPConnectEntry{{ IP: "1.1.1.1", Port: 853, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Failure: &failure, Success: false, }, @@ -281,10 +281,10 @@ func TestComputeTCPBlocking(t *testing.T) { }, }, }, - want: []archival.TCPConnectEntry{{ + want: []tracex.TCPConnectEntry{{ IP: "1.1.1.1", Port: 853, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Blocked: &falseValue, Failure: &failure, Success: false, @@ -293,10 +293,10 @@ func TestComputeTCPBlocking(t *testing.T) { }, { name: "with failure on the probe side", args: args{ - measurement: []archival.TCPConnectEntry{{ + measurement: []tracex.TCPConnectEntry{{ IP: "1.1.1.1", Port: 853, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Failure: &failure, Success: false, }, @@ -308,10 +308,10 @@ func TestComputeTCPBlocking(t *testing.T) { }, }, }, - want: []archival.TCPConnectEntry{{ + want: []tracex.TCPConnectEntry{{ IP: "1.1.1.1", Port: 853, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Blocked: &trueValue, Failure: &failure, Success: false, @@ -320,10 +320,10 @@ func TestComputeTCPBlocking(t *testing.T) { }, { name: "with failure on the control side", args: args{ - measurement: []archival.TCPConnectEntry{{ + measurement: []tracex.TCPConnectEntry{{ IP: "1.1.1.1", Port: 853, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Failure: nil, Success: true, }, @@ -335,10 +335,10 @@ func TestComputeTCPBlocking(t *testing.T) { }, }, }, - want: []archival.TCPConnectEntry{{ + want: []tracex.TCPConnectEntry{{ IP: "1.1.1.1", Port: 853, - Status: archival.TCPConnectStatus{ + Status: tracex.TCPConnectStatus{ Blocked: &falseValue, Failure: nil, Success: true, diff --git a/internal/engine/netx/archival/archival_internal_test.go b/internal/engine/netx/archival/archival_internal_test.go deleted file mode 100644 index e927f9f..0000000 --- a/internal/engine/netx/archival/archival_internal_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package archival - -import "testing" - -func TestDNSQueryIPOfType(t *testing.T) { - type expectation struct { - qtype dnsQueryType - ip string - output bool - } - var expectations = []expectation{{ - qtype: "A", - ip: "8.8.8.8", - output: true, - }, { - qtype: "A", - ip: "2a00:1450:4002:801::2004", - output: false, - }, { - qtype: "AAAA", - ip: "8.8.8.8", - output: false, - }, { - qtype: "AAAA", - ip: "2a00:1450:4002:801::2004", - output: true, - }, { - qtype: "ANTANI", - ip: "2a00:1450:4002:801::2004", - output: false, - }, { - qtype: "ANTANI", - ip: "8.8.8.8", - output: false, - }} - for _, exp := range expectations { - if exp.qtype.ipoftype(exp.ip) != exp.output { - t.Fatalf("failure for %+v", exp) - } - } -} diff --git a/internal/engine/netx/dialer/dialer.go b/internal/engine/netx/dialer/dialer.go index f134c56..567d61b 100644 --- a/internal/engine/netx/dialer/dialer.go +++ b/internal/engine/netx/dialer/dialer.go @@ -7,7 +7,7 @@ import ( "net/url" "github.com/ooni/probe-cli/v3/internal/bytecounter" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -35,7 +35,7 @@ type Config struct { // DialSaver is the optional saver for dialing events. If not // set, we will not save any dialing event. - DialSaver *trace.Saver + DialSaver *tracex.Saver // Logger is the optional logger. If not set, there // will be no logging from the new dialer. @@ -45,7 +45,7 @@ type Config struct { ProxyURL *url.URL // ReadWriteSaver is like DialSaver but for I/O events. - ReadWriteSaver *trace.Saver + ReadWriteSaver *tracex.Saver } // New creates a new Dialer from the specified config and resolver. @@ -57,13 +57,13 @@ func New(config *Config, resolver model.Resolver) model.Dialer { modifiers := []netxlite.DialerWrapper{ func(dialer model.Dialer) model.Dialer { if config.DialSaver != nil { - dialer = &saverDialer{Dialer: dialer, Saver: config.DialSaver} + dialer = &tracex.SaverDialer{Dialer: dialer, Saver: config.DialSaver} } return dialer }, func(dialer model.Dialer) model.Dialer { if config.ReadWriteSaver != nil { - dialer = &saverConnDialer{Dialer: dialer, Saver: config.ReadWriteSaver} + dialer = &tracex.SaverConnDialer{Dialer: dialer, Saver: config.ReadWriteSaver} } return dialer }, diff --git a/internal/engine/netx/dialer/dialer_test.go b/internal/engine/netx/dialer/dialer_test.go index 2ecf349..ff2da14 100644 --- a/internal/engine/netx/dialer/dialer_test.go +++ b/internal/engine/netx/dialer/dialer_test.go @@ -7,12 +7,12 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/bytecounter" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/netxlite" ) func TestNewCreatesTheExpectedChain(t *testing.T) { - saver := &trace.Saver{} + saver := &tracex.Saver{} dlr := New(&Config{ ContextByteCounting: true, DialSaver: saver, diff --git a/internal/engine/netx/integration_test.go b/internal/engine/netx/integration_test.go index 050686b..203401e 100644 --- a/internal/engine/netx/integration_test.go +++ b/internal/engine/netx/integration_test.go @@ -9,7 +9,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -24,12 +24,12 @@ func TestSuccess(t *testing.T) { ByteCounter: counter, CacheResolutions: true, ContextByteCounting: true, - DialSaver: &trace.Saver{}, - HTTPSaver: &trace.Saver{}, + DialSaver: &tracex.Saver{}, + HTTPSaver: &tracex.Saver{}, Logger: log.Log, - ReadWriteSaver: &trace.Saver{}, - ResolveSaver: &trace.Saver{}, - TLSSaver: &trace.Saver{}, + ReadWriteSaver: &tracex.Saver{}, + ResolveSaver: &tracex.Saver{}, + TLSSaver: &tracex.Saver{}, } txp := netx.NewHTTPTransport(config) client := &http.Client{Transport: txp} @@ -67,7 +67,7 @@ func TestSuccess(t *testing.T) { } func TestBogonResolutionNotBroken(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) r := netx.NewResolver(netx.Config{ BogonIsError: true, DNSCache: map[string][]string{ diff --git a/internal/engine/netx/netx.go b/internal/engine/netx/netx.go index c76ba33..088cf70 100644 --- a/internal/engine/netx/netx.go +++ b/internal/engine/netx/netx.go @@ -33,10 +33,8 @@ import ( "github.com/ooni/probe-cli/v3/internal/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/netx/dialer" "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" - "github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer" "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" - "github.com/ooni/probe-cli/v3/internal/engine/netx/tlsdialer" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -54,20 +52,20 @@ type Config struct { CertPool *x509.CertPool // default: use vendored gocertifi ContextByteCounting bool // default: no implicit byte counting DNSCache map[string][]string // default: cache is empty - DialSaver *trace.Saver // default: not saving dials + DialSaver *tracex.Saver // default: not saving dials Dialer model.Dialer // default: dialer.DNSDialer FullResolver model.Resolver // default: base resolver + goodies QUICDialer model.QUICDialer // default: quicdialer.DNSDialer HTTP3Enabled bool // default: disabled - HTTPSaver *trace.Saver // default: not saving HTTP + HTTPSaver *tracex.Saver // default: not saving HTTP Logger model.DebugLogger // default: no logging NoTLSVerify bool // default: perform TLS verify ProxyURL *url.URL // default: no proxy - ReadWriteSaver *trace.Saver // default: not saving read/write - ResolveSaver *trace.Saver // default: not saving resolves + ReadWriteSaver *tracex.Saver // default: not saving read/write + ResolveSaver *tracex.Saver // default: not saving resolves TLSConfig *tls.Config // default: attempt using h2 TLSDialer model.TLSDialer // default: dialer.TLSDialer - TLSSaver *trace.Saver // default: not saving TLS + TLSSaver *tracex.Saver // default: not saving TLS } type tlsHandshaker interface { @@ -107,7 +105,7 @@ func NewResolver(config Config) model.Resolver { } } if config.ResolveSaver != nil { - r = resolver.SaverResolver{Resolver: r, Saver: config.ResolveSaver} + r = tracex.SaverResolver{Resolver: r, Saver: config.ResolveSaver} } return &netxlite.ResolverIDNA{Resolver: r} } @@ -133,7 +131,7 @@ func NewQUICDialer(config Config) model.QUICDialer { } ql := netxlite.NewQUICListener() if config.ReadWriteSaver != nil { - ql = &quicdialer.QUICListenerSaver{ + ql = &tracex.QUICListenerSaver{ QUICListener: ql, Saver: config.ReadWriteSaver, } @@ -145,7 +143,7 @@ func NewQUICDialer(config Config) model.QUICDialer { extensions := []netxlite.QUICDialerWrapper{ func(dialer model.QUICDialer) model.QUICDialer { if config.TLSSaver != nil { - dialer = quicdialer.HandshakeSaver{Saver: config.TLSSaver, QUICDialer: dialer} + dialer = tracex.QUICHandshakeSaver{Saver: config.TLSSaver, QUICDialer: dialer} } return dialer }, @@ -164,7 +162,7 @@ func NewTLSDialer(config Config) model.TLSDialer { h = &netxlite.TLSHandshakerLogger{DebugLogger: config.Logger, TLSHandshaker: h} } if config.TLSSaver != nil { - h = tlsdialer.SaverTLSHandshaker{TLSHandshaker: h, Saver: config.TLSSaver} + h = tracex.SaverTLSHandshaker{TLSHandshaker: h, Saver: config.TLSSaver} } if config.TLSConfig == nil { config.TLSConfig = &tls.Config{NextProtos: []string{"h2", "http/1.1"}} @@ -207,11 +205,11 @@ func NewHTTPTransport(config Config) model.HTTPTransport { txp = &netxlite.HTTPTransportLogger{Logger: config.Logger, HTTPTransport: txp} } if config.HTTPSaver != nil { - txp = httptransport.SaverMetadataHTTPTransport{ + txp = tracex.SaverMetadataHTTPTransport{ HTTPTransport: txp, Saver: config.HTTPSaver} - txp = httptransport.SaverBodyHTTPTransport{ + txp = tracex.SaverBodyHTTPTransport{ HTTPTransport: txp, Saver: config.HTTPSaver} - txp = httptransport.SaverTransactionHTTPTransport{ + txp = tracex.SaverTransactionHTTPTransport{ HTTPTransport: txp, Saver: config.HTTPSaver} } return txp @@ -287,7 +285,7 @@ func NewDNSClientWithOverrides(config Config, URL, hostOverride, SNIOverride, var txp model.DNSTransport = netxlite.NewDNSOverHTTPSTransportWithHostOverride( httpClient, URL, hostOverride) if config.ResolveSaver != nil { - txp = resolver.SaverDNSTransport{ + txp = tracex.SaverDNSTransport{ DNSTransport: txp, Saver: config.ResolveSaver, } @@ -302,7 +300,7 @@ func NewDNSClientWithOverrides(config Config, URL, hostOverride, SNIOverride, var txp model.DNSTransport = netxlite.NewDNSOverUDPTransport( dialer, endpoint) if config.ResolveSaver != nil { - txp = resolver.SaverDNSTransport{ + txp = tracex.SaverDNSTransport{ DNSTransport: txp, Saver: config.ResolveSaver, } @@ -318,7 +316,7 @@ func NewDNSClientWithOverrides(config Config, URL, hostOverride, SNIOverride, var txp model.DNSTransport = netxlite.NewDNSOverTLSTransport( tlsDialer.DialTLSContext, endpoint) if config.ResolveSaver != nil { - txp = resolver.SaverDNSTransport{ + txp = tracex.SaverDNSTransport{ DNSTransport: txp, Saver: config.ResolveSaver, } @@ -333,7 +331,7 @@ func NewDNSClientWithOverrides(config Config, URL, hostOverride, SNIOverride, var txp model.DNSTransport = netxlite.NewDNSOverTCPTransport( dialer.DialContext, endpoint) if config.ResolveSaver != nil { - txp = resolver.SaverDNSTransport{ + txp = tracex.SaverDNSTransport{ DNSTransport: txp, Saver: config.ResolveSaver, } diff --git a/internal/engine/netx/netx_test.go b/internal/engine/netx/netx_test.go index 4f2cd78..642a71f 100644 --- a/internal/engine/netx/netx_test.go +++ b/internal/engine/netx/netx_test.go @@ -12,10 +12,8 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/netx" - "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" - "github.com/ooni/probe-cli/v3/internal/engine/netx/tlsdialer" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" "github.com/ooni/probe-cli/v3/internal/model/mocks" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -120,7 +118,7 @@ func TestNewResolverWithLogging(t *testing.T) { } func TestNewResolverWithSaver(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) r := netx.NewResolver(netx.Config{ ResolveSaver: saver, }) @@ -128,7 +126,7 @@ func TestNewResolverWithSaver(t *testing.T) { if !ok { t.Fatal("not the resolver we expected") } - sr, ok := ir.Resolver.(resolver.SaverResolver) + sr, ok := ir.Resolver.(tracex.SaverResolver) if !ok { t.Fatal("not the resolver we expected") } @@ -311,7 +309,7 @@ func TestNewTLSDialerWithLogging(t *testing.T) { } func TestNewTLSDialerWithSaver(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) td := netx.NewTLSDialer(netx.Config{ TLSSaver: saver, }) @@ -334,7 +332,7 @@ func TestNewTLSDialerWithSaver(t *testing.T) { if rtd.TLSHandshaker == nil { t.Fatal("invalid TLSHandshaker") } - sth, ok := rtd.TLSHandshaker.(tlsdialer.SaverTLSHandshaker) + sth, ok := rtd.TLSHandshaker.(tracex.SaverTLSHandshaker) if !ok { t.Fatal("not the TLSHandshaker we expected") } @@ -502,25 +500,25 @@ func TestNewWithLogger(t *testing.T) { } func TestNewWithSaver(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) txp := netx.NewHTTPTransport(netx.Config{ HTTPSaver: saver, }) - stxptxp, ok := txp.(httptransport.SaverTransactionHTTPTransport) + stxptxp, ok := txp.(tracex.SaverTransactionHTTPTransport) if !ok { t.Fatal("not the transport we expected") } if stxptxp.Saver != saver { t.Fatal("not the logger we expected") } - sbtxp, ok := stxptxp.HTTPTransport.(httptransport.SaverBodyHTTPTransport) + sbtxp, ok := stxptxp.HTTPTransport.(tracex.SaverBodyHTTPTransport) if !ok { t.Fatal("not the transport we expected") } if sbtxp.Saver != saver { t.Fatal("not the logger we expected") } - smtxp, ok := sbtxp.HTTPTransport.(httptransport.SaverMetadataHTTPTransport) + smtxp, ok := sbtxp.HTTPTransport.(tracex.SaverMetadataHTTPTransport) if !ok { t.Fatal("not the transport we expected") } @@ -625,7 +623,7 @@ func TestNewDNSClientCloudflareDoH(t *testing.T) { } func TestNewDNSClientCloudflareDoHSaver(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) dnsclient, err := netx.NewDNSClient( netx.Config{ResolveSaver: saver}, "doh://cloudflare") if err != nil { @@ -635,7 +633,7 @@ func TestNewDNSClientCloudflareDoHSaver(t *testing.T) { if !ok { t.Fatal("not the resolver we expected") } - txp, ok := r.Transport().(resolver.SaverDNSTransport) + txp, ok := r.Transport().(tracex.SaverDNSTransport) if !ok { t.Fatal("not the transport we expected") } @@ -662,7 +660,7 @@ func TestNewDNSClientUDP(t *testing.T) { } func TestNewDNSClientUDPDNSSaver(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) dnsclient, err := netx.NewDNSClient( netx.Config{ResolveSaver: saver}, "udp://8.8.8.8:53") if err != nil { @@ -672,7 +670,7 @@ func TestNewDNSClientUDPDNSSaver(t *testing.T) { if !ok { t.Fatal("not the resolver we expected") } - txp, ok := r.Transport().(resolver.SaverDNSTransport) + txp, ok := r.Transport().(tracex.SaverDNSTransport) if !ok { t.Fatal("not the transport we expected") } @@ -703,7 +701,7 @@ func TestNewDNSClientTCP(t *testing.T) { } func TestNewDNSClientTCPDNSSaver(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) dnsclient, err := netx.NewDNSClient( netx.Config{ResolveSaver: saver}, "tcp://8.8.8.8:53") if err != nil { @@ -713,7 +711,7 @@ func TestNewDNSClientTCPDNSSaver(t *testing.T) { if !ok { t.Fatal("not the resolver we expected") } - txp, ok := r.Transport().(resolver.SaverDNSTransport) + txp, ok := r.Transport().(tracex.SaverDNSTransport) if !ok { t.Fatal("not the transport we expected") } @@ -748,7 +746,7 @@ func TestNewDNSClientDoT(t *testing.T) { } func TestNewDNSClientDoTDNSSaver(t *testing.T) { - saver := new(trace.Saver) + saver := new(tracex.Saver) dnsclient, err := netx.NewDNSClient( netx.Config{ResolveSaver: saver}, "dot://8.8.8.8:53") if err != nil { @@ -758,7 +756,7 @@ func TestNewDNSClientDoTDNSSaver(t *testing.T) { if !ok { t.Fatal("not the resolver we expected") } - txp, ok := r.Transport().(resolver.SaverDNSTransport) + txp, ok := r.Transport().(tracex.SaverDNSTransport) if !ok { t.Fatal("not the transport we expected") } diff --git a/internal/engine/netx/quicdialer/saver.go b/internal/engine/netx/quicdialer/saver.go deleted file mode 100644 index 470f8fe..0000000 --- a/internal/engine/netx/quicdialer/saver.go +++ /dev/null @@ -1,68 +0,0 @@ -package quicdialer - -import ( - "context" - "crypto/tls" - "time" - - "github.com/lucas-clemente/quic-go" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// HandshakeSaver saves events occurring during the handshake -type HandshakeSaver struct { - Saver *trace.Saver - model.QUICDialer -} - -// DialContext implements ContextDialer.DialContext -func (h HandshakeSaver) DialContext(ctx context.Context, network string, - host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) { - start := time.Now() - // TODO(bassosimone): in the future we probably want to also save - // information about what versions we're willing to accept. - h.Saver.Write(trace.Event{ - Address: host, - Name: "quic_handshake_start", - NoTLSVerify: tlsCfg.InsecureSkipVerify, - Proto: network, - TLSNextProtos: tlsCfg.NextProtos, - TLSServerName: tlsCfg.ServerName, - Time: start, - }) - sess, err := h.QUICDialer.DialContext(ctx, network, host, tlsCfg, cfg) - stop := time.Now() - if err != nil { - h.Saver.Write(trace.Event{ - Duration: stop.Sub(start), - Err: err, - Name: "quic_handshake_done", - NoTLSVerify: tlsCfg.InsecureSkipVerify, - TLSNextProtos: tlsCfg.NextProtos, - TLSServerName: tlsCfg.ServerName, - Time: stop, - }) - return nil, err - } - state := connectionState(sess) - h.Saver.Write(trace.Event{ - Duration: stop.Sub(start), - Name: "quic_handshake_done", - NoTLSVerify: tlsCfg.InsecureSkipVerify, - TLSCipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), - TLSNegotiatedProto: state.NegotiatedProtocol, - TLSNextProtos: tlsCfg.NextProtos, - TLSPeerCerts: trace.PeerCerts(state, err), - TLSServerName: tlsCfg.ServerName, - TLSVersion: netxlite.TLSVersionString(state.Version), - Time: stop, - }) - return sess, nil -} - -// connectionState returns the ConnectionState of a QUIC Session. -func connectionState(sess quic.EarlyConnection) tls.ConnectionState { - return sess.ConnectionState().TLS.ConnectionState -} diff --git a/internal/engine/netx/quicdialer/system.go b/internal/engine/netx/quicdialer/system.go deleted file mode 100644 index a805bd7..0000000 --- a/internal/engine/netx/quicdialer/system.go +++ /dev/null @@ -1,81 +0,0 @@ -package quicdialer - -import ( - "net" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// QUICListenerSaver is a QUICListener that also implements saving events. -type QUICListenerSaver struct { - // QUICListener is the underlying QUICListener. - model.QUICListener - - // Saver is the underlying Saver. - Saver *trace.Saver -} - -// Listen implements QUICListener.Listen. -func (qls *QUICListenerSaver) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { - pconn, err := qls.QUICListener.Listen(addr) - if err != nil { - return nil, err - } - return &saverUDPConn{ - UDPLikeConn: pconn, - saver: qls.Saver, - }, nil -} - -type saverUDPConn struct { - model.UDPLikeConn - saver *trace.Saver -} - -var _ model.UDPLikeConn = &saverUDPConn{} - -func (c *saverUDPConn) WriteTo(p []byte, addr net.Addr) (int, error) { - start := time.Now() - count, err := c.UDPLikeConn.WriteTo(p, addr) - stop := time.Now() - c.saver.Write(trace.Event{ - Address: addr.String(), - Data: p[:count], - Duration: stop.Sub(start), - Err: err, - NumBytes: count, - Name: netxlite.WriteToOperation, - Time: stop, - }) - return count, err -} - -func (c *saverUDPConn) ReadFrom(b []byte) (int, net.Addr, error) { - start := time.Now() - n, addr, err := c.UDPLikeConn.ReadFrom(b) - stop := time.Now() - var data []byte - if n > 0 { - data = b[:n] - } - c.saver.Write(trace.Event{ - Address: c.safeAddrString(addr), - Data: data, - Duration: stop.Sub(start), - Err: err, - NumBytes: n, - Name: netxlite.ReadFromOperation, - Time: stop, - }) - return n, addr, err -} - -func (c *saverUDPConn) safeAddrString(addr net.Addr) (out string) { - if addr != nil { - out = addr.String() - } - return -} diff --git a/internal/engine/netx/quicdialer/system_test.go b/internal/engine/netx/quicdialer/system_test.go deleted file mode 100644 index 28a429b..0000000 --- a/internal/engine/netx/quicdialer/system_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package quicdialer_test - -import ( - "context" - "crypto/tls" - "errors" - "net" - "testing" - - "github.com/lucas-clemente/quic-go" - "github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/model/mocks" - "github.com/ooni/probe-cli/v3/internal/netxlite" - "github.com/ooni/probe-cli/v3/internal/netxlite/quictesting" -) - -func TestQUICListenerSaverCannotListen(t *testing.T) { - expected := errors.New("mocked error") - qls := &quicdialer.QUICListenerSaver{ - QUICListener: &mocks.QUICListener{ - MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { - return nil, expected - }, - }, - Saver: &trace.Saver{}, - } - pconn, err := qls.Listen(&net.UDPAddr{ - IP: []byte{}, - Port: 8080, - Zone: "", - }) - if !errors.Is(err, expected) { - t.Fatal("unexpected error", err) - } - if pconn != nil { - t.Fatal("expected nil pconn here") - } -} - -func TestSystemDialerSuccessWithReadWrite(t *testing.T) { - // This is the most common use case for collecting reads, writes - tlsConf := &tls.Config{ - NextProtos: []string{"h3"}, - ServerName: quictesting.Domain, - } - saver := &trace.Saver{} - systemdialer := &netxlite.QUICDialerQUICGo{ - QUICListener: &quicdialer.QUICListenerSaver{ - QUICListener: &netxlite.QUICListenerStdlib{}, - Saver: saver, - }, - } - _, err := systemdialer.DialContext(context.Background(), "udp", - quictesting.Endpoint("443"), tlsConf, &quic.Config{}) - if err != nil { - t.Fatal(err) - } - ev := saver.Read() - if len(ev) < 2 { - t.Fatal("unexpected number of events") - } - last := len(ev) - 1 - for idx := 1; idx < last; idx++ { - if ev[idx].Data == nil { - t.Fatal("unexpected Data") - } - if ev[idx].Duration <= 0 { - t.Fatal("unexpected Duration") - } - if ev[idx].Err != nil { - t.Fatal("unexpected Err") - } - if ev[idx].NumBytes <= 0 { - t.Fatal("unexpected NumBytes") - } - switch ev[idx].Name { - case netxlite.ReadFromOperation, netxlite.WriteToOperation: - default: - t.Fatal("unexpected Name") - } - if ev[idx].Time.Before(ev[idx-1].Time) { - t.Fatal("unexpected Time", ev[idx].Time, ev[idx-1].Time) - } - } -} diff --git a/internal/engine/netx/archival/archival.go b/internal/engine/netx/tracex/archival.go similarity index 92% rename from internal/engine/netx/archival/archival.go rename to internal/engine/netx/tracex/archival.go index f99af94..4a75836 100644 --- a/internal/engine/netx/archival/archival.go +++ b/internal/engine/netx/tracex/archival.go @@ -1,7 +1,4 @@ -// Package archival contains data formats used for archival. -// -// See https://github.com/ooni/spec. -package archival +package tracex import ( "crypto/x509" @@ -14,7 +11,6 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/engine/geolocate" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -47,7 +43,7 @@ var ( ) // NewTCPConnectList creates a new TCPConnectList -func NewTCPConnectList(begin time.Time, events []trace.Event) []TCPConnectEntry { +func NewTCPConnectList(begin time.Time, events []Event) []TCPConnectEntry { var out []TCPConnectEntry for _, event := range events { if event.Name != netxlite.ConnectOperation { @@ -129,7 +125,7 @@ func addheaders( } // NewRequestList returns the list for "requests" -func NewRequestList(begin time.Time, events []trace.Event) []RequestEntry { +func NewRequestList(begin time.Time, events []Event) []RequestEntry { // OONI wants the last request to appear first var out []RequestEntry tmp := newRequestList(begin, events) @@ -139,7 +135,7 @@ func NewRequestList(begin time.Time, events []trace.Event) []RequestEntry { return out } -func newRequestList(begin time.Time, events []trace.Event) []RequestEntry { +func newRequestList(begin time.Time, events []Event) []RequestEntry { var ( out []RequestEntry entry RequestEntry @@ -179,7 +175,7 @@ func newRequestList(begin time.Time, events []trace.Event) []RequestEntry { type dnsQueryType string // NewDNSQueriesList returns a list of DNS queries. -func NewDNSQueriesList(begin time.Time, events []trace.Event) []DNSQueryEntry { +func NewDNSQueriesList(begin time.Time, events []Event) []DNSQueryEntry { // TODO(bassosimone): add support for CNAME lookups. var out []DNSQueryEntry for _, ev := range events { @@ -234,7 +230,7 @@ func (qtype dnsQueryType) makeanswerentry(addr string) DNSAnswerEntry { return answer } -func (qtype dnsQueryType) makequeryentry(begin time.Time, ev trace.Event) DNSQueryEntry { +func (qtype dnsQueryType) makequeryentry(begin time.Time, ev Event) DNSQueryEntry { return DNSQueryEntry{ Engine: ev.Proto, Failure: NewFailure(ev.Err), @@ -246,7 +242,7 @@ func (qtype dnsQueryType) makequeryentry(begin time.Time, ev trace.Event) DNSQue } // NewNetworkEventsList returns a list of DNS queries. -func NewNetworkEventsList(begin time.Time, events []trace.Event) []NetworkEvent { +func NewNetworkEventsList(begin time.Time, events []Event) []NetworkEvent { var out []NetworkEvent for _, ev := range events { if ev.Name == netxlite.ConnectOperation { @@ -307,7 +303,7 @@ func NewNetworkEventsList(begin time.Time, events []trace.Event) []NetworkEvent } // NewTLSHandshakesList creates a new TLSHandshakesList -func NewTLSHandshakesList(begin time.Time, events []trace.Event) []TLSHandshake { +func NewTLSHandshakesList(begin time.Time, events []Event) []TLSHandshake { var out []TLSHandshake for _, ev := range events { if !strings.Contains(ev.Name, "_handshake_done") { diff --git a/internal/engine/netx/archival/archival_test.go b/internal/engine/netx/tracex/archival_test.go similarity index 81% rename from internal/engine/netx/archival/archival_test.go rename to internal/engine/netx/tracex/archival_test.go index 3196aa7..dad3c58 100644 --- a/internal/engine/netx/archival/archival_test.go +++ b/internal/engine/netx/tracex/archival_test.go @@ -1,4 +1,4 @@ -package archival_test +package tracex import ( "context" @@ -12,21 +12,57 @@ import ( "github.com/google/go-cmp/cmp" "github.com/gorilla/websocket" - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/netxlite" ) +func TestDNSQueryIPOfType(t *testing.T) { + type expectation struct { + qtype dnsQueryType + ip string + output bool + } + var expectations = []expectation{{ + qtype: "A", + ip: "8.8.8.8", + output: true, + }, { + qtype: "A", + ip: "2a00:1450:4002:801::2004", + output: false, + }, { + qtype: "AAAA", + ip: "8.8.8.8", + output: false, + }, { + qtype: "AAAA", + ip: "2a00:1450:4002:801::2004", + output: true, + }, { + qtype: "ANTANI", + ip: "2a00:1450:4002:801::2004", + output: false, + }, { + qtype: "ANTANI", + ip: "8.8.8.8", + output: false, + }} + for _, exp := range expectations { + if exp.qtype.ipoftype(exp.ip) != exp.output { + t.Fatalf("failure for %+v", exp) + } + } +} + func TestNewTCPConnectList(t *testing.T) { begin := time.Now() type args struct { begin time.Time - events []trace.Event + events []Event } tests := []struct { name string args args - want []archival.TCPConnectEntry + want []TCPConnectEntry }{{ name: "empty run", args: args{ @@ -38,7 +74,7 @@ func TestNewTCPConnectList(t *testing.T) { name: "realistic run", args: args{ begin: begin, - events: []trace.Event{{ + events: []Event{{ Addresses: []string{"8.8.8.8", "8.8.4.4"}, Hostname: "dns.google.com", Name: "resolve_done", @@ -64,18 +100,18 @@ func TestNewTCPConnectList(t *testing.T) { Time: begin.Add(180 * time.Millisecond), }}, }, - want: []archival.TCPConnectEntry{{ + want: []TCPConnectEntry{{ IP: "8.8.8.8", Port: 853, - Status: archival.TCPConnectStatus{ + Status: TCPConnectStatus{ Success: true, }, T: 0.13, }, { IP: "8.8.4.4", Port: 53, - Status: archival.TCPConnectStatus{ - Failure: archival.NewFailure(io.EOF), + Status: TCPConnectStatus{ + Failure: NewFailure(io.EOF), Success: false, }, T: 0.18, @@ -83,7 +119,7 @@ func TestNewTCPConnectList(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := archival.NewTCPConnectList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) { + if got := NewTCPConnectList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) { t.Error(cmp.Diff(got, tt.want)) } }) @@ -94,12 +130,12 @@ func TestNewRequestList(t *testing.T) { begin := time.Now() type args struct { begin time.Time - events []trace.Event + events []Event } tests := []struct { name string args args - want []archival.RequestEntry + want []RequestEntry }{{ name: "empty run", args: args{ @@ -111,7 +147,7 @@ func TestNewRequestList(t *testing.T) { name: "realistic run", args: args{ begin: begin, - events: []trace.Event{{ + events: []Event{{ Name: "http_transaction_start", Time: begin.Add(10 * time.Millisecond), }, { @@ -152,16 +188,16 @@ func TestNewRequestList(t *testing.T) { Err: io.EOF, }}, }, - want: []archival.RequestEntry{{ - Failure: archival.NewFailure(io.EOF), - Request: archival.HTTPRequest{ - HeadersList: []archival.HTTPHeader{{ + want: []RequestEntry{{ + Failure: NewFailure(io.EOF), + Request: HTTPRequest{ + HeadersList: []HTTPHeader{{ Key: "User-Agent", - Value: archival.MaybeBinaryValue{ + Value: MaybeBinaryValue{ Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]archival.MaybeBinaryValue{ + Headers: map[string]MaybeBinaryValue{ "User-Agent": {Value: "miniooni/0.1.0-dev"}, }, Method: "GET", @@ -169,34 +205,34 @@ func TestNewRequestList(t *testing.T) { }, T: 0.02, }, { - Request: archival.HTTPRequest{ - Body: archival.MaybeBinaryValue{ + Request: HTTPRequest{ + Body: MaybeBinaryValue{ Value: "deadbeef", }, - HeadersList: []archival.HTTPHeader{{ + HeadersList: []HTTPHeader{{ Key: "User-Agent", - Value: archival.MaybeBinaryValue{ + Value: MaybeBinaryValue{ Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]archival.MaybeBinaryValue{ + Headers: map[string]MaybeBinaryValue{ "User-Agent": {Value: "miniooni/0.1.0-dev"}, }, Method: "POST", URL: "https://www.example.com/submit", }, - Response: archival.HTTPResponse{ - Body: archival.MaybeBinaryValue{ + Response: HTTPResponse{ + Body: MaybeBinaryValue{ Value: "{}", }, Code: 200, - HeadersList: []archival.HTTPHeader{{ + HeadersList: []HTTPHeader{{ Key: "Server", - Value: archival.MaybeBinaryValue{ + Value: MaybeBinaryValue{ Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]archival.MaybeBinaryValue{ + Headers: map[string]MaybeBinaryValue{ "Server": {Value: "miniooni/0.1.0-dev"}, }, Locations: nil, @@ -209,7 +245,7 @@ func TestNewRequestList(t *testing.T) { name: "run with redirect and headers to sort", args: args{ begin: begin, - events: []trace.Event{{ + events: []Event{{ Name: "http_transaction_start", Time: begin.Add(10 * time.Millisecond), }, { @@ -230,39 +266,39 @@ func TestNewRequestList(t *testing.T) { Name: "http_transaction_done", }}, }, - want: []archival.RequestEntry{{ - Request: archival.HTTPRequest{ - HeadersList: []archival.HTTPHeader{{ + want: []RequestEntry{{ + Request: HTTPRequest{ + HeadersList: []HTTPHeader{{ Key: "User-Agent", - Value: archival.MaybeBinaryValue{ + Value: MaybeBinaryValue{ Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]archival.MaybeBinaryValue{ + Headers: map[string]MaybeBinaryValue{ "User-Agent": {Value: "miniooni/0.1.0-dev"}, }, Method: "GET", URL: "https://www.example.com/", }, - Response: archival.HTTPResponse{ + Response: HTTPResponse{ Code: 302, - HeadersList: []archival.HTTPHeader{{ + HeadersList: []HTTPHeader{{ Key: "Location", - Value: archival.MaybeBinaryValue{ + Value: MaybeBinaryValue{ Value: "https://x.example.com", }, }, { Key: "Location", - Value: archival.MaybeBinaryValue{ + Value: MaybeBinaryValue{ Value: "https://y.example.com", }, }, { Key: "Server", - Value: archival.MaybeBinaryValue{ + Value: MaybeBinaryValue{ Value: "miniooni/0.1.0-dev", }, }}, - Headers: map[string]archival.MaybeBinaryValue{ + Headers: map[string]MaybeBinaryValue{ "Server": {Value: "miniooni/0.1.0-dev"}, "Location": {Value: "https://x.example.com"}, }, @@ -275,7 +311,7 @@ func TestNewRequestList(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := archival.NewRequestList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) { + if got := NewRequestList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) { t.Error(cmp.Diff(got, tt.want)) } }) @@ -286,12 +322,12 @@ func TestNewDNSQueriesList(t *testing.T) { begin := time.Now() type args struct { begin time.Time - events []trace.Event + events []Event } tests := []struct { name string args args - want []archival.DNSQueryEntry + want []DNSQueryEntry }{{ name: "empty run", args: args{ @@ -303,7 +339,7 @@ func TestNewDNSQueriesList(t *testing.T) { name: "realistic run", args: args{ begin: begin, - events: []trace.Event{{ + events: []Event{{ Address: "1.1.1.1:853", Addresses: []string{"8.8.8.8", "8.8.4.4"}, Hostname: "dns.google.com", @@ -325,8 +361,8 @@ func TestNewDNSQueriesList(t *testing.T) { Time: begin.Add(180 * time.Millisecond), }}, }, - want: []archival.DNSQueryEntry{{ - Answers: []archival.DNSAnswerEntry{{ + want: []DNSQueryEntry{{ + Answers: []DNSAnswerEntry{{ ASN: 15169, ASOrgName: "Google LLC", AnswerType: "A", @@ -347,15 +383,15 @@ func TestNewDNSQueriesList(t *testing.T) { name: "run with IPv6 results", args: args{ begin: begin, - events: []trace.Event{{ + events: []Event{{ Addresses: []string{"2001:4860:4860::8888"}, Hostname: "dns.google.com", Name: "resolve_done", Time: begin.Add(200 * time.Millisecond), }}, }, - want: []archival.DNSQueryEntry{{ - Answers: []archival.DNSAnswerEntry{{ + want: []DNSQueryEntry{{ + Answers: []DNSAnswerEntry{{ ASN: 15169, ASOrgName: "Google LLC", AnswerType: "AAAA", @@ -369,23 +405,23 @@ func TestNewDNSQueriesList(t *testing.T) { name: "run with errors", args: args{ begin: begin, - events: []trace.Event{{ + events: []Event{{ Err: &netxlite.ErrWrapper{Failure: netxlite.FailureDNSNXDOMAINError}, Hostname: "dns.google.com", Name: "resolve_done", Time: begin.Add(200 * time.Millisecond), }}, }, - want: []archival.DNSQueryEntry{{ + want: []DNSQueryEntry{{ Answers: nil, - Failure: archival.NewFailure( + Failure: NewFailure( &netxlite.ErrWrapper{Failure: netxlite.FailureDNSNXDOMAINError}), Hostname: "dns.google.com", QueryType: "A", T: 0.2, }, { Answers: nil, - Failure: archival.NewFailure( + Failure: NewFailure( &netxlite.ErrWrapper{Failure: netxlite.FailureDNSNXDOMAINError}), Hostname: "dns.google.com", QueryType: "AAAA", @@ -394,7 +430,7 @@ func TestNewDNSQueriesList(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := archival.NewDNSQueriesList(tt.args.begin, tt.args.events) + got := NewDNSQueriesList(tt.args.begin, tt.args.events) if diff := cmp.Diff(tt.want, got); diff != "" { t.Fatal(diff) } @@ -406,12 +442,12 @@ func TestNewNetworkEventsList(t *testing.T) { begin := time.Now() type args struct { begin time.Time - events []trace.Event + events []Event } tests := []struct { name string args args - want []archival.NetworkEvent + want []NetworkEvent }{{ name: "empty run", args: args{ @@ -423,7 +459,7 @@ func TestNewNetworkEventsList(t *testing.T) { name: "realistic run", args: args{ begin: begin, - events: []trace.Event{{ + events: []Event{{ Name: netxlite.ConnectOperation, Address: "8.8.8.8:853", Err: io.EOF, @@ -457,43 +493,43 @@ func TestNewNetworkEventsList(t *testing.T) { Time: begin.Add(17 * time.Millisecond), }}, }, - want: []archival.NetworkEvent{{ + want: []NetworkEvent{{ Address: "8.8.8.8:853", - Failure: archival.NewFailure(io.EOF), + Failure: NewFailure(io.EOF), Operation: netxlite.ConnectOperation, Proto: "tcp", T: 0.007, }, { - Failure: archival.NewFailure(context.Canceled), + Failure: NewFailure(context.Canceled), NumBytes: 7117, Operation: netxlite.ReadOperation, T: 0.011, }, { Address: "8.8.8.8:853", - Failure: archival.NewFailure(context.Canceled), + Failure: NewFailure(context.Canceled), NumBytes: 7117, Operation: netxlite.ReadFromOperation, T: 0.011, }, { - Failure: archival.NewFailure(websocket.ErrBadHandshake), + Failure: NewFailure(websocket.ErrBadHandshake), NumBytes: 4114, Operation: netxlite.WriteOperation, T: 0.014, }, { Address: "8.8.8.8:853", - Failure: archival.NewFailure(websocket.ErrBadHandshake), + Failure: NewFailure(websocket.ErrBadHandshake), NumBytes: 4114, Operation: netxlite.WriteToOperation, T: 0.014, }, { - Failure: archival.NewFailure(websocket.ErrReadLimit), + Failure: NewFailure(websocket.ErrReadLimit), Operation: netxlite.CloseOperation, T: 0.017, }}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := archival.NewNetworkEventsList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) { + if got := NewNetworkEventsList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) { t.Error(cmp.Diff(got, tt.want)) } }) @@ -504,12 +540,12 @@ func TestNewTLSHandshakesList(t *testing.T) { begin := time.Now() type args struct { begin time.Time - events []trace.Event + events []Event } tests := []struct { name string args args - want []archival.TLSHandshake + want []TLSHandshake }{{ name: "empty run", args: args{ @@ -521,7 +557,7 @@ func TestNewTLSHandshakesList(t *testing.T) { name: "realistic run", args: args{ begin: begin, - events: []trace.Event{{ + events: []Event{{ Name: netxlite.CloseOperation, Err: websocket.ErrReadLimit, Time: begin.Add(17 * time.Millisecond), @@ -542,13 +578,13 @@ func TestNewTLSHandshakesList(t *testing.T) { Time: begin.Add(55 * time.Millisecond), }}, }, - want: []archival.TLSHandshake{{ + want: []TLSHandshake{{ Address: "131.252.210.176:443", CipherSuite: "SUITE", - Failure: archival.NewFailure(io.EOF), + Failure: NewFailure(io.EOF), NegotiatedProtocol: "h2", NoTLSVerify: false, - PeerCertificates: []archival.MaybeBinaryValue{{ + PeerCertificates: []MaybeBinaryValue{{ Value: "deadbeef", }, { Value: "abad1dea", @@ -560,7 +596,7 @@ func TestNewTLSHandshakesList(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := archival.NewTLSHandshakesList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) { + if got := NewTLSHandshakesList(tt.args.begin, tt.args.events); !reflect.DeepEqual(got, tt.want) { t.Error(cmp.Diff(got, tt.want)) } }) @@ -620,7 +656,7 @@ func TestNewFailure(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := archival.NewFailure(tt.args.err) + got := NewFailure(tt.args.err) if tt.want == nil && got == nil { return } @@ -689,7 +725,7 @@ func TestNewFailedOperation(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := archival.NewFailedOperation(tt.args.err) + got := NewFailedOperation(tt.args.err) if got == nil && tt.want == nil { return } diff --git a/internal/engine/netx/dialer/saver.go b/internal/engine/netx/tracex/dialer.go similarity index 70% rename from internal/engine/netx/dialer/saver.go rename to internal/engine/netx/tracex/dialer.go index 42f2e5c..ec6d025 100644 --- a/internal/engine/netx/dialer/saver.go +++ b/internal/engine/netx/tracex/dialer.go @@ -1,27 +1,26 @@ -package dialer +package tracex import ( "context" "net" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) -// saverDialer saves events occurring during the dial -type saverDialer struct { +// SaverDialer saves events occurring during the dial +type SaverDialer struct { model.Dialer - Saver *trace.Saver + Saver *Saver } // DialContext implements Dialer.DialContext -func (d *saverDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { +func (d *SaverDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { start := time.Now() conn, err := d.Dialer.DialContext(ctx, network, address) stop := time.Now() - d.Saver.Write(trace.Event{ + d.Saver.Write(Event{ Address: address, Duration: stop.Sub(start), Err: err, @@ -32,15 +31,15 @@ func (d *saverDialer) DialContext(ctx context.Context, network, address string) return conn, err } -// saverConnDialer wraps the returned connection such that we +// SaverConnDialer wraps the returned connection such that we // collect all the read/write events that occur. -type saverConnDialer struct { +type SaverConnDialer struct { model.Dialer - Saver *trace.Saver + Saver *Saver } // DialContext implements Dialer.DialContext -func (d *saverConnDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { +func (d *SaverConnDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { conn, err := d.Dialer.DialContext(ctx, network, address) if err != nil { return nil, err @@ -50,14 +49,14 @@ func (d *saverConnDialer) DialContext(ctx context.Context, network, address stri type saverConn struct { net.Conn - saver *trace.Saver + saver *Saver } func (c *saverConn) Read(p []byte) (int, error) { start := time.Now() count, err := c.Conn.Read(p) stop := time.Now() - c.saver.Write(trace.Event{ + c.saver.Write(Event{ Data: p[:count], Duration: stop.Sub(start), Err: err, @@ -72,7 +71,7 @@ func (c *saverConn) Write(p []byte) (int, error) { start := time.Now() count, err := c.Conn.Write(p) stop := time.Now() - c.saver.Write(trace.Event{ + c.saver.Write(Event{ Data: p[:count], Duration: stop.Sub(start), Err: err, @@ -83,6 +82,6 @@ func (c *saverConn) Write(p []byte) (int, error) { return count, err } -var _ model.Dialer = &saverDialer{} -var _ model.Dialer = &saverConnDialer{} +var _ model.Dialer = &SaverDialer{} +var _ model.Dialer = &SaverConnDialer{} var _ net.Conn = &saverConn{} diff --git a/internal/engine/netx/dialer/saver_test.go b/internal/engine/netx/tracex/dialer_test.go similarity index 87% rename from internal/engine/netx/dialer/saver_test.go rename to internal/engine/netx/tracex/dialer_test.go index 0439d81..18299a6 100644 --- a/internal/engine/netx/dialer/saver_test.go +++ b/internal/engine/netx/tracex/dialer_test.go @@ -1,4 +1,4 @@ -package dialer +package tracex import ( "context" @@ -8,15 +8,14 @@ import ( "testing" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model/mocks" "github.com/ooni/probe-cli/v3/internal/netxlite" ) func TestSaverDialerFailure(t *testing.T) { expected := errors.New("mocked error") - saver := &trace.Saver{} - dlr := &saverDialer{ + saver := &Saver{} + dlr := &SaverDialer{ Dialer: &mocks.Dialer{ MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) { return nil, expected @@ -57,8 +56,8 @@ func TestSaverDialerFailure(t *testing.T) { func TestSaverConnDialerFailure(t *testing.T) { expected := errors.New("mocked error") - saver := &trace.Saver{} - dlr := &saverConnDialer{ + saver := &Saver{} + dlr := &SaverConnDialer{ Dialer: &mocks.Dialer{ MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) { return nil, expected @@ -76,9 +75,9 @@ func TestSaverConnDialerFailure(t *testing.T) { } func TestSaverConnDialerSuccess(t *testing.T) { - saver := &trace.Saver{} - dlr := &saverConnDialer{ - Dialer: &saverDialer{ + saver := &Saver{} + dlr := &SaverConnDialer{ + Dialer: &SaverDialer{ Dialer: &mocks.Dialer{ MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) { return &mocks.Conn{ @@ -126,14 +125,14 @@ func TestSaverConnDialerSuccess(t *testing.T) { saverCheckWriteEvent(t, &events[2]) } -func saverCheckConnectEvent(t *testing.T, ev *trace.Event) { +func saverCheckConnectEvent(t *testing.T, ev *Event) { // TODO(bassosimone): implement } -func saverCheckReadEvent(t *testing.T, ev *trace.Event) { +func saverCheckReadEvent(t *testing.T, ev *Event) { // TODO(bassosimone): implement } -func saverCheckWriteEvent(t *testing.T, ev *trace.Event) { +func saverCheckWriteEvent(t *testing.T, ev *Event) { // TODO(bassosimone): implement } diff --git a/internal/engine/netx/tracex/doc.go b/internal/engine/netx/tracex/doc.go new file mode 100644 index 0000000..399e1a1 --- /dev/null +++ b/internal/engine/netx/tracex/doc.go @@ -0,0 +1,2 @@ +// Package tracex contains code to perform measurements using tracing. +package tracex diff --git a/internal/engine/netx/trace/event.go b/internal/engine/netx/tracex/event.go similarity index 99% rename from internal/engine/netx/trace/event.go rename to internal/engine/netx/tracex/event.go index f98a84c..cfdc4b3 100644 --- a/internal/engine/netx/trace/event.go +++ b/internal/engine/netx/tracex/event.go @@ -1,4 +1,4 @@ -package trace +package tracex import ( "crypto/tls" diff --git a/internal/engine/netx/httptransport/saver.go b/internal/engine/netx/tracex/http.go similarity index 91% rename from internal/engine/netx/httptransport/saver.go rename to internal/engine/netx/tracex/http.go index 772aac1..c7db07c 100644 --- a/internal/engine/netx/httptransport/saver.go +++ b/internal/engine/netx/tracex/http.go @@ -1,4 +1,4 @@ -package httptransport +package tracex import ( "bytes" @@ -7,7 +7,6 @@ import ( "net/http" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -16,12 +15,12 @@ import ( // events related to HTTP request and response metadata type SaverMetadataHTTPTransport struct { model.HTTPTransport - Saver *trace.Saver + Saver *Saver } // RoundTrip implements RoundTripper.RoundTrip func (txp SaverMetadataHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) { - txp.Saver.Write(trace.Event{ + txp.Saver.Write(Event{ HTTPHeaders: txp.CloneHeaders(req), HTTPMethod: req.Method, HTTPURL: req.URL.String(), @@ -33,7 +32,7 @@ func (txp SaverMetadataHTTPTransport) RoundTrip(req *http.Request) (*http.Respon if err != nil { return nil, err } - txp.Saver.Write(trace.Event{ + txp.Saver.Write(Event{ HTTPHeaders: resp.Header, HTTPStatusCode: resp.StatusCode, Name: "http_response_metadata", @@ -59,17 +58,17 @@ func (txp SaverMetadataHTTPTransport) CloneHeaders(req *http.Request) http.Heade // events related to the HTTP transaction type SaverTransactionHTTPTransport struct { model.HTTPTransport - Saver *trace.Saver + Saver *Saver } // RoundTrip implements RoundTripper.RoundTrip func (txp SaverTransactionHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) { - txp.Saver.Write(trace.Event{ + txp.Saver.Write(Event{ Name: "http_transaction_start", Time: time.Now(), }) resp, err := txp.HTTPTransport.RoundTrip(req) - txp.Saver.Write(trace.Event{ + txp.Saver.Write(Event{ Err: err, Name: "http_transaction_done", Time: time.Now(), @@ -81,7 +80,7 @@ func (txp SaverTransactionHTTPTransport) RoundTrip(req *http.Request) (*http.Res // body events occurring during the round trip type SaverBodyHTTPTransport struct { model.HTTPTransport - Saver *trace.Saver + Saver *Saver SnapshotSize int } @@ -98,7 +97,7 @@ func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response, return nil, err } req.Body = saverCompose(data, req.Body) - txp.Saver.Write(trace.Event{ + txp.Saver.Write(Event{ DataIsTruncated: len(data) >= snapsize, Data: data, Name: "http_request_body_snapshot", @@ -115,7 +114,7 @@ func (txp SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response, return nil, err } resp.Body = saverCompose(data, resp.Body) - txp.Saver.Write(trace.Event{ + txp.Saver.Write(Event{ DataIsTruncated: len(data) >= snapsize, Data: data, Name: "http_response_body_snapshot", diff --git a/internal/engine/netx/httptransport/saver_test.go b/internal/engine/netx/tracex/http_test.go similarity index 92% rename from internal/engine/netx/httptransport/saver_test.go rename to internal/engine/netx/tracex/http_test.go index 23868c4..4b854e5 100644 --- a/internal/engine/netx/httptransport/saver_test.go +++ b/internal/engine/netx/tracex/http_test.go @@ -1,4 +1,4 @@ -package httptransport_test +package tracex import ( "context" @@ -11,8 +11,6 @@ import ( "testing" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/httptransport" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -21,8 +19,8 @@ func TestSaverMetadataSuccess(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - saver := &trace.Saver{} - txp := httptransport.SaverMetadataHTTPTransport{ + saver := &Saver{} + txp := SaverMetadataHTTPTransport{ HTTPTransport: netxlite.NewHTTPTransportStdlib(model.DiscardLogger), Saver: saver, } @@ -75,8 +73,8 @@ func TestSaverMetadataSuccess(t *testing.T) { func TestSaverMetadataFailure(t *testing.T) { expected := errors.New("mocked error") - saver := &trace.Saver{} - txp := httptransport.SaverMetadataHTTPTransport{ + saver := &Saver{} + txp := SaverMetadataHTTPTransport{ HTTPTransport: FakeTransport{ Err: expected, }, @@ -119,8 +117,8 @@ func TestSaverTransactionSuccess(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - saver := &trace.Saver{} - txp := httptransport.SaverTransactionHTTPTransport{ + saver := &Saver{} + txp := SaverTransactionHTTPTransport{ HTTPTransport: netxlite.NewHTTPTransportStdlib(model.DiscardLogger), Saver: saver, } @@ -160,8 +158,8 @@ func TestSaverTransactionSuccess(t *testing.T) { func TestSaverTransactionFailure(t *testing.T) { expected := errors.New("mocked error") - saver := &trace.Saver{} - txp := httptransport.SaverTransactionHTTPTransport{ + saver := &Saver{} + txp := SaverTransactionHTTPTransport{ HTTPTransport: FakeTransport{ Err: expected, }, @@ -200,8 +198,8 @@ func TestSaverTransactionFailure(t *testing.T) { } func TestSaverBodySuccess(t *testing.T) { - saver := new(trace.Saver) - txp := httptransport.SaverBodyHTTPTransport{ + saver := new(Saver) + txp := SaverBodyHTTPTransport{ HTTPTransport: FakeTransport{ Func: func(req *http.Request) (*http.Response, error) { data, err := netxlite.ReadAllContext(context.Background(), req.Body) @@ -271,8 +269,8 @@ func TestSaverBodySuccess(t *testing.T) { } func TestSaverBodyRequestReadError(t *testing.T) { - saver := new(trace.Saver) - txp := httptransport.SaverBodyHTTPTransport{ + saver := new(Saver) + txp := SaverBodyHTTPTransport{ HTTPTransport: FakeTransport{ Func: func(req *http.Request) (*http.Response, error) { panic("should not be called") @@ -301,9 +299,9 @@ func TestSaverBodyRequestReadError(t *testing.T) { } func TestSaverBodyRoundTripError(t *testing.T) { - saver := new(trace.Saver) + saver := new(Saver) expected := errors.New("mocked error") - txp := httptransport.SaverBodyHTTPTransport{ + txp := SaverBodyHTTPTransport{ HTTPTransport: FakeTransport{ Err: expected, }, @@ -341,9 +339,9 @@ func TestSaverBodyRoundTripError(t *testing.T) { } func TestSaverBodyResponseReadError(t *testing.T) { - saver := new(trace.Saver) + saver := new(Saver) expected := errors.New("mocked error") - txp := httptransport.SaverBodyHTTPTransport{ + txp := SaverBodyHTTPTransport{ HTTPTransport: FakeTransport{ Func: func(req *http.Request) (*http.Response, error) { return &http.Response{ @@ -396,7 +394,7 @@ func TestCloneHeaders(t *testing.T) { }, Header: http.Header{}, } - txp := httptransport.SaverMetadataHTTPTransport{} + txp := SaverMetadataHTTPTransport{} header := txp.CloneHeaders(req) if header.Get("Host") != "www.example.com" { t.Fatal("did not set Host header correctly") @@ -411,7 +409,7 @@ func TestCloneHeaders(t *testing.T) { }, Header: http.Header{}, } - txp := httptransport.SaverMetadataHTTPTransport{} + txp := SaverMetadataHTTPTransport{} header := txp.CloneHeaders(req) if header.Get("Host") != "www.kernel.org" { t.Fatal("did not set Host header correctly") diff --git a/internal/engine/netx/tracex/quic.go b/internal/engine/netx/tracex/quic.go new file mode 100644 index 0000000..be60c85 --- /dev/null +++ b/internal/engine/netx/tracex/quic.go @@ -0,0 +1,139 @@ +package tracex + +import ( + "context" + "crypto/tls" + "net" + "time" + + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// QUICHandshakeSaver saves events occurring during the handshake +type QUICHandshakeSaver struct { + Saver *Saver + model.QUICDialer +} + +// DialContext implements ContextDialer.DialContext +func (h QUICHandshakeSaver) DialContext(ctx context.Context, network string, + host string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) { + start := time.Now() + // TODO(bassosimone): in the future we probably want to also save + // information about what versions we're willing to accept. + h.Saver.Write(Event{ + Address: host, + Name: "quic_handshake_start", + NoTLSVerify: tlsCfg.InsecureSkipVerify, + Proto: network, + TLSNextProtos: tlsCfg.NextProtos, + TLSServerName: tlsCfg.ServerName, + Time: start, + }) + sess, err := h.QUICDialer.DialContext(ctx, network, host, tlsCfg, cfg) + stop := time.Now() + if err != nil { + h.Saver.Write(Event{ + Duration: stop.Sub(start), + Err: err, + Name: "quic_handshake_done", + NoTLSVerify: tlsCfg.InsecureSkipVerify, + TLSNextProtos: tlsCfg.NextProtos, + TLSServerName: tlsCfg.ServerName, + Time: stop, + }) + return nil, err + } + state := quicConnectionState(sess) + h.Saver.Write(Event{ + Duration: stop.Sub(start), + Name: "quic_handshake_done", + NoTLSVerify: tlsCfg.InsecureSkipVerify, + TLSCipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), + TLSNegotiatedProto: state.NegotiatedProtocol, + TLSNextProtos: tlsCfg.NextProtos, + TLSPeerCerts: PeerCerts(state, err), + TLSServerName: tlsCfg.ServerName, + TLSVersion: netxlite.TLSVersionString(state.Version), + Time: stop, + }) + return sess, nil +} + +// quicConnectionState returns the ConnectionState of a QUIC Session. +func quicConnectionState(sess quic.EarlyConnection) tls.ConnectionState { + return sess.ConnectionState().TLS.ConnectionState +} + +// QUICListenerSaver is a QUICListener that also implements saving events. +type QUICListenerSaver struct { + // QUICListener is the underlying QUICListener. + model.QUICListener + + // Saver is the underlying Saver. + Saver *Saver +} + +// Listen implements QUICListener.Listen. +func (qls *QUICListenerSaver) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { + pconn, err := qls.QUICListener.Listen(addr) + if err != nil { + return nil, err + } + return &saverUDPConn{ + UDPLikeConn: pconn, + saver: qls.Saver, + }, nil +} + +type saverUDPConn struct { + model.UDPLikeConn + saver *Saver +} + +var _ model.UDPLikeConn = &saverUDPConn{} + +func (c *saverUDPConn) WriteTo(p []byte, addr net.Addr) (int, error) { + start := time.Now() + count, err := c.UDPLikeConn.WriteTo(p, addr) + stop := time.Now() + c.saver.Write(Event{ + Address: addr.String(), + Data: p[:count], + Duration: stop.Sub(start), + Err: err, + NumBytes: count, + Name: netxlite.WriteToOperation, + Time: stop, + }) + return count, err +} + +func (c *saverUDPConn) ReadFrom(b []byte) (int, net.Addr, error) { + start := time.Now() + n, addr, err := c.UDPLikeConn.ReadFrom(b) + stop := time.Now() + var data []byte + if n > 0 { + data = b[:n] + } + c.saver.Write(Event{ + Address: c.safeAddrString(addr), + Data: data, + Duration: stop.Sub(start), + Err: err, + NumBytes: n, + Name: netxlite.ReadFromOperation, + Time: stop, + }) + return n, addr, err +} + +func (c *saverUDPConn) safeAddrString(addr net.Addr) (out string) { + if addr != nil { + out = addr.String() + } + return +} diff --git a/internal/engine/netx/quicdialer/saver_test.go b/internal/engine/netx/tracex/quic_test.go similarity index 60% rename from internal/engine/netx/quicdialer/saver_test.go rename to internal/engine/netx/tracex/quic_test.go index 6aa7ace..33058ab 100644 --- a/internal/engine/netx/quicdialer/saver_test.go +++ b/internal/engine/netx/tracex/quic_test.go @@ -1,17 +1,18 @@ -package quicdialer_test +package tracex import ( "context" "crypto/tls" + "errors" + "net" "reflect" "strings" "testing" "time" "github.com/lucas-clemente/quic-go" - "github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/ooni/probe-cli/v3/internal/netxlite/quictesting" ) @@ -37,8 +38,8 @@ func TestHandshakeSaverSuccess(t *testing.T) { NextProtos: nextprotos, ServerName: servername, } - saver := &trace.Saver{} - dlr := quicdialer.HandshakeSaver{ + saver := &Saver{} + dlr := QUICHandshakeSaver{ QUICDialer: &netxlite.QUICDialerQUICGo{ QUICListener: &netxlite.QUICListenerStdlib{}, }, @@ -95,8 +96,8 @@ func TestHandshakeSaverHostNameError(t *testing.T) { NextProtos: nextprotos, ServerName: servername, } - saver := &trace.Saver{} - dlr := quicdialer.HandshakeSaver{ + saver := &Saver{} + dlr := QUICHandshakeSaver{ QUICDialer: &netxlite.QUICDialerQUICGo{ QUICListener: &netxlite.QUICListenerStdlib{}, }, @@ -122,3 +123,73 @@ func TestHandshakeSaverHostNameError(t *testing.T) { } } } + +func TestQUICListenerSaverCannotListen(t *testing.T) { + expected := errors.New("mocked error") + qls := &QUICListenerSaver{ + QUICListener: &mocks.QUICListener{ + MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { + return nil, expected + }, + }, + Saver: &Saver{}, + } + pconn, err := qls.Listen(&net.UDPAddr{ + IP: []byte{}, + Port: 8080, + Zone: "", + }) + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + if pconn != nil { + t.Fatal("expected nil pconn here") + } +} + +func TestSystemDialerSuccessWithReadWrite(t *testing.T) { + // This is the most common use case for collecting reads, writes + tlsConf := &tls.Config{ + NextProtos: []string{"h3"}, + ServerName: quictesting.Domain, + } + saver := &Saver{} + systemdialer := &netxlite.QUICDialerQUICGo{ + QUICListener: &QUICListenerSaver{ + QUICListener: &netxlite.QUICListenerStdlib{}, + Saver: saver, + }, + } + _, err := systemdialer.DialContext(context.Background(), "udp", + quictesting.Endpoint("443"), tlsConf, &quic.Config{}) + if err != nil { + t.Fatal(err) + } + ev := saver.Read() + if len(ev) < 2 { + t.Fatal("unexpected number of events") + } + last := len(ev) - 1 + for idx := 1; idx < last; idx++ { + if ev[idx].Data == nil { + t.Fatal("unexpected Data") + } + if ev[idx].Duration <= 0 { + t.Fatal("unexpected Duration") + } + if ev[idx].Err != nil { + t.Fatal("unexpected Err") + } + if ev[idx].NumBytes <= 0 { + t.Fatal("unexpected NumBytes") + } + switch ev[idx].Name { + case netxlite.ReadFromOperation, netxlite.WriteToOperation: + default: + t.Fatal("unexpected Name") + } + if ev[idx].Time.Before(ev[idx-1].Time) { + t.Fatal("unexpected Time", ev[idx].Time, ev[idx-1].Time) + } + } +} diff --git a/internal/engine/netx/resolver/saver.go b/internal/engine/netx/tracex/resolver.go similarity index 89% rename from internal/engine/netx/resolver/saver.go rename to internal/engine/netx/tracex/resolver.go index ba11a9b..4bbdc95 100644 --- a/internal/engine/netx/resolver/saver.go +++ b/internal/engine/netx/tracex/resolver.go @@ -1,23 +1,22 @@ -package resolver +package tracex import ( "context" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model" ) // SaverResolver is a resolver that saves events type SaverResolver struct { model.Resolver - Saver *trace.Saver + Saver *Saver } // LookupHost implements Resolver.LookupHost func (r SaverResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) { start := time.Now() - r.Saver.Write(trace.Event{ + r.Saver.Write(Event{ Address: r.Resolver.Address(), Hostname: hostname, Name: "resolve_start", @@ -26,7 +25,7 @@ func (r SaverResolver) LookupHost(ctx context.Context, hostname string) ([]strin }) addrs, err := r.Resolver.LookupHost(ctx, hostname) stop := time.Now() - r.Saver.Write(trace.Event{ + r.Saver.Write(Event{ Addresses: addrs, Address: r.Resolver.Address(), Duration: stop.Sub(start), @@ -42,14 +41,14 @@ func (r SaverResolver) LookupHost(ctx context.Context, hostname string) ([]strin // SaverDNSTransport is a DNS transport that saves events type SaverDNSTransport struct { model.DNSTransport - Saver *trace.Saver + Saver *Saver } // RoundTrip implements RoundTripper.RoundTrip func (txp SaverDNSTransport) RoundTrip( ctx context.Context, query model.DNSQuery) (model.DNSResponse, error) { start := time.Now() - txp.Saver.Write(trace.Event{ + txp.Saver.Write(Event{ Address: txp.Address(), DNSQuery: txp.maybeQueryBytes(query), Name: "dns_round_trip_start", @@ -58,7 +57,7 @@ func (txp SaverDNSTransport) RoundTrip( }) response, err := txp.DNSTransport.RoundTrip(ctx, query) stop := time.Now() - txp.Saver.Write(trace.Event{ + txp.Saver.Write(Event{ Address: txp.Address(), DNSQuery: txp.maybeQueryBytes(query), DNSReply: txp.maybeResponseBytes(response), diff --git a/internal/engine/netx/resolver/saver_test.go b/internal/engine/netx/tracex/resolver_test.go similarity index 76% rename from internal/engine/netx/resolver/saver_test.go rename to internal/engine/netx/tracex/resolver_test.go index 01f35cb..73a0590 100644 --- a/internal/engine/netx/resolver/saver_test.go +++ b/internal/engine/netx/tracex/resolver_test.go @@ -1,24 +1,24 @@ -package resolver_test +package tracex import ( "bytes" "context" "errors" + "net" "reflect" "testing" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/resolver" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/model/mocks" + "github.com/ooni/probe-cli/v3/internal/runtimex" ) func TestSaverResolverFailure(t *testing.T) { expected := errors.New("no such host") - saver := &trace.Saver{} - reso := resolver.SaverResolver{ - Resolver: resolver.NewFakeResolverWithExplicitError(expected), + saver := &Saver{} + reso := SaverResolver{ + Resolver: NewFakeResolverWithExplicitError(expected), Saver: saver, } addrs, err := reso.LookupHost(context.Background(), "www.google.com") @@ -63,9 +63,9 @@ func TestSaverResolverFailure(t *testing.T) { func TestSaverResolverSuccess(t *testing.T) { expected := []string{"8.8.8.8", "8.8.4.4"} - saver := &trace.Saver{} - reso := resolver.SaverResolver{ - Resolver: resolver.NewFakeResolverWithResult(expected), + saver := &Saver{} + reso := SaverResolver{ + Resolver: NewFakeResolverWithResult(expected), Saver: saver, } addrs, err := reso.LookupHost(context.Background(), "www.google.com") @@ -110,8 +110,8 @@ func TestSaverResolverSuccess(t *testing.T) { func TestSaverDNSTransportFailure(t *testing.T) { expected := errors.New("no such host") - saver := &trace.Saver{} - txp := resolver.SaverDNSTransport{ + saver := &Saver{} + txp := SaverDNSTransport{ DNSTransport: &mocks.DNSTransport{ MockRoundTrip: func(ctx context.Context, query model.DNSQuery) (model.DNSResponse, error) { return nil, expected @@ -173,13 +173,13 @@ func TestSaverDNSTransportFailure(t *testing.T) { func TestSaverDNSTransportSuccess(t *testing.T) { expected := []byte{0xef, 0xbe, 0xad, 0xde} - saver := &trace.Saver{} + saver := &Saver{} response := &mocks.DNSResponse{ MockBytes: func() []byte { return expected }, } - txp := resolver.SaverDNSTransport{ + txp := SaverDNSTransport{ DNSTransport: &mocks.DNSTransport{ MockRoundTrip: func(ctx context.Context, query model.DNSQuery) (model.DNSResponse, error) { return response, nil @@ -238,3 +238,50 @@ func TestSaverDNSTransportSuccess(t *testing.T) { t.Fatal("the saved time is wrong") } } + +func NewFakeResolverWithExplicitError(err error) model.Resolver { + runtimex.PanicIfNil(err, "passed nil error") + return &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, err + }, + MockNetwork: func() string { + return "fake" + }, + MockAddress: func() string { + return "" + }, + MockCloseIdleConnections: func() { + // nothing + }, + MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) { + return nil, errors.New("not implemented") + }, + MockLookupNS: func(ctx context.Context, domain string) ([]*net.NS, error) { + return nil, errors.New("not implemented") + }, + } +} + +func NewFakeResolverWithResult(r []string) model.Resolver { + return &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return r, nil + }, + MockNetwork: func() string { + return "fake" + }, + MockAddress: func() string { + return "" + }, + MockCloseIdleConnections: func() { + // nothing + }, + MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) { + return nil, errors.New("not implemented") + }, + MockLookupNS: func(ctx context.Context, domain string) ([]*net.NS, error) { + return nil, errors.New("not implemented") + }, + } +} diff --git a/internal/engine/netx/trace/saver.go b/internal/engine/netx/tracex/saver.go similarity index 97% rename from internal/engine/netx/trace/saver.go rename to internal/engine/netx/tracex/saver.go index c05dda9..53d9f8e 100644 --- a/internal/engine/netx/trace/saver.go +++ b/internal/engine/netx/tracex/saver.go @@ -1,4 +1,4 @@ -package trace +package tracex import "sync" diff --git a/internal/engine/netx/trace/trace_test.go b/internal/engine/netx/tracex/saver_test.go similarity index 69% rename from internal/engine/netx/trace/trace_test.go rename to internal/engine/netx/tracex/saver_test.go index 1cf1029..b63e642 100644 --- a/internal/engine/netx/trace/trace_test.go +++ b/internal/engine/netx/tracex/saver_test.go @@ -1,20 +1,18 @@ -package trace_test +package tracex import ( "sync" "testing" - - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" ) func TestGood(t *testing.T) { - saver := trace.Saver{} + saver := Saver{} var wg sync.WaitGroup const parallel = 10 wg.Add(parallel) for idx := 0; idx < parallel; idx++ { go func() { - saver.Write(trace.Event{}) + saver.Write(Event{}) wg.Done() }() } diff --git a/internal/engine/netx/tlsdialer/saver.go b/internal/engine/netx/tracex/tls.go similarity index 87% rename from internal/engine/netx/tlsdialer/saver.go rename to internal/engine/netx/tracex/tls.go index 117244b..621c316 100644 --- a/internal/engine/netx/tlsdialer/saver.go +++ b/internal/engine/netx/tracex/tls.go @@ -1,4 +1,4 @@ -package tlsdialer +package tracex import ( "context" @@ -6,7 +6,6 @@ import ( "net" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -14,7 +13,7 @@ import ( // SaverTLSHandshaker saves events occurring during the handshake type SaverTLSHandshaker struct { model.TLSHandshaker - Saver *trace.Saver + Saver *Saver } // Handshake implements TLSHandshaker.Handshake @@ -22,7 +21,7 @@ func (h SaverTLSHandshaker) Handshake( ctx context.Context, conn net.Conn, config *tls.Config, ) (net.Conn, tls.ConnectionState, error) { start := time.Now() - h.Saver.Write(trace.Event{ + h.Saver.Write(Event{ Name: "tls_handshake_start", NoTLSVerify: config.InsecureSkipVerify, TLSNextProtos: config.NextProtos, @@ -32,7 +31,7 @@ func (h SaverTLSHandshaker) Handshake( remoteAddr := conn.RemoteAddr().String() tlsconn, state, err := h.TLSHandshaker.Handshake(ctx, conn, config) stop := time.Now() - h.Saver.Write(trace.Event{ + h.Saver.Write(Event{ Address: remoteAddr, Duration: stop.Sub(start), Err: err, @@ -41,7 +40,7 @@ func (h SaverTLSHandshaker) Handshake( TLSCipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), TLSNegotiatedProto: state.NegotiatedProtocol, TLSNextProtos: config.NextProtos, - TLSPeerCerts: trace.PeerCerts(state, err), + TLSPeerCerts: PeerCerts(state, err), TLSServerName: config.ServerName, TLSVersion: netxlite.TLSVersionString(state.Version), Time: stop, diff --git a/internal/engine/netx/tlsdialer/saver_test.go b/internal/engine/netx/tracex/tls_test.go similarity index 90% rename from internal/engine/netx/tlsdialer/saver_test.go rename to internal/engine/netx/tracex/tls_test.go index 29d4657..c5093ca 100644 --- a/internal/engine/netx/tlsdialer/saver_test.go +++ b/internal/engine/netx/tracex/tls_test.go @@ -1,4 +1,4 @@ -package tlsdialer_test +package tracex import ( "context" @@ -7,9 +7,7 @@ import ( "testing" "time" - "github.com/ooni/probe-cli/v3/internal/engine/netx/dialer" - "github.com/ooni/probe-cli/v3/internal/engine/netx/tlsdialer" - "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -19,11 +17,20 @@ func TestSaverTLSHandshakerSuccessWithReadWrite(t *testing.T) { t.Skip("skip test in short mode") } nextprotos := []string{"h2"} - saver := &trace.Saver{} + saver := &Saver{} tlsdlr := &netxlite.TLSDialerLegacy{ Config: &tls.Config{NextProtos: nextprotos}, - Dialer: dialer.New(&dialer.Config{ReadWriteSaver: saver}, netxlite.DefaultResolver), - TLSHandshaker: tlsdialer.SaverTLSHandshaker{ + Dialer: netxlite.NewDialerWithResolver( + model.DiscardLogger, + netxlite.NewResolverStdlib(model.DiscardLogger), + func(dialer model.Dialer) model.Dialer { + return &SaverConnDialer{ + Dialer: dialer, + Saver: saver, + } + }, + ), + TLSHandshaker: SaverTLSHandshaker{ TLSHandshaker: &netxlite.TLSHandshakerConfigurable{}, Saver: saver, }, @@ -112,11 +119,11 @@ func TestSaverTLSHandshakerSuccess(t *testing.T) { t.Skip("skip test in short mode") } nextprotos := []string{"h2"} - saver := &trace.Saver{} + saver := &Saver{} tlsdlr := &netxlite.TLSDialerLegacy{ Config: &tls.Config{NextProtos: nextprotos}, Dialer: netxlite.DefaultDialer, - TLSHandshaker: tlsdialer.SaverTLSHandshaker{ + TLSHandshaker: SaverTLSHandshaker{ TLSHandshaker: &netxlite.TLSHandshakerConfigurable{}, Saver: saver, }, @@ -178,10 +185,10 @@ func TestSaverTLSHandshakerHostnameError(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - saver := &trace.Saver{} + saver := &Saver{} tlsdlr := &netxlite.TLSDialerLegacy{ Dialer: netxlite.DefaultDialer, - TLSHandshaker: tlsdialer.SaverTLSHandshaker{ + TLSHandshaker: SaverTLSHandshaker{ TLSHandshaker: &netxlite.TLSHandshakerConfigurable{}, Saver: saver, }, @@ -211,10 +218,10 @@ func TestSaverTLSHandshakerInvalidCertError(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - saver := &trace.Saver{} + saver := &Saver{} tlsdlr := &netxlite.TLSDialerLegacy{ Dialer: netxlite.DefaultDialer, - TLSHandshaker: tlsdialer.SaverTLSHandshaker{ + TLSHandshaker: SaverTLSHandshaker{ TLSHandshaker: &netxlite.TLSHandshakerConfigurable{}, Saver: saver, }, @@ -244,10 +251,10 @@ func TestSaverTLSHandshakerAuthorityError(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - saver := &trace.Saver{} + saver := &Saver{} tlsdlr := &netxlite.TLSDialerLegacy{ Dialer: netxlite.DefaultDialer, - TLSHandshaker: tlsdialer.SaverTLSHandshaker{ + TLSHandshaker: SaverTLSHandshaker{ TLSHandshaker: &netxlite.TLSHandshakerConfigurable{}, Saver: saver, }, @@ -277,11 +284,11 @@ func TestSaverTLSHandshakerNoTLSVerify(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - saver := &trace.Saver{} + saver := &Saver{} tlsdlr := &netxlite.TLSDialerLegacy{ Config: &tls.Config{InsecureSkipVerify: true}, Dialer: netxlite.DefaultDialer, - TLSHandshaker: tlsdialer.SaverTLSHandshaker{ + TLSHandshaker: SaverTLSHandshaker{ TLSHandshaker: &netxlite.TLSHandshakerConfigurable{}, Saver: saver, }, diff --git a/internal/tutorial/experiment/torsf/chapter04/README.md b/internal/tutorial/experiment/torsf/chapter04/README.md index f1d9e0d..1fd8fbc 100644 --- a/internal/tutorial/experiment/torsf/chapter04/README.md +++ b/internal/tutorial/experiment/torsf/chapter04/README.md @@ -34,11 +34,11 @@ generic data model used by all experiments. ``` -The `archival` package contains code used to format internal +The `tracex` package contains code used to format internal measurements representations to the OONI data format. ```Go - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" ``` @@ -104,7 +104,7 @@ starting the experiment. Therefore, it's consistent with the ```Go if err := ptl.Start(); err != nil { - testkeys.Failure = archival.NewFailure(err) + testkeys.Failure = tracex.NewFailure(err) errch <- err return } @@ -116,7 +116,7 @@ pass specific `TorArgs` that cause `tor` to know about the pluggable transport created by `ptl` and `sfdialer`. ```Go - tun, err := tunnel.Start(ctx, &tunnel.Config{ + tun, _, err := tunnel.Start(ctx, &tunnel.Config{ Name: "tor", Session: sess, TunnelDir: path.Join(sess.TempDir(), "torsf"), @@ -130,7 +130,7 @@ pluggable transport created by `ptl` and `sfdialer`. ``` In case of error, we convert `err` to a OONI failure using -the `NewFailure` function of `archival`. This function reduces +the `NewFailure` function of `tracex`. This function reduces Go error strings to the error strings used by OONI. You can read the [errors spec](https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md) at the [github.com/ooni/spec repo](https://github.com/ooni/spec). @@ -141,7 +141,7 @@ experiment, but rather a possibly interesting anomaly. ```Go if err != nil { - testkeys.Failure = archival.NewFailure(err) + testkeys.Failure = tracex.NewFailure(err) errch <- nil return } diff --git a/internal/tutorial/experiment/torsf/chapter04/torsf.go b/internal/tutorial/experiment/torsf/chapter04/torsf.go index 5268578..1ca6399 100644 --- a/internal/tutorial/experiment/torsf/chapter04/torsf.go +++ b/internal/tutorial/experiment/torsf/chapter04/torsf.go @@ -37,11 +37,11 @@ import ( // ``` // - // The `archival` package contains code used to format internal + // The `tracex` package contains code used to format internal // measurements representations to the OONI data format. // // ```Go - "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" + "github.com/ooni/probe-cli/v3/internal/engine/netx/tracex" // ``` // @@ -169,7 +169,7 @@ func (m *Measurer) run(ctx context.Context, // // ```Go if err := ptl.Start(); err != nil { - testkeys.Failure = archival.NewFailure(err) + testkeys.Failure = tracex.NewFailure(err) errch <- err return } @@ -195,7 +195,7 @@ func (m *Measurer) run(ctx context.Context, // ``` // // In case of error, we convert `err` to a OONI failure using - // the `NewFailure` function of `archival`. This function reduces + // the `NewFailure` function of `tracex`. This function reduces // Go error strings to the error strings used by OONI. You can // read the [errors spec](https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md) // at the [github.com/ooni/spec repo](https://github.com/ooni/spec). @@ -206,7 +206,7 @@ func (m *Measurer) run(ctx context.Context, // // ```Go if err != nil { - testkeys.Failure = archival.NewFailure(err) + testkeys.Failure = tracex.NewFailure(err) errch <- nil return }