c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(netx): merge archival, trace, and the savers (#772)

Browse Source 
This diff creates a new package under netx called tracex that
contains everything we need to perform measurements using events
tracing and postprocessing (which is the technique with which
we implement most network experiments).

The general idea here is to (1) create a unique package out of
all of these packages; (2) clean up the code a bit (improve tests,
docs, apply more recent code patterns); (3) move the resulting
code as a toplevel package inside of internal.

Once this is done, netx can be further refactored to avoid
subpackages and we can search for more code to salvage/refactor.

See https://github.com/ooni/probe/issues/2121
This commit is contained in:
Simone Basso
2022-05-31 21:53:01 +02:00
committed by GitHub
parent dd5655eaee
commit bbcd2e2280
53 changed files with 869 additions and 865 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/experiment/hhfm/hhfm.go
+23 -23
View File
@@ -15,7 +15,7 @@ import (
"time"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/tracex"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/randx"
@@ -34,11 +34,11 @@ type Config struct{}
// Here we are emitting for the same set of test keys that are
// produced by the MK implementation.
type TestKeys struct {
Agent string `json:"agent"`
Failure *string `json:"failure"`
Requests []archival.RequestEntry `json:"requests"`
SOCKSProxy *string `json:"socksproxy"`
Tampering Tampering `json:"tampering"`
Agent string `json:"agent"`
Failure *string `json:"failure"`
Requests []tracex.RequestEntry `json:"requests"`
SOCKSProxy *string `json:"socksproxy"`
Tampering Tampering `json:"tampering"`
}
// Tampering describes the detected forms of tampering.
@@ -151,7 +151,7 @@ func (m Measurer) Run(
// from that and then see to improve the robustness in the future.
resp, data, err := Transact(txp, req.WithContext(ctx), callbacks)
if err != nil {
tk.Failure = archival.NewFailure(err)
tk.Failure = tracex.NewFailure(err)
tk.Requests[0].Failure = tk.Failure
tk.Tampering.Total = true
return nil // measurement did not fail, we measured tampering
@@ -246,23 +246,23 @@ func (tk *TestKeys) FillTampering(
}
}
// NewRequestEntryList creates a new []archival.RequestEntry given a
// NewRequestEntryList creates a new []tracex.RequestEntry given a
// specific *http.Request and headers with random case.
func NewRequestEntryList(req *http.Request, headers map[string]string) (out []archival.RequestEntry) {
out = []archival.RequestEntry{{
Request: archival.HTTPRequest{
Headers: make(map[string]archival.MaybeBinaryValue),
HeadersList: []archival.HTTPHeader{},
func NewRequestEntryList(req *http.Request, headers map[string]string) (out []tracex.RequestEntry) {
out = []tracex.RequestEntry{{
Request: tracex.HTTPRequest{
Headers: make(map[string]tracex.MaybeBinaryValue),
HeadersList: []tracex.HTTPHeader{},
Method: req.Method,
URL: req.URL.String(),
},
}}
for key, value := range headers {
// Using the random capitalization headers here
mbv := archival.MaybeBinaryValue{Value: value}
mbv := tracex.MaybeBinaryValue{Value: value}
out[0].Request.Headers[key] = mbv
out[0].Request.HeadersList = append(out[0].Request.HeadersList,
archival.HTTPHeader{Key: key, Value: mbv})
tracex.HTTPHeader{Key: key, Value: mbv})
}
sort.Slice(out[0].Request.HeadersList, func(i, j int) bool {
return out[0].Request.HeadersList[i].Key < out[0].Request.HeadersList[j].Key
@@ -270,19 +270,19 @@ func NewRequestEntryList(req *http.Request, headers map[string]string) (out []ar
return
}
// NewHTTPResponse creates a new archival.HTTPResponse given a
// NewHTTPResponse creates a new tracex.HTTPResponse given a
// specific *http.Response instance and its body.
func NewHTTPResponse(resp *http.Response, data []byte) (out archival.HTTPResponse) {
out = archival.HTTPResponse{
Body: archival.HTTPBody{Value: string(data)},
func NewHTTPResponse(resp *http.Response, data []byte) (out tracex.HTTPResponse) {
out = tracex.HTTPResponse{
Body: tracex.HTTPBody{Value: string(data)},
Code: int64(resp.StatusCode),
Headers: make(map[string]archival.MaybeBinaryValue),
HeadersList: []archival.HTTPHeader{},
Headers: make(map[string]tracex.MaybeBinaryValue),
HeadersList: []tracex.HTTPHeader{},
}
for key := range resp.Header {
mbv := archival.MaybeBinaryValue{Value: resp.Header.Get(key)}
mbv := tracex.MaybeBinaryValue{Value: resp.Header.Get(key)}
out.Headers[key] = mbv
out.HeadersList = append(out.HeadersList, archival.HTTPHeader{Key: key, Value: mbv})
out.HeadersList = append(out.HeadersList, tracex.HTTPHeader{Key: key, Value: mbv})
}
sort.Slice(out.HeadersList, func(i, j int) bool {
return out.HeadersList[i].Key < out.HeadersList[j].Key
internal/engine/experiment/hhfm/hhfm_test.go
+24 -24
View File
@@ -16,7 +16,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/tracex"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
@@ -554,7 +554,7 @@ func TestTestKeys_FillTampering(t *testing.T) {
type fields struct {
Agent string
Failure *string
Requests []archival.RequestEntry
Requests []tracex.RequestEntry
SOCKSProxy *string
Tampering hhfm.Tampering
}
@@ -689,7 +689,7 @@ func TestNewRequestEntryList(t *testing.T) {
tests := []struct {
name string
args args
wantOut []archival.RequestEntry
wantOut []tracex.RequestEntry
}{{
name: "common case",
args: args{
@@ -706,16 +706,16 @@ func TestNewRequestEntryList(t *testing.T) {
"User-aGENT": "foo/1.0",
},
},
wantOut: []archival.RequestEntry{{
Request: archival.HTTPRequest{
HeadersList: []archival.HTTPHeader{{
wantOut: []tracex.RequestEntry{{
Request: tracex.HTTPRequest{
HeadersList: []tracex.HTTPHeader{{
Key: "ContENt-tYPE",
Value: archival.MaybeBinaryValue{Value: "text/plain"},
Value: tracex.MaybeBinaryValue{Value: "text/plain"},
}, {
Key: "User-aGENT",
Value: archival.MaybeBinaryValue{Value: "foo/1.0"},
Value: tracex.MaybeBinaryValue{Value: "foo/1.0"},
}},
Headers: map[string]archival.MaybeBinaryValue{
Headers: map[string]tracex.MaybeBinaryValue{
"ContENt-tYPE": {Value: "text/plain"},
"User-aGENT": {Value: "foo/1.0"},
},
@@ -735,11 +735,11 @@ func TestNewRequestEntryList(t *testing.T) {
},
},
},
wantOut: []archival.RequestEntry{{
Request: archival.HTTPRequest{
wantOut: []tracex.RequestEntry{{
Request: tracex.HTTPRequest{
Method: "GeT",
Headers: make(map[string]archival.MaybeBinaryValue),
HeadersList: []archival.HTTPHeader{},
Headers: make(map[string]tracex.MaybeBinaryValue),
HeadersList: []tracex.HTTPHeader{},
URL: "http://10.0.0.1/",
},
}},
@@ -762,7 +762,7 @@ func TestNewHTTPResponse(t *testing.T) {
tests := []struct {
name string
args args
wantOut archival.HTTPResponse
wantOut tracex.HTTPResponse
}{{
name: "common case",
args: args{
@@ -775,17 +775,17 @@ func TestNewHTTPResponse(t *testing.T) {
},
data: []byte("deadbeef"),
},
wantOut: archival.HTTPResponse{
Body: archival.MaybeBinaryValue{Value: "deadbeef"},
wantOut: tracex.HTTPResponse{
Body: tracex.MaybeBinaryValue{Value: "deadbeef"},
Code: 200,
HeadersList: []archival.HTTPHeader{{
HeadersList: []tracex.HTTPHeader{{
Key: "Content-Type",
Value: archival.MaybeBinaryValue{Value: "text/plain"},
Value: tracex.MaybeBinaryValue{Value: "text/plain"},
}, {
Key: "User-Agent",
Value: archival.MaybeBinaryValue{Value: "foo/1.0"},
Value: tracex.MaybeBinaryValue{Value: "foo/1.0"},
}},
Headers: map[string]archival.MaybeBinaryValue{
Headers: map[string]tracex.MaybeBinaryValue{
"Content-Type": {Value: "text/plain"},
"User-Agent": {Value: "foo/1.0"},
},
@@ -795,11 +795,11 @@ func TestNewHTTPResponse(t *testing.T) {
args: args{
resp: &http.Response{StatusCode: 200},
},
wantOut: archival.HTTPResponse{
Body: archival.MaybeBinaryValue{Value: ""},
wantOut: tracex.HTTPResponse{
Body: tracex.MaybeBinaryValue{Value: ""},
Code: 200,
HeadersList: []archival.HTTPHeader{},
Headers: map[string]archival.MaybeBinaryValue{},
HeadersList: []tracex.HTTPHeader{},
Headers: map[string]tracex.MaybeBinaryValue{},
},
}}
for _, tt := range tests {