c/ooni-probe-cli
Watch
1
0
Fork
You've already forked ooni-probe-cli
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix(measurex): use same keys of the OONI data format (#572)

Browse source 
This change should simplify the pipeline's job.

Reference issue: https://github.com/ooni/probe/issues/1817.

I previously dismissed this possibility, but now it seems clear it
is simpler to have a very tabular data format internally and to
convert such a format to OONI's data format when serializing.

The OONI data format is what the pipeline expects, but processing
is easier with a more linear/tabular format.
This commit is contained in:
Simone Basso 2021-11-05 10:46:45 +01:00 • committed by GitHub
commit aa27bbe33f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 1539 additions and 993 deletions
Download patch file Download diff file Expand all files Collapse all files

49
internal/engine/experiment/webstepsx/th.go
View file

@ -41,14 +41,7 @@ type THClientRequest struct {
}
// THServerResponse is the response from the test helper.
type THServerResponse struct {
// DNS contains all the DNS related measurements.
DNS []*measurex.DNSMeasurement `json:"dns"`
// Endpoints contains a measurement for each endpoint
// that was discovered by the probe or the TH.
Endpoints []*measurex.HTTPEndpointMeasurement `json:"endpoints"`
}
type THServerResponse = measurex.THMeasurement
// thMaxAcceptableBodySize is the maximum acceptable body size by TH code.
const thMaxAcceptableBodySize = 1 << 20
@ -294,9 +287,9 @@ func (h *THHandler) simplifyMeasurement(in *measurex.Measurement) (out *measurex
}
func (h *THHandler) simplifyHandshake(
in []*measurex.TLSHandshakeEvent) (out []*measurex.TLSHandshakeEvent) {
in []*measurex.QUICTLSHandshakeEvent) (out []*measurex.QUICTLSHandshakeEvent) {
for _, ev := range in {
out = append(out, &measurex.TLSHandshakeEvent{
out = append(out, &measurex.QUICTLSHandshakeEvent{
CipherSuite: ev.CipherSuite,
Failure: ev.Failure,
NegotiatedProto: ev.NegotiatedProto,
@ -319,32 +312,24 @@ func (h *THHandler) simplifyHTTPRoundTrip(
in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) {
for _, ev := range in {
out = append(out, &measurex.HTTPRoundTripEvent{
Failure: ev.Failure,
Request: ev.Request,
Response: h.simplifyHTTPResponse(ev.Response),
Finished: 0,
Started: 0,
Oddity: ev.Oddity,
Failure: ev.Failure,
Method: ev.Method,
URL: ev.URL,
RequestHeaders: ev.RequestHeaders,
StatusCode: ev.StatusCode,
ResponseHeaders: ev.ResponseHeaders,
ResponseBody: nil, // we don't transfer the body
ResponseBodyLength: ev.ResponseBodyLength,
ResponseBodyIsTruncated: ev.ResponseBodyIsTruncated,
ResponseBodyIsUTF8: ev.ResponseBodyIsUTF8,
Finished: ev.Finished,
Started: ev.Started,
Oddity: ev.Oddity,
})
}
return
}
func (h *THHandler) simplifyHTTPResponse(
in *measurex.HTTPResponse) (out *measurex.HTTPResponse) {
if in != nil {
out = &measurex.HTTPResponse{
Code: in.Code,
Headers: in.Headers,
Body: nil,
BodyIsTruncated: in.BodyIsTruncated,
BodyLength: in.BodyLength,
BodyIsUTF8: in.BodyIsUTF8,
}
}
return
}
type thMeasureURLHelper struct {
epnts []*measurex.Endpoint
}
@ -352,7 +337,7 @@ type thMeasureURLHelper struct {
func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header,
serverEpnts ...*measurex.HTTPEndpoint) (
epnts []*measurex.HTTPEndpoint, thMeaurement interface{}, err error) {
epnts []*measurex.HTTPEndpoint, thMeaurement *measurex.THMeasurement, err error) {
for _, epnt := range thh.epnts {
epnts = append(epnts, &measurex.HTTPEndpoint{
Domain: URL.Hostname(),