fix(measurex): use same keys of the OONI data format (#572)

This change should simplify the pipeline's job.

Reference issue: https://github.com/ooni/probe/issues/1817.

I previously dismissed this possibility, but now it seems clear it
is simpler to have a very tabular data format internally and to
convert such a format to OONI's data format when serializing.

The OONI data format is what the pipeline expects, but processing
is easier with a more linear/tabular format.

internal/engine/experiment/webstepsx/measurer.go

@@ -29,7 +29,7 @@ type Config struct{}
 
 // TestKeys contains the experiment's test keys.
 type TestKeys struct {
-	*measurex.URLMeasurement
+	*measurex.ArchivalURLMeasurement
 }
 
 // Measurer performs the measurement.
@@ -142,7 +142,9 @@ func (mx *Measurer) runAsync(ctx context.Context, sess model.ExperimentSession,
 			},
 			Input:              model.MeasurementTarget(m.URL),
 			MeasurementRuntime: m.TotalRuntime.Seconds(),
-			TestKeys:           &TestKeys{URLMeasurement: m},
+			TestKeys: &TestKeys{
+				ArchivalURLMeasurement: measurex.NewArchivalURLMeasurement(m),
+			},
 		}
 	}
 }
@@ -163,7 +165,7 @@ type measurerMeasureURLHelper struct {
 func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints(
 	ctx context.Context, URL *url.URL, headers http.Header,
 	curEndpoints ...*measurex.HTTPEndpoint) (
-	[]*measurex.HTTPEndpoint, interface{}, error) {
+	[]*measurex.HTTPEndpoint, *measurex.THMeasurement, error) {
 	cc := &THClientCall{
 		Endpoints:  measurex.HTTPEndpointsToEndpoints(curEndpoints),
 		HTTPClient: mth.Clnt,

internal/engine/experiment/webstepsx/th.go

@@ -41,14 +41,7 @@ type THClientRequest struct {
 }
 
 // THServerResponse is the response from the test helper.
-type THServerResponse struct {
-	// DNS contains all the DNS related measurements.
-	DNS []*measurex.DNSMeasurement `json:"dns"`
-
-	// Endpoints contains a measurement for each endpoint
-	// that was discovered by the probe or the TH.
-	Endpoints []*measurex.HTTPEndpointMeasurement `json:"endpoints"`
-}
+type THServerResponse = measurex.THMeasurement
 
 // thMaxAcceptableBodySize is the maximum acceptable body size by TH code.
 const thMaxAcceptableBodySize = 1 << 20
@@ -294,9 +287,9 @@ func (h *THHandler) simplifyMeasurement(in *measurex.Measurement) (out *measurex
 }
 
 func (h *THHandler) simplifyHandshake(
-	in []*measurex.TLSHandshakeEvent) (out []*measurex.TLSHandshakeEvent) {
+	in []*measurex.QUICTLSHandshakeEvent) (out []*measurex.QUICTLSHandshakeEvent) {
 	for _, ev := range in {
-		out = append(out, &measurex.TLSHandshakeEvent{
+		out = append(out, &measurex.QUICTLSHandshakeEvent{
 			CipherSuite:     ev.CipherSuite,
 			Failure:         ev.Failure,
 			NegotiatedProto: ev.NegotiatedProto,
@@ -319,32 +312,24 @@ func (h *THHandler) simplifyHTTPRoundTrip(
 	in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) {
 	for _, ev := range in {
 		out = append(out, &measurex.HTTPRoundTripEvent{
-			Failure:  ev.Failure,
-			Request:  ev.Request,
-			Response: h.simplifyHTTPResponse(ev.Response),
-			Finished: 0,
-			Started:  0,
-			Oddity:   ev.Oddity,
+			Failure:                 ev.Failure,
+			Method:                  ev.Method,
+			URL:                     ev.URL,
+			RequestHeaders:          ev.RequestHeaders,
+			StatusCode:              ev.StatusCode,
+			ResponseHeaders:         ev.ResponseHeaders,
+			ResponseBody:            nil, // we don't transfer the body
+			ResponseBodyLength:      ev.ResponseBodyLength,
+			ResponseBodyIsTruncated: ev.ResponseBodyIsTruncated,
+			ResponseBodyIsUTF8:      ev.ResponseBodyIsUTF8,
+			Finished:                ev.Finished,
+			Started:                 ev.Started,
+			Oddity:                  ev.Oddity,
 		})
 	}
 	return
 }
 
-func (h *THHandler) simplifyHTTPResponse(
-	in *measurex.HTTPResponse) (out *measurex.HTTPResponse) {
-	if in != nil {
-		out = &measurex.HTTPResponse{
-			Code:            in.Code,
-			Headers:         in.Headers,
-			Body:            nil,
-			BodyIsTruncated: in.BodyIsTruncated,
-			BodyLength:      in.BodyLength,
-			BodyIsUTF8:      in.BodyIsUTF8,
-		}
-	}
-	return
-}
-
 type thMeasureURLHelper struct {
 	epnts []*measurex.Endpoint
 }
@@ -352,7 +337,7 @@ type thMeasureURLHelper struct {
 func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints(
 	ctx context.Context, URL *url.URL, headers http.Header,
 	serverEpnts ...*measurex.HTTPEndpoint) (
-	epnts []*measurex.HTTPEndpoint, thMeaurement interface{}, err error) {
+	epnts []*measurex.HTTPEndpoint, thMeaurement *measurex.THMeasurement, err error) {
 	for _, epnt := range thh.epnts {
 		epnts = append(epnts, &measurex.HTTPEndpoint{
 			Domain:  URL.Hostname(),