fix(measurex): use same keys of the OONI data format (#572)

This change should simplify the pipeline's job.

Reference issue: https://github.com/ooni/probe/issues/1817.

I previously dismissed this possibility, but now it seems clear it
is simpler to have a very tabular data format internally and to
convert such a format to OONI's data format when serializing.

The OONI data format is what the pipeline expects, but processing
is easier with a more linear/tabular format.
This commit is contained in:
Simone Basso 2021-11-05 10:46:45 +01:00 committed by GitHub
commit aa27bbe33f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 1539 additions and 993 deletions

View file

@ -29,7 +29,7 @@ type Config struct{}
// TestKeys contains the experiment's test keys.
type TestKeys struct {
*measurex.URLMeasurement
*measurex.ArchivalURLMeasurement
}
// Measurer performs the measurement.
@ -142,7 +142,9 @@ func (mx *Measurer) runAsync(ctx context.Context, sess model.ExperimentSession,
},
Input: model.MeasurementTarget(m.URL),
MeasurementRuntime: m.TotalRuntime.Seconds(),
TestKeys: &TestKeys{URLMeasurement: m},
TestKeys: &TestKeys{
ArchivalURLMeasurement: measurex.NewArchivalURLMeasurement(m),
},
}
}
}
@ -163,7 +165,7 @@ type measurerMeasureURLHelper struct {
func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header,
curEndpoints ...*measurex.HTTPEndpoint) (
[]*measurex.HTTPEndpoint, interface{}, error) {
[]*measurex.HTTPEndpoint, *measurex.THMeasurement, error) {
cc := &THClientCall{
Endpoints: measurex.HTTPEndpointsToEndpoints(curEndpoints),
HTTPClient: mth.Clnt,