c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(measurex): use same keys of the OONI data format (#572)

Browse Source 
This change should simplify the pipeline's job.

Reference issue: https://github.com/ooni/probe/issues/1817.

I previously dismissed this possibility, but now it seems clear it
is simpler to have a very tabular data format internally and to
convert such a format to OONI's data format when serializing.

The OONI data format is what the pipeline expects, but processing
is easier with a more linear/tabular format.
This commit is contained in:
Simone Basso
2021-11-05 10:46:45 +01:00
committed by GitHub
parent 6f90d29bfa
commit aa27bbe33f
35 changed files with 1571 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tutorial/measurex/chapter14/README.md
+14 -9
View File
@@ -45,10 +45,10 @@ that a Web Connectivity measurement should have.
```Go
type measurement struct {
Queries []*measurex.DNSLookupEvent `json:"queries"`
TCPConnect []*measurex.NetworkEvent `json:"tcp_connect"`
TLSHandshakes []*measurex.TLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.HTTPRoundTripEvent `json:"requests"`
Queries []*measurex.ArchivalDNSLookupEvent `json:"queries"`
TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"`
TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.ArchivalHTTPRoundTripEvent `json:"requests"`
}
```
@@ -96,7 +96,8 @@ the input URL's domain using the system resolver.
```Go
dns := mx.LookupHostSystem(ctx, parsedURL.Hostname())
m.Queries = append(m.Queries, dns.LookupHost...)
m.Queries = append(
m.Queries, measurex.NewArchivalDNSLookupEventList(dns.LookupHost)...)
```
@@ -128,7 +129,8 @@ whether the input URL is HTTP or HTTPS.
switch parsedURL.Scheme {
case "http":
tcp := mx.TCPConnect(ctx, epnt.Address)
m.TCPConnect = append(m.TCPConnect, tcp.Connect...)
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...)
case "https":
config := &tls.Config{
ServerName: parsedURL.Hostname(),
@@ -136,8 +138,10 @@ whether the input URL is HTTP or HTTPS.
RootCAs: netxlite.NewDefaultCertPool(),
}
tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config)
m.TCPConnect = append(m.TCPConnect, tls.Connect...)
m.TLSHandshakes = append(m.TLSHandshakes, tls.TLSHandshake...)
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tls.Connect)...)
m.TLSHandshakes = append(m.TLSHandshakes,
measurex.NewArchivalQUICTLSHandshakeEventList(tls.TLSHandshake)...)
}
}
@@ -222,7 +226,8 @@ the chapters we have seen so far.
```Go
m.Requests = append(m.Requests, db.AsMeasurement().HTTPRoundTrip...)
m.Requests = append(m.Requests, measurex.NewArchivalHTTPRoundTripEventList(
db.AsMeasurement().HTTPRoundTrip)...)
return m, nil
}
internal/tutorial/measurex/chapter14/main.go
+14 -9
View File
@@ -46,10 +46,10 @@ func print(v interface{}) {
// ```Go
type measurement struct {
Queries []*measurex.DNSLookupEvent `json:"queries"`
TCPConnect []*measurex.NetworkEvent `json:"tcp_connect"`
TLSHandshakes []*measurex.TLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.HTTPRoundTripEvent `json:"requests"`
Queries []*measurex.ArchivalDNSLookupEvent `json:"queries"`
TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"`
TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.ArchivalHTTPRoundTripEvent `json:"requests"`
}
// ```
@@ -97,7 +97,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
//
// ```Go
dns := mx.LookupHostSystem(ctx, parsedURL.Hostname())
m.Queries = append(m.Queries, dns.LookupHost...)
m.Queries = append(
m.Queries, measurex.NewArchivalDNSLookupEventList(dns.LookupHost)...)
// ```
//
@@ -129,7 +130,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
switch parsedURL.Scheme {
case "http":
tcp := mx.TCPConnect(ctx, epnt.Address)
m.TCPConnect = append(m.TCPConnect, tcp.Connect...)
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...)
case "https":
config := &tls.Config{
ServerName: parsedURL.Hostname(),
@@ -137,8 +139,10 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
RootCAs: netxlite.NewDefaultCertPool(),
}
tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config)
m.TCPConnect = append(m.TCPConnect, tls.Connect...)
m.TLSHandshakes = append(m.TLSHandshakes, tls.TLSHandshake...)
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tls.Connect)...)
m.TLSHandshakes = append(m.TLSHandshakes,
measurex.NewArchivalQUICTLSHandshakeEventList(tls.TLSHandshake)...)
}
}
@@ -223,7 +227,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
//
// ```Go
m.Requests = append(m.Requests, db.AsMeasurement().HTTPRoundTrip...)
m.Requests = append(m.Requests, measurex.NewArchivalHTTPRoundTripEventList(
db.AsMeasurement().HTTPRoundTrip)...)
return m, nil
}