c/ooni-probe-cli
Watch
1
0
Fork
You've already forked ooni-probe-cli
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix(webconnectivity): expose network events (#258)

Browse source 
* fix(webconnectivity): expose network events

By not exposing network events in webconnectivity, we are missing
several interesting, explanatory data points.

This diff fixes the issue by:

1. enriching the definition of network events to include extra
data useful for performing (manual) data analysis;

2. adding a tags field to network events such that we can add
tags to specific events and understand where they come from;

3. exposing all the (tagged) network events that happen when running
a webconnectivity experiment.

See https://github.com/ooni/probe-engine/issues/1157.

* progress

* more work towards landing this diff

* Apply suggestions from code review
This commit is contained in:
Simone Basso 2021-03-23 16:46:46 +01:00 • committed by GitHub
commit fc19c9901a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 67 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

45
internal/engine/experiment/webconnectivity/webconnectivity.go
View file

@ -19,7 +19,7 @@ import (
const (
testName = "web_connectivity"
testVersion = "0.3.0"
testVersion = "0.4.0"
)
// Config contains the experiment config.
@ -32,6 +32,15 @@ type TestKeys struct {
Retries *int64 `json:"retries"` // unused
SOCKSProxy *string `json:"socksproxy"` // unused
// For now mostly TCP/TLS "connect" experiment but we are
// considering adding more events. An open question is
// currently how to properly tag these events so that it
// is rather obvious where they come from.
//
// See https://github.com/ooni/probe/issues/1413.
NetworkEvents []archival.NetworkEvent `json:"network_events"`
TLSHandshakes []archival.TLSHandshake `json:"tls_handshakes"`
// DNS experiment
Queries []archival.DNSQueryEntry `json:"queries"`
DNSExperimentFailure *string `json:"dns_experiment_failure"`
@ -42,7 +51,7 @@ type TestKeys struct {
ControlRequest ControlRequest `json:"-"`
Control ControlResponse `json:"control"`
// TCP connect experiment
// TCP/TLS "connect" experiment
TCPConnect []archival.TCPConnectEntry `json:"tcp_connect"`
TCPConnectSuccesses int `json:"-"`
TCPConnectAttempts int `json:"-"`
@ -90,6 +99,19 @@ var (
ErrUnsupportedInput = errors.New("unsupported input scheme")
)
// Tags describing the section of this experiment in which
// the data has been collected.
const (
// DNSExperimentTag is a tag indicating the DNS experiment.
DNSExperimentTag = "dns_experiment"
// TCPTLSExperimentTag is a tag indicating the connect experiment.
TCPTLSExperimentTag = "tcptls_experiment"
// HTTPExperimentTag is a tag indicating the HTTP experiment.
HTTPExperimentTag = "http_experiment"
)
// Run implements ExperimentMeasurer.Run.
func (m Measurer) Run(
ctx context.Context,
@ -129,7 +151,9 @@ func (m Measurer) Run(
"backend": testhelper,
}
// 2. perform the DNS lookup step
dnsResult := DNSLookup(ctx, DNSLookupConfig{Session: sess, URL: URL})
dnsResult := DNSLookup(ctx, DNSLookupConfig{
Begin: measurement.MeasurementStartTimeSaved,
Session: sess, URL: URL})
tk.Queries = append(tk.Queries, dnsResult.TestKeys.Queries...)
tk.DNSExperimentFailure = dnsResult.Failure
epnts := NewEndpoints(URL, dnsResult.Addresses())
@ -152,7 +176,13 @@ func (m Measurer) Run(
sess.Logger().Infof("DNS analysis result: %+v", internal.StringPointerToString(
tk.DNSAnalysisResult.DNSConsistency))
// 5. perform TCP/TLS connects
//
// TODO(bassosimone): here we should also follow the IP addresses
// returned by the control experiment.
//
// See https://github.com/ooni/probe/issues/1414
connectsResult := Connects(ctx, ConnectsConfig{
Begin: measurement.MeasurementStartTimeSaved,
Session: sess,
TargetURL: URL,
URLGetterURLs: epnts.URLs(),
@ -164,12 +194,21 @@ func (m Measurer) Run(
// sad that we're storing analysis result inside the measurement
tk.TCPConnect = append(tk.TCPConnect, ComputeTCPBlocking(
tcpkeys.TCPConnect, tk.Control.TCPConnect)...)
for _, ev := range tcpkeys.NetworkEvents {
ev.Tags = []string{TCPTLSExperimentTag}
tk.NetworkEvents = append(tk.NetworkEvents, ev)
}
for _, ev := range tcpkeys.TLSHandshakes {
ev.Tags = []string{TCPTLSExperimentTag}
tk.TLSHandshakes = append(tk.TLSHandshakes, ev)
}
}
tk.TCPConnectAttempts = connectsResult.Total
tk.TCPConnectSuccesses = connectsResult.Successes
// 6. perform HTTP/HTTPS measurement
httpResult := HTTPGet(ctx, HTTPGetConfig{
Addresses: dnsResult.Addresses(),
Begin: measurement.MeasurementStartTimeSaved,
Session: sess,
TargetURL: URL,
})