feat(measurex): refactored measurement library (#528)
This commit introduce a measurement library that consists of
refactored code from earlier websteps experiments.
I am not going to add tests for the time being, because this library
is still a bit in flux, as we finalize websteps.
I will soon though commit documentation explaining in detail how
to use it, which currrently is at https://github.com/ooni/probe-cli/pull/506
and adds a new directory to internal/tutorial.
The core idea of this measurement library is to allow two
measurement modes:
1. tracing, which is what we're currently doing now, and the
tutorial shows how we can rewrite the measurement part of web
connectivity with measurex using less code. Under a tracing
approach, we construct a normal http.Client that however has
tracing configured, we gather events for resolve, connect, TLS
handshake, QUIC handshake, HTTP round trip, etc. and then we
try to make sense of what happened from the events stream;
2. step-by-step, which is what websteps does, and basically
means that after each operation you immediately write into
a Measurement structure its results and immediately draw the
conclusions on what seems odd (which later may become an
anomaly if we see what the test helper measured).
This library is also such that it produces a data format
compatible with the current OONI spec.
This work is part of https://github.com/ooni/probe/issues/1733.
2021-09-30 01:24:08 +02:00
|
|
|
package measurex
|
|
|
|
|
|
|
|
//
|
|
|
|
// Dialer
|
|
|
|
//
|
|
|
|
// Wrappers for Dialer and Conn to store events into a WritableDB.
|
|
|
|
//
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"net"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Conn is a network connection.
|
|
|
|
type Conn = net.Conn
|
|
|
|
|
|
|
|
// Dialer dials network connections.
|
|
|
|
type Dialer = netxlite.Dialer
|
|
|
|
|
|
|
|
// WrapDialer creates a new dialer that writes events
|
|
|
|
// into the given WritableDB. The net.Conns created by
|
|
|
|
// a wrapped dialer also write into the WritableDB.
|
|
|
|
func (mx *Measurer) WrapDialer(db WritableDB, dialer netxlite.Dialer) Dialer {
|
|
|
|
return WrapDialer(mx.Begin, db, dialer)
|
|
|
|
}
|
|
|
|
|
|
|
|
// WrapDialer wraps a dialer.
|
|
|
|
func WrapDialer(begin time.Time, db WritableDB, dialer netxlite.Dialer) Dialer {
|
|
|
|
return &dialerDB{Dialer: dialer, db: db, begin: begin}
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewDialerWithSystemResolver creates a
|
|
|
|
func (mx *Measurer) NewDialerWithSystemResolver(db WritableDB, logger Logger) Dialer {
|
|
|
|
r := mx.NewResolverSystem(db, logger)
|
|
|
|
return mx.WrapDialer(db, netxlite.NewDialerWithResolver(logger, r))
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewDialerWithoutResolver is a convenience factory for creating
|
|
|
|
// a dialer that saves measurements into the DB and that is not attached
|
|
|
|
// to any resolver (hence only works when passed IP addresses).
|
|
|
|
func (mx *Measurer) NewDialerWithoutResolver(db WritableDB, logger Logger) Dialer {
|
|
|
|
return mx.WrapDialer(db, netxlite.NewDialerWithoutResolver(logger))
|
|
|
|
}
|
|
|
|
|
|
|
|
type dialerDB struct {
|
|
|
|
netxlite.Dialer
|
|
|
|
begin time.Time
|
|
|
|
db WritableDB
|
|
|
|
}
|
|
|
|
|
|
|
|
// NetworkEvent contains a network event. This kind of events
|
|
|
|
// are generated by Dialer, QUICDialer, Conn, QUICConn.
|
|
|
|
type NetworkEvent struct {
|
2021-11-05 10:46:45 +01:00
|
|
|
RemoteAddr string
|
|
|
|
Failure *string
|
|
|
|
Count int
|
|
|
|
Operation string
|
|
|
|
Network string
|
|
|
|
Oddity Oddity
|
|
|
|
Finished float64
|
|
|
|
Started float64
|
feat(measurex): refactored measurement library (#528)
This commit introduce a measurement library that consists of
refactored code from earlier websteps experiments.
I am not going to add tests for the time being, because this library
is still a bit in flux, as we finalize websteps.
I will soon though commit documentation explaining in detail how
to use it, which currrently is at https://github.com/ooni/probe-cli/pull/506
and adds a new directory to internal/tutorial.
The core idea of this measurement library is to allow two
measurement modes:
1. tracing, which is what we're currently doing now, and the
tutorial shows how we can rewrite the measurement part of web
connectivity with measurex using less code. Under a tracing
approach, we construct a normal http.Client that however has
tracing configured, we gather events for resolve, connect, TLS
handshake, QUIC handshake, HTTP round trip, etc. and then we
try to make sense of what happened from the events stream;
2. step-by-step, which is what websteps does, and basically
means that after each operation you immediately write into
a Measurement structure its results and immediately draw the
conclusions on what seems odd (which later may become an
anomaly if we see what the test helper measured).
This library is also such that it produces a data format
compatible with the current OONI spec.
This work is part of https://github.com/ooni/probe/issues/1733.
2021-09-30 01:24:08 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func (d *dialerDB) DialContext(
|
|
|
|
ctx context.Context, network, address string) (Conn, error) {
|
|
|
|
started := time.Since(d.begin).Seconds()
|
|
|
|
conn, err := d.Dialer.DialContext(ctx, network, address)
|
|
|
|
finished := time.Since(d.begin).Seconds()
|
|
|
|
d.db.InsertIntoDial(&NetworkEvent{
|
|
|
|
Operation: "connect",
|
|
|
|
Network: network,
|
|
|
|
RemoteAddr: address,
|
|
|
|
Started: started,
|
|
|
|
Finished: finished,
|
2021-11-05 10:46:45 +01:00
|
|
|
Failure: NewFailure(err),
|
feat(measurex): refactored measurement library (#528)
This commit introduce a measurement library that consists of
refactored code from earlier websteps experiments.
I am not going to add tests for the time being, because this library
is still a bit in flux, as we finalize websteps.
I will soon though commit documentation explaining in detail how
to use it, which currrently is at https://github.com/ooni/probe-cli/pull/506
and adds a new directory to internal/tutorial.
The core idea of this measurement library is to allow two
measurement modes:
1. tracing, which is what we're currently doing now, and the
tutorial shows how we can rewrite the measurement part of web
connectivity with measurex using less code. Under a tracing
approach, we construct a normal http.Client that however has
tracing configured, we gather events for resolve, connect, TLS
handshake, QUIC handshake, HTTP round trip, etc. and then we
try to make sense of what happened from the events stream;
2. step-by-step, which is what websteps does, and basically
means that after each operation you immediately write into
a Measurement structure its results and immediately draw the
conclusions on what seems odd (which later may become an
anomaly if we see what the test helper measured).
This library is also such that it produces a data format
compatible with the current OONI spec.
This work is part of https://github.com/ooni/probe/issues/1733.
2021-09-30 01:24:08 +02:00
|
|
|
Oddity: d.computeOddity(err),
|
|
|
|
Count: 0,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return &connDB{
|
|
|
|
Conn: conn,
|
|
|
|
begin: d.begin,
|
|
|
|
db: d.db,
|
|
|
|
network: network,
|
|
|
|
remoteAddr: address,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *dialerDB) computeOddity(err error) Oddity {
|
|
|
|
if err == nil {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
switch err.Error() {
|
|
|
|
case netxlite.FailureGenericTimeoutError:
|
|
|
|
return OddityTCPConnectTimeout
|
|
|
|
case netxlite.FailureConnectionRefused:
|
|
|
|
return OddityTCPConnectRefused
|
|
|
|
case netxlite.FailureHostUnreachable:
|
|
|
|
return OddityTCPConnectHostUnreachable
|
|
|
|
default:
|
|
|
|
return OddityTCPConnectOher
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
type connDB struct {
|
|
|
|
net.Conn
|
|
|
|
begin time.Time
|
|
|
|
db WritableDB
|
|
|
|
network string
|
|
|
|
remoteAddr string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *connDB) Read(b []byte) (int, error) {
|
|
|
|
started := time.Since(c.begin).Seconds()
|
|
|
|
count, err := c.Conn.Read(b)
|
|
|
|
finished := time.Since(c.begin).Seconds()
|
|
|
|
c.db.InsertIntoReadWrite(&NetworkEvent{
|
|
|
|
Operation: "read",
|
|
|
|
Network: c.network,
|
|
|
|
RemoteAddr: c.remoteAddr,
|
|
|
|
Started: started,
|
|
|
|
Finished: finished,
|
2021-11-05 10:46:45 +01:00
|
|
|
Failure: NewFailure(err),
|
feat(measurex): refactored measurement library (#528)
This commit introduce a measurement library that consists of
refactored code from earlier websteps experiments.
I am not going to add tests for the time being, because this library
is still a bit in flux, as we finalize websteps.
I will soon though commit documentation explaining in detail how
to use it, which currrently is at https://github.com/ooni/probe-cli/pull/506
and adds a new directory to internal/tutorial.
The core idea of this measurement library is to allow two
measurement modes:
1. tracing, which is what we're currently doing now, and the
tutorial shows how we can rewrite the measurement part of web
connectivity with measurex using less code. Under a tracing
approach, we construct a normal http.Client that however has
tracing configured, we gather events for resolve, connect, TLS
handshake, QUIC handshake, HTTP round trip, etc. and then we
try to make sense of what happened from the events stream;
2. step-by-step, which is what websteps does, and basically
means that after each operation you immediately write into
a Measurement structure its results and immediately draw the
conclusions on what seems odd (which later may become an
anomaly if we see what the test helper measured).
This library is also such that it produces a data format
compatible with the current OONI spec.
This work is part of https://github.com/ooni/probe/issues/1733.
2021-09-30 01:24:08 +02:00
|
|
|
Count: count,
|
|
|
|
})
|
|
|
|
return count, err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *connDB) Write(b []byte) (int, error) {
|
|
|
|
started := time.Since(c.begin).Seconds()
|
|
|
|
count, err := c.Conn.Write(b)
|
|
|
|
finished := time.Since(c.begin).Seconds()
|
|
|
|
c.db.InsertIntoReadWrite(&NetworkEvent{
|
|
|
|
Operation: "write",
|
|
|
|
Network: c.network,
|
|
|
|
RemoteAddr: c.remoteAddr,
|
|
|
|
Started: started,
|
|
|
|
Finished: finished,
|
2021-11-05 10:46:45 +01:00
|
|
|
Failure: NewFailure(err),
|
feat(measurex): refactored measurement library (#528)
This commit introduce a measurement library that consists of
refactored code from earlier websteps experiments.
I am not going to add tests for the time being, because this library
is still a bit in flux, as we finalize websteps.
I will soon though commit documentation explaining in detail how
to use it, which currrently is at https://github.com/ooni/probe-cli/pull/506
and adds a new directory to internal/tutorial.
The core idea of this measurement library is to allow two
measurement modes:
1. tracing, which is what we're currently doing now, and the
tutorial shows how we can rewrite the measurement part of web
connectivity with measurex using less code. Under a tracing
approach, we construct a normal http.Client that however has
tracing configured, we gather events for resolve, connect, TLS
handshake, QUIC handshake, HTTP round trip, etc. and then we
try to make sense of what happened from the events stream;
2. step-by-step, which is what websteps does, and basically
means that after each operation you immediately write into
a Measurement structure its results and immediately draw the
conclusions on what seems odd (which later may become an
anomaly if we see what the test helper measured).
This library is also such that it produces a data format
compatible with the current OONI spec.
This work is part of https://github.com/ooni/probe/issues/1733.
2021-09-30 01:24:08 +02:00
|
|
|
Count: count,
|
|
|
|
})
|
|
|
|
return count, err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *connDB) Close() error {
|
|
|
|
started := time.Since(c.begin).Seconds()
|
|
|
|
err := c.Conn.Close()
|
|
|
|
finished := time.Since(c.begin).Seconds()
|
|
|
|
c.db.InsertIntoClose(&NetworkEvent{
|
|
|
|
Operation: "close",
|
|
|
|
Network: c.network,
|
|
|
|
RemoteAddr: c.remoteAddr,
|
|
|
|
Started: started,
|
|
|
|
Finished: finished,
|
2021-11-05 10:46:45 +01:00
|
|
|
Failure: NewFailure(err),
|
feat(measurex): refactored measurement library (#528)
This commit introduce a measurement library that consists of
refactored code from earlier websteps experiments.
I am not going to add tests for the time being, because this library
is still a bit in flux, as we finalize websteps.
I will soon though commit documentation explaining in detail how
to use it, which currrently is at https://github.com/ooni/probe-cli/pull/506
and adds a new directory to internal/tutorial.
The core idea of this measurement library is to allow two
measurement modes:
1. tracing, which is what we're currently doing now, and the
tutorial shows how we can rewrite the measurement part of web
connectivity with measurex using less code. Under a tracing
approach, we construct a normal http.Client that however has
tracing configured, we gather events for resolve, connect, TLS
handshake, QUIC handshake, HTTP round trip, etc. and then we
try to make sense of what happened from the events stream;
2. step-by-step, which is what websteps does, and basically
means that after each operation you immediately write into
a Measurement structure its results and immediately draw the
conclusions on what seems odd (which later may become an
anomaly if we see what the test helper measured).
This library is also such that it produces a data format
compatible with the current OONI spec.
This work is part of https://github.com/ooni/probe/issues/1733.
2021-09-30 01:24:08 +02:00
|
|
|
Count: 0,
|
|
|
|
})
|
|
|
|
return err
|
|
|
|
}
|