c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(datafmt): use "udp" instead of "quic" (#946)

Browse Source 
This diff changes the data format to prefer "udp" to "quic" everywhere we were previously using "quic".

Previously, the code inconsistently used "quic" for operations where we knew we were using "quic" and "udp" otherwise (e.g., for generic operations like ReadFrom).

While it would be more correct to say that a specific HTTP request used "quic" rather than "udp", using "udp" consistently allows one to see how distinct events such as ReadFrom and an handshake all refer to the same address, port, and protocol triple. Therefore, this change makes it easier to programmatically unpack a single measurement and create endpoint stats.

Before implementing this change, I discussed the problem with @hellais who mentioned that ooni/data is not currently using the "quic" string anywhere. I know that ooni/pipeline also doesn't rely on this string. The only users of this feature have been research-oriented experiments such as urlgetter, for which such a change would actually be acceptable.

See https://github.com/ooni/probe/issues/2238 and https://github.com/ooni/spec/pull/262.
This commit is contained in:
Simone Basso 2022-09-08 17:19:59 +02:00 committed by GitHub
parent 800217d15b
commit b78b9aca51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 66 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/engine/netx/http.go
View File

@ -55,7 +55,7 @@ var allTransportsInfo = map[bool]httpTransportInfo{
},
true: {
Factory: newHTTP3Transport,
TransportName: "quic",
TransportName: "udp",
},
}

8
internal/measurex/endpoint.go
View File

@ -19,13 +19,13 @@ const (
// NetworkTCP identifies endpoints using TCP.
NetworkTCP = EndpointNetwork("tcp")
// NetworkQUIC identifies endpoints using QUIC.
NetworkQUIC = EndpointNetwork("quic")
// NetworkUDP identifies endpoints using UDP.
NetworkUDP = EndpointNetwork("udp")
)
// Endpoint is an endpoint for a domain.
type Endpoint struct {
// Network is the network (e.g., "tcp", "quic")
// Network is the network (e.g., "tcp", "udp")
Network EndpointNetwork
// Address is the endpoint address (e.g., "8.8.8.8:443")
@ -42,7 +42,7 @@ type HTTPEndpoint struct {
// Domain is the endpoint domain (e.g., "dns.google").
Domain string
// Network is the network (e.g., "tcp" or "quic").
// Network is the network (e.g., "tcp" or "udp").
Network EndpointNetwork
// Address is the endpoint address (e.g., "8.8.8.8:443").

4
internal/measurex/measurement.go
View File

@ -156,7 +156,7 @@ func (m *DNSMeasurement) allQUICEndpoints(domain, port string) (out []*Endpoint)
continue
}
for _, addr := range entry.Addrs() {
out = append(out, m.newEndpoint(addr, port, NetworkQUIC))
out = append(out, m.newEndpoint(addr, port, NetworkUDP))
}
}
return
@ -186,7 +186,7 @@ func (m *DNSMeasurement) allHTTPEndpointsForURL(
epnts := m.allEndpointsForDomain(domain, port)
var out []*HTTPEndpoint
for _, epnt := range epnts {
if URL.Scheme != "https" && epnt.Network == NetworkQUIC {
if URL.Scheme != "https" && epnt.Network == NetworkUDP {
continue // we'll only use QUIC with HTTPS
}
out = append(out, &HTTPEndpoint{

8
internal/measurex/measurer.go
View File

@ -394,7 +394,7 @@ func (mx *Measurer) QUICHandshake(ctx context.Context, address string,
qconn.CloseWithError(0, "")
}
return &EndpointMeasurement{
Network: NetworkQUIC,
Network: NetworkUDP,
Address: address,
Measurement: measurement,
}
@ -502,7 +502,7 @@ func (mx *Measurer) httpEndpointGetMeasurement(ctx context.Context, epnt *HTTPEn
func (mx *Measurer) HTTPEndpointGetWithDB(ctx context.Context, epnt *HTTPEndpoint,
db WritableDB, jar http.CookieJar) (err error) {
switch epnt.Network {
case NetworkQUIC:
case NetworkUDP:
_, err = mx.httpEndpointGetQUIC(ctx, db, epnt, jar)
case NetworkTCP:
_, err = mx.httpEndpointGetTCP(ctx, db, epnt, jar)
@ -517,7 +517,7 @@ func (mx *Measurer) HTTPEndpointGetWithDB(ctx context.Context, epnt *HTTPEndpoin
func (mx *Measurer) httpEndpointGetWithDB(ctx context.Context, epnt *HTTPEndpoint,
db WritableDB, jar http.CookieJar) (resp *http.Response, err error) {
switch epnt.Network {
case NetworkQUIC:
case NetworkUDP:
resp, err = mx.httpEndpointGetQUIC(ctx, db, epnt, jar)
case NetworkTCP:
resp, err = mx.httpEndpointGetTCP(ctx, db, epnt, jar)
@ -971,7 +971,7 @@ func (mx *Measurer) doQUICFollowUp(ctx context.Context, parallelism int,
for _, epnt := range epnts {
quicEpnts = append(quicEpnts, &HTTPEndpoint{
Domain: epnt.Domain,
Network: NetworkQUIC,
Network: NetworkUDP,
Address: epnt.Address,
SNI: epnt.SNI,
ALPN: []string{"h3"},

8
internal/measurex/quic.go
View File

@ -47,7 +47,7 @@ func (c *udpLikeConnDB) WriteTo(p []byte, addr net.Addr) (int, error) {
finished := time.Since(c.begin).Seconds()
c.db.InsertIntoReadWrite(&NetworkEvent{
Operation: "write_to",
Network: "quic",
Network: "udp",
RemoteAddr: addr.String(),
Started: started,
Finished: finished,
@ -63,7 +63,7 @@ func (c *udpLikeConnDB) ReadFrom(b []byte) (int, net.Addr, error) {
finished := time.Since(c.begin).Seconds()
c.db.InsertIntoReadWrite(&NetworkEvent{
Operation: "read_from",
Network: "quic",
Network: "udp",
RemoteAddr: addrStringIfNotNil(addr),
Started: started,
Finished: finished,
@ -79,7 +79,7 @@ func (c *udpLikeConnDB) Close() error {
finished := time.Since(c.begin).Seconds()
c.db.InsertIntoClose(&NetworkEvent{
Operation: "close",
Network: "quic",
Network: "udp",
RemoteAddr: "",
Started: started,
Finished: finished,
@ -123,7 +123,7 @@ func (qh *quicDialerDB) DialContext(ctx context.Context, address string,
}
finished := time.Since(qh.begin).Seconds()
qh.db.InsertIntoQUICHandshake(&QUICTLSHandshakeEvent{
Network: "quic",
Network: "udp",
RemoteAddr: address,
SNI: tlsConfig.ServerName,
ALPN: tlsConfig.NextProtos,

2
internal/measurex/utils.go
View File

@ -16,7 +16,7 @@ import (
// given the network. On failure, we return a nil list.
func ALPNForHTTPEndpoint(network EndpointNetwork) []string {
switch network {
case NetworkQUIC:
case NetworkUDP:
return []string{"h3"}
case NetworkTCP:
return []string{"h2", "http/1.1"}

2
internal/measurexlite/http.go
View File

@ -27,7 +27,7 @@ import (
//
// - alpn is the negotiated ALPN or an empty string when not applicable;
//
// - transport is the HTTP transport's protocol we're using ("quic" or "tcp"): this field
// - transport is the HTTP transport's protocol we're using ("udp" or "tcp"): this field
// was introduced a long time ago to support QUIC measurements and we keep it for backwards
// compatibility but network, address, and alpn are much more informative;
//

8
internal/measurexlite/http_test.go
View File

@ -155,7 +155,7 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) {
network: "udp",
address: "8.8.8.8:443",
alpn: "h3",
transport: "quic",
transport: "udp",
req: &http.Request{
Method: "GET",
URL: &url.URL{
@ -205,7 +205,7 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) {
},
Method: "GET",
Tor: model.ArchivalHTTPTor{},
Transport: "quic",
Transport: "udp",
URL: "https://dns.google/",
},
Response: model.ArchivalHTTPResponse{
@ -243,7 +243,7 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) {
network: "udp",
address: "8.8.8.8:443",
alpn: "h3",
transport: "quic",
transport: "udp",
req: &http.Request{
Method: "GET",
URL: &url.URL{
@ -301,7 +301,7 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) {
},
Method: "GET",
Tor: model.ArchivalHTTPTor{},
Transport: "quic",
Transport: "udp",
URL: "https://dns.google/",
},
Response: model.ArchivalHTTPResponse{

2
internal/measurexlite/quic.go
View File

@ -64,7 +64,7 @@ func (tx *Trace) OnQUICHandshakeDone(started time.Time, remoteAddr string, qconn
case tx.quicHandshake <- NewArchivalTLSOrQUICHandshakeResult(
tx.Index,
started.Sub(tx.ZeroTime),
"quic",
"udp",
remoteAddr,
config,
state,

6
internal/measurexlite/quic_test.go
View File

@ -137,7 +137,7 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) {
}
expectedFailure := "unknown_failure: mocked"
expect := &model.ArchivalTLSOrQUICHandshakeResult{
Network: "quic",
Network: "udp",
Address: "1.1.1.1:443",
CipherSuite: "",
Failure: &expectedFailure,
@ -275,7 +275,7 @@ func TestFirstQUICHandshake(t *testing.T) {
zeroTime := time.Now()
trace := NewTrace(0, zeroTime)
expect := []*model.ArchivalTLSOrQUICHandshakeResult{{
Network: "quic",
Network: "udp",
Address: "1.1.1.1:443",
CipherSuite: "",
Failure: nil,
@ -287,7 +287,7 @@ func TestFirstQUICHandshake(t *testing.T) {
Tags: []string{},
TLSVersion: "",
}, {
Network: "quic",
Network: "udp",
Address: "8.8.8.8:443",
CipherSuite: "",
Failure: nil,

2
internal/model/mocks/http_test.go
View File

@ -10,7 +10,7 @@ import (
func TestHTTPTransport(t *testing.T) {
t.Run("Network", func(t *testing.T) {
expected := "quic"
expected := "udp"
txp := &HTTPTransport{
MockNetwork: func() string {
return expected

2
internal/model/netx.go
View File

@ -158,7 +158,7 @@ type HTTPClient interface {
// HTTPTransport is an http.Transport-like structure.
type HTTPTransport interface {
// Network returns the network used by the transport, which
// should be one of "tcp" and "quic".
// should be one of "tcp" and "udp".
Network() string
// RoundTrip performs the HTTP round trip.

2
internal/netxlite/http3.go
View File

@ -29,7 +29,7 @@ var _ model.HTTPTransport = &http3Transport{}
// Network implements HTTPTransport.Network.
func (txp *http3Transport) Network() string {
return "quic"
return "udp"
}
// RoundTrip implements HTTPTransport.RoundTrip.

2
internal/netxlite/http3_test.go
View File

@ -39,7 +39,7 @@ func TestHTTP3Transport(t *testing.T) {
t.Run("Network", func(t *testing.T) {
txp := &http3Transport{}
if txp.Network() != "quic" {
if txp.Network() != "udp" {
t.Fatal("unexpected .Network return value")
}
})

2
internal/tracex/archival_test.go
View File

@ -564,7 +564,7 @@ func TestNewTLSHandshakesList(t *testing.T) {
Address: "131.252.210.176:443",
Err: netxlite.FailureEOFError,
NoTLSVerify: false,
Proto: "quic",
Proto: "udp",
TLSCipherSuite: "SUITE",
TLSNegotiatedProto: "h3",
TLSPeerCerts: [][]byte{

32
internal/tutorial/measurex/chapter05/README.md
View File

@ -89,12 +89,12 @@ Produces this JSON:
```JavaScript
{
// In chapter02 these two fields were similar but
// the network was "tcp" as opposed to "quic"
"network": "quic",
// the network was "tcp" as opposed to "udp"
"network": "udp",
"address": "8.8.4.4:443",
// This block contains I/O operations. Note that
// the protocol is "quic" and that the syscalls
// the protocol is "udp" and that the syscalls
// are "read_from" and "write_to" because QUIC does
// not bind/connect sockets. (The real syscalls
// are actually `recvfrom` and `sendto` but here
@ -106,7 +106,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 1252,
"operation": "write_to",
"proto": "quic",
"proto": "udp",
"t": 0.027184208,
"started": 0.027127208,
"oddity": ""
@ -116,7 +116,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 1252,
"operation": "read_from",
"proto": "quic",
"proto": "udp",
"t": 0.053116458,
"started": 0.025626583,
"oddity": ""
@ -126,7 +126,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 1252,
"operation": "write_to",
"proto": "quic",
"proto": "udp",
"t": 0.054538792,
"started": 0.054517542,
"oddity": ""
@ -136,7 +136,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 1252,
"operation": "read_from",
"proto": "quic",
"proto": "udp",
"t": 0.069144958,
"started": 0.053194208,
"oddity": ""
@ -146,7 +146,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 1252,
"operation": "read_from",
"proto": "quic",
"proto": "udp",
"t": 0.069183458,
"started": 0.069173292,
"oddity": ""
@ -156,7 +156,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 1252,
"operation": "read_from",
"proto": "quic",
"proto": "udp",
"t": 0.06920225,
"started": 0.069197875,
"oddity": ""
@ -166,7 +166,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 1216,
"operation": "read_from",
"proto": "quic",
"proto": "udp",
"t": 0.069210958,
"started": 0.069206875,
"oddity": ""
@ -176,7 +176,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 65,
"operation": "read_from",
"proto": "quic",
"proto": "udp",
"t": 0.069220667,
"started": 0.069217375,
"oddity": ""
@ -186,7 +186,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 44,
"operation": "write_to",
"proto": "quic",
"proto": "udp",
"t": 0.069433417,
"started": 0.069417625,
"oddity": ""
@ -196,7 +196,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 44,
"operation": "write_to",
"proto": "quic",
"proto": "udp",
"t": 0.069677625,
"started": 0.069647458,
"oddity": ""
@ -206,7 +206,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 83,
"operation": "write_to",
"proto": "quic",
"proto": "udp",
"t": 0.073461917,
"started": 0.073432875,
"oddity": ""
@ -216,7 +216,7 @@ Produces this JSON:
"failure": null,
"num_bytes": 33,
"operation": "write_to",
"proto": "quic",
"proto": "udp",
"t": 0.073559417,
"started": 0.073542542,
"oddity": ""
@ -253,7 +253,7 @@ Produces this JSON:
],
"no_tls_verify": false,
"oddity": "",
"proto": "quic",
"proto": "udp",
"started": 0.025061583
}
]

32
internal/tutorial/measurex/chapter05/main.go
View File

@ -90,12 +90,12 @@ func main() {
// ```JavaScript
// {
// // In chapter02 these two fields were similar but
// // the network was "tcp" as opposed to "quic"
// "network": "quic",
// // the network was "tcp" as opposed to "udp"
// "network": "udp",
// "address": "8.8.4.4:443",
//
// // This block contains I/O operations. Note that
// // the protocol is "quic" and that the syscalls
// // the protocol is "udp" and that the syscalls
// // are "read_from" and "write_to" because QUIC does
// // not bind/connect sockets. (The real syscalls
// // are actually `recvfrom` and `sendto` but here
@ -107,7 +107,7 @@ func main() {
// "failure": null,
// "num_bytes": 1252,
// "operation": "write_to",
// "proto": "quic",
// "proto": "udp",
// "t": 0.027184208,
// "started": 0.027127208,
// "oddity": ""
@ -117,7 +117,7 @@ func main() {
// "failure": null,
// "num_bytes": 1252,
// "operation": "read_from",
// "proto": "quic",
// "proto": "udp",
// "t": 0.053116458,
// "started": 0.025626583,
// "oddity": ""
@ -127,7 +127,7 @@ func main() {
// "failure": null,
// "num_bytes": 1252,
// "operation": "write_to",
// "proto": "quic",
// "proto": "udp",
// "t": 0.054538792,
// "started": 0.054517542,
// "oddity": ""
@ -137,7 +137,7 @@ func main() {
// "failure": null,
// "num_bytes": 1252,
// "operation": "read_from",
// "proto": "quic",
// "proto": "udp",
// "t": 0.069144958,
// "started": 0.053194208,
// "oddity": ""
@ -147,7 +147,7 @@ func main() {
// "failure": null,
// "num_bytes": 1252,
// "operation": "read_from",
// "proto": "quic",
// "proto": "udp",
// "t": 0.069183458,
// "started": 0.069173292,
// "oddity": ""
@ -157,7 +157,7 @@ func main() {
// "failure": null,
// "num_bytes": 1252,
// "operation": "read_from",
// "proto": "quic",
// "proto": "udp",
// "t": 0.06920225,
// "started": 0.069197875,
// "oddity": ""
@ -167,7 +167,7 @@ func main() {
// "failure": null,
// "num_bytes": 1216,
// "operation": "read_from",
// "proto": "quic",
// "proto": "udp",
// "t": 0.069210958,
// "started": 0.069206875,
// "oddity": ""
@ -177,7 +177,7 @@ func main() {
// "failure": null,
// "num_bytes": 65,
// "operation": "read_from",
// "proto": "quic",
// "proto": "udp",
// "t": 0.069220667,
// "started": 0.069217375,
// "oddity": ""
@ -187,7 +187,7 @@ func main() {
// "failure": null,
// "num_bytes": 44,
// "operation": "write_to",
// "proto": "quic",
// "proto": "udp",
// "t": 0.069433417,
// "started": 0.069417625,
// "oddity": ""
@ -197,7 +197,7 @@ func main() {
// "failure": null,
// "num_bytes": 44,
// "operation": "write_to",
// "proto": "quic",
// "proto": "udp",
// "t": 0.069677625,
// "started": 0.069647458,
// "oddity": ""
@ -207,7 +207,7 @@ func main() {
// "failure": null,
// "num_bytes": 83,
// "operation": "write_to",
// "proto": "quic",
// "proto": "udp",
// "t": 0.073461917,
// "started": 0.073432875,
// "oddity": ""
@ -217,7 +217,7 @@ func main() {
// "failure": null,
// "num_bytes": 33,
// "operation": "write_to",
// "proto": "quic",
// "proto": "udp",
// "t": 0.073559417,
// "started": 0.073542542,
// "oddity": ""
@ -254,7 +254,7 @@ func main() {
// ],
// "no_tls_verify": false,
// "oddity": "",
// "proto": "quic",
// "proto": "udp",
// "started": 0.025061583
// }
// ]

2
internal/tutorial/measurex/chapter06/README.md
View File

@ -97,7 +97,7 @@ to need more information to characterize the endpoint.
In fact, in the above definition we recognize fields
we have already discussed, such as:
- `Network`, describing whether to use "tcp" or "quic";
- `Network`, describing whether to use "tcp" or "udp";
- `Address`, which is the endpoint address.

2
internal/tutorial/measurex/chapter06/main.go
View File

@ -98,7 +98,7 @@ func main() {
// In fact, in the above definition we recognize fields
// we have already discussed, such as:
//
// - `Network`, describing whether to use "tcp" or "quic";
// - `Network`, describing whether to use "tcp" or "udp";
//
// - `Address`, which is the endpoint address.
//

2
internal/tutorial/measurex/chapter08/README.md
View File

@ -86,7 +86,7 @@ A/AAAA lookups results but also the HTTPS lookup results.
The `AllHTTPEndpointsForURL` function will recognize that
we also have HTTPS lookups and, if the "h3" ALPN is
present, will _also_ build HTTP/3 endpoints using "quic"
present, will _also_ build HTTP/3 endpoints using "udp"
as the `HTTPEndpoint.Network`.
```Go

2
internal/tutorial/measurex/chapter08/main.go
View File

@ -87,7 +87,7 @@ func main() {
//
// The `AllHTTPEndpointsForURL` function will recognize that
// we also have HTTPS lookups and, if the "h3" ALPN is
// present, will _also_ build HTTP/3 endpoints using "quic"
// present, will _also_ build HTTP/3 endpoints using "udp"
// as the `HTTPEndpoint.Network`.
//
// ```Go