c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(measurex): use same keys of the OONI data format (#572)

Browse Source 
This change should simplify the pipeline's job.

Reference issue: https://github.com/ooni/probe/issues/1817.

I previously dismissed this possibility, but now it seems clear it
is simpler to have a very tabular data format internally and to
convert such a format to OONI's data format when serializing.

The OONI data format is what the pipeline expects, but processing
is easier with a more linear/tabular format.
This commit is contained in:
Simone Basso 2021-11-05 10:46:45 +01:00 committed by GitHub
parent 6f90d29bfa
commit aa27bbe33f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 1571 additions and 1025 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/engine/experiment/webstepsx/measurer.go
View File

@ -29,7 +29,7 @@ type Config struct{}
// TestKeys contains the experiment's test keys. // TestKeys contains the experiment's test keys.
type TestKeys struct { type TestKeys struct {
*measurex.URLMeasurement *measurex.ArchivalURLMeasurement
} }
// Measurer performs the measurement. // Measurer performs the measurement.
@ -142,7 +142,9 @@ func (mx *Measurer) runAsync(ctx context.Context, sess model.ExperimentSession,
}, },
Input: model.MeasurementTarget(m.URL), Input: model.MeasurementTarget(m.URL),
MeasurementRuntime: m.TotalRuntime.Seconds(), MeasurementRuntime: m.TotalRuntime.Seconds(),
TestKeys: &TestKeys{URLMeasurement: m}, TestKeys: &TestKeys{
ArchivalURLMeasurement: measurex.NewArchivalURLMeasurement(m),
},
} }
} }
} }
@ -163,7 +165,7 @@ type measurerMeasureURLHelper struct {
func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints( func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header, ctx context.Context, URL *url.URL, headers http.Header,
curEndpoints ...*measurex.HTTPEndpoint) ( curEndpoints ...*measurex.HTTPEndpoint) (
[]*measurex.HTTPEndpoint, interface{}, error) { []*measurex.HTTPEndpoint, *measurex.THMeasurement, error) {
cc := &THClientCall{ cc := &THClientCall{
Endpoints: measurex.HTTPEndpointsToEndpoints(curEndpoints), Endpoints: measurex.HTTPEndpointsToEndpoints(curEndpoints),
HTTPClient: mth.Clnt, HTTPClient: mth.Clnt,

49
internal/engine/experiment/webstepsx/th.go
View File

@ -41,14 +41,7 @@ type THClientRequest struct {
} }
// THServerResponse is the response from the test helper. // THServerResponse is the response from the test helper.
type THServerResponse struct { type THServerResponse = measurex.THMeasurement
// DNS contains all the DNS related measurements.
DNS []*measurex.DNSMeasurement `json:"dns"`
// Endpoints contains a measurement for each endpoint
// that was discovered by the probe or the TH.
Endpoints []*measurex.HTTPEndpointMeasurement `json:"endpoints"`
}
// thMaxAcceptableBodySize is the maximum acceptable body size by TH code. // thMaxAcceptableBodySize is the maximum acceptable body size by TH code.
const thMaxAcceptableBodySize = 1 << 20 const thMaxAcceptableBodySize = 1 << 20
@ -294,9 +287,9 @@ func (h *THHandler) simplifyMeasurement(in *measurex.Measurement) (out *measurex
} }
func (h *THHandler) simplifyHandshake( func (h *THHandler) simplifyHandshake(
in []*measurex.TLSHandshakeEvent) (out []*measurex.TLSHandshakeEvent) { in []*measurex.QUICTLSHandshakeEvent) (out []*measurex.QUICTLSHandshakeEvent) {
for _, ev := range in { for _, ev := range in {
out = append(out, &measurex.TLSHandshakeEvent{ out = append(out, &measurex.QUICTLSHandshakeEvent{
CipherSuite: ev.CipherSuite, CipherSuite: ev.CipherSuite,
Failure: ev.Failure, Failure: ev.Failure,
NegotiatedProto: ev.NegotiatedProto, NegotiatedProto: ev.NegotiatedProto,
@ -319,32 +312,24 @@ func (h *THHandler) simplifyHTTPRoundTrip(
in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) { in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) {
for _, ev := range in { for _, ev := range in {
out = append(out, &measurex.HTTPRoundTripEvent{ out = append(out, &measurex.HTTPRoundTripEvent{
Failure: ev.Failure, Failure: ev.Failure,
Request: ev.Request, Method: ev.Method,
Response: h.simplifyHTTPResponse(ev.Response), URL: ev.URL,
Finished: 0, RequestHeaders: ev.RequestHeaders,
Started: 0, StatusCode: ev.StatusCode,
Oddity: ev.Oddity, ResponseHeaders: ev.ResponseHeaders,
ResponseBody: nil, // we don't transfer the body
ResponseBodyLength: ev.ResponseBodyLength,
ResponseBodyIsTruncated: ev.ResponseBodyIsTruncated,
ResponseBodyIsUTF8: ev.ResponseBodyIsUTF8,
Finished: ev.Finished,
Started: ev.Started,
Oddity: ev.Oddity,
}) })
} }
return return
} }
func (h *THHandler) simplifyHTTPResponse(
in *measurex.HTTPResponse) (out *measurex.HTTPResponse) {
if in != nil {
out = &measurex.HTTPResponse{
Code: in.Code,
Headers: in.Headers,
Body: nil,
BodyIsTruncated: in.BodyIsTruncated,
BodyLength: in.BodyLength,
BodyIsUTF8: in.BodyIsUTF8,
}
}
return
}
type thMeasureURLHelper struct { type thMeasureURLHelper struct {
epnts []*measurex.Endpoint epnts []*measurex.Endpoint
} }
@ -352,7 +337,7 @@ type thMeasureURLHelper struct {
func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints( func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header, ctx context.Context, URL *url.URL, headers http.Header,
serverEpnts ...*measurex.HTTPEndpoint) ( serverEpnts ...*measurex.HTTPEndpoint) (
epnts []*measurex.HTTPEndpoint, thMeaurement interface{}, err error) { epnts []*measurex.HTTPEndpoint, thMeaurement *measurex.THMeasurement, err error) {
for _, epnt := range thh.epnts { for _, epnt := range thh.epnts {
epnts = append(epnts, &measurex.HTTPEndpoint{ epnts = append(epnts, &measurex.HTTPEndpoint{
Domain: URL.Hostname(), Domain: URL.Hostname(),

533
internal/measurex/archival.go
View File

@ -1,8 +1,11 @@
package measurex package measurex
import ( import (
"net"
"net/http" "net/http"
"strconv"
"strings" "strings"
"time"
) )
// //
@ -11,6 +14,10 @@ import (
// This file defines helpers to serialize to the OONI data format. // This file defines helpers to serialize to the OONI data format.
// //
//
// BinaryData
//
// ArchivalBinaryData is the archival format for binary data. // ArchivalBinaryData is the archival format for binary data.
type ArchivalBinaryData struct { type ArchivalBinaryData struct {
Data []byte `json:"data"` Data []byte `json:"data"`
@ -29,6 +36,130 @@ func NewArchivalBinaryData(data []byte) (out *ArchivalBinaryData) {
return return
} }
//
// NetworkEvent
//
// ArchivalNetworkEvent is the OONI data format representation
// of a network event according to df-008-netevents.
type ArchivalNetworkEvent struct {
// JSON names compatible with df-008-netevents
RemoteAddr string `json:"address"`
Failure *string `json:"failure"`
Count int `json:"num_bytes,omitempty"`
Operation string `json:"operation"`
Network string `json:"proto"`
Finished float64 `json:"t"`
Started float64 `json:"started"`
// Names that are not part of the spec.
Oddity Oddity `json:"oddity"`
}
// NewArchivalNetworkEvent converts a network event to its archival format.
func NewArchivalNetworkEvent(in *NetworkEvent) *ArchivalNetworkEvent {
return &ArchivalNetworkEvent{
RemoteAddr: in.RemoteAddr,
Failure: in.Failure,
Count: in.Count,
Operation: in.Operation,
Network: in.Network,
Finished: in.Finished,
Started: in.Started,
Oddity: in.Oddity,
}
}
// NewArchivalNetworkEventList converts a list of NetworkEvent
// to a list of ArchivalNetworkEvent.
func NewArchivalNetworkEventList(in []*NetworkEvent) (out []*ArchivalNetworkEvent) {
for _, ev := range in {
out = append(out, NewArchivalNetworkEvent(ev))
}
return
}
//
// DNSRoundTripEvent
//
// ArchivalDNSRoundTripEvent is the OONI data format representation
// of a DNS round trip, which is currently not specified.
//
// We are trying to use names compatible with the names currently
// used by other specifications we currently use.
type ArchivalDNSRoundTripEvent struct {
Network string `json:"engine"`
Address string `json:"resolver_address"`
Query *ArchivalBinaryData `json:"raw_query"`
Started float64 `json:"started"`
Finished float64 `json:"t"`
Failure *string `json:"failure"`
Reply *ArchivalBinaryData `json:"raw_reply"`
}
// NewArchivalDNSRoundTripEvent converts a DNSRoundTripEvent into is archival format.
func NewArchivalDNSRoundTripEvent(in *DNSRoundTripEvent) *ArchivalDNSRoundTripEvent {
return &ArchivalDNSRoundTripEvent{
Network: in.Network,
Address: in.Address,
Query: NewArchivalBinaryData(in.Query),
Started: in.Started,
Finished: in.Finished,
Failure: in.Failure,
Reply: NewArchivalBinaryData(in.Reply),
}
}
// NewArchivalDNSRoundTripEventList converts a DNSRoundTripEvent
// list to the corresponding archival format.
func NewArchivalDNSRoundTripEventList(in []*DNSRoundTripEvent) (out []*ArchivalDNSRoundTripEvent) {
for _, ev := range in {
out = append(out, NewArchivalDNSRoundTripEvent(ev))
}
return
}
//
// HTTPRoundTrip
//
// ArchivalHTTPRequest is the archival format of an HTTP
// request according to df-001-http.md.
type ArchivalHTTPRequest struct {
Method string `json:"method"`
URL string `json:"url"`
Headers ArchivalHeaders `json:"headers"`
}
// ArchivalHTTPResponse is the archival format of an HTTP
// response according to df-001-http.md.
type ArchivalHTTPResponse struct {
// Names consistent with df-001-http.md
Code int64 `json:"code"`
Headers ArchivalHeaders `json:"headers"`
Body *ArchivalBinaryData `json:"body"`
BodyIsTruncated bool `json:"body_is_truncated"`
// Fields not part of the spec
BodyLength int64 `json:"x_body_length"`
BodyIsUTF8 bool `json:"x_body_is_utf8"`
}
// ArchivalHTTPRoundTripEvent is the archival format of an
// HTTP response according to df-001-http.md.
type ArchivalHTTPRoundTripEvent struct {
// JSON names following the df-001-httpt data format.
Failure *string `json:"failure"`
Request *HTTPRequest `json:"request"`
Response *HTTPResponse `json:"response"`
Finished float64 `json:"t"`
Started float64 `json:"started"`
// Names not in the specification
Oddity Oddity `json:"oddity"`
}
// ArchivalHeaders is a list of HTTP headers. // ArchivalHeaders is a list of HTTP headers.
type ArchivalHeaders map[string]string type ArchivalHeaders map[string]string
@ -54,6 +185,64 @@ func NewArchivalHeaders(in http.Header) (out ArchivalHeaders) {
return return
} }
// NewArchivalHTTPRoundTripEvent converts an HTTPRoundTrip to its archival format.
func NewArchivalHTTPRoundTripEvent(in *HTTPRoundTripEvent) *ArchivalHTTPRoundTripEvent {
return &ArchivalHTTPRoundTripEvent{
Failure: in.Failure,
Request: &HTTPRequest{
Method: in.Method,
URL: in.URL,
Headers: NewArchivalHeaders(in.RequestHeaders),
},
Response: &HTTPResponse{
Code: in.StatusCode,
Headers: NewArchivalHeaders(in.ResponseHeaders),
Body: NewArchivalBinaryData(in.ResponseBody),
BodyLength: in.ResponseBodyLength,
BodyIsTruncated: in.ResponseBodyIsTruncated,
BodyIsUTF8: in.ResponseBodyIsUTF8,
},
Finished: in.Finished,
Started: in.Started,
Oddity: in.Oddity,
}
}
// NewArchivalHTTPRoundTripEventList converts a list of
// HTTPRoundTripEvent to a list of ArchivalRoundTripEvent.
func NewArchivalHTTPRoundTripEventList(in []*HTTPRoundTripEvent) (out []*ArchivalHTTPRoundTripEvent) {
for _, ev := range in {
out = append(out, NewArchivalHTTPRoundTripEvent(ev))
}
return
}
//
// QUICTLSHandshakeEvent
//
// ArchivalQUICTLSHandshakeEvent is the archival data format for a
// QUIC or TLS handshake event according to df-006-tlshandshake.
type ArchivalQUICTLSHandshakeEvent struct {
// JSON names compatible with df-006-tlshandshake
CipherSuite string `json:"cipher_suite"`
Failure *string `json:"failure"`
NegotiatedProto string `json:"negotiated_proto"`
TLSVersion string `json:"tls_version"`
PeerCerts []*ArchivalBinaryData `json:"peer_certificates"`
Finished float64 `json:"t"`
// JSON names that are consistent with the
// spirit of the spec but are not in it
RemoteAddr string `json:"address"`
SNI string `json:"server_name"` // used in prod
ALPN []string `json:"alpn"`
SkipVerify bool `json:"no_tls_verify"` // used in prod
Oddity Oddity `json:"oddity"`
Network string `json:"proto"`
Started float64 `json:"started"`
}
// NewArchivalTLSCertList builds a new []ArchivalBinaryData // NewArchivalTLSCertList builds a new []ArchivalBinaryData
// from a list of raw x509 certificates data. // from a list of raw x509 certificates data.
func NewArchivalTLSCerts(in [][]byte) (out []*ArchivalBinaryData) { func NewArchivalTLSCerts(in [][]byte) (out []*ArchivalBinaryData) {
@ -66,13 +255,341 @@ func NewArchivalTLSCerts(in [][]byte) (out []*ArchivalBinaryData) {
return return
} }
// NewArchivalFailure creates an archival failure from an error. We // NewArchivalQUICTLSHandshakeEvent converts a QUICTLSHandshakeEvent
// cannot round trip an error using JSON, so we serialize to this // to its archival data format.
// intermediate format that is a sort of Optional<string>. func NewArchivalQUICTLSHandshakeEvent(in *QUICTLSHandshakeEvent) *ArchivalQUICTLSHandshakeEvent {
func NewArchivalFailure(err error) *string { return &ArchivalQUICTLSHandshakeEvent{
if err == nil { CipherSuite: in.CipherSuite,
return nil Failure: in.Failure,
NegotiatedProto: in.NegotiatedProto,
TLSVersion: in.TLSVersion,
PeerCerts: NewArchivalTLSCerts(in.PeerCerts),
Finished: in.Finished,
RemoteAddr: in.RemoteAddr,
SNI: in.SNI,
ALPN: in.ALPN,
SkipVerify: in.SkipVerify,
Oddity: in.Oddity,
Network: in.Network,
Started: in.Started,
} }
s := err.Error() }
return &s
// NewArchivalQUICTLSHandshakeEventList converts a list of
// QUICTLSHandshakeEvent to a list of ArchivalQUICTLSHandshakeEvent.
func NewArchivalQUICTLSHandshakeEventList(in []*QUICTLSHandshakeEvent) (out []*ArchivalQUICTLSHandshakeEvent) {
for _, ev := range in {
out = append(out, NewArchivalQUICTLSHandshakeEvent(ev))
}
return
}
//
// DNSLookup
//
// ArchivalDNSLookupAnswer is the archival format of a
// DNS lookup answer according to df-002-dnst.
type ArchivalDNSLookupAnswer struct {
// JSON names compatible with df-002-dnst's spec
Type string `json:"answer_type"`
IPv4 string `json:"ipv4,omitempty"`
IPv6 string `json:"ivp6,omitempty"`
// Names not part of the spec.
ALPN string `json:"alpn,omitempty"`
}
// ArchivalDNSLookupEvent is the archival data format
// of a DNS lookup according to df-002-dnst.
type ArchivalDNSLookupEvent struct {
// fields inside df-002-dnst
Answers []ArchivalDNSLookupAnswer `json:"answers"`
Network string `json:"engine"`
Failure *string `json:"failure"`
Domain string `json:"hostname"`
QueryType string `json:"query_type"`
Address string `json:"resolver_address"`
Finished float64 `json:"t"`
// Names not part of the spec.
Started float64 `json:"started"`
Oddity Oddity `json:"oddity"`
}
// NewArchivalDNSLookupAnswers creates a list of ArchivalDNSLookupAnswer.
func NewArchivalDNSLookupAnswers(in *DNSLookupEvent) (out []ArchivalDNSLookupAnswer) {
for _, ip := range in.A {
out = append(out, ArchivalDNSLookupAnswer{
Type: "A",
IPv4: ip,
})
}
for _, ip := range in.AAAA {
out = append(out, ArchivalDNSLookupAnswer{
Type: "AAAA",
IPv6: ip,
})
}
for _, alpn := range in.ALPN {
out = append(out, ArchivalDNSLookupAnswer{
Type: "ALPN",
ALPN: alpn,
})
}
return
}
// NewArchivalDNSLookupEvent converts a DNSLookupEvent
// to its archival representation.
func NewArchivalDNSLookupEvent(in *DNSLookupEvent) *ArchivalDNSLookupEvent {
return &ArchivalDNSLookupEvent{
Answers: NewArchivalDNSLookupAnswers(in),
Network: in.Network,
Failure: in.Failure,
Domain: in.Domain,
QueryType: in.QueryType,
Address: in.Address,
Finished: in.Finished,
Started: in.Started,
Oddity: in.Oddity,
}
}
// NewArchivalDNSLookupEventList converts a list of DNSLookupEvent
// to a list of ArchivalDNSLookupEvent.
func NewArchivalDNSLookupEventList(in []*DNSLookupEvent) (out []*ArchivalDNSLookupEvent) {
for _, ev := range in {
out = append(out, NewArchivalDNSLookupEvent(ev))
}
return
}
//
// TCPConnect
//
// ArchivalTCPConnect is the archival form of TCP connect
// events in compliance with df-005-tcpconnect.
type ArchivalTCPConnect struct {
// Names part of the spec.
IP string `json:"ip"`
Port int64 `json:"port"`
Finished float64 `json:"t"`
Status *ArchivalTCPConnectStatus `json:"status"`
// Names not part of the spec.
Started float64 `json:"started"`
Oddity Oddity `json:"oddity"`
}
// ArchivalTCPConnectStatus contains the status of a TCP connect.
type ArchivalTCPConnectStatus struct {
Blocked bool `json:"blocked"`
Failure *string `json:"failure"`
Success bool `json:"success"`
}
// NewArchivalTCPConnect converts a NetworkEvent to an ArchivalTCPConnect.
func NewArchivalTCPConnect(in *NetworkEvent) *ArchivalTCPConnect {
// We ignore errors because values come from Go code that
// emits correct serialization of TCP/UDP addresses.
addr, port, _ := net.SplitHostPort(in.RemoteAddr)
portnum, _ := strconv.Atoi(port)
return &ArchivalTCPConnect{
IP: addr,
Port: int64(portnum),
Finished: in.Finished,
Status: &ArchivalTCPConnectStatus{
Blocked: in.Failure != nil,
Failure: in.Failure,
Success: in.Failure == nil,
},
Started: in.Started,
Oddity: in.Oddity,
}
}
// NewArchivalTCPConnectList converts a list of NetworkEvent
// to a list of ArchivalTCPConnect. In doing that, the code
// only considers "connect" events using the TCP protocol.
func NewArchivalTCPConnectList(in []*NetworkEvent) (out []*ArchivalTCPConnect) {
for _, ev := range in {
if ev.Operation != "connect" {
continue
}
switch ev.Network {
case "tcp", "tcp4", "tcp6":
out = append(out, NewArchivalTCPConnect(ev))
default:
// nothing
}
}
return
}
//
// URLMeasurement
//
// ArchivalURLMeasurement is the archival representation of URLMeasurement
type ArchivalURLMeasurement struct {
URL string `json:"url"`
DNS []*ArchivalDNSMeasurement `json:"dns"`
Endpoints []*ArchivalHTTPEndpointMeasurement `json:"endpoints"`
TH *ArchivalTHMeasurement `json:"th"`
TotalRuntime time.Duration `json:"x_total_runtime"`
DNSRuntime time.Duration `json:"x_dns_runtime"`
THRuntime time.Duration `json:"x_th_runtime"`
EpntsRuntime time.Duration `json:"x_epnts_runtime"`
}
// NewArchivalURLMeasurement creates the archival representation
// of an URLMeasurement data structure.
func NewArchivalURLMeasurement(in *URLMeasurement) *ArchivalURLMeasurement {
return &ArchivalURLMeasurement{
URL: in.URL,
DNS: NewArchivalDNSMeasurementList(in.DNS),
Endpoints: NewArchivalHTTPEndpointMeasurementList(in.Endpoints),
TH: NewArchivalTHMeasurement(in.TH),
TotalRuntime: in.TotalRuntime,
DNSRuntime: in.DNSRuntime,
THRuntime: in.THRuntime,
EpntsRuntime: in.EpntsRuntime,
}
}
//
// EndpointMeasurement
//
// ArchivalEndpointMeasurement is the archival representation of EndpointMeasurement.
type ArchivalEndpointMeasurement struct {
// Network is the network of this endpoint.
Network EndpointNetwork `json:"network"`
// Address is the address of this endpoint.
Address string `json:"address"`
// An EndpointMeasurement is a Measurement.
*ArchivalMeasurement
}
// NewArchivalEndpointMeasurement converts an EndpointMeasurement
// to the corresponding archival data format.
func NewArchivalEndpointMeasurement(in *EndpointMeasurement) *ArchivalEndpointMeasurement {
return &ArchivalEndpointMeasurement{
Network: in.Network,
Address: in.Address,
ArchivalMeasurement: NewArchivalMeasurement(in.Measurement),
}
}
//
// THMeasurement
//
// ArchivalTHMeasurement is the archival representation of THMeasurement.
type ArchivalTHMeasurement struct {
DNS []*ArchivalDNSMeasurement `json:"dns"`
Endpoints []*ArchivalHTTPEndpointMeasurement `json:"endpoints"`
}
// NewArchivalTHMeasurement creates the archival representation of THMeasurement.
func NewArchivalTHMeasurement(in *THMeasurement) *ArchivalTHMeasurement {
return &ArchivalTHMeasurement{
DNS: NewArchivalDNSMeasurementList(in.DNS),
Endpoints: NewArchivalHTTPEndpointMeasurementList(in.Endpoints),
}
}
//
// DNSMeasurement
//
// ArchivalDNSMeasurement is the archival representation of DNSMeasurement.
type ArchivalDNSMeasurement struct {
Domain string `json:"domain"`
*ArchivalMeasurement
}
// NewArchivalDNSMeasurement converts a DNSMeasurement to an ArchivalDNSMeasurement.
func NewArchivalDNSMeasurement(in *DNSMeasurement) *ArchivalDNSMeasurement {
return &ArchivalDNSMeasurement{
Domain: in.Domain,
ArchivalMeasurement: NewArchivalMeasurement(in.Measurement),
}
}
// NewArchivalDNSMeasurementList converts a list of DNSMeasurement
// to a list of ArchivalDNSMeasurement.
func NewArchivalDNSMeasurementList(in []*DNSMeasurement) (out []*ArchivalDNSMeasurement) {
for _, m := range in {
out = append(out, NewArchivalDNSMeasurement(m))
}
return
}
//
// HTTPEndpointMeasurement
//
// ArchivalHTTPEndpointMeasurement is the archival representation
// of an HTTPEndpointMeasurement.
type ArchivalHTTPEndpointMeasurement struct {
URL string `json:"url"`
Network EndpointNetwork `json:"network"`
Address string `json:"address"`
*ArchivalMeasurement
}
// NewArchivalHTTPEndpointMeasurement converts an HTTPEndpointMeasurement
// to an ArchivalHTTPEndpointMeasurement.
func NewArchivalHTTPEndpointMeasurement(in *HTTPEndpointMeasurement) *ArchivalHTTPEndpointMeasurement {
return &ArchivalHTTPEndpointMeasurement{
URL: in.URL,
Network: in.Network,
Address: in.Address,
ArchivalMeasurement: NewArchivalMeasurement(in.Measurement),
}
}
// NewArchivalHTTPEndpointMeasurementList converts a list of HTTPEndpointMeasurement
// to a list of ArchivalHTTPEndpointMeasurement.
func NewArchivalHTTPEndpointMeasurementList(in []*HTTPEndpointMeasurement) (out []*ArchivalHTTPEndpointMeasurement) {
for _, m := range in {
out = append(out, NewArchivalHTTPEndpointMeasurement(m))
}
return
}
//
// Measurement
//
// ArchivalMeasurement is the archival representation of a Measurement.
type ArchivalMeasurement struct {
NetworkEvents []*ArchivalNetworkEvent `json:"network_events,omitempty"`
DNSEvents []*ArchivalDNSRoundTripEvent `json:"dns_events,omitempty"`
Queries []*ArchivalDNSLookupEvent `json:"queries,omitempty"`
TCPConnect []*ArchivalTCPConnect `json:"tcp_connect,omitempty"`
TLSHandshakes []*ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes,omitempty"`
QUICHandshakes []*ArchivalQUICTLSHandshakeEvent `json:"quic_handshakes,omitempty"`
Requests []*ArchivalHTTPRoundTripEvent `json:"requests,omitempty"`
}
// NewArchivalMeasurement converts a Measurement to ArchivalMeasurement.
func NewArchivalMeasurement(in *Measurement) *ArchivalMeasurement {
out := &ArchivalMeasurement{
NetworkEvents: NewArchivalNetworkEventList(in.ReadWrite),
DNSEvents: NewArchivalDNSRoundTripEventList(in.DNSRoundTrip),
Queries: nil, // done below
TCPConnect: NewArchivalTCPConnectList(in.Connect),
TLSHandshakes: NewArchivalQUICTLSHandshakeEventList(in.TLSHandshake),
QUICHandshakes: NewArchivalQUICTLSHandshakeEventList(in.QUICHandshake),
Requests: NewArchivalHTTPRoundTripEventList(in.HTTPRoundTrip),
}
out.Queries = append(out.Queries, NewArchivalDNSLookupEventList(in.LookupHost)...)
out.Queries = append(out.Queries, NewArchivalDNSLookupEventList(in.LookupHTTPSSvc)...)
return out
} }

16
internal/measurex/db.go
View File

@ -28,7 +28,7 @@ type WritableDB interface {
InsertIntoClose(ev *NetworkEvent) InsertIntoClose(ev *NetworkEvent)
// InsertIntoTLSHandshake saves a TLS handshake event. // InsertIntoTLSHandshake saves a TLS handshake event.
InsertIntoTLSHandshake(ev *TLSHandshakeEvent) InsertIntoTLSHandshake(ev *QUICTLSHandshakeEvent)
// InsertIntoLookupHost saves a lookup host event. // InsertIntoLookupHost saves a lookup host event.
InsertIntoLookupHost(ev *DNSLookupEvent) InsertIntoLookupHost(ev *DNSLookupEvent)
@ -46,7 +46,7 @@ type WritableDB interface {
InsertIntoHTTPRedirect(ev *HTTPRedirectEvent) InsertIntoHTTPRedirect(ev *HTTPRedirectEvent)
// InsertIntoQUICHandshake saves a QUIC handshake event. // InsertIntoQUICHandshake saves a QUIC handshake event.
InsertIntoQUICHandshake(ev *QUICHandshakeEvent) InsertIntoQUICHandshake(ev *QUICTLSHandshakeEvent)
} }
// MeasurementDB is a WritableDB that also allows high-level code // MeasurementDB is a WritableDB that also allows high-level code
@ -56,13 +56,13 @@ type MeasurementDB struct {
dialTable []*NetworkEvent dialTable []*NetworkEvent
readWriteTable []*NetworkEvent readWriteTable []*NetworkEvent
closeTable []*NetworkEvent closeTable []*NetworkEvent
tlsHandshakeTable []*TLSHandshakeEvent tlsHandshakeTable []*QUICTLSHandshakeEvent
lookupHostTable []*DNSLookupEvent lookupHostTable []*DNSLookupEvent
lookupHTTPSvcTable []*DNSLookupEvent lookupHTTPSvcTable []*DNSLookupEvent
dnsRoundTripTable []*DNSRoundTripEvent dnsRoundTripTable []*DNSRoundTripEvent
httpRoundTripTable []*HTTPRoundTripEvent httpRoundTripTable []*HTTPRoundTripEvent
httpRedirectTable []*HTTPRedirectEvent httpRedirectTable []*HTTPRedirectEvent
quicHandshakeTable []*QUICHandshakeEvent quicHandshakeTable []*QUICTLSHandshakeEvent
// mu protects all the fields // mu protects all the fields
mu sync.Mutex mu sync.Mutex
@ -126,14 +126,14 @@ func (db *MeasurementDB) selectAllFromCloseUnlocked() (out []*NetworkEvent) {
} }
// InsertIntoTLSHandshake implements EventDB.InsertIntoTLSHandshake. // InsertIntoTLSHandshake implements EventDB.InsertIntoTLSHandshake.
func (db *MeasurementDB) InsertIntoTLSHandshake(ev *TLSHandshakeEvent) { func (db *MeasurementDB) InsertIntoTLSHandshake(ev *QUICTLSHandshakeEvent) {
db.mu.Lock() db.mu.Lock()
db.tlsHandshakeTable = append(db.tlsHandshakeTable, ev) db.tlsHandshakeTable = append(db.tlsHandshakeTable, ev)
db.mu.Unlock() db.mu.Unlock()
} }
// selectAllFromTLSHandshakeUnlocked returns all TLS handshake events. // selectAllFromTLSHandshakeUnlocked returns all TLS handshake events.
func (db *MeasurementDB) selectAllFromTLSHandshakeUnlocked() (out []*TLSHandshakeEvent) { func (db *MeasurementDB) selectAllFromTLSHandshakeUnlocked() (out []*QUICTLSHandshakeEvent) {
out = append(out, db.tlsHandshakeTable...) out = append(out, db.tlsHandshakeTable...)
return return
} }
@ -204,14 +204,14 @@ func (db *MeasurementDB) selectAllFromHTTPRedirectUnlocked() (out []*HTTPRedirec
} }
// InsertIntoQUICHandshake implements EventDB.InsertIntoQUICHandshake. // InsertIntoQUICHandshake implements EventDB.InsertIntoQUICHandshake.
func (db *MeasurementDB) InsertIntoQUICHandshake(ev *QUICHandshakeEvent) { func (db *MeasurementDB) InsertIntoQUICHandshake(ev *QUICTLSHandshakeEvent) {
db.mu.Lock() db.mu.Lock()
db.quicHandshakeTable = append(db.quicHandshakeTable, ev) db.quicHandshakeTable = append(db.quicHandshakeTable, ev)
db.mu.Unlock() db.mu.Unlock()
} }
// selectAllFromQUICHandshakeUnlocked returns all QUIC handshake events. // selectAllFromQUICHandshakeUnlocked returns all QUIC handshake events.
func (db *MeasurementDB) selectAllFromQUICHandshakeUnlocked() (out []*QUICHandshakeEvent) { func (db *MeasurementDB) selectAllFromQUICHandshakeUnlocked() (out []*QUICTLSHandshakeEvent) {
out = append(out, db.quicHandshakeTable...) out = append(out, db.quicHandshakeTable...)
return return
} }

27
internal/measurex/dialer.go
View File

@ -54,17 +54,14 @@ type dialerDB struct {
// NetworkEvent contains a network event. This kind of events // NetworkEvent contains a network event. This kind of events
// are generated by Dialer, QUICDialer, Conn, QUICConn. // are generated by Dialer, QUICDialer, Conn, QUICConn.
type NetworkEvent struct { type NetworkEvent struct {
// JSON names compatible with df-008-netevents RemoteAddr string
RemoteAddr string `json:"address"` Failure *string
Failure *string `json:"failure"` Count int
Count int `json:"num_bytes,omitempty"` Operation string
Operation string `json:"operation"` Network string
Network string `json:"proto"` Oddity Oddity
Finished float64 `json:"t"` Finished float64
Started float64 `json:"started"` Started float64
// Names that are not part of the spec.
Oddity Oddity `json:"oddity"`
} }
func (d *dialerDB) DialContext( func (d *dialerDB) DialContext(
@ -78,7 +75,7 @@ func (d *dialerDB) DialContext(
RemoteAddr: address, RemoteAddr: address,
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Oddity: d.computeOddity(err), Oddity: d.computeOddity(err),
Count: 0, Count: 0,
}) })
@ -128,7 +125,7 @@ func (c *connDB) Read(b []byte) (int, error) {
RemoteAddr: c.remoteAddr, RemoteAddr: c.remoteAddr,
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Count: count, Count: count,
}) })
return count, err return count, err
@ -144,7 +141,7 @@ func (c *connDB) Write(b []byte) (int, error) {
RemoteAddr: c.remoteAddr, RemoteAddr: c.remoteAddr,
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Count: count, Count: count,
}) })
return count, err return count, err
@ -160,7 +157,7 @@ func (c *connDB) Close() error {
RemoteAddr: c.remoteAddr, RemoteAddr: c.remoteAddr,
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Count: 0, Count: 0,
}) })
return err return err

22
internal/measurex/dnsx.go
View File

@ -32,15 +32,13 @@ type dnsxRoundTripperDB struct {
// DNSRoundTripEvent contains the result of a DNS round trip. // DNSRoundTripEvent contains the result of a DNS round trip.
type DNSRoundTripEvent struct { type DNSRoundTripEvent struct {
// This data structure is not in df-002-dns but the names and Network string
// semantics try to be consistent with such a spec. Address string
Network string `json:"engine"` Query []byte
Address string `json:"resolver_address"` Started float64
Query *ArchivalBinaryData `json:"raw_query"` Finished float64
Started float64 `json:"started"` Failure *string
Finished float64 `json:"t"` Reply []byte
Failure *string `json:"failure"`
Reply *ArchivalBinaryData `json:"raw_reply"`
} }
func (txp *dnsxRoundTripperDB) RoundTrip(ctx context.Context, query []byte) ([]byte, error) { func (txp *dnsxRoundTripperDB) RoundTrip(ctx context.Context, query []byte) ([]byte, error) {
@ -50,11 +48,11 @@ func (txp *dnsxRoundTripperDB) RoundTrip(ctx context.Context, query []byte) ([]b
txp.db.InsertIntoDNSRoundTrip(&DNSRoundTripEvent{ txp.db.InsertIntoDNSRoundTrip(&DNSRoundTripEvent{
Network: txp.DNSTransport.Network(), Network: txp.DNSTransport.Network(),
Address: txp.DNSTransport.Address(), Address: txp.DNSTransport.Address(),
Query: NewArchivalBinaryData(query), Query: query,
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Reply: NewArchivalBinaryData(reply), Reply: reply,
}) })
return reply, err return reply, err
} }

12
internal/measurex/failure.go Normal file
View File

@ -0,0 +1,12 @@
package measurex
// NewFailure creates a serializable failure from an error. We
// cannot round trip an error using JSON, so we serialize to this
// intermediate format that is a sort of Optional<string>.
func NewFailure(err error) *string {
if err == nil {
return nil
}
s := err.Error()
return &s
}

50
internal/measurex/http.go
View File

@ -124,31 +124,33 @@ type HTTPResponse struct {
// HTTPRoundTripEvent contains information about an HTTP round trip. // HTTPRoundTripEvent contains information about an HTTP round trip.
type HTTPRoundTripEvent struct { type HTTPRoundTripEvent struct {
// JSON names following the df-001-httpt data format. Failure *string
Failure *string `json:"failure"` Method string
Request *HTTPRequest `json:"request"` URL string
Response *HTTPResponse `json:"response"` RequestHeaders http.Header
Finished float64 `json:"t"` StatusCode int64
Started float64 `json:"started"` ResponseHeaders http.Header
ResponseBody []byte
// Names not in the specification ResponseBodyLength int64
Oddity Oddity `json:"oddity"` ResponseBodyIsTruncated bool
ResponseBodyIsUTF8 bool
Finished float64
Started float64
Oddity Oddity
} }
func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error) { func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error) {
started := time.Since(txp.Begin).Seconds() started := time.Since(txp.Begin).Seconds()
resp, err := txp.HTTPTransport.RoundTrip(req) resp, err := txp.HTTPTransport.RoundTrip(req)
rt := &HTTPRoundTripEvent{ rt := &HTTPRoundTripEvent{
Request: &HTTPRequest{ Method: req.Method,
Method: req.Method, URL: req.URL.String(),
URL: req.URL.String(), RequestHeaders: req.Header,
Headers: NewArchivalHeaders(req.Header), Started: started,
},
Started: started,
} }
if err != nil { if err != nil {
rt.Finished = time.Since(txp.Begin).Seconds() rt.Finished = time.Since(txp.Begin).Seconds()
rt.Failure = NewArchivalFailure(err) rt.Failure = NewFailure(err)
txp.DB.InsertIntoHTTPRoundTrip(rt) txp.DB.InsertIntoHTTPRoundTrip(rt)
return nil, err return nil, err
} }
@ -162,10 +164,8 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error)
case resp.StatusCode >= 400: case resp.StatusCode >= 400:
rt.Oddity = OddityStatusOther rt.Oddity = OddityStatusOther
} }
rt.Response = &HTTPResponse{ rt.StatusCode = int64(resp.StatusCode)
Code: int64(resp.StatusCode), rt.ResponseHeaders = resp.Header
Headers: NewArchivalHeaders(resp.Header),
}
r := io.LimitReader(resp.Body, txp.MaxBodySnapshotSize) r := io.LimitReader(resp.Body, txp.MaxBodySnapshotSize)
body, err := netxlite.ReadAllContext(req.Context(), r) body, err := netxlite.ReadAllContext(req.Context(), r)
if errors.Is(err, io.EOF) && resp.Close { if errors.Is(err, io.EOF) && resp.Close {
@ -173,7 +173,7 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error)
} }
if err != nil { if err != nil {
rt.Finished = time.Since(txp.Begin).Seconds() rt.Finished = time.Since(txp.Begin).Seconds()
rt.Failure = NewArchivalFailure(err) rt.Failure = NewFailure(err)
txp.DB.InsertIntoHTTPRoundTrip(rt) txp.DB.InsertIntoHTTPRoundTrip(rt)
return nil, err return nil, err
} }
@ -181,10 +181,10 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error)
Reader: io.MultiReader(bytes.NewReader(body), resp.Body), Reader: io.MultiReader(bytes.NewReader(body), resp.Body),
Closer: resp.Body, Closer: resp.Body,
} }
rt.Response.Body = NewArchivalBinaryData(body) rt.ResponseBody = body
rt.Response.BodyLength = int64(len(body)) rt.ResponseBodyLength = int64(len(body))
rt.Response.BodyIsTruncated = int64(len(body)) >= txp.MaxBodySnapshotSize rt.ResponseBodyIsTruncated = int64(len(body)) >= txp.MaxBodySnapshotSize
rt.Response.BodyIsUTF8 = utf8.Valid(body) rt.ResponseBodyIsUTF8 = utf8.Valid(body)
rt.Finished = time.Since(txp.Begin).Seconds() rt.Finished = time.Since(txp.Begin).Seconds()
txp.DB.InsertIntoHTTPRoundTrip(rt) txp.DB.InsertIntoHTTPRoundTrip(rt)
return resp, nil return resp, nil

62
internal/measurex/measurement.go
View File

@ -18,33 +18,33 @@ import (
// a bunch of measurements detailing each measurement step. // a bunch of measurements detailing each measurement step.
type URLMeasurement struct { type URLMeasurement struct {
// URL is the URL we're measuring. // URL is the URL we're measuring.
URL string `json:"url"` URL string
// DNS contains all the DNS related measurements. // DNS contains all the DNS related measurements.
DNS []*DNSMeasurement `json:"dns"` DNS []*DNSMeasurement
// Endpoints contains a measurement for each endpoint // Endpoints contains a measurement for each endpoint
// that we discovered via DNS or TH. // that we discovered via DNS or TH.
Endpoints []*HTTPEndpointMeasurement `json:"endpoints"` Endpoints []*HTTPEndpointMeasurement
// RedirectURLs contain the URLs to which we should fetch // RedirectURLs contain the URLs to which we should fetch
// if we choose to follow redirections. // if we choose to follow redirections.
RedirectURLs []string `json:"-"` RedirectURLs []string
// THMeasurement is the measurement collected by the TH. // TH is the measurement collected by the TH.
TH interface{} `json:"th,omitempty"` TH *THMeasurement
// TotalRuntime is the total time to measure this URL. // TotalRuntime is the total time to measure this URL.
TotalRuntime time.Duration `json:"-"` TotalRuntime time.Duration
// DNSRuntime is the time to run all DNS checks. // DNSRuntime is the time to run all DNS checks.
DNSRuntime time.Duration `json:"x_dns_runtime"` DNSRuntime time.Duration
// THRuntime is the total time to invoke all test helpers. // THRuntime is the total time to invoke all test helpers.
THRuntime time.Duration `json:"x_th_runtime"` THRuntime time.Duration
// EpntsRuntime is the total time to check all the endpoints. // EpntsRuntime is the total time to check all the endpoints.
EpntsRuntime time.Duration `json:"x_epnts_runtime"` EpntsRuntime time.Duration
} }
// fillRedirects takes in input a complete URLMeasurement and fills // fillRedirects takes in input a complete URLMeasurement and fills
@ -67,40 +67,40 @@ func (m *URLMeasurement) fillRedirects() {
// data format is not compatible with the OONI data format. // data format is not compatible with the OONI data format.
type Measurement struct { type Measurement struct {
// Connect contains all the connect operations. // Connect contains all the connect operations.
Connect []*NetworkEvent `json:"connect,omitempty"` Connect []*NetworkEvent
// ReadWrite contains all the read and write operations. // ReadWrite contains all the read and write operations.
ReadWrite []*NetworkEvent `json:"read_write,omitempty"` ReadWrite []*NetworkEvent
// Close contains all the close operations. // Close contains all the close operations.
Close []*NetworkEvent `json:"-"` Close []*NetworkEvent
// TLSHandshake contains all the TLS handshakes. // TLSHandshake contains all the TLS handshakes.
TLSHandshake []*TLSHandshakeEvent `json:"tls_handshake,omitempty"` TLSHandshake []*QUICTLSHandshakeEvent
// QUICHandshake contains all the QUIC handshakes. // QUICHandshake contains all the QUIC handshakes.
QUICHandshake []*QUICHandshakeEvent `json:"quic_handshake,omitempty"` QUICHandshake []*QUICTLSHandshakeEvent
// LookupHost contains all the host lookups. // LookupHost contains all the host lookups.
LookupHost []*DNSLookupEvent `json:"lookup_host,omitempty"` LookupHost []*DNSLookupEvent
// LookupHTTPSSvc contains all the HTTPSSvc lookups. // LookupHTTPSSvc contains all the HTTPSSvc lookups.
LookupHTTPSSvc []*DNSLookupEvent `json:"lookup_httpssvc,omitempty"` LookupHTTPSSvc []*DNSLookupEvent
// DNSRoundTrip contains all the DNS round trips. // DNSRoundTrip contains all the DNS round trips.
DNSRoundTrip []*DNSRoundTripEvent `json:"dns_round_trip,omitempty"` DNSRoundTrip []*DNSRoundTripEvent
// HTTPRoundTrip contains all the HTTP round trips. // HTTPRoundTrip contains all the HTTP round trips.
HTTPRoundTrip []*HTTPRoundTripEvent `json:"http_round_trip,omitempty"` HTTPRoundTrip []*HTTPRoundTripEvent
// HTTPRedirect contains all the redirections. // HTTPRedirect contains all the redirections.
HTTPRedirect []*HTTPRedirectEvent `json:"-"` HTTPRedirect []*HTTPRedirectEvent
} }
// DNSMeasurement is a DNS measurement. // DNSMeasurement is a DNS measurement.
type DNSMeasurement struct { type DNSMeasurement struct {
// Domain is the domain this measurement refers to. // Domain is the domain this measurement refers to.
Domain string `json:"domain"` Domain string
// A DNSMeasurement is a Measurement. // A DNSMeasurement is a Measurement.
*Measurement *Measurement
@ -239,10 +239,10 @@ func AllHTTPEndpointsForURL(URL *url.URL,
// EndpointMeasurement is an endpoint measurement. // EndpointMeasurement is an endpoint measurement.
type EndpointMeasurement struct { type EndpointMeasurement struct {
// Network is the network of this endpoint. // Network is the network of this endpoint.
Network EndpointNetwork `json:"network"` Network EndpointNetwork
// Address is the address of this endpoint. // Address is the address of this endpoint.
Address string `json:"address"` Address string
// An EndpointMeasurement is a Measurement. // An EndpointMeasurement is a Measurement.
*Measurement *Measurement
@ -251,14 +251,24 @@ type EndpointMeasurement struct {
// HTTPEndpointMeasurement is an HTTP endpoint measurement. // HTTPEndpointMeasurement is an HTTP endpoint measurement.
type HTTPEndpointMeasurement struct { type HTTPEndpointMeasurement struct {
// URL is the URL this measurement refers to. // URL is the URL this measurement refers to.
URL string `json:"url"` URL string
// Network is the network of this endpoint. // Network is the network of this endpoint.
Network EndpointNetwork `json:"network"` Network EndpointNetwork
// Address is the address of this endpoint. // Address is the address of this endpoint.
Address string `json:"address"` Address string
// An HTTPEndpointMeasurement is a Measurement. // An HTTPEndpointMeasurement is a Measurement.
*Measurement *Measurement
} }
// THMeasurement is the measurement performed by the TH.
type THMeasurement struct {
// DNS contains all the DNS related measurements.
DNS []*DNSMeasurement
// Endpoints contains a measurement for each endpoint
// that was discovered by the probe or the TH.
Endpoints []*HTTPEndpointMeasurement
}

11
internal/measurex/measurer.go
View File

@ -13,7 +13,6 @@ import (
"context" "context"
"crypto/tls" "crypto/tls"
"errors" "errors"
stdlog "log"
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
@ -712,7 +711,7 @@ type MeasureURLHelper interface {
// test helper protocol allows one to set. // test helper protocol allows one to set.
LookupExtraHTTPEndpoints(ctx context.Context, URL *url.URL, LookupExtraHTTPEndpoints(ctx context.Context, URL *url.URL,
headers http.Header, epnts ...*HTTPEndpoint) ( headers http.Header, epnts ...*HTTPEndpoint) (
newEpnts []*HTTPEndpoint, thMeasurement interface{}, err error) newEpnts []*HTTPEndpoint, thMeasurement *THMeasurement, err error)
} }
// MeasureURL measures an HTTP or HTTPS URL. The DNS resolvers // MeasureURL measures an HTTP or HTTPS URL. The DNS resolvers
@ -802,12 +801,8 @@ func (mx *Measurer) maybeQUICFollowUp(ctx context.Context,
if epnt.QUICHandshake != nil { if epnt.QUICHandshake != nil {
return return
} }
for idx, rtrip := range epnt.HTTPRoundTrip { for _, rtrip := range epnt.HTTPRoundTrip {
if rtrip.Response == nil { if v := rtrip.ResponseHeaders.Get("alt-svc"); v != "" {
stdlog.Printf("malformed HTTPRoundTrip@%d: %+v", idx, rtrip)
continue
}
if v := rtrip.Response.Headers.Get("alt-svc"); v != "" {
altsvc = append(altsvc, v) altsvc = append(altsvc, v)
} }
} }

15
internal/measurex/quic.go
View File

@ -60,7 +60,7 @@ func (c *udpLikeConnDB) WriteTo(p []byte, addr net.Addr) (int, error) {
RemoteAddr: addr.String(), RemoteAddr: addr.String(),
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Count: count, Count: count,
}) })
return count, err return count, err
@ -76,7 +76,7 @@ func (c *udpLikeConnDB) ReadFrom(b []byte) (int, net.Addr, error) {
RemoteAddr: addrStringIfNotNil(addr), RemoteAddr: addrStringIfNotNil(addr),
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Count: count, Count: count,
}) })
return count, addr, err return count, addr, err
@ -92,15 +92,12 @@ func (c *udpLikeConnDB) Close() error {
RemoteAddr: "", RemoteAddr: "",
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Count: 0, Count: 0,
}) })
return err return err
} }
// QUICHandshakeEvent is the result of QUICHandshake.
type QUICHandshakeEvent = TLSHandshakeEvent
// NewQUICDialerWithoutResolver creates a new QUICDialer that is not // NewQUICDialerWithoutResolver creates a new QUICDialer that is not
// attached to any resolver. This means that every attempt to dial any // attached to any resolver. This means that every attempt to dial any
// address containing a domain name will fail. This QUICDialer will // address containing a domain name will fail. This QUICDialer will
@ -138,7 +135,7 @@ func (qh *quicDialerDB) DialContext(ctx context.Context, network, address string
} }
} }
finished := time.Since(qh.begin).Seconds() finished := time.Since(qh.begin).Seconds()
qh.db.InsertIntoQUICHandshake(&QUICHandshakeEvent{ qh.db.InsertIntoQUICHandshake(&QUICTLSHandshakeEvent{
Network: "quic", Network: "quic",
RemoteAddr: address, RemoteAddr: address,
SNI: tlsConfig.ServerName, SNI: tlsConfig.ServerName,
@ -146,12 +143,12 @@ func (qh *quicDialerDB) DialContext(ctx context.Context, network, address string
SkipVerify: tlsConfig.InsecureSkipVerify, SkipVerify: tlsConfig.InsecureSkipVerify,
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Oddity: qh.computeOddity(err), Oddity: qh.computeOddity(err),
TLSVersion: netxlite.TLSVersionString(state.Version), TLSVersion: netxlite.TLSVersionString(state.Version),
CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite),
NegotiatedProto: state.NegotiatedProtocol, NegotiatedProto: state.NegotiatedProtocol,
PeerCerts: NewArchivalTLSCerts(peerCerts(nil, &state)), PeerCerts: peerCerts(nil, &state),
}) })
return sess, err return sess, err
} }

121
internal/measurex/resolver.go
View File

@ -8,7 +8,6 @@ package measurex
import ( import (
"context" "context"
"net"
"strings" "strings"
"time" "time"
@ -65,31 +64,19 @@ type resolverDB struct {
db WritableDB db WritableDB
} }
// DNSLookupAnswer is a DNS lookup answer.
type DNSLookupAnswer struct {
// JSON names compatible with df-002-dnst's spec
Type string `json:"answer_type"`
IPv4 string `json:"ipv4,omitempty"`
IPv6 string `json:"ivp6,omitempty"`
// Names not part of the spec.
ALPN string `json:"alpn,omitempty"`
}
// DNSLookupEvent contains the results of a DNS lookup. // DNSLookupEvent contains the results of a DNS lookup.
type DNSLookupEvent struct { type DNSLookupEvent struct {
// fields inside df-002-dnst Network string
Answers []DNSLookupAnswer `json:"answers"` Failure *string
Network string `json:"engine"` Domain string
Failure *string `json:"failure"` QueryType string
Domain string `json:"hostname"` Address string
QueryType string `json:"query_type"` Finished float64
Address string `json:"resolver_address"` Started float64
Finished float64 `json:"t"` Oddity Oddity
A []string
// Names not part of the spec. AAAA []string
Started float64 `json:"started"` ALPN []string
Oddity Oddity `json:"oddity"`
} }
// SupportsHTTP3 returns true if this query is for HTTPS and // SupportsHTTP3 returns true if this query is for HTTPS and
@ -98,12 +85,9 @@ func (ev *DNSLookupEvent) SupportsHTTP3() bool {
if ev.QueryType != "HTTPS" { if ev.QueryType != "HTTPS" {
return false return false
} }
for _, ans := range ev.Answers { for _, alpn := range ev.ALPN {
switch ans.Type { if alpn == "h3" {
case "ALPN": return true
if ans.ALPN == "h3" {
return true
}
} }
} }
return false return false
@ -111,18 +95,8 @@ func (ev *DNSLookupEvent) SupportsHTTP3() bool {
// Addrs returns all the IPv4/IPv6 addresses // Addrs returns all the IPv4/IPv6 addresses
func (ev *DNSLookupEvent) Addrs() (out []string) { func (ev *DNSLookupEvent) Addrs() (out []string) {
for _, ans := range ev.Answers { out = append(out, ev.A...)
switch ans.Type { out = append(out, ev.AAAA...)
case "A":
if net.ParseIP(ans.IPv4) != nil {
out = append(out, ans.IPv4)
}
case "AAAA":
if net.ParseIP(ans.IPv6) != nil {
out = append(out, ans.IPv6)
}
}
}
return return
} }
@ -130,35 +104,39 @@ func (r *resolverDB) LookupHost(ctx context.Context, domain string) ([]string, e
started := time.Since(r.begin).Seconds() started := time.Since(r.begin).Seconds()
addrs, err := r.Resolver.LookupHost(ctx, domain) addrs, err := r.Resolver.LookupHost(ctx, domain)
finished := time.Since(r.begin).Seconds() finished := time.Since(r.begin).Seconds()
for _, qtype := range []string{"A", "AAAA"} { r.saveLookupResults(domain, started, finished, err, addrs, "A")
ev := &DNSLookupEvent{ r.saveLookupResults(domain, started, finished, err, addrs, "AAAA")
Answers: r.computeAnswers(addrs, qtype),
Network: r.Resolver.Network(),
Address: r.Resolver.Address(),
Failure: NewArchivalFailure(err),
Domain: domain,
QueryType: qtype,
Finished: finished,
Started: started,
Oddity: r.computeOddityLookupHost(addrs, err),
}
r.db.InsertIntoLookupHost(ev)
}
return addrs, err return addrs, err
} }
func (r *resolverDB) computeAnswers(addrs []string, qtype string) (out []DNSLookupAnswer) { func (r *resolverDB) saveLookupResults(domain string, started, finished float64,
err error, addrs []string, qtype string) {
ev := &DNSLookupEvent{
Network: r.Resolver.Network(),
Address: r.Resolver.Address(),
Failure: NewFailure(err),
Domain: domain,
QueryType: qtype,
Finished: finished,
Started: started,
}
for _, addr := range addrs { for _, addr := range addrs {
if qtype == "A" && !strings.Contains(addr, ":") { if qtype == "A" && !strings.Contains(addr, ":") {
out = append(out, DNSLookupAnswer{Type: qtype, IPv4: addr}) ev.A = append(ev.A, addr)
continue continue
} }
if qtype == "AAAA" && strings.Contains(addr, ":") { if qtype == "AAAA" && strings.Contains(addr, ":") {
out = append(out, DNSLookupAnswer{Type: qtype, IPv6: addr}) ev.AAAA = append(ev.AAAA, addr)
continue continue
} }
} }
return switch qtype {
case "A":
ev.Oddity = r.computeOddityLookupHost(ev.A, err)
case "AAAA":
ev.Oddity = r.computeOddityLookupHost(ev.AAAA, err)
}
r.db.InsertIntoLookupHost(ev)
} }
func (r *resolverDB) computeOddityLookupHost(addrs []string, err error) Oddity { func (r *resolverDB) computeOddityLookupHost(addrs []string, err error) Oddity {
@ -193,28 +171,13 @@ func (r *resolverDB) LookupHTTPS(ctx context.Context, domain string) (*HTTPSSvc,
QueryType: "HTTPS", QueryType: "HTTPS",
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Oddity: Oddity(r.computeOddityHTTPSSvc(https, err)), Oddity: Oddity(r.computeOddityHTTPSSvc(https, err)),
} }
if err == nil { if err == nil {
for _, addr := range https.IPv4 { ev.A = append(ev.A, https.IPv4...)
ev.Answers = append(ev.Answers, DNSLookupAnswer{ ev.AAAA = append(ev.AAAA, https.IPv6...)
Type: "A", ev.ALPN = append(ev.ALPN, https.ALPN...)
IPv4: addr,
})
}
for _, addr := range https.IPv6 {
ev.Answers = append(ev.Answers, DNSLookupAnswer{
Type: "AAAA",
IPv6: addr,
})
}
for _, alpn := range https.ALPN {
ev.Answers = append(ev.Answers, DNSLookupAnswer{
Type: "ALPN",
ALPN: alpn,
})
}
} }
r.db.InsertIntoLookupHTTPSSvc(ev) r.db.InsertIntoLookupHTTPSSvc(ev)
return https, err return https, err

40
internal/measurex/tls.go
View File

@ -38,25 +38,21 @@ type tlsHandshakerDB struct {
db WritableDB db WritableDB
} }
// TLSHandshakeEvent contains a TLS handshake event. // QUICTLSHandshakeEvent contains a QUIC or TLS handshake event.
type TLSHandshakeEvent struct { type QUICTLSHandshakeEvent struct {
// JSON names compatible with df-006-tlshandshake CipherSuite string
CipherSuite string `json:"cipher_suite"` Failure *string
Failure *string `json:"failure"` NegotiatedProto string
NegotiatedProto string `json:"negotiated_proto"` TLSVersion string
TLSVersion string `json:"tls_version"` PeerCerts [][]byte
PeerCerts []*ArchivalBinaryData `json:"peer_certificates"` Finished float64
Finished float64 `json:"t"` RemoteAddr string
SNI string
// JSON names that are consistent with the ALPN []string
// spirit of the spec but are not in it SkipVerify bool
RemoteAddr string `json:"address"` Oddity Oddity
SNI string `json:"server_name"` // used in prod Network string
ALPN []string `json:"alpn"` Started float64
SkipVerify bool `json:"no_tls_verify"` // used in prod
Oddity Oddity `json:"oddity"`
Network string `json:"proto"`
Started float64 `json:"started"`
} }
func (thx *tlsHandshakerDB) Handshake(ctx context.Context, func (thx *tlsHandshakerDB) Handshake(ctx context.Context,
@ -66,7 +62,7 @@ func (thx *tlsHandshakerDB) Handshake(ctx context.Context,
started := time.Since(thx.begin).Seconds() started := time.Since(thx.begin).Seconds()
tconn, state, err := thx.TLSHandshaker.Handshake(ctx, conn, config) tconn, state, err := thx.TLSHandshaker.Handshake(ctx, conn, config)
finished := time.Since(thx.begin).Seconds() finished := time.Since(thx.begin).Seconds()
thx.db.InsertIntoTLSHandshake(&TLSHandshakeEvent{ thx.db.InsertIntoTLSHandshake(&QUICTLSHandshakeEvent{
Network: network, Network: network,
RemoteAddr: remoteAddr, RemoteAddr: remoteAddr,
SNI: config.ServerName, SNI: config.ServerName,
@ -74,12 +70,12 @@ func (thx *tlsHandshakerDB) Handshake(ctx context.Context,
SkipVerify: config.InsecureSkipVerify, SkipVerify: config.InsecureSkipVerify,
Started: started, Started: started,
Finished: finished, Finished: finished,
Failure: NewArchivalFailure(err), Failure: NewFailure(err),
Oddity: thx.computeOddity(err), Oddity: thx.computeOddity(err),
TLSVersion: netxlite.TLSVersionString(state.Version), TLSVersion: netxlite.TLSVersionString(state.Version),
CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite),
NegotiatedProto: state.NegotiatedProtocol, NegotiatedProto: state.NegotiatedProtocol,
PeerCerts: NewArchivalTLSCerts(peerCerts(err, &state)), PeerCerts: peerCerts(err, &state),
}) })
return tconn, state, err return tconn, state, err
} }

16
internal/tutorial/measurex/chapter01/README.md
View File

@ -167,8 +167,12 @@ format, in the remainder of this program we're
going to serialize the `Measurement` to JSON and going to serialize the `Measurement` to JSON and
print it to the standard output. print it to the standard output.
Rather than serializing the raw `Measurement` struct,
we first convert it to the "archival" format. This is the
data format specified at [ooni/spec](https://github.com/ooni/spec/tree/master/data-formats).
```Go ```Go
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
``` ```
@ -194,13 +198,14 @@ go run -race ./internal/tutorial/measurex/chapter01 | jq
``` ```
Where `jq` is being used to make the output more presentable. Where `jq` is being used to make the output more presentable.
If you do that you obtain some logging messages, which are out of If you do that you obtain some logging messages, which are out of
the scope of this tutorial, and the following JSON: the scope of this tutorial, and the following JSON:
```JSON ```JSON
{ {
"domain": "example.com", "domain": "example.com",
"lookup_host": [ "queries": [
{ {
"answers": [ "answers": [
{ {
@ -237,6 +242,9 @@ the scope of this tutorial, and the following JSON:
} }
``` ```
This JSON [implements the df-002-dnst](https://github.com/ooni/spec/blob/master/data-formats/df-002-dnst.md)
OONI data format.
You see that we have two messages here. OONI splits a DNS You see that we have two messages here. OONI splits a DNS
resolution performed using the system resolver into two "fake" resolution performed using the system resolver into two "fake"
DNS resolutions for A and AAAA. (Under the hood, this is DNS resolutions for A and AAAA. (Under the hood, this is
@ -266,7 +274,7 @@ This is the output JSON:
```JSON ```JSON
{ {
"domain": "antani.ooni.org", "domain": "antani.ooni.org",
"lookup_host": [ "queries": [
{ {
"answers": null, "answers": null,
"engine": "system", "engine": "system",
@ -327,7 +335,7 @@ To get this JSON:
```JSON ```JSON
{ {
"domain": "example.com", "domain": "example.com",
"lookup_host": [ "queries": [
{ {
"answers": null, "answers": null,
"engine": "system", "engine": "system",

16
internal/tutorial/measurex/chapter01/main.go
View File

@ -168,8 +168,12 @@ func main() {
// going to serialize the `Measurement` to JSON and // going to serialize the `Measurement` to JSON and
// print it to the standard output. // print it to the standard output.
// //
// Rather than serializing the raw `Measurement` struct,
// we first convert it to the "archival" format. This is the
// data format specified at [ooni/spec](https://github.com/ooni/spec/tree/master/data-formats).
//
// ```Go // ```Go
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
// ``` // ```
@ -195,13 +199,14 @@ func main() {
// ``` // ```
// //
// Where `jq` is being used to make the output more presentable. // Where `jq` is being used to make the output more presentable.
//
// If you do that you obtain some logging messages, which are out of // If you do that you obtain some logging messages, which are out of
// the scope of this tutorial, and the following JSON: // the scope of this tutorial, and the following JSON:
// //
// ```JSON // ```JSON
// { // {
// "domain": "example.com", // "domain": "example.com",
// "lookup_host": [ // "queries": [
// { // {
// "answers": [ // "answers": [
// { // {
@ -238,6 +243,9 @@ func main() {
// } // }
// ``` // ```
// //
// This JSON [implements the df-002-dnst](https://github.com/ooni/spec/blob/master/data-formats/df-002-dnst.md)
// OONI data format.
//
// You see that we have two messages here. OONI splits a DNS // You see that we have two messages here. OONI splits a DNS
// resolution performed using the system resolver into two "fake" // resolution performed using the system resolver into two "fake"
// DNS resolutions for A and AAAA. (Under the hood, this is // DNS resolutions for A and AAAA. (Under the hood, this is
@ -267,7 +275,7 @@ func main() {
// ```JSON // ```JSON
// { // {
// "domain": "antani.ooni.org", // "domain": "antani.ooni.org",
// "lookup_host": [ // "queries": [
// { // {
// "answers": null, // "answers": null,
// "engine": "system", // "engine": "system",
@ -328,7 +336,7 @@ func main() {
// ```JSON // ```JSON
// { // {
// "domain": "example.com", // "domain": "example.com",
// "lookup_host": [ // "queries": [
// { // {
// "answers": null, // "answers": null,
// "engine": "system", // "engine": "system",

88
internal/tutorial/measurex/chapter02/README.md
View File

@ -69,10 +69,12 @@ address is quoted with "[" and "]" if IPv6, e.g., `[::1]:53`.)
### Printing the measurement ### Printing the measurement
The rest of the main function is just like in the previous chapter. The rest of the main function is just like in the previous
chapter. Like we did before, we convert the obtained measurement
to the "archival" data format before printing.
```Go ```Go
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
@ -98,27 +100,31 @@ Here is the JSON we obtain in output:
// This block contains the results of the connect syscall // This block contains the results of the connect syscall
// using the df-008-netevents data format. // using the df-008-netevents data format.
"connect": [ "tcp_connect": [{
{ "ip": "8.8.4.4",
"address": "8.8.4.4:443", "port": 443,
"t": 0.020303,
"status": {
"blocked": false,
"failure": null, "failure": null,
"operation": "connect", "success": true
"proto": "tcp", },
"t": 0.026879041, "started": 0.000109292,
"started": 8.8625e-05, "oddity": ""
"oddity": "" }]
}
]
} }
``` ```
This JSON implements [the df-005-tcpconnect](https://github.com/ooni/spec/blob/master/data-formats/df-005-tcpconnect.md)
OONI data format.
This is what it says: This is what it says:
- we are connecting a "tcp" socket; - we are connecting a "tcp" socket;
- the destination endpoint address is "8.8.4.4:443"; - the destination endpoint address is "8.8.4.4:443";
- connect terminated ~0.027 seconds into the program's life; - connect terminated ~0.020 seconds into the program's life (see `t`);
- the operation succeeded (`failure` is `nil`). - the operation succeeded (`failure` is `nil`).
@ -138,19 +144,21 @@ We get this JSON:
{ {
"network": "tcp", "network": "tcp",
"address": "127.0.0.1:1", "address": "127.0.0.1:1",
"connect": [ "tcp_connect": [
{ {
"address": "127.0.0.1:1", "ip": "127.0.0.1",
"failure": "connection_refused", "port": 1,
"operation": "connect", "t": 0.000457584,
"proto": "tcp", "status": {
"t": 0.000372167, "blocked": true,
"started": 8.4917e-05, "failure": "connection_refused",
"success": false
},
"started": 0.000104792,
"oddity": "tcp.connect.refused" "oddity": "tcp.connect.refused"
} }
] ]
} }
``` ```
And here's an error telling us the connection was refused and And here's an error telling us the connection was refused and
@ -170,14 +178,17 @@ We get this JSON:
{ {
"network": "tcp", "network": "tcp",
"address": "8.8.4.4:1", "address": "8.8.4.4:1",
"connect": [ "tcp_connect": [
{ {
"address": "8.8.4.4:1", "ip": "8.8.4.4",
"failure": "generic_timeout_error", "port": 1,
"operation": "connect", "t": 10.006558625,
"proto": "tcp", "status": {
"t": 10.005494583, "blocked": true,
"started": 8.4833e-05, "failure": "generic_timeout_error",
"success": false
},
"started": 9.55e-05,
"oddity": "tcp.connect.timeout" "oddity": "tcp.connect.timeout"
} }
] ]
@ -201,21 +212,26 @@ To get this JSON:
{ {
"network": "tcp", "network": "tcp",
"address": "8.8.4.4:1", "address": "8.8.4.4:1",
"connect": [ "tcp_connect": [
{ {
"address": "8.8.4.4:1", "ip": "8.8.4.4",
"failure": "generic_timeout_error", "port": 1,
"operation": "connect", "t": 0.105445125,
"proto": "tcp", "status": {
"t": 0.10148025, "blocked": true,
"started": 0.000122375, "failure": "generic_timeout_error",
"success": false
},
"started": 9.4083e-05,
"oddity": "tcp.connect.timeout" "oddity": "tcp.connect.timeout"
} }
] ]
} }
``` ```
We see a timeout after ~0.1s. We enforce a reasonably small We see a timeout after ~0.1s.
We enforce a reasonably small
timeout for connecting, equal to 10 s, because we want to timeout for connecting, equal to 10 s, because we want to
guarantee that measurements eventually terminate. Also, since guarantee that measurements eventually terminate. Also, since
often censorship is implemented by timing out, we don't want often censorship is implemented by timing out, we don't want

88
internal/tutorial/measurex/chapter02/main.go
View File

@ -70,10 +70,12 @@ func main() {
// //
// ### Printing the measurement // ### Printing the measurement
// //
// The rest of the main function is just like in the previous chapter. // The rest of the main function is just like in the previous
// chapter. Like we did before, we convert the obtained measurement
// to the "archival" data format before printing.
// //
// ```Go // ```Go
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
@ -99,27 +101,31 @@ func main() {
// //
// // This block contains the results of the connect syscall // // This block contains the results of the connect syscall
// // using the df-008-netevents data format. // // using the df-008-netevents data format.
// "connect": [ // "tcp_connect": [{
// { // "ip": "8.8.4.4",
// "address": "8.8.4.4:443", // "port": 443,
// "t": 0.020303,
// "status": {
// "blocked": false,
// "failure": null, // "failure": null,
// "operation": "connect", // "success": true
// "proto": "tcp", // },
// "t": 0.026879041, // "started": 0.000109292,
// "started": 8.8625e-05, // "oddity": ""
// "oddity": "" // }]
// }
// ]
// } // }
// ``` // ```
// //
// This JSON implements [the df-005-tcpconnect](https://github.com/ooni/spec/blob/master/data-formats/df-005-tcpconnect.md)
// OONI data format.
//
// This is what it says: // This is what it says:
// //
// - we are connecting a "tcp" socket; // - we are connecting a "tcp" socket;
// //
// - the destination endpoint address is "8.8.4.4:443"; // - the destination endpoint address is "8.8.4.4:443";
// //
// - connect terminated ~0.027 seconds into the program's life; // - connect terminated ~0.020 seconds into the program's life (see `t`);
// //
// - the operation succeeded (`failure` is `nil`). // - the operation succeeded (`failure` is `nil`).
// //
@ -139,19 +145,21 @@ func main() {
// { // {
// "network": "tcp", // "network": "tcp",
// "address": "127.0.0.1:1", // "address": "127.0.0.1:1",
// "connect": [ // "tcp_connect": [
// { // {
// "address": "127.0.0.1:1", // "ip": "127.0.0.1",
// "failure": "connection_refused", // "port": 1,
// "operation": "connect", // "t": 0.000457584,
// "proto": "tcp", // "status": {
// "t": 0.000372167, // "blocked": true,
// "started": 8.4917e-05, // "failure": "connection_refused",
// "success": false
// },
// "started": 0.000104792,
// "oddity": "tcp.connect.refused" // "oddity": "tcp.connect.refused"
// } // }
// ] // ]
// } // }
//
// ``` // ```
// //
// And here's an error telling us the connection was refused and // And here's an error telling us the connection was refused and
@ -171,14 +179,17 @@ func main() {
// { // {
// "network": "tcp", // "network": "tcp",
// "address": "8.8.4.4:1", // "address": "8.8.4.4:1",
// "connect": [ // "tcp_connect": [
// { // {
// "address": "8.8.4.4:1", // "ip": "8.8.4.4",
// "failure": "generic_timeout_error", // "port": 1,
// "operation": "connect", // "t": 10.006558625,
// "proto": "tcp", // "status": {
// "t": 10.005494583, // "blocked": true,
// "started": 8.4833e-05, // "failure": "generic_timeout_error",
// "success": false
// },
// "started": 9.55e-05,
// "oddity": "tcp.connect.timeout" // "oddity": "tcp.connect.timeout"
// } // }
// ] // ]
@ -202,21 +213,26 @@ func main() {
// { // {
// "network": "tcp", // "network": "tcp",
// "address": "8.8.4.4:1", // "address": "8.8.4.4:1",
// "connect": [ // "tcp_connect": [
// { // {
// "address": "8.8.4.4:1", // "ip": "8.8.4.4",
// "failure": "generic_timeout_error", // "port": 1,
// "operation": "connect", // "t": 0.105445125,
// "proto": "tcp", // "status": {
// "t": 0.10148025, // "blocked": true,
// "started": 0.000122375, // "failure": "generic_timeout_error",
// "success": false
// },
// "started": 9.4083e-05,
// "oddity": "tcp.connect.timeout" // "oddity": "tcp.connect.timeout"
// } // }
// ] // ]
// } // }
// ``` // ```
// //
// We see a timeout after ~0.1s. We enforce a reasonably small // We see a timeout after ~0.1s.
//
// We enforce a reasonably small
// timeout for connecting, equal to 10 s, because we want to // timeout for connecting, equal to 10 s, because we want to
// guarantee that measurements eventually terminate. Also, since // guarantee that measurements eventually terminate. Also, since
// often censorship is implemented by timing out, we don't want // often censorship is implemented by timing out, we don't want

395
internal/tutorial/measurex/chapter03/README.md
View File

@ -57,7 +57,7 @@ Also this operation returns a measurement, which
we print using the usual three-liner. we print using the usual three-liner.
```Go ```Go
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
@ -83,42 +83,22 @@ be generated and inserted into a `Measurement`.)
{ {
"domain": "example.com", "domain": "example.com",
// This block tells us about the UDP connect events
// where we bind to the server's endpoint
"connect": [
{
"address": "8.8.4.4:53",
"failure": null,
"operation": "connect",
"proto": "udp",
"t": 0.00043175,
"started": 0.000191958,
"oddity": ""
},
{
"address": "8.8.4.4:53",
"failure": null,
"operation": "connect",
"proto": "udp",
"t": 0.042198458,
"started": 0.042113208,
"oddity": ""
}
],
// This block shows the read and write events // This block shows the read and write events
// occurred on the sockets (because we control // occurred on the sockets (because we control
// in full the implementation of this DNS // in full the implementation of this DNS
// over UDP resolver, we can see these events) // over UDP resolver, we can see these events)
"read_write": [ //
// See https://github.com/ooni/spec/blob/master/data-formats/df-008-netevents.md
// for a description of this data format.
"network_events": [
{ {
"address": "8.8.4.4:53", "address": "8.8.4.4:53",
"failure": null, "failure": null,
"num_bytes": 29, "num_bytes": 29,
"operation": "write", "operation": "write",
"proto": "udp", "proto": "udp",
"t": 0.000459583, "t": 0.00048825,
"started": 0.00043825, "started": 0.000462917,
"oddity": "" "oddity": ""
}, },
{ {
@ -127,8 +107,8 @@ be generated and inserted into a `Measurement`.)
"num_bytes": 45, "num_bytes": 45,
"operation": "read", "operation": "read",
"proto": "udp", "proto": "udp",
"t": 0.041955792, "t": 0.022081833,
"started": 0.000471833, "started": 0.000502625,
"oddity": "" "oddity": ""
}, },
{ {
@ -137,8 +117,8 @@ be generated and inserted into a `Measurement`.)
"num_bytes": 29, "num_bytes": 29,
"operation": "write", "operation": "write",
"proto": "udp", "proto": "udp",
"t": 0.042218917, "t": 0.022433083,
"started": 0.042203, "started": 0.022423875,
"oddity": "" "oddity": ""
}, },
{ {
@ -147,19 +127,59 @@ be generated and inserted into a `Measurement`.)
"num_bytes": 57, "num_bytes": 57,
"operation": "read", "operation": "read",
"proto": "udp", "proto": "udp",
"t": 0.196646583, "t": 0.046706,
"started": 0.042233167, "started": 0.022443833,
"oddity": "" "oddity": ""
} }
], ],
// This block shows the query we sent (encoded as base64)
// and the response we received. Here we clearly see
// that we perform two DNS "round trip" (i.e., send request
// and receive response) to resolve a domain: one for
// A and the other for AAAA.
//
// We don't have a specification for this data format yet.
"dns_events": [
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "dGwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000205083,
"t": 0.022141333,
"failure": null,
"raw_reply": {
"data": "dGyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAEuIAARduNgi",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "Ts8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
},
"started": 0.022221417,
"t": 0.046733125,
"failure": null,
"raw_reply": {
"data": "Ts+BgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAFOQABAmBigAAiAAAQJIGJMlyBlG",
"format": "base64"
}
}
],
// This is the same kind of result as before, we // This is the same kind of result as before, we
// show the emitted queries and the resolved addrs. // show the emitted queries and the resolved addrs.
// //
// Also note how here the resolver_address is the // Also note how here the resolver_address is the
// correct endpoint address and the engine tells us // correct endpoint address and the engine tells us
// that we're using DNS over UDP. // that we're using DNS over UDP.
"lookup_host": [ "queries": [
{ {
"answers": [ "answers": [
{ {
@ -172,8 +192,8 @@ be generated and inserted into a `Measurement`.)
"hostname": "example.com", "hostname": "example.com",
"query_type": "A", "query_type": "A",
"resolver_address": "8.8.4.4:53", "resolver_address": "8.8.4.4:53",
"t": 0.196777042, "t": 0.046766833,
"started": 0.000118542, "started": 0.000124375,
"oddity": "" "oddity": ""
}, },
{ {
@ -188,48 +208,10 @@ be generated and inserted into a `Measurement`.)
"hostname": "example.com", "hostname": "example.com",
"query_type": "AAAA", "query_type": "AAAA",
"resolver_address": "8.8.4.4:53", "resolver_address": "8.8.4.4:53",
"t": 0.196777042, "t": 0.046766833,
"started": 0.000118542, "started": 0.000124375,
"oddity": "" "oddity": ""
} }
],
// This block shows the query we sent (encoded as base64)
// and the response we received. Here we clearly see
// that we perform two DNS "round trip" (i.e., send request
// and receive response) to resolve a domain: one for
// A and the other for AAAA.
"dns_round_trip": [
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "PrcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000191625,
"t": 0.041998667,
"failure": null,
"raw_reply": {
"data": "PreBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAE8BAARduNgi",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "LAwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
},
"started": 0.04210775,
"t": 0.196701333,
"failure": null,
"raw_reply": {
"data": "LAyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAE6nABAmBigAAiAAAQJIGJMlyBlG",
"format": "base64"
}
}
] ]
} }
``` ```
@ -254,9 +236,42 @@ This produces the following JSON:
```JavaScript ```JavaScript
{ {
"domain": "antani.ooni.org", "domain": "antani.ooni.org",
"connect": [ /* snip */ ], "network_events": [ /* snip */ ],
"read_write": [ /* snip */ ],
"lookup_host": [ "dns_events": [
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "p7YBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB",
"format": "base64"
},
"started": 0.000152959,
"t": 0.051650917,
"failure": null,
"raw_reply": {
"data": "p7aBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "ILkBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB",
"format": "base64"
},
"started": 0.051755209,
"t": 0.101094375,
"failure": null,
"raw_reply": {
"data": "ILmBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==",
"format": "base64"
}
}
],
"queries": [
{ {
"answers": null, "answers": null,
"engine": "udp", "engine": "udp",
@ -264,8 +279,8 @@ This produces the following JSON:
"hostname": "antani.ooni.org", "hostname": "antani.ooni.org",
"query_type": "A", "query_type": "A",
"resolver_address": "8.8.4.4:53", "resolver_address": "8.8.4.4:53",
"t": 0.098208709, "t": 0.101241667,
"started": 8.95e-05, "started": 8.8e-05,
"oddity": "dns.lookup.nxdomain" "oddity": "dns.lookup.nxdomain"
}, },
{ {
@ -275,42 +290,10 @@ This produces the following JSON:
"hostname": "antani.ooni.org", "hostname": "antani.ooni.org",
"query_type": "AAAA", "query_type": "AAAA",
"resolver_address": "8.8.4.4:53", "resolver_address": "8.8.4.4:53",
"t": 0.098208709, "t": 0.101241667,
"started": 8.95e-05, "started": 8.8e-05,
"oddity": "dns.lookup.nxdomain" "oddity": "dns.lookup.nxdomain"
} }
],
"dns_round_trip": [
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "jLIBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB",
"format": "base64"
},
"started": 0.000141542,
"t": 0.034689417,
"failure": null,
"raw_reply": {
"data": "jLKBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "azEBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB",
"format": "base64"
},
"started": 0.034776709,
"t": 0.098170542,
"failure": null,
"raw_reply": {
"data": "azGBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==",
"format": "base64"
}
}
] ]
} }
``` ```
@ -323,10 +306,10 @@ Let us now decode one of the replies by using this program:
package main package main
import ( import (
"fmt" "fmt"
"encoding/base64" "encoding/base64"
"github.com/miekg/dns" "github.com/miekg/dns"
) )
func main() { func main() {
@ -366,16 +349,15 @@ Here's the corresponding JSON:
```JavaScript ```JavaScript
{ {
"domain": "example.com", "domain": "example.com",
"connect": [ /* snip */ ], "network_events": [
"read_write": [
{ {
"address": "182.92.22.222:53", "address": "182.92.22.222:53",
"failure": null, "failure": null,
"num_bytes": 29, "num_bytes": 29,
"operation": "write", "operation": "write",
"proto": "udp", "proto": "udp",
"t": 0.0005275, "t": 0.000479583,
"started": 0.000500209, "started": 0.00045525,
"oddity": "" "oddity": ""
}, },
{ {
@ -383,56 +365,58 @@ Here's the corresponding JSON:
"failure": "generic_timeout_error", /* <--- */ "failure": "generic_timeout_error", /* <--- */
"operation": "read", "operation": "read",
"proto": "udp", "proto": "udp",
"t": 5.001140125, "t": 5.006016292,
"started": 0.000544042, "started": 0.000491792,
"oddity": "" "oddity": ""
} }
], ],
"lookup_host": [ "dns_events": [
{
"answers": null,
"engine": "udp",
"failure": "generic_timeout_error", /* <--- */
"hostname": "example.com",
"query_type": "A",
"resolver_address": "182.92.22.222:53",
"t": 5.001462084,
"started": 0.000127917,
"oddity": "dns.lookup.timeout" /* <--- */
},
{
"answers": null,
"engine": "udp",
"failure": "generic_timeout_error",
"hostname": "example.com",
"query_type": "AAAA",
"resolver_address": "182.92.22.222:53",
"t": 5.001462084,
"started": 0.000127917,
"oddity": "dns.lookup.timeout"
}
],
"dns_round_trip": [
{ {
"engine": "udp", "engine": "udp",
"resolver_address": "182.92.22.222:53", "resolver_address": "182.92.22.222:53",
"raw_query": { "raw_query": {
"data": "ej8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", "data": "GRUBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64" "format": "base64"
}, },
"started": 0.000220584, "started": 0.00018225,
"t": 5.001317417, "t": 5.006185667,
"failure": "generic_timeout_error", "failure": "generic_timeout_error", /* <--- */
"raw_reply": null "raw_reply": null
} }
/* snip */
],
"queries": [
{
"answers": null,
"engine": "udp",
"failure": "generic_timeout_error", /* <--- */
"hostname": "example.com",
"query_type": "A",
"resolver_address": "182.92.22.222:53",
"t": 5.007385458,
"started": 0.000107583,
"oddity": "dns.lookup.timeout"
},
{
"answers": null,
"engine": "udp",
"failure": "generic_timeout_error", /* <--- */
"hostname": "example.com",
"query_type": "AAAA",
"resolver_address": "182.92.22.222:53",
"t": 5.007385458,
"started": 0.000107583,
"oddity": "dns.lookup.timeout"
}
] ]
} }
``` ```
We see that we fail with a timeout (I have marked some of them We see that we fail with a timeout (I have marked some of them
with comments inside the JSON). We see the timeout at three different with comments inside the JSON). We see the timeout at three different
levels of abstractions (from lower to higher abstraction): at the socket layer, levels of abstractions (from lower to higher abstraction): at the socket
during the DNS round trip, during the DNS lookup. layer (`network_events`), during the DNS round trip (`dns_events`),
during the DNS lookup (`queries`).
What we also see is that `t`'s value is ~5s when the `read` event What we also see is that `t`'s value is ~5s when the `read` event
fails, which tells us about the socket's read timeout. fails, which tells us about the socket's read timeout.
@ -451,18 +435,17 @@ Here's the answer I get:
```JavaScript ```JavaScript
{ {
"domain": "example.com", "domain": "example.com",
"connect": [ /* snip */ ],
// The I/O events look normal this time // The network events look normal this time
"read_write": [ "network_events": [
{ {
"address": "180.97.36.63:53", "address": "180.97.36.63:53",
"failure": null, "failure": null,
"num_bytes": 29, "num_bytes": 29,
"operation": "write", "operation": "write",
"proto": "udp", "proto": "udp",
"t": 0.000333583, "t": 0.000492125,
"started": 0.000312125, "started": 0.000467042,
"oddity": "" "oddity": ""
}, },
{ {
@ -471,8 +454,8 @@ Here's the answer I get:
"num_bytes": 29, "num_bytes": 29,
"operation": "read", "operation": "read",
"proto": "udp", "proto": "udp",
"t": 0.334948125, "t": 0.321373542,
"started": 0.000366625, "started": 0.000504833,
"oddity": "" "oddity": ""
}, },
{ {
@ -481,8 +464,8 @@ Here's the answer I get:
"num_bytes": 29, "num_bytes": 29,
"operation": "write", "operation": "write",
"proto": "udp", "proto": "udp",
"t": 0.3358025, "t": 0.322500875,
"started": 0.335725958, "started": 0.322450042,
"oddity": "" "oddity": ""
}, },
{ {
@ -491,15 +474,49 @@ Here's the answer I get:
"num_bytes": 29, "num_bytes": 29,
"operation": "read", "operation": "read",
"proto": "udp", "proto": "udp",
"t": 0.739987666, "t": 0.655514542,
"started": 0.335863875, "started": 0.322557667,
"oddity": "" "oddity": ""
} }
], ],
// But we see both in the error and in the oddity // Exercise: do like I did before and decode the messages
"dns_events": [
{
"engine": "udp",
"resolver_address": "180.97.36.63:53",
"raw_query": {
"data": "WcgBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000209875,
"t": 0.321504042,
"failure": null,
"raw_reply": {
"data": "WciBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "180.97.36.63:53",
"raw_query": {
"data": "I9oBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
},
"started": 0.321672042,
"t": 0.655680792,
"failure": null,
"raw_reply": {
"data": "I9qBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
}
}
],
// We see both in the error and in the oddity
// that the response was "REFUSED" // that the response was "REFUSED"
"lookup_host": [ "queries": [
{ {
"answers": null, "answers": null,
"engine": "udp", "engine": "udp",
@ -507,8 +524,8 @@ Here's the answer I get:
"hostname": "example.com", "hostname": "example.com",
"query_type": "A", "query_type": "A",
"resolver_address": "180.97.36.63:53", "resolver_address": "180.97.36.63:53",
"t": 0.7402975, "t": 0.655814875,
"started": 7.2291e-05, "started": 0.000107417,
"oddity": "dns.lookup.refused" "oddity": "dns.lookup.refused"
}, },
{ {
@ -518,44 +535,10 @@ Here's the answer I get:
"hostname": "example.com", "hostname": "example.com",
"query_type": "AAAA", "query_type": "AAAA",
"resolver_address": "180.97.36.63:53", "resolver_address": "180.97.36.63:53",
"t": 0.7402975, "t": 0.655814875,
"started": 7.2291e-05, "started": 0.000107417,
"oddity": "dns.lookup.refused" "oddity": "dns.lookup.refused"
} }
],
// Exercise: do like I did before and decode the messages
"dns_round_trip": [
{
"engine": "udp",
"resolver_address": "180.97.36.63:53",
"raw_query": {
"data": "crkBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000130666,
"t": 0.33509925,
"failure": null,
"raw_reply": {
"data": "crmBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "180.97.36.63:53",
"raw_query": {
"data": "ywcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
},
"started": 0.335321333,
"t": 0.740152375,
"failure": null,
"raw_reply": {
"data": "yweBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
}
}
] ]
} }
``` ```

395
internal/tutorial/measurex/chapter03/main.go
View File

@ -58,7 +58,7 @@ func main() {
// we print using the usual three-liner. // we print using the usual three-liner.
// //
// ```Go // ```Go
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
@ -84,42 +84,22 @@ func main() {
// { // {
// "domain": "example.com", // "domain": "example.com",
// //
// // This block tells us about the UDP connect events
// // where we bind to the server's endpoint
// "connect": [
// {
// "address": "8.8.4.4:53",
// "failure": null,
// "operation": "connect",
// "proto": "udp",
// "t": 0.00043175,
// "started": 0.000191958,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:53",
// "failure": null,
// "operation": "connect",
// "proto": "udp",
// "t": 0.042198458,
// "started": 0.042113208,
// "oddity": ""
// }
// ],
//
// // This block shows the read and write events // // This block shows the read and write events
// // occurred on the sockets (because we control // // occurred on the sockets (because we control
// // in full the implementation of this DNS // // in full the implementation of this DNS
// // over UDP resolver, we can see these events) // // over UDP resolver, we can see these events)
// "read_write": [ // //
// // See https://github.com/ooni/spec/blob/master/data-formats/df-008-netevents.md
// // for a description of this data format.
// "network_events": [
// { // {
// "address": "8.8.4.4:53", // "address": "8.8.4.4:53",
// "failure": null, // "failure": null,
// "num_bytes": 29, // "num_bytes": 29,
// "operation": "write", // "operation": "write",
// "proto": "udp", // "proto": "udp",
// "t": 0.000459583, // "t": 0.00048825,
// "started": 0.00043825, // "started": 0.000462917,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -128,8 +108,8 @@ func main() {
// "num_bytes": 45, // "num_bytes": 45,
// "operation": "read", // "operation": "read",
// "proto": "udp", // "proto": "udp",
// "t": 0.041955792, // "t": 0.022081833,
// "started": 0.000471833, // "started": 0.000502625,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -138,8 +118,8 @@ func main() {
// "num_bytes": 29, // "num_bytes": 29,
// "operation": "write", // "operation": "write",
// "proto": "udp", // "proto": "udp",
// "t": 0.042218917, // "t": 0.022433083,
// "started": 0.042203, // "started": 0.022423875,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -148,19 +128,59 @@ func main() {
// "num_bytes": 57, // "num_bytes": 57,
// "operation": "read", // "operation": "read",
// "proto": "udp", // "proto": "udp",
// "t": 0.196646583, // "t": 0.046706,
// "started": 0.042233167, // "started": 0.022443833,
// "oddity": "" // "oddity": ""
// } // }
// ], // ],
// //
// // This block shows the query we sent (encoded as base64)
// // and the response we received. Here we clearly see
// // that we perform two DNS "round trip" (i.e., send request
// // and receive response) to resolve a domain: one for
// // A and the other for AAAA.
// //
// // We don't have a specification for this data format yet.
// "dns_events": [
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "dGwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000205083,
// "t": 0.022141333,
// "failure": null,
// "raw_reply": {
// "data": "dGyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAEuIAARduNgi",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "Ts8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// },
// "started": 0.022221417,
// "t": 0.046733125,
// "failure": null,
// "raw_reply": {
// "data": "Ts+BgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAFOQABAmBigAAiAAAQJIGJMlyBlG",
// "format": "base64"
// }
// }
// ],
//
// // This is the same kind of result as before, we // // This is the same kind of result as before, we
// // show the emitted queries and the resolved addrs. // // show the emitted queries and the resolved addrs.
// // // //
// // Also note how here the resolver_address is the // // Also note how here the resolver_address is the
// // correct endpoint address and the engine tells us // // correct endpoint address and the engine tells us
// // that we're using DNS over UDP. // // that we're using DNS over UDP.
// "lookup_host": [ // "queries": [
// { // {
// "answers": [ // "answers": [
// { // {
@ -173,8 +193,8 @@ func main() {
// "hostname": "example.com", // "hostname": "example.com",
// "query_type": "A", // "query_type": "A",
// "resolver_address": "8.8.4.4:53", // "resolver_address": "8.8.4.4:53",
// "t": 0.196777042, // "t": 0.046766833,
// "started": 0.000118542, // "started": 0.000124375,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -189,48 +209,10 @@ func main() {
// "hostname": "example.com", // "hostname": "example.com",
// "query_type": "AAAA", // "query_type": "AAAA",
// "resolver_address": "8.8.4.4:53", // "resolver_address": "8.8.4.4:53",
// "t": 0.196777042, // "t": 0.046766833,
// "started": 0.000118542, // "started": 0.000124375,
// "oddity": "" // "oddity": ""
// } // }
// ],
//
// // This block shows the query we sent (encoded as base64)
// // and the response we received. Here we clearly see
// // that we perform two DNS "round trip" (i.e., send request
// // and receive response) to resolve a domain: one for
// // A and the other for AAAA.
// "dns_round_trip": [
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "PrcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000191625,
// "t": 0.041998667,
// "failure": null,
// "raw_reply": {
// "data": "PreBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAE8BAARduNgi",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "LAwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// },
// "started": 0.04210775,
// "t": 0.196701333,
// "failure": null,
// "raw_reply": {
// "data": "LAyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAE6nABAmBigAAiAAAQJIGJMlyBlG",
// "format": "base64"
// }
// }
// ] // ]
// } // }
// ``` // ```
@ -255,9 +237,42 @@ func main() {
// ```JavaScript // ```JavaScript
// { // {
// "domain": "antani.ooni.org", // "domain": "antani.ooni.org",
// "connect": [ /* snip */ ], // "network_events": [ /* snip */ ],
// "read_write": [ /* snip */ ], //
// "lookup_host": [ // "dns_events": [
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "p7YBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB",
// "format": "base64"
// },
// "started": 0.000152959,
// "t": 0.051650917,
// "failure": null,
// "raw_reply": {
// "data": "p7aBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "ILkBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB",
// "format": "base64"
// },
// "started": 0.051755209,
// "t": 0.101094375,
// "failure": null,
// "raw_reply": {
// "data": "ILmBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==",
// "format": "base64"
// }
// }
// ],
//
// "queries": [
// { // {
// "answers": null, // "answers": null,
// "engine": "udp", // "engine": "udp",
@ -265,8 +280,8 @@ func main() {
// "hostname": "antani.ooni.org", // "hostname": "antani.ooni.org",
// "query_type": "A", // "query_type": "A",
// "resolver_address": "8.8.4.4:53", // "resolver_address": "8.8.4.4:53",
// "t": 0.098208709, // "t": 0.101241667,
// "started": 8.95e-05, // "started": 8.8e-05,
// "oddity": "dns.lookup.nxdomain" // "oddity": "dns.lookup.nxdomain"
// }, // },
// { // {
@ -276,42 +291,10 @@ func main() {
// "hostname": "antani.ooni.org", // "hostname": "antani.ooni.org",
// "query_type": "AAAA", // "query_type": "AAAA",
// "resolver_address": "8.8.4.4:53", // "resolver_address": "8.8.4.4:53",
// "t": 0.098208709, // "t": 0.101241667,
// "started": 8.95e-05, // "started": 8.8e-05,
// "oddity": "dns.lookup.nxdomain" // "oddity": "dns.lookup.nxdomain"
// } // }
// ],
// "dns_round_trip": [
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "jLIBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB",
// "format": "base64"
// },
// "started": 0.000141542,
// "t": 0.034689417,
// "failure": null,
// "raw_reply": {
// "data": "jLKBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "azEBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB",
// "format": "base64"
// },
// "started": 0.034776709,
// "t": 0.098170542,
// "failure": null,
// "raw_reply": {
// "data": "azGBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==",
// "format": "base64"
// }
// }
// ] // ]
// } // }
// ``` // ```
@ -324,10 +307,10 @@ func main() {
// package main // package main
// //
// import ( // import (
// "fmt" // "fmt"
// "encoding/base64" // "encoding/base64"
// //
// "github.com/miekg/dns" // "github.com/miekg/dns"
// ) // )
// //
// func main() { // func main() {
@ -367,16 +350,15 @@ func main() {
// ```JavaScript // ```JavaScript
// { // {
// "domain": "example.com", // "domain": "example.com",
// "connect": [ /* snip */ ], // "network_events": [
// "read_write": [
// { // {
// "address": "182.92.22.222:53", // "address": "182.92.22.222:53",
// "failure": null, // "failure": null,
// "num_bytes": 29, // "num_bytes": 29,
// "operation": "write", // "operation": "write",
// "proto": "udp", // "proto": "udp",
// "t": 0.0005275, // "t": 0.000479583,
// "started": 0.000500209, // "started": 0.00045525,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -384,56 +366,58 @@ func main() {
// "failure": "generic_timeout_error", /* <--- */ // "failure": "generic_timeout_error", /* <--- */
// "operation": "read", // "operation": "read",
// "proto": "udp", // "proto": "udp",
// "t": 5.001140125, // "t": 5.006016292,
// "started": 0.000544042, // "started": 0.000491792,
// "oddity": "" // "oddity": ""
// } // }
// ], // ],
// "lookup_host": [ // "dns_events": [
// {
// "answers": null,
// "engine": "udp",
// "failure": "generic_timeout_error", /* <--- */
// "hostname": "example.com",
// "query_type": "A",
// "resolver_address": "182.92.22.222:53",
// "t": 5.001462084,
// "started": 0.000127917,
// "oddity": "dns.lookup.timeout" /* <--- */
// },
// {
// "answers": null,
// "engine": "udp",
// "failure": "generic_timeout_error",
// "hostname": "example.com",
// "query_type": "AAAA",
// "resolver_address": "182.92.22.222:53",
// "t": 5.001462084,
// "started": 0.000127917,
// "oddity": "dns.lookup.timeout"
// }
// ],
// "dns_round_trip": [
// { // {
// "engine": "udp", // "engine": "udp",
// "resolver_address": "182.92.22.222:53", // "resolver_address": "182.92.22.222:53",
// "raw_query": { // "raw_query": {
// "data": "ej8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", // "data": "GRUBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64" // "format": "base64"
// }, // },
// "started": 0.000220584, // "started": 0.00018225,
// "t": 5.001317417, // "t": 5.006185667,
// "failure": "generic_timeout_error", // "failure": "generic_timeout_error", /* <--- */
// "raw_reply": null // "raw_reply": null
// } // }
// /* snip */
// ],
// "queries": [
// {
// "answers": null,
// "engine": "udp",
// "failure": "generic_timeout_error", /* <--- */
// "hostname": "example.com",
// "query_type": "A",
// "resolver_address": "182.92.22.222:53",
// "t": 5.007385458,
// "started": 0.000107583,
// "oddity": "dns.lookup.timeout"
// },
// {
// "answers": null,
// "engine": "udp",
// "failure": "generic_timeout_error", /* <--- */
// "hostname": "example.com",
// "query_type": "AAAA",
// "resolver_address": "182.92.22.222:53",
// "t": 5.007385458,
// "started": 0.000107583,
// "oddity": "dns.lookup.timeout"
// }
// ] // ]
// } // }
// ``` // ```
// //
// We see that we fail with a timeout (I have marked some of them // We see that we fail with a timeout (I have marked some of them
// with comments inside the JSON). We see the timeout at three different // with comments inside the JSON). We see the timeout at three different
// levels of abstractions (from lower to higher abstraction): at the socket layer, // levels of abstractions (from lower to higher abstraction): at the socket
// during the DNS round trip, during the DNS lookup. // layer (`network_events`), during the DNS round trip (`dns_events`),
// during the DNS lookup (`queries`).
// //
// What we also see is that `t`'s value is ~5s when the `read` event // What we also see is that `t`'s value is ~5s when the `read` event
// fails, which tells us about the socket's read timeout. // fails, which tells us about the socket's read timeout.
@ -452,18 +436,17 @@ func main() {
// ```JavaScript // ```JavaScript
// { // {
// "domain": "example.com", // "domain": "example.com",
// "connect": [ /* snip */ ],
// //
// // The I/O events look normal this time // // The network events look normal this time
// "read_write": [ // "network_events": [
// { // {
// "address": "180.97.36.63:53", // "address": "180.97.36.63:53",
// "failure": null, // "failure": null,
// "num_bytes": 29, // "num_bytes": 29,
// "operation": "write", // "operation": "write",
// "proto": "udp", // "proto": "udp",
// "t": 0.000333583, // "t": 0.000492125,
// "started": 0.000312125, // "started": 0.000467042,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -472,8 +455,8 @@ func main() {
// "num_bytes": 29, // "num_bytes": 29,
// "operation": "read", // "operation": "read",
// "proto": "udp", // "proto": "udp",
// "t": 0.334948125, // "t": 0.321373542,
// "started": 0.000366625, // "started": 0.000504833,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -482,8 +465,8 @@ func main() {
// "num_bytes": 29, // "num_bytes": 29,
// "operation": "write", // "operation": "write",
// "proto": "udp", // "proto": "udp",
// "t": 0.3358025, // "t": 0.322500875,
// "started": 0.335725958, // "started": 0.322450042,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -492,15 +475,49 @@ func main() {
// "num_bytes": 29, // "num_bytes": 29,
// "operation": "read", // "operation": "read",
// "proto": "udp", // "proto": "udp",
// "t": 0.739987666, // "t": 0.655514542,
// "started": 0.335863875, // "started": 0.322557667,
// "oddity": "" // "oddity": ""
// } // }
// ], // ],
// //
// // But we see both in the error and in the oddity // // Exercise: do like I did before and decode the messages
// "dns_events": [
// {
// "engine": "udp",
// "resolver_address": "180.97.36.63:53",
// "raw_query": {
// "data": "WcgBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000209875,
// "t": 0.321504042,
// "failure": null,
// "raw_reply": {
// "data": "WciBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "180.97.36.63:53",
// "raw_query": {
// "data": "I9oBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// },
// "started": 0.321672042,
// "t": 0.655680792,
// "failure": null,
// "raw_reply": {
// "data": "I9qBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// }
// }
// ],
//
// // We see both in the error and in the oddity
// // that the response was "REFUSED" // // that the response was "REFUSED"
// "lookup_host": [ // "queries": [
// { // {
// "answers": null, // "answers": null,
// "engine": "udp", // "engine": "udp",
@ -508,8 +525,8 @@ func main() {
// "hostname": "example.com", // "hostname": "example.com",
// "query_type": "A", // "query_type": "A",
// "resolver_address": "180.97.36.63:53", // "resolver_address": "180.97.36.63:53",
// "t": 0.7402975, // "t": 0.655814875,
// "started": 7.2291e-05, // "started": 0.000107417,
// "oddity": "dns.lookup.refused" // "oddity": "dns.lookup.refused"
// }, // },
// { // {
@ -519,44 +536,10 @@ func main() {
// "hostname": "example.com", // "hostname": "example.com",
// "query_type": "AAAA", // "query_type": "AAAA",
// "resolver_address": "180.97.36.63:53", // "resolver_address": "180.97.36.63:53",
// "t": 0.7402975, // "t": 0.655814875,
// "started": 7.2291e-05, // "started": 0.000107417,
// "oddity": "dns.lookup.refused" // "oddity": "dns.lookup.refused"
// } // }
// ],
//
// // Exercise: do like I did before and decode the messages
// "dns_round_trip": [
// {
// "engine": "udp",
// "resolver_address": "180.97.36.63:53",
// "raw_query": {
// "data": "crkBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000130666,
// "t": 0.33509925,
// "failure": null,
// "raw_reply": {
// "data": "crmBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "180.97.36.63:53",
// "raw_query": {
// "data": "ywcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// },
// "started": 0.335321333,
// "t": 0.740152375,
// "failure": null,
// "raw_reply": {
// "data": "yweBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// }
// }
// ] // ]
// } // }
// ``` // ```

69
internal/tutorial/measurex/chapter04/README.md
View File

@ -69,7 +69,7 @@ As usual, the method to perform a measurement returns
the measurement itself, which we print below. the measurement itself, which we print below.
``` ```
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
@ -90,30 +90,16 @@ Let us comment the JSON in detail:
"network": "tcp", "network": "tcp",
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
// This block is generated when connecting to a TCP
// socket, as we've already seen in chapter02
"connect": [
{
"address": "8.8.4.4:443",
"failure": null,
"operation": "connect",
"proto": "tcp",
"t": 0.046959084,
"started": 0.022998875,
"oddity": ""
}
],
// These are the I/O events during the handshake // These are the I/O events during the handshake
"read_write": [ "network_events": [
{ {
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"failure": null, "failure": null,
"num_bytes": 280, "num_bytes": 280,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.048752875, "t": 0.048268333,
"started": 0.04874125, "started": 0.048246666,
"oddity": "" "oddity": ""
}, },
{ {
@ -122,18 +108,18 @@ Let us comment the JSON in detail:
"num_bytes": 517, "num_bytes": 517,
"operation": "read", "operation": "read",
"proto": "tcp", "proto": "tcp",
"t": 0.087221334, "t": 0.086214708,
"started": 0.048760417, "started": 0.048287791,
"oddity": "" "oddity": ""
}, },
{ {
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"failure": null, "failure": null,
"num_bytes": 4301, "num_bytes": 4303,
"operation": "read", "operation": "read",
"proto": "tcp", "proto": "tcp",
"t": 0.088843584, "t": 0.087951708,
"started": 0.088830959, "started": 0.08792725,
"oddity": "" "oddity": ""
}, },
{ {
@ -142,14 +128,33 @@ Let us comment the JSON in detail:
"num_bytes": 64, "num_bytes": 64,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.092078042, "t": 0.090097833,
"started": 0.092064042, "started": 0.090083875,
"oddity": "" "oddity": ""
} }
], ],
// This block contains information about the handshake // This block is generated when connecting to a TCP
"tls_handshake": [ // socket, as we've already seen in chapter02
"tcp_connect": [
{
"ip": "8.8.4.4",
"port": 443,
"t": 0.046022583,
"status": {
"blocked": false,
"failure": null,
"success": true
},
"started": 0.024424916,
"oddity": ""
}
],
// This block contains information about the handshake.
//
// See https://github.com/ooni/spec/blob/master/data-formats/df-006-tlshandshake.md
"tls_handshakes": [
{ {
"cipher_suite": "TLS_AES_128_GCM_SHA256", "cipher_suite": "TLS_AES_128_GCM_SHA256",
"failure": null, "failure": null,
@ -157,7 +162,7 @@ Let us comment the JSON in detail:
"tls_version": "TLSv1.3", "tls_version": "TLSv1.3",
"peer_certificates": [ "peer_certificates": [
{ {
"data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=",
"format": "base64" "format": "base64"
}, },
{ {
@ -169,7 +174,7 @@ Let us comment the JSON in detail:
"format": "base64" "format": "base64"
} }
], ],
"t": 0.092117709, "t": 0.090150666,
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"server_name": "dns.google", "server_name": "dns.google",
"alpn": [ "alpn": [
@ -179,16 +184,12 @@ Let us comment the JSON in detail:
"no_tls_verify": false, "no_tls_verify": false,
"oddity": "", "oddity": "",
"proto": "tcp", "proto": "tcp",
"started": 0.047288542 "started": 0.04635975
} }
] ]
} }
``` ```
All the data formats we're using here are, by the way,
compatible with the data formats specified at
https://github.com/ooni/spec/tree/master/data-formats.
### Suggested follow-up experiments ### Suggested follow-up experiments
Try to run experiments in the following scenarios, and Try to run experiments in the following scenarios, and

69
internal/tutorial/measurex/chapter04/main.go
View File

@ -70,7 +70,7 @@ func main() {
// the measurement itself, which we print below. // the measurement itself, which we print below.
// //
// ``` // ```
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
@ -91,30 +91,16 @@ func main() {
// "network": "tcp", // "network": "tcp",
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// //
// // This block is generated when connecting to a TCP
// // socket, as we've already seen in chapter02
// "connect": [
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "operation": "connect",
// "proto": "tcp",
// "t": 0.046959084,
// "started": 0.022998875,
// "oddity": ""
// }
// ],
//
// // These are the I/O events during the handshake // // These are the I/O events during the handshake
// "read_write": [ // "network_events": [
// { // {
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "failure": null, // "failure": null,
// "num_bytes": 280, // "num_bytes": 280,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.048752875, // "t": 0.048268333,
// "started": 0.04874125, // "started": 0.048246666,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -123,18 +109,18 @@ func main() {
// "num_bytes": 517, // "num_bytes": 517,
// "operation": "read", // "operation": "read",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.087221334, // "t": 0.086214708,
// "started": 0.048760417, // "started": 0.048287791,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "failure": null, // "failure": null,
// "num_bytes": 4301, // "num_bytes": 4303,
// "operation": "read", // "operation": "read",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.088843584, // "t": 0.087951708,
// "started": 0.088830959, // "started": 0.08792725,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -143,14 +129,33 @@ func main() {
// "num_bytes": 64, // "num_bytes": 64,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.092078042, // "t": 0.090097833,
// "started": 0.092064042, // "started": 0.090083875,
// "oddity": "" // "oddity": ""
// } // }
// ], // ],
// //
// // This block contains information about the handshake // // This block is generated when connecting to a TCP
// "tls_handshake": [ // // socket, as we've already seen in chapter02
// "tcp_connect": [
// {
// "ip": "8.8.4.4",
// "port": 443,
// "t": 0.046022583,
// "status": {
// "blocked": false,
// "failure": null,
// "success": true
// },
// "started": 0.024424916,
// "oddity": ""
// }
// ],
//
// // This block contains information about the handshake.
// //
// // See https://github.com/ooni/spec/blob/master/data-formats/df-006-tlshandshake.md
// "tls_handshakes": [
// { // {
// "cipher_suite": "TLS_AES_128_GCM_SHA256", // "cipher_suite": "TLS_AES_128_GCM_SHA256",
// "failure": null, // "failure": null,
@ -158,7 +163,7 @@ func main() {
// "tls_version": "TLSv1.3", // "tls_version": "TLSv1.3",
// "peer_certificates": [ // "peer_certificates": [
// { // {
// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", // "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=",
// "format": "base64" // "format": "base64"
// }, // },
// { // {
@ -170,7 +175,7 @@ func main() {
// "format": "base64" // "format": "base64"
// } // }
// ], // ],
// "t": 0.092117709, // "t": 0.090150666,
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "server_name": "dns.google", // "server_name": "dns.google",
// "alpn": [ // "alpn": [
@ -180,16 +185,12 @@ func main() {
// "no_tls_verify": false, // "no_tls_verify": false,
// "oddity": "", // "oddity": "",
// "proto": "tcp", // "proto": "tcp",
// "started": 0.047288542 // "started": 0.04635975
// } // }
// ] // ]
// } // }
// ``` // ```
// //
// All the data formats we're using here are, by the way,
// compatible with the data formats specified at
// https://github.com/ooni/spec/tree/master/data-formats.
//
// ### Suggested follow-up experiments // ### Suggested follow-up experiments
// //
// Try to run experiments in the following scenarios, and // Try to run experiments in the following scenarios, and

64
internal/tutorial/measurex/chapter05/README.md
View File

@ -69,7 +69,7 @@ As we did in the previous chapters, here's the usual three
lines of code for printing the resulting measurement. lines of code for printing the resulting measurement.
``` ```
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
@ -100,15 +100,15 @@ Produces this JSON:
// are actually `recvfrom` and `sendto` but here // are actually `recvfrom` and `sendto` but here
// we follow the Go convention of using read/write // we follow the Go convention of using read/write
// more frequently than send/recv.) // more frequently than send/recv.)
"read_write": [ "network_events": [
{ {
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"failure": null, "failure": null,
"num_bytes": 1252, "num_bytes": 1252,
"operation": "write_to", "operation": "write_to",
"proto": "quic", "proto": "quic",
"t": 0.003903167, "t": 0.027184208,
"started": 0.0037395, "started": 0.027127208,
"oddity": "" "oddity": ""
}, },
{ {
@ -117,8 +117,8 @@ Produces this JSON:
"num_bytes": 1252, "num_bytes": 1252,
"operation": "read_from", "operation": "read_from",
"proto": "quic", "proto": "quic",
"t": 0.029389125, "t": 0.053116458,
"started": 0.002954792, "started": 0.025626583,
"oddity": "" "oddity": ""
}, },
{ {
@ -127,8 +127,8 @@ Produces this JSON:
"num_bytes": 1252, "num_bytes": 1252,
"operation": "write_to", "operation": "write_to",
"proto": "quic", "proto": "quic",
"t": 0.029757584, "t": 0.054538792,
"started": 0.02972325, "started": 0.054517542,
"oddity": "" "oddity": ""
}, },
{ {
@ -137,8 +137,8 @@ Produces this JSON:
"num_bytes": 1252, "num_bytes": 1252,
"operation": "read_from", "operation": "read_from",
"proto": "quic", "proto": "quic",
"t": 0.045039875, "t": 0.069144958,
"started": 0.029424792, "started": 0.053194208,
"oddity": "" "oddity": ""
}, },
{ {
@ -147,8 +147,8 @@ Produces this JSON:
"num_bytes": 1252, "num_bytes": 1252,
"operation": "read_from", "operation": "read_from",
"proto": "quic", "proto": "quic",
"t": 0.045055334, "t": 0.069183458,
"started": 0.045049625, "started": 0.069173292,
"oddity": "" "oddity": ""
}, },
{ {
@ -157,28 +157,28 @@ Produces this JSON:
"num_bytes": 1252, "num_bytes": 1252,
"operation": "read_from", "operation": "read_from",
"proto": "quic", "proto": "quic",
"t": 0.045073917, "t": 0.06920225,
"started": 0.045069667, "started": 0.069197875,
"oddity": "" "oddity": ""
}, },
{ {
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"failure": null, "failure": null,
"num_bytes": 1233, "num_bytes": 1216,
"operation": "read_from", "operation": "read_from",
"proto": "quic", "proto": "quic",
"t": 0.04508, "t": 0.069210958,
"started": 0.045075292, "started": 0.069206875,
"oddity": "" "oddity": ""
}, },
{ {
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"failure": null, "failure": null,
"num_bytes": 64, "num_bytes": 65,
"operation": "read_from", "operation": "read_from",
"proto": "quic", "proto": "quic",
"t": 0.045088167, "t": 0.069220667,
"started": 0.045081167, "started": 0.069217375,
"oddity": "" "oddity": ""
}, },
{ {
@ -187,8 +187,8 @@ Produces this JSON:
"num_bytes": 44, "num_bytes": 44,
"operation": "write_to", "operation": "write_to",
"proto": "quic", "proto": "quic",
"t": 0.045370417, "t": 0.069433417,
"started": 0.045338667, "started": 0.069417625,
"oddity": "" "oddity": ""
}, },
{ {
@ -197,8 +197,8 @@ Produces this JSON:
"num_bytes": 44, "num_bytes": 44,
"operation": "write_to", "operation": "write_to",
"proto": "quic", "proto": "quic",
"t": 0.045392125, "t": 0.069677625,
"started": 0.045380959, "started": 0.069647458,
"oddity": "" "oddity": ""
}, },
{ {
@ -207,8 +207,8 @@ Produces this JSON:
"num_bytes": 83, "num_bytes": 83,
"operation": "write_to", "operation": "write_to",
"proto": "quic", "proto": "quic",
"t": 0.047042542, "t": 0.073461917,
"started": 0.047001917, "started": 0.073432875,
"oddity": "" "oddity": ""
}, },
{ {
@ -217,15 +217,15 @@ Produces this JSON:
"num_bytes": 33, "num_bytes": 33,
"operation": "write_to", "operation": "write_to",
"proto": "quic", "proto": "quic",
"t": 0.047060834, "t": 0.073559417,
"started": 0.047046875, "started": 0.073542542,
"oddity": "" "oddity": ""
} }
], ],
// This section describes the QUIC handshake and it has // This section describes the QUIC handshake and it has
// basically the same fields as the TLS handshake. // basically the same fields as the TLS handshake.
"quic_handshake": [ "quic_handshakes": [
{ {
"cipher_suite": "TLS_CHACHA20_POLY1305_SHA256", "cipher_suite": "TLS_CHACHA20_POLY1305_SHA256",
"failure": null, "failure": null,
@ -233,7 +233,7 @@ Produces this JSON:
"tls_version": "TLSv1.3", "tls_version": "TLSv1.3",
"peer_certificates": [ "peer_certificates": [
{ {
"data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", "data": "MIIF4DCCBMigAwIBAgIQWiqDOVk83wAKAAAAAQivAjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTEwMDQwMjQzMzVaFw0yMTEyMjcwMjQzMzRaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUBr7csGnwBJnnh4nRPQFux/52IXuOIIUao2X4JQH7chvtuNyR4rxpeyL/zAYbFpohSdSfxrCFxrFWae+LzANL/VTPZPzPH2S2p1TNWy8rFWMU0OnRd3Ym5s1xOLBG9Lu28iH3h5+5yk5pGmvSnzA0Ra9Q5pjLJlUfnGi9ceP2uFM/SDRo1IEqNB7ukIeE5lmw7CJCyDafbWy8eFFPG69YRAjPb13NF4ijingt2JsUckgXJdWdoRf3KghM5ddvQhv5rdELjVUbI550MjfBQWoEUIxWEUsYNj/L4IbK46E4RpMnU+XnwTtinQYYgHVQxLKPtPdfYHp145DjiGXE9LMDAgMBAAGjggL5MIIC9TAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUo61UxicmPa371PCog8zI7NbrdkQwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQIGCisGAQQB1nkCBAIEgfMEgfAA7gB1AESUZS6w7s6vxEAH2Kj+KMDa5oK+2MsxtT/TM5a1toGoAAABfElmWjQAAAQDAEYwRAIgZpBA5ih3hRoZ1749kEcxdEcpzHUV3cS2zDHuz1WMy7gCIGtMqROCc/wrP01x1GXrk3M/qbHssnvhelxhNVbi4FTUAHUA9lyUL9F3MCIUVBgIMJRWjuNNExkzv98MLyALzE7xZOMAAAF8SWZaAwAABAMARjBEAiBRWeLjcLIQwBFdPEXa15s++b7kAKit86em9GR23F+7KQIgTHZgL7inapApbw5WJNhtEI78a5HHPsI+kU5LIDgpv7swDQYJKoZIhvcNAQELBQADggEBAGki3+h4nn12Ef449NirIIP5APx+1NQk3DDKT6PrpH3m+s/wQKXlJ8eNg6zJEBEtHxBdO0xI+/te2Bh1s6RU/iJZrVRAtol/xmn0AMvPQNL+JZUnYuryz2mwTpk5ZHnGHyZknbJDspB2oZkItDeDbvkMws+JKrCYbbHG4ZtcsoFYPrkjfyMLRemhj+qWvvMtUKUldsXbYhugQL44N+pWAJNyxEWUBcuEldItww/gSrl/O2alfGOTNvdcXT/nedrw+SJnci4m4oMTz+XWFkbf3yPXEikvUqcvQFe10F1wBr8wW6soM9nR7vKq7WUlnx1m4lghw/jStp2mWenT6YFp5Tw=",
"format": "base64" "format": "base64"
}, },
{ {
@ -245,7 +245,7 @@ Produces this JSON:
"format": "base64" "format": "base64"
} }
], ],
"t": 0.047042459, "t": 0.073469208,
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"server_name": "dns.google", "server_name": "dns.google",
"alpn": [ "alpn": [
@ -254,7 +254,7 @@ Produces this JSON:
"no_tls_verify": false, "no_tls_verify": false,
"oddity": "", "oddity": "",
"proto": "quic", "proto": "quic",
"started": 0.002154834 "started": 0.025061583
} }
] ]
} }

64
internal/tutorial/measurex/chapter05/main.go
View File

@ -70,7 +70,7 @@ func main() {
// lines of code for printing the resulting measurement. // lines of code for printing the resulting measurement.
// //
// ``` // ```
data, err := json.Marshal(m) data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed") runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
@ -101,15 +101,15 @@ func main() {
// // are actually `recvfrom` and `sendto` but here // // are actually `recvfrom` and `sendto` but here
// // we follow the Go convention of using read/write // // we follow the Go convention of using read/write
// // more frequently than send/recv.) // // more frequently than send/recv.)
// "read_write": [ // "network_events": [
// { // {
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "failure": null, // "failure": null,
// "num_bytes": 1252, // "num_bytes": 1252,
// "operation": "write_to", // "operation": "write_to",
// "proto": "quic", // "proto": "quic",
// "t": 0.003903167, // "t": 0.027184208,
// "started": 0.0037395, // "started": 0.027127208,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -118,8 +118,8 @@ func main() {
// "num_bytes": 1252, // "num_bytes": 1252,
// "operation": "read_from", // "operation": "read_from",
// "proto": "quic", // "proto": "quic",
// "t": 0.029389125, // "t": 0.053116458,
// "started": 0.002954792, // "started": 0.025626583,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -128,8 +128,8 @@ func main() {
// "num_bytes": 1252, // "num_bytes": 1252,
// "operation": "write_to", // "operation": "write_to",
// "proto": "quic", // "proto": "quic",
// "t": 0.029757584, // "t": 0.054538792,
// "started": 0.02972325, // "started": 0.054517542,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -138,8 +138,8 @@ func main() {
// "num_bytes": 1252, // "num_bytes": 1252,
// "operation": "read_from", // "operation": "read_from",
// "proto": "quic", // "proto": "quic",
// "t": 0.045039875, // "t": 0.069144958,
// "started": 0.029424792, // "started": 0.053194208,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -148,8 +148,8 @@ func main() {
// "num_bytes": 1252, // "num_bytes": 1252,
// "operation": "read_from", // "operation": "read_from",
// "proto": "quic", // "proto": "quic",
// "t": 0.045055334, // "t": 0.069183458,
// "started": 0.045049625, // "started": 0.069173292,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -158,28 +158,28 @@ func main() {
// "num_bytes": 1252, // "num_bytes": 1252,
// "operation": "read_from", // "operation": "read_from",
// "proto": "quic", // "proto": "quic",
// "t": 0.045073917, // "t": 0.06920225,
// "started": 0.045069667, // "started": 0.069197875,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "failure": null, // "failure": null,
// "num_bytes": 1233, // "num_bytes": 1216,
// "operation": "read_from", // "operation": "read_from",
// "proto": "quic", // "proto": "quic",
// "t": 0.04508, // "t": 0.069210958,
// "started": 0.045075292, // "started": 0.069206875,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "failure": null, // "failure": null,
// "num_bytes": 64, // "num_bytes": 65,
// "operation": "read_from", // "operation": "read_from",
// "proto": "quic", // "proto": "quic",
// "t": 0.045088167, // "t": 0.069220667,
// "started": 0.045081167, // "started": 0.069217375,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -188,8 +188,8 @@ func main() {
// "num_bytes": 44, // "num_bytes": 44,
// "operation": "write_to", // "operation": "write_to",
// "proto": "quic", // "proto": "quic",
// "t": 0.045370417, // "t": 0.069433417,
// "started": 0.045338667, // "started": 0.069417625,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -198,8 +198,8 @@ func main() {
// "num_bytes": 44, // "num_bytes": 44,
// "operation": "write_to", // "operation": "write_to",
// "proto": "quic", // "proto": "quic",
// "t": 0.045392125, // "t": 0.069677625,
// "started": 0.045380959, // "started": 0.069647458,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -208,8 +208,8 @@ func main() {
// "num_bytes": 83, // "num_bytes": 83,
// "operation": "write_to", // "operation": "write_to",
// "proto": "quic", // "proto": "quic",
// "t": 0.047042542, // "t": 0.073461917,
// "started": 0.047001917, // "started": 0.073432875,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -218,15 +218,15 @@ func main() {
// "num_bytes": 33, // "num_bytes": 33,
// "operation": "write_to", // "operation": "write_to",
// "proto": "quic", // "proto": "quic",
// "t": 0.047060834, // "t": 0.073559417,
// "started": 0.047046875, // "started": 0.073542542,
// "oddity": "" // "oddity": ""
// } // }
// ], // ],
// //
// // This section describes the QUIC handshake and it has // // This section describes the QUIC handshake and it has
// // basically the same fields as the TLS handshake. // // basically the same fields as the TLS handshake.
// "quic_handshake": [ // "quic_handshakes": [
// { // {
// "cipher_suite": "TLS_CHACHA20_POLY1305_SHA256", // "cipher_suite": "TLS_CHACHA20_POLY1305_SHA256",
// "failure": null, // "failure": null,
@ -234,7 +234,7 @@ func main() {
// "tls_version": "TLSv1.3", // "tls_version": "TLSv1.3",
// "peer_certificates": [ // "peer_certificates": [
// { // {
// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", // "data": "MIIF4DCCBMigAwIBAgIQWiqDOVk83wAKAAAAAQivAjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTEwMDQwMjQzMzVaFw0yMTEyMjcwMjQzMzRaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUBr7csGnwBJnnh4nRPQFux/52IXuOIIUao2X4JQH7chvtuNyR4rxpeyL/zAYbFpohSdSfxrCFxrFWae+LzANL/VTPZPzPH2S2p1TNWy8rFWMU0OnRd3Ym5s1xOLBG9Lu28iH3h5+5yk5pGmvSnzA0Ra9Q5pjLJlUfnGi9ceP2uFM/SDRo1IEqNB7ukIeE5lmw7CJCyDafbWy8eFFPG69YRAjPb13NF4ijingt2JsUckgXJdWdoRf3KghM5ddvQhv5rdELjVUbI550MjfBQWoEUIxWEUsYNj/L4IbK46E4RpMnU+XnwTtinQYYgHVQxLKPtPdfYHp145DjiGXE9LMDAgMBAAGjggL5MIIC9TAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUo61UxicmPa371PCog8zI7NbrdkQwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQIGCisGAQQB1nkCBAIEgfMEgfAA7gB1AESUZS6w7s6vxEAH2Kj+KMDa5oK+2MsxtT/TM5a1toGoAAABfElmWjQAAAQDAEYwRAIgZpBA5ih3hRoZ1749kEcxdEcpzHUV3cS2zDHuz1WMy7gCIGtMqROCc/wrP01x1GXrk3M/qbHssnvhelxhNVbi4FTUAHUA9lyUL9F3MCIUVBgIMJRWjuNNExkzv98MLyALzE7xZOMAAAF8SWZaAwAABAMARjBEAiBRWeLjcLIQwBFdPEXa15s++b7kAKit86em9GR23F+7KQIgTHZgL7inapApbw5WJNhtEI78a5HHPsI+kU5LIDgpv7swDQYJKoZIhvcNAQELBQADggEBAGki3+h4nn12Ef449NirIIP5APx+1NQk3DDKT6PrpH3m+s/wQKXlJ8eNg6zJEBEtHxBdO0xI+/te2Bh1s6RU/iJZrVRAtol/xmn0AMvPQNL+JZUnYuryz2mwTpk5ZHnGHyZknbJDspB2oZkItDeDbvkMws+JKrCYbbHG4ZtcsoFYPrkjfyMLRemhj+qWvvMtUKUldsXbYhugQL44N+pWAJNyxEWUBcuEldItww/gSrl/O2alfGOTNvdcXT/nedrw+SJnci4m4oMTz+XWFkbf3yPXEikvUqcvQFe10F1wBr8wW6soM9nR7vKq7WUlnx1m4lghw/jStp2mWenT6YFp5Tw=",
// "format": "base64" // "format": "base64"
// }, // },
// { // {
@ -246,7 +246,7 @@ func main() {
// "format": "base64" // "format": "base64"
// } // }
// ], // ],
// "t": 0.047042459, // "t": 0.073469208,
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "server_name": "dns.google", // "server_name": "dns.google",
// "alpn": [ // "alpn": [
@ -255,7 +255,7 @@ func main() {
// "no_tls_verify": false, // "no_tls_verify": false,
// "oddity": "", // "oddity": "",
// "proto": "quic", // "proto": "quic",
// "started": 0.002154834 // "started": 0.025061583
// } // }
// ] // ]
// } // }

131
internal/tutorial/measurex/chapter06/README.md
View File

@ -172,7 +172,7 @@ go doc ./internal/measurex.HTTPEndpointMeasurement
Let us now print the resulting measurement. Let us now print the resulting measurement.
```Go ```Go
print(m) print(measurex.NewArchivalHTTPEndpointMeasurement(m))
} }
``` ```
@ -182,7 +182,7 @@ Let us now print the resulting measurement.
Let us perform a vanilla run first: Let us perform a vanilla run first:
```bash ```bash
go run -race ./internal/tutorial/measurex/chapter06 go run -race ./internal/tutorial/measurex/chapter06 | jq
``` ```
This is the JSON output. Let us comment it in detail: This is the JSON output. Let us comment it in detail:
@ -196,31 +196,17 @@ This is the JSON output. Let us comment it in detail:
"network": "tcp", "network": "tcp",
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
// Internally, HTTPEndpointGetWithoutCookies calls
// TCPConnect and here we see the corresponding event
"connect": [
{
"address": "8.8.4.4:443",
"failure": null,
"operation": "connect",
"proto": "tcp",
"t": 0.02422375,
"started": 0.002269291,
"oddity": ""
}
],
// These are the I/O operations we have already seen // These are the I/O operations we have already seen
// in previous chapters // in previous chapters
"read_write": [ "network_events": [
{ {
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"failure": null, "failure": null,
"num_bytes": 280, "num_bytes": 280,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.024931791, "t": 0.045800292,
"started": 0.024910416, "started": 0.045782167,
"oddity": "" "oddity": ""
}, },
{ {
@ -229,18 +215,18 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 517, "num_bytes": 517,
"operation": "read", "operation": "read",
"proto": "tcp", "proto": "tcp",
"t": 0.063629791, "t": 0.082571,
"started": 0.024935666, "started": 0.045805458,
"oddity": "" "oddity": ""
}, },
{ {
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"failure": null, "failure": null,
"num_bytes": 4301, "num_bytes": 4303,
"operation": "read", "operation": "read",
"proto": "tcp", "proto": "tcp",
"t": 0.064183, "t": 0.084400542,
"started": 0.064144208, "started": 0.084372667,
"oddity": "" "oddity": ""
}, },
{ {
@ -249,8 +235,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 64, "num_bytes": 64,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.065464041, "t": 0.086762625,
"started": 0.065441333, "started": 0.086748292,
"oddity": "" "oddity": ""
}, },
{ {
@ -259,8 +245,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 86, "num_bytes": 86,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.067256083, "t": 0.087851,
"started": 0.067224375, "started": 0.087837625,
"oddity": "" "oddity": ""
}, },
{ {
@ -269,8 +255,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 201, "num_bytes": 201,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.067674416, "t": 0.089527292,
"started": 0.067652375, "started": 0.089507958,
"oddity": "" "oddity": ""
}, },
{ {
@ -279,8 +265,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 93, "num_bytes": 93,
"operation": "read", "operation": "read",
"proto": "tcp", "proto": "tcp",
"t": 0.086618708, "t": 0.168585625,
"started": 0.067599208, "started": 0.088068375,
"oddity": "" "oddity": ""
}, },
{ {
@ -289,18 +275,28 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 31, "num_bytes": 31,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.086703625, "t": 0.168713542,
"started": 0.0866745, "started": 0.168671417,
"oddity": "" "oddity": ""
}, },
{ {
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"failure": null, "failure": null,
"num_bytes": 2028, "num_bytes": 2000,
"operation": "read", "operation": "read",
"proto": "tcp", "proto": "tcp",
"t": 0.337785916, "t": 0.468671417,
"started": 0.086717333, "started": 0.168759333,
"oddity": ""
},
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 39,
"operation": "read",
"proto": "tcp",
"t": 0.47118175,
"started": 0.471169667,
"oddity": "" "oddity": ""
}, },
{ {
@ -309,8 +305,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 39, "num_bytes": 39,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.338514916, "t": 0.471335458,
"started": 0.338485375, "started": 0.471268583,
"oddity": "" "oddity": ""
}, },
{ {
@ -319,17 +315,25 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 24, "num_bytes": 24,
"operation": "write", "operation": "write",
"proto": "tcp", "proto": "tcp",
"t": 0.338800833, "t": 0.471865,
"started": 0.338788625, "started": 0.471836292,
"oddity": "" "oddity": ""
}, }
],
// Internally, HTTPEndpointGetWithoutCookies calls
// TCPConnect and here we see the corresponding event
"tcp_connect": [
{ {
"address": "8.8.4.4:443", "ip": "8.8.4.4",
"failure": "connection_already_closed", "port": 443,
"operation": "read", "t": 0.043644958,
"proto": "tcp", "status": {
"t": 0.338888041, "blocked": false,
"started": 0.338523291, "failure": null,
"success": true
},
"started": 0.022849458,
"oddity": "" "oddity": ""
} }
], ],
@ -339,7 +343,7 @@ This is the JSON output. Let us comment it in detail:
// specified a QUIC endpoint we would instead see here a // specified a QUIC endpoint we would instead see here a
// QUIC handshake event. And, we would not see any handshake // QUIC handshake event. And, we would not see any handshake
// if the URL was instead an HTTP URL. // if the URL was instead an HTTP URL.
"tls_handshake": [ "tls_handshakes": [
{ {
"cipher_suite": "TLS_AES_128_GCM_SHA256", "cipher_suite": "TLS_AES_128_GCM_SHA256",
"failure": null, "failure": null,
@ -347,7 +351,7 @@ This is the JSON output. Let us comment it in detail:
"tls_version": "TLSv1.3", "tls_version": "TLSv1.3",
"peer_certificates": [ "peer_certificates": [
{ {
"data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=",
"format": "base64" "format": "base64"
}, },
{ {
@ -359,7 +363,7 @@ This is the JSON output. Let us comment it in detail:
"format": "base64" "format": "base64"
} }
], ],
"t": 0.065514708, "t": 0.086816667,
"address": "8.8.4.4:443", "address": "8.8.4.4:443",
"server_name": "dns.google", "server_name": "dns.google",
"alpn": [ "alpn": [
@ -369,13 +373,13 @@ This is the JSON output. Let us comment it in detail:
"no_tls_verify": false, "no_tls_verify": false,
"oddity": "", "oddity": "",
"proto": "tcp", "proto": "tcp",
"started": 0.024404083 "started": 0.043971083
} }
], ],
// Finally here we see information about the round trip, which // Finally here we see information about the round trip, which
// is formatted according the df-001-httpt data format: // is formatted according to https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md:
"http_round_trip": [ "requests": [
{ {
// This field indicates whether there was an error during // This field indicates whether there was an error during
@ -389,21 +393,20 @@ This is the JSON output. Let us comment it in detail:
"headers": { "headers": {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"accept-language": "en-US;q=0.8,en;q=0.5", "accept-language": "en-US;q=0.8,en;q=0.5",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36" "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36"
} }
}, },
// This field contains the response status code, body, // This field contains the response status code, body, and headers.
// and headers.
"response": { "response": {
"code": 200, "code": 200,
"headers": { "headers": {
"accept-ranges": "none", "accept-ranges": "none",
"alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
"cache-control": "private", "cache-control": "private",
"content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-bSLcJjaotppZl3Y2moIaxg==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'", "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-y/OGliLR2gbEfTG2i4MAaw==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'",
"content-type": "text/html; charset=UTF-8", "content-type": "text/html; charset=UTF-8",
"date": "Fri, 24 Sep 2021 08:51:01 GMT", "date": "Fri, 05 Nov 2021 08:59:37 GMT",
"server": "scaffolding on HTTPServer2", "server": "scaffolding on HTTPServer2",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload", "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"vary": "Accept-Encoding", "vary": "Accept-Encoding",
@ -416,7 +419,7 @@ This is the JSON output. Let us comment it in detail:
// body: we don't want to read and submit to the OONI // body: we don't want to read and submit to the OONI
// collector large bodies. // collector large bodies.
"body": { "body": {
"data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJiU0xjSmphb3RwcFpsM1kybW9JYXhnPT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+", "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJ5L09HbGlMUjJnYkVmVEcyaTRNQWF3PT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+",
"format": "base64" "format": "base64"
}, },
@ -436,14 +439,14 @@ This is the JSON output. Let us comment it in detail:
// The t field is the moment where we finished the // The t field is the moment where we finished the
// round trip and saved the event. The started field // round trip and saved the event. The started field
// is instead when we started the round trip. // is instead when we started the round trip.
//
// You may notice that the start of the round trip // You may notice that the start of the round trip
// if after the `t` of the handshake. This tells us // if after the `t` of the handshake. This tells us
// that the code first connects, then handshakes, and // that the code first connects, then handshakes, and
// finally creates HTTP code for performing the // finally creates HTTP code for performing the
// round trip. // round trip.
"t": 0.338674625, "t": 0.471535167,
"started": 0.065926625, "started": 0.087176458,
// As usual we also compute an oddity value related // As usual we also compute an oddity value related
// in this case to the HTTP round trip. // in this case to the HTTP round trip.

131
internal/tutorial/measurex/chapter06/main.go
View File

@ -173,7 +173,7 @@ func main() {
// Let us now print the resulting measurement. // Let us now print the resulting measurement.
// //
// ```Go // ```Go
print(m) print(measurex.NewArchivalHTTPEndpointMeasurement(m))
} }
// ``` // ```
@ -183,7 +183,7 @@ func main() {
// Let us perform a vanilla run first: // Let us perform a vanilla run first:
// //
// ```bash // ```bash
// go run -race ./internal/tutorial/measurex/chapter06 // go run -race ./internal/tutorial/measurex/chapter06 | jq
// ``` // ```
// //
// This is the JSON output. Let us comment it in detail: // This is the JSON output. Let us comment it in detail:
@ -197,31 +197,17 @@ func main() {
// "network": "tcp", // "network": "tcp",
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// //
// // Internally, HTTPEndpointGetWithoutCookies calls
// // TCPConnect and here we see the corresponding event
// "connect": [
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "operation": "connect",
// "proto": "tcp",
// "t": 0.02422375,
// "started": 0.002269291,
// "oddity": ""
// }
// ],
//
// // These are the I/O operations we have already seen // // These are the I/O operations we have already seen
// // in previous chapters // // in previous chapters
// "read_write": [ // "network_events": [
// { // {
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "failure": null, // "failure": null,
// "num_bytes": 280, // "num_bytes": 280,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.024931791, // "t": 0.045800292,
// "started": 0.024910416, // "started": 0.045782167,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -230,18 +216,18 @@ func main() {
// "num_bytes": 517, // "num_bytes": 517,
// "operation": "read", // "operation": "read",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.063629791, // "t": 0.082571,
// "started": 0.024935666, // "started": 0.045805458,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "failure": null, // "failure": null,
// "num_bytes": 4301, // "num_bytes": 4303,
// "operation": "read", // "operation": "read",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.064183, // "t": 0.084400542,
// "started": 0.064144208, // "started": 0.084372667,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -250,8 +236,8 @@ func main() {
// "num_bytes": 64, // "num_bytes": 64,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.065464041, // "t": 0.086762625,
// "started": 0.065441333, // "started": 0.086748292,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -260,8 +246,8 @@ func main() {
// "num_bytes": 86, // "num_bytes": 86,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.067256083, // "t": 0.087851,
// "started": 0.067224375, // "started": 0.087837625,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -270,8 +256,8 @@ func main() {
// "num_bytes": 201, // "num_bytes": 201,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.067674416, // "t": 0.089527292,
// "started": 0.067652375, // "started": 0.089507958,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -280,8 +266,8 @@ func main() {
// "num_bytes": 93, // "num_bytes": 93,
// "operation": "read", // "operation": "read",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.086618708, // "t": 0.168585625,
// "started": 0.067599208, // "started": 0.088068375,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -290,18 +276,28 @@ func main() {
// "num_bytes": 31, // "num_bytes": 31,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.086703625, // "t": 0.168713542,
// "started": 0.0866745, // "started": 0.168671417,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "failure": null, // "failure": null,
// "num_bytes": 2028, // "num_bytes": 2000,
// "operation": "read", // "operation": "read",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.337785916, // "t": 0.468671417,
// "started": 0.086717333, // "started": 0.168759333,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 39,
// "operation": "read",
// "proto": "tcp",
// "t": 0.47118175,
// "started": 0.471169667,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -310,8 +306,8 @@ func main() {
// "num_bytes": 39, // "num_bytes": 39,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.338514916, // "t": 0.471335458,
// "started": 0.338485375, // "started": 0.471268583,
// "oddity": "" // "oddity": ""
// }, // },
// { // {
@ -320,17 +316,25 @@ func main() {
// "num_bytes": 24, // "num_bytes": 24,
// "operation": "write", // "operation": "write",
// "proto": "tcp", // "proto": "tcp",
// "t": 0.338800833, // "t": 0.471865,
// "started": 0.338788625, // "started": 0.471836292,
// "oddity": "" // "oddity": ""
// }, // }
// ],
//
// // Internally, HTTPEndpointGetWithoutCookies calls
// // TCPConnect and here we see the corresponding event
// "tcp_connect": [
// { // {
// "address": "8.8.4.4:443", // "ip": "8.8.4.4",
// "failure": "connection_already_closed", // "port": 443,
// "operation": "read", // "t": 0.043644958,
// "proto": "tcp", // "status": {
// "t": 0.338888041, // "blocked": false,
// "started": 0.338523291, // "failure": null,
// "success": true
// },
// "started": 0.022849458,
// "oddity": "" // "oddity": ""
// } // }
// ], // ],
@ -340,7 +344,7 @@ func main() {
// // specified a QUIC endpoint we would instead see here a // // specified a QUIC endpoint we would instead see here a
// // QUIC handshake event. And, we would not see any handshake // // QUIC handshake event. And, we would not see any handshake
// // if the URL was instead an HTTP URL. // // if the URL was instead an HTTP URL.
// "tls_handshake": [ // "tls_handshakes": [
// { // {
// "cipher_suite": "TLS_AES_128_GCM_SHA256", // "cipher_suite": "TLS_AES_128_GCM_SHA256",
// "failure": null, // "failure": null,
@ -348,7 +352,7 @@ func main() {
// "tls_version": "TLSv1.3", // "tls_version": "TLSv1.3",
// "peer_certificates": [ // "peer_certificates": [
// { // {
// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", // "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=",
// "format": "base64" // "format": "base64"
// }, // },
// { // {
@ -360,7 +364,7 @@ func main() {
// "format": "base64" // "format": "base64"
// } // }
// ], // ],
// "t": 0.065514708, // "t": 0.086816667,
// "address": "8.8.4.4:443", // "address": "8.8.4.4:443",
// "server_name": "dns.google", // "server_name": "dns.google",
// "alpn": [ // "alpn": [
@ -370,13 +374,13 @@ func main() {
// "no_tls_verify": false, // "no_tls_verify": false,
// "oddity": "", // "oddity": "",
// "proto": "tcp", // "proto": "tcp",
// "started": 0.024404083 // "started": 0.043971083
// } // }
// ], // ],
// //
// // Finally here we see information about the round trip, which // // Finally here we see information about the round trip, which
// // is formatted according the df-001-httpt data format: // // is formatted according to https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md:
// "http_round_trip": [ // "requests": [
// { // {
// //
// // This field indicates whether there was an error during // // This field indicates whether there was an error during
@ -390,21 +394,20 @@ func main() {
// "headers": { // "headers": {
// "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", // "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
// "accept-language": "en-US;q=0.8,en;q=0.5", // "accept-language": "en-US;q=0.8,en;q=0.5",
// "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36" // "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36"
// } // }
// }, // },
// //
// // This field contains the response status code, body, // // This field contains the response status code, body, and headers.
// // and headers.
// "response": { // "response": {
// "code": 200, // "code": 200,
// "headers": { // "headers": {
// "accept-ranges": "none", // "accept-ranges": "none",
// "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", // "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
// "cache-control": "private", // "cache-control": "private",
// "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-bSLcJjaotppZl3Y2moIaxg==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'", // "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-y/OGliLR2gbEfTG2i4MAaw==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'",
// "content-type": "text/html; charset=UTF-8", // "content-type": "text/html; charset=UTF-8",
// "date": "Fri, 24 Sep 2021 08:51:01 GMT", // "date": "Fri, 05 Nov 2021 08:59:37 GMT",
// "server": "scaffolding on HTTPServer2", // "server": "scaffolding on HTTPServer2",
// "strict-transport-security": "max-age=31536000; includeSubDomains; preload", // "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
// "vary": "Accept-Encoding", // "vary": "Accept-Encoding",
@ -417,7 +420,7 @@ func main() {
// // body: we don't want to read and submit to the OONI // // body: we don't want to read and submit to the OONI
// // collector large bodies. // // collector large bodies.
// "body": { // "body": {
// "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJiU0xjSmphb3RwcFpsM1kybW9JYXhnPT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+", // "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJ5L09HbGlMUjJnYkVmVEcyaTRNQWF3PT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+",
// "format": "base64" // "format": "base64"
// }, // },
// //
@ -437,14 +440,14 @@ func main() {
// // The t field is the moment where we finished the // // The t field is the moment where we finished the
// // round trip and saved the event. The started field // // round trip and saved the event. The started field
// // is instead when we started the round trip. // // is instead when we started the round trip.
// // //
// // You may notice that the start of the round trip // // You may notice that the start of the round trip
// // if after the `t` of the handshake. This tells us // // if after the `t` of the handshake. This tells us
// // that the code first connects, then handshakes, and // // that the code first connects, then handshakes, and
// // finally creates HTTP code for performing the // // finally creates HTTP code for performing the
// // round trip. // // round trip.
// "t": 0.338674625, // "t": 0.471535167,
// "started": 0.065926625, // "started": 0.087176458,
// //
// // As usual we also compute an oddity value related // // As usual we also compute an oddity value related
// // in this case to the HTTP round trip. // // in this case to the HTTP round trip.

8
internal/tutorial/measurex/chapter07/README.md
View File

@ -111,7 +111,8 @@ We are almost done now: we loop over all the endpoints and apply the
} }
``` ```
Finally, we print the results. Finally, we print the results. (Note that here we are not
converting to the OONI archival data format.)
```Go ```Go
print(m) print(m)
@ -128,7 +129,10 @@ go run -race ./internal/tutorial/measurex/chapter07 | jq
``` ```
Please, check the JSON output. Do you recognize the fields Please, check the JSON output. Do you recognize the fields
we have described in previous chapters? we have described in previous chapters, even though we didn't
convert to the OONI data format? Can you modify the code to
use the OONI data format in the output by calling the proper
conversion functions exported by `measurex`?
Can you provoke common errors such as DNS resolution Can you provoke common errors such as DNS resolution
errors, TCP connect errors, TLS handshake errors, and errors, TCP connect errors, TLS handshake errors, and

8
internal/tutorial/measurex/chapter07/main.go
View File

@ -112,7 +112,8 @@ func main() {
} }
// ``` // ```
// //
// Finally, we print the results. // Finally, we print the results. (Note that here we are not
// converting to the OONI archival data format.)
// //
// ```Go // ```Go
print(m) print(m)
@ -129,7 +130,10 @@ func main() {
// ``` // ```
// //
// Please, check the JSON output. Do you recognize the fields // Please, check the JSON output. Do you recognize the fields
// we have described in previous chapters? // we have described in previous chapters, even though we didn't
// convert to the OONI data format? Can you modify the code to
// use the OONI data format in the output by calling the proper
// conversion functions exported by `measurex`?
// //
// Can you provoke common errors such as DNS resolution // Can you provoke common errors such as DNS resolution
// errors, TCP connect errors, TLS handshake errors, and // errors, TCP connect errors, TLS handshake errors, and

9
internal/tutorial/measurex/chapter08/README.md
View File

@ -101,6 +101,15 @@ This is it. The rest of the program is exactly the same.
for _, epnt := range httpEndpoints { for _, epnt := range httpEndpoints {
m.Endpoints = append(m.Endpoints, mx.HTTPEndpointGetWithoutCookies(ctx, epnt)) m.Endpoints = append(m.Endpoints, mx.HTTPEndpointGetWithoutCookies(ctx, epnt))
} }
```
(Note that here, like in the previous chapter, we are not converting
to the OONI data format. Rather, we're just dumping the internally
used data structures. Exercise: can you modify this program to emit
a JSON compliant with the OONI data format by using the proper]
conversion functions exported by `measurex`?)
```Go
print(m) print(m)
} }

9
internal/tutorial/measurex/chapter08/main.go
View File

@ -102,6 +102,15 @@ func main() {
for _, epnt := range httpEndpoints { for _, epnt := range httpEndpoints {
m.Endpoints = append(m.Endpoints, mx.HTTPEndpointGetWithoutCookies(ctx, epnt)) m.Endpoints = append(m.Endpoints, mx.HTTPEndpointGetWithoutCookies(ctx, epnt))
} }
// ```
//
// (Note that here, like in the previous chapter, we are not converting
// to the OONI data format. Rather, we're just dumping the internally
// used data structures. Exercise: can you modify this program to emit
// a JSON compliant with the OONI data format by using the proper]
// conversion functions exported by `measurex`?)
//
// ```Go
print(m) print(m)
} }

5
internal/tutorial/measurex/chapter09/README.md
View File

@ -101,6 +101,11 @@ fully measured, this method closes the returned channel.
Like we did before, we append the resulting measurements to Like we did before, we append the resulting measurements to
our `m` container and we print it. our `m` container and we print it.
Exercise: here we're not using the OONI data format and we're
instead printing the internally used data structures. Can
you modify the code to emit data using OONI's data format here?
(Hint: there are conversion functions in `measurex`.)
```Go ```Go
print(m) print(m)
} }

5
internal/tutorial/measurex/chapter09/main.go
View File

@ -102,6 +102,11 @@ func main() {
// Like we did before, we append the resulting measurements to // Like we did before, we append the resulting measurements to
// our `m` container and we print it. // our `m` container and we print it.
// //
// Exercise: here we're not using the OONI data format and we're
// instead printing the internally used data structures. Can
// you modify the code to emit data using OONI's data format here?
// (Hint: there are conversion functions in `measurex`.)
//
// ```Go // ```Go
print(m) print(m)
} }

7
internal/tutorial/measurex/chapter12/README.md
View File

@ -32,7 +32,7 @@ import (
) )
type measurement struct { type measurement struct {
URLs []*measurex.URLMeasurement URLs []*measurex.ArchivalURLMeasurement
} }
func print(v interface{}) { func print(v interface{}) {
@ -67,7 +67,7 @@ is closed when done by `MeasureURLAndFollowRedirections`, so we leave the loop.
```Go ```Go
for m := range mx.MeasureURLAndFollowRedirections(ctx, *URL, headers, cookies) { for m := range mx.MeasureURLAndFollowRedirections(ctx, *URL, headers, cookies) {
all.URLs = append(all.URLs, m) all.URLs = append(all.URLs, measurex.NewArchivalURLMeasurement(m))
} }
print(all) print(all)
} }
@ -86,6 +86,9 @@ Take a look at the JSON. You should see several redirects
and that we measure each endpoint of each redirect, including and that we measure each endpoint of each redirect, including
QUIC endpoints that we discover on the way. QUIC endpoints that we discover on the way.
Exercise: remove code for converting to OONI data format
and compare output with previous chapter. See any difference?
## Conclusion ## Conclusion
We have introduced `MeasureURLAndFollowRedirect`, the We have introduced `MeasureURLAndFollowRedirect`, the

7
internal/tutorial/measurex/chapter12/main.go
View File

@ -33,7 +33,7 @@ import (
) )
type measurement struct { type measurement struct {
URLs []*measurex.URLMeasurement URLs []*measurex.ArchivalURLMeasurement
} }
func print(v interface{}) { func print(v interface{}) {
@ -68,7 +68,7 @@ func main() {
// //
// ```Go // ```Go
for m := range mx.MeasureURLAndFollowRedirections(ctx, *URL, headers, cookies) { for m := range mx.MeasureURLAndFollowRedirections(ctx, *URL, headers, cookies) {
all.URLs = append(all.URLs, m) all.URLs = append(all.URLs, measurex.NewArchivalURLMeasurement(m))
} }
print(all) print(all)
} }
@ -87,6 +87,9 @@ func main() {
// and that we measure each endpoint of each redirect, including // and that we measure each endpoint of each redirect, including
// QUIC endpoints that we discover on the way. // QUIC endpoints that we discover on the way.
// //
// Exercise: remove code for converting to OONI data format
// and compare output with previous chapter. See any difference?
//
// ## Conclusion // ## Conclusion
// //
// We have introduced `MeasureURLAndFollowRedirect`, the // We have introduced `MeasureURLAndFollowRedirect`, the

23
internal/tutorial/measurex/chapter14/README.md
View File

@ -45,10 +45,10 @@ that a Web Connectivity measurement should have.
```Go ```Go
type measurement struct { type measurement struct {
Queries []*measurex.DNSLookupEvent `json:"queries"` Queries []*measurex.ArchivalDNSLookupEvent `json:"queries"`
TCPConnect []*measurex.NetworkEvent `json:"tcp_connect"` TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"`
TLSHandshakes []*measurex.TLSHandshakeEvent `json:"tls_handshakes"` TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.HTTPRoundTripEvent `json:"requests"` Requests []*measurex.ArchivalHTTPRoundTripEvent `json:"requests"`
} }
``` ```
@ -96,7 +96,8 @@ the input URL's domain using the system resolver.
```Go ```Go
dns := mx.LookupHostSystem(ctx, parsedURL.Hostname()) dns := mx.LookupHostSystem(ctx, parsedURL.Hostname())
m.Queries = append(m.Queries, dns.LookupHost...) m.Queries = append(
m.Queries, measurex.NewArchivalDNSLookupEventList(dns.LookupHost)...)
``` ```
@ -128,7 +129,8 @@ whether the input URL is HTTP or HTTPS.
switch parsedURL.Scheme { switch parsedURL.Scheme {
case "http": case "http":
tcp := mx.TCPConnect(ctx, epnt.Address) tcp := mx.TCPConnect(ctx, epnt.Address)
m.TCPConnect = append(m.TCPConnect, tcp.Connect...) m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...)
case "https": case "https":
config := &tls.Config{ config := &tls.Config{
ServerName: parsedURL.Hostname(), ServerName: parsedURL.Hostname(),
@ -136,8 +138,10 @@ whether the input URL is HTTP or HTTPS.
RootCAs: netxlite.NewDefaultCertPool(), RootCAs: netxlite.NewDefaultCertPool(),
} }
tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config) tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config)
m.TCPConnect = append(m.TCPConnect, tls.Connect...) m.TCPConnect = append(
m.TLSHandshakes = append(m.TLSHandshakes, tls.TLSHandshake...) m.TCPConnect, measurex.NewArchivalTCPConnectList(tls.Connect)...)
m.TLSHandshakes = append(m.TLSHandshakes,
measurex.NewArchivalQUICTLSHandshakeEventList(tls.TLSHandshake)...)
} }
} }
@ -222,7 +226,8 @@ the chapters we have seen so far.
```Go ```Go
m.Requests = append(m.Requests, db.AsMeasurement().HTTPRoundTrip...) m.Requests = append(m.Requests, measurex.NewArchivalHTTPRoundTripEventList(
db.AsMeasurement().HTTPRoundTrip)...)
return m, nil return m, nil
} }

23
internal/tutorial/measurex/chapter14/main.go
View File

@ -46,10 +46,10 @@ func print(v interface{}) {
// ```Go // ```Go
type measurement struct { type measurement struct {
Queries []*measurex.DNSLookupEvent `json:"queries"` Queries []*measurex.ArchivalDNSLookupEvent `json:"queries"`
TCPConnect []*measurex.NetworkEvent `json:"tcp_connect"` TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"`
TLSHandshakes []*measurex.TLSHandshakeEvent `json:"tls_handshakes"` TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.HTTPRoundTripEvent `json:"requests"` Requests []*measurex.ArchivalHTTPRoundTripEvent `json:"requests"`
} }
// ``` // ```
@ -97,7 +97,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
// //
// ```Go // ```Go
dns := mx.LookupHostSystem(ctx, parsedURL.Hostname()) dns := mx.LookupHostSystem(ctx, parsedURL.Hostname())
m.Queries = append(m.Queries, dns.LookupHost...) m.Queries = append(
m.Queries, measurex.NewArchivalDNSLookupEventList(dns.LookupHost)...)
// ``` // ```
// //
@ -129,7 +130,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
switch parsedURL.Scheme { switch parsedURL.Scheme {
case "http": case "http":
tcp := mx.TCPConnect(ctx, epnt.Address) tcp := mx.TCPConnect(ctx, epnt.Address)
m.TCPConnect = append(m.TCPConnect, tcp.Connect...) m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...)
case "https": case "https":
config := &tls.Config{ config := &tls.Config{
ServerName: parsedURL.Hostname(), ServerName: parsedURL.Hostname(),
@ -137,8 +139,10 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
RootCAs: netxlite.NewDefaultCertPool(), RootCAs: netxlite.NewDefaultCertPool(),
} }
tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config) tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config)
m.TCPConnect = append(m.TCPConnect, tls.Connect...) m.TCPConnect = append(
m.TLSHandshakes = append(m.TLSHandshakes, tls.TLSHandshake...) m.TCPConnect, measurex.NewArchivalTCPConnectList(tls.Connect)...)
m.TLSHandshakes = append(m.TLSHandshakes,
measurex.NewArchivalQUICTLSHandshakeEventList(tls.TLSHandshake)...)
} }
} }
@ -223,7 +227,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
// //
// ```Go // ```Go
m.Requests = append(m.Requests, db.AsMeasurement().HTTPRoundTrip...) m.Requests = append(m.Requests, measurex.NewArchivalHTTPRoundTripEventList(
db.AsMeasurement().HTTPRoundTrip)...)
return m, nil return m, nil
} }