diff --git a/internal/engine/experiment/webstepsx/measurer.go b/internal/engine/experiment/webstepsx/measurer.go index 491f2d7..3238394 100644 --- a/internal/engine/experiment/webstepsx/measurer.go +++ b/internal/engine/experiment/webstepsx/measurer.go @@ -29,7 +29,7 @@ type Config struct{} // TestKeys contains the experiment's test keys. type TestKeys struct { - *measurex.URLMeasurement + *measurex.ArchivalURLMeasurement } // Measurer performs the measurement. @@ -142,7 +142,9 @@ func (mx *Measurer) runAsync(ctx context.Context, sess model.ExperimentSession, }, Input: model.MeasurementTarget(m.URL), MeasurementRuntime: m.TotalRuntime.Seconds(), - TestKeys: &TestKeys{URLMeasurement: m}, + TestKeys: &TestKeys{ + ArchivalURLMeasurement: measurex.NewArchivalURLMeasurement(m), + }, } } } @@ -163,7 +165,7 @@ type measurerMeasureURLHelper struct { func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints( ctx context.Context, URL *url.URL, headers http.Header, curEndpoints ...*measurex.HTTPEndpoint) ( - []*measurex.HTTPEndpoint, interface{}, error) { + []*measurex.HTTPEndpoint, *measurex.THMeasurement, error) { cc := &THClientCall{ Endpoints: measurex.HTTPEndpointsToEndpoints(curEndpoints), HTTPClient: mth.Clnt, diff --git a/internal/engine/experiment/webstepsx/th.go b/internal/engine/experiment/webstepsx/th.go index 20978b4..ee1fc3f 100644 --- a/internal/engine/experiment/webstepsx/th.go +++ b/internal/engine/experiment/webstepsx/th.go @@ -41,14 +41,7 @@ type THClientRequest struct { } // THServerResponse is the response from the test helper. -type THServerResponse struct { - // DNS contains all the DNS related measurements. - DNS []*measurex.DNSMeasurement `json:"dns"` - - // Endpoints contains a measurement for each endpoint - // that was discovered by the probe or the TH. - Endpoints []*measurex.HTTPEndpointMeasurement `json:"endpoints"` -} +type THServerResponse = measurex.THMeasurement // thMaxAcceptableBodySize is the maximum acceptable body size by TH code. const thMaxAcceptableBodySize = 1 << 20 @@ -294,9 +287,9 @@ func (h *THHandler) simplifyMeasurement(in *measurex.Measurement) (out *measurex } func (h *THHandler) simplifyHandshake( - in []*measurex.TLSHandshakeEvent) (out []*measurex.TLSHandshakeEvent) { + in []*measurex.QUICTLSHandshakeEvent) (out []*measurex.QUICTLSHandshakeEvent) { for _, ev := range in { - out = append(out, &measurex.TLSHandshakeEvent{ + out = append(out, &measurex.QUICTLSHandshakeEvent{ CipherSuite: ev.CipherSuite, Failure: ev.Failure, NegotiatedProto: ev.NegotiatedProto, @@ -319,32 +312,24 @@ func (h *THHandler) simplifyHTTPRoundTrip( in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) { for _, ev := range in { out = append(out, &measurex.HTTPRoundTripEvent{ - Failure: ev.Failure, - Request: ev.Request, - Response: h.simplifyHTTPResponse(ev.Response), - Finished: 0, - Started: 0, - Oddity: ev.Oddity, + Failure: ev.Failure, + Method: ev.Method, + URL: ev.URL, + RequestHeaders: ev.RequestHeaders, + StatusCode: ev.StatusCode, + ResponseHeaders: ev.ResponseHeaders, + ResponseBody: nil, // we don't transfer the body + ResponseBodyLength: ev.ResponseBodyLength, + ResponseBodyIsTruncated: ev.ResponseBodyIsTruncated, + ResponseBodyIsUTF8: ev.ResponseBodyIsUTF8, + Finished: ev.Finished, + Started: ev.Started, + Oddity: ev.Oddity, }) } return } -func (h *THHandler) simplifyHTTPResponse( - in *measurex.HTTPResponse) (out *measurex.HTTPResponse) { - if in != nil { - out = &measurex.HTTPResponse{ - Code: in.Code, - Headers: in.Headers, - Body: nil, - BodyIsTruncated: in.BodyIsTruncated, - BodyLength: in.BodyLength, - BodyIsUTF8: in.BodyIsUTF8, - } - } - return -} - type thMeasureURLHelper struct { epnts []*measurex.Endpoint } @@ -352,7 +337,7 @@ type thMeasureURLHelper struct { func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints( ctx context.Context, URL *url.URL, headers http.Header, serverEpnts ...*measurex.HTTPEndpoint) ( - epnts []*measurex.HTTPEndpoint, thMeaurement interface{}, err error) { + epnts []*measurex.HTTPEndpoint, thMeaurement *measurex.THMeasurement, err error) { for _, epnt := range thh.epnts { epnts = append(epnts, &measurex.HTTPEndpoint{ Domain: URL.Hostname(), diff --git a/internal/measurex/archival.go b/internal/measurex/archival.go index 1fe8136..2feb65f 100644 --- a/internal/measurex/archival.go +++ b/internal/measurex/archival.go @@ -1,8 +1,11 @@ package measurex import ( + "net" "net/http" + "strconv" "strings" + "time" ) // @@ -11,6 +14,10 @@ import ( // This file defines helpers to serialize to the OONI data format. // +// +// BinaryData +// + // ArchivalBinaryData is the archival format for binary data. type ArchivalBinaryData struct { Data []byte `json:"data"` @@ -29,6 +36,130 @@ func NewArchivalBinaryData(data []byte) (out *ArchivalBinaryData) { return } +// +// NetworkEvent +// + +// ArchivalNetworkEvent is the OONI data format representation +// of a network event according to df-008-netevents. +type ArchivalNetworkEvent struct { + // JSON names compatible with df-008-netevents + RemoteAddr string `json:"address"` + Failure *string `json:"failure"` + Count int `json:"num_bytes,omitempty"` + Operation string `json:"operation"` + Network string `json:"proto"` + Finished float64 `json:"t"` + Started float64 `json:"started"` + + // Names that are not part of the spec. + Oddity Oddity `json:"oddity"` +} + +// NewArchivalNetworkEvent converts a network event to its archival format. +func NewArchivalNetworkEvent(in *NetworkEvent) *ArchivalNetworkEvent { + return &ArchivalNetworkEvent{ + RemoteAddr: in.RemoteAddr, + Failure: in.Failure, + Count: in.Count, + Operation: in.Operation, + Network: in.Network, + Finished: in.Finished, + Started: in.Started, + Oddity: in.Oddity, + } +} + +// NewArchivalNetworkEventList converts a list of NetworkEvent +// to a list of ArchivalNetworkEvent. +func NewArchivalNetworkEventList(in []*NetworkEvent) (out []*ArchivalNetworkEvent) { + for _, ev := range in { + out = append(out, NewArchivalNetworkEvent(ev)) + } + return +} + +// +// DNSRoundTripEvent +// + +// ArchivalDNSRoundTripEvent is the OONI data format representation +// of a DNS round trip, which is currently not specified. +// +// We are trying to use names compatible with the names currently +// used by other specifications we currently use. +type ArchivalDNSRoundTripEvent struct { + Network string `json:"engine"` + Address string `json:"resolver_address"` + Query *ArchivalBinaryData `json:"raw_query"` + Started float64 `json:"started"` + Finished float64 `json:"t"` + Failure *string `json:"failure"` + Reply *ArchivalBinaryData `json:"raw_reply"` +} + +// NewArchivalDNSRoundTripEvent converts a DNSRoundTripEvent into is archival format. +func NewArchivalDNSRoundTripEvent(in *DNSRoundTripEvent) *ArchivalDNSRoundTripEvent { + return &ArchivalDNSRoundTripEvent{ + Network: in.Network, + Address: in.Address, + Query: NewArchivalBinaryData(in.Query), + Started: in.Started, + Finished: in.Finished, + Failure: in.Failure, + Reply: NewArchivalBinaryData(in.Reply), + } +} + +// NewArchivalDNSRoundTripEventList converts a DNSRoundTripEvent +// list to the corresponding archival format. +func NewArchivalDNSRoundTripEventList(in []*DNSRoundTripEvent) (out []*ArchivalDNSRoundTripEvent) { + for _, ev := range in { + out = append(out, NewArchivalDNSRoundTripEvent(ev)) + } + return +} + +// +// HTTPRoundTrip +// + +// ArchivalHTTPRequest is the archival format of an HTTP +// request according to df-001-http.md. +type ArchivalHTTPRequest struct { + Method string `json:"method"` + URL string `json:"url"` + Headers ArchivalHeaders `json:"headers"` +} + +// ArchivalHTTPResponse is the archival format of an HTTP +// response according to df-001-http.md. +type ArchivalHTTPResponse struct { + // Names consistent with df-001-http.md + Code int64 `json:"code"` + Headers ArchivalHeaders `json:"headers"` + Body *ArchivalBinaryData `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + + // Fields not part of the spec + BodyLength int64 `json:"x_body_length"` + BodyIsUTF8 bool `json:"x_body_is_utf8"` +} + +// ArchivalHTTPRoundTripEvent is the archival format of an +// HTTP response according to df-001-http.md. +type ArchivalHTTPRoundTripEvent struct { + // JSON names following the df-001-httpt data format. + Failure *string `json:"failure"` + Request *HTTPRequest `json:"request"` + Response *HTTPResponse `json:"response"` + Finished float64 `json:"t"` + Started float64 `json:"started"` + + // Names not in the specification + Oddity Oddity `json:"oddity"` +} + // ArchivalHeaders is a list of HTTP headers. type ArchivalHeaders map[string]string @@ -54,6 +185,64 @@ func NewArchivalHeaders(in http.Header) (out ArchivalHeaders) { return } +// NewArchivalHTTPRoundTripEvent converts an HTTPRoundTrip to its archival format. +func NewArchivalHTTPRoundTripEvent(in *HTTPRoundTripEvent) *ArchivalHTTPRoundTripEvent { + return &ArchivalHTTPRoundTripEvent{ + Failure: in.Failure, + Request: &HTTPRequest{ + Method: in.Method, + URL: in.URL, + Headers: NewArchivalHeaders(in.RequestHeaders), + }, + Response: &HTTPResponse{ + Code: in.StatusCode, + Headers: NewArchivalHeaders(in.ResponseHeaders), + Body: NewArchivalBinaryData(in.ResponseBody), + BodyLength: in.ResponseBodyLength, + BodyIsTruncated: in.ResponseBodyIsTruncated, + BodyIsUTF8: in.ResponseBodyIsUTF8, + }, + Finished: in.Finished, + Started: in.Started, + Oddity: in.Oddity, + } +} + +// NewArchivalHTTPRoundTripEventList converts a list of +// HTTPRoundTripEvent to a list of ArchivalRoundTripEvent. +func NewArchivalHTTPRoundTripEventList(in []*HTTPRoundTripEvent) (out []*ArchivalHTTPRoundTripEvent) { + for _, ev := range in { + out = append(out, NewArchivalHTTPRoundTripEvent(ev)) + } + return +} + +// +// QUICTLSHandshakeEvent +// + +// ArchivalQUICTLSHandshakeEvent is the archival data format for a +// QUIC or TLS handshake event according to df-006-tlshandshake. +type ArchivalQUICTLSHandshakeEvent struct { + // JSON names compatible with df-006-tlshandshake + CipherSuite string `json:"cipher_suite"` + Failure *string `json:"failure"` + NegotiatedProto string `json:"negotiated_proto"` + TLSVersion string `json:"tls_version"` + PeerCerts []*ArchivalBinaryData `json:"peer_certificates"` + Finished float64 `json:"t"` + + // JSON names that are consistent with the + // spirit of the spec but are not in it + RemoteAddr string `json:"address"` + SNI string `json:"server_name"` // used in prod + ALPN []string `json:"alpn"` + SkipVerify bool `json:"no_tls_verify"` // used in prod + Oddity Oddity `json:"oddity"` + Network string `json:"proto"` + Started float64 `json:"started"` +} + // NewArchivalTLSCertList builds a new []ArchivalBinaryData // from a list of raw x509 certificates data. func NewArchivalTLSCerts(in [][]byte) (out []*ArchivalBinaryData) { @@ -66,13 +255,341 @@ func NewArchivalTLSCerts(in [][]byte) (out []*ArchivalBinaryData) { return } -// NewArchivalFailure creates an archival failure from an error. We -// cannot round trip an error using JSON, so we serialize to this -// intermediate format that is a sort of Optional. -func NewArchivalFailure(err error) *string { - if err == nil { - return nil +// NewArchivalQUICTLSHandshakeEvent converts a QUICTLSHandshakeEvent +// to its archival data format. +func NewArchivalQUICTLSHandshakeEvent(in *QUICTLSHandshakeEvent) *ArchivalQUICTLSHandshakeEvent { + return &ArchivalQUICTLSHandshakeEvent{ + CipherSuite: in.CipherSuite, + Failure: in.Failure, + NegotiatedProto: in.NegotiatedProto, + TLSVersion: in.TLSVersion, + PeerCerts: NewArchivalTLSCerts(in.PeerCerts), + Finished: in.Finished, + RemoteAddr: in.RemoteAddr, + SNI: in.SNI, + ALPN: in.ALPN, + SkipVerify: in.SkipVerify, + Oddity: in.Oddity, + Network: in.Network, + Started: in.Started, } - s := err.Error() - return &s +} + +// NewArchivalQUICTLSHandshakeEventList converts a list of +// QUICTLSHandshakeEvent to a list of ArchivalQUICTLSHandshakeEvent. +func NewArchivalQUICTLSHandshakeEventList(in []*QUICTLSHandshakeEvent) (out []*ArchivalQUICTLSHandshakeEvent) { + for _, ev := range in { + out = append(out, NewArchivalQUICTLSHandshakeEvent(ev)) + } + return +} + +// +// DNSLookup +// + +// ArchivalDNSLookupAnswer is the archival format of a +// DNS lookup answer according to df-002-dnst. +type ArchivalDNSLookupAnswer struct { + // JSON names compatible with df-002-dnst's spec + Type string `json:"answer_type"` + IPv4 string `json:"ipv4,omitempty"` + IPv6 string `json:"ivp6,omitempty"` + + // Names not part of the spec. + ALPN string `json:"alpn,omitempty"` +} + +// ArchivalDNSLookupEvent is the archival data format +// of a DNS lookup according to df-002-dnst. +type ArchivalDNSLookupEvent struct { + // fields inside df-002-dnst + Answers []ArchivalDNSLookupAnswer `json:"answers"` + Network string `json:"engine"` + Failure *string `json:"failure"` + Domain string `json:"hostname"` + QueryType string `json:"query_type"` + Address string `json:"resolver_address"` + Finished float64 `json:"t"` + + // Names not part of the spec. + Started float64 `json:"started"` + Oddity Oddity `json:"oddity"` +} + +// NewArchivalDNSLookupAnswers creates a list of ArchivalDNSLookupAnswer. +func NewArchivalDNSLookupAnswers(in *DNSLookupEvent) (out []ArchivalDNSLookupAnswer) { + for _, ip := range in.A { + out = append(out, ArchivalDNSLookupAnswer{ + Type: "A", + IPv4: ip, + }) + } + for _, ip := range in.AAAA { + out = append(out, ArchivalDNSLookupAnswer{ + Type: "AAAA", + IPv6: ip, + }) + } + for _, alpn := range in.ALPN { + out = append(out, ArchivalDNSLookupAnswer{ + Type: "ALPN", + ALPN: alpn, + }) + } + return +} + +// NewArchivalDNSLookupEvent converts a DNSLookupEvent +// to its archival representation. +func NewArchivalDNSLookupEvent(in *DNSLookupEvent) *ArchivalDNSLookupEvent { + return &ArchivalDNSLookupEvent{ + Answers: NewArchivalDNSLookupAnswers(in), + Network: in.Network, + Failure: in.Failure, + Domain: in.Domain, + QueryType: in.QueryType, + Address: in.Address, + Finished: in.Finished, + Started: in.Started, + Oddity: in.Oddity, + } +} + +// NewArchivalDNSLookupEventList converts a list of DNSLookupEvent +// to a list of ArchivalDNSLookupEvent. +func NewArchivalDNSLookupEventList(in []*DNSLookupEvent) (out []*ArchivalDNSLookupEvent) { + for _, ev := range in { + out = append(out, NewArchivalDNSLookupEvent(ev)) + } + return +} + +// +// TCPConnect +// + +// ArchivalTCPConnect is the archival form of TCP connect +// events in compliance with df-005-tcpconnect. +type ArchivalTCPConnect struct { + // Names part of the spec. + IP string `json:"ip"` + Port int64 `json:"port"` + Finished float64 `json:"t"` + Status *ArchivalTCPConnectStatus `json:"status"` + + // Names not part of the spec. + Started float64 `json:"started"` + Oddity Oddity `json:"oddity"` +} + +// ArchivalTCPConnectStatus contains the status of a TCP connect. +type ArchivalTCPConnectStatus struct { + Blocked bool `json:"blocked"` + Failure *string `json:"failure"` + Success bool `json:"success"` +} + +// NewArchivalTCPConnect converts a NetworkEvent to an ArchivalTCPConnect. +func NewArchivalTCPConnect(in *NetworkEvent) *ArchivalTCPConnect { + // We ignore errors because values come from Go code that + // emits correct serialization of TCP/UDP addresses. + addr, port, _ := net.SplitHostPort(in.RemoteAddr) + portnum, _ := strconv.Atoi(port) + return &ArchivalTCPConnect{ + IP: addr, + Port: int64(portnum), + Finished: in.Finished, + Status: &ArchivalTCPConnectStatus{ + Blocked: in.Failure != nil, + Failure: in.Failure, + Success: in.Failure == nil, + }, + Started: in.Started, + Oddity: in.Oddity, + } +} + +// NewArchivalTCPConnectList converts a list of NetworkEvent +// to a list of ArchivalTCPConnect. In doing that, the code +// only considers "connect" events using the TCP protocol. +func NewArchivalTCPConnectList(in []*NetworkEvent) (out []*ArchivalTCPConnect) { + for _, ev := range in { + if ev.Operation != "connect" { + continue + } + switch ev.Network { + case "tcp", "tcp4", "tcp6": + out = append(out, NewArchivalTCPConnect(ev)) + default: + // nothing + } + } + return +} + +// +// URLMeasurement +// + +// ArchivalURLMeasurement is the archival representation of URLMeasurement +type ArchivalURLMeasurement struct { + URL string `json:"url"` + DNS []*ArchivalDNSMeasurement `json:"dns"` + Endpoints []*ArchivalHTTPEndpointMeasurement `json:"endpoints"` + TH *ArchivalTHMeasurement `json:"th"` + TotalRuntime time.Duration `json:"x_total_runtime"` + DNSRuntime time.Duration `json:"x_dns_runtime"` + THRuntime time.Duration `json:"x_th_runtime"` + EpntsRuntime time.Duration `json:"x_epnts_runtime"` +} + +// NewArchivalURLMeasurement creates the archival representation +// of an URLMeasurement data structure. +func NewArchivalURLMeasurement(in *URLMeasurement) *ArchivalURLMeasurement { + return &ArchivalURLMeasurement{ + URL: in.URL, + DNS: NewArchivalDNSMeasurementList(in.DNS), + Endpoints: NewArchivalHTTPEndpointMeasurementList(in.Endpoints), + TH: NewArchivalTHMeasurement(in.TH), + TotalRuntime: in.TotalRuntime, + DNSRuntime: in.DNSRuntime, + THRuntime: in.THRuntime, + EpntsRuntime: in.EpntsRuntime, + } +} + +// +// EndpointMeasurement +// + +// ArchivalEndpointMeasurement is the archival representation of EndpointMeasurement. +type ArchivalEndpointMeasurement struct { + // Network is the network of this endpoint. + Network EndpointNetwork `json:"network"` + + // Address is the address of this endpoint. + Address string `json:"address"` + + // An EndpointMeasurement is a Measurement. + *ArchivalMeasurement +} + +// NewArchivalEndpointMeasurement converts an EndpointMeasurement +// to the corresponding archival data format. +func NewArchivalEndpointMeasurement(in *EndpointMeasurement) *ArchivalEndpointMeasurement { + return &ArchivalEndpointMeasurement{ + Network: in.Network, + Address: in.Address, + ArchivalMeasurement: NewArchivalMeasurement(in.Measurement), + } +} + +// +// THMeasurement +// + +// ArchivalTHMeasurement is the archival representation of THMeasurement. +type ArchivalTHMeasurement struct { + DNS []*ArchivalDNSMeasurement `json:"dns"` + Endpoints []*ArchivalHTTPEndpointMeasurement `json:"endpoints"` +} + +// NewArchivalTHMeasurement creates the archival representation of THMeasurement. +func NewArchivalTHMeasurement(in *THMeasurement) *ArchivalTHMeasurement { + return &ArchivalTHMeasurement{ + DNS: NewArchivalDNSMeasurementList(in.DNS), + Endpoints: NewArchivalHTTPEndpointMeasurementList(in.Endpoints), + } +} + +// +// DNSMeasurement +// + +// ArchivalDNSMeasurement is the archival representation of DNSMeasurement. +type ArchivalDNSMeasurement struct { + Domain string `json:"domain"` + *ArchivalMeasurement +} + +// NewArchivalDNSMeasurement converts a DNSMeasurement to an ArchivalDNSMeasurement. +func NewArchivalDNSMeasurement(in *DNSMeasurement) *ArchivalDNSMeasurement { + return &ArchivalDNSMeasurement{ + Domain: in.Domain, + ArchivalMeasurement: NewArchivalMeasurement(in.Measurement), + } +} + +// NewArchivalDNSMeasurementList converts a list of DNSMeasurement +// to a list of ArchivalDNSMeasurement. +func NewArchivalDNSMeasurementList(in []*DNSMeasurement) (out []*ArchivalDNSMeasurement) { + for _, m := range in { + out = append(out, NewArchivalDNSMeasurement(m)) + } + return +} + +// +// HTTPEndpointMeasurement +// + +// ArchivalHTTPEndpointMeasurement is the archival representation +// of an HTTPEndpointMeasurement. +type ArchivalHTTPEndpointMeasurement struct { + URL string `json:"url"` + Network EndpointNetwork `json:"network"` + Address string `json:"address"` + *ArchivalMeasurement +} + +// NewArchivalHTTPEndpointMeasurement converts an HTTPEndpointMeasurement +// to an ArchivalHTTPEndpointMeasurement. +func NewArchivalHTTPEndpointMeasurement(in *HTTPEndpointMeasurement) *ArchivalHTTPEndpointMeasurement { + return &ArchivalHTTPEndpointMeasurement{ + URL: in.URL, + Network: in.Network, + Address: in.Address, + ArchivalMeasurement: NewArchivalMeasurement(in.Measurement), + } +} + +// NewArchivalHTTPEndpointMeasurementList converts a list of HTTPEndpointMeasurement +// to a list of ArchivalHTTPEndpointMeasurement. +func NewArchivalHTTPEndpointMeasurementList(in []*HTTPEndpointMeasurement) (out []*ArchivalHTTPEndpointMeasurement) { + for _, m := range in { + out = append(out, NewArchivalHTTPEndpointMeasurement(m)) + } + return +} + +// +// Measurement +// + +// ArchivalMeasurement is the archival representation of a Measurement. +type ArchivalMeasurement struct { + NetworkEvents []*ArchivalNetworkEvent `json:"network_events,omitempty"` + DNSEvents []*ArchivalDNSRoundTripEvent `json:"dns_events,omitempty"` + Queries []*ArchivalDNSLookupEvent `json:"queries,omitempty"` + TCPConnect []*ArchivalTCPConnect `json:"tcp_connect,omitempty"` + TLSHandshakes []*ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes,omitempty"` + QUICHandshakes []*ArchivalQUICTLSHandshakeEvent `json:"quic_handshakes,omitempty"` + Requests []*ArchivalHTTPRoundTripEvent `json:"requests,omitempty"` +} + +// NewArchivalMeasurement converts a Measurement to ArchivalMeasurement. +func NewArchivalMeasurement(in *Measurement) *ArchivalMeasurement { + out := &ArchivalMeasurement{ + NetworkEvents: NewArchivalNetworkEventList(in.ReadWrite), + DNSEvents: NewArchivalDNSRoundTripEventList(in.DNSRoundTrip), + Queries: nil, // done below + TCPConnect: NewArchivalTCPConnectList(in.Connect), + TLSHandshakes: NewArchivalQUICTLSHandshakeEventList(in.TLSHandshake), + QUICHandshakes: NewArchivalQUICTLSHandshakeEventList(in.QUICHandshake), + Requests: NewArchivalHTTPRoundTripEventList(in.HTTPRoundTrip), + } + out.Queries = append(out.Queries, NewArchivalDNSLookupEventList(in.LookupHost)...) + out.Queries = append(out.Queries, NewArchivalDNSLookupEventList(in.LookupHTTPSSvc)...) + return out } diff --git a/internal/measurex/db.go b/internal/measurex/db.go index 7260e07..2eca1c5 100644 --- a/internal/measurex/db.go +++ b/internal/measurex/db.go @@ -28,7 +28,7 @@ type WritableDB interface { InsertIntoClose(ev *NetworkEvent) // InsertIntoTLSHandshake saves a TLS handshake event. - InsertIntoTLSHandshake(ev *TLSHandshakeEvent) + InsertIntoTLSHandshake(ev *QUICTLSHandshakeEvent) // InsertIntoLookupHost saves a lookup host event. InsertIntoLookupHost(ev *DNSLookupEvent) @@ -46,7 +46,7 @@ type WritableDB interface { InsertIntoHTTPRedirect(ev *HTTPRedirectEvent) // InsertIntoQUICHandshake saves a QUIC handshake event. - InsertIntoQUICHandshake(ev *QUICHandshakeEvent) + InsertIntoQUICHandshake(ev *QUICTLSHandshakeEvent) } // MeasurementDB is a WritableDB that also allows high-level code @@ -56,13 +56,13 @@ type MeasurementDB struct { dialTable []*NetworkEvent readWriteTable []*NetworkEvent closeTable []*NetworkEvent - tlsHandshakeTable []*TLSHandshakeEvent + tlsHandshakeTable []*QUICTLSHandshakeEvent lookupHostTable []*DNSLookupEvent lookupHTTPSvcTable []*DNSLookupEvent dnsRoundTripTable []*DNSRoundTripEvent httpRoundTripTable []*HTTPRoundTripEvent httpRedirectTable []*HTTPRedirectEvent - quicHandshakeTable []*QUICHandshakeEvent + quicHandshakeTable []*QUICTLSHandshakeEvent // mu protects all the fields mu sync.Mutex @@ -126,14 +126,14 @@ func (db *MeasurementDB) selectAllFromCloseUnlocked() (out []*NetworkEvent) { } // InsertIntoTLSHandshake implements EventDB.InsertIntoTLSHandshake. -func (db *MeasurementDB) InsertIntoTLSHandshake(ev *TLSHandshakeEvent) { +func (db *MeasurementDB) InsertIntoTLSHandshake(ev *QUICTLSHandshakeEvent) { db.mu.Lock() db.tlsHandshakeTable = append(db.tlsHandshakeTable, ev) db.mu.Unlock() } // selectAllFromTLSHandshakeUnlocked returns all TLS handshake events. -func (db *MeasurementDB) selectAllFromTLSHandshakeUnlocked() (out []*TLSHandshakeEvent) { +func (db *MeasurementDB) selectAllFromTLSHandshakeUnlocked() (out []*QUICTLSHandshakeEvent) { out = append(out, db.tlsHandshakeTable...) return } @@ -204,14 +204,14 @@ func (db *MeasurementDB) selectAllFromHTTPRedirectUnlocked() (out []*HTTPRedirec } // InsertIntoQUICHandshake implements EventDB.InsertIntoQUICHandshake. -func (db *MeasurementDB) InsertIntoQUICHandshake(ev *QUICHandshakeEvent) { +func (db *MeasurementDB) InsertIntoQUICHandshake(ev *QUICTLSHandshakeEvent) { db.mu.Lock() db.quicHandshakeTable = append(db.quicHandshakeTable, ev) db.mu.Unlock() } // selectAllFromQUICHandshakeUnlocked returns all QUIC handshake events. -func (db *MeasurementDB) selectAllFromQUICHandshakeUnlocked() (out []*QUICHandshakeEvent) { +func (db *MeasurementDB) selectAllFromQUICHandshakeUnlocked() (out []*QUICTLSHandshakeEvent) { out = append(out, db.quicHandshakeTable...) return } diff --git a/internal/measurex/dialer.go b/internal/measurex/dialer.go index 6484857..2920c10 100644 --- a/internal/measurex/dialer.go +++ b/internal/measurex/dialer.go @@ -54,17 +54,14 @@ type dialerDB struct { // NetworkEvent contains a network event. This kind of events // are generated by Dialer, QUICDialer, Conn, QUICConn. type NetworkEvent struct { - // JSON names compatible with df-008-netevents - RemoteAddr string `json:"address"` - Failure *string `json:"failure"` - Count int `json:"num_bytes,omitempty"` - Operation string `json:"operation"` - Network string `json:"proto"` - Finished float64 `json:"t"` - Started float64 `json:"started"` - - // Names that are not part of the spec. - Oddity Oddity `json:"oddity"` + RemoteAddr string + Failure *string + Count int + Operation string + Network string + Oddity Oddity + Finished float64 + Started float64 } func (d *dialerDB) DialContext( @@ -78,7 +75,7 @@ func (d *dialerDB) DialContext( RemoteAddr: address, Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Oddity: d.computeOddity(err), Count: 0, }) @@ -128,7 +125,7 @@ func (c *connDB) Read(b []byte) (int, error) { RemoteAddr: c.remoteAddr, Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Count: count, }) return count, err @@ -144,7 +141,7 @@ func (c *connDB) Write(b []byte) (int, error) { RemoteAddr: c.remoteAddr, Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Count: count, }) return count, err @@ -160,7 +157,7 @@ func (c *connDB) Close() error { RemoteAddr: c.remoteAddr, Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Count: 0, }) return err diff --git a/internal/measurex/dnsx.go b/internal/measurex/dnsx.go index 8eae174..47db106 100644 --- a/internal/measurex/dnsx.go +++ b/internal/measurex/dnsx.go @@ -32,15 +32,13 @@ type dnsxRoundTripperDB struct { // DNSRoundTripEvent contains the result of a DNS round trip. type DNSRoundTripEvent struct { - // This data structure is not in df-002-dns but the names and - // semantics try to be consistent with such a spec. - Network string `json:"engine"` - Address string `json:"resolver_address"` - Query *ArchivalBinaryData `json:"raw_query"` - Started float64 `json:"started"` - Finished float64 `json:"t"` - Failure *string `json:"failure"` - Reply *ArchivalBinaryData `json:"raw_reply"` + Network string + Address string + Query []byte + Started float64 + Finished float64 + Failure *string + Reply []byte } func (txp *dnsxRoundTripperDB) RoundTrip(ctx context.Context, query []byte) ([]byte, error) { @@ -50,11 +48,11 @@ func (txp *dnsxRoundTripperDB) RoundTrip(ctx context.Context, query []byte) ([]b txp.db.InsertIntoDNSRoundTrip(&DNSRoundTripEvent{ Network: txp.DNSTransport.Network(), Address: txp.DNSTransport.Address(), - Query: NewArchivalBinaryData(query), + Query: query, Started: started, Finished: finished, - Failure: NewArchivalFailure(err), - Reply: NewArchivalBinaryData(reply), + Failure: NewFailure(err), + Reply: reply, }) return reply, err } diff --git a/internal/measurex/failure.go b/internal/measurex/failure.go new file mode 100644 index 0000000..b7ddf84 --- /dev/null +++ b/internal/measurex/failure.go @@ -0,0 +1,12 @@ +package measurex + +// NewFailure creates a serializable failure from an error. We +// cannot round trip an error using JSON, so we serialize to this +// intermediate format that is a sort of Optional. +func NewFailure(err error) *string { + if err == nil { + return nil + } + s := err.Error() + return &s +} diff --git a/internal/measurex/http.go b/internal/measurex/http.go index 2003e20..872449e 100644 --- a/internal/measurex/http.go +++ b/internal/measurex/http.go @@ -124,31 +124,33 @@ type HTTPResponse struct { // HTTPRoundTripEvent contains information about an HTTP round trip. type HTTPRoundTripEvent struct { - // JSON names following the df-001-httpt data format. - Failure *string `json:"failure"` - Request *HTTPRequest `json:"request"` - Response *HTTPResponse `json:"response"` - Finished float64 `json:"t"` - Started float64 `json:"started"` - - // Names not in the specification - Oddity Oddity `json:"oddity"` + Failure *string + Method string + URL string + RequestHeaders http.Header + StatusCode int64 + ResponseHeaders http.Header + ResponseBody []byte + ResponseBodyLength int64 + ResponseBodyIsTruncated bool + ResponseBodyIsUTF8 bool + Finished float64 + Started float64 + Oddity Oddity } func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error) { started := time.Since(txp.Begin).Seconds() resp, err := txp.HTTPTransport.RoundTrip(req) rt := &HTTPRoundTripEvent{ - Request: &HTTPRequest{ - Method: req.Method, - URL: req.URL.String(), - Headers: NewArchivalHeaders(req.Header), - }, - Started: started, + Method: req.Method, + URL: req.URL.String(), + RequestHeaders: req.Header, + Started: started, } if err != nil { rt.Finished = time.Since(txp.Begin).Seconds() - rt.Failure = NewArchivalFailure(err) + rt.Failure = NewFailure(err) txp.DB.InsertIntoHTTPRoundTrip(rt) return nil, err } @@ -162,10 +164,8 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error) case resp.StatusCode >= 400: rt.Oddity = OddityStatusOther } - rt.Response = &HTTPResponse{ - Code: int64(resp.StatusCode), - Headers: NewArchivalHeaders(resp.Header), - } + rt.StatusCode = int64(resp.StatusCode) + rt.ResponseHeaders = resp.Header r := io.LimitReader(resp.Body, txp.MaxBodySnapshotSize) body, err := netxlite.ReadAllContext(req.Context(), r) if errors.Is(err, io.EOF) && resp.Close { @@ -173,7 +173,7 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error) } if err != nil { rt.Finished = time.Since(txp.Begin).Seconds() - rt.Failure = NewArchivalFailure(err) + rt.Failure = NewFailure(err) txp.DB.InsertIntoHTTPRoundTrip(rt) return nil, err } @@ -181,10 +181,10 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error) Reader: io.MultiReader(bytes.NewReader(body), resp.Body), Closer: resp.Body, } - rt.Response.Body = NewArchivalBinaryData(body) - rt.Response.BodyLength = int64(len(body)) - rt.Response.BodyIsTruncated = int64(len(body)) >= txp.MaxBodySnapshotSize - rt.Response.BodyIsUTF8 = utf8.Valid(body) + rt.ResponseBody = body + rt.ResponseBodyLength = int64(len(body)) + rt.ResponseBodyIsTruncated = int64(len(body)) >= txp.MaxBodySnapshotSize + rt.ResponseBodyIsUTF8 = utf8.Valid(body) rt.Finished = time.Since(txp.Begin).Seconds() txp.DB.InsertIntoHTTPRoundTrip(rt) return resp, nil diff --git a/internal/measurex/measurement.go b/internal/measurex/measurement.go index 0141a22..1d04652 100644 --- a/internal/measurex/measurement.go +++ b/internal/measurex/measurement.go @@ -18,33 +18,33 @@ import ( // a bunch of measurements detailing each measurement step. type URLMeasurement struct { // URL is the URL we're measuring. - URL string `json:"url"` + URL string // DNS contains all the DNS related measurements. - DNS []*DNSMeasurement `json:"dns"` + DNS []*DNSMeasurement // Endpoints contains a measurement for each endpoint // that we discovered via DNS or TH. - Endpoints []*HTTPEndpointMeasurement `json:"endpoints"` + Endpoints []*HTTPEndpointMeasurement // RedirectURLs contain the URLs to which we should fetch // if we choose to follow redirections. - RedirectURLs []string `json:"-"` + RedirectURLs []string - // THMeasurement is the measurement collected by the TH. - TH interface{} `json:"th,omitempty"` + // TH is the measurement collected by the TH. + TH *THMeasurement // TotalRuntime is the total time to measure this URL. - TotalRuntime time.Duration `json:"-"` + TotalRuntime time.Duration // DNSRuntime is the time to run all DNS checks. - DNSRuntime time.Duration `json:"x_dns_runtime"` + DNSRuntime time.Duration // THRuntime is the total time to invoke all test helpers. - THRuntime time.Duration `json:"x_th_runtime"` + THRuntime time.Duration // EpntsRuntime is the total time to check all the endpoints. - EpntsRuntime time.Duration `json:"x_epnts_runtime"` + EpntsRuntime time.Duration } // fillRedirects takes in input a complete URLMeasurement and fills @@ -67,40 +67,40 @@ func (m *URLMeasurement) fillRedirects() { // data format is not compatible with the OONI data format. type Measurement struct { // Connect contains all the connect operations. - Connect []*NetworkEvent `json:"connect,omitempty"` + Connect []*NetworkEvent // ReadWrite contains all the read and write operations. - ReadWrite []*NetworkEvent `json:"read_write,omitempty"` + ReadWrite []*NetworkEvent // Close contains all the close operations. - Close []*NetworkEvent `json:"-"` + Close []*NetworkEvent // TLSHandshake contains all the TLS handshakes. - TLSHandshake []*TLSHandshakeEvent `json:"tls_handshake,omitempty"` + TLSHandshake []*QUICTLSHandshakeEvent // QUICHandshake contains all the QUIC handshakes. - QUICHandshake []*QUICHandshakeEvent `json:"quic_handshake,omitempty"` + QUICHandshake []*QUICTLSHandshakeEvent // LookupHost contains all the host lookups. - LookupHost []*DNSLookupEvent `json:"lookup_host,omitempty"` + LookupHost []*DNSLookupEvent // LookupHTTPSSvc contains all the HTTPSSvc lookups. - LookupHTTPSSvc []*DNSLookupEvent `json:"lookup_httpssvc,omitempty"` + LookupHTTPSSvc []*DNSLookupEvent // DNSRoundTrip contains all the DNS round trips. - DNSRoundTrip []*DNSRoundTripEvent `json:"dns_round_trip,omitempty"` + DNSRoundTrip []*DNSRoundTripEvent // HTTPRoundTrip contains all the HTTP round trips. - HTTPRoundTrip []*HTTPRoundTripEvent `json:"http_round_trip,omitempty"` + HTTPRoundTrip []*HTTPRoundTripEvent // HTTPRedirect contains all the redirections. - HTTPRedirect []*HTTPRedirectEvent `json:"-"` + HTTPRedirect []*HTTPRedirectEvent } // DNSMeasurement is a DNS measurement. type DNSMeasurement struct { // Domain is the domain this measurement refers to. - Domain string `json:"domain"` + Domain string // A DNSMeasurement is a Measurement. *Measurement @@ -239,10 +239,10 @@ func AllHTTPEndpointsForURL(URL *url.URL, // EndpointMeasurement is an endpoint measurement. type EndpointMeasurement struct { // Network is the network of this endpoint. - Network EndpointNetwork `json:"network"` + Network EndpointNetwork // Address is the address of this endpoint. - Address string `json:"address"` + Address string // An EndpointMeasurement is a Measurement. *Measurement @@ -251,14 +251,24 @@ type EndpointMeasurement struct { // HTTPEndpointMeasurement is an HTTP endpoint measurement. type HTTPEndpointMeasurement struct { // URL is the URL this measurement refers to. - URL string `json:"url"` + URL string // Network is the network of this endpoint. - Network EndpointNetwork `json:"network"` + Network EndpointNetwork // Address is the address of this endpoint. - Address string `json:"address"` + Address string // An HTTPEndpointMeasurement is a Measurement. *Measurement } + +// THMeasurement is the measurement performed by the TH. +type THMeasurement struct { + // DNS contains all the DNS related measurements. + DNS []*DNSMeasurement + + // Endpoints contains a measurement for each endpoint + // that was discovered by the probe or the TH. + Endpoints []*HTTPEndpointMeasurement +} diff --git a/internal/measurex/measurer.go b/internal/measurex/measurer.go index 66e7f34..285b852 100644 --- a/internal/measurex/measurer.go +++ b/internal/measurex/measurer.go @@ -13,7 +13,6 @@ import ( "context" "crypto/tls" "errors" - stdlog "log" "net" "net/http" "net/url" @@ -712,7 +711,7 @@ type MeasureURLHelper interface { // test helper protocol allows one to set. LookupExtraHTTPEndpoints(ctx context.Context, URL *url.URL, headers http.Header, epnts ...*HTTPEndpoint) ( - newEpnts []*HTTPEndpoint, thMeasurement interface{}, err error) + newEpnts []*HTTPEndpoint, thMeasurement *THMeasurement, err error) } // MeasureURL measures an HTTP or HTTPS URL. The DNS resolvers @@ -802,12 +801,8 @@ func (mx *Measurer) maybeQUICFollowUp(ctx context.Context, if epnt.QUICHandshake != nil { return } - for idx, rtrip := range epnt.HTTPRoundTrip { - if rtrip.Response == nil { - stdlog.Printf("malformed HTTPRoundTrip@%d: %+v", idx, rtrip) - continue - } - if v := rtrip.Response.Headers.Get("alt-svc"); v != "" { + for _, rtrip := range epnt.HTTPRoundTrip { + if v := rtrip.ResponseHeaders.Get("alt-svc"); v != "" { altsvc = append(altsvc, v) } } diff --git a/internal/measurex/quic.go b/internal/measurex/quic.go index eee7a7f..18bd833 100644 --- a/internal/measurex/quic.go +++ b/internal/measurex/quic.go @@ -60,7 +60,7 @@ func (c *udpLikeConnDB) WriteTo(p []byte, addr net.Addr) (int, error) { RemoteAddr: addr.String(), Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Count: count, }) return count, err @@ -76,7 +76,7 @@ func (c *udpLikeConnDB) ReadFrom(b []byte) (int, net.Addr, error) { RemoteAddr: addrStringIfNotNil(addr), Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Count: count, }) return count, addr, err @@ -92,15 +92,12 @@ func (c *udpLikeConnDB) Close() error { RemoteAddr: "", Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Count: 0, }) return err } -// QUICHandshakeEvent is the result of QUICHandshake. -type QUICHandshakeEvent = TLSHandshakeEvent - // NewQUICDialerWithoutResolver creates a new QUICDialer that is not // attached to any resolver. This means that every attempt to dial any // address containing a domain name will fail. This QUICDialer will @@ -138,7 +135,7 @@ func (qh *quicDialerDB) DialContext(ctx context.Context, network, address string } } finished := time.Since(qh.begin).Seconds() - qh.db.InsertIntoQUICHandshake(&QUICHandshakeEvent{ + qh.db.InsertIntoQUICHandshake(&QUICTLSHandshakeEvent{ Network: "quic", RemoteAddr: address, SNI: tlsConfig.ServerName, @@ -146,12 +143,12 @@ func (qh *quicDialerDB) DialContext(ctx context.Context, network, address string SkipVerify: tlsConfig.InsecureSkipVerify, Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Oddity: qh.computeOddity(err), TLSVersion: netxlite.TLSVersionString(state.Version), CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), NegotiatedProto: state.NegotiatedProtocol, - PeerCerts: NewArchivalTLSCerts(peerCerts(nil, &state)), + PeerCerts: peerCerts(nil, &state), }) return sess, err } diff --git a/internal/measurex/resolver.go b/internal/measurex/resolver.go index c079ec0..b519cb7 100644 --- a/internal/measurex/resolver.go +++ b/internal/measurex/resolver.go @@ -8,7 +8,6 @@ package measurex import ( "context" - "net" "strings" "time" @@ -65,31 +64,19 @@ type resolverDB struct { db WritableDB } -// DNSLookupAnswer is a DNS lookup answer. -type DNSLookupAnswer struct { - // JSON names compatible with df-002-dnst's spec - Type string `json:"answer_type"` - IPv4 string `json:"ipv4,omitempty"` - IPv6 string `json:"ivp6,omitempty"` - - // Names not part of the spec. - ALPN string `json:"alpn,omitempty"` -} - // DNSLookupEvent contains the results of a DNS lookup. type DNSLookupEvent struct { - // fields inside df-002-dnst - Answers []DNSLookupAnswer `json:"answers"` - Network string `json:"engine"` - Failure *string `json:"failure"` - Domain string `json:"hostname"` - QueryType string `json:"query_type"` - Address string `json:"resolver_address"` - Finished float64 `json:"t"` - - // Names not part of the spec. - Started float64 `json:"started"` - Oddity Oddity `json:"oddity"` + Network string + Failure *string + Domain string + QueryType string + Address string + Finished float64 + Started float64 + Oddity Oddity + A []string + AAAA []string + ALPN []string } // SupportsHTTP3 returns true if this query is for HTTPS and @@ -98,12 +85,9 @@ func (ev *DNSLookupEvent) SupportsHTTP3() bool { if ev.QueryType != "HTTPS" { return false } - for _, ans := range ev.Answers { - switch ans.Type { - case "ALPN": - if ans.ALPN == "h3" { - return true - } + for _, alpn := range ev.ALPN { + if alpn == "h3" { + return true } } return false @@ -111,18 +95,8 @@ func (ev *DNSLookupEvent) SupportsHTTP3() bool { // Addrs returns all the IPv4/IPv6 addresses func (ev *DNSLookupEvent) Addrs() (out []string) { - for _, ans := range ev.Answers { - switch ans.Type { - case "A": - if net.ParseIP(ans.IPv4) != nil { - out = append(out, ans.IPv4) - } - case "AAAA": - if net.ParseIP(ans.IPv6) != nil { - out = append(out, ans.IPv6) - } - } - } + out = append(out, ev.A...) + out = append(out, ev.AAAA...) return } @@ -130,35 +104,39 @@ func (r *resolverDB) LookupHost(ctx context.Context, domain string) ([]string, e started := time.Since(r.begin).Seconds() addrs, err := r.Resolver.LookupHost(ctx, domain) finished := time.Since(r.begin).Seconds() - for _, qtype := range []string{"A", "AAAA"} { - ev := &DNSLookupEvent{ - Answers: r.computeAnswers(addrs, qtype), - Network: r.Resolver.Network(), - Address: r.Resolver.Address(), - Failure: NewArchivalFailure(err), - Domain: domain, - QueryType: qtype, - Finished: finished, - Started: started, - Oddity: r.computeOddityLookupHost(addrs, err), - } - r.db.InsertIntoLookupHost(ev) - } + r.saveLookupResults(domain, started, finished, err, addrs, "A") + r.saveLookupResults(domain, started, finished, err, addrs, "AAAA") return addrs, err } -func (r *resolverDB) computeAnswers(addrs []string, qtype string) (out []DNSLookupAnswer) { +func (r *resolverDB) saveLookupResults(domain string, started, finished float64, + err error, addrs []string, qtype string) { + ev := &DNSLookupEvent{ + Network: r.Resolver.Network(), + Address: r.Resolver.Address(), + Failure: NewFailure(err), + Domain: domain, + QueryType: qtype, + Finished: finished, + Started: started, + } for _, addr := range addrs { if qtype == "A" && !strings.Contains(addr, ":") { - out = append(out, DNSLookupAnswer{Type: qtype, IPv4: addr}) + ev.A = append(ev.A, addr) continue } if qtype == "AAAA" && strings.Contains(addr, ":") { - out = append(out, DNSLookupAnswer{Type: qtype, IPv6: addr}) + ev.AAAA = append(ev.AAAA, addr) continue } } - return + switch qtype { + case "A": + ev.Oddity = r.computeOddityLookupHost(ev.A, err) + case "AAAA": + ev.Oddity = r.computeOddityLookupHost(ev.AAAA, err) + } + r.db.InsertIntoLookupHost(ev) } func (r *resolverDB) computeOddityLookupHost(addrs []string, err error) Oddity { @@ -193,28 +171,13 @@ func (r *resolverDB) LookupHTTPS(ctx context.Context, domain string) (*HTTPSSvc, QueryType: "HTTPS", Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Oddity: Oddity(r.computeOddityHTTPSSvc(https, err)), } if err == nil { - for _, addr := range https.IPv4 { - ev.Answers = append(ev.Answers, DNSLookupAnswer{ - Type: "A", - IPv4: addr, - }) - } - for _, addr := range https.IPv6 { - ev.Answers = append(ev.Answers, DNSLookupAnswer{ - Type: "AAAA", - IPv6: addr, - }) - } - for _, alpn := range https.ALPN { - ev.Answers = append(ev.Answers, DNSLookupAnswer{ - Type: "ALPN", - ALPN: alpn, - }) - } + ev.A = append(ev.A, https.IPv4...) + ev.AAAA = append(ev.AAAA, https.IPv6...) + ev.ALPN = append(ev.ALPN, https.ALPN...) } r.db.InsertIntoLookupHTTPSSvc(ev) return https, err diff --git a/internal/measurex/tls.go b/internal/measurex/tls.go index 8ef85fa..afeada6 100644 --- a/internal/measurex/tls.go +++ b/internal/measurex/tls.go @@ -38,25 +38,21 @@ type tlsHandshakerDB struct { db WritableDB } -// TLSHandshakeEvent contains a TLS handshake event. -type TLSHandshakeEvent struct { - // JSON names compatible with df-006-tlshandshake - CipherSuite string `json:"cipher_suite"` - Failure *string `json:"failure"` - NegotiatedProto string `json:"negotiated_proto"` - TLSVersion string `json:"tls_version"` - PeerCerts []*ArchivalBinaryData `json:"peer_certificates"` - Finished float64 `json:"t"` - - // JSON names that are consistent with the - // spirit of the spec but are not in it - RemoteAddr string `json:"address"` - SNI string `json:"server_name"` // used in prod - ALPN []string `json:"alpn"` - SkipVerify bool `json:"no_tls_verify"` // used in prod - Oddity Oddity `json:"oddity"` - Network string `json:"proto"` - Started float64 `json:"started"` +// QUICTLSHandshakeEvent contains a QUIC or TLS handshake event. +type QUICTLSHandshakeEvent struct { + CipherSuite string + Failure *string + NegotiatedProto string + TLSVersion string + PeerCerts [][]byte + Finished float64 + RemoteAddr string + SNI string + ALPN []string + SkipVerify bool + Oddity Oddity + Network string + Started float64 } func (thx *tlsHandshakerDB) Handshake(ctx context.Context, @@ -66,7 +62,7 @@ func (thx *tlsHandshakerDB) Handshake(ctx context.Context, started := time.Since(thx.begin).Seconds() tconn, state, err := thx.TLSHandshaker.Handshake(ctx, conn, config) finished := time.Since(thx.begin).Seconds() - thx.db.InsertIntoTLSHandshake(&TLSHandshakeEvent{ + thx.db.InsertIntoTLSHandshake(&QUICTLSHandshakeEvent{ Network: network, RemoteAddr: remoteAddr, SNI: config.ServerName, @@ -74,12 +70,12 @@ func (thx *tlsHandshakerDB) Handshake(ctx context.Context, SkipVerify: config.InsecureSkipVerify, Started: started, Finished: finished, - Failure: NewArchivalFailure(err), + Failure: NewFailure(err), Oddity: thx.computeOddity(err), TLSVersion: netxlite.TLSVersionString(state.Version), CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), NegotiatedProto: state.NegotiatedProtocol, - PeerCerts: NewArchivalTLSCerts(peerCerts(err, &state)), + PeerCerts: peerCerts(err, &state), }) return tconn, state, err } diff --git a/internal/tutorial/measurex/chapter01/README.md b/internal/tutorial/measurex/chapter01/README.md index 2efa2b9..4c04d4a 100644 --- a/internal/tutorial/measurex/chapter01/README.md +++ b/internal/tutorial/measurex/chapter01/README.md @@ -167,8 +167,12 @@ format, in the remainder of this program we're going to serialize the `Measurement` to JSON and print it to the standard output. +Rather than serializing the raw `Measurement` struct, +we first convert it to the "archival" format. This is the +data format specified at [ooni/spec](https://github.com/ooni/spec/tree/master/data-formats). + ```Go - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) ``` @@ -194,13 +198,14 @@ go run -race ./internal/tutorial/measurex/chapter01 | jq ``` Where `jq` is being used to make the output more presentable. + If you do that you obtain some logging messages, which are out of the scope of this tutorial, and the following JSON: ```JSON { "domain": "example.com", - "lookup_host": [ + "queries": [ { "answers": [ { @@ -237,6 +242,9 @@ the scope of this tutorial, and the following JSON: } ``` +This JSON [implements the df-002-dnst](https://github.com/ooni/spec/blob/master/data-formats/df-002-dnst.md) +OONI data format. + You see that we have two messages here. OONI splits a DNS resolution performed using the system resolver into two "fake" DNS resolutions for A and AAAA. (Under the hood, this is @@ -266,7 +274,7 @@ This is the output JSON: ```JSON { "domain": "antani.ooni.org", - "lookup_host": [ + "queries": [ { "answers": null, "engine": "system", @@ -327,7 +335,7 @@ To get this JSON: ```JSON { "domain": "example.com", - "lookup_host": [ + "queries": [ { "answers": null, "engine": "system", diff --git a/internal/tutorial/measurex/chapter01/main.go b/internal/tutorial/measurex/chapter01/main.go index e88c7a7..fc14d9b 100644 --- a/internal/tutorial/measurex/chapter01/main.go +++ b/internal/tutorial/measurex/chapter01/main.go @@ -168,8 +168,12 @@ func main() { // going to serialize the `Measurement` to JSON and // print it to the standard output. // + // Rather than serializing the raw `Measurement` struct, + // we first convert it to the "archival" format. This is the + // data format specified at [ooni/spec](https://github.com/ooni/spec/tree/master/data-formats). + // // ```Go - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) // ``` @@ -195,13 +199,14 @@ func main() { // ``` // // Where `jq` is being used to make the output more presentable. +// // If you do that you obtain some logging messages, which are out of // the scope of this tutorial, and the following JSON: // // ```JSON // { // "domain": "example.com", -// "lookup_host": [ +// "queries": [ // { // "answers": [ // { @@ -238,6 +243,9 @@ func main() { // } // ``` // +// This JSON [implements the df-002-dnst](https://github.com/ooni/spec/blob/master/data-formats/df-002-dnst.md) +// OONI data format. +// // You see that we have two messages here. OONI splits a DNS // resolution performed using the system resolver into two "fake" // DNS resolutions for A and AAAA. (Under the hood, this is @@ -267,7 +275,7 @@ func main() { // ```JSON // { // "domain": "antani.ooni.org", -// "lookup_host": [ +// "queries": [ // { // "answers": null, // "engine": "system", @@ -328,7 +336,7 @@ func main() { // ```JSON // { // "domain": "example.com", -// "lookup_host": [ +// "queries": [ // { // "answers": null, // "engine": "system", diff --git a/internal/tutorial/measurex/chapter02/README.md b/internal/tutorial/measurex/chapter02/README.md index cde5ab0..7edc3b2 100644 --- a/internal/tutorial/measurex/chapter02/README.md +++ b/internal/tutorial/measurex/chapter02/README.md @@ -69,10 +69,12 @@ address is quoted with "[" and "]" if IPv6, e.g., `[::1]:53`.) ### Printing the measurement -The rest of the main function is just like in the previous chapter. +The rest of the main function is just like in the previous +chapter. Like we did before, we convert the obtained measurement +to the "archival" data format before printing. ```Go - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) } @@ -98,27 +100,31 @@ Here is the JSON we obtain in output: // This block contains the results of the connect syscall // using the df-008-netevents data format. - "connect": [ - { - "address": "8.8.4.4:443", + "tcp_connect": [{ + "ip": "8.8.4.4", + "port": 443, + "t": 0.020303, + "status": { + "blocked": false, "failure": null, - "operation": "connect", - "proto": "tcp", - "t": 0.026879041, - "started": 8.8625e-05, - "oddity": "" - } - ] + "success": true + }, + "started": 0.000109292, + "oddity": "" + }] } ``` +This JSON implements [the df-005-tcpconnect](https://github.com/ooni/spec/blob/master/data-formats/df-005-tcpconnect.md) +OONI data format. + This is what it says: - we are connecting a "tcp" socket; - the destination endpoint address is "8.8.4.4:443"; -- connect terminated ~0.027 seconds into the program's life; +- connect terminated ~0.020 seconds into the program's life (see `t`); - the operation succeeded (`failure` is `nil`). @@ -138,19 +144,21 @@ We get this JSON: { "network": "tcp", "address": "127.0.0.1:1", - "connect": [ + "tcp_connect": [ { - "address": "127.0.0.1:1", - "failure": "connection_refused", - "operation": "connect", - "proto": "tcp", - "t": 0.000372167, - "started": 8.4917e-05, + "ip": "127.0.0.1", + "port": 1, + "t": 0.000457584, + "status": { + "blocked": true, + "failure": "connection_refused", + "success": false + }, + "started": 0.000104792, "oddity": "tcp.connect.refused" } ] } - ``` And here's an error telling us the connection was refused and @@ -170,14 +178,17 @@ We get this JSON: { "network": "tcp", "address": "8.8.4.4:1", - "connect": [ + "tcp_connect": [ { - "address": "8.8.4.4:1", - "failure": "generic_timeout_error", - "operation": "connect", - "proto": "tcp", - "t": 10.005494583, - "started": 8.4833e-05, + "ip": "8.8.4.4", + "port": 1, + "t": 10.006558625, + "status": { + "blocked": true, + "failure": "generic_timeout_error", + "success": false + }, + "started": 9.55e-05, "oddity": "tcp.connect.timeout" } ] @@ -201,21 +212,26 @@ To get this JSON: { "network": "tcp", "address": "8.8.4.4:1", - "connect": [ + "tcp_connect": [ { - "address": "8.8.4.4:1", - "failure": "generic_timeout_error", - "operation": "connect", - "proto": "tcp", - "t": 0.10148025, - "started": 0.000122375, + "ip": "8.8.4.4", + "port": 1, + "t": 0.105445125, + "status": { + "blocked": true, + "failure": "generic_timeout_error", + "success": false + }, + "started": 9.4083e-05, "oddity": "tcp.connect.timeout" } ] } ``` -We see a timeout after ~0.1s. We enforce a reasonably small +We see a timeout after ~0.1s. + +We enforce a reasonably small timeout for connecting, equal to 10 s, because we want to guarantee that measurements eventually terminate. Also, since often censorship is implemented by timing out, we don't want diff --git a/internal/tutorial/measurex/chapter02/main.go b/internal/tutorial/measurex/chapter02/main.go index b3a4742..d39336a 100644 --- a/internal/tutorial/measurex/chapter02/main.go +++ b/internal/tutorial/measurex/chapter02/main.go @@ -70,10 +70,12 @@ func main() { // // ### Printing the measurement // - // The rest of the main function is just like in the previous chapter. + // The rest of the main function is just like in the previous + // chapter. Like we did before, we convert the obtained measurement + // to the "archival" data format before printing. // // ```Go - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) } @@ -99,27 +101,31 @@ func main() { // // // This block contains the results of the connect syscall // // using the df-008-netevents data format. -// "connect": [ -// { -// "address": "8.8.4.4:443", +// "tcp_connect": [{ +// "ip": "8.8.4.4", +// "port": 443, +// "t": 0.020303, +// "status": { +// "blocked": false, // "failure": null, -// "operation": "connect", -// "proto": "tcp", -// "t": 0.026879041, -// "started": 8.8625e-05, -// "oddity": "" -// } -// ] +// "success": true +// }, +// "started": 0.000109292, +// "oddity": "" +// }] // } // ``` // +// This JSON implements [the df-005-tcpconnect](https://github.com/ooni/spec/blob/master/data-formats/df-005-tcpconnect.md) +// OONI data format. +// // This is what it says: // // - we are connecting a "tcp" socket; // // - the destination endpoint address is "8.8.4.4:443"; // -// - connect terminated ~0.027 seconds into the program's life; +// - connect terminated ~0.020 seconds into the program's life (see `t`); // // - the operation succeeded (`failure` is `nil`). // @@ -139,19 +145,21 @@ func main() { // { // "network": "tcp", // "address": "127.0.0.1:1", -// "connect": [ +// "tcp_connect": [ // { -// "address": "127.0.0.1:1", -// "failure": "connection_refused", -// "operation": "connect", -// "proto": "tcp", -// "t": 0.000372167, -// "started": 8.4917e-05, +// "ip": "127.0.0.1", +// "port": 1, +// "t": 0.000457584, +// "status": { +// "blocked": true, +// "failure": "connection_refused", +// "success": false +// }, +// "started": 0.000104792, // "oddity": "tcp.connect.refused" // } // ] // } -// // ``` // // And here's an error telling us the connection was refused and @@ -171,14 +179,17 @@ func main() { // { // "network": "tcp", // "address": "8.8.4.4:1", -// "connect": [ +// "tcp_connect": [ // { -// "address": "8.8.4.4:1", -// "failure": "generic_timeout_error", -// "operation": "connect", -// "proto": "tcp", -// "t": 10.005494583, -// "started": 8.4833e-05, +// "ip": "8.8.4.4", +// "port": 1, +// "t": 10.006558625, +// "status": { +// "blocked": true, +// "failure": "generic_timeout_error", +// "success": false +// }, +// "started": 9.55e-05, // "oddity": "tcp.connect.timeout" // } // ] @@ -202,21 +213,26 @@ func main() { // { // "network": "tcp", // "address": "8.8.4.4:1", -// "connect": [ +// "tcp_connect": [ // { -// "address": "8.8.4.4:1", -// "failure": "generic_timeout_error", -// "operation": "connect", -// "proto": "tcp", -// "t": 0.10148025, -// "started": 0.000122375, +// "ip": "8.8.4.4", +// "port": 1, +// "t": 0.105445125, +// "status": { +// "blocked": true, +// "failure": "generic_timeout_error", +// "success": false +// }, +// "started": 9.4083e-05, // "oddity": "tcp.connect.timeout" // } // ] // } // ``` // -// We see a timeout after ~0.1s. We enforce a reasonably small +// We see a timeout after ~0.1s. +// +// We enforce a reasonably small // timeout for connecting, equal to 10 s, because we want to // guarantee that measurements eventually terminate. Also, since // often censorship is implemented by timing out, we don't want diff --git a/internal/tutorial/measurex/chapter03/README.md b/internal/tutorial/measurex/chapter03/README.md index 6063733..0f0b4f3 100644 --- a/internal/tutorial/measurex/chapter03/README.md +++ b/internal/tutorial/measurex/chapter03/README.md @@ -57,7 +57,7 @@ Also this operation returns a measurement, which we print using the usual three-liner. ```Go - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) } @@ -83,42 +83,22 @@ be generated and inserted into a `Measurement`.) { "domain": "example.com", - // This block tells us about the UDP connect events - // where we bind to the server's endpoint - "connect": [ - { - "address": "8.8.4.4:53", - "failure": null, - "operation": "connect", - "proto": "udp", - "t": 0.00043175, - "started": 0.000191958, - "oddity": "" - }, - { - "address": "8.8.4.4:53", - "failure": null, - "operation": "connect", - "proto": "udp", - "t": 0.042198458, - "started": 0.042113208, - "oddity": "" - } - ], - // This block shows the read and write events // occurred on the sockets (because we control // in full the implementation of this DNS // over UDP resolver, we can see these events) - "read_write": [ + // + // See https://github.com/ooni/spec/blob/master/data-formats/df-008-netevents.md + // for a description of this data format. + "network_events": [ { "address": "8.8.4.4:53", "failure": null, "num_bytes": 29, "operation": "write", "proto": "udp", - "t": 0.000459583, - "started": 0.00043825, + "t": 0.00048825, + "started": 0.000462917, "oddity": "" }, { @@ -127,8 +107,8 @@ be generated and inserted into a `Measurement`.) "num_bytes": 45, "operation": "read", "proto": "udp", - "t": 0.041955792, - "started": 0.000471833, + "t": 0.022081833, + "started": 0.000502625, "oddity": "" }, { @@ -137,8 +117,8 @@ be generated and inserted into a `Measurement`.) "num_bytes": 29, "operation": "write", "proto": "udp", - "t": 0.042218917, - "started": 0.042203, + "t": 0.022433083, + "started": 0.022423875, "oddity": "" }, { @@ -147,19 +127,59 @@ be generated and inserted into a `Measurement`.) "num_bytes": 57, "operation": "read", "proto": "udp", - "t": 0.196646583, - "started": 0.042233167, + "t": 0.046706, + "started": 0.022443833, "oddity": "" } ], + // This block shows the query we sent (encoded as base64) + // and the response we received. Here we clearly see + // that we perform two DNS "round trip" (i.e., send request + // and receive response) to resolve a domain: one for + // A and the other for AAAA. + // + // We don't have a specification for this data format yet. + "dns_events": [ + { + "engine": "udp", + "resolver_address": "8.8.4.4:53", + "raw_query": { + "data": "dGwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", + "format": "base64" + }, + "started": 0.000205083, + "t": 0.022141333, + "failure": null, + "raw_reply": { + "data": "dGyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAEuIAARduNgi", + "format": "base64" + } + }, + { + "engine": "udp", + "resolver_address": "8.8.4.4:53", + "raw_query": { + "data": "Ts8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", + "format": "base64" + }, + "started": 0.022221417, + "t": 0.046733125, + "failure": null, + "raw_reply": { + "data": "Ts+BgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAFOQABAmBigAAiAAAQJIGJMlyBlG", + "format": "base64" + } + } + ], + // This is the same kind of result as before, we // show the emitted queries and the resolved addrs. // // Also note how here the resolver_address is the // correct endpoint address and the engine tells us // that we're using DNS over UDP. - "lookup_host": [ + "queries": [ { "answers": [ { @@ -172,8 +192,8 @@ be generated and inserted into a `Measurement`.) "hostname": "example.com", "query_type": "A", "resolver_address": "8.8.4.4:53", - "t": 0.196777042, - "started": 0.000118542, + "t": 0.046766833, + "started": 0.000124375, "oddity": "" }, { @@ -188,48 +208,10 @@ be generated and inserted into a `Measurement`.) "hostname": "example.com", "query_type": "AAAA", "resolver_address": "8.8.4.4:53", - "t": 0.196777042, - "started": 0.000118542, + "t": 0.046766833, + "started": 0.000124375, "oddity": "" } - ], - - // This block shows the query we sent (encoded as base64) - // and the response we received. Here we clearly see - // that we perform two DNS "round trip" (i.e., send request - // and receive response) to resolve a domain: one for - // A and the other for AAAA. - "dns_round_trip": [ - { - "engine": "udp", - "resolver_address": "8.8.4.4:53", - "raw_query": { - "data": "PrcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", - "format": "base64" - }, - "started": 0.000191625, - "t": 0.041998667, - "failure": null, - "raw_reply": { - "data": "PreBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAE8BAARduNgi", - "format": "base64" - } - }, - { - "engine": "udp", - "resolver_address": "8.8.4.4:53", - "raw_query": { - "data": "LAwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", - "format": "base64" - }, - "started": 0.04210775, - "t": 0.196701333, - "failure": null, - "raw_reply": { - "data": "LAyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAE6nABAmBigAAiAAAQJIGJMlyBlG", - "format": "base64" - } - } ] } ``` @@ -254,9 +236,42 @@ This produces the following JSON: ```JavaScript { "domain": "antani.ooni.org", - "connect": [ /* snip */ ], - "read_write": [ /* snip */ ], - "lookup_host": [ + "network_events": [ /* snip */ ], + + "dns_events": [ + { + "engine": "udp", + "resolver_address": "8.8.4.4:53", + "raw_query": { + "data": "p7YBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB", + "format": "base64" + }, + "started": 0.000152959, + "t": 0.051650917, + "failure": null, + "raw_reply": { + "data": "p7aBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==", + "format": "base64" + } + }, + { + "engine": "udp", + "resolver_address": "8.8.4.4:53", + "raw_query": { + "data": "ILkBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB", + "format": "base64" + }, + "started": 0.051755209, + "t": 0.101094375, + "failure": null, + "raw_reply": { + "data": "ILmBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==", + "format": "base64" + } + } + ], + + "queries": [ { "answers": null, "engine": "udp", @@ -264,8 +279,8 @@ This produces the following JSON: "hostname": "antani.ooni.org", "query_type": "A", "resolver_address": "8.8.4.4:53", - "t": 0.098208709, - "started": 8.95e-05, + "t": 0.101241667, + "started": 8.8e-05, "oddity": "dns.lookup.nxdomain" }, { @@ -275,42 +290,10 @@ This produces the following JSON: "hostname": "antani.ooni.org", "query_type": "AAAA", "resolver_address": "8.8.4.4:53", - "t": 0.098208709, - "started": 8.95e-05, + "t": 0.101241667, + "started": 8.8e-05, "oddity": "dns.lookup.nxdomain" } - ], - "dns_round_trip": [ - { - "engine": "udp", - "resolver_address": "8.8.4.4:53", - "raw_query": { - "data": "jLIBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB", - "format": "base64" - }, - "started": 0.000141542, - "t": 0.034689417, - "failure": null, - "raw_reply": { - "data": "jLKBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==", - "format": "base64" - } - }, - { - "engine": "udp", - "resolver_address": "8.8.4.4:53", - "raw_query": { - "data": "azEBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB", - "format": "base64" - }, - "started": 0.034776709, - "t": 0.098170542, - "failure": null, - "raw_reply": { - "data": "azGBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==", - "format": "base64" - } - } ] } ``` @@ -323,10 +306,10 @@ Let us now decode one of the replies by using this program: package main import ( -"fmt" -"encoding/base64" + "fmt" + "encoding/base64" -"github.com/miekg/dns" + "github.com/miekg/dns" ) func main() { @@ -366,16 +349,15 @@ Here's the corresponding JSON: ```JavaScript { "domain": "example.com", - "connect": [ /* snip */ ], - "read_write": [ + "network_events": [ { "address": "182.92.22.222:53", "failure": null, "num_bytes": 29, "operation": "write", "proto": "udp", - "t": 0.0005275, - "started": 0.000500209, + "t": 0.000479583, + "started": 0.00045525, "oddity": "" }, { @@ -383,56 +365,58 @@ Here's the corresponding JSON: "failure": "generic_timeout_error", /* <--- */ "operation": "read", "proto": "udp", - "t": 5.001140125, - "started": 0.000544042, + "t": 5.006016292, + "started": 0.000491792, "oddity": "" } ], - "lookup_host": [ - { - "answers": null, - "engine": "udp", - "failure": "generic_timeout_error", /* <--- */ - "hostname": "example.com", - "query_type": "A", - "resolver_address": "182.92.22.222:53", - "t": 5.001462084, - "started": 0.000127917, - "oddity": "dns.lookup.timeout" /* <--- */ - }, - { - "answers": null, - "engine": "udp", - "failure": "generic_timeout_error", - "hostname": "example.com", - "query_type": "AAAA", - "resolver_address": "182.92.22.222:53", - "t": 5.001462084, - "started": 0.000127917, - "oddity": "dns.lookup.timeout" - } - ], - "dns_round_trip": [ + "dns_events": [ { "engine": "udp", "resolver_address": "182.92.22.222:53", "raw_query": { - "data": "ej8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", + "data": "GRUBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", "format": "base64" }, - "started": 0.000220584, - "t": 5.001317417, - "failure": "generic_timeout_error", + "started": 0.00018225, + "t": 5.006185667, + "failure": "generic_timeout_error", /* <--- */ "raw_reply": null } + /* snip */ + ], + "queries": [ + { + "answers": null, + "engine": "udp", + "failure": "generic_timeout_error", /* <--- */ + "hostname": "example.com", + "query_type": "A", + "resolver_address": "182.92.22.222:53", + "t": 5.007385458, + "started": 0.000107583, + "oddity": "dns.lookup.timeout" + }, + { + "answers": null, + "engine": "udp", + "failure": "generic_timeout_error", /* <--- */ + "hostname": "example.com", + "query_type": "AAAA", + "resolver_address": "182.92.22.222:53", + "t": 5.007385458, + "started": 0.000107583, + "oddity": "dns.lookup.timeout" + } ] } ``` We see that we fail with a timeout (I have marked some of them with comments inside the JSON). We see the timeout at three different -levels of abstractions (from lower to higher abstraction): at the socket layer, -during the DNS round trip, during the DNS lookup. +levels of abstractions (from lower to higher abstraction): at the socket +layer (`network_events`), during the DNS round trip (`dns_events`), +during the DNS lookup (`queries`). What we also see is that `t`'s value is ~5s when the `read` event fails, which tells us about the socket's read timeout. @@ -451,18 +435,17 @@ Here's the answer I get: ```JavaScript { "domain": "example.com", - "connect": [ /* snip */ ], - // The I/O events look normal this time - "read_write": [ + // The network events look normal this time + "network_events": [ { "address": "180.97.36.63:53", "failure": null, "num_bytes": 29, "operation": "write", "proto": "udp", - "t": 0.000333583, - "started": 0.000312125, + "t": 0.000492125, + "started": 0.000467042, "oddity": "" }, { @@ -471,8 +454,8 @@ Here's the answer I get: "num_bytes": 29, "operation": "read", "proto": "udp", - "t": 0.334948125, - "started": 0.000366625, + "t": 0.321373542, + "started": 0.000504833, "oddity": "" }, { @@ -481,8 +464,8 @@ Here's the answer I get: "num_bytes": 29, "operation": "write", "proto": "udp", - "t": 0.3358025, - "started": 0.335725958, + "t": 0.322500875, + "started": 0.322450042, "oddity": "" }, { @@ -491,15 +474,49 @@ Here's the answer I get: "num_bytes": 29, "operation": "read", "proto": "udp", - "t": 0.739987666, - "started": 0.335863875, + "t": 0.655514542, + "started": 0.322557667, "oddity": "" } ], - // But we see both in the error and in the oddity + // Exercise: do like I did before and decode the messages + "dns_events": [ + { + "engine": "udp", + "resolver_address": "180.97.36.63:53", + "raw_query": { + "data": "WcgBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", + "format": "base64" + }, + "started": 0.000209875, + "t": 0.321504042, + "failure": null, + "raw_reply": { + "data": "WciBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", + "format": "base64" + } + }, + { + "engine": "udp", + "resolver_address": "180.97.36.63:53", + "raw_query": { + "data": "I9oBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", + "format": "base64" + }, + "started": 0.321672042, + "t": 0.655680792, + "failure": null, + "raw_reply": { + "data": "I9qBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", + "format": "base64" + } + } + ], + + // We see both in the error and in the oddity // that the response was "REFUSED" - "lookup_host": [ + "queries": [ { "answers": null, "engine": "udp", @@ -507,8 +524,8 @@ Here's the answer I get: "hostname": "example.com", "query_type": "A", "resolver_address": "180.97.36.63:53", - "t": 0.7402975, - "started": 7.2291e-05, + "t": 0.655814875, + "started": 0.000107417, "oddity": "dns.lookup.refused" }, { @@ -518,44 +535,10 @@ Here's the answer I get: "hostname": "example.com", "query_type": "AAAA", "resolver_address": "180.97.36.63:53", - "t": 0.7402975, - "started": 7.2291e-05, + "t": 0.655814875, + "started": 0.000107417, "oddity": "dns.lookup.refused" } - ], - - // Exercise: do like I did before and decode the messages - "dns_round_trip": [ - { - "engine": "udp", - "resolver_address": "180.97.36.63:53", - "raw_query": { - "data": "crkBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", - "format": "base64" - }, - "started": 0.000130666, - "t": 0.33509925, - "failure": null, - "raw_reply": { - "data": "crmBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", - "format": "base64" - } - }, - { - "engine": "udp", - "resolver_address": "180.97.36.63:53", - "raw_query": { - "data": "ywcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", - "format": "base64" - }, - "started": 0.335321333, - "t": 0.740152375, - "failure": null, - "raw_reply": { - "data": "yweBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", - "format": "base64" - } - } ] } ``` diff --git a/internal/tutorial/measurex/chapter03/main.go b/internal/tutorial/measurex/chapter03/main.go index 645e2b0..b309980 100644 --- a/internal/tutorial/measurex/chapter03/main.go +++ b/internal/tutorial/measurex/chapter03/main.go @@ -58,7 +58,7 @@ func main() { // we print using the usual three-liner. // // ```Go - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) } @@ -84,42 +84,22 @@ func main() { // { // "domain": "example.com", // -// // This block tells us about the UDP connect events -// // where we bind to the server's endpoint -// "connect": [ -// { -// "address": "8.8.4.4:53", -// "failure": null, -// "operation": "connect", -// "proto": "udp", -// "t": 0.00043175, -// "started": 0.000191958, -// "oddity": "" -// }, -// { -// "address": "8.8.4.4:53", -// "failure": null, -// "operation": "connect", -// "proto": "udp", -// "t": 0.042198458, -// "started": 0.042113208, -// "oddity": "" -// } -// ], -// // // This block shows the read and write events // // occurred on the sockets (because we control // // in full the implementation of this DNS // // over UDP resolver, we can see these events) -// "read_write": [ +// // +// // See https://github.com/ooni/spec/blob/master/data-formats/df-008-netevents.md +// // for a description of this data format. +// "network_events": [ // { // "address": "8.8.4.4:53", // "failure": null, // "num_bytes": 29, // "operation": "write", // "proto": "udp", -// "t": 0.000459583, -// "started": 0.00043825, +// "t": 0.00048825, +// "started": 0.000462917, // "oddity": "" // }, // { @@ -128,8 +108,8 @@ func main() { // "num_bytes": 45, // "operation": "read", // "proto": "udp", -// "t": 0.041955792, -// "started": 0.000471833, +// "t": 0.022081833, +// "started": 0.000502625, // "oddity": "" // }, // { @@ -138,8 +118,8 @@ func main() { // "num_bytes": 29, // "operation": "write", // "proto": "udp", -// "t": 0.042218917, -// "started": 0.042203, +// "t": 0.022433083, +// "started": 0.022423875, // "oddity": "" // }, // { @@ -148,19 +128,59 @@ func main() { // "num_bytes": 57, // "operation": "read", // "proto": "udp", -// "t": 0.196646583, -// "started": 0.042233167, +// "t": 0.046706, +// "started": 0.022443833, // "oddity": "" // } // ], // +// // This block shows the query we sent (encoded as base64) +// // and the response we received. Here we clearly see +// // that we perform two DNS "round trip" (i.e., send request +// // and receive response) to resolve a domain: one for +// // A and the other for AAAA. +// // +// // We don't have a specification for this data format yet. +// "dns_events": [ +// { +// "engine": "udp", +// "resolver_address": "8.8.4.4:53", +// "raw_query": { +// "data": "dGwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", +// "format": "base64" +// }, +// "started": 0.000205083, +// "t": 0.022141333, +// "failure": null, +// "raw_reply": { +// "data": "dGyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAEuIAARduNgi", +// "format": "base64" +// } +// }, +// { +// "engine": "udp", +// "resolver_address": "8.8.4.4:53", +// "raw_query": { +// "data": "Ts8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", +// "format": "base64" +// }, +// "started": 0.022221417, +// "t": 0.046733125, +// "failure": null, +// "raw_reply": { +// "data": "Ts+BgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAFOQABAmBigAAiAAAQJIGJMlyBlG", +// "format": "base64" +// } +// } +// ], +// // // This is the same kind of result as before, we // // show the emitted queries and the resolved addrs. // // // // Also note how here the resolver_address is the // // correct endpoint address and the engine tells us // // that we're using DNS over UDP. -// "lookup_host": [ +// "queries": [ // { // "answers": [ // { @@ -173,8 +193,8 @@ func main() { // "hostname": "example.com", // "query_type": "A", // "resolver_address": "8.8.4.4:53", -// "t": 0.196777042, -// "started": 0.000118542, +// "t": 0.046766833, +// "started": 0.000124375, // "oddity": "" // }, // { @@ -189,48 +209,10 @@ func main() { // "hostname": "example.com", // "query_type": "AAAA", // "resolver_address": "8.8.4.4:53", -// "t": 0.196777042, -// "started": 0.000118542, +// "t": 0.046766833, +// "started": 0.000124375, // "oddity": "" // } -// ], -// -// // This block shows the query we sent (encoded as base64) -// // and the response we received. Here we clearly see -// // that we perform two DNS "round trip" (i.e., send request -// // and receive response) to resolve a domain: one for -// // A and the other for AAAA. -// "dns_round_trip": [ -// { -// "engine": "udp", -// "resolver_address": "8.8.4.4:53", -// "raw_query": { -// "data": "PrcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", -// "format": "base64" -// }, -// "started": 0.000191625, -// "t": 0.041998667, -// "failure": null, -// "raw_reply": { -// "data": "PreBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAE8BAARduNgi", -// "format": "base64" -// } -// }, -// { -// "engine": "udp", -// "resolver_address": "8.8.4.4:53", -// "raw_query": { -// "data": "LAwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", -// "format": "base64" -// }, -// "started": 0.04210775, -// "t": 0.196701333, -// "failure": null, -// "raw_reply": { -// "data": "LAyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAE6nABAmBigAAiAAAQJIGJMlyBlG", -// "format": "base64" -// } -// } // ] // } // ``` @@ -255,9 +237,42 @@ func main() { // ```JavaScript // { // "domain": "antani.ooni.org", -// "connect": [ /* snip */ ], -// "read_write": [ /* snip */ ], -// "lookup_host": [ +// "network_events": [ /* snip */ ], +// +// "dns_events": [ +// { +// "engine": "udp", +// "resolver_address": "8.8.4.4:53", +// "raw_query": { +// "data": "p7YBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB", +// "format": "base64" +// }, +// "started": 0.000152959, +// "t": 0.051650917, +// "failure": null, +// "raw_reply": { +// "data": "p7aBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==", +// "format": "base64" +// } +// }, +// { +// "engine": "udp", +// "resolver_address": "8.8.4.4:53", +// "raw_query": { +// "data": "ILkBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB", +// "format": "base64" +// }, +// "started": 0.051755209, +// "t": 0.101094375, +// "failure": null, +// "raw_reply": { +// "data": "ILmBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==", +// "format": "base64" +// } +// } +// ], +// +// "queries": [ // { // "answers": null, // "engine": "udp", @@ -265,8 +280,8 @@ func main() { // "hostname": "antani.ooni.org", // "query_type": "A", // "resolver_address": "8.8.4.4:53", -// "t": 0.098208709, -// "started": 8.95e-05, +// "t": 0.101241667, +// "started": 8.8e-05, // "oddity": "dns.lookup.nxdomain" // }, // { @@ -276,42 +291,10 @@ func main() { // "hostname": "antani.ooni.org", // "query_type": "AAAA", // "resolver_address": "8.8.4.4:53", -// "t": 0.098208709, -// "started": 8.95e-05, +// "t": 0.101241667, +// "started": 8.8e-05, // "oddity": "dns.lookup.nxdomain" // } -// ], -// "dns_round_trip": [ -// { -// "engine": "udp", -// "resolver_address": "8.8.4.4:53", -// "raw_query": { -// "data": "jLIBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB", -// "format": "base64" -// }, -// "started": 0.000141542, -// "t": 0.034689417, -// "failure": null, -// "raw_reply": { -// "data": "jLKBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==", -// "format": "base64" -// } -// }, -// { -// "engine": "udp", -// "resolver_address": "8.8.4.4:53", -// "raw_query": { -// "data": "azEBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB", -// "format": "base64" -// }, -// "started": 0.034776709, -// "t": 0.098170542, -// "failure": null, -// "raw_reply": { -// "data": "azGBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==", -// "format": "base64" -// } -// } // ] // } // ``` @@ -324,10 +307,10 @@ func main() { // package main // // import ( -// "fmt" -// "encoding/base64" +// "fmt" +// "encoding/base64" // -// "github.com/miekg/dns" +// "github.com/miekg/dns" // ) // // func main() { @@ -367,16 +350,15 @@ func main() { // ```JavaScript // { // "domain": "example.com", -// "connect": [ /* snip */ ], -// "read_write": [ +// "network_events": [ // { // "address": "182.92.22.222:53", // "failure": null, // "num_bytes": 29, // "operation": "write", // "proto": "udp", -// "t": 0.0005275, -// "started": 0.000500209, +// "t": 0.000479583, +// "started": 0.00045525, // "oddity": "" // }, // { @@ -384,56 +366,58 @@ func main() { // "failure": "generic_timeout_error", /* <--- */ // "operation": "read", // "proto": "udp", -// "t": 5.001140125, -// "started": 0.000544042, +// "t": 5.006016292, +// "started": 0.000491792, // "oddity": "" // } // ], -// "lookup_host": [ -// { -// "answers": null, -// "engine": "udp", -// "failure": "generic_timeout_error", /* <--- */ -// "hostname": "example.com", -// "query_type": "A", -// "resolver_address": "182.92.22.222:53", -// "t": 5.001462084, -// "started": 0.000127917, -// "oddity": "dns.lookup.timeout" /* <--- */ -// }, -// { -// "answers": null, -// "engine": "udp", -// "failure": "generic_timeout_error", -// "hostname": "example.com", -// "query_type": "AAAA", -// "resolver_address": "182.92.22.222:53", -// "t": 5.001462084, -// "started": 0.000127917, -// "oddity": "dns.lookup.timeout" -// } -// ], -// "dns_round_trip": [ +// "dns_events": [ // { // "engine": "udp", // "resolver_address": "182.92.22.222:53", // "raw_query": { -// "data": "ej8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", +// "data": "GRUBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", // "format": "base64" // }, -// "started": 0.000220584, -// "t": 5.001317417, -// "failure": "generic_timeout_error", +// "started": 0.00018225, +// "t": 5.006185667, +// "failure": "generic_timeout_error", /* <--- */ // "raw_reply": null // } +// /* snip */ +// ], +// "queries": [ +// { +// "answers": null, +// "engine": "udp", +// "failure": "generic_timeout_error", /* <--- */ +// "hostname": "example.com", +// "query_type": "A", +// "resolver_address": "182.92.22.222:53", +// "t": 5.007385458, +// "started": 0.000107583, +// "oddity": "dns.lookup.timeout" +// }, +// { +// "answers": null, +// "engine": "udp", +// "failure": "generic_timeout_error", /* <--- */ +// "hostname": "example.com", +// "query_type": "AAAA", +// "resolver_address": "182.92.22.222:53", +// "t": 5.007385458, +// "started": 0.000107583, +// "oddity": "dns.lookup.timeout" +// } // ] // } // ``` // // We see that we fail with a timeout (I have marked some of them // with comments inside the JSON). We see the timeout at three different -// levels of abstractions (from lower to higher abstraction): at the socket layer, -// during the DNS round trip, during the DNS lookup. +// levels of abstractions (from lower to higher abstraction): at the socket +// layer (`network_events`), during the DNS round trip (`dns_events`), +// during the DNS lookup (`queries`). // // What we also see is that `t`'s value is ~5s when the `read` event // fails, which tells us about the socket's read timeout. @@ -452,18 +436,17 @@ func main() { // ```JavaScript // { // "domain": "example.com", -// "connect": [ /* snip */ ], // -// // The I/O events look normal this time -// "read_write": [ +// // The network events look normal this time +// "network_events": [ // { // "address": "180.97.36.63:53", // "failure": null, // "num_bytes": 29, // "operation": "write", // "proto": "udp", -// "t": 0.000333583, -// "started": 0.000312125, +// "t": 0.000492125, +// "started": 0.000467042, // "oddity": "" // }, // { @@ -472,8 +455,8 @@ func main() { // "num_bytes": 29, // "operation": "read", // "proto": "udp", -// "t": 0.334948125, -// "started": 0.000366625, +// "t": 0.321373542, +// "started": 0.000504833, // "oddity": "" // }, // { @@ -482,8 +465,8 @@ func main() { // "num_bytes": 29, // "operation": "write", // "proto": "udp", -// "t": 0.3358025, -// "started": 0.335725958, +// "t": 0.322500875, +// "started": 0.322450042, // "oddity": "" // }, // { @@ -492,15 +475,49 @@ func main() { // "num_bytes": 29, // "operation": "read", // "proto": "udp", -// "t": 0.739987666, -// "started": 0.335863875, +// "t": 0.655514542, +// "started": 0.322557667, // "oddity": "" // } // ], // -// // But we see both in the error and in the oddity +// // Exercise: do like I did before and decode the messages +// "dns_events": [ +// { +// "engine": "udp", +// "resolver_address": "180.97.36.63:53", +// "raw_query": { +// "data": "WcgBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", +// "format": "base64" +// }, +// "started": 0.000209875, +// "t": 0.321504042, +// "failure": null, +// "raw_reply": { +// "data": "WciBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", +// "format": "base64" +// } +// }, +// { +// "engine": "udp", +// "resolver_address": "180.97.36.63:53", +// "raw_query": { +// "data": "I9oBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", +// "format": "base64" +// }, +// "started": 0.321672042, +// "t": 0.655680792, +// "failure": null, +// "raw_reply": { +// "data": "I9qBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", +// "format": "base64" +// } +// } +// ], +// +// // We see both in the error and in the oddity // // that the response was "REFUSED" -// "lookup_host": [ +// "queries": [ // { // "answers": null, // "engine": "udp", @@ -508,8 +525,8 @@ func main() { // "hostname": "example.com", // "query_type": "A", // "resolver_address": "180.97.36.63:53", -// "t": 0.7402975, -// "started": 7.2291e-05, +// "t": 0.655814875, +// "started": 0.000107417, // "oddity": "dns.lookup.refused" // }, // { @@ -519,44 +536,10 @@ func main() { // "hostname": "example.com", // "query_type": "AAAA", // "resolver_address": "180.97.36.63:53", -// "t": 0.7402975, -// "started": 7.2291e-05, +// "t": 0.655814875, +// "started": 0.000107417, // "oddity": "dns.lookup.refused" // } -// ], -// -// // Exercise: do like I did before and decode the messages -// "dns_round_trip": [ -// { -// "engine": "udp", -// "resolver_address": "180.97.36.63:53", -// "raw_query": { -// "data": "crkBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", -// "format": "base64" -// }, -// "started": 0.000130666, -// "t": 0.33509925, -// "failure": null, -// "raw_reply": { -// "data": "crmBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=", -// "format": "base64" -// } -// }, -// { -// "engine": "udp", -// "resolver_address": "180.97.36.63:53", -// "raw_query": { -// "data": "ywcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", -// "format": "base64" -// }, -// "started": 0.335321333, -// "t": 0.740152375, -// "failure": null, -// "raw_reply": { -// "data": "yweBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=", -// "format": "base64" -// } -// } // ] // } // ``` diff --git a/internal/tutorial/measurex/chapter04/README.md b/internal/tutorial/measurex/chapter04/README.md index ca1fbc8..f4f1118 100644 --- a/internal/tutorial/measurex/chapter04/README.md +++ b/internal/tutorial/measurex/chapter04/README.md @@ -69,7 +69,7 @@ As usual, the method to perform a measurement returns the measurement itself, which we print below. ``` - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) } @@ -90,30 +90,16 @@ Let us comment the JSON in detail: "network": "tcp", "address": "8.8.4.4:443", - // This block is generated when connecting to a TCP - // socket, as we've already seen in chapter02 - "connect": [ - { - "address": "8.8.4.4:443", - "failure": null, - "operation": "connect", - "proto": "tcp", - "t": 0.046959084, - "started": 0.022998875, - "oddity": "" - } - ], - // These are the I/O events during the handshake - "read_write": [ + "network_events": [ { "address": "8.8.4.4:443", "failure": null, "num_bytes": 280, "operation": "write", "proto": "tcp", - "t": 0.048752875, - "started": 0.04874125, + "t": 0.048268333, + "started": 0.048246666, "oddity": "" }, { @@ -122,18 +108,18 @@ Let us comment the JSON in detail: "num_bytes": 517, "operation": "read", "proto": "tcp", - "t": 0.087221334, - "started": 0.048760417, + "t": 0.086214708, + "started": 0.048287791, "oddity": "" }, { "address": "8.8.4.4:443", "failure": null, - "num_bytes": 4301, + "num_bytes": 4303, "operation": "read", "proto": "tcp", - "t": 0.088843584, - "started": 0.088830959, + "t": 0.087951708, + "started": 0.08792725, "oddity": "" }, { @@ -142,14 +128,33 @@ Let us comment the JSON in detail: "num_bytes": 64, "operation": "write", "proto": "tcp", - "t": 0.092078042, - "started": 0.092064042, + "t": 0.090097833, + "started": 0.090083875, "oddity": "" } ], - // This block contains information about the handshake - "tls_handshake": [ + // This block is generated when connecting to a TCP + // socket, as we've already seen in chapter02 + "tcp_connect": [ + { + "ip": "8.8.4.4", + "port": 443, + "t": 0.046022583, + "status": { + "blocked": false, + "failure": null, + "success": true + }, + "started": 0.024424916, + "oddity": "" + } + ], + + // This block contains information about the handshake. + // + // See https://github.com/ooni/spec/blob/master/data-formats/df-006-tlshandshake.md + "tls_handshakes": [ { "cipher_suite": "TLS_AES_128_GCM_SHA256", "failure": null, @@ -157,7 +162,7 @@ Let us comment the JSON in detail: "tls_version": "TLSv1.3", "peer_certificates": [ { - "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", + "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=", "format": "base64" }, { @@ -169,7 +174,7 @@ Let us comment the JSON in detail: "format": "base64" } ], - "t": 0.092117709, + "t": 0.090150666, "address": "8.8.4.4:443", "server_name": "dns.google", "alpn": [ @@ -179,16 +184,12 @@ Let us comment the JSON in detail: "no_tls_verify": false, "oddity": "", "proto": "tcp", - "started": 0.047288542 + "started": 0.04635975 } ] } ``` -All the data formats we're using here are, by the way, -compatible with the data formats specified at -https://github.com/ooni/spec/tree/master/data-formats. - ### Suggested follow-up experiments Try to run experiments in the following scenarios, and diff --git a/internal/tutorial/measurex/chapter04/main.go b/internal/tutorial/measurex/chapter04/main.go index f399a5f..28b2553 100644 --- a/internal/tutorial/measurex/chapter04/main.go +++ b/internal/tutorial/measurex/chapter04/main.go @@ -70,7 +70,7 @@ func main() { // the measurement itself, which we print below. // // ``` - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) } @@ -91,30 +91,16 @@ func main() { // "network": "tcp", // "address": "8.8.4.4:443", // -// // This block is generated when connecting to a TCP -// // socket, as we've already seen in chapter02 -// "connect": [ -// { -// "address": "8.8.4.4:443", -// "failure": null, -// "operation": "connect", -// "proto": "tcp", -// "t": 0.046959084, -// "started": 0.022998875, -// "oddity": "" -// } -// ], -// // // These are the I/O events during the handshake -// "read_write": [ +// "network_events": [ // { // "address": "8.8.4.4:443", // "failure": null, // "num_bytes": 280, // "operation": "write", // "proto": "tcp", -// "t": 0.048752875, -// "started": 0.04874125, +// "t": 0.048268333, +// "started": 0.048246666, // "oddity": "" // }, // { @@ -123,18 +109,18 @@ func main() { // "num_bytes": 517, // "operation": "read", // "proto": "tcp", -// "t": 0.087221334, -// "started": 0.048760417, +// "t": 0.086214708, +// "started": 0.048287791, // "oddity": "" // }, // { // "address": "8.8.4.4:443", // "failure": null, -// "num_bytes": 4301, +// "num_bytes": 4303, // "operation": "read", // "proto": "tcp", -// "t": 0.088843584, -// "started": 0.088830959, +// "t": 0.087951708, +// "started": 0.08792725, // "oddity": "" // }, // { @@ -143,14 +129,33 @@ func main() { // "num_bytes": 64, // "operation": "write", // "proto": "tcp", -// "t": 0.092078042, -// "started": 0.092064042, +// "t": 0.090097833, +// "started": 0.090083875, // "oddity": "" // } // ], // -// // This block contains information about the handshake -// "tls_handshake": [ +// // This block is generated when connecting to a TCP +// // socket, as we've already seen in chapter02 +// "tcp_connect": [ +// { +// "ip": "8.8.4.4", +// "port": 443, +// "t": 0.046022583, +// "status": { +// "blocked": false, +// "failure": null, +// "success": true +// }, +// "started": 0.024424916, +// "oddity": "" +// } +// ], +// +// // This block contains information about the handshake. +// // +// // See https://github.com/ooni/spec/blob/master/data-formats/df-006-tlshandshake.md +// "tls_handshakes": [ // { // "cipher_suite": "TLS_AES_128_GCM_SHA256", // "failure": null, @@ -158,7 +163,7 @@ func main() { // "tls_version": "TLSv1.3", // "peer_certificates": [ // { -// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", +// "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=", // "format": "base64" // }, // { @@ -170,7 +175,7 @@ func main() { // "format": "base64" // } // ], -// "t": 0.092117709, +// "t": 0.090150666, // "address": "8.8.4.4:443", // "server_name": "dns.google", // "alpn": [ @@ -180,16 +185,12 @@ func main() { // "no_tls_verify": false, // "oddity": "", // "proto": "tcp", -// "started": 0.047288542 +// "started": 0.04635975 // } // ] // } // ``` // -// All the data formats we're using here are, by the way, -// compatible with the data formats specified at -// https://github.com/ooni/spec/tree/master/data-formats. -// // ### Suggested follow-up experiments // // Try to run experiments in the following scenarios, and diff --git a/internal/tutorial/measurex/chapter05/README.md b/internal/tutorial/measurex/chapter05/README.md index 2124b37..aac5593 100644 --- a/internal/tutorial/measurex/chapter05/README.md +++ b/internal/tutorial/measurex/chapter05/README.md @@ -69,7 +69,7 @@ As we did in the previous chapters, here's the usual three lines of code for printing the resulting measurement. ``` - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) } @@ -100,15 +100,15 @@ Produces this JSON: // are actually `recvfrom` and `sendto` but here // we follow the Go convention of using read/write // more frequently than send/recv.) - "read_write": [ + "network_events": [ { "address": "8.8.4.4:443", "failure": null, "num_bytes": 1252, "operation": "write_to", "proto": "quic", - "t": 0.003903167, - "started": 0.0037395, + "t": 0.027184208, + "started": 0.027127208, "oddity": "" }, { @@ -117,8 +117,8 @@ Produces this JSON: "num_bytes": 1252, "operation": "read_from", "proto": "quic", - "t": 0.029389125, - "started": 0.002954792, + "t": 0.053116458, + "started": 0.025626583, "oddity": "" }, { @@ -127,8 +127,8 @@ Produces this JSON: "num_bytes": 1252, "operation": "write_to", "proto": "quic", - "t": 0.029757584, - "started": 0.02972325, + "t": 0.054538792, + "started": 0.054517542, "oddity": "" }, { @@ -137,8 +137,8 @@ Produces this JSON: "num_bytes": 1252, "operation": "read_from", "proto": "quic", - "t": 0.045039875, - "started": 0.029424792, + "t": 0.069144958, + "started": 0.053194208, "oddity": "" }, { @@ -147,8 +147,8 @@ Produces this JSON: "num_bytes": 1252, "operation": "read_from", "proto": "quic", - "t": 0.045055334, - "started": 0.045049625, + "t": 0.069183458, + "started": 0.069173292, "oddity": "" }, { @@ -157,28 +157,28 @@ Produces this JSON: "num_bytes": 1252, "operation": "read_from", "proto": "quic", - "t": 0.045073917, - "started": 0.045069667, + "t": 0.06920225, + "started": 0.069197875, "oddity": "" }, { "address": "8.8.4.4:443", "failure": null, - "num_bytes": 1233, + "num_bytes": 1216, "operation": "read_from", "proto": "quic", - "t": 0.04508, - "started": 0.045075292, + "t": 0.069210958, + "started": 0.069206875, "oddity": "" }, { "address": "8.8.4.4:443", "failure": null, - "num_bytes": 64, + "num_bytes": 65, "operation": "read_from", "proto": "quic", - "t": 0.045088167, - "started": 0.045081167, + "t": 0.069220667, + "started": 0.069217375, "oddity": "" }, { @@ -187,8 +187,8 @@ Produces this JSON: "num_bytes": 44, "operation": "write_to", "proto": "quic", - "t": 0.045370417, - "started": 0.045338667, + "t": 0.069433417, + "started": 0.069417625, "oddity": "" }, { @@ -197,8 +197,8 @@ Produces this JSON: "num_bytes": 44, "operation": "write_to", "proto": "quic", - "t": 0.045392125, - "started": 0.045380959, + "t": 0.069677625, + "started": 0.069647458, "oddity": "" }, { @@ -207,8 +207,8 @@ Produces this JSON: "num_bytes": 83, "operation": "write_to", "proto": "quic", - "t": 0.047042542, - "started": 0.047001917, + "t": 0.073461917, + "started": 0.073432875, "oddity": "" }, { @@ -217,15 +217,15 @@ Produces this JSON: "num_bytes": 33, "operation": "write_to", "proto": "quic", - "t": 0.047060834, - "started": 0.047046875, + "t": 0.073559417, + "started": 0.073542542, "oddity": "" } ], // This section describes the QUIC handshake and it has // basically the same fields as the TLS handshake. - "quic_handshake": [ + "quic_handshakes": [ { "cipher_suite": "TLS_CHACHA20_POLY1305_SHA256", "failure": null, @@ -233,7 +233,7 @@ Produces this JSON: "tls_version": "TLSv1.3", "peer_certificates": [ { - "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", + "data": "MIIF4DCCBMigAwIBAgIQWiqDOVk83wAKAAAAAQivAjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTEwMDQwMjQzMzVaFw0yMTEyMjcwMjQzMzRaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUBr7csGnwBJnnh4nRPQFux/52IXuOIIUao2X4JQH7chvtuNyR4rxpeyL/zAYbFpohSdSfxrCFxrFWae+LzANL/VTPZPzPH2S2p1TNWy8rFWMU0OnRd3Ym5s1xOLBG9Lu28iH3h5+5yk5pGmvSnzA0Ra9Q5pjLJlUfnGi9ceP2uFM/SDRo1IEqNB7ukIeE5lmw7CJCyDafbWy8eFFPG69YRAjPb13NF4ijingt2JsUckgXJdWdoRf3KghM5ddvQhv5rdELjVUbI550MjfBQWoEUIxWEUsYNj/L4IbK46E4RpMnU+XnwTtinQYYgHVQxLKPtPdfYHp145DjiGXE9LMDAgMBAAGjggL5MIIC9TAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUo61UxicmPa371PCog8zI7NbrdkQwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQIGCisGAQQB1nkCBAIEgfMEgfAA7gB1AESUZS6w7s6vxEAH2Kj+KMDa5oK+2MsxtT/TM5a1toGoAAABfElmWjQAAAQDAEYwRAIgZpBA5ih3hRoZ1749kEcxdEcpzHUV3cS2zDHuz1WMy7gCIGtMqROCc/wrP01x1GXrk3M/qbHssnvhelxhNVbi4FTUAHUA9lyUL9F3MCIUVBgIMJRWjuNNExkzv98MLyALzE7xZOMAAAF8SWZaAwAABAMARjBEAiBRWeLjcLIQwBFdPEXa15s++b7kAKit86em9GR23F+7KQIgTHZgL7inapApbw5WJNhtEI78a5HHPsI+kU5LIDgpv7swDQYJKoZIhvcNAQELBQADggEBAGki3+h4nn12Ef449NirIIP5APx+1NQk3DDKT6PrpH3m+s/wQKXlJ8eNg6zJEBEtHxBdO0xI+/te2Bh1s6RU/iJZrVRAtol/xmn0AMvPQNL+JZUnYuryz2mwTpk5ZHnGHyZknbJDspB2oZkItDeDbvkMws+JKrCYbbHG4ZtcsoFYPrkjfyMLRemhj+qWvvMtUKUldsXbYhugQL44N+pWAJNyxEWUBcuEldItww/gSrl/O2alfGOTNvdcXT/nedrw+SJnci4m4oMTz+XWFkbf3yPXEikvUqcvQFe10F1wBr8wW6soM9nR7vKq7WUlnx1m4lghw/jStp2mWenT6YFp5Tw=", "format": "base64" }, { @@ -245,7 +245,7 @@ Produces this JSON: "format": "base64" } ], - "t": 0.047042459, + "t": 0.073469208, "address": "8.8.4.4:443", "server_name": "dns.google", "alpn": [ @@ -254,7 +254,7 @@ Produces this JSON: "no_tls_verify": false, "oddity": "", "proto": "quic", - "started": 0.002154834 + "started": 0.025061583 } ] } diff --git a/internal/tutorial/measurex/chapter05/main.go b/internal/tutorial/measurex/chapter05/main.go index 31a2819..f070e24 100644 --- a/internal/tutorial/measurex/chapter05/main.go +++ b/internal/tutorial/measurex/chapter05/main.go @@ -70,7 +70,7 @@ func main() { // lines of code for printing the resulting measurement. // // ``` - data, err := json.Marshal(m) + data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m)) runtimex.PanicOnError(err, "json.Marshal failed") fmt.Printf("%s\n", string(data)) } @@ -101,15 +101,15 @@ func main() { // // are actually `recvfrom` and `sendto` but here // // we follow the Go convention of using read/write // // more frequently than send/recv.) -// "read_write": [ +// "network_events": [ // { // "address": "8.8.4.4:443", // "failure": null, // "num_bytes": 1252, // "operation": "write_to", // "proto": "quic", -// "t": 0.003903167, -// "started": 0.0037395, +// "t": 0.027184208, +// "started": 0.027127208, // "oddity": "" // }, // { @@ -118,8 +118,8 @@ func main() { // "num_bytes": 1252, // "operation": "read_from", // "proto": "quic", -// "t": 0.029389125, -// "started": 0.002954792, +// "t": 0.053116458, +// "started": 0.025626583, // "oddity": "" // }, // { @@ -128,8 +128,8 @@ func main() { // "num_bytes": 1252, // "operation": "write_to", // "proto": "quic", -// "t": 0.029757584, -// "started": 0.02972325, +// "t": 0.054538792, +// "started": 0.054517542, // "oddity": "" // }, // { @@ -138,8 +138,8 @@ func main() { // "num_bytes": 1252, // "operation": "read_from", // "proto": "quic", -// "t": 0.045039875, -// "started": 0.029424792, +// "t": 0.069144958, +// "started": 0.053194208, // "oddity": "" // }, // { @@ -148,8 +148,8 @@ func main() { // "num_bytes": 1252, // "operation": "read_from", // "proto": "quic", -// "t": 0.045055334, -// "started": 0.045049625, +// "t": 0.069183458, +// "started": 0.069173292, // "oddity": "" // }, // { @@ -158,28 +158,28 @@ func main() { // "num_bytes": 1252, // "operation": "read_from", // "proto": "quic", -// "t": 0.045073917, -// "started": 0.045069667, +// "t": 0.06920225, +// "started": 0.069197875, // "oddity": "" // }, // { // "address": "8.8.4.4:443", // "failure": null, -// "num_bytes": 1233, +// "num_bytes": 1216, // "operation": "read_from", // "proto": "quic", -// "t": 0.04508, -// "started": 0.045075292, +// "t": 0.069210958, +// "started": 0.069206875, // "oddity": "" // }, // { // "address": "8.8.4.4:443", // "failure": null, -// "num_bytes": 64, +// "num_bytes": 65, // "operation": "read_from", // "proto": "quic", -// "t": 0.045088167, -// "started": 0.045081167, +// "t": 0.069220667, +// "started": 0.069217375, // "oddity": "" // }, // { @@ -188,8 +188,8 @@ func main() { // "num_bytes": 44, // "operation": "write_to", // "proto": "quic", -// "t": 0.045370417, -// "started": 0.045338667, +// "t": 0.069433417, +// "started": 0.069417625, // "oddity": "" // }, // { @@ -198,8 +198,8 @@ func main() { // "num_bytes": 44, // "operation": "write_to", // "proto": "quic", -// "t": 0.045392125, -// "started": 0.045380959, +// "t": 0.069677625, +// "started": 0.069647458, // "oddity": "" // }, // { @@ -208,8 +208,8 @@ func main() { // "num_bytes": 83, // "operation": "write_to", // "proto": "quic", -// "t": 0.047042542, -// "started": 0.047001917, +// "t": 0.073461917, +// "started": 0.073432875, // "oddity": "" // }, // { @@ -218,15 +218,15 @@ func main() { // "num_bytes": 33, // "operation": "write_to", // "proto": "quic", -// "t": 0.047060834, -// "started": 0.047046875, +// "t": 0.073559417, +// "started": 0.073542542, // "oddity": "" // } // ], // // // This section describes the QUIC handshake and it has // // basically the same fields as the TLS handshake. -// "quic_handshake": [ +// "quic_handshakes": [ // { // "cipher_suite": "TLS_CHACHA20_POLY1305_SHA256", // "failure": null, @@ -234,7 +234,7 @@ func main() { // "tls_version": "TLSv1.3", // "peer_certificates": [ // { -// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", +// "data": "MIIF4DCCBMigAwIBAgIQWiqDOVk83wAKAAAAAQivAjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTEwMDQwMjQzMzVaFw0yMTEyMjcwMjQzMzRaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUBr7csGnwBJnnh4nRPQFux/52IXuOIIUao2X4JQH7chvtuNyR4rxpeyL/zAYbFpohSdSfxrCFxrFWae+LzANL/VTPZPzPH2S2p1TNWy8rFWMU0OnRd3Ym5s1xOLBG9Lu28iH3h5+5yk5pGmvSnzA0Ra9Q5pjLJlUfnGi9ceP2uFM/SDRo1IEqNB7ukIeE5lmw7CJCyDafbWy8eFFPG69YRAjPb13NF4ijingt2JsUckgXJdWdoRf3KghM5ddvQhv5rdELjVUbI550MjfBQWoEUIxWEUsYNj/L4IbK46E4RpMnU+XnwTtinQYYgHVQxLKPtPdfYHp145DjiGXE9LMDAgMBAAGjggL5MIIC9TAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUo61UxicmPa371PCog8zI7NbrdkQwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQIGCisGAQQB1nkCBAIEgfMEgfAA7gB1AESUZS6w7s6vxEAH2Kj+KMDa5oK+2MsxtT/TM5a1toGoAAABfElmWjQAAAQDAEYwRAIgZpBA5ih3hRoZ1749kEcxdEcpzHUV3cS2zDHuz1WMy7gCIGtMqROCc/wrP01x1GXrk3M/qbHssnvhelxhNVbi4FTUAHUA9lyUL9F3MCIUVBgIMJRWjuNNExkzv98MLyALzE7xZOMAAAF8SWZaAwAABAMARjBEAiBRWeLjcLIQwBFdPEXa15s++b7kAKit86em9GR23F+7KQIgTHZgL7inapApbw5WJNhtEI78a5HHPsI+kU5LIDgpv7swDQYJKoZIhvcNAQELBQADggEBAGki3+h4nn12Ef449NirIIP5APx+1NQk3DDKT6PrpH3m+s/wQKXlJ8eNg6zJEBEtHxBdO0xI+/te2Bh1s6RU/iJZrVRAtol/xmn0AMvPQNL+JZUnYuryz2mwTpk5ZHnGHyZknbJDspB2oZkItDeDbvkMws+JKrCYbbHG4ZtcsoFYPrkjfyMLRemhj+qWvvMtUKUldsXbYhugQL44N+pWAJNyxEWUBcuEldItww/gSrl/O2alfGOTNvdcXT/nedrw+SJnci4m4oMTz+XWFkbf3yPXEikvUqcvQFe10F1wBr8wW6soM9nR7vKq7WUlnx1m4lghw/jStp2mWenT6YFp5Tw=", // "format": "base64" // }, // { @@ -246,7 +246,7 @@ func main() { // "format": "base64" // } // ], -// "t": 0.047042459, +// "t": 0.073469208, // "address": "8.8.4.4:443", // "server_name": "dns.google", // "alpn": [ @@ -255,7 +255,7 @@ func main() { // "no_tls_verify": false, // "oddity": "", // "proto": "quic", -// "started": 0.002154834 +// "started": 0.025061583 // } // ] // } diff --git a/internal/tutorial/measurex/chapter06/README.md b/internal/tutorial/measurex/chapter06/README.md index ae4d759..2a767e7 100644 --- a/internal/tutorial/measurex/chapter06/README.md +++ b/internal/tutorial/measurex/chapter06/README.md @@ -172,7 +172,7 @@ go doc ./internal/measurex.HTTPEndpointMeasurement Let us now print the resulting measurement. ```Go - print(m) + print(measurex.NewArchivalHTTPEndpointMeasurement(m)) } ``` @@ -182,7 +182,7 @@ Let us now print the resulting measurement. Let us perform a vanilla run first: ```bash -go run -race ./internal/tutorial/measurex/chapter06 +go run -race ./internal/tutorial/measurex/chapter06 | jq ``` This is the JSON output. Let us comment it in detail: @@ -196,31 +196,17 @@ This is the JSON output. Let us comment it in detail: "network": "tcp", "address": "8.8.4.4:443", - // Internally, HTTPEndpointGetWithoutCookies calls - // TCPConnect and here we see the corresponding event - "connect": [ - { - "address": "8.8.4.4:443", - "failure": null, - "operation": "connect", - "proto": "tcp", - "t": 0.02422375, - "started": 0.002269291, - "oddity": "" - } - ], - // These are the I/O operations we have already seen // in previous chapters - "read_write": [ + "network_events": [ { "address": "8.8.4.4:443", "failure": null, "num_bytes": 280, "operation": "write", "proto": "tcp", - "t": 0.024931791, - "started": 0.024910416, + "t": 0.045800292, + "started": 0.045782167, "oddity": "" }, { @@ -229,18 +215,18 @@ This is the JSON output. Let us comment it in detail: "num_bytes": 517, "operation": "read", "proto": "tcp", - "t": 0.063629791, - "started": 0.024935666, + "t": 0.082571, + "started": 0.045805458, "oddity": "" }, { "address": "8.8.4.4:443", "failure": null, - "num_bytes": 4301, + "num_bytes": 4303, "operation": "read", "proto": "tcp", - "t": 0.064183, - "started": 0.064144208, + "t": 0.084400542, + "started": 0.084372667, "oddity": "" }, { @@ -249,8 +235,8 @@ This is the JSON output. Let us comment it in detail: "num_bytes": 64, "operation": "write", "proto": "tcp", - "t": 0.065464041, - "started": 0.065441333, + "t": 0.086762625, + "started": 0.086748292, "oddity": "" }, { @@ -259,8 +245,8 @@ This is the JSON output. Let us comment it in detail: "num_bytes": 86, "operation": "write", "proto": "tcp", - "t": 0.067256083, - "started": 0.067224375, + "t": 0.087851, + "started": 0.087837625, "oddity": "" }, { @@ -269,8 +255,8 @@ This is the JSON output. Let us comment it in detail: "num_bytes": 201, "operation": "write", "proto": "tcp", - "t": 0.067674416, - "started": 0.067652375, + "t": 0.089527292, + "started": 0.089507958, "oddity": "" }, { @@ -279,8 +265,8 @@ This is the JSON output. Let us comment it in detail: "num_bytes": 93, "operation": "read", "proto": "tcp", - "t": 0.086618708, - "started": 0.067599208, + "t": 0.168585625, + "started": 0.088068375, "oddity": "" }, { @@ -289,18 +275,28 @@ This is the JSON output. Let us comment it in detail: "num_bytes": 31, "operation": "write", "proto": "tcp", - "t": 0.086703625, - "started": 0.0866745, + "t": 0.168713542, + "started": 0.168671417, "oddity": "" }, { "address": "8.8.4.4:443", "failure": null, - "num_bytes": 2028, + "num_bytes": 2000, "operation": "read", "proto": "tcp", - "t": 0.337785916, - "started": 0.086717333, + "t": 0.468671417, + "started": 0.168759333, + "oddity": "" + }, + { + "address": "8.8.4.4:443", + "failure": null, + "num_bytes": 39, + "operation": "read", + "proto": "tcp", + "t": 0.47118175, + "started": 0.471169667, "oddity": "" }, { @@ -309,8 +305,8 @@ This is the JSON output. Let us comment it in detail: "num_bytes": 39, "operation": "write", "proto": "tcp", - "t": 0.338514916, - "started": 0.338485375, + "t": 0.471335458, + "started": 0.471268583, "oddity": "" }, { @@ -319,17 +315,25 @@ This is the JSON output. Let us comment it in detail: "num_bytes": 24, "operation": "write", "proto": "tcp", - "t": 0.338800833, - "started": 0.338788625, + "t": 0.471865, + "started": 0.471836292, "oddity": "" - }, + } + ], + + // Internally, HTTPEndpointGetWithoutCookies calls + // TCPConnect and here we see the corresponding event + "tcp_connect": [ { - "address": "8.8.4.4:443", - "failure": "connection_already_closed", - "operation": "read", - "proto": "tcp", - "t": 0.338888041, - "started": 0.338523291, + "ip": "8.8.4.4", + "port": 443, + "t": 0.043644958, + "status": { + "blocked": false, + "failure": null, + "success": true + }, + "started": 0.022849458, "oddity": "" } ], @@ -339,7 +343,7 @@ This is the JSON output. Let us comment it in detail: // specified a QUIC endpoint we would instead see here a // QUIC handshake event. And, we would not see any handshake // if the URL was instead an HTTP URL. - "tls_handshake": [ + "tls_handshakes": [ { "cipher_suite": "TLS_AES_128_GCM_SHA256", "failure": null, @@ -347,7 +351,7 @@ This is the JSON output. Let us comment it in detail: "tls_version": "TLSv1.3", "peer_certificates": [ { - "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", + "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=", "format": "base64" }, { @@ -359,7 +363,7 @@ This is the JSON output. Let us comment it in detail: "format": "base64" } ], - "t": 0.065514708, + "t": 0.086816667, "address": "8.8.4.4:443", "server_name": "dns.google", "alpn": [ @@ -369,13 +373,13 @@ This is the JSON output. Let us comment it in detail: "no_tls_verify": false, "oddity": "", "proto": "tcp", - "started": 0.024404083 + "started": 0.043971083 } ], // Finally here we see information about the round trip, which - // is formatted according the df-001-httpt data format: - "http_round_trip": [ + // is formatted according to https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md: + "requests": [ { // This field indicates whether there was an error during @@ -389,21 +393,20 @@ This is the JSON output. Let us comment it in detail: "headers": { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "accept-language": "en-US;q=0.8,en;q=0.5", - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36" + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36" } }, - // This field contains the response status code, body, - // and headers. + // This field contains the response status code, body, and headers. "response": { "code": 200, "headers": { "accept-ranges": "none", - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", "cache-control": "private", - "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-bSLcJjaotppZl3Y2moIaxg==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'", + "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-y/OGliLR2gbEfTG2i4MAaw==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'", "content-type": "text/html; charset=UTF-8", - "date": "Fri, 24 Sep 2021 08:51:01 GMT", + "date": "Fri, 05 Nov 2021 08:59:37 GMT", "server": "scaffolding on HTTPServer2", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "vary": "Accept-Encoding", @@ -416,7 +419,7 @@ This is the JSON output. Let us comment it in detail: // body: we don't want to read and submit to the OONI // collector large bodies. "body": { - "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJiU0xjSmphb3RwcFpsM1kybW9JYXhnPT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+", + "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJ5L09HbGlMUjJnYkVmVEcyaTRNQWF3PT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+", "format": "base64" }, @@ -436,14 +439,14 @@ This is the JSON output. Let us comment it in detail: // The t field is the moment where we finished the // round trip and saved the event. The started field // is instead when we started the round trip. - + // // You may notice that the start of the round trip // if after the `t` of the handshake. This tells us // that the code first connects, then handshakes, and // finally creates HTTP code for performing the // round trip. - "t": 0.338674625, - "started": 0.065926625, + "t": 0.471535167, + "started": 0.087176458, // As usual we also compute an oddity value related // in this case to the HTTP round trip. diff --git a/internal/tutorial/measurex/chapter06/main.go b/internal/tutorial/measurex/chapter06/main.go index a5a2333..a69b0f9 100644 --- a/internal/tutorial/measurex/chapter06/main.go +++ b/internal/tutorial/measurex/chapter06/main.go @@ -173,7 +173,7 @@ func main() { // Let us now print the resulting measurement. // // ```Go - print(m) + print(measurex.NewArchivalHTTPEndpointMeasurement(m)) } // ``` @@ -183,7 +183,7 @@ func main() { // Let us perform a vanilla run first: // // ```bash -// go run -race ./internal/tutorial/measurex/chapter06 +// go run -race ./internal/tutorial/measurex/chapter06 | jq // ``` // // This is the JSON output. Let us comment it in detail: @@ -197,31 +197,17 @@ func main() { // "network": "tcp", // "address": "8.8.4.4:443", // -// // Internally, HTTPEndpointGetWithoutCookies calls -// // TCPConnect and here we see the corresponding event -// "connect": [ -// { -// "address": "8.8.4.4:443", -// "failure": null, -// "operation": "connect", -// "proto": "tcp", -// "t": 0.02422375, -// "started": 0.002269291, -// "oddity": "" -// } -// ], -// // // These are the I/O operations we have already seen // // in previous chapters -// "read_write": [ +// "network_events": [ // { // "address": "8.8.4.4:443", // "failure": null, // "num_bytes": 280, // "operation": "write", // "proto": "tcp", -// "t": 0.024931791, -// "started": 0.024910416, +// "t": 0.045800292, +// "started": 0.045782167, // "oddity": "" // }, // { @@ -230,18 +216,18 @@ func main() { // "num_bytes": 517, // "operation": "read", // "proto": "tcp", -// "t": 0.063629791, -// "started": 0.024935666, +// "t": 0.082571, +// "started": 0.045805458, // "oddity": "" // }, // { // "address": "8.8.4.4:443", // "failure": null, -// "num_bytes": 4301, +// "num_bytes": 4303, // "operation": "read", // "proto": "tcp", -// "t": 0.064183, -// "started": 0.064144208, +// "t": 0.084400542, +// "started": 0.084372667, // "oddity": "" // }, // { @@ -250,8 +236,8 @@ func main() { // "num_bytes": 64, // "operation": "write", // "proto": "tcp", -// "t": 0.065464041, -// "started": 0.065441333, +// "t": 0.086762625, +// "started": 0.086748292, // "oddity": "" // }, // { @@ -260,8 +246,8 @@ func main() { // "num_bytes": 86, // "operation": "write", // "proto": "tcp", -// "t": 0.067256083, -// "started": 0.067224375, +// "t": 0.087851, +// "started": 0.087837625, // "oddity": "" // }, // { @@ -270,8 +256,8 @@ func main() { // "num_bytes": 201, // "operation": "write", // "proto": "tcp", -// "t": 0.067674416, -// "started": 0.067652375, +// "t": 0.089527292, +// "started": 0.089507958, // "oddity": "" // }, // { @@ -280,8 +266,8 @@ func main() { // "num_bytes": 93, // "operation": "read", // "proto": "tcp", -// "t": 0.086618708, -// "started": 0.067599208, +// "t": 0.168585625, +// "started": 0.088068375, // "oddity": "" // }, // { @@ -290,18 +276,28 @@ func main() { // "num_bytes": 31, // "operation": "write", // "proto": "tcp", -// "t": 0.086703625, -// "started": 0.0866745, +// "t": 0.168713542, +// "started": 0.168671417, // "oddity": "" // }, // { // "address": "8.8.4.4:443", // "failure": null, -// "num_bytes": 2028, +// "num_bytes": 2000, // "operation": "read", // "proto": "tcp", -// "t": 0.337785916, -// "started": 0.086717333, +// "t": 0.468671417, +// "started": 0.168759333, +// "oddity": "" +// }, +// { +// "address": "8.8.4.4:443", +// "failure": null, +// "num_bytes": 39, +// "operation": "read", +// "proto": "tcp", +// "t": 0.47118175, +// "started": 0.471169667, // "oddity": "" // }, // { @@ -310,8 +306,8 @@ func main() { // "num_bytes": 39, // "operation": "write", // "proto": "tcp", -// "t": 0.338514916, -// "started": 0.338485375, +// "t": 0.471335458, +// "started": 0.471268583, // "oddity": "" // }, // { @@ -320,17 +316,25 @@ func main() { // "num_bytes": 24, // "operation": "write", // "proto": "tcp", -// "t": 0.338800833, -// "started": 0.338788625, +// "t": 0.471865, +// "started": 0.471836292, // "oddity": "" -// }, +// } +// ], +// +// // Internally, HTTPEndpointGetWithoutCookies calls +// // TCPConnect and here we see the corresponding event +// "tcp_connect": [ // { -// "address": "8.8.4.4:443", -// "failure": "connection_already_closed", -// "operation": "read", -// "proto": "tcp", -// "t": 0.338888041, -// "started": 0.338523291, +// "ip": "8.8.4.4", +// "port": 443, +// "t": 0.043644958, +// "status": { +// "blocked": false, +// "failure": null, +// "success": true +// }, +// "started": 0.022849458, // "oddity": "" // } // ], @@ -340,7 +344,7 @@ func main() { // // specified a QUIC endpoint we would instead see here a // // QUIC handshake event. And, we would not see any handshake // // if the URL was instead an HTTP URL. -// "tls_handshake": [ +// "tls_handshakes": [ // { // "cipher_suite": "TLS_AES_128_GCM_SHA256", // "failure": null, @@ -348,7 +352,7 @@ func main() { // "tls_version": "TLSv1.3", // "peer_certificates": [ // { -// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc", +// "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=", // "format": "base64" // }, // { @@ -360,7 +364,7 @@ func main() { // "format": "base64" // } // ], -// "t": 0.065514708, +// "t": 0.086816667, // "address": "8.8.4.4:443", // "server_name": "dns.google", // "alpn": [ @@ -370,13 +374,13 @@ func main() { // "no_tls_verify": false, // "oddity": "", // "proto": "tcp", -// "started": 0.024404083 +// "started": 0.043971083 // } // ], // // // Finally here we see information about the round trip, which -// // is formatted according the df-001-httpt data format: -// "http_round_trip": [ +// // is formatted according to https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md: +// "requests": [ // { // // // This field indicates whether there was an error during @@ -390,21 +394,20 @@ func main() { // "headers": { // "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", // "accept-language": "en-US;q=0.8,en;q=0.5", -// "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36" +// "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36" // } // }, // -// // This field contains the response status code, body, -// // and headers. +// // This field contains the response status code, body, and headers. // "response": { // "code": 200, // "headers": { // "accept-ranges": "none", -// "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", +// "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", // "cache-control": "private", -// "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-bSLcJjaotppZl3Y2moIaxg==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'", +// "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-y/OGliLR2gbEfTG2i4MAaw==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'", // "content-type": "text/html; charset=UTF-8", -// "date": "Fri, 24 Sep 2021 08:51:01 GMT", +// "date": "Fri, 05 Nov 2021 08:59:37 GMT", // "server": "scaffolding on HTTPServer2", // "strict-transport-security": "max-age=31536000; includeSubDomains; preload", // "vary": "Accept-Encoding", @@ -417,7 +420,7 @@ func main() { // // body: we don't want to read and submit to the OONI // // collector large bodies. // "body": { -// "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJiU0xjSmphb3RwcFpsM1kybW9JYXhnPT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+", +// "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJ5L09HbGlMUjJnYkVmVEcyaTRNQWF3PT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+", // "format": "base64" // }, // @@ -437,14 +440,14 @@ func main() { // // The t field is the moment where we finished the // // round trip and saved the event. The started field // // is instead when we started the round trip. -// +// // // // You may notice that the start of the round trip // // if after the `t` of the handshake. This tells us // // that the code first connects, then handshakes, and // // finally creates HTTP code for performing the // // round trip. -// "t": 0.338674625, -// "started": 0.065926625, +// "t": 0.471535167, +// "started": 0.087176458, // // // As usual we also compute an oddity value related // // in this case to the HTTP round trip. diff --git a/internal/tutorial/measurex/chapter07/README.md b/internal/tutorial/measurex/chapter07/README.md index 8392c3a..2adefae 100644 --- a/internal/tutorial/measurex/chapter07/README.md +++ b/internal/tutorial/measurex/chapter07/README.md @@ -111,7 +111,8 @@ We are almost done now: we loop over all the endpoints and apply the } ``` -Finally, we print the results. +Finally, we print the results. (Note that here we are not +converting to the OONI archival data format.) ```Go print(m) @@ -128,7 +129,10 @@ go run -race ./internal/tutorial/measurex/chapter07 | jq ``` Please, check the JSON output. Do you recognize the fields -we have described in previous chapters? +we have described in previous chapters, even though we didn't +convert to the OONI data format? Can you modify the code to +use the OONI data format in the output by calling the proper +conversion functions exported by `measurex`? Can you provoke common errors such as DNS resolution errors, TCP connect errors, TLS handshake errors, and diff --git a/internal/tutorial/measurex/chapter07/main.go b/internal/tutorial/measurex/chapter07/main.go index 7cc0ba6..1917248 100644 --- a/internal/tutorial/measurex/chapter07/main.go +++ b/internal/tutorial/measurex/chapter07/main.go @@ -112,7 +112,8 @@ func main() { } // ``` // - // Finally, we print the results. + // Finally, we print the results. (Note that here we are not + // converting to the OONI archival data format.) // // ```Go print(m) @@ -129,7 +130,10 @@ func main() { // ``` // // Please, check the JSON output. Do you recognize the fields -// we have described in previous chapters? +// we have described in previous chapters, even though we didn't +// convert to the OONI data format? Can you modify the code to +// use the OONI data format in the output by calling the proper +// conversion functions exported by `measurex`? // // Can you provoke common errors such as DNS resolution // errors, TCP connect errors, TLS handshake errors, and diff --git a/internal/tutorial/measurex/chapter08/README.md b/internal/tutorial/measurex/chapter08/README.md index 361db53..700414d 100644 --- a/internal/tutorial/measurex/chapter08/README.md +++ b/internal/tutorial/measurex/chapter08/README.md @@ -101,6 +101,15 @@ This is it. The rest of the program is exactly the same. for _, epnt := range httpEndpoints { m.Endpoints = append(m.Endpoints, mx.HTTPEndpointGetWithoutCookies(ctx, epnt)) } +``` + +(Note that here, like in the previous chapter, we are not converting +to the OONI data format. Rather, we're just dumping the internally +used data structures. Exercise: can you modify this program to emit +a JSON compliant with the OONI data format by using the proper] +conversion functions exported by `measurex`?) + +```Go print(m) } diff --git a/internal/tutorial/measurex/chapter08/main.go b/internal/tutorial/measurex/chapter08/main.go index 7300eae..9db3a9c 100644 --- a/internal/tutorial/measurex/chapter08/main.go +++ b/internal/tutorial/measurex/chapter08/main.go @@ -102,6 +102,15 @@ func main() { for _, epnt := range httpEndpoints { m.Endpoints = append(m.Endpoints, mx.HTTPEndpointGetWithoutCookies(ctx, epnt)) } + // ``` + // + // (Note that here, like in the previous chapter, we are not converting + // to the OONI data format. Rather, we're just dumping the internally + // used data structures. Exercise: can you modify this program to emit + // a JSON compliant with the OONI data format by using the proper] + // conversion functions exported by `measurex`?) + // + // ```Go print(m) } diff --git a/internal/tutorial/measurex/chapter09/README.md b/internal/tutorial/measurex/chapter09/README.md index 39ef26a..1a984d7 100644 --- a/internal/tutorial/measurex/chapter09/README.md +++ b/internal/tutorial/measurex/chapter09/README.md @@ -101,6 +101,11 @@ fully measured, this method closes the returned channel. Like we did before, we append the resulting measurements to our `m` container and we print it. +Exercise: here we're not using the OONI data format and we're +instead printing the internally used data structures. Can +you modify the code to emit data using OONI's data format here? +(Hint: there are conversion functions in `measurex`.) + ```Go print(m) } diff --git a/internal/tutorial/measurex/chapter09/main.go b/internal/tutorial/measurex/chapter09/main.go index 67280a7..9b8f44f 100644 --- a/internal/tutorial/measurex/chapter09/main.go +++ b/internal/tutorial/measurex/chapter09/main.go @@ -102,6 +102,11 @@ func main() { // Like we did before, we append the resulting measurements to // our `m` container and we print it. // + // Exercise: here we're not using the OONI data format and we're + // instead printing the internally used data structures. Can + // you modify the code to emit data using OONI's data format here? + // (Hint: there are conversion functions in `measurex`.) + // // ```Go print(m) } diff --git a/internal/tutorial/measurex/chapter12/README.md b/internal/tutorial/measurex/chapter12/README.md index d7c428f..cba87cf 100644 --- a/internal/tutorial/measurex/chapter12/README.md +++ b/internal/tutorial/measurex/chapter12/README.md @@ -32,7 +32,7 @@ import ( ) type measurement struct { - URLs []*measurex.URLMeasurement + URLs []*measurex.ArchivalURLMeasurement } func print(v interface{}) { @@ -67,7 +67,7 @@ is closed when done by `MeasureURLAndFollowRedirections`, so we leave the loop. ```Go for m := range mx.MeasureURLAndFollowRedirections(ctx, *URL, headers, cookies) { - all.URLs = append(all.URLs, m) + all.URLs = append(all.URLs, measurex.NewArchivalURLMeasurement(m)) } print(all) } @@ -86,6 +86,9 @@ Take a look at the JSON. You should see several redirects and that we measure each endpoint of each redirect, including QUIC endpoints that we discover on the way. +Exercise: remove code for converting to OONI data format +and compare output with previous chapter. See any difference? + ## Conclusion We have introduced `MeasureURLAndFollowRedirect`, the diff --git a/internal/tutorial/measurex/chapter12/main.go b/internal/tutorial/measurex/chapter12/main.go index fbee259..899c978 100644 --- a/internal/tutorial/measurex/chapter12/main.go +++ b/internal/tutorial/measurex/chapter12/main.go @@ -33,7 +33,7 @@ import ( ) type measurement struct { - URLs []*measurex.URLMeasurement + URLs []*measurex.ArchivalURLMeasurement } func print(v interface{}) { @@ -68,7 +68,7 @@ func main() { // // ```Go for m := range mx.MeasureURLAndFollowRedirections(ctx, *URL, headers, cookies) { - all.URLs = append(all.URLs, m) + all.URLs = append(all.URLs, measurex.NewArchivalURLMeasurement(m)) } print(all) } @@ -87,6 +87,9 @@ func main() { // and that we measure each endpoint of each redirect, including // QUIC endpoints that we discover on the way. // +// Exercise: remove code for converting to OONI data format +// and compare output with previous chapter. See any difference? +// // ## Conclusion // // We have introduced `MeasureURLAndFollowRedirect`, the diff --git a/internal/tutorial/measurex/chapter14/README.md b/internal/tutorial/measurex/chapter14/README.md index fbce4c4..620e59c 100644 --- a/internal/tutorial/measurex/chapter14/README.md +++ b/internal/tutorial/measurex/chapter14/README.md @@ -45,10 +45,10 @@ that a Web Connectivity measurement should have. ```Go type measurement struct { - Queries []*measurex.DNSLookupEvent `json:"queries"` - TCPConnect []*measurex.NetworkEvent `json:"tcp_connect"` - TLSHandshakes []*measurex.TLSHandshakeEvent `json:"tls_handshakes"` - Requests []*measurex.HTTPRoundTripEvent `json:"requests"` + Queries []*measurex.ArchivalDNSLookupEvent `json:"queries"` + TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"` + TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"` + Requests []*measurex.ArchivalHTTPRoundTripEvent `json:"requests"` } ``` @@ -96,7 +96,8 @@ the input URL's domain using the system resolver. ```Go dns := mx.LookupHostSystem(ctx, parsedURL.Hostname()) - m.Queries = append(m.Queries, dns.LookupHost...) + m.Queries = append( + m.Queries, measurex.NewArchivalDNSLookupEventList(dns.LookupHost)...) ``` @@ -128,7 +129,8 @@ whether the input URL is HTTP or HTTPS. switch parsedURL.Scheme { case "http": tcp := mx.TCPConnect(ctx, epnt.Address) - m.TCPConnect = append(m.TCPConnect, tcp.Connect...) + m.TCPConnect = append( + m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...) case "https": config := &tls.Config{ ServerName: parsedURL.Hostname(), @@ -136,8 +138,10 @@ whether the input URL is HTTP or HTTPS. RootCAs: netxlite.NewDefaultCertPool(), } tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config) - m.TCPConnect = append(m.TCPConnect, tls.Connect...) - m.TLSHandshakes = append(m.TLSHandshakes, tls.TLSHandshake...) + m.TCPConnect = append( + m.TCPConnect, measurex.NewArchivalTCPConnectList(tls.Connect)...) + m.TLSHandshakes = append(m.TLSHandshakes, + measurex.NewArchivalQUICTLSHandshakeEventList(tls.TLSHandshake)...) } } @@ -222,7 +226,8 @@ the chapters we have seen so far. ```Go - m.Requests = append(m.Requests, db.AsMeasurement().HTTPRoundTrip...) + m.Requests = append(m.Requests, measurex.NewArchivalHTTPRoundTripEventList( + db.AsMeasurement().HTTPRoundTrip)...) return m, nil } diff --git a/internal/tutorial/measurex/chapter14/main.go b/internal/tutorial/measurex/chapter14/main.go index c197585..1447bae 100644 --- a/internal/tutorial/measurex/chapter14/main.go +++ b/internal/tutorial/measurex/chapter14/main.go @@ -46,10 +46,10 @@ func print(v interface{}) { // ```Go type measurement struct { - Queries []*measurex.DNSLookupEvent `json:"queries"` - TCPConnect []*measurex.NetworkEvent `json:"tcp_connect"` - TLSHandshakes []*measurex.TLSHandshakeEvent `json:"tls_handshakes"` - Requests []*measurex.HTTPRoundTripEvent `json:"requests"` + Queries []*measurex.ArchivalDNSLookupEvent `json:"queries"` + TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"` + TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"` + Requests []*measurex.ArchivalHTTPRoundTripEvent `json:"requests"` } // ``` @@ -97,7 +97,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) { // // ```Go dns := mx.LookupHostSystem(ctx, parsedURL.Hostname()) - m.Queries = append(m.Queries, dns.LookupHost...) + m.Queries = append( + m.Queries, measurex.NewArchivalDNSLookupEventList(dns.LookupHost)...) // ``` // @@ -129,7 +130,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) { switch parsedURL.Scheme { case "http": tcp := mx.TCPConnect(ctx, epnt.Address) - m.TCPConnect = append(m.TCPConnect, tcp.Connect...) + m.TCPConnect = append( + m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...) case "https": config := &tls.Config{ ServerName: parsedURL.Hostname(), @@ -137,8 +139,10 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) { RootCAs: netxlite.NewDefaultCertPool(), } tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config) - m.TCPConnect = append(m.TCPConnect, tls.Connect...) - m.TLSHandshakes = append(m.TLSHandshakes, tls.TLSHandshake...) + m.TCPConnect = append( + m.TCPConnect, measurex.NewArchivalTCPConnectList(tls.Connect)...) + m.TLSHandshakes = append(m.TLSHandshakes, + measurex.NewArchivalQUICTLSHandshakeEventList(tls.TLSHandshake)...) } } @@ -223,7 +227,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) { // // ```Go - m.Requests = append(m.Requests, db.AsMeasurement().HTTPRoundTrip...) + m.Requests = append(m.Requests, measurex.NewArchivalHTTPRoundTripEventList( + db.AsMeasurement().HTTPRoundTrip)...) return m, nil }