c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(measurex): use same keys of the OONI data format (#572)

Browse Source 
This change should simplify the pipeline's job.

Reference issue: https://github.com/ooni/probe/issues/1817.

I previously dismissed this possibility, but now it seems clear it
is simpler to have a very tabular data format internally and to
convert such a format to OONI's data format when serializing.

The OONI data format is what the pipeline expects, but processing
is easier with a more linear/tabular format.
This commit is contained in:
Simone Basso 2021-11-05 10:46:45 +01:00 committed by GitHub
parent 6f90d29bfa
commit aa27bbe33f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 1571 additions and 1025 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/engine/experiment/webstepsx/measurer.go
View File

@ -29,7 +29,7 @@ type Config struct{}
// TestKeys contains the experiment's test keys.
type TestKeys struct {
*measurex.URLMeasurement
*measurex.ArchivalURLMeasurement
}
// Measurer performs the measurement.
@ -142,7 +142,9 @@ func (mx *Measurer) runAsync(ctx context.Context, sess model.ExperimentSession,
},
Input: model.MeasurementTarget(m.URL),
MeasurementRuntime: m.TotalRuntime.Seconds(),
TestKeys: &TestKeys{URLMeasurement: m},
TestKeys: &TestKeys{
ArchivalURLMeasurement: measurex.NewArchivalURLMeasurement(m),
},
}
}
}
@ -163,7 +165,7 @@ type measurerMeasureURLHelper struct {
func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header,
curEndpoints ...*measurex.HTTPEndpoint) (
[]*measurex.HTTPEndpoint, interface{}, error) {
[]*measurex.HTTPEndpoint, *measurex.THMeasurement, error) {
cc := &THClientCall{
Endpoints: measurex.HTTPEndpointsToEndpoints(curEndpoints),
HTTPClient: mth.Clnt,

49
internal/engine/experiment/webstepsx/th.go
View File

@ -41,14 +41,7 @@ type THClientRequest struct {
}
// THServerResponse is the response from the test helper.
type THServerResponse struct {
// DNS contains all the DNS related measurements.
DNS []*measurex.DNSMeasurement `json:"dns"`
// Endpoints contains a measurement for each endpoint
// that was discovered by the probe or the TH.
Endpoints []*measurex.HTTPEndpointMeasurement `json:"endpoints"`
}
type THServerResponse = measurex.THMeasurement
// thMaxAcceptableBodySize is the maximum acceptable body size by TH code.
const thMaxAcceptableBodySize = 1 << 20
@ -294,9 +287,9 @@ func (h *THHandler) simplifyMeasurement(in *measurex.Measurement) (out *measurex
}
func (h *THHandler) simplifyHandshake(
in []*measurex.TLSHandshakeEvent) (out []*measurex.TLSHandshakeEvent) {
in []*measurex.QUICTLSHandshakeEvent) (out []*measurex.QUICTLSHandshakeEvent) {
for _, ev := range in {
out = append(out, &measurex.TLSHandshakeEvent{
out = append(out, &measurex.QUICTLSHandshakeEvent{
CipherSuite: ev.CipherSuite,
Failure: ev.Failure,
NegotiatedProto: ev.NegotiatedProto,
@ -319,32 +312,24 @@ func (h *THHandler) simplifyHTTPRoundTrip(
in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) {
for _, ev := range in {
out = append(out, &measurex.HTTPRoundTripEvent{
Failure: ev.Failure,
Request: ev.Request,
Response: h.simplifyHTTPResponse(ev.Response),
Finished: 0,
Started: 0,
Oddity: ev.Oddity,
Failure: ev.Failure,
Method: ev.Method,
URL: ev.URL,
RequestHeaders: ev.RequestHeaders,
StatusCode: ev.StatusCode,
ResponseHeaders: ev.ResponseHeaders,
ResponseBody: nil, // we don't transfer the body
ResponseBodyLength: ev.ResponseBodyLength,
ResponseBodyIsTruncated: ev.ResponseBodyIsTruncated,
ResponseBodyIsUTF8: ev.ResponseBodyIsUTF8,
Finished: ev.Finished,
Started: ev.Started,
Oddity: ev.Oddity,
})
}
return
}
func (h *THHandler) simplifyHTTPResponse(
in *measurex.HTTPResponse) (out *measurex.HTTPResponse) {
if in != nil {
out = &measurex.HTTPResponse{
Code: in.Code,
Headers: in.Headers,
Body: nil,
BodyIsTruncated: in.BodyIsTruncated,
BodyLength: in.BodyLength,
BodyIsUTF8: in.BodyIsUTF8,
}
}
return
}
type thMeasureURLHelper struct {
epnts []*measurex.Endpoint
}
@ -352,7 +337,7 @@ type thMeasureURLHelper struct {
func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header,
serverEpnts ...*measurex.HTTPEndpoint) (
epnts []*measurex.HTTPEndpoint, thMeaurement interface{}, err error) {
epnts []*measurex.HTTPEndpoint, thMeaurement *measurex.THMeasurement, err error) {
for _, epnt := range thh.epnts {
epnts = append(epnts, &measurex.HTTPEndpoint{
Domain: URL.Hostname(),

533
internal/measurex/archival.go
View File

@ -1,8 +1,11 @@
package measurex
import (
"net"
"net/http"
"strconv"
"strings"
"time"
)
//
@ -11,6 +14,10 @@ import (
// This file defines helpers to serialize to the OONI data format.
//
//
// BinaryData
//
// ArchivalBinaryData is the archival format for binary data.
type ArchivalBinaryData struct {
Data []byte `json:"data"`
@ -29,6 +36,130 @@ func NewArchivalBinaryData(data []byte) (out *ArchivalBinaryData) {
return
}
//
// NetworkEvent
//
// ArchivalNetworkEvent is the OONI data format representation
// of a network event according to df-008-netevents.
type ArchivalNetworkEvent struct {
// JSON names compatible with df-008-netevents
RemoteAddr string `json:"address"`
Failure *string `json:"failure"`
Count int `json:"num_bytes,omitempty"`
Operation string `json:"operation"`
Network string `json:"proto"`
Finished float64 `json:"t"`
Started float64 `json:"started"`
// Names that are not part of the spec.
Oddity Oddity `json:"oddity"`
}
// NewArchivalNetworkEvent converts a network event to its archival format.
func NewArchivalNetworkEvent(in *NetworkEvent) *ArchivalNetworkEvent {
return &ArchivalNetworkEvent{
RemoteAddr: in.RemoteAddr,
Failure: in.Failure,
Count: in.Count,
Operation: in.Operation,
Network: in.Network,
Finished: in.Finished,
Started: in.Started,
Oddity: in.Oddity,
}
}
// NewArchivalNetworkEventList converts a list of NetworkEvent
// to a list of ArchivalNetworkEvent.
func NewArchivalNetworkEventList(in []*NetworkEvent) (out []*ArchivalNetworkEvent) {
for _, ev := range in {
out = append(out, NewArchivalNetworkEvent(ev))
}
return
}
//
// DNSRoundTripEvent
//
// ArchivalDNSRoundTripEvent is the OONI data format representation
// of a DNS round trip, which is currently not specified.
//
// We are trying to use names compatible with the names currently
// used by other specifications we currently use.
type ArchivalDNSRoundTripEvent struct {
Network string `json:"engine"`
Address string `json:"resolver_address"`
Query *ArchivalBinaryData `json:"raw_query"`
Started float64 `json:"started"`
Finished float64 `json:"t"`
Failure *string `json:"failure"`
Reply *ArchivalBinaryData `json:"raw_reply"`
}
// NewArchivalDNSRoundTripEvent converts a DNSRoundTripEvent into is archival format.
func NewArchivalDNSRoundTripEvent(in *DNSRoundTripEvent) *ArchivalDNSRoundTripEvent {
return &ArchivalDNSRoundTripEvent{
Network: in.Network,
Address: in.Address,
Query: NewArchivalBinaryData(in.Query),
Started: in.Started,
Finished: in.Finished,
Failure: in.Failure,
Reply: NewArchivalBinaryData(in.Reply),
}
}
// NewArchivalDNSRoundTripEventList converts a DNSRoundTripEvent
// list to the corresponding archival format.
func NewArchivalDNSRoundTripEventList(in []*DNSRoundTripEvent) (out []*ArchivalDNSRoundTripEvent) {
for _, ev := range in {
out = append(out, NewArchivalDNSRoundTripEvent(ev))
}
return
}
//
// HTTPRoundTrip
//
// ArchivalHTTPRequest is the archival format of an HTTP
// request according to df-001-http.md.
type ArchivalHTTPRequest struct {
Method string `json:"method"`
URL string `json:"url"`
Headers ArchivalHeaders `json:"headers"`
}
// ArchivalHTTPResponse is the archival format of an HTTP
// response according to df-001-http.md.
type ArchivalHTTPResponse struct {
// Names consistent with df-001-http.md
Code int64 `json:"code"`
Headers ArchivalHeaders `json:"headers"`
Body *ArchivalBinaryData `json:"body"`
BodyIsTruncated bool `json:"body_is_truncated"`
// Fields not part of the spec
BodyLength int64 `json:"x_body_length"`
BodyIsUTF8 bool `json:"x_body_is_utf8"`
}
// ArchivalHTTPRoundTripEvent is the archival format of an
// HTTP response according to df-001-http.md.
type ArchivalHTTPRoundTripEvent struct {
// JSON names following the df-001-httpt data format.
Failure *string `json:"failure"`
Request *HTTPRequest `json:"request"`
Response *HTTPResponse `json:"response"`
Finished float64 `json:"t"`
Started float64 `json:"started"`
// Names not in the specification
Oddity Oddity `json:"oddity"`
}
// ArchivalHeaders is a list of HTTP headers.
type ArchivalHeaders map[string]string
@ -54,6 +185,64 @@ func NewArchivalHeaders(in http.Header) (out ArchivalHeaders) {
return
}
// NewArchivalHTTPRoundTripEvent converts an HTTPRoundTrip to its archival format.
func NewArchivalHTTPRoundTripEvent(in *HTTPRoundTripEvent) *ArchivalHTTPRoundTripEvent {
return &ArchivalHTTPRoundTripEvent{
Failure: in.Failure,
Request: &HTTPRequest{
Method: in.Method,
URL: in.URL,
Headers: NewArchivalHeaders(in.RequestHeaders),
},
Response: &HTTPResponse{
Code: in.StatusCode,
Headers: NewArchivalHeaders(in.ResponseHeaders),
Body: NewArchivalBinaryData(in.ResponseBody),
BodyLength: in.ResponseBodyLength,
BodyIsTruncated: in.ResponseBodyIsTruncated,
BodyIsUTF8: in.ResponseBodyIsUTF8,
},
Finished: in.Finished,
Started: in.Started,
Oddity: in.Oddity,
}
}
// NewArchivalHTTPRoundTripEventList converts a list of
// HTTPRoundTripEvent to a list of ArchivalRoundTripEvent.
func NewArchivalHTTPRoundTripEventList(in []*HTTPRoundTripEvent) (out []*ArchivalHTTPRoundTripEvent) {
for _, ev := range in {
out = append(out, NewArchivalHTTPRoundTripEvent(ev))
}
return
}
//
// QUICTLSHandshakeEvent
//
// ArchivalQUICTLSHandshakeEvent is the archival data format for a
// QUIC or TLS handshake event according to df-006-tlshandshake.
type ArchivalQUICTLSHandshakeEvent struct {
// JSON names compatible with df-006-tlshandshake
CipherSuite string `json:"cipher_suite"`
Failure *string `json:"failure"`
NegotiatedProto string `json:"negotiated_proto"`
TLSVersion string `json:"tls_version"`
PeerCerts []*ArchivalBinaryData `json:"peer_certificates"`
Finished float64 `json:"t"`
// JSON names that are consistent with the
// spirit of the spec but are not in it
RemoteAddr string `json:"address"`
SNI string `json:"server_name"` // used in prod
ALPN []string `json:"alpn"`
SkipVerify bool `json:"no_tls_verify"` // used in prod
Oddity Oddity `json:"oddity"`
Network string `json:"proto"`
Started float64 `json:"started"`
}
// NewArchivalTLSCertList builds a new []ArchivalBinaryData
// from a list of raw x509 certificates data.
func NewArchivalTLSCerts(in [][]byte) (out []*ArchivalBinaryData) {
@ -66,13 +255,341 @@ func NewArchivalTLSCerts(in [][]byte) (out []*ArchivalBinaryData) {
return
}
// NewArchivalFailure creates an archival failure from an error. We
// cannot round trip an error using JSON, so we serialize to this
// intermediate format that is a sort of Optional<string>.
func NewArchivalFailure(err error) *string {
if err == nil {
return nil
// NewArchivalQUICTLSHandshakeEvent converts a QUICTLSHandshakeEvent
// to its archival data format.
func NewArchivalQUICTLSHandshakeEvent(in *QUICTLSHandshakeEvent) *ArchivalQUICTLSHandshakeEvent {
return &ArchivalQUICTLSHandshakeEvent{
CipherSuite: in.CipherSuite,
Failure: in.Failure,
NegotiatedProto: in.NegotiatedProto,
TLSVersion: in.TLSVersion,
PeerCerts: NewArchivalTLSCerts(in.PeerCerts),
Finished: in.Finished,
RemoteAddr: in.RemoteAddr,
SNI: in.SNI,
ALPN: in.ALPN,
SkipVerify: in.SkipVerify,
Oddity: in.Oddity,
Network: in.Network,
Started: in.Started,
}
s := err.Error()
return &s
}
// NewArchivalQUICTLSHandshakeEventList converts a list of
// QUICTLSHandshakeEvent to a list of ArchivalQUICTLSHandshakeEvent.
func NewArchivalQUICTLSHandshakeEventList(in []*QUICTLSHandshakeEvent) (out []*ArchivalQUICTLSHandshakeEvent) {
for _, ev := range in {
out = append(out, NewArchivalQUICTLSHandshakeEvent(ev))
}
return
}
//
// DNSLookup
//
// ArchivalDNSLookupAnswer is the archival format of a
// DNS lookup answer according to df-002-dnst.
type ArchivalDNSLookupAnswer struct {
// JSON names compatible with df-002-dnst's spec
Type string `json:"answer_type"`
IPv4 string `json:"ipv4,omitempty"`
IPv6 string `json:"ivp6,omitempty"`
// Names not part of the spec.
ALPN string `json:"alpn,omitempty"`
}
// ArchivalDNSLookupEvent is the archival data format
// of a DNS lookup according to df-002-dnst.
type ArchivalDNSLookupEvent struct {
// fields inside df-002-dnst
Answers []ArchivalDNSLookupAnswer `json:"answers"`
Network string `json:"engine"`
Failure *string `json:"failure"`
Domain string `json:"hostname"`
QueryType string `json:"query_type"`
Address string `json:"resolver_address"`
Finished float64 `json:"t"`
// Names not part of the spec.
Started float64 `json:"started"`
Oddity Oddity `json:"oddity"`
}
// NewArchivalDNSLookupAnswers creates a list of ArchivalDNSLookupAnswer.
func NewArchivalDNSLookupAnswers(in *DNSLookupEvent) (out []ArchivalDNSLookupAnswer) {
for _, ip := range in.A {
out = append(out, ArchivalDNSLookupAnswer{
Type: "A",
IPv4: ip,
})
}
for _, ip := range in.AAAA {
out = append(out, ArchivalDNSLookupAnswer{
Type: "AAAA",
IPv6: ip,
})
}
for _, alpn := range in.ALPN {
out = append(out, ArchivalDNSLookupAnswer{
Type: "ALPN",
ALPN: alpn,
})
}
return
}
// NewArchivalDNSLookupEvent converts a DNSLookupEvent
// to its archival representation.
func NewArchivalDNSLookupEvent(in *DNSLookupEvent) *ArchivalDNSLookupEvent {
return &ArchivalDNSLookupEvent{
Answers: NewArchivalDNSLookupAnswers(in),
Network: in.Network,
Failure: in.Failure,
Domain: in.Domain,
QueryType: in.QueryType,
Address: in.Address,
Finished: in.Finished,
Started: in.Started,
Oddity: in.Oddity,
}
}
// NewArchivalDNSLookupEventList converts a list of DNSLookupEvent
// to a list of ArchivalDNSLookupEvent.
func NewArchivalDNSLookupEventList(in []*DNSLookupEvent) (out []*ArchivalDNSLookupEvent) {
for _, ev := range in {
out = append(out, NewArchivalDNSLookupEvent(ev))
}
return
}
//
// TCPConnect
//
// ArchivalTCPConnect is the archival form of TCP connect
// events in compliance with df-005-tcpconnect.
type ArchivalTCPConnect struct {
// Names part of the spec.
IP string `json:"ip"`
Port int64 `json:"port"`
Finished float64 `json:"t"`
Status *ArchivalTCPConnectStatus `json:"status"`
// Names not part of the spec.
Started float64 `json:"started"`
Oddity Oddity `json:"oddity"`
}
// ArchivalTCPConnectStatus contains the status of a TCP connect.
type ArchivalTCPConnectStatus struct {
Blocked bool `json:"blocked"`
Failure *string `json:"failure"`
Success bool `json:"success"`
}
// NewArchivalTCPConnect converts a NetworkEvent to an ArchivalTCPConnect.
func NewArchivalTCPConnect(in *NetworkEvent) *ArchivalTCPConnect {
// We ignore errors because values come from Go code that
// emits correct serialization of TCP/UDP addresses.
addr, port, _ := net.SplitHostPort(in.RemoteAddr)
portnum, _ := strconv.Atoi(port)
return &ArchivalTCPConnect{
IP: addr,
Port: int64(portnum),
Finished: in.Finished,
Status: &ArchivalTCPConnectStatus{
Blocked: in.Failure != nil,
Failure: in.Failure,
Success: in.Failure == nil,
},
Started: in.Started,
Oddity: in.Oddity,
}
}
// NewArchivalTCPConnectList converts a list of NetworkEvent
// to a list of ArchivalTCPConnect. In doing that, the code
// only considers "connect" events using the TCP protocol.
func NewArchivalTCPConnectList(in []*NetworkEvent) (out []*ArchivalTCPConnect) {
for _, ev := range in {
if ev.Operation != "connect" {
continue
}
switch ev.Network {
case "tcp", "tcp4", "tcp6":
out = append(out, NewArchivalTCPConnect(ev))
default:
// nothing
}
}
return
}
//
// URLMeasurement
//
// ArchivalURLMeasurement is the archival representation of URLMeasurement
type ArchivalURLMeasurement struct {
URL string `json:"url"`
DNS []*ArchivalDNSMeasurement `json:"dns"`
Endpoints []*ArchivalHTTPEndpointMeasurement `json:"endpoints"`
TH *ArchivalTHMeasurement `json:"th"`
TotalRuntime time.Duration `json:"x_total_runtime"`
DNSRuntime time.Duration `json:"x_dns_runtime"`
THRuntime time.Duration `json:"x_th_runtime"`
EpntsRuntime time.Duration `json:"x_epnts_runtime"`
}
// NewArchivalURLMeasurement creates the archival representation
// of an URLMeasurement data structure.
func NewArchivalURLMeasurement(in *URLMeasurement) *ArchivalURLMeasurement {
return &ArchivalURLMeasurement{
URL: in.URL,
DNS: NewArchivalDNSMeasurementList(in.DNS),
Endpoints: NewArchivalHTTPEndpointMeasurementList(in.Endpoints),
TH: NewArchivalTHMeasurement(in.TH),
TotalRuntime: in.TotalRuntime,
DNSRuntime: in.DNSRuntime,
THRuntime: in.THRuntime,
EpntsRuntime: in.EpntsRuntime,
}
}
//
// EndpointMeasurement
//
// ArchivalEndpointMeasurement is the archival representation of EndpointMeasurement.
type ArchivalEndpointMeasurement struct {
// Network is the network of this endpoint.
Network EndpointNetwork `json:"network"`
// Address is the address of this endpoint.
Address string `json:"address"`
// An EndpointMeasurement is a Measurement.
*ArchivalMeasurement
}
// NewArchivalEndpointMeasurement converts an EndpointMeasurement
// to the corresponding archival data format.
func NewArchivalEndpointMeasurement(in *EndpointMeasurement) *ArchivalEndpointMeasurement {
return &ArchivalEndpointMeasurement{
Network: in.Network,
Address: in.Address,
ArchivalMeasurement: NewArchivalMeasurement(in.Measurement),
}
}
//
// THMeasurement
//
// ArchivalTHMeasurement is the archival representation of THMeasurement.
type ArchivalTHMeasurement struct {
DNS []*ArchivalDNSMeasurement `json:"dns"`
Endpoints []*ArchivalHTTPEndpointMeasurement `json:"endpoints"`
}
// NewArchivalTHMeasurement creates the archival representation of THMeasurement.
func NewArchivalTHMeasurement(in *THMeasurement) *ArchivalTHMeasurement {
return &ArchivalTHMeasurement{
DNS: NewArchivalDNSMeasurementList(in.DNS),
Endpoints: NewArchivalHTTPEndpointMeasurementList(in.Endpoints),
}
}
//
// DNSMeasurement
//
// ArchivalDNSMeasurement is the archival representation of DNSMeasurement.
type ArchivalDNSMeasurement struct {
Domain string `json:"domain"`
*ArchivalMeasurement
}
// NewArchivalDNSMeasurement converts a DNSMeasurement to an ArchivalDNSMeasurement.
func NewArchivalDNSMeasurement(in *DNSMeasurement) *ArchivalDNSMeasurement {
return &ArchivalDNSMeasurement{
Domain: in.Domain,
ArchivalMeasurement: NewArchivalMeasurement(in.Measurement),
}
}
// NewArchivalDNSMeasurementList converts a list of DNSMeasurement
// to a list of ArchivalDNSMeasurement.
func NewArchivalDNSMeasurementList(in []*DNSMeasurement) (out []*ArchivalDNSMeasurement) {
for _, m := range in {
out = append(out, NewArchivalDNSMeasurement(m))
}
return
}
//
// HTTPEndpointMeasurement
//
// ArchivalHTTPEndpointMeasurement is the archival representation
// of an HTTPEndpointMeasurement.
type ArchivalHTTPEndpointMeasurement struct {
URL string `json:"url"`
Network EndpointNetwork `json:"network"`
Address string `json:"address"`
*ArchivalMeasurement
}
// NewArchivalHTTPEndpointMeasurement converts an HTTPEndpointMeasurement
// to an ArchivalHTTPEndpointMeasurement.
func NewArchivalHTTPEndpointMeasurement(in *HTTPEndpointMeasurement) *ArchivalHTTPEndpointMeasurement {
return &ArchivalHTTPEndpointMeasurement{
URL: in.URL,
Network: in.Network,
Address: in.Address,
ArchivalMeasurement: NewArchivalMeasurement(in.Measurement),
}
}
// NewArchivalHTTPEndpointMeasurementList converts a list of HTTPEndpointMeasurement
// to a list of ArchivalHTTPEndpointMeasurement.
func NewArchivalHTTPEndpointMeasurementList(in []*HTTPEndpointMeasurement) (out []*ArchivalHTTPEndpointMeasurement) {
for _, m := range in {
out = append(out, NewArchivalHTTPEndpointMeasurement(m))
}
return
}
//
// Measurement
//
// ArchivalMeasurement is the archival representation of a Measurement.
type ArchivalMeasurement struct {
NetworkEvents []*ArchivalNetworkEvent `json:"network_events,omitempty"`
DNSEvents []*ArchivalDNSRoundTripEvent `json:"dns_events,omitempty"`
Queries []*ArchivalDNSLookupEvent `json:"queries,omitempty"`
TCPConnect []*ArchivalTCPConnect `json:"tcp_connect,omitempty"`
TLSHandshakes []*ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes,omitempty"`
QUICHandshakes []*ArchivalQUICTLSHandshakeEvent `json:"quic_handshakes,omitempty"`
Requests []*ArchivalHTTPRoundTripEvent `json:"requests,omitempty"`
}
// NewArchivalMeasurement converts a Measurement to ArchivalMeasurement.
func NewArchivalMeasurement(in *Measurement) *ArchivalMeasurement {
out := &ArchivalMeasurement{
NetworkEvents: NewArchivalNetworkEventList(in.ReadWrite),
DNSEvents: NewArchivalDNSRoundTripEventList(in.DNSRoundTrip),
Queries: nil, // done below
TCPConnect: NewArchivalTCPConnectList(in.Connect),
TLSHandshakes: NewArchivalQUICTLSHandshakeEventList(in.TLSHandshake),
QUICHandshakes: NewArchivalQUICTLSHandshakeEventList(in.QUICHandshake),
Requests: NewArchivalHTTPRoundTripEventList(in.HTTPRoundTrip),
}
out.Queries = append(out.Queries, NewArchivalDNSLookupEventList(in.LookupHost)...)
out.Queries = append(out.Queries, NewArchivalDNSLookupEventList(in.LookupHTTPSSvc)...)
return out
}

16
internal/measurex/db.go
View File

@ -28,7 +28,7 @@ type WritableDB interface {
InsertIntoClose(ev *NetworkEvent)
// InsertIntoTLSHandshake saves a TLS handshake event.
InsertIntoTLSHandshake(ev *TLSHandshakeEvent)
InsertIntoTLSHandshake(ev *QUICTLSHandshakeEvent)
// InsertIntoLookupHost saves a lookup host event.
InsertIntoLookupHost(ev *DNSLookupEvent)
@ -46,7 +46,7 @@ type WritableDB interface {
InsertIntoHTTPRedirect(ev *HTTPRedirectEvent)
// InsertIntoQUICHandshake saves a QUIC handshake event.
InsertIntoQUICHandshake(ev *QUICHandshakeEvent)
InsertIntoQUICHandshake(ev *QUICTLSHandshakeEvent)
}
// MeasurementDB is a WritableDB that also allows high-level code
@ -56,13 +56,13 @@ type MeasurementDB struct {
dialTable []*NetworkEvent
readWriteTable []*NetworkEvent
closeTable []*NetworkEvent
tlsHandshakeTable []*TLSHandshakeEvent
tlsHandshakeTable []*QUICTLSHandshakeEvent
lookupHostTable []*DNSLookupEvent
lookupHTTPSvcTable []*DNSLookupEvent
dnsRoundTripTable []*DNSRoundTripEvent
httpRoundTripTable []*HTTPRoundTripEvent
httpRedirectTable []*HTTPRedirectEvent
quicHandshakeTable []*QUICHandshakeEvent
quicHandshakeTable []*QUICTLSHandshakeEvent
// mu protects all the fields
mu sync.Mutex
@ -126,14 +126,14 @@ func (db *MeasurementDB) selectAllFromCloseUnlocked() (out []*NetworkEvent) {
}
// InsertIntoTLSHandshake implements EventDB.InsertIntoTLSHandshake.
func (db *MeasurementDB) InsertIntoTLSHandshake(ev *TLSHandshakeEvent) {
func (db *MeasurementDB) InsertIntoTLSHandshake(ev *QUICTLSHandshakeEvent) {
db.mu.Lock()
db.tlsHandshakeTable = append(db.tlsHandshakeTable, ev)
db.mu.Unlock()
}
// selectAllFromTLSHandshakeUnlocked returns all TLS handshake events.
func (db *MeasurementDB) selectAllFromTLSHandshakeUnlocked() (out []*TLSHandshakeEvent) {
func (db *MeasurementDB) selectAllFromTLSHandshakeUnlocked() (out []*QUICTLSHandshakeEvent) {
out = append(out, db.tlsHandshakeTable...)
return
}
@ -204,14 +204,14 @@ func (db *MeasurementDB) selectAllFromHTTPRedirectUnlocked() (out []*HTTPRedirec
}
// InsertIntoQUICHandshake implements EventDB.InsertIntoQUICHandshake.
func (db *MeasurementDB) InsertIntoQUICHandshake(ev *QUICHandshakeEvent) {
func (db *MeasurementDB) InsertIntoQUICHandshake(ev *QUICTLSHandshakeEvent) {
db.mu.Lock()
db.quicHandshakeTable = append(db.quicHandshakeTable, ev)
db.mu.Unlock()
}
// selectAllFromQUICHandshakeUnlocked returns all QUIC handshake events.
func (db *MeasurementDB) selectAllFromQUICHandshakeUnlocked() (out []*QUICHandshakeEvent) {
func (db *MeasurementDB) selectAllFromQUICHandshakeUnlocked() (out []*QUICTLSHandshakeEvent) {
out = append(out, db.quicHandshakeTable...)
return
}

27
internal/measurex/dialer.go
View File

@ -54,17 +54,14 @@ type dialerDB struct {
// NetworkEvent contains a network event. This kind of events
// are generated by Dialer, QUICDialer, Conn, QUICConn.
type NetworkEvent struct {
// JSON names compatible with df-008-netevents
RemoteAddr string `json:"address"`
Failure *string `json:"failure"`
Count int `json:"num_bytes,omitempty"`
Operation string `json:"operation"`
Network string `json:"proto"`
Finished float64 `json:"t"`
Started float64 `json:"started"`
// Names that are not part of the spec.
Oddity Oddity `json:"oddity"`
RemoteAddr string
Failure *string
Count int
Operation string
Network string
Oddity Oddity
Finished float64
Started float64
}
func (d *dialerDB) DialContext(
@ -78,7 +75,7 @@ func (d *dialerDB) DialContext(
RemoteAddr: address,
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Oddity: d.computeOddity(err),
Count: 0,
})
@ -128,7 +125,7 @@ func (c *connDB) Read(b []byte) (int, error) {
RemoteAddr: c.remoteAddr,
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Count: count,
})
return count, err
@ -144,7 +141,7 @@ func (c *connDB) Write(b []byte) (int, error) {
RemoteAddr: c.remoteAddr,
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Count: count,
})
return count, err
@ -160,7 +157,7 @@ func (c *connDB) Close() error {
RemoteAddr: c.remoteAddr,
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Count: 0,
})
return err

22
internal/measurex/dnsx.go
View File

@ -32,15 +32,13 @@ type dnsxRoundTripperDB struct {
// DNSRoundTripEvent contains the result of a DNS round trip.
type DNSRoundTripEvent struct {
// This data structure is not in df-002-dns but the names and
// semantics try to be consistent with such a spec.
Network string `json:"engine"`
Address string `json:"resolver_address"`
Query *ArchivalBinaryData `json:"raw_query"`
Started float64 `json:"started"`
Finished float64 `json:"t"`
Failure *string `json:"failure"`
Reply *ArchivalBinaryData `json:"raw_reply"`
Network string
Address string
Query []byte
Started float64
Finished float64
Failure *string
Reply []byte
}
func (txp *dnsxRoundTripperDB) RoundTrip(ctx context.Context, query []byte) ([]byte, error) {
@ -50,11 +48,11 @@ func (txp *dnsxRoundTripperDB) RoundTrip(ctx context.Context, query []byte) ([]b
txp.db.InsertIntoDNSRoundTrip(&DNSRoundTripEvent{
Network: txp.DNSTransport.Network(),
Address: txp.DNSTransport.Address(),
Query: NewArchivalBinaryData(query),
Query: query,
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Reply: NewArchivalBinaryData(reply),
Failure: NewFailure(err),
Reply: reply,
})
return reply, err
}

12
internal/measurex/failure.go Normal file
View File

@ -0,0 +1,12 @@
package measurex
// NewFailure creates a serializable failure from an error. We
// cannot round trip an error using JSON, so we serialize to this
// intermediate format that is a sort of Optional<string>.
func NewFailure(err error) *string {
if err == nil {
return nil
}
s := err.Error()
return &s
}

50
internal/measurex/http.go
View File

@ -124,31 +124,33 @@ type HTTPResponse struct {
// HTTPRoundTripEvent contains information about an HTTP round trip.
type HTTPRoundTripEvent struct {
// JSON names following the df-001-httpt data format.
Failure *string `json:"failure"`
Request *HTTPRequest `json:"request"`
Response *HTTPResponse `json:"response"`
Finished float64 `json:"t"`
Started float64 `json:"started"`
// Names not in the specification
Oddity Oddity `json:"oddity"`
Failure *string
Method string
URL string
RequestHeaders http.Header
StatusCode int64
ResponseHeaders http.Header
ResponseBody []byte
ResponseBodyLength int64
ResponseBodyIsTruncated bool
ResponseBodyIsUTF8 bool
Finished float64
Started float64
Oddity Oddity
}
func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error) {
started := time.Since(txp.Begin).Seconds()
resp, err := txp.HTTPTransport.RoundTrip(req)
rt := &HTTPRoundTripEvent{
Request: &HTTPRequest{
Method: req.Method,
URL: req.URL.String(),
Headers: NewArchivalHeaders(req.Header),
},
Started: started,
Method: req.Method,
URL: req.URL.String(),
RequestHeaders: req.Header,
Started: started,
}
if err != nil {
rt.Finished = time.Since(txp.Begin).Seconds()
rt.Failure = NewArchivalFailure(err)
rt.Failure = NewFailure(err)
txp.DB.InsertIntoHTTPRoundTrip(rt)
return nil, err
}
@ -162,10 +164,8 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error)
case resp.StatusCode >= 400:
rt.Oddity = OddityStatusOther
}
rt.Response = &HTTPResponse{
Code: int64(resp.StatusCode),
Headers: NewArchivalHeaders(resp.Header),
}
rt.StatusCode = int64(resp.StatusCode)
rt.ResponseHeaders = resp.Header
r := io.LimitReader(resp.Body, txp.MaxBodySnapshotSize)
body, err := netxlite.ReadAllContext(req.Context(), r)
if errors.Is(err, io.EOF) && resp.Close {
@ -173,7 +173,7 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error)
}
if err != nil {
rt.Finished = time.Since(txp.Begin).Seconds()
rt.Failure = NewArchivalFailure(err)
rt.Failure = NewFailure(err)
txp.DB.InsertIntoHTTPRoundTrip(rt)
return nil, err
}
@ -181,10 +181,10 @@ func (txp *HTTPTransportDB) RoundTrip(req *http.Request) (*http.Response, error)
Reader: io.MultiReader(bytes.NewReader(body), resp.Body),
Closer: resp.Body,
}
rt.Response.Body = NewArchivalBinaryData(body)
rt.Response.BodyLength = int64(len(body))
rt.Response.BodyIsTruncated = int64(len(body)) >= txp.MaxBodySnapshotSize
rt.Response.BodyIsUTF8 = utf8.Valid(body)
rt.ResponseBody = body
rt.ResponseBodyLength = int64(len(body))
rt.ResponseBodyIsTruncated = int64(len(body)) >= txp.MaxBodySnapshotSize
rt.ResponseBodyIsUTF8 = utf8.Valid(body)
rt.Finished = time.Since(txp.Begin).Seconds()
txp.DB.InsertIntoHTTPRoundTrip(rt)
return resp, nil

62
internal/measurex/measurement.go
View File

@ -18,33 +18,33 @@ import (
// a bunch of measurements detailing each measurement step.
type URLMeasurement struct {
// URL is the URL we're measuring.
URL string `json:"url"`
URL string
// DNS contains all the DNS related measurements.
DNS []*DNSMeasurement `json:"dns"`
DNS []*DNSMeasurement
// Endpoints contains a measurement for each endpoint
// that we discovered via DNS or TH.
Endpoints []*HTTPEndpointMeasurement `json:"endpoints"`
Endpoints []*HTTPEndpointMeasurement
// RedirectURLs contain the URLs to which we should fetch
// if we choose to follow redirections.
RedirectURLs []string `json:"-"`
RedirectURLs []string
// THMeasurement is the measurement collected by the TH.
TH interface{} `json:"th,omitempty"`
// TH is the measurement collected by the TH.
TH *THMeasurement
// TotalRuntime is the total time to measure this URL.
TotalRuntime time.Duration `json:"-"`
TotalRuntime time.Duration
// DNSRuntime is the time to run all DNS checks.
DNSRuntime time.Duration `json:"x_dns_runtime"`
DNSRuntime time.Duration
// THRuntime is the total time to invoke all test helpers.
THRuntime time.Duration `json:"x_th_runtime"`
THRuntime time.Duration
// EpntsRuntime is the total time to check all the endpoints.
EpntsRuntime time.Duration `json:"x_epnts_runtime"`
EpntsRuntime time.Duration
}
// fillRedirects takes in input a complete URLMeasurement and fills
@ -67,40 +67,40 @@ func (m *URLMeasurement) fillRedirects() {
// data format is not compatible with the OONI data format.
type Measurement struct {
// Connect contains all the connect operations.
Connect []*NetworkEvent `json:"connect,omitempty"`
Connect []*NetworkEvent
// ReadWrite contains all the read and write operations.
ReadWrite []*NetworkEvent `json:"read_write,omitempty"`
ReadWrite []*NetworkEvent
// Close contains all the close operations.
Close []*NetworkEvent `json:"-"`
Close []*NetworkEvent
// TLSHandshake contains all the TLS handshakes.
TLSHandshake []*TLSHandshakeEvent `json:"tls_handshake,omitempty"`
TLSHandshake []*QUICTLSHandshakeEvent
// QUICHandshake contains all the QUIC handshakes.
QUICHandshake []*QUICHandshakeEvent `json:"quic_handshake,omitempty"`
QUICHandshake []*QUICTLSHandshakeEvent
// LookupHost contains all the host lookups.
LookupHost []*DNSLookupEvent `json:"lookup_host,omitempty"`
LookupHost []*DNSLookupEvent
// LookupHTTPSSvc contains all the HTTPSSvc lookups.
LookupHTTPSSvc []*DNSLookupEvent `json:"lookup_httpssvc,omitempty"`
LookupHTTPSSvc []*DNSLookupEvent
// DNSRoundTrip contains all the DNS round trips.
DNSRoundTrip []*DNSRoundTripEvent `json:"dns_round_trip,omitempty"`
DNSRoundTrip []*DNSRoundTripEvent
// HTTPRoundTrip contains all the HTTP round trips.
HTTPRoundTrip []*HTTPRoundTripEvent `json:"http_round_trip,omitempty"`
HTTPRoundTrip []*HTTPRoundTripEvent
// HTTPRedirect contains all the redirections.
HTTPRedirect []*HTTPRedirectEvent `json:"-"`
HTTPRedirect []*HTTPRedirectEvent
}
// DNSMeasurement is a DNS measurement.
type DNSMeasurement struct {
// Domain is the domain this measurement refers to.
Domain string `json:"domain"`
Domain string
// A DNSMeasurement is a Measurement.
*Measurement
@ -239,10 +239,10 @@ func AllHTTPEndpointsForURL(URL *url.URL,
// EndpointMeasurement is an endpoint measurement.
type EndpointMeasurement struct {
// Network is the network of this endpoint.
Network EndpointNetwork `json:"network"`
Network EndpointNetwork
// Address is the address of this endpoint.
Address string `json:"address"`
Address string
// An EndpointMeasurement is a Measurement.
*Measurement
@ -251,14 +251,24 @@ type EndpointMeasurement struct {
// HTTPEndpointMeasurement is an HTTP endpoint measurement.
type HTTPEndpointMeasurement struct {
// URL is the URL this measurement refers to.
URL string `json:"url"`
URL string
// Network is the network of this endpoint.
Network EndpointNetwork `json:"network"`
Network EndpointNetwork
// Address is the address of this endpoint.
Address string `json:"address"`
Address string
// An HTTPEndpointMeasurement is a Measurement.
*Measurement
}
// THMeasurement is the measurement performed by the TH.
type THMeasurement struct {
// DNS contains all the DNS related measurements.
DNS []*DNSMeasurement
// Endpoints contains a measurement for each endpoint
// that was discovered by the probe or the TH.
Endpoints []*HTTPEndpointMeasurement
}

11
internal/measurex/measurer.go
View File

@ -13,7 +13,6 @@ import (
"context"
"crypto/tls"
"errors"
stdlog "log"
"net"
"net/http"
"net/url"
@ -712,7 +711,7 @@ type MeasureURLHelper interface {
// test helper protocol allows one to set.
LookupExtraHTTPEndpoints(ctx context.Context, URL *url.URL,
headers http.Header, epnts ...*HTTPEndpoint) (
newEpnts []*HTTPEndpoint, thMeasurement interface{}, err error)
newEpnts []*HTTPEndpoint, thMeasurement *THMeasurement, err error)
}
// MeasureURL measures an HTTP or HTTPS URL. The DNS resolvers
@ -802,12 +801,8 @@ func (mx *Measurer) maybeQUICFollowUp(ctx context.Context,
if epnt.QUICHandshake != nil {
return
}
for idx, rtrip := range epnt.HTTPRoundTrip {
if rtrip.Response == nil {
stdlog.Printf("malformed HTTPRoundTrip@%d: %+v", idx, rtrip)
continue
}
if v := rtrip.Response.Headers.Get("alt-svc"); v != "" {
for _, rtrip := range epnt.HTTPRoundTrip {
if v := rtrip.ResponseHeaders.Get("alt-svc"); v != "" {
altsvc = append(altsvc, v)
}
}

15
internal/measurex/quic.go
View File

@ -60,7 +60,7 @@ func (c *udpLikeConnDB) WriteTo(p []byte, addr net.Addr) (int, error) {
RemoteAddr: addr.String(),
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Count: count,
})
return count, err
@ -76,7 +76,7 @@ func (c *udpLikeConnDB) ReadFrom(b []byte) (int, net.Addr, error) {
RemoteAddr: addrStringIfNotNil(addr),
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Count: count,
})
return count, addr, err
@ -92,15 +92,12 @@ func (c *udpLikeConnDB) Close() error {
RemoteAddr: "",
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Count: 0,
})
return err
}
// QUICHandshakeEvent is the result of QUICHandshake.
type QUICHandshakeEvent = TLSHandshakeEvent
// NewQUICDialerWithoutResolver creates a new QUICDialer that is not
// attached to any resolver. This means that every attempt to dial any
// address containing a domain name will fail. This QUICDialer will
@ -138,7 +135,7 @@ func (qh *quicDialerDB) DialContext(ctx context.Context, network, address string
}
}
finished := time.Since(qh.begin).Seconds()
qh.db.InsertIntoQUICHandshake(&QUICHandshakeEvent{
qh.db.InsertIntoQUICHandshake(&QUICTLSHandshakeEvent{
Network: "quic",
RemoteAddr: address,
SNI: tlsConfig.ServerName,
@ -146,12 +143,12 @@ func (qh *quicDialerDB) DialContext(ctx context.Context, network, address string
SkipVerify: tlsConfig.InsecureSkipVerify,
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Oddity: qh.computeOddity(err),
TLSVersion: netxlite.TLSVersionString(state.Version),
CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite),
NegotiatedProto: state.NegotiatedProtocol,
PeerCerts: NewArchivalTLSCerts(peerCerts(nil, &state)),
PeerCerts: peerCerts(nil, &state),
})
return sess, err
}

121
internal/measurex/resolver.go
View File

@ -8,7 +8,6 @@ package measurex
import (
"context"
"net"
"strings"
"time"
@ -65,31 +64,19 @@ type resolverDB struct {
db WritableDB
}
// DNSLookupAnswer is a DNS lookup answer.
type DNSLookupAnswer struct {
// JSON names compatible with df-002-dnst's spec
Type string `json:"answer_type"`
IPv4 string `json:"ipv4,omitempty"`
IPv6 string `json:"ivp6,omitempty"`
// Names not part of the spec.
ALPN string `json:"alpn,omitempty"`
}
// DNSLookupEvent contains the results of a DNS lookup.
type DNSLookupEvent struct {
// fields inside df-002-dnst
Answers []DNSLookupAnswer `json:"answers"`
Network string `json:"engine"`
Failure *string `json:"failure"`
Domain string `json:"hostname"`
QueryType string `json:"query_type"`
Address string `json:"resolver_address"`
Finished float64 `json:"t"`
// Names not part of the spec.
Started float64 `json:"started"`
Oddity Oddity `json:"oddity"`
Network string
Failure *string
Domain string
QueryType string
Address string
Finished float64
Started float64
Oddity Oddity
A []string
AAAA []string
ALPN []string
}
// SupportsHTTP3 returns true if this query is for HTTPS and
@ -98,12 +85,9 @@ func (ev *DNSLookupEvent) SupportsHTTP3() bool {
if ev.QueryType != "HTTPS" {
return false
}
for _, ans := range ev.Answers {
switch ans.Type {
case "ALPN":
if ans.ALPN == "h3" {
return true
}
for _, alpn := range ev.ALPN {
if alpn == "h3" {
return true
}
}
return false
@ -111,18 +95,8 @@ func (ev *DNSLookupEvent) SupportsHTTP3() bool {
// Addrs returns all the IPv4/IPv6 addresses
func (ev *DNSLookupEvent) Addrs() (out []string) {
for _, ans := range ev.Answers {
switch ans.Type {
case "A":
if net.ParseIP(ans.IPv4) != nil {
out = append(out, ans.IPv4)
}
case "AAAA":
if net.ParseIP(ans.IPv6) != nil {
out = append(out, ans.IPv6)
}
}
}
out = append(out, ev.A...)
out = append(out, ev.AAAA...)
return
}
@ -130,35 +104,39 @@ func (r *resolverDB) LookupHost(ctx context.Context, domain string) ([]string, e
started := time.Since(r.begin).Seconds()
addrs, err := r.Resolver.LookupHost(ctx, domain)
finished := time.Since(r.begin).Seconds()
for _, qtype := range []string{"A", "AAAA"} {
ev := &DNSLookupEvent{
Answers: r.computeAnswers(addrs, qtype),
Network: r.Resolver.Network(),
Address: r.Resolver.Address(),
Failure: NewArchivalFailure(err),
Domain: domain,
QueryType: qtype,
Finished: finished,
Started: started,
Oddity: r.computeOddityLookupHost(addrs, err),
}
r.db.InsertIntoLookupHost(ev)
}
r.saveLookupResults(domain, started, finished, err, addrs, "A")
r.saveLookupResults(domain, started, finished, err, addrs, "AAAA")
return addrs, err
}
func (r *resolverDB) computeAnswers(addrs []string, qtype string) (out []DNSLookupAnswer) {
func (r *resolverDB) saveLookupResults(domain string, started, finished float64,
err error, addrs []string, qtype string) {
ev := &DNSLookupEvent{
Network: r.Resolver.Network(),
Address: r.Resolver.Address(),
Failure: NewFailure(err),
Domain: domain,
QueryType: qtype,
Finished: finished,
Started: started,
}
for _, addr := range addrs {
if qtype == "A" && !strings.Contains(addr, ":") {
out = append(out, DNSLookupAnswer{Type: qtype, IPv4: addr})
ev.A = append(ev.A, addr)
continue
}
if qtype == "AAAA" && strings.Contains(addr, ":") {
out = append(out, DNSLookupAnswer{Type: qtype, IPv6: addr})
ev.AAAA = append(ev.AAAA, addr)
continue
}
}
return
switch qtype {
case "A":
ev.Oddity = r.computeOddityLookupHost(ev.A, err)
case "AAAA":
ev.Oddity = r.computeOddityLookupHost(ev.AAAA, err)
}
r.db.InsertIntoLookupHost(ev)
}
func (r *resolverDB) computeOddityLookupHost(addrs []string, err error) Oddity {
@ -193,28 +171,13 @@ func (r *resolverDB) LookupHTTPS(ctx context.Context, domain string) (*HTTPSSvc,
QueryType: "HTTPS",
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Oddity: Oddity(r.computeOddityHTTPSSvc(https, err)),
}
if err == nil {
for _, addr := range https.IPv4 {
ev.Answers = append(ev.Answers, DNSLookupAnswer{
Type: "A",
IPv4: addr,
})
}
for _, addr := range https.IPv6 {
ev.Answers = append(ev.Answers, DNSLookupAnswer{
Type: "AAAA",
IPv6: addr,
})
}
for _, alpn := range https.ALPN {
ev.Answers = append(ev.Answers, DNSLookupAnswer{
Type: "ALPN",
ALPN: alpn,
})
}
ev.A = append(ev.A, https.IPv4...)
ev.AAAA = append(ev.AAAA, https.IPv6...)
ev.ALPN = append(ev.ALPN, https.ALPN...)
}
r.db.InsertIntoLookupHTTPSSvc(ev)
return https, err

40
internal/measurex/tls.go
View File

@ -38,25 +38,21 @@ type tlsHandshakerDB struct {
db WritableDB
}
// TLSHandshakeEvent contains a TLS handshake event.
type TLSHandshakeEvent struct {
// JSON names compatible with df-006-tlshandshake
CipherSuite string `json:"cipher_suite"`
Failure *string `json:"failure"`
NegotiatedProto string `json:"negotiated_proto"`
TLSVersion string `json:"tls_version"`
PeerCerts []*ArchivalBinaryData `json:"peer_certificates"`
Finished float64 `json:"t"`
// JSON names that are consistent with the
// spirit of the spec but are not in it
RemoteAddr string `json:"address"`
SNI string `json:"server_name"` // used in prod
ALPN []string `json:"alpn"`
SkipVerify bool `json:"no_tls_verify"` // used in prod
Oddity Oddity `json:"oddity"`
Network string `json:"proto"`
Started float64 `json:"started"`
// QUICTLSHandshakeEvent contains a QUIC or TLS handshake event.
type QUICTLSHandshakeEvent struct {
CipherSuite string
Failure *string
NegotiatedProto string
TLSVersion string
PeerCerts [][]byte
Finished float64
RemoteAddr string
SNI string
ALPN []string
SkipVerify bool
Oddity Oddity
Network string
Started float64
}
func (thx *tlsHandshakerDB) Handshake(ctx context.Context,
@ -66,7 +62,7 @@ func (thx *tlsHandshakerDB) Handshake(ctx context.Context,
started := time.Since(thx.begin).Seconds()
tconn, state, err := thx.TLSHandshaker.Handshake(ctx, conn, config)
finished := time.Since(thx.begin).Seconds()
thx.db.InsertIntoTLSHandshake(&TLSHandshakeEvent{
thx.db.InsertIntoTLSHandshake(&QUICTLSHandshakeEvent{
Network: network,
RemoteAddr: remoteAddr,
SNI: config.ServerName,
@ -74,12 +70,12 @@ func (thx *tlsHandshakerDB) Handshake(ctx context.Context,
SkipVerify: config.InsecureSkipVerify,
Started: started,
Finished: finished,
Failure: NewArchivalFailure(err),
Failure: NewFailure(err),
Oddity: thx.computeOddity(err),
TLSVersion: netxlite.TLSVersionString(state.Version),
CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite),
NegotiatedProto: state.NegotiatedProtocol,
PeerCerts: NewArchivalTLSCerts(peerCerts(err, &state)),
PeerCerts: peerCerts(err, &state),
})
return tconn, state, err
}

16
internal/tutorial/measurex/chapter01/README.md
View File

@ -167,8 +167,12 @@ format, in the remainder of this program we're
going to serialize the `Measurement` to JSON and
print it to the standard output.
Rather than serializing the raw `Measurement` struct,
we first convert it to the "archival" format. This is the
data format specified at [ooni/spec](https://github.com/ooni/spec/tree/master/data-formats).
```Go
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
```
@ -194,13 +198,14 @@ go run -race ./internal/tutorial/measurex/chapter01 | jq
```
Where `jq` is being used to make the output more presentable.
If you do that you obtain some logging messages, which are out of
the scope of this tutorial, and the following JSON:
```JSON
{
"domain": "example.com",
"lookup_host": [
"queries": [
{
"answers": [
{
@ -237,6 +242,9 @@ the scope of this tutorial, and the following JSON:
}
```
This JSON [implements the df-002-dnst](https://github.com/ooni/spec/blob/master/data-formats/df-002-dnst.md)
OONI data format.
You see that we have two messages here. OONI splits a DNS
resolution performed using the system resolver into two "fake"
DNS resolutions for A and AAAA. (Under the hood, this is
@ -266,7 +274,7 @@ This is the output JSON:
```JSON
{
"domain": "antani.ooni.org",
"lookup_host": [
"queries": [
{
"answers": null,
"engine": "system",
@ -327,7 +335,7 @@ To get this JSON:
```JSON
{
"domain": "example.com",
"lookup_host": [
"queries": [
{
"answers": null,
"engine": "system",

16
internal/tutorial/measurex/chapter01/main.go
View File

@ -168,8 +168,12 @@ func main() {
// going to serialize the `Measurement` to JSON and
// print it to the standard output.
//
// Rather than serializing the raw `Measurement` struct,
// we first convert it to the "archival" format. This is the
// data format specified at [ooni/spec](https://github.com/ooni/spec/tree/master/data-formats).
//
// ```Go
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
// ```
@ -195,13 +199,14 @@ func main() {
// ```
//
// Where `jq` is being used to make the output more presentable.
//
// If you do that you obtain some logging messages, which are out of
// the scope of this tutorial, and the following JSON:
//
// ```JSON
// {
// "domain": "example.com",
// "lookup_host": [
// "queries": [
// {
// "answers": [
// {
@ -238,6 +243,9 @@ func main() {
// }
// ```
//
// This JSON [implements the df-002-dnst](https://github.com/ooni/spec/blob/master/data-formats/df-002-dnst.md)
// OONI data format.
//
// You see that we have two messages here. OONI splits a DNS
// resolution performed using the system resolver into two "fake"
// DNS resolutions for A and AAAA. (Under the hood, this is
@ -267,7 +275,7 @@ func main() {
// ```JSON
// {
// "domain": "antani.ooni.org",
// "lookup_host": [
// "queries": [
// {
// "answers": null,
// "engine": "system",
@ -328,7 +336,7 @@ func main() {
// ```JSON
// {
// "domain": "example.com",
// "lookup_host": [
// "queries": [
// {
// "answers": null,
// "engine": "system",

88
internal/tutorial/measurex/chapter02/README.md
View File

@ -69,10 +69,12 @@ address is quoted with "[" and "]" if IPv6, e.g., `[::1]:53`.)
### Printing the measurement
The rest of the main function is just like in the previous chapter.
The rest of the main function is just like in the previous
chapter. Like we did before, we convert the obtained measurement
to the "archival" data format before printing.
```Go
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
}
@ -98,27 +100,31 @@ Here is the JSON we obtain in output:
// This block contains the results of the connect syscall
// using the df-008-netevents data format.
"connect": [
{
"address": "8.8.4.4:443",
"tcp_connect": [{
"ip": "8.8.4.4",
"port": 443,
"t": 0.020303,
"status": {
"blocked": false,
"failure": null,
"operation": "connect",
"proto": "tcp",
"t": 0.026879041,
"started": 8.8625e-05,
"oddity": ""
}
]
"success": true
},
"started": 0.000109292,
"oddity": ""
}]
}
```
This JSON implements [the df-005-tcpconnect](https://github.com/ooni/spec/blob/master/data-formats/df-005-tcpconnect.md)
OONI data format.
This is what it says:
- we are connecting a "tcp" socket;
- the destination endpoint address is "8.8.4.4:443";
- connect terminated ~0.027 seconds into the program's life;
- connect terminated ~0.020 seconds into the program's life (see `t`);
- the operation succeeded (`failure` is `nil`).
@ -138,19 +144,21 @@ We get this JSON:
{
"network": "tcp",
"address": "127.0.0.1:1",
"connect": [
"tcp_connect": [
{
"address": "127.0.0.1:1",
"failure": "connection_refused",
"operation": "connect",
"proto": "tcp",
"t": 0.000372167,
"started": 8.4917e-05,
"ip": "127.0.0.1",
"port": 1,
"t": 0.000457584,
"status": {
"blocked": true,
"failure": "connection_refused",
"success": false
},
"started": 0.000104792,
"oddity": "tcp.connect.refused"
}
]
}
```
And here's an error telling us the connection was refused and
@ -170,14 +178,17 @@ We get this JSON:
{
"network": "tcp",
"address": "8.8.4.4:1",
"connect": [
"tcp_connect": [
{
"address": "8.8.4.4:1",
"failure": "generic_timeout_error",
"operation": "connect",
"proto": "tcp",
"t": 10.005494583,
"started": 8.4833e-05,
"ip": "8.8.4.4",
"port": 1,
"t": 10.006558625,
"status": {
"blocked": true,
"failure": "generic_timeout_error",
"success": false
},
"started": 9.55e-05,
"oddity": "tcp.connect.timeout"
}
]
@ -201,21 +212,26 @@ To get this JSON:
{
"network": "tcp",
"address": "8.8.4.4:1",
"connect": [
"tcp_connect": [
{
"address": "8.8.4.4:1",
"failure": "generic_timeout_error",
"operation": "connect",
"proto": "tcp",
"t": 0.10148025,
"started": 0.000122375,
"ip": "8.8.4.4",
"port": 1,
"t": 0.105445125,
"status": {
"blocked": true,
"failure": "generic_timeout_error",
"success": false
},
"started": 9.4083e-05,
"oddity": "tcp.connect.timeout"
}
]
}
```
We see a timeout after ~0.1s. We enforce a reasonably small
We see a timeout after ~0.1s.
We enforce a reasonably small
timeout for connecting, equal to 10 s, because we want to
guarantee that measurements eventually terminate. Also, since
often censorship is implemented by timing out, we don't want

88
internal/tutorial/measurex/chapter02/main.go
View File

@ -70,10 +70,12 @@ func main() {
//
// ### Printing the measurement
//
// The rest of the main function is just like in the previous chapter.
// The rest of the main function is just like in the previous
// chapter. Like we did before, we convert the obtained measurement
// to the "archival" data format before printing.
//
// ```Go
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
}
@ -99,27 +101,31 @@ func main() {
//
// // This block contains the results of the connect syscall
// // using the df-008-netevents data format.
// "connect": [
// {
// "address": "8.8.4.4:443",
// "tcp_connect": [{
// "ip": "8.8.4.4",
// "port": 443,
// "t": 0.020303,
// "status": {
// "blocked": false,
// "failure": null,
// "operation": "connect",
// "proto": "tcp",
// "t": 0.026879041,
// "started": 8.8625e-05,
// "oddity": ""
// }
// ]
// "success": true
// },
// "started": 0.000109292,
// "oddity": ""
// }]
// }
// ```
//
// This JSON implements [the df-005-tcpconnect](https://github.com/ooni/spec/blob/master/data-formats/df-005-tcpconnect.md)
// OONI data format.
//
// This is what it says:
//
// - we are connecting a "tcp" socket;
//
// - the destination endpoint address is "8.8.4.4:443";
//
// - connect terminated ~0.027 seconds into the program's life;
// - connect terminated ~0.020 seconds into the program's life (see `t`);
//
// - the operation succeeded (`failure` is `nil`).
//
@ -139,19 +145,21 @@ func main() {
// {
// "network": "tcp",
// "address": "127.0.0.1:1",
// "connect": [
// "tcp_connect": [
// {
// "address": "127.0.0.1:1",
// "failure": "connection_refused",
// "operation": "connect",
// "proto": "tcp",
// "t": 0.000372167,
// "started": 8.4917e-05,
// "ip": "127.0.0.1",
// "port": 1,
// "t": 0.000457584,
// "status": {
// "blocked": true,
// "failure": "connection_refused",
// "success": false
// },
// "started": 0.000104792,
// "oddity": "tcp.connect.refused"
// }
// ]
// }
//
// ```
//
// And here's an error telling us the connection was refused and
@ -171,14 +179,17 @@ func main() {
// {
// "network": "tcp",
// "address": "8.8.4.4:1",
// "connect": [
// "tcp_connect": [
// {
// "address": "8.8.4.4:1",
// "failure": "generic_timeout_error",
// "operation": "connect",
// "proto": "tcp",
// "t": 10.005494583,
// "started": 8.4833e-05,
// "ip": "8.8.4.4",
// "port": 1,
// "t": 10.006558625,
// "status": {
// "blocked": true,
// "failure": "generic_timeout_error",
// "success": false
// },
// "started": 9.55e-05,
// "oddity": "tcp.connect.timeout"
// }
// ]
@ -202,21 +213,26 @@ func main() {
// {
// "network": "tcp",
// "address": "8.8.4.4:1",
// "connect": [
// "tcp_connect": [
// {
// "address": "8.8.4.4:1",
// "failure": "generic_timeout_error",
// "operation": "connect",
// "proto": "tcp",
// "t": 0.10148025,
// "started": 0.000122375,
// "ip": "8.8.4.4",
// "port": 1,
// "t": 0.105445125,
// "status": {
// "blocked": true,
// "failure": "generic_timeout_error",
// "success": false
// },
// "started": 9.4083e-05,
// "oddity": "tcp.connect.timeout"
// }
// ]
// }
// ```
//
// We see a timeout after ~0.1s. We enforce a reasonably small
// We see a timeout after ~0.1s.
//
// We enforce a reasonably small
// timeout for connecting, equal to 10 s, because we want to
// guarantee that measurements eventually terminate. Also, since
// often censorship is implemented by timing out, we don't want

395
internal/tutorial/measurex/chapter03/README.md
View File

@ -57,7 +57,7 @@ Also this operation returns a measurement, which
we print using the usual three-liner.
```Go
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
}
@ -83,42 +83,22 @@ be generated and inserted into a `Measurement`.)
{
"domain": "example.com",
// This block tells us about the UDP connect events
// where we bind to the server's endpoint
"connect": [
{
"address": "8.8.4.4:53",
"failure": null,
"operation": "connect",
"proto": "udp",
"t": 0.00043175,
"started": 0.000191958,
"oddity": ""
},
{
"address": "8.8.4.4:53",
"failure": null,
"operation": "connect",
"proto": "udp",
"t": 0.042198458,
"started": 0.042113208,
"oddity": ""
}
],
// This block shows the read and write events
// occurred on the sockets (because we control
// in full the implementation of this DNS
// over UDP resolver, we can see these events)
"read_write": [
//
// See https://github.com/ooni/spec/blob/master/data-formats/df-008-netevents.md
// for a description of this data format.
"network_events": [
{
"address": "8.8.4.4:53",
"failure": null,
"num_bytes": 29,
"operation": "write",
"proto": "udp",
"t": 0.000459583,
"started": 0.00043825,
"t": 0.00048825,
"started": 0.000462917,
"oddity": ""
},
{
@ -127,8 +107,8 @@ be generated and inserted into a `Measurement`.)
"num_bytes": 45,
"operation": "read",
"proto": "udp",
"t": 0.041955792,
"started": 0.000471833,
"t": 0.022081833,
"started": 0.000502625,
"oddity": ""
},
{
@ -137,8 +117,8 @@ be generated and inserted into a `Measurement`.)
"num_bytes": 29,
"operation": "write",
"proto": "udp",
"t": 0.042218917,
"started": 0.042203,
"t": 0.022433083,
"started": 0.022423875,
"oddity": ""
},
{
@ -147,19 +127,59 @@ be generated and inserted into a `Measurement`.)
"num_bytes": 57,
"operation": "read",
"proto": "udp",
"t": 0.196646583,
"started": 0.042233167,
"t": 0.046706,
"started": 0.022443833,
"oddity": ""
}
],
// This block shows the query we sent (encoded as base64)
// and the response we received. Here we clearly see
// that we perform two DNS "round trip" (i.e., send request
// and receive response) to resolve a domain: one for
// A and the other for AAAA.
//
// We don't have a specification for this data format yet.
"dns_events": [
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "dGwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000205083,
"t": 0.022141333,
"failure": null,
"raw_reply": {
"data": "dGyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAEuIAARduNgi",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "Ts8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
},
"started": 0.022221417,
"t": 0.046733125,
"failure": null,
"raw_reply": {
"data": "Ts+BgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAFOQABAmBigAAiAAAQJIGJMlyBlG",
"format": "base64"
}
}
],
// This is the same kind of result as before, we
// show the emitted queries and the resolved addrs.
//
// Also note how here the resolver_address is the
// correct endpoint address and the engine tells us
// that we're using DNS over UDP.
"lookup_host": [
"queries": [
{
"answers": [
{
@ -172,8 +192,8 @@ be generated and inserted into a `Measurement`.)
"hostname": "example.com",
"query_type": "A",
"resolver_address": "8.8.4.4:53",
"t": 0.196777042,
"started": 0.000118542,
"t": 0.046766833,
"started": 0.000124375,
"oddity": ""
},
{
@ -188,48 +208,10 @@ be generated and inserted into a `Measurement`.)
"hostname": "example.com",
"query_type": "AAAA",
"resolver_address": "8.8.4.4:53",
"t": 0.196777042,
"started": 0.000118542,
"t": 0.046766833,
"started": 0.000124375,
"oddity": ""
}
],
// This block shows the query we sent (encoded as base64)
// and the response we received. Here we clearly see
// that we perform two DNS "round trip" (i.e., send request
// and receive response) to resolve a domain: one for
// A and the other for AAAA.
"dns_round_trip": [
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "PrcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000191625,
"t": 0.041998667,
"failure": null,
"raw_reply": {
"data": "PreBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAE8BAARduNgi",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "LAwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
},
"started": 0.04210775,
"t": 0.196701333,
"failure": null,
"raw_reply": {
"data": "LAyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAE6nABAmBigAAiAAAQJIGJMlyBlG",
"format": "base64"
}
}
]
}
```
@ -254,9 +236,42 @@ This produces the following JSON:
```JavaScript
{
"domain": "antani.ooni.org",
"connect": [ /* snip */ ],
"read_write": [ /* snip */ ],
"lookup_host": [
"network_events": [ /* snip */ ],
"dns_events": [
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "p7YBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB",
"format": "base64"
},
"started": 0.000152959,
"t": 0.051650917,
"failure": null,
"raw_reply": {
"data": "p7aBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "ILkBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB",
"format": "base64"
},
"started": 0.051755209,
"t": 0.101094375,
"failure": null,
"raw_reply": {
"data": "ILmBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==",
"format": "base64"
}
}
],
"queries": [
{
"answers": null,
"engine": "udp",
@ -264,8 +279,8 @@ This produces the following JSON:
"hostname": "antani.ooni.org",
"query_type": "A",
"resolver_address": "8.8.4.4:53",
"t": 0.098208709,
"started": 8.95e-05,
"t": 0.101241667,
"started": 8.8e-05,
"oddity": "dns.lookup.nxdomain"
},
{
@ -275,42 +290,10 @@ This produces the following JSON:
"hostname": "antani.ooni.org",
"query_type": "AAAA",
"resolver_address": "8.8.4.4:53",
"t": 0.098208709,
"started": 8.95e-05,
"t": 0.101241667,
"started": 8.8e-05,
"oddity": "dns.lookup.nxdomain"
}
],
"dns_round_trip": [
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "jLIBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB",
"format": "base64"
},
"started": 0.000141542,
"t": 0.034689417,
"failure": null,
"raw_reply": {
"data": "jLKBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "8.8.4.4:53",
"raw_query": {
"data": "azEBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB",
"format": "base64"
},
"started": 0.034776709,
"t": 0.098170542,
"failure": null,
"raw_reply": {
"data": "azGBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==",
"format": "base64"
}
}
]
}
```
@ -323,10 +306,10 @@ Let us now decode one of the replies by using this program:
package main
import (
"fmt"
"encoding/base64"
"fmt"
"encoding/base64"
"github.com/miekg/dns"
"github.com/miekg/dns"
)
func main() {
@ -366,16 +349,15 @@ Here's the corresponding JSON:
```JavaScript
{
"domain": "example.com",
"connect": [ /* snip */ ],
"read_write": [
"network_events": [
{
"address": "182.92.22.222:53",
"failure": null,
"num_bytes": 29,
"operation": "write",
"proto": "udp",
"t": 0.0005275,
"started": 0.000500209,
"t": 0.000479583,
"started": 0.00045525,
"oddity": ""
},
{
@ -383,56 +365,58 @@ Here's the corresponding JSON:
"failure": "generic_timeout_error", /* <--- */
"operation": "read",
"proto": "udp",
"t": 5.001140125,
"started": 0.000544042,
"t": 5.006016292,
"started": 0.000491792,
"oddity": ""
}
],
"lookup_host": [
{
"answers": null,
"engine": "udp",
"failure": "generic_timeout_error", /* <--- */
"hostname": "example.com",
"query_type": "A",
"resolver_address": "182.92.22.222:53",
"t": 5.001462084,
"started": 0.000127917,
"oddity": "dns.lookup.timeout" /* <--- */
},
{
"answers": null,
"engine": "udp",
"failure": "generic_timeout_error",
"hostname": "example.com",
"query_type": "AAAA",
"resolver_address": "182.92.22.222:53",
"t": 5.001462084,
"started": 0.000127917,
"oddity": "dns.lookup.timeout"
}
],
"dns_round_trip": [
"dns_events": [
{
"engine": "udp",
"resolver_address": "182.92.22.222:53",
"raw_query": {
"data": "ej8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"data": "GRUBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000220584,
"t": 5.001317417,
"failure": "generic_timeout_error",
"started": 0.00018225,
"t": 5.006185667,
"failure": "generic_timeout_error", /* <--- */
"raw_reply": null
}
/* snip */
],
"queries": [
{
"answers": null,
"engine": "udp",
"failure": "generic_timeout_error", /* <--- */
"hostname": "example.com",
"query_type": "A",
"resolver_address": "182.92.22.222:53",
"t": 5.007385458,
"started": 0.000107583,
"oddity": "dns.lookup.timeout"
},
{
"answers": null,
"engine": "udp",
"failure": "generic_timeout_error", /* <--- */
"hostname": "example.com",
"query_type": "AAAA",
"resolver_address": "182.92.22.222:53",
"t": 5.007385458,
"started": 0.000107583,
"oddity": "dns.lookup.timeout"
}
]
}
```
We see that we fail with a timeout (I have marked some of them
with comments inside the JSON). We see the timeout at three different
levels of abstractions (from lower to higher abstraction): at the socket layer,
during the DNS round trip, during the DNS lookup.
levels of abstractions (from lower to higher abstraction): at the socket
layer (`network_events`), during the DNS round trip (`dns_events`),
during the DNS lookup (`queries`).
What we also see is that `t`'s value is ~5s when the `read` event
fails, which tells us about the socket's read timeout.
@ -451,18 +435,17 @@ Here's the answer I get:
```JavaScript
{
"domain": "example.com",
"connect": [ /* snip */ ],
// The I/O events look normal this time
"read_write": [
// The network events look normal this time
"network_events": [
{
"address": "180.97.36.63:53",
"failure": null,
"num_bytes": 29,
"operation": "write",
"proto": "udp",
"t": 0.000333583,
"started": 0.000312125,
"t": 0.000492125,
"started": 0.000467042,
"oddity": ""
},
{
@ -471,8 +454,8 @@ Here's the answer I get:
"num_bytes": 29,
"operation": "read",
"proto": "udp",
"t": 0.334948125,
"started": 0.000366625,
"t": 0.321373542,
"started": 0.000504833,
"oddity": ""
},
{
@ -481,8 +464,8 @@ Here's the answer I get:
"num_bytes": 29,
"operation": "write",
"proto": "udp",
"t": 0.3358025,
"started": 0.335725958,
"t": 0.322500875,
"started": 0.322450042,
"oddity": ""
},
{
@ -491,15 +474,49 @@ Here's the answer I get:
"num_bytes": 29,
"operation": "read",
"proto": "udp",
"t": 0.739987666,
"started": 0.335863875,
"t": 0.655514542,
"started": 0.322557667,
"oddity": ""
}
],
// But we see both in the error and in the oddity
// Exercise: do like I did before and decode the messages
"dns_events": [
{
"engine": "udp",
"resolver_address": "180.97.36.63:53",
"raw_query": {
"data": "WcgBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000209875,
"t": 0.321504042,
"failure": null,
"raw_reply": {
"data": "WciBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "180.97.36.63:53",
"raw_query": {
"data": "I9oBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
},
"started": 0.321672042,
"t": 0.655680792,
"failure": null,
"raw_reply": {
"data": "I9qBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
}
}
],
// We see both in the error and in the oddity
// that the response was "REFUSED"
"lookup_host": [
"queries": [
{
"answers": null,
"engine": "udp",
@ -507,8 +524,8 @@ Here's the answer I get:
"hostname": "example.com",
"query_type": "A",
"resolver_address": "180.97.36.63:53",
"t": 0.7402975,
"started": 7.2291e-05,
"t": 0.655814875,
"started": 0.000107417,
"oddity": "dns.lookup.refused"
},
{
@ -518,44 +535,10 @@ Here's the answer I get:
"hostname": "example.com",
"query_type": "AAAA",
"resolver_address": "180.97.36.63:53",
"t": 0.7402975,
"started": 7.2291e-05,
"t": 0.655814875,
"started": 0.000107417,
"oddity": "dns.lookup.refused"
}
],
// Exercise: do like I did before and decode the messages
"dns_round_trip": [
{
"engine": "udp",
"resolver_address": "180.97.36.63:53",
"raw_query": {
"data": "crkBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
},
"started": 0.000130666,
"t": 0.33509925,
"failure": null,
"raw_reply": {
"data": "crmBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
"format": "base64"
}
},
{
"engine": "udp",
"resolver_address": "180.97.36.63:53",
"raw_query": {
"data": "ywcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
},
"started": 0.335321333,
"t": 0.740152375,
"failure": null,
"raw_reply": {
"data": "yweBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
"format": "base64"
}
}
]
}
```

395
internal/tutorial/measurex/chapter03/main.go
View File

@ -58,7 +58,7 @@ func main() {
// we print using the usual three-liner.
//
// ```Go
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalDNSMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
}
@ -84,42 +84,22 @@ func main() {
// {
// "domain": "example.com",
//
// // This block tells us about the UDP connect events
// // where we bind to the server's endpoint
// "connect": [
// {
// "address": "8.8.4.4:53",
// "failure": null,
// "operation": "connect",
// "proto": "udp",
// "t": 0.00043175,
// "started": 0.000191958,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:53",
// "failure": null,
// "operation": "connect",
// "proto": "udp",
// "t": 0.042198458,
// "started": 0.042113208,
// "oddity": ""
// }
// ],
//
// // This block shows the read and write events
// // occurred on the sockets (because we control
// // in full the implementation of this DNS
// // over UDP resolver, we can see these events)
// "read_write": [
// //
// // See https://github.com/ooni/spec/blob/master/data-formats/df-008-netevents.md
// // for a description of this data format.
// "network_events": [
// {
// "address": "8.8.4.4:53",
// "failure": null,
// "num_bytes": 29,
// "operation": "write",
// "proto": "udp",
// "t": 0.000459583,
// "started": 0.00043825,
// "t": 0.00048825,
// "started": 0.000462917,
// "oddity": ""
// },
// {
@ -128,8 +108,8 @@ func main() {
// "num_bytes": 45,
// "operation": "read",
// "proto": "udp",
// "t": 0.041955792,
// "started": 0.000471833,
// "t": 0.022081833,
// "started": 0.000502625,
// "oddity": ""
// },
// {
@ -138,8 +118,8 @@ func main() {
// "num_bytes": 29,
// "operation": "write",
// "proto": "udp",
// "t": 0.042218917,
// "started": 0.042203,
// "t": 0.022433083,
// "started": 0.022423875,
// "oddity": ""
// },
// {
@ -148,19 +128,59 @@ func main() {
// "num_bytes": 57,
// "operation": "read",
// "proto": "udp",
// "t": 0.196646583,
// "started": 0.042233167,
// "t": 0.046706,
// "started": 0.022443833,
// "oddity": ""
// }
// ],
//
// // This block shows the query we sent (encoded as base64)
// // and the response we received. Here we clearly see
// // that we perform two DNS "round trip" (i.e., send request
// // and receive response) to resolve a domain: one for
// // A and the other for AAAA.
// //
// // We don't have a specification for this data format yet.
// "dns_events": [
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "dGwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000205083,
// "t": 0.022141333,
// "failure": null,
// "raw_reply": {
// "data": "dGyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAEuIAARduNgi",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "Ts8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// },
// "started": 0.022221417,
// "t": 0.046733125,
// "failure": null,
// "raw_reply": {
// "data": "Ts+BgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAFOQABAmBigAAiAAAQJIGJMlyBlG",
// "format": "base64"
// }
// }
// ],
//
// // This is the same kind of result as before, we
// // show the emitted queries and the resolved addrs.
// //
// // Also note how here the resolver_address is the
// // correct endpoint address and the engine tells us
// // that we're using DNS over UDP.
// "lookup_host": [
// "queries": [
// {
// "answers": [
// {
@ -173,8 +193,8 @@ func main() {
// "hostname": "example.com",
// "query_type": "A",
// "resolver_address": "8.8.4.4:53",
// "t": 0.196777042,
// "started": 0.000118542,
// "t": 0.046766833,
// "started": 0.000124375,
// "oddity": ""
// },
// {
@ -189,48 +209,10 @@ func main() {
// "hostname": "example.com",
// "query_type": "AAAA",
// "resolver_address": "8.8.4.4:53",
// "t": 0.196777042,
// "started": 0.000118542,
// "t": 0.046766833,
// "started": 0.000124375,
// "oddity": ""
// }
// ],
//
// // This block shows the query we sent (encoded as base64)
// // and the response we received. Here we clearly see
// // that we perform two DNS "round trip" (i.e., send request
// // and receive response) to resolve a domain: one for
// // A and the other for AAAA.
// "dns_round_trip": [
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "PrcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000191625,
// "t": 0.041998667,
// "failure": null,
// "raw_reply": {
// "data": "PreBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAE8BAARduNgi",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "LAwBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// },
// "started": 0.04210775,
// "t": 0.196701333,
// "failure": null,
// "raw_reply": {
// "data": "LAyBgAABAAEAAAAAB2V4YW1wbGUDY29tAAAcAAHADAAcAAEAAE6nABAmBigAAiAAAQJIGJMlyBlG",
// "format": "base64"
// }
// }
// ]
// }
// ```
@ -255,9 +237,42 @@ func main() {
// ```JavaScript
// {
// "domain": "antani.ooni.org",
// "connect": [ /* snip */ ],
// "read_write": [ /* snip */ ],
// "lookup_host": [
// "network_events": [ /* snip */ ],
//
// "dns_events": [
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "p7YBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB",
// "format": "base64"
// },
// "started": 0.000152959,
// "t": 0.051650917,
// "failure": null,
// "raw_reply": {
// "data": "p7aBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "ILkBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB",
// "format": "base64"
// },
// "started": 0.051755209,
// "t": 0.101094375,
// "failure": null,
// "raw_reply": {
// "data": "ILmBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhbwqOAACowAAADhAACTqAAAAOEQ==",
// "format": "base64"
// }
// }
// ],
//
// "queries": [
// {
// "answers": null,
// "engine": "udp",
@ -265,8 +280,8 @@ func main() {
// "hostname": "antani.ooni.org",
// "query_type": "A",
// "resolver_address": "8.8.4.4:53",
// "t": 0.098208709,
// "started": 8.95e-05,
// "t": 0.101241667,
// "started": 8.8e-05,
// "oddity": "dns.lookup.nxdomain"
// },
// {
@ -276,42 +291,10 @@ func main() {
// "hostname": "antani.ooni.org",
// "query_type": "AAAA",
// "resolver_address": "8.8.4.4:53",
// "t": 0.098208709,
// "started": 8.95e-05,
// "t": 0.101241667,
// "started": 8.8e-05,
// "oddity": "dns.lookup.nxdomain"
// }
// ],
// "dns_round_trip": [
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "jLIBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAAQAB",
// "format": "base64"
// },
// "started": 0.000141542,
// "t": 0.034689417,
// "failure": null,
// "raw_reply": {
// "data": "jLKBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAAQABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "8.8.4.4:53",
// "raw_query": {
// "data": "azEBAAABAAAAAAAABmFudGFuaQRvb25pA29yZwAAHAAB",
// "format": "base64"
// },
// "started": 0.034776709,
// "t": 0.098170542,
// "failure": null,
// "raw_reply": {
// "data": "azGBgwABAAAAAQAABmFudGFuaQRvb25pA29yZwAAHAABwBMABgABAAAHCAA9BGRuczERcmVnaXN0cmFyLXNlcnZlcnMDY29tAApob3N0bWFzdGVywDJhABz8AACowAAADhAACTqAAAAOEQ==",
// "format": "base64"
// }
// }
// ]
// }
// ```
@ -324,10 +307,10 @@ func main() {
// package main
//
// import (
// "fmt"
// "encoding/base64"
// "fmt"
// "encoding/base64"
//
// "github.com/miekg/dns"
// "github.com/miekg/dns"
// )
//
// func main() {
@ -367,16 +350,15 @@ func main() {
// ```JavaScript
// {
// "domain": "example.com",
// "connect": [ /* snip */ ],
// "read_write": [
// "network_events": [
// {
// "address": "182.92.22.222:53",
// "failure": null,
// "num_bytes": 29,
// "operation": "write",
// "proto": "udp",
// "t": 0.0005275,
// "started": 0.000500209,
// "t": 0.000479583,
// "started": 0.00045525,
// "oddity": ""
// },
// {
@ -384,56 +366,58 @@ func main() {
// "failure": "generic_timeout_error", /* <--- */
// "operation": "read",
// "proto": "udp",
// "t": 5.001140125,
// "started": 0.000544042,
// "t": 5.006016292,
// "started": 0.000491792,
// "oddity": ""
// }
// ],
// "lookup_host": [
// {
// "answers": null,
// "engine": "udp",
// "failure": "generic_timeout_error", /* <--- */
// "hostname": "example.com",
// "query_type": "A",
// "resolver_address": "182.92.22.222:53",
// "t": 5.001462084,
// "started": 0.000127917,
// "oddity": "dns.lookup.timeout" /* <--- */
// },
// {
// "answers": null,
// "engine": "udp",
// "failure": "generic_timeout_error",
// "hostname": "example.com",
// "query_type": "AAAA",
// "resolver_address": "182.92.22.222:53",
// "t": 5.001462084,
// "started": 0.000127917,
// "oddity": "dns.lookup.timeout"
// }
// ],
// "dns_round_trip": [
// "dns_events": [
// {
// "engine": "udp",
// "resolver_address": "182.92.22.222:53",
// "raw_query": {
// "data": "ej8BAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "data": "GRUBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000220584,
// "t": 5.001317417,
// "failure": "generic_timeout_error",
// "started": 0.00018225,
// "t": 5.006185667,
// "failure": "generic_timeout_error", /* <--- */
// "raw_reply": null
// }
// /* snip */
// ],
// "queries": [
// {
// "answers": null,
// "engine": "udp",
// "failure": "generic_timeout_error", /* <--- */
// "hostname": "example.com",
// "query_type": "A",
// "resolver_address": "182.92.22.222:53",
// "t": 5.007385458,
// "started": 0.000107583,
// "oddity": "dns.lookup.timeout"
// },
// {
// "answers": null,
// "engine": "udp",
// "failure": "generic_timeout_error", /* <--- */
// "hostname": "example.com",
// "query_type": "AAAA",
// "resolver_address": "182.92.22.222:53",
// "t": 5.007385458,
// "started": 0.000107583,
// "oddity": "dns.lookup.timeout"
// }
// ]
// }
// ```
//
// We see that we fail with a timeout (I have marked some of them
// with comments inside the JSON). We see the timeout at three different
// levels of abstractions (from lower to higher abstraction): at the socket layer,
// during the DNS round trip, during the DNS lookup.
// levels of abstractions (from lower to higher abstraction): at the socket
// layer (`network_events`), during the DNS round trip (`dns_events`),
// during the DNS lookup (`queries`).
//
// What we also see is that `t`'s value is ~5s when the `read` event
// fails, which tells us about the socket's read timeout.
@ -452,18 +436,17 @@ func main() {
// ```JavaScript
// {
// "domain": "example.com",
// "connect": [ /* snip */ ],
//
// // The I/O events look normal this time
// "read_write": [
// // The network events look normal this time
// "network_events": [
// {
// "address": "180.97.36.63:53",
// "failure": null,
// "num_bytes": 29,
// "operation": "write",
// "proto": "udp",
// "t": 0.000333583,
// "started": 0.000312125,
// "t": 0.000492125,
// "started": 0.000467042,
// "oddity": ""
// },
// {
@ -472,8 +455,8 @@ func main() {
// "num_bytes": 29,
// "operation": "read",
// "proto": "udp",
// "t": 0.334948125,
// "started": 0.000366625,
// "t": 0.321373542,
// "started": 0.000504833,
// "oddity": ""
// },
// {
@ -482,8 +465,8 @@ func main() {
// "num_bytes": 29,
// "operation": "write",
// "proto": "udp",
// "t": 0.3358025,
// "started": 0.335725958,
// "t": 0.322500875,
// "started": 0.322450042,
// "oddity": ""
// },
// {
@ -492,15 +475,49 @@ func main() {
// "num_bytes": 29,
// "operation": "read",
// "proto": "udp",
// "t": 0.739987666,
// "started": 0.335863875,
// "t": 0.655514542,
// "started": 0.322557667,
// "oddity": ""
// }
// ],
//
// // But we see both in the error and in the oddity
// // Exercise: do like I did before and decode the messages
// "dns_events": [
// {
// "engine": "udp",
// "resolver_address": "180.97.36.63:53",
// "raw_query": {
// "data": "WcgBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000209875,
// "t": 0.321504042,
// "failure": null,
// "raw_reply": {
// "data": "WciBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "180.97.36.63:53",
// "raw_query": {
// "data": "I9oBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// },
// "started": 0.321672042,
// "t": 0.655680792,
// "failure": null,
// "raw_reply": {
// "data": "I9qBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// }
// }
// ],
//
// // We see both in the error and in the oddity
// // that the response was "REFUSED"
// "lookup_host": [
// "queries": [
// {
// "answers": null,
// "engine": "udp",
@ -508,8 +525,8 @@ func main() {
// "hostname": "example.com",
// "query_type": "A",
// "resolver_address": "180.97.36.63:53",
// "t": 0.7402975,
// "started": 7.2291e-05,
// "t": 0.655814875,
// "started": 0.000107417,
// "oddity": "dns.lookup.refused"
// },
// {
@ -519,44 +536,10 @@ func main() {
// "hostname": "example.com",
// "query_type": "AAAA",
// "resolver_address": "180.97.36.63:53",
// "t": 0.7402975,
// "started": 7.2291e-05,
// "t": 0.655814875,
// "started": 0.000107417,
// "oddity": "dns.lookup.refused"
// }
// ],
//
// // Exercise: do like I did before and decode the messages
// "dns_round_trip": [
// {
// "engine": "udp",
// "resolver_address": "180.97.36.63:53",
// "raw_query": {
// "data": "crkBAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// },
// "started": 0.000130666,
// "t": 0.33509925,
// "failure": null,
// "raw_reply": {
// "data": "crmBBQABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE=",
// "format": "base64"
// }
// },
// {
// "engine": "udp",
// "resolver_address": "180.97.36.63:53",
// "raw_query": {
// "data": "ywcBAAABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// },
// "started": 0.335321333,
// "t": 0.740152375,
// "failure": null,
// "raw_reply": {
// "data": "yweBBQABAAAAAAAAB2V4YW1wbGUDY29tAAAcAAE=",
// "format": "base64"
// }
// }
// ]
// }
// ```

69
internal/tutorial/measurex/chapter04/README.md
View File

@ -69,7 +69,7 @@ As usual, the method to perform a measurement returns
the measurement itself, which we print below.
```
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
}
@ -90,30 +90,16 @@ Let us comment the JSON in detail:
"network": "tcp",
"address": "8.8.4.4:443",
// This block is generated when connecting to a TCP
// socket, as we've already seen in chapter02
"connect": [
{
"address": "8.8.4.4:443",
"failure": null,
"operation": "connect",
"proto": "tcp",
"t": 0.046959084,
"started": 0.022998875,
"oddity": ""
}
],
// These are the I/O events during the handshake
"read_write": [
"network_events": [
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 280,
"operation": "write",
"proto": "tcp",
"t": 0.048752875,
"started": 0.04874125,
"t": 0.048268333,
"started": 0.048246666,
"oddity": ""
},
{
@ -122,18 +108,18 @@ Let us comment the JSON in detail:
"num_bytes": 517,
"operation": "read",
"proto": "tcp",
"t": 0.087221334,
"started": 0.048760417,
"t": 0.086214708,
"started": 0.048287791,
"oddity": ""
},
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 4301,
"num_bytes": 4303,
"operation": "read",
"proto": "tcp",
"t": 0.088843584,
"started": 0.088830959,
"t": 0.087951708,
"started": 0.08792725,
"oddity": ""
},
{
@ -142,14 +128,33 @@ Let us comment the JSON in detail:
"num_bytes": 64,
"operation": "write",
"proto": "tcp",
"t": 0.092078042,
"started": 0.092064042,
"t": 0.090097833,
"started": 0.090083875,
"oddity": ""
}
],
// This block contains information about the handshake
"tls_handshake": [
// This block is generated when connecting to a TCP
// socket, as we've already seen in chapter02
"tcp_connect": [
{
"ip": "8.8.4.4",
"port": 443,
"t": 0.046022583,
"status": {
"blocked": false,
"failure": null,
"success": true
},
"started": 0.024424916,
"oddity": ""
}
],
// This block contains information about the handshake.
//
// See https://github.com/ooni/spec/blob/master/data-formats/df-006-tlshandshake.md
"tls_handshakes": [
{
"cipher_suite": "TLS_AES_128_GCM_SHA256",
"failure": null,
@ -157,7 +162,7 @@ Let us comment the JSON in detail:
"tls_version": "TLSv1.3",
"peer_certificates": [
{
"data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc",
"data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=",
"format": "base64"
},
{
@ -169,7 +174,7 @@ Let us comment the JSON in detail:
"format": "base64"
}
],
"t": 0.092117709,
"t": 0.090150666,
"address": "8.8.4.4:443",
"server_name": "dns.google",
"alpn": [
@ -179,16 +184,12 @@ Let us comment the JSON in detail:
"no_tls_verify": false,
"oddity": "",
"proto": "tcp",
"started": 0.047288542
"started": 0.04635975
}
]
}
```
All the data formats we're using here are, by the way,
compatible with the data formats specified at
https://github.com/ooni/spec/tree/master/data-formats.
### Suggested follow-up experiments
Try to run experiments in the following scenarios, and

69
internal/tutorial/measurex/chapter04/main.go
View File

@ -70,7 +70,7 @@ func main() {
// the measurement itself, which we print below.
//
// ```
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
}
@ -91,30 +91,16 @@ func main() {
// "network": "tcp",
// "address": "8.8.4.4:443",
//
// // This block is generated when connecting to a TCP
// // socket, as we've already seen in chapter02
// "connect": [
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "operation": "connect",
// "proto": "tcp",
// "t": 0.046959084,
// "started": 0.022998875,
// "oddity": ""
// }
// ],
//
// // These are the I/O events during the handshake
// "read_write": [
// "network_events": [
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 280,
// "operation": "write",
// "proto": "tcp",
// "t": 0.048752875,
// "started": 0.04874125,
// "t": 0.048268333,
// "started": 0.048246666,
// "oddity": ""
// },
// {
@ -123,18 +109,18 @@ func main() {
// "num_bytes": 517,
// "operation": "read",
// "proto": "tcp",
// "t": 0.087221334,
// "started": 0.048760417,
// "t": 0.086214708,
// "started": 0.048287791,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 4301,
// "num_bytes": 4303,
// "operation": "read",
// "proto": "tcp",
// "t": 0.088843584,
// "started": 0.088830959,
// "t": 0.087951708,
// "started": 0.08792725,
// "oddity": ""
// },
// {
@ -143,14 +129,33 @@ func main() {
// "num_bytes": 64,
// "operation": "write",
// "proto": "tcp",
// "t": 0.092078042,
// "started": 0.092064042,
// "t": 0.090097833,
// "started": 0.090083875,
// "oddity": ""
// }
// ],
//
// // This block contains information about the handshake
// "tls_handshake": [
// // This block is generated when connecting to a TCP
// // socket, as we've already seen in chapter02
// "tcp_connect": [
// {
// "ip": "8.8.4.4",
// "port": 443,
// "t": 0.046022583,
// "status": {
// "blocked": false,
// "failure": null,
// "success": true
// },
// "started": 0.024424916,
// "oddity": ""
// }
// ],
//
// // This block contains information about the handshake.
// //
// // See https://github.com/ooni/spec/blob/master/data-formats/df-006-tlshandshake.md
// "tls_handshakes": [
// {
// "cipher_suite": "TLS_AES_128_GCM_SHA256",
// "failure": null,
@ -158,7 +163,7 @@ func main() {
// "tls_version": "TLSv1.3",
// "peer_certificates": [
// {
// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc",
// "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=",
// "format": "base64"
// },
// {
@ -170,7 +175,7 @@ func main() {
// "format": "base64"
// }
// ],
// "t": 0.092117709,
// "t": 0.090150666,
// "address": "8.8.4.4:443",
// "server_name": "dns.google",
// "alpn": [
@ -180,16 +185,12 @@ func main() {
// "no_tls_verify": false,
// "oddity": "",
// "proto": "tcp",
// "started": 0.047288542
// "started": 0.04635975
// }
// ]
// }
// ```
//
// All the data formats we're using here are, by the way,
// compatible with the data formats specified at
// https://github.com/ooni/spec/tree/master/data-formats.
//
// ### Suggested follow-up experiments
//
// Try to run experiments in the following scenarios, and

64
internal/tutorial/measurex/chapter05/README.md
View File

@ -69,7 +69,7 @@ As we did in the previous chapters, here's the usual three
lines of code for printing the resulting measurement.
```
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
}
@ -100,15 +100,15 @@ Produces this JSON:
// are actually `recvfrom` and `sendto` but here
// we follow the Go convention of using read/write
// more frequently than send/recv.)
"read_write": [
"network_events": [
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 1252,
"operation": "write_to",
"proto": "quic",
"t": 0.003903167,
"started": 0.0037395,
"t": 0.027184208,
"started": 0.027127208,
"oddity": ""
},
{
@ -117,8 +117,8 @@ Produces this JSON:
"num_bytes": 1252,
"operation": "read_from",
"proto": "quic",
"t": 0.029389125,
"started": 0.002954792,
"t": 0.053116458,
"started": 0.025626583,
"oddity": ""
},
{
@ -127,8 +127,8 @@ Produces this JSON:
"num_bytes": 1252,
"operation": "write_to",
"proto": "quic",
"t": 0.029757584,
"started": 0.02972325,
"t": 0.054538792,
"started": 0.054517542,
"oddity": ""
},
{
@ -137,8 +137,8 @@ Produces this JSON:
"num_bytes": 1252,
"operation": "read_from",
"proto": "quic",
"t": 0.045039875,
"started": 0.029424792,
"t": 0.069144958,
"started": 0.053194208,
"oddity": ""
},
{
@ -147,8 +147,8 @@ Produces this JSON:
"num_bytes": 1252,
"operation": "read_from",
"proto": "quic",
"t": 0.045055334,
"started": 0.045049625,
"t": 0.069183458,
"started": 0.069173292,
"oddity": ""
},
{
@ -157,28 +157,28 @@ Produces this JSON:
"num_bytes": 1252,
"operation": "read_from",
"proto": "quic",
"t": 0.045073917,
"started": 0.045069667,
"t": 0.06920225,
"started": 0.069197875,
"oddity": ""
},
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 1233,
"num_bytes": 1216,
"operation": "read_from",
"proto": "quic",
"t": 0.04508,
"started": 0.045075292,
"t": 0.069210958,
"started": 0.069206875,
"oddity": ""
},
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 64,
"num_bytes": 65,
"operation": "read_from",
"proto": "quic",
"t": 0.045088167,
"started": 0.045081167,
"t": 0.069220667,
"started": 0.069217375,
"oddity": ""
},
{
@ -187,8 +187,8 @@ Produces this JSON:
"num_bytes": 44,
"operation": "write_to",
"proto": "quic",
"t": 0.045370417,
"started": 0.045338667,
"t": 0.069433417,
"started": 0.069417625,
"oddity": ""
},
{
@ -197,8 +197,8 @@ Produces this JSON:
"num_bytes": 44,
"operation": "write_to",
"proto": "quic",
"t": 0.045392125,
"started": 0.045380959,
"t": 0.069677625,
"started": 0.069647458,
"oddity": ""
},
{
@ -207,8 +207,8 @@ Produces this JSON:
"num_bytes": 83,
"operation": "write_to",
"proto": "quic",
"t": 0.047042542,
"started": 0.047001917,
"t": 0.073461917,
"started": 0.073432875,
"oddity": ""
},
{
@ -217,15 +217,15 @@ Produces this JSON:
"num_bytes": 33,
"operation": "write_to",
"proto": "quic",
"t": 0.047060834,
"started": 0.047046875,
"t": 0.073559417,
"started": 0.073542542,
"oddity": ""
}
],
// This section describes the QUIC handshake and it has
// basically the same fields as the TLS handshake.
"quic_handshake": [
"quic_handshakes": [
{
"cipher_suite": "TLS_CHACHA20_POLY1305_SHA256",
"failure": null,
@ -233,7 +233,7 @@ Produces this JSON:
"tls_version": "TLSv1.3",
"peer_certificates": [
{
"data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc",
"data": "MIIF4DCCBMigAwIBAgIQWiqDOVk83wAKAAAAAQivAjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTEwMDQwMjQzMzVaFw0yMTEyMjcwMjQzMzRaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUBr7csGnwBJnnh4nRPQFux/52IXuOIIUao2X4JQH7chvtuNyR4rxpeyL/zAYbFpohSdSfxrCFxrFWae+LzANL/VTPZPzPH2S2p1TNWy8rFWMU0OnRd3Ym5s1xOLBG9Lu28iH3h5+5yk5pGmvSnzA0Ra9Q5pjLJlUfnGi9ceP2uFM/SDRo1IEqNB7ukIeE5lmw7CJCyDafbWy8eFFPG69YRAjPb13NF4ijingt2JsUckgXJdWdoRf3KghM5ddvQhv5rdELjVUbI550MjfBQWoEUIxWEUsYNj/L4IbK46E4RpMnU+XnwTtinQYYgHVQxLKPtPdfYHp145DjiGXE9LMDAgMBAAGjggL5MIIC9TAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUo61UxicmPa371PCog8zI7NbrdkQwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQIGCisGAQQB1nkCBAIEgfMEgfAA7gB1AESUZS6w7s6vxEAH2Kj+KMDa5oK+2MsxtT/TM5a1toGoAAABfElmWjQAAAQDAEYwRAIgZpBA5ih3hRoZ1749kEcxdEcpzHUV3cS2zDHuz1WMy7gCIGtMqROCc/wrP01x1GXrk3M/qbHssnvhelxhNVbi4FTUAHUA9lyUL9F3MCIUVBgIMJRWjuNNExkzv98MLyALzE7xZOMAAAF8SWZaAwAABAMARjBEAiBRWeLjcLIQwBFdPEXa15s++b7kAKit86em9GR23F+7KQIgTHZgL7inapApbw5WJNhtEI78a5HHPsI+kU5LIDgpv7swDQYJKoZIhvcNAQELBQADggEBAGki3+h4nn12Ef449NirIIP5APx+1NQk3DDKT6PrpH3m+s/wQKXlJ8eNg6zJEBEtHxBdO0xI+/te2Bh1s6RU/iJZrVRAtol/xmn0AMvPQNL+JZUnYuryz2mwTpk5ZHnGHyZknbJDspB2oZkItDeDbvkMws+JKrCYbbHG4ZtcsoFYPrkjfyMLRemhj+qWvvMtUKUldsXbYhugQL44N+pWAJNyxEWUBcuEldItww/gSrl/O2alfGOTNvdcXT/nedrw+SJnci4m4oMTz+XWFkbf3yPXEikvUqcvQFe10F1wBr8wW6soM9nR7vKq7WUlnx1m4lghw/jStp2mWenT6YFp5Tw=",
"format": "base64"
},
{
@ -245,7 +245,7 @@ Produces this JSON:
"format": "base64"
}
],
"t": 0.047042459,
"t": 0.073469208,
"address": "8.8.4.4:443",
"server_name": "dns.google",
"alpn": [
@ -254,7 +254,7 @@ Produces this JSON:
"no_tls_verify": false,
"oddity": "",
"proto": "quic",
"started": 0.002154834
"started": 0.025061583
}
]
}

64
internal/tutorial/measurex/chapter05/main.go
View File

@ -70,7 +70,7 @@ func main() {
// lines of code for printing the resulting measurement.
//
// ```
data, err := json.Marshal(m)
data, err := json.Marshal(measurex.NewArchivalEndpointMeasurement(m))
runtimex.PanicOnError(err, "json.Marshal failed")
fmt.Printf("%s\n", string(data))
}
@ -101,15 +101,15 @@ func main() {
// // are actually `recvfrom` and `sendto` but here
// // we follow the Go convention of using read/write
// // more frequently than send/recv.)
// "read_write": [
// "network_events": [
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 1252,
// "operation": "write_to",
// "proto": "quic",
// "t": 0.003903167,
// "started": 0.0037395,
// "t": 0.027184208,
// "started": 0.027127208,
// "oddity": ""
// },
// {
@ -118,8 +118,8 @@ func main() {
// "num_bytes": 1252,
// "operation": "read_from",
// "proto": "quic",
// "t": 0.029389125,
// "started": 0.002954792,
// "t": 0.053116458,
// "started": 0.025626583,
// "oddity": ""
// },
// {
@ -128,8 +128,8 @@ func main() {
// "num_bytes": 1252,
// "operation": "write_to",
// "proto": "quic",
// "t": 0.029757584,
// "started": 0.02972325,
// "t": 0.054538792,
// "started": 0.054517542,
// "oddity": ""
// },
// {
@ -138,8 +138,8 @@ func main() {
// "num_bytes": 1252,
// "operation": "read_from",
// "proto": "quic",
// "t": 0.045039875,
// "started": 0.029424792,
// "t": 0.069144958,
// "started": 0.053194208,
// "oddity": ""
// },
// {
@ -148,8 +148,8 @@ func main() {
// "num_bytes": 1252,
// "operation": "read_from",
// "proto": "quic",
// "t": 0.045055334,
// "started": 0.045049625,
// "t": 0.069183458,
// "started": 0.069173292,
// "oddity": ""
// },
// {
@ -158,28 +158,28 @@ func main() {
// "num_bytes": 1252,
// "operation": "read_from",
// "proto": "quic",
// "t": 0.045073917,
// "started": 0.045069667,
// "t": 0.06920225,
// "started": 0.069197875,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 1233,
// "num_bytes": 1216,
// "operation": "read_from",
// "proto": "quic",
// "t": 0.04508,
// "started": 0.045075292,
// "t": 0.069210958,
// "started": 0.069206875,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 64,
// "num_bytes": 65,
// "operation": "read_from",
// "proto": "quic",
// "t": 0.045088167,
// "started": 0.045081167,
// "t": 0.069220667,
// "started": 0.069217375,
// "oddity": ""
// },
// {
@ -188,8 +188,8 @@ func main() {
// "num_bytes": 44,
// "operation": "write_to",
// "proto": "quic",
// "t": 0.045370417,
// "started": 0.045338667,
// "t": 0.069433417,
// "started": 0.069417625,
// "oddity": ""
// },
// {
@ -198,8 +198,8 @@ func main() {
// "num_bytes": 44,
// "operation": "write_to",
// "proto": "quic",
// "t": 0.045392125,
// "started": 0.045380959,
// "t": 0.069677625,
// "started": 0.069647458,
// "oddity": ""
// },
// {
@ -208,8 +208,8 @@ func main() {
// "num_bytes": 83,
// "operation": "write_to",
// "proto": "quic",
// "t": 0.047042542,
// "started": 0.047001917,
// "t": 0.073461917,
// "started": 0.073432875,
// "oddity": ""
// },
// {
@ -218,15 +218,15 @@ func main() {
// "num_bytes": 33,
// "operation": "write_to",
// "proto": "quic",
// "t": 0.047060834,
// "started": 0.047046875,
// "t": 0.073559417,
// "started": 0.073542542,
// "oddity": ""
// }
// ],
//
// // This section describes the QUIC handshake and it has
// // basically the same fields as the TLS handshake.
// "quic_handshake": [
// "quic_handshakes": [
// {
// "cipher_suite": "TLS_CHACHA20_POLY1305_SHA256",
// "failure": null,
@ -234,7 +234,7 @@ func main() {
// "tls_version": "TLSv1.3",
// "peer_certificates": [
// {
// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc",
// "data": "MIIF4DCCBMigAwIBAgIQWiqDOVk83wAKAAAAAQivAjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTEwMDQwMjQzMzVaFw0yMTEyMjcwMjQzMzRaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUBr7csGnwBJnnh4nRPQFux/52IXuOIIUao2X4JQH7chvtuNyR4rxpeyL/zAYbFpohSdSfxrCFxrFWae+LzANL/VTPZPzPH2S2p1TNWy8rFWMU0OnRd3Ym5s1xOLBG9Lu28iH3h5+5yk5pGmvSnzA0Ra9Q5pjLJlUfnGi9ceP2uFM/SDRo1IEqNB7ukIeE5lmw7CJCyDafbWy8eFFPG69YRAjPb13NF4ijingt2JsUckgXJdWdoRf3KghM5ddvQhv5rdELjVUbI550MjfBQWoEUIxWEUsYNj/L4IbK46E4RpMnU+XnwTtinQYYgHVQxLKPtPdfYHp145DjiGXE9LMDAgMBAAGjggL5MIIC9TAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUo61UxicmPa371PCog8zI7NbrdkQwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQIGCisGAQQB1nkCBAIEgfMEgfAA7gB1AESUZS6w7s6vxEAH2Kj+KMDa5oK+2MsxtT/TM5a1toGoAAABfElmWjQAAAQDAEYwRAIgZpBA5ih3hRoZ1749kEcxdEcpzHUV3cS2zDHuz1WMy7gCIGtMqROCc/wrP01x1GXrk3M/qbHssnvhelxhNVbi4FTUAHUA9lyUL9F3MCIUVBgIMJRWjuNNExkzv98MLyALzE7xZOMAAAF8SWZaAwAABAMARjBEAiBRWeLjcLIQwBFdPEXa15s++b7kAKit86em9GR23F+7KQIgTHZgL7inapApbw5WJNhtEI78a5HHPsI+kU5LIDgpv7swDQYJKoZIhvcNAQELBQADggEBAGki3+h4nn12Ef449NirIIP5APx+1NQk3DDKT6PrpH3m+s/wQKXlJ8eNg6zJEBEtHxBdO0xI+/te2Bh1s6RU/iJZrVRAtol/xmn0AMvPQNL+JZUnYuryz2mwTpk5ZHnGHyZknbJDspB2oZkItDeDbvkMws+JKrCYbbHG4ZtcsoFYPrkjfyMLRemhj+qWvvMtUKUldsXbYhugQL44N+pWAJNyxEWUBcuEldItww/gSrl/O2alfGOTNvdcXT/nedrw+SJnci4m4oMTz+XWFkbf3yPXEikvUqcvQFe10F1wBr8wW6soM9nR7vKq7WUlnx1m4lghw/jStp2mWenT6YFp5Tw=",
// "format": "base64"
// },
// {
@ -246,7 +246,7 @@ func main() {
// "format": "base64"
// }
// ],
// "t": 0.047042459,
// "t": 0.073469208,
// "address": "8.8.4.4:443",
// "server_name": "dns.google",
// "alpn": [
@ -255,7 +255,7 @@ func main() {
// "no_tls_verify": false,
// "oddity": "",
// "proto": "quic",
// "started": 0.002154834
// "started": 0.025061583
// }
// ]
// }

131
internal/tutorial/measurex/chapter06/README.md
View File

@ -172,7 +172,7 @@ go doc ./internal/measurex.HTTPEndpointMeasurement
Let us now print the resulting measurement.
```Go
print(m)
print(measurex.NewArchivalHTTPEndpointMeasurement(m))
}
```
@ -182,7 +182,7 @@ Let us now print the resulting measurement.
Let us perform a vanilla run first:
```bash
go run -race ./internal/tutorial/measurex/chapter06
go run -race ./internal/tutorial/measurex/chapter06 | jq
```
This is the JSON output. Let us comment it in detail:
@ -196,31 +196,17 @@ This is the JSON output. Let us comment it in detail:
"network": "tcp",
"address": "8.8.4.4:443",
// Internally, HTTPEndpointGetWithoutCookies calls
// TCPConnect and here we see the corresponding event
"connect": [
{
"address": "8.8.4.4:443",
"failure": null,
"operation": "connect",
"proto": "tcp",
"t": 0.02422375,
"started": 0.002269291,
"oddity": ""
}
],
// These are the I/O operations we have already seen
// in previous chapters
"read_write": [
"network_events": [
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 280,
"operation": "write",
"proto": "tcp",
"t": 0.024931791,
"started": 0.024910416,
"t": 0.045800292,
"started": 0.045782167,
"oddity": ""
},
{
@ -229,18 +215,18 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 517,
"operation": "read",
"proto": "tcp",
"t": 0.063629791,
"started": 0.024935666,
"t": 0.082571,
"started": 0.045805458,
"oddity": ""
},
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 4301,
"num_bytes": 4303,
"operation": "read",
"proto": "tcp",
"t": 0.064183,
"started": 0.064144208,
"t": 0.084400542,
"started": 0.084372667,
"oddity": ""
},
{
@ -249,8 +235,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 64,
"operation": "write",
"proto": "tcp",
"t": 0.065464041,
"started": 0.065441333,
"t": 0.086762625,
"started": 0.086748292,
"oddity": ""
},
{
@ -259,8 +245,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 86,
"operation": "write",
"proto": "tcp",
"t": 0.067256083,
"started": 0.067224375,
"t": 0.087851,
"started": 0.087837625,
"oddity": ""
},
{
@ -269,8 +255,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 201,
"operation": "write",
"proto": "tcp",
"t": 0.067674416,
"started": 0.067652375,
"t": 0.089527292,
"started": 0.089507958,
"oddity": ""
},
{
@ -279,8 +265,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 93,
"operation": "read",
"proto": "tcp",
"t": 0.086618708,
"started": 0.067599208,
"t": 0.168585625,
"started": 0.088068375,
"oddity": ""
},
{
@ -289,18 +275,28 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 31,
"operation": "write",
"proto": "tcp",
"t": 0.086703625,
"started": 0.0866745,
"t": 0.168713542,
"started": 0.168671417,
"oddity": ""
},
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 2028,
"num_bytes": 2000,
"operation": "read",
"proto": "tcp",
"t": 0.337785916,
"started": 0.086717333,
"t": 0.468671417,
"started": 0.168759333,
"oddity": ""
},
{
"address": "8.8.4.4:443",
"failure": null,
"num_bytes": 39,
"operation": "read",
"proto": "tcp",
"t": 0.47118175,
"started": 0.471169667,
"oddity": ""
},
{
@ -309,8 +305,8 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 39,
"operation": "write",
"proto": "tcp",
"t": 0.338514916,
"started": 0.338485375,
"t": 0.471335458,
"started": 0.471268583,
"oddity": ""
},
{
@ -319,17 +315,25 @@ This is the JSON output. Let us comment it in detail:
"num_bytes": 24,
"operation": "write",
"proto": "tcp",
"t": 0.338800833,
"started": 0.338788625,
"t": 0.471865,
"started": 0.471836292,
"oddity": ""
},
}
],
// Internally, HTTPEndpointGetWithoutCookies calls
// TCPConnect and here we see the corresponding event
"tcp_connect": [
{
"address": "8.8.4.4:443",
"failure": "connection_already_closed",
"operation": "read",
"proto": "tcp",
"t": 0.338888041,
"started": 0.338523291,
"ip": "8.8.4.4",
"port": 443,
"t": 0.043644958,
"status": {
"blocked": false,
"failure": null,
"success": true
},
"started": 0.022849458,
"oddity": ""
}
],
@ -339,7 +343,7 @@ This is the JSON output. Let us comment it in detail:
// specified a QUIC endpoint we would instead see here a
// QUIC handshake event. And, we would not see any handshake
// if the URL was instead an HTTP URL.
"tls_handshake": [
"tls_handshakes": [
{
"cipher_suite": "TLS_AES_128_GCM_SHA256",
"failure": null,
@ -347,7 +351,7 @@ This is the JSON output. Let us comment it in detail:
"tls_version": "TLSv1.3",
"peer_certificates": [
{
"data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc",
"data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=",
"format": "base64"
},
{
@ -359,7 +363,7 @@ This is the JSON output. Let us comment it in detail:
"format": "base64"
}
],
"t": 0.065514708,
"t": 0.086816667,
"address": "8.8.4.4:443",
"server_name": "dns.google",
"alpn": [
@ -369,13 +373,13 @@ This is the JSON output. Let us comment it in detail:
"no_tls_verify": false,
"oddity": "",
"proto": "tcp",
"started": 0.024404083
"started": 0.043971083
}
],
// Finally here we see information about the round trip, which
// is formatted according the df-001-httpt data format:
"http_round_trip": [
// is formatted according to https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md:
"requests": [
{
// This field indicates whether there was an error during
@ -389,21 +393,20 @@ This is the JSON output. Let us comment it in detail:
"headers": {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"accept-language": "en-US;q=0.8,en;q=0.5",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36"
}
},
// This field contains the response status code, body,
// and headers.
// This field contains the response status code, body, and headers.
"response": {
"code": 200,
"headers": {
"accept-ranges": "none",
"alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
"alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
"cache-control": "private",
"content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-bSLcJjaotppZl3Y2moIaxg==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'",
"content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-y/OGliLR2gbEfTG2i4MAaw==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'",
"content-type": "text/html; charset=UTF-8",
"date": "Fri, 24 Sep 2021 08:51:01 GMT",
"date": "Fri, 05 Nov 2021 08:59:37 GMT",
"server": "scaffolding on HTTPServer2",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"vary": "Accept-Encoding",
@ -416,7 +419,7 @@ This is the JSON output. Let us comment it in detail:
// body: we don't want to read and submit to the OONI
// collector large bodies.
"body": {
"data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJiU0xjSmphb3RwcFpsM1kybW9JYXhnPT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+",
"data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJ5L09HbGlMUjJnYkVmVEcyaTRNQWF3PT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+",
"format": "base64"
},
@ -436,14 +439,14 @@ This is the JSON output. Let us comment it in detail:
// The t field is the moment where we finished the
// round trip and saved the event. The started field
// is instead when we started the round trip.
//
// You may notice that the start of the round trip
// if after the `t` of the handshake. This tells us
// that the code first connects, then handshakes, and
// finally creates HTTP code for performing the
// round trip.
"t": 0.338674625,
"started": 0.065926625,
"t": 0.471535167,
"started": 0.087176458,
// As usual we also compute an oddity value related
// in this case to the HTTP round trip.

131
internal/tutorial/measurex/chapter06/main.go
View File

@ -173,7 +173,7 @@ func main() {
// Let us now print the resulting measurement.
//
// ```Go
print(m)
print(measurex.NewArchivalHTTPEndpointMeasurement(m))
}
// ```
@ -183,7 +183,7 @@ func main() {
// Let us perform a vanilla run first:
//
// ```bash
// go run -race ./internal/tutorial/measurex/chapter06
// go run -race ./internal/tutorial/measurex/chapter06 | jq
// ```
//
// This is the JSON output. Let us comment it in detail:
@ -197,31 +197,17 @@ func main() {
// "network": "tcp",
// "address": "8.8.4.4:443",
//
// // Internally, HTTPEndpointGetWithoutCookies calls
// // TCPConnect and here we see the corresponding event
// "connect": [
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "operation": "connect",
// "proto": "tcp",
// "t": 0.02422375,
// "started": 0.002269291,
// "oddity": ""
// }
// ],
//
// // These are the I/O operations we have already seen
// // in previous chapters
// "read_write": [
// "network_events": [
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 280,
// "operation": "write",
// "proto": "tcp",
// "t": 0.024931791,
// "started": 0.024910416,
// "t": 0.045800292,
// "started": 0.045782167,
// "oddity": ""
// },
// {
@ -230,18 +216,18 @@ func main() {
// "num_bytes": 517,
// "operation": "read",
// "proto": "tcp",
// "t": 0.063629791,
// "started": 0.024935666,
// "t": 0.082571,
// "started": 0.045805458,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 4301,
// "num_bytes": 4303,
// "operation": "read",
// "proto": "tcp",
// "t": 0.064183,
// "started": 0.064144208,
// "t": 0.084400542,
// "started": 0.084372667,
// "oddity": ""
// },
// {
@ -250,8 +236,8 @@ func main() {
// "num_bytes": 64,
// "operation": "write",
// "proto": "tcp",
// "t": 0.065464041,
// "started": 0.065441333,
// "t": 0.086762625,
// "started": 0.086748292,
// "oddity": ""
// },
// {
@ -260,8 +246,8 @@ func main() {
// "num_bytes": 86,
// "operation": "write",
// "proto": "tcp",
// "t": 0.067256083,
// "started": 0.067224375,
// "t": 0.087851,
// "started": 0.087837625,
// "oddity": ""
// },
// {
@ -270,8 +256,8 @@ func main() {
// "num_bytes": 201,
// "operation": "write",
// "proto": "tcp",
// "t": 0.067674416,
// "started": 0.067652375,
// "t": 0.089527292,
// "started": 0.089507958,
// "oddity": ""
// },
// {
@ -280,8 +266,8 @@ func main() {
// "num_bytes": 93,
// "operation": "read",
// "proto": "tcp",
// "t": 0.086618708,
// "started": 0.067599208,
// "t": 0.168585625,
// "started": 0.088068375,
// "oddity": ""
// },
// {
@ -290,18 +276,28 @@ func main() {
// "num_bytes": 31,
// "operation": "write",
// "proto": "tcp",
// "t": 0.086703625,
// "started": 0.0866745,
// "t": 0.168713542,
// "started": 0.168671417,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 2028,
// "num_bytes": 2000,
// "operation": "read",
// "proto": "tcp",
// "t": 0.337785916,
// "started": 0.086717333,
// "t": 0.468671417,
// "started": 0.168759333,
// "oddity": ""
// },
// {
// "address": "8.8.4.4:443",
// "failure": null,
// "num_bytes": 39,
// "operation": "read",
// "proto": "tcp",
// "t": 0.47118175,
// "started": 0.471169667,
// "oddity": ""
// },
// {
@ -310,8 +306,8 @@ func main() {
// "num_bytes": 39,
// "operation": "write",
// "proto": "tcp",
// "t": 0.338514916,
// "started": 0.338485375,
// "t": 0.471335458,
// "started": 0.471268583,
// "oddity": ""
// },
// {
@ -320,17 +316,25 @@ func main() {
// "num_bytes": 24,
// "operation": "write",
// "proto": "tcp",
// "t": 0.338800833,
// "started": 0.338788625,
// "t": 0.471865,
// "started": 0.471836292,
// "oddity": ""
// },
// }
// ],
//
// // Internally, HTTPEndpointGetWithoutCookies calls
// // TCPConnect and here we see the corresponding event
// "tcp_connect": [
// {
// "address": "8.8.4.4:443",
// "failure": "connection_already_closed",
// "operation": "read",
// "proto": "tcp",
// "t": 0.338888041,
// "started": 0.338523291,
// "ip": "8.8.4.4",
// "port": 443,
// "t": 0.043644958,
// "status": {
// "blocked": false,
// "failure": null,
// "success": true
// },
// "started": 0.022849458,
// "oddity": ""
// }
// ],
@ -340,7 +344,7 @@ func main() {
// // specified a QUIC endpoint we would instead see here a
// // QUIC handshake event. And, we would not see any handshake
// // if the URL was instead an HTTP URL.
// "tls_handshake": [
// "tls_handshakes": [
// {
// "cipher_suite": "TLS_AES_128_GCM_SHA256",
// "failure": null,
@ -348,7 +352,7 @@ func main() {
// "tls_version": "TLSv1.3",
// "peer_certificates": [
// {
// "data": "MIIF4TCCBMmgAwIBAgIQGa7QSAXLo6sKAAAAAPz4cjANBgkqhkiG9w0BAQsFADBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yMTA4MzAwNDAwMDBaFw0yMTExMjIwMzU5NTlaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8cttrGHp3SS9YGYgsNLXt43dhW4d8FPULk0n6WYWC+EbMLkLnYXHLZHXJEz1Tor5hrCfHEVyX4xmhY2LCt0jprP6Gfo+gkKyjSV3LO65aWx6ezejvIdQBiLhSo/R5E3NwjMUAbm9PoNfSZSLiP3RjC3Px1vXFVmlcap4bUHnv9OvcPvwV1wmw5IMVzCuGBjCzJ4c4fxgyyggES1mbXZpYcDO4YKhSqIJx2D0gop9wzBQevI/kb35miN1pAvIKK2lgf7kZvYa7HH5vJ+vtn3Vkr34dKUAc/cO62t+NVufADPwn2/Tx8y8fPxlnCmoJeI+MPsw+StTYDawxajkjvZfdAgMBAAGjggL6MIIC9jAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUooaIxGAth6+bJh0JHYVWccyuoUcwHwYDVR0jBBgwFoAUinR/r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdvb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29vZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAAAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAAAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1AH0+8viP/4hVaCTCwMqeUol5K8UOeAl/LmqXaJl+IvDXAAABe5VtuiwAAAQDAEYwRAIgAwzr02ayTnNk/G+HDP50WTZUls3g+9P1fTGR9PEywpYCIAIOIQJ7nJTlcJdSyyOvgzX4BxJDr18mOKJPHlJs1naIAHYAXNxDkv7mq0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAF7lW26IQAABAMARzBFAiAtlIkbCH+QgiO6T6Y/+UAf+eqHB2wdzMNfOoo4SnUhVgIhALPiRtyPMo8fPPxN3VgiXBqVF7tzLWTJUjprOe4kQUCgMA0GCSqGSIb3DQEBCwUAA4IBAQDVq3WWgg6eYSpFLfNgo2KzLKDPkWZx42gW2Tum6JZd6O/Nj+mjYGOyXyryTslUwmONxiq2Ip3PLA/qlbPdYic1F1mDwMHSzRteSe7axwEP6RkoxhMy5zuI4hfijhSrfhVUZF299PesDf2gI+Vh30s6muHVfQjbXOl/AkAqIPLSetv2mS9MHQLeHcCCXpwsXQJwusZ3+ILrgCRAGv6NLXwbfE0t3OjXV0gnNRp3DWEaF+yrfjE0oU1myeYDNtugsw8VRwTzCM53Nqf/BJffnuShmBBZfZ2jlsPnLys0UqCZo2dg5wdwj3DaKtHO5Pofq6P8r4w6W/aUZCTLUi1jZ3Gc",
// "data": "MIIF4zCCBMugAwIBAgIRAJiMfOq7Or/8CgAAAAEQN9cwDQYJKoZIhvcNAQELBQAwRjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjExMDE4MTAxODI0WhcNMjIwMTEwMTAxODIzWjAVMRMwEQYDVQQDEwpkbnMuZ29vZ2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApihvr5NGRpea4ykYeyoKpbnwCr/YGp0Annb2T+DvTNmxWimJopYn7g9xbcZO3MRDWk4mbPX1TFqBg0YmVpPglaFVn8E03DjJakBdD20zF8cUmjUg2CrPwMbubSIecCLH4i5BfRTjs4hNLLBS2577b1o3oNU9rGsSkXoPs30XFuYJrJdcuVeU3uEx1ZDNIcrYIHcr1S+j0b1jtwHisy8N22wdLFUBTmeEw1NH7kamPFZgK+aXHxq8Z+htmrZpIesgBcfggyhYFU9SjSUHvIwoqCxuP1P5YUvcJBkrvMFjNRkUiFVAyEKmvKELGNOLOVkWeh9A9D+OBm9LdUOnHo42kQIDAQABo4IC+zCCAvcwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFD9wNtP27HXKprvm/76s/71s9fRbMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJGFPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AucGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVwby9jZXJ0cy9ndHMxYzMuZGVyMIGsBgNVHREEgaQwgaGCCmRucy5nb29nbGWCDmRucy5nb29nbGUuY29tghAqLmRucy5nb29nbGUuY29tggs4ODg4Lmdvb2dsZYIQZG5zNjQuZG5zLmdvb2dsZYcECAgICIcECAgEBIcQIAFIYEhgAAAAAAAAAACIiIcQIAFIYEhgAAAAAAAAAACIRIcQIAFIYEhgAAAAAAAAAABkZIcQIAFIYEhgAAAAAAAAAAAAZDAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RcUZ4Ymk5TTQ4Yy5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdgBRo7D1/QF5nFZtuDd4jwykeswbJ8v3nohCmg3+1IsF5QAAAXyTH8eGAAAEAwBHMEUCIQCDizVHW4ZqmkNxlrWhxDuzQjUg0uAfjvjPAgcPLIH/oAIgAaM2ihtIp6+6wAOP4NjScTZ3GXxvz9BPH6fHyZY0qQMAdgBGpVXrdfqRIDC1oolp9PN9ESxBdL79SbiFq/L8cP5tRwAAAXyTH8e4AAAEAwBHMEUCIHjpmWJyqK/RNqDX/15iUo70FgqvHoM1KeqXUcOnb4aIAiEA64ioBWLIwVYWAwt8xjX+Oy1fQ7ynTyCMvleFBTTC7kowDQYJKoZIhvcNAQELBQADggEBAMBLHXkhCXAyCb7oez8/6yV6R7L58/ArV0yqLMMNK+uL5rK/kVa36m/H+5eew8HP8+qB/bpoLq46S+YFDQMr9CCX1ip8oD2jrA91X2nrzhles6L58mIIDvTksOTl4FiMDyXtK/V3g9EXqG8CMgQVj2fZTjMyUC33nxmSUp4Zq0QVSeZCLgIbuBCKdMtkRzol2m/e3XJ6PD/ByezhG+E8N+o2GmeB2Ooq4Ur/vZg/QoN/tIMT//TbmNH0pY7BkMsTKMokfX5iygCAOvjsBRB52wUokMsC1qkWzxK4ToXhl5HPECMqf/nGZSkFsUHEM3Y7HKEVkhhO9YZJnR1bE6UFCMI=",
// "format": "base64"
// },
// {
@ -360,7 +364,7 @@ func main() {
// "format": "base64"
// }
// ],
// "t": 0.065514708,
// "t": 0.086816667,
// "address": "8.8.4.4:443",
// "server_name": "dns.google",
// "alpn": [
@ -370,13 +374,13 @@ func main() {
// "no_tls_verify": false,
// "oddity": "",
// "proto": "tcp",
// "started": 0.024404083
// "started": 0.043971083
// }
// ],
//
// // Finally here we see information about the round trip, which
// // is formatted according the df-001-httpt data format:
// "http_round_trip": [
// // is formatted according to https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md:
// "requests": [
// {
//
// // This field indicates whether there was an error during
@ -390,21 +394,20 @@ func main() {
// "headers": {
// "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
// "accept-language": "en-US;q=0.8,en;q=0.5",
// "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"
// "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36"
// }
// },
//
// // This field contains the response status code, body,
// // and headers.
// // This field contains the response status code, body, and headers.
// "response": {
// "code": 200,
// "headers": {
// "accept-ranges": "none",
// "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
// "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
// "cache-control": "private",
// "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-bSLcJjaotppZl3Y2moIaxg==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'",
// "content-security-policy": "object-src 'none';base-uri 'self';script-src 'nonce-y/OGliLR2gbEfTG2i4MAaw==' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'",
// "content-type": "text/html; charset=UTF-8",
// "date": "Fri, 24 Sep 2021 08:51:01 GMT",
// "date": "Fri, 05 Nov 2021 08:59:37 GMT",
// "server": "scaffolding on HTTPServer2",
// "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
// "vary": "Accept-Encoding",
@ -417,7 +420,7 @@ func main() {
// // body: we don't want to read and submit to the OONI
// // collector large bodies.
// "body": {
// "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJiU0xjSmphb3RwcFpsM1kybW9JYXhnPT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+",
// "data": "PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4gPGhlYWQ+IDx0aXRsZT5Hb29nbGUgUHVibGljIEROUzwvdGl0bGU+ICA8bWV0YSBjaGFyc2V0PSJVVEYtOCI+IDxsaW5rIGhyZWY9Ii9zdGF0aWMvOTNkZDU5NTQvZmF2aWNvbi5wbmciIHJlbD0ic2hvcnRjdXQgaWNvbiIgdHlwZT0iaW1hZ2UvcG5nIj4gPGxpbmsgaHJlZj0iL3N0YXRpYy84MzZhZWJjNi9tYXR0ZXIubWluLmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPGxpbmsgaHJlZj0iL3N0YXRpYy9iODUzNmMzNy9zaGFyZWQuY3NzIiByZWw9InN0eWxlc2hlZXQiPiA8bWV0YSBuYW1lPSJ2aWV3cG9ydCIgY29udGVudD0id2lkdGg9ZGV2aWNlLXdpZHRoLCBpbml0aWFsLXNjYWxlPTEiPiAgPGxpbmsgaHJlZj0iL3N0YXRpYy9kMDVjZDZiYS9yb290LmNzcyIgcmVsPSJzdHlsZXNoZWV0Ij4gPC9oZWFkPiA8Ym9keT4gPHNwYW4gY2xhc3M9ImZpbGxlciB0b3AiPjwvc3Bhbj4gICA8ZGl2IGNsYXNzPSJsb2dvIiB0aXRsZT0iR29vZ2xlIFB1YmxpYyBETlMiPiA8ZGl2IGNsYXNzPSJsb2dvLXRleHQiPjxzcGFuPlB1YmxpYyBETlM8L3NwYW4+PC9kaXY+IDwvZGl2PiAgPGZvcm0gYWN0aW9uPSIvcXVlcnkiIG1ldGhvZD0iR0VUIj4gIDxkaXYgY2xhc3M9InJvdyI+IDxsYWJlbCBjbGFzcz0ibWF0dGVyLXRleHRmaWVsZC1vdXRsaW5lZCI+IDxpbnB1dCB0eXBlPSJ0ZXh0IiBuYW1lPSJuYW1lIiBwbGFjZWhvbGRlcj0iJm5ic3A7Ij4gPHNwYW4+RE5TIE5hbWU8L3NwYW4+IDxwIGNsYXNzPSJoZWxwIj4gRW50ZXIgYSBkb21haW4gKGxpa2UgZXhhbXBsZS5jb20pIG9yIElQIGFkZHJlc3MgKGxpa2UgOC44LjguOCBvciAyMDAxOjQ4NjA6NDg2MDo6ODg0NCkgaGVyZS4gPC9wPiA8L2xhYmVsPiA8YnV0dG9uIGNsYXNzPSJtYXR0ZXItYnV0dG9uLWNvbnRhaW5lZCBtYXR0ZXItcHJpbWFyeSIgdHlwZT0ic3VibWl0Ij5SZXNvbHZlPC9idXR0b24+IDwvZGl2PiA8L2Zvcm0+ICA8c3BhbiBjbGFzcz0iZmlsbGVyIGJvdHRvbSI+PC9zcGFuPiA8Zm9vdGVyIGNsYXNzPSJyb3ciPiA8YSBocmVmPSJodHRwczovL2RldmVsb3BlcnMuZ29vZ2xlLmNvbS9zcGVlZC9wdWJsaWMtZG5zIj5IZWxwPC9hPiA8YSBocmVmPSIvY2FjaGUiPkNhY2hlIEZsdXNoPC9hPiA8c3BhbiBjbGFzcz0iZmlsbGVyIj48L3NwYW4+IDxhIGhyZWY9Imh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL3NwZWVkL3B1YmxpYy1kbnMvZG9jcy91c2luZyI+IEdldCBTdGFydGVkIHdpdGggR29vZ2xlIFB1YmxpYyBETlMgPC9hPiA8L2Zvb3Rlcj4gICA8c2NyaXB0IG5vbmNlPSJ5L09HbGlMUjJnYkVmVEcyaTRNQWF3PT0iPmRvY3VtZW50LmZvcm1zWzBdLm5hbWUuZm9jdXMoKTs8L3NjcmlwdD4gPC9ib2R5PiA8L2h0bWw+",
// "format": "base64"
// },
//
@ -437,14 +440,14 @@ func main() {
// // The t field is the moment where we finished the
// // round trip and saved the event. The started field
// // is instead when we started the round trip.
//
// //
// // You may notice that the start of the round trip
// // if after the `t` of the handshake. This tells us
// // that the code first connects, then handshakes, and
// // finally creates HTTP code for performing the
// // round trip.
// "t": 0.338674625,
// "started": 0.065926625,
// "t": 0.471535167,
// "started": 0.087176458,
//
// // As usual we also compute an oddity value related
// // in this case to the HTTP round trip.

8
internal/tutorial/measurex/chapter07/README.md
View File

@ -111,7 +111,8 @@ We are almost done now: we loop over all the endpoints and apply the
}
```
Finally, we print the results.
Finally, we print the results. (Note that here we are not
converting to the OONI archival data format.)
```Go
print(m)
@ -128,7 +129,10 @@ go run -race ./internal/tutorial/measurex/chapter07 | jq
```
Please, check the JSON output. Do you recognize the fields
we have described in previous chapters?
we have described in previous chapters, even though we didn't
convert to the OONI data format? Can you modify the code to
use the OONI data format in the output by calling the proper
conversion functions exported by `measurex`?
Can you provoke common errors such as DNS resolution
errors, TCP connect errors, TLS handshake errors, and

8
internal/tutorial/measurex/chapter07/main.go
View File

@ -112,7 +112,8 @@ func main() {
}
// ```
//
// Finally, we print the results.
// Finally, we print the results. (Note that here we are not
// converting to the OONI archival data format.)
//
// ```Go
print(m)
@ -129,7 +130,10 @@ func main() {
// ```
//
// Please, check the JSON output. Do you recognize the fields
// we have described in previous chapters?
// we have described in previous chapters, even though we didn't
// convert to the OONI data format? Can you modify the code to
// use the OONI data format in the output by calling the proper
// conversion functions exported by `measurex`?
//
// Can you provoke common errors such as DNS resolution
// errors, TCP connect errors, TLS handshake errors, and

9
internal/tutorial/measurex/chapter08/README.md
View File

@ -101,6 +101,15 @@ This is it. The rest of the program is exactly the same.
for _, epnt := range httpEndpoints {
m.Endpoints = append(m.Endpoints, mx.HTTPEndpointGetWithoutCookies(ctx, epnt))
}
```
(Note that here, like in the previous chapter, we are not converting
to the OONI data format. Rather, we're just dumping the internally
used data structures. Exercise: can you modify this program to emit
a JSON compliant with the OONI data format by using the proper]
conversion functions exported by `measurex`?)
```Go
print(m)
}

9
internal/tutorial/measurex/chapter08/main.go
View File

@ -102,6 +102,15 @@ func main() {
for _, epnt := range httpEndpoints {
m.Endpoints = append(m.Endpoints, mx.HTTPEndpointGetWithoutCookies(ctx, epnt))
}
// ```
//
// (Note that here, like in the previous chapter, we are not converting
// to the OONI data format. Rather, we're just dumping the internally
// used data structures. Exercise: can you modify this program to emit
// a JSON compliant with the OONI data format by using the proper]
// conversion functions exported by `measurex`?)
//
// ```Go
print(m)
}

5
internal/tutorial/measurex/chapter09/README.md
View File

@ -101,6 +101,11 @@ fully measured, this method closes the returned channel.
Like we did before, we append the resulting measurements to
our `m` container and we print it.
Exercise: here we're not using the OONI data format and we're
instead printing the internally used data structures. Can
you modify the code to emit data using OONI's data format here?
(Hint: there are conversion functions in `measurex`.)
```Go
print(m)
}

5
internal/tutorial/measurex/chapter09/main.go
View File

@ -102,6 +102,11 @@ func main() {
// Like we did before, we append the resulting measurements to
// our `m` container and we print it.
//
// Exercise: here we're not using the OONI data format and we're
// instead printing the internally used data structures. Can
// you modify the code to emit data using OONI's data format here?
// (Hint: there are conversion functions in `measurex`.)
//
// ```Go
print(m)
}

7
internal/tutorial/measurex/chapter12/README.md
View File

@ -32,7 +32,7 @@ import (
)
type measurement struct {
URLs []*measurex.URLMeasurement
URLs []*measurex.ArchivalURLMeasurement
}
func print(v interface{}) {
@ -67,7 +67,7 @@ is closed when done by `MeasureURLAndFollowRedirections`, so we leave the loop.
```Go
for m := range mx.MeasureURLAndFollowRedirections(ctx, *URL, headers, cookies) {
all.URLs = append(all.URLs, m)
all.URLs = append(all.URLs, measurex.NewArchivalURLMeasurement(m))
}
print(all)
}
@ -86,6 +86,9 @@ Take a look at the JSON. You should see several redirects
and that we measure each endpoint of each redirect, including
QUIC endpoints that we discover on the way.
Exercise: remove code for converting to OONI data format
and compare output with previous chapter. See any difference?
## Conclusion
We have introduced `MeasureURLAndFollowRedirect`, the

7
internal/tutorial/measurex/chapter12/main.go
View File

@ -33,7 +33,7 @@ import (
)
type measurement struct {
URLs []*measurex.URLMeasurement
URLs []*measurex.ArchivalURLMeasurement
}
func print(v interface{}) {
@ -68,7 +68,7 @@ func main() {
//
// ```Go
for m := range mx.MeasureURLAndFollowRedirections(ctx, *URL, headers, cookies) {
all.URLs = append(all.URLs, m)
all.URLs = append(all.URLs, measurex.NewArchivalURLMeasurement(m))
}
print(all)
}
@ -87,6 +87,9 @@ func main() {
// and that we measure each endpoint of each redirect, including
// QUIC endpoints that we discover on the way.
//
// Exercise: remove code for converting to OONI data format
// and compare output with previous chapter. See any difference?
//
// ## Conclusion
//
// We have introduced `MeasureURLAndFollowRedirect`, the

23
internal/tutorial/measurex/chapter14/README.md
View File

@ -45,10 +45,10 @@ that a Web Connectivity measurement should have.
```Go
type measurement struct {
Queries []*measurex.DNSLookupEvent `json:"queries"`
TCPConnect []*measurex.NetworkEvent `json:"tcp_connect"`
TLSHandshakes []*measurex.TLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.HTTPRoundTripEvent `json:"requests"`
Queries []*measurex.ArchivalDNSLookupEvent `json:"queries"`
TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"`
TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.ArchivalHTTPRoundTripEvent `json:"requests"`
}
```
@ -96,7 +96,8 @@ the input URL's domain using the system resolver.
```Go
dns := mx.LookupHostSystem(ctx, parsedURL.Hostname())
m.Queries = append(m.Queries, dns.LookupHost...)
m.Queries = append(
m.Queries, measurex.NewArchivalDNSLookupEventList(dns.LookupHost)...)
```
@ -128,7 +129,8 @@ whether the input URL is HTTP or HTTPS.
switch parsedURL.Scheme {
case "http":
tcp := mx.TCPConnect(ctx, epnt.Address)
m.TCPConnect = append(m.TCPConnect, tcp.Connect...)
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...)
case "https":
config := &tls.Config{
ServerName: parsedURL.Hostname(),
@ -136,8 +138,10 @@ whether the input URL is HTTP or HTTPS.
RootCAs: netxlite.NewDefaultCertPool(),
}
tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config)
m.TCPConnect = append(m.TCPConnect, tls.Connect...)
m.TLSHandshakes = append(m.TLSHandshakes, tls.TLSHandshake...)
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tls.Connect)...)
m.TLSHandshakes = append(m.TLSHandshakes,
measurex.NewArchivalQUICTLSHandshakeEventList(tls.TLSHandshake)...)
}
}
@ -222,7 +226,8 @@ the chapters we have seen so far.
```Go
m.Requests = append(m.Requests, db.AsMeasurement().HTTPRoundTrip...)
m.Requests = append(m.Requests, measurex.NewArchivalHTTPRoundTripEventList(
db.AsMeasurement().HTTPRoundTrip)...)
return m, nil
}

23
internal/tutorial/measurex/chapter14/main.go
View File

@ -46,10 +46,10 @@ func print(v interface{}) {
// ```Go
type measurement struct {
Queries []*measurex.DNSLookupEvent `json:"queries"`
TCPConnect []*measurex.NetworkEvent `json:"tcp_connect"`
TLSHandshakes []*measurex.TLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.HTTPRoundTripEvent `json:"requests"`
Queries []*measurex.ArchivalDNSLookupEvent `json:"queries"`
TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"`
TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"`
Requests []*measurex.ArchivalHTTPRoundTripEvent `json:"requests"`
}
// ```
@ -97,7 +97,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
//
// ```Go
dns := mx.LookupHostSystem(ctx, parsedURL.Hostname())
m.Queries = append(m.Queries, dns.LookupHost...)
m.Queries = append(
m.Queries, measurex.NewArchivalDNSLookupEventList(dns.LookupHost)...)
// ```
//
@ -129,7 +130,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
switch parsedURL.Scheme {
case "http":
tcp := mx.TCPConnect(ctx, epnt.Address)
m.TCPConnect = append(m.TCPConnect, tcp.Connect...)
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tcp.Connect)...)
case "https":
config := &tls.Config{
ServerName: parsedURL.Hostname(),
@ -137,8 +139,10 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
RootCAs: netxlite.NewDefaultCertPool(),
}
tls := mx.TLSConnectAndHandshake(ctx, epnt.Address, config)
m.TCPConnect = append(m.TCPConnect, tls.Connect...)
m.TLSHandshakes = append(m.TLSHandshakes, tls.TLSHandshake...)
m.TCPConnect = append(
m.TCPConnect, measurex.NewArchivalTCPConnectList(tls.Connect)...)
m.TLSHandshakes = append(m.TLSHandshakes,
measurex.NewArchivalQUICTLSHandshakeEventList(tls.TLSHandshake)...)
}
}
@ -223,7 +227,8 @@ func webConnectivity(ctx context.Context, URL string) (*measurement, error) {
//
// ```Go
m.Requests = append(m.Requests, db.AsMeasurement().HTTPRoundTrip...)
m.Requests = append(m.Requests, measurex.NewArchivalHTTPRoundTripEventList(
db.AsMeasurement().HTTPRoundTrip)...)
return m, nil
}