From b2b1a4b2f1782ac6f81d8c2f1a1e898d2abb1250 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 29 Sep 2021 20:21:25 +0200 Subject: [PATCH] doc(netxlite): revamp the documentation (#523) Part of https://github.com/ooni/probe-cli/pull/506. In parallel with tutorials, we also need to make sure we have good documentation. --- internal/README.md | 16 +++++++ internal/netxlite/certifi.go | 2 +- internal/netxlite/classify.go | 52 +++++++++++---------- internal/netxlite/dialer.go | 35 ++++++++------ internal/netxlite/dnsdecoder.go | 25 ++++++++++ internal/netxlite/dnsencoder.go | 13 +++++- internal/netxlite/dnsoverhttps.go | 38 +++++++++------ internal/netxlite/dnsovertcp.go | 29 ++++++++---- internal/netxlite/dnsoverudp.go | 16 +++++-- internal/netxlite/dnstransport.go | 8 ++-- internal/netxlite/dnsx/dnsx.go | 10 ++-- internal/netxlite/doc.go | 8 +++- internal/netxlite/errno.go | 5 +- internal/netxlite/errno_android.go | 6 ++- internal/netxlite/errno_android_test.go | 2 +- internal/netxlite/errno_darwin.go | 6 ++- internal/netxlite/errno_darwin_test.go | 2 +- internal/netxlite/errno_freebsd.go | 6 ++- internal/netxlite/errno_freebsd_test.go | 2 +- internal/netxlite/errno_ios.go | 6 ++- internal/netxlite/errno_ios_test.go | 2 +- internal/netxlite/errno_linux.go | 6 ++- internal/netxlite/errno_linux_test.go | 2 +- internal/netxlite/errno_windows.go | 6 ++- internal/netxlite/errno_windows_test.go | 2 +- internal/netxlite/errwrapper.go | 10 ++-- internal/netxlite/http.go | 35 ++++++++------ internal/netxlite/internal/generrno/main.go | 7 ++- internal/netxlite/iox.go | 9 ++-- internal/netxlite/legacy.go | 25 +++++----- internal/netxlite/operations.go | 2 +- internal/netxlite/quic.go | 48 ++++++++++--------- internal/netxlite/quic_test.go | 7 ++- internal/netxlite/quicx/quicx.go | 9 +--- internal/netxlite/resolver.go | 33 ++++++++----- internal/netxlite/serialresolver.go | 30 ++++++++---- internal/netxlite/tls.go | 51 ++++++++++++++------ internal/netxlite/utls.go | 9 ++-- internal/tutorial/netxlite/README.md | 29 +++++++++++- 39 files changed, 399 insertions(+), 210 deletions(-) diff --git a/internal/README.md b/internal/README.md index 54b137f..11d37e6 100644 --- a/internal/README.md +++ b/internal/README.md @@ -1,3 +1,19 @@ # Directory github.com/ooni/probe-cli/internal This directory contains private Go packages. + +As a reminder, you can always check the Go documentation of +a package by using + +```bash +go doc -all ./internal/$package +``` + +where `$package` is the name of the package. + +Some notable packages: + +- [netxlite](netxlite) is the underlying networking library; + +- [tutorial](tutorial) contains tutorials on writing new experiments, +using measurements libraries, and networking code. diff --git a/internal/netxlite/certifi.go b/internal/netxlite/certifi.go index 79b255f..c99138f 100644 --- a/internal/netxlite/certifi.go +++ b/internal/netxlite/certifi.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2021-09-29 10:21:32.800846 +0200 CEST m=+0.427651209 +// 2021-09-29 16:40:31.464953 +0200 CEST m=+0.466236210 // https://curl.haxx.se/ca/cacert.pem package netxlite diff --git a/internal/netxlite/classify.go b/internal/netxlite/classify.go index 0b61daa..56a4f4d 100644 --- a/internal/netxlite/classify.go +++ b/internal/netxlite/classify.go @@ -11,28 +11,29 @@ import ( "github.com/ooni/probe-cli/v3/internal/scrubber" ) -// ClassifyGenericError is the generic classifier mapping an error -// occurred during an operation to an OONI failure string. +// ClassifyGenericError is maps an error occurred during an operation +// to an OONI failure string. This specific classifier is the most +// generic one. You usually use it when mapping I/O errors. You should +// check whether there is a specific classifier for more specific +// operations (e.g., DNS resolution, TLS handshake). // -// If the input error is already an ErrWrapper we don't perform -// the classification again and we return its Failure to the caller. -// -// Classification rules +// If the input error is an *ErrWrapper we don't perform +// the classification again and we return its Failure. // // We put inside this classifier: // -// - system call errors +// - system call errors; // -// - generic errors that can occur in multiple places +// - generic errors that can occur in multiple places; // -// - all the errors that depend on strings +// - all the errors that depend on strings. // // The more specific classifiers will call this classifier if // they fail to find a mapping for the input error. // // If everything else fails, this classifier returns a string // like "unknown_failure: XXX" where XXX has been scrubbed -// so to remove any network endpoints from its value. +// so to remove any network endpoints from the original error string. func ClassifyGenericError(err error) string { // The list returned here matches the values used by MK unless // explicitly noted otherwise with a comment. @@ -133,14 +134,14 @@ const ( quicTLSUnrecognizedName = 112 ) -// ClassifyQUICHandshakeError maps an error occurred during the QUIC -// handshake to an OONI failure string. +// ClassifyQUICHandshakeError maps errors during a QUIC +// handshake to OONI failure strings. // -// If the input error is already an ErrWrapper we don't perform -// the classification again and we return its Failure to the caller. +// If the input error is an *ErrWrapper we don't perform +// the classification again and we return its Failure. // -// If this classifier fails, it calls ClassifyGenericError and -// returns to the caller its return value. +// If this classifier fails, it calls ClassifyGenericError +// and returns to the caller its return value. func ClassifyQUICHandshakeError(err error) string { var errwrapper *ErrWrapper if errors.As(err, &errwrapper) { @@ -229,14 +230,17 @@ func quicIsCertificateError(alert uint8) bool { // filters for DNS bogons MUST use this error. var ErrDNSBogon = errors.New("dns: detected bogon address") -// These strings are same as the standard library. +// We use these strings to string-match errors in the standard library +// and map such errors to OONI failures. const ( DNSNoSuchHostSuffix = "no such host" DNSServerMisbehavingSuffix = "server misbehaving" DNSNoAnswerSuffix = "no answer from DNS server" ) -// These errors are returned by the decoder and/or the serial resolver. +// These errors are returned by custom DNSTransport instances (e.g., +// DNSOverHTTPS and DNSOverUDP). Their suffix matches the equivalent +// unexported errors used by the Go standard library. var ( ErrOODNSNoSuchHost = fmt.Errorf("ooniresolver: %s", DNSNoSuchHostSuffix) ErrOODNSRefused = errors.New("ooniresolver: refused") @@ -244,11 +248,11 @@ var ( ErrOODNSNoAnswer = fmt.Errorf("ooniresolver: %s", DNSNoAnswerSuffix) ) -// ClassifyResolverError maps an error occurred during a domain name -// resolution to the corresponding OONI failure string. +// ClassifyResolverError maps DNS resolution errors to +// OONI failure strings. // -// If the input error is already an ErrWrapper we don't perform -// the classification again and we return its Failure to the caller. +// If the input error is an *ErrWrapper we don't perform +// the classification again and we return its Failure. // // If this classifier fails, it calls ClassifyGenericError and // returns to the caller its return value. @@ -271,8 +275,8 @@ func ClassifyResolverError(err error) string { // ClassifyTLSHandshakeError maps an error occurred during the TLS // handshake to an OONI failure string. // -// If the input error is already an ErrWrapper we don't perform -// the classification again and we return its Failure to the caller. +// If the input error is an *ErrWrapper we don't perform +// the classification again and we return its Failure. // // If this classifier fails, it calls ClassifyGenericError and // returns to the caller its return value. diff --git a/internal/netxlite/dialer.go b/internal/netxlite/dialer.go index 9ba9afa..d4e3672 100644 --- a/internal/netxlite/dialer.go +++ b/internal/netxlite/dialer.go @@ -17,8 +17,7 @@ type Dialer interface { CloseIdleConnections() } -// NewDialerWithResolver is a convenience factory that calls -// WrapDialer for a stdlib dialer type. +// NewDialerWithResolver calls WrapDialer for the stdlib dialer. func NewDialerWithResolver(logger Logger, resolver Resolver) Dialer { return WrapDialer(logger, resolver, &dialerSystem{}) } @@ -30,14 +29,14 @@ func NewDialerWithResolver(logger Logger, resolver Resolver) Dialer { // // 2. resolves domain names using the givern resolver; // -// 3. when using a resolver, each available enpoint is tried +// 3. when the resolver is not a "null" resolver, +// each available enpoint is tried // sequentially. On error, the code will return what it believes // to be the most representative error in the pack. Most often, -// such an error is the first one that occurred. Choosing the +// the first error that occurred. Choosing the // error to return using this logic is a QUIRK that we owe // to the original implementation of netx. We cannot change -// this behavior until all the legacy code that relies on -// it has been migrated to more sane patterns. +// this behavior until we refactor legacy code using it. // // Removing this quirk from the codebase is documented as // TODO(https://github.com/ooni/probe/issues/1779). @@ -49,6 +48,9 @@ func NewDialerWithResolver(logger Logger, resolver Resolver) Dialer { // 6. if a dialer wraps a resolver, the dialer will forward // the CloseIdleConnection call to its resolver (which is // instrumental to manage a DoH resolver connections properly). +// +// In general, do not use WrapDialer directly but try to use +// more high-level factories, e.g., NewDialerWithResolver. func WrapDialer(logger Logger, resolver Resolver, dialer Dialer) Dialer { return &dialerLogger{ Dialer: &dialerResolver{ @@ -65,8 +67,9 @@ func WrapDialer(logger Logger, resolver Resolver, dialer Dialer) Dialer { } } -// NewDialerWithoutResolver is like NewDialerWithResolver except that -// it will fail with ErrNoResolver if passed a domain name. +// NewDialerWithoutResolver calls NewDialerWithResolver with a "null" resolver. +// +// The returned dialer fails with ErrNoResolver if passed a domain name. func NewDialerWithoutResolver(logger Logger) Dialer { return NewDialerWithResolver(logger, &nullResolver{}) } @@ -183,12 +186,15 @@ func (d *dialerLogger) CloseIdleConnections() { d.Dialer.CloseIdleConnections() } -// ErrNoConnReuse indicates we cannot reuse the connection provided -// to a single use (possibly TLS) dialer. +// ErrNoConnReuse is the type of error returned when you create a +// "single use" dialer or a "single use" TLS dialer and you dial +// more than once, which is not supported by such a dialer. var ErrNoConnReuse = errors.New("cannot reuse connection") -// NewSingleUseDialer returns a dialer that returns the given connection once -// and after that always fails with the ErrNoConnReuse error. +// NewSingleUseDialer returns a "single use" dialer. The first +// dial will succed and return conn regardless of the network +// and address arguments passed to DialContext. Any subsequent +// dial returns ErrNoConnReuse. func NewSingleUseDialer(conn net.Conn) Dialer { return &dialerSingleUse{conn: conn} } @@ -263,10 +269,11 @@ func (c *dialerErrWrapperConn) Close() error { return nil } -// ErrNoDialer indicates that no dialer is configured. +// ErrNoDialer is the type of error returned by "null" dialers +// when you attempt to dial with them. var ErrNoDialer = errors.New("no configured dialer") -// NewNullDialer returns a dialer that always fails. +// NewNullDialer returns a dialer that always fails with ErrNoDialer. func NewNullDialer() Dialer { return &nullDialer{} } diff --git a/internal/netxlite/dnsdecoder.go b/internal/netxlite/dnsdecoder.go index 5976d74..a7117b6 100644 --- a/internal/netxlite/dnsdecoder.go +++ b/internal/netxlite/dnsdecoder.go @@ -5,9 +5,34 @@ import "github.com/miekg/dns" // The DNSDecoder decodes DNS replies. type DNSDecoder interface { // DecodeLookupHost decodes an A or AAAA reply. + // + // Arguments: + // + // - qtype is the query type (e.g., dns.TypeAAAA) + // + // - data contains the reply bytes read from a DNSTransport + // + // Returns: + // + // - on success, a list of IP addrs inside the reply and a nil error + // + // - on failure, a nil list and an error. + // + // Note that this function will return an error if there is no + // IP address inside of the reply. DecodeLookupHost(qtype uint16, data []byte) ([]string, error) // DecodeHTTPS decodes an HTTPS reply. + // + // The argument is the reply as read by the DNSTransport. + // + // On success, this function returns an HTTPSSvc structure and + // a nil error. On failure, the HTTPSSvc pointer is nil and + // the error points to the error that occurred. + // + // This function will return an error if the HTTPS reply does not + // contain at least a valid ALPN entry. It will not return + // an error, though, when there are no IPv4/IPv6 hints in the reply. DecodeHTTPS(data []byte) (*HTTPSSvc, error) } diff --git a/internal/netxlite/dnsencoder.go b/internal/netxlite/dnsencoder.go index 6a96ab5..08844e4 100644 --- a/internal/netxlite/dnsencoder.go +++ b/internal/netxlite/dnsencoder.go @@ -4,6 +4,18 @@ import "github.com/miekg/dns" // The DNSEncoder encodes DNS queries to bytes type DNSEncoder interface { + // Encode transforms its arguments into a serialized DNS query. + // + // Arguments: + // + // - domain is the domain for the query (e.g., x.org); + // + // - qtype is the query type (e.g., dns.TypeA); + // + // - padding is whether to add padding to the query. + // + // On success, this function returns a valid byte array and + // a nil error. On failure, we have an error and the byte array is nil. Encode(domain string, qtype uint16, padding bool) ([]byte, error) } @@ -21,7 +33,6 @@ const ( dnsDNSSECEnabled = true ) -// Encode implements Encoder.Encode func (e *DNSEncoderMiekg) Encode(domain string, qtype uint16, padding bool) ([]byte, error) { question := dns.Question{ Name: dns.Fqdn(domain), diff --git a/internal/netxlite/dnsoverhttps.go b/internal/netxlite/dnsoverhttps.go index f63b9b1..70bacfc 100644 --- a/internal/netxlite/dnsoverhttps.go +++ b/internal/netxlite/dnsoverhttps.go @@ -10,34 +10,44 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/httpheader" ) -// HTTPClient is the HTTP client expected by DNSOverHTTPS. +// HTTPClient is an http.Client-like interface. type HTTPClient interface { Do(req *http.Request) (*http.Response, error) CloseIdleConnections() } -// DNSOverHTTPS is a DNS over HTTPS RoundTripper. Requests are submitted over -// an HTTP/HTTPS channel provided by URL using the Do function. +// DNSOverHTTPS is a DNS-over-HTTPS DNSTransport. type DNSOverHTTPS struct { - Client HTTPClient - URL string + // Client is the MANDATORY http client to use. + Client HTTPClient + + // URL is the MANDATORY URL of the DNS-over-HTTPS server. + URL string + + // HostOverride is OPTIONAL and allows to override the + // Host header sent in every request. HostOverride string } -// NewDNSOverHTTPS creates a new DNSOverHTTP instance from the -// specified http.Client and URL, as a convenience. +// NewDNSOverHTTPS creates a new DNSOverHTTPS instance. +// +// Arguments: +// +// - client in http.Client-like type (e.g., http.DefaultClient); +// +// - URL is the DoH resolver URL (e.g., https://1.1.1.1/dns-query). func NewDNSOverHTTPS(client HTTPClient, URL string) *DNSOverHTTPS { return NewDNSOverHTTPSWithHostOverride(client, URL, "") } -// NewDNSOverHTTPSWithHostOverride is like NewDNSOverHTTPS except that -// it's creating a resolver where we use the specified host. +// NewDNSOverHTTPSWithHostOverride creates a new DNSOverHTTPS +// with the given Host header override. func NewDNSOverHTTPSWithHostOverride( client HTTPClient, URL, hostOverride string) *DNSOverHTTPS { return &DNSOverHTTPS{Client: client, URL: URL, HostOverride: hostOverride} } -// RoundTrip implements RoundTripper.RoundTrip. +// RoundTrip sends a query and receives a reply. func (t *DNSOverHTTPS) RoundTrip(ctx context.Context, query []byte) ([]byte, error) { ctx, cancel := context.WithTimeout(ctx, 45*time.Second) defer cancel() @@ -65,22 +75,22 @@ func (t *DNSOverHTTPS) RoundTrip(ctx context.Context, query []byte) ([]byte, err return ReadAllContext(ctx, resp.Body) } -// RequiresPadding returns true for DoH according to RFC8467 +// RequiresPadding returns true for DoH according to RFC8467. func (t *DNSOverHTTPS) RequiresPadding() bool { return true } -// Network returns the transport network (e.g., doh, dot) +// Network returns the transport network, i.e., "doh". func (t *DNSOverHTTPS) Network() string { return "doh" } -// Address returns the upstream server address. +// Address returns the URL we're using for the DoH server. func (t *DNSOverHTTPS) Address() string { return t.URL } -// CloseIdleConnections closes idle connections. +// CloseIdleConnections closes idle connections, if any. func (t *DNSOverHTTPS) CloseIdleConnections() { t.Client.CloseIdleConnections() } diff --git a/internal/netxlite/dnsovertcp.go b/internal/netxlite/dnsovertcp.go index 77f9c7c..9f66a59 100644 --- a/internal/netxlite/dnsovertcp.go +++ b/internal/netxlite/dnsovertcp.go @@ -9,15 +9,12 @@ import ( "time" ) -// DialContextFunc is a generic function for dialing a connection. +// DialContextFunc is the type of net.Dialer.DialContext. type DialContextFunc func(context.Context, string, string) (net.Conn, error) -// DNSOverTCP is a DNS over TCP/TLS RoundTripper. Use NewDNSOverTCP -// and NewDNSOverTLS to create specific instances that use plaintext -// queries or encrypted queries over TLS. +// DNSOverTCP is a DNS-over-{TCP,TLS} DNSTransport. // -// As a known bug, this implementation always creates a new connection -// for each incoming query, thus increasing the response delay. +// Bug: this implementation always creates a new connection for each query. type DNSOverTCP struct { dial DialContextFunc address string @@ -26,6 +23,12 @@ type DNSOverTCP struct { } // NewDNSOverTCP creates a new DNSOverTCP transport. +// +// Arguments: +// +// - dial is a function with the net.Dialer.DialContext's signature; +// +// - address is the endpoint address (e.g., 8.8.8.8:53). func NewDNSOverTCP(dial DialContextFunc, address string) *DNSOverTCP { return &DNSOverTCP{ dial: dial, @@ -36,6 +39,12 @@ func NewDNSOverTCP(dial DialContextFunc, address string) *DNSOverTCP { } // NewDNSOverTLS creates a new DNSOverTLS transport. +// +// Arguments: +// +// - dial is a function with the net.Dialer.DialContext's signature; +// +// - address is the endpoint address (e.g., 8.8.8.8:853). func NewDNSOverTLS(dial DialContextFunc, address string) *DNSOverTCP { return &DNSOverTCP{ dial: dial, @@ -45,7 +54,7 @@ func NewDNSOverTLS(dial DialContextFunc, address string) *DNSOverTCP { } } -// RoundTrip implements RoundTripper.RoundTrip. +// RoundTrip sends a query and receives a reply. func (t *DNSOverTCP) RoundTrip(ctx context.Context, query []byte) ([]byte, error) { if len(query) > math.MaxUint16 { return nil, errors.New("query too long") @@ -84,17 +93,17 @@ func (t *DNSOverTCP) RequiresPadding() bool { return t.requiresPadding } -// Network returns the transport network (e.g., doh, dot) +// Network returns the transport network, i.e., "dot" or "tcp". func (t *DNSOverTCP) Network() string { return t.network } -// Address returns the upstream server address. +// Address returns the upstream server endpoint (e.g., "1.1.1.1:853"). func (t *DNSOverTCP) Address() string { return t.address } -// CloseIdleConnections closes idle connections. +// CloseIdleConnections closes idle connections, if any. func (t *DNSOverTCP) CloseIdleConnections() { // nothing to do } diff --git a/internal/netxlite/dnsoverudp.go b/internal/netxlite/dnsoverudp.go index 5eae24e..2d66d4a 100644 --- a/internal/netxlite/dnsoverudp.go +++ b/internal/netxlite/dnsoverudp.go @@ -5,18 +5,24 @@ import ( "time" ) -// DNSOverUDP is a DNS over UDP RoundTripper. +// DNSOverUDP is a DNS-over-UDP DNSTransport. type DNSOverUDP struct { dialer Dialer address string } // NewDNSOverUDP creates a DNSOverUDP instance. +// +// Arguments: +// +// - dialer is any type that implements the Dialer interface; +// +// - address is the endpoint address (e.g., 8.8.8.8:53). func NewDNSOverUDP(dialer Dialer, address string) *DNSOverUDP { return &DNSOverUDP{dialer: dialer, address: address} } -// RoundTrip implements RoundTripper.RoundTrip. +// RoundTrip sends a query and receives a reply. func (t *DNSOverUDP) RoundTrip(ctx context.Context, query []byte) ([]byte, error) { conn, err := t.dialer.DialContext(ctx, "udp", t.address) if err != nil { @@ -40,12 +46,12 @@ func (t *DNSOverUDP) RoundTrip(ctx context.Context, query []byte) ([]byte, error return reply[:n], nil } -// RequiresPadding returns false for UDP according to RFC8467 +// RequiresPadding returns false for UDP according to RFC8467. func (t *DNSOverUDP) RequiresPadding() bool { return false } -// Network returns the transport network (e.g., doh, dot) +// Network returns the transport network, i.e., "udp". func (t *DNSOverUDP) Network() string { return "udp" } @@ -55,7 +61,7 @@ func (t *DNSOverUDP) Address() string { return t.address } -// CloseIdleConnections closes idle connections. +// CloseIdleConnections closes idle connections, if any. func (t *DNSOverUDP) CloseIdleConnections() { // nothing to do } diff --git a/internal/netxlite/dnstransport.go b/internal/netxlite/dnstransport.go index a9cc144..cf7b8d5 100644 --- a/internal/netxlite/dnstransport.go +++ b/internal/netxlite/dnstransport.go @@ -7,15 +7,15 @@ type DNSTransport interface { // RoundTrip sends a DNS query and receives the reply. RoundTrip(ctx context.Context, query []byte) (reply []byte, err error) - // RequiresPadding return true for DoH and DoT according to RFC8467 + // RequiresPadding returns whether this transport needs padding. RequiresPadding() bool - // Network is the network of the round tripper (e.g. "dot") + // Network is the network of the round tripper (e.g. "dot"). Network() string - // Address is the address of the round tripper (e.g. "1.1.1.1:853") + // Address is the address of the round tripper (e.g. "1.1.1.1:853"). Address() string - // CloseIdleConnections closes idle connections. + // CloseIdleConnections closes idle connections, if any. CloseIdleConnections() } diff --git a/internal/netxlite/dnsx/dnsx.go b/internal/netxlite/dnsx/dnsx.go index 25d2640..b78249f 100644 --- a/internal/netxlite/dnsx/dnsx.go +++ b/internal/netxlite/dnsx/dnsx.go @@ -1,14 +1,14 @@ -// Package dnsx contains the dnsx model. +// Package dnsx contains DNS extension types. package dnsx -// HTTPSSvc is an HTTPSSvc reply. +// HTTPSSvc is the reply to an HTTPS DNS query. type HTTPSSvc struct { - // ALPN contains the ALPNs inside the HTTPS reply + // ALPN contains the ALPNs inside the HTTPS reply. ALPN []string - // IPv4 contains the IPv4 hints. + // IPv4 contains the IPv4 hints (which may be empty). IPv4 []string - // IPv6 contains the IPv6 hints. + // IPv6 contains the IPv6 hints (which may be empty). IPv6 []string } diff --git a/internal/netxlite/doc.go b/internal/netxlite/doc.go index db6feb6..2ce9e27 100644 --- a/internal/netxlite/doc.go +++ b/internal/netxlite/doc.go @@ -3,6 +3,9 @@ // This package is the basic networking building block that you // should be using every time you need networking. // +// You should consider checking the tutorial explaining how to use this package +// for network measurements: https://github.com/ooni/probe-cli/tree/master/internal/tutorial/netxlite. +// // Naming and history // // Previous versions of this package were called netx. Compared to such @@ -27,13 +30,16 @@ // We also want to map errors to OONI failures, which are described by // https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md. // +// We want to have reasonable watchdog timeouts for each operation. +// // Operations // // This package implements the following operations: // // 1. establishing a TCP connection; // -// 2. performing a domain name resolution; +// 2. performing a domain name resolution with the "system" resolver +// (i.e., getaddrinfo on Unix) or custom DNS transports (e.g., DoT, DoH); // // 3. performing the TLS handshake; // diff --git a/internal/netxlite/errno.go b/internal/netxlite/errno.go index 29786f3..6a0a936 100644 --- a/internal/netxlite/errno.go +++ b/internal/netxlite/errno.go @@ -1,12 +1,13 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.711301 +0200 CEST m=+0.645971001 +// Generated: 2021-09-29 16:40:32.42792 +0200 CEST m=+0.613746876 package netxlite //go:generate go run ./internal/generrno/ // This enumeration lists the failures defined at -// https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md +// https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md. +// Please, refer to such a document for more information. const ( FailureAddressFamilyNotSupported = "address_family_not_supported" FailureAddressInUse = "address_in_use" diff --git a/internal/netxlite/errno_android.go b/internal/netxlite/errno_android.go index be0bdf1..61dafa9 100644 --- a/internal/netxlite/errno_android.go +++ b/internal/netxlite/errno_android.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.065909 +0200 CEST m=+0.000565085 +// Generated: 2021-09-29 16:40:31.814543 +0200 CEST m=+0.000360918 package netxlite @@ -11,7 +11,9 @@ import ( ) // This enumeration provides a canonical name for -// every system-call error we support on this systems. +// every system-call error we support. Note: this list +// is system dependent. You're currently looking at +// the list of errors for android. const ( ECONNREFUSED = unix.ECONNREFUSED ECONNRESET = unix.ECONNRESET diff --git a/internal/netxlite/errno_android_test.go b/internal/netxlite/errno_android_test.go index a119f5c..e8f89e0 100644 --- a/internal/netxlite/errno_android_test.go +++ b/internal/netxlite/errno_android_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.171735 +0200 CEST m=+0.106392751 +// Generated: 2021-09-29 16:40:31.912358 +0200 CEST m=+0.098177043 package netxlite diff --git a/internal/netxlite/errno_darwin.go b/internal/netxlite/errno_darwin.go index c626dd8..6a8f6d0 100644 --- a/internal/netxlite/errno_darwin.go +++ b/internal/netxlite/errno_darwin.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.208597 +0200 CEST m=+0.143256126 +// Generated: 2021-09-29 16:40:31.947263 +0200 CEST m=+0.133082793 package netxlite @@ -11,7 +11,9 @@ import ( ) // This enumeration provides a canonical name for -// every system-call error we support on this systems. +// every system-call error we support. Note: this list +// is system dependent. You're currently looking at +// the list of errors for darwin. const ( ECONNREFUSED = unix.ECONNREFUSED ECONNRESET = unix.ECONNRESET diff --git a/internal/netxlite/errno_darwin_test.go b/internal/netxlite/errno_darwin_test.go index d940430..49c5cdd 100644 --- a/internal/netxlite/errno_darwin_test.go +++ b/internal/netxlite/errno_darwin_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.27181 +0200 CEST m=+0.206469960 +// Generated: 2021-09-29 16:40:32.010423 +0200 CEST m=+0.196243584 package netxlite diff --git a/internal/netxlite/errno_freebsd.go b/internal/netxlite/errno_freebsd.go index e1c98ca..343c036 100644 --- a/internal/netxlite/errno_freebsd.go +++ b/internal/netxlite/errno_freebsd.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.310236 +0200 CEST m=+0.244897126 +// Generated: 2021-09-29 16:40:32.047214 +0200 CEST m=+0.233035084 package netxlite @@ -11,7 +11,9 @@ import ( ) // This enumeration provides a canonical name for -// every system-call error we support on this systems. +// every system-call error we support. Note: this list +// is system dependent. You're currently looking at +// the list of errors for freebsd. const ( ECONNREFUSED = unix.ECONNREFUSED ECONNRESET = unix.ECONNRESET diff --git a/internal/netxlite/errno_freebsd_test.go b/internal/netxlite/errno_freebsd_test.go index 13cd7bf..ceb927c 100644 --- a/internal/netxlite/errno_freebsd_test.go +++ b/internal/netxlite/errno_freebsd_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.373611 +0200 CEST m=+0.308273168 +// Generated: 2021-09-29 16:40:32.110555 +0200 CEST m=+0.296377043 package netxlite diff --git a/internal/netxlite/errno_ios.go b/internal/netxlite/errno_ios.go index e2eadf0..2ed9fd2 100644 --- a/internal/netxlite/errno_ios.go +++ b/internal/netxlite/errno_ios.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.410358 +0200 CEST m=+0.345021835 +// Generated: 2021-09-29 16:40:32.146781 +0200 CEST m=+0.332603876 package netxlite @@ -11,7 +11,9 @@ import ( ) // This enumeration provides a canonical name for -// every system-call error we support on this systems. +// every system-call error we support. Note: this list +// is system dependent. You're currently looking at +// the list of errors for ios. const ( ECONNREFUSED = unix.ECONNREFUSED ECONNRESET = unix.ECONNRESET diff --git a/internal/netxlite/errno_ios_test.go b/internal/netxlite/errno_ios_test.go index 103134d..a181a7d 100644 --- a/internal/netxlite/errno_ios_test.go +++ b/internal/netxlite/errno_ios_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.491959 +0200 CEST m=+0.426624626 +// Generated: 2021-09-29 16:40:32.210176 +0200 CEST m=+0.395999709 package netxlite diff --git a/internal/netxlite/errno_linux.go b/internal/netxlite/errno_linux.go index 0f1638e..2bf209d 100644 --- a/internal/netxlite/errno_linux.go +++ b/internal/netxlite/errno_linux.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.529823 +0200 CEST m=+0.464489585 +// Generated: 2021-09-29 16:40:32.246508 +0200 CEST m=+0.432331918 package netxlite @@ -11,7 +11,9 @@ import ( ) // This enumeration provides a canonical name for -// every system-call error we support on this systems. +// every system-call error we support. Note: this list +// is system dependent. You're currently looking at +// the list of errors for linux. const ( ECONNREFUSED = unix.ECONNREFUSED ECONNRESET = unix.ECONNRESET diff --git a/internal/netxlite/errno_linux_test.go b/internal/netxlite/errno_linux_test.go index c1c384c..927ba93 100644 --- a/internal/netxlite/errno_linux_test.go +++ b/internal/netxlite/errno_linux_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.592275 +0200 CEST m=+0.526942210 +// Generated: 2021-09-29 16:40:32.309745 +0200 CEST m=+0.495570709 package netxlite diff --git a/internal/netxlite/errno_windows.go b/internal/netxlite/errno_windows.go index 7d23f76..c83a083 100644 --- a/internal/netxlite/errno_windows.go +++ b/internal/netxlite/errno_windows.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.628771 +0200 CEST m=+0.563439293 +// Generated: 2021-09-29 16:40:32.346817 +0200 CEST m=+0.532646709 package netxlite @@ -11,7 +11,9 @@ import ( ) // This enumeration provides a canonical name for -// every system-call error we support on this systems. +// every system-call error we support. Note: this list +// is system dependent. You're currently looking at +// the list of errors for windows. const ( ECONNREFUSED = windows.WSAECONNREFUSED ECONNRESET = windows.WSAECONNRESET diff --git a/internal/netxlite/errno_windows_test.go b/internal/netxlite/errno_windows_test.go index c411eeb..fdc6ae8 100644 --- a/internal/netxlite/errno_windows_test.go +++ b/internal/netxlite/errno_windows_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2021-09-29 10:33:56.675065 +0200 CEST m=+0.609734418 +// Generated: 2021-09-29 16:40:32.393519 +0200 CEST m=+0.579345709 package netxlite diff --git a/internal/netxlite/errwrapper.go b/internal/netxlite/errwrapper.go index e9b4c5c..9d604f8 100644 --- a/internal/netxlite/errwrapper.go +++ b/internal/netxlite/errwrapper.go @@ -53,12 +53,12 @@ type ErrWrapper struct { WrappedErr error } -// Error returns a description of the error that occurred. +// Error returns the OONI failure string for this error. func (e *ErrWrapper) Error() string { return e.Failure } -// Unwrap allows to access the underlying error +// Unwrap allows to access the underlying error. func (e *ErrWrapper) Unwrap() error { return e.WrappedErr } @@ -68,7 +68,9 @@ func (e *ErrWrapper) MarshalJSON() ([]byte, error) { return json.Marshal(e.Failure) } -// Classifier is the type of function that performs classification. +// Classifier is the type of the function that maps a Go error +// to a OONI failure string defined at +// https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md. type Classifier func(err error) string // NewErrWrapper creates a new ErrWrapper using the given @@ -94,7 +96,7 @@ func NewErrWrapper(c Classifier, op string, err error) *ErrWrapper { } // NewTopLevelGenericErrWrapper wraps an error occurring at top -// level using the most generic available classifier. +// level using ClassifyGenericError as classifier. func NewTopLevelGenericErrWrapper(err error) *ErrWrapper { return NewErrWrapper(ClassifyGenericError, TopLevelOperation, err) } diff --git a/internal/netxlite/http.go b/internal/netxlite/http.go index c9fb9c5..8ccbf06 100644 --- a/internal/netxlite/http.go +++ b/internal/netxlite/http.go @@ -72,17 +72,19 @@ func (txp *httpTransportConnectionsCloser) CloseIdleConnections() { txp.TLSDialer.CloseIdleConnections() } -// NewHTTPTransport combines NewOOHTTPBaseTransport and -// WrapHTTPTransport to construct a new HTTPTransport. +// NewHTTPTransport combines NewOOHTTPBaseTransport and WrapHTTPTransport. +// +// This factory and NewHTTPTransportStdlib are the recommended +// ways of creating a new HTTPTransport. func NewHTTPTransport(logger Logger, dialer Dialer, tlsDialer TLSDialer) HTTPTransport { return WrapHTTPTransport(logger, NewOOHTTPBaseTransport(dialer, tlsDialer)) } -// NewOOHTTPBaseTransport creates a new HTTP transport using the given -// dialer and TLS dialer to create connections. +// NewOOHTTPBaseTransport creates an HTTPTransport using the given dialers. // // The returned transport will gracefully handle TLS connections -// created using gitlab.com/yawning/utls.git. +// created using gitlab.com/yawning/utls.git, if the TLS dialer +// is a dialer using such library for TLS operations. // // The returned transport will not have a configured proxy, not // even the proxy configurable from the environment. @@ -100,6 +102,8 @@ func NewHTTPTransport(logger Logger, dialer Dialer, tlsDialer TLSDialer) HTTPTra // necessary to perform sane measurements with tracing. We will be // able to possibly relax this requirement after we change the // way in which we perform measurements. +// +// This is a low level factory. Consider not using it directly. func NewOOHTTPBaseTransport(dialer Dialer, tlsDialer TLSDialer) HTTPTransport { // Using oohttp to support any TLS library. txp := oohttp.DefaultTransport.(*oohttp.Transport).Clone() @@ -137,8 +141,9 @@ func NewOOHTTPBaseTransport(dialer Dialer, tlsDialer TLSDialer) HTTPTransport { } } -// WrapHTTPTransport creates a new HTTP transport using -// the given logger for logging. +// WrapHTTPTransport creates an HTTPTransport using the given logger. +// +// This is a low level factory. Consider not using it directly. func WrapHTTPTransport(logger Logger, txp HTTPTransport) HTTPTransport { return &httpTransportLogger{ HTTPTransport: txp, @@ -168,9 +173,9 @@ type httpTLSDialerWithReadTimeout struct { TLSDialer } -// ErrNotTLSConn indicates that a TLSDialer returns a net.Conn -// that does not implement the TLSConn interface. This error should -// only happen when we do something wrong setting up HTTP code. +// ErrNotTLSConn occur when an interface accepts a net.Conn but +// internally needs a TLSConn and you pass a net.Conn that doesn't +// implement TLSConn to such an interface. var ErrNotTLSConn = errors.New("not a TLSConn") // DialTLSContext implements TLSDialer's DialTLSContext. @@ -233,9 +238,13 @@ func (c *httpTLSConnWithReadTimeout) Read(b []byte) (int, error) { return c.TLSConn.Read(b) } -// NewHTTPTransportStdlib creates a new HTTPTransport that uses -// the Go standard library for all operations, including DNS -// resolutions and TLS handshakes. +// NewHTTPTransportStdlib creates a new HTTPTransport using +// the stdlib for DNS resolutions and TLS. +// +// This factory calls NewHTTPTransport with suitable dialers. +// +// This factory and NewHTTPTransport are the recommended +// ways of creating a new HTTPTransport. func NewHTTPTransportStdlib(logger Logger) HTTPTransport { dialer := NewDialerWithResolver(logger, NewResolverStdlib(logger)) tlsDialer := NewTLSDialer(dialer, NewTLSHandshakerStdlib(logger)) diff --git a/internal/netxlite/internal/generrno/main.go b/internal/netxlite/internal/generrno/main.go index 536754e..e05c6bc 100644 --- a/internal/netxlite/internal/generrno/main.go +++ b/internal/netxlite/internal/generrno/main.go @@ -228,7 +228,9 @@ func writeSystemSpecificFile(system string) { fileWrite(filep, ")\n\n") fileWrite(filep, "// This enumeration provides a canonical name for\n") - fileWrite(filep, "// every system-call error we support on this systems.\n") + fileWrite(filep, "// every system-call error we support. Note: this list\n") + fileWrite(filep, "// is system dependent. You're currently looking at\n") + filePrintf(filep, "// the list of errors for %s.\n", system) fileWrite(filep, "const (\n") for _, spec := range Specs { if !spec.IsSystemError() || !spec.IsForSystem(system) { @@ -272,7 +274,8 @@ func writeGenericFile() { fileWrite(filep, "//go:generate go run ./internal/generrno/\n\n") fileWrite(filep, "// This enumeration lists the failures defined at\n") - fileWrite(filep, "// https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md\n") + fileWrite(filep, "// https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md.\n") + fileWrite(filep, "// Please, refer to that document for more information.\n") fileWrite(filep, "const (\n") names := make(map[string]string) for _, spec := range Specs { diff --git a/internal/netxlite/iox.go b/internal/netxlite/iox.go index a9b8f58..710e1cb 100644 --- a/internal/netxlite/iox.go +++ b/internal/netxlite/iox.go @@ -8,10 +8,11 @@ import ( // ReadAllContext is like io.ReadAll but reads r in a // background goroutine. This function will return // earlier if the context is cancelled. In which case -// we will continue reading from r in the background +// we will continue reading from the reader in the background // goroutine, and we will discard the result. To stop -// the long-running goroutine, you need to close the -// connection bound to the r reader, if possible. +// the long-running goroutine, close the connection +// bound to the reader. Until such a connection is closed, +// you're leaking the backround goroutine and doing I/O. func ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) { datach, errch := make(chan []byte, 1), make(chan error, 1) // buffers go func() { @@ -35,7 +36,7 @@ func ReadAllContext(ctx context.Context, r io.Reader) ([]byte, error) { // CopyContext is like io.Copy but may terminate earlier // when the context expires. This function has the same // caveats of ReadAllContext regarding the temporary leaking -// of the background goroutine used to do I/O. +// of the background I/O goroutine. func CopyContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { countch, errch := make(chan int64, 1), make(chan error, 1) // buffers go func() { diff --git a/internal/netxlite/legacy.go b/internal/netxlite/legacy.go index 9af9092..7e5c0e1 100644 --- a/internal/netxlite/legacy.go +++ b/internal/netxlite/legacy.go @@ -9,6 +9,8 @@ import ( ) // These vars export internal names to legacy ooni/probe-cli code. +// +// Deprecated: do not use these names in new code. var ( DefaultDialer = &dialerSystem{} DefaultTLSHandshaker = defaultTLSHandshaker @@ -17,6 +19,8 @@ var ( ) // These types export internal names to legacy ooni/probe-cli code. +// +// Deprecated: do not use these names in new code. type ( DialerResolver = dialerResolver DialerLogger = dialerLogger @@ -37,8 +41,7 @@ type ( // ResolverLegacy performs domain name resolutions. // -// This definition of Resolver is DEPRECATED. New code should use -// the more complete definition in the new Resolver interface. +// Deprecated: new code should use Resolver. // // Existing code in ooni/probe-cli is still using this definition. type ResolverLegacy interface { @@ -52,10 +55,7 @@ func NewResolverLegacyAdapter(reso ResolverLegacy) Resolver { return &ResolverLegacyAdapter{reso} } -// ResolverLegacyAdapter makes a ResolverLegacy behave like -// it was a Resolver type. If ResolverLegacy is actually also -// a Resolver, this adapter will just forward missing calls, -// otherwise it will implement a sensible default action. +// ResolverLegacyAdapter makes a ResolverLegacy behave like a Resolver. type ResolverLegacyAdapter struct { ResolverLegacy } @@ -97,6 +97,7 @@ func (r *ResolverLegacyAdapter) CloseIdleConnections() { } } +// LookupHTTPS always returns ErrDNSNoTransport. func (r *ResolverLegacyAdapter) LookupHTTPS( ctx context.Context, domain string) (*HTTPSSvc, error) { return nil, ErrNoDNSTransport @@ -104,7 +105,7 @@ func (r *ResolverLegacyAdapter) LookupHTTPS( // DialerLegacy establishes network connections. // -// This definition is DEPRECATED. Please, use Dialer. +// Deprecated: please use Dialer instead. // // Existing code in probe-cli can use it until we // have finished refactoring it. @@ -115,6 +116,8 @@ type DialerLegacy interface { // NewDialerLegacyAdapter adapts a DialerrLegacy to // become compatible with the Dialer definition. +// +// Deprecated: do not use this function in new code. func NewDialerLegacyAdapter(d DialerLegacy) Dialer { return &DialerLegacyAdapter{d} } @@ -133,7 +136,7 @@ type dialerLegacyIdleConnectionsCloser interface { CloseIdleConnections() } -// CloseIdleConnections implements Resolver.CloseIdleConnections. +// CloseIdleConnections implements Dialer.CloseIdleConnections. func (d *DialerLegacyAdapter) CloseIdleConnections() { if ra, ok := d.DialerLegacy.(dialerLegacyIdleConnectionsCloser); ok { ra.CloseIdleConnections() @@ -142,10 +145,10 @@ func (d *DialerLegacyAdapter) CloseIdleConnections() { // QUICContextDialer is a dialer for QUIC using Context. // -// This is a LEGACY name. New code should use QUICDialer directly. +// Deprecated: new code should use QUICDialer. // // Use NewQUICDialerFromContextDialerAdapter if you need to -// adapt an existing QUICContextDialer to a QUICDialer. +// adapt to QUICDialer. type QUICContextDialer interface { // DialContext establishes a new QUIC session using the given // network and address. The tlsConfig and the quicConfig arguments @@ -160,7 +163,7 @@ func NewQUICDialerFromContextDialerAdapter(d QUICContextDialer) QUICDialer { return &QUICContextDialerAdapter{d} } -// QUICContextDialerAdapter adapta a QUICContextDialer to be a QUICDialer. +// QUICContextDialerAdapter adapts a QUICContextDialer to be a QUICDialer. type QUICContextDialerAdapter struct { QUICContextDialer } diff --git a/internal/netxlite/operations.go b/internal/netxlite/operations.go index 256fc03..e675074 100644 --- a/internal/netxlite/operations.go +++ b/internal/netxlite/operations.go @@ -1,6 +1,6 @@ package netxlite -// Operations that we measure. They are the possibly values of +// Operations that we measure. They are the possible values of // the ErrWrapper.Operation field. const ( // ResolveOperation is the operation where we resolve a domain name. diff --git a/internal/netxlite/quic.go b/internal/netxlite/quic.go index 710cd91..25a3670 100644 --- a/internal/netxlite/quic.go +++ b/internal/netxlite/quic.go @@ -12,10 +12,13 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite/quicx" ) +// UDPLikeConn is the kind of UDP socket used by QUIC. +type UDPLikeConn = quicx.UDPLikeConn + // QUICListener listens for QUIC connections. type QUICListener interface { - // Listen creates a new listening UDPConn. - Listen(addr *net.UDPAddr) (quicx.UDPLikeConn, error) + // Listen creates a new listening UDPLikeConn. + Listen(addr *net.UDPAddr) (UDPLikeConn, error) } // NewQUICListener creates a new QUICListener using the standard @@ -30,7 +33,7 @@ type quicListenerStdlib struct{} var _ QUICListener = &quicListenerStdlib{} // Listen implements QUICListener.Listen. -func (qls *quicListenerStdlib) Listen(addr *net.UDPAddr) (quicx.UDPLikeConn, error) { +func (qls *quicListenerStdlib) Listen(addr *net.UDPAddr) (UDPLikeConn, error) { return net.ListenUDP("udp", addr) } @@ -39,6 +42,16 @@ type QUICDialer interface { // DialContext establishes a new QUIC session using the given // network and address. The tlsConfig and the quicConfig arguments // MUST NOT be nil. Returns either the session or an error. + // + // Recommended tlsConfig setup: + // + // - set ServerName to be the SNI; + // + // - set RootCAs to NewDefaultCertPool(); + // + // - set NextProtos to []string{"h3"}. + // + // Typically, you want to pass `&quic.Config{}` as quicConfig. DialContext(ctx context.Context, network, address string, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlySession, error) @@ -66,15 +79,7 @@ type QUICDialer interface { // // 6. if a dialer wraps a resolver, the dialer will forward // the CloseIdleConnection call to its resolver (which is -// instrumental to manage a DoH resolver connections properly); -// -// 7. will use the bundled CA unless you provide another CA; -// -// 8. will attempt to guess SNI when resolving domain names -// and otherwise will not set the SNI; -// -// 9. will attempt to guess ALPN when the port is known and -// otherwise will not set the ALPN. +// instrumental to manage a DoH resolver connections properly). func NewQUICDialerWithResolver(listener QUICListener, logger Logger, resolver Resolver) QUICDialer { return &quicDialerLogger{ @@ -195,7 +200,7 @@ type quicSessionOwnsConn struct { quic.EarlySession // conn is the connection we own - conn quicx.UDPLikeConn + conn UDPLikeConn } // CloseWithError implements quic.EarlySession.CloseWithError. @@ -314,8 +319,7 @@ func (d *quicDialerLogger) CloseIdleConnections() { d.Dialer.CloseIdleConnections() } -// NewSingleUseQUICDialer returns a dialer that returns the given connection -// once and after that always fails with the ErrNoConnReuse error. +// NewSingleUseQUICDialer is like NewSingleUseDialer but for QUIC. func NewSingleUseQUICDialer(sess quic.EarlySession) QUICDialer { return &quicDialerSingleUse{sess: sess} } @@ -356,7 +360,7 @@ type quicListenerErrWrapper struct { var _ QUICListener = &quicListenerErrWrapper{} // Listen implements QUICListener.Listen. -func (qls *quicListenerErrWrapper) Listen(addr *net.UDPAddr) (quicx.UDPLikeConn, error) { +func (qls *quicListenerErrWrapper) Listen(addr *net.UDPAddr) (UDPLikeConn, error) { pconn, err := qls.QUICListener.Listen(addr) if err != nil { return nil, NewErrWrapper(ClassifyGenericError, QUICListenOperation, err) @@ -364,15 +368,15 @@ func (qls *quicListenerErrWrapper) Listen(addr *net.UDPAddr) (quicx.UDPLikeConn, return &quicErrWrapperUDPLikeConn{pconn}, nil } -// quicErrWrapperUDPLikeConn is a quicx.UDPLikeConn that wraps errors. +// quicErrWrapperUDPLikeConn is a UDPLikeConn that wraps errors. type quicErrWrapperUDPLikeConn struct { // UDPLikeConn is the underlying conn. - quicx.UDPLikeConn + UDPLikeConn } -var _ quicx.UDPLikeConn = &quicErrWrapperUDPLikeConn{} +var _ UDPLikeConn = &quicErrWrapperUDPLikeConn{} -// WriteTo implements quicx.UDPLikeConn.WriteTo. +// WriteTo implements UDPLikeConn.WriteTo. func (c *quicErrWrapperUDPLikeConn) WriteTo(p []byte, addr net.Addr) (int, error) { count, err := c.UDPLikeConn.WriteTo(p, addr) if err != nil { @@ -381,7 +385,7 @@ func (c *quicErrWrapperUDPLikeConn) WriteTo(p []byte, addr net.Addr) (int, error return count, nil } -// ReadFrom implements quicx.UDPLikeConn.ReadFrom. +// ReadFrom implements UDPLikeConn.ReadFrom. func (c *quicErrWrapperUDPLikeConn) ReadFrom(b []byte) (int, net.Addr, error) { n, addr, err := c.UDPLikeConn.ReadFrom(b) if err != nil { @@ -390,7 +394,7 @@ func (c *quicErrWrapperUDPLikeConn) ReadFrom(b []byte) (int, net.Addr, error) { return n, addr, nil } -// Close implements quicx.UDPLikeConn.Close. +// Close implements UDPLikeConn.Close. func (c *quicErrWrapperUDPLikeConn) Close() error { err := c.UDPLikeConn.Close() if err != nil { diff --git a/internal/netxlite/quic_test.go b/internal/netxlite/quic_test.go index ce7dd88..3188be6 100644 --- a/internal/netxlite/quic_test.go +++ b/internal/netxlite/quic_test.go @@ -13,7 +13,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/lucas-clemente/quic-go" "github.com/ooni/probe-cli/v3/internal/netxlite/mocks" - "github.com/ooni/probe-cli/v3/internal/netxlite/quicx" ) func TestNewQUICListener(t *testing.T) { @@ -108,7 +107,7 @@ func TestQUICDialerQUICGo(t *testing.T) { } systemdialer := quicDialerQUICGo{ QUICListener: &mocks.QUICListener{ - MockListen: func(addr *net.UDPAddr) (quicx.UDPLikeConn, error) { + MockListen: func(addr *net.UDPAddr) (UDPLikeConn, error) { return nil, expected }, }, @@ -478,7 +477,7 @@ func TestQUICListenerErrWrapper(t *testing.T) { expectedConn := &mocks.QUICUDPLikeConn{} ql := &quicListenerErrWrapper{ QUICListener: &mocks.QUICListener{ - MockListen: func(addr *net.UDPAddr) (quicx.UDPLikeConn, error) { + MockListen: func(addr *net.UDPAddr) (UDPLikeConn, error) { return expectedConn, nil }, }, @@ -497,7 +496,7 @@ func TestQUICListenerErrWrapper(t *testing.T) { expectedErr := io.EOF ql := &quicListenerErrWrapper{ QUICListener: &mocks.QUICListener{ - MockListen: func(addr *net.UDPAddr) (quicx.UDPLikeConn, error) { + MockListen: func(addr *net.UDPAddr) (UDPLikeConn, error) { return nil, expectedErr }, }, diff --git a/internal/netxlite/quicx/quicx.go b/internal/netxlite/quicx/quicx.go index c186164..d8fe06d 100644 --- a/internal/netxlite/quicx/quicx.go +++ b/internal/netxlite/quicx/quicx.go @@ -1,9 +1,4 @@ // Package quicx contains lucas-clemente/quic-go extensions. -// -// This code introduces the UDPLikeConn, whose documentation explain -// why we need to introduce this new type. We could not put this -// code inside an existing package because it's used (as of 20 Aug 2021) -// by the netxlite package as well as by the mocks package. package quicx import ( @@ -17,8 +12,8 @@ import ( // // The QUIC library will treat this connection as a "dumb" // net.PacketConn, calling its ReadFrom and WriteTo methods -// as opposed to more advanced methods that are available -// under Linux and FreeBSD and improve the performance. +// as opposed to more efficient methods that are available +// under Linux and (maybe?) FreeBSD. // // It seems fine to avoid performance optimizations, because // they would complicate the implementation on our side and diff --git a/internal/netxlite/resolver.go b/internal/netxlite/resolver.go index 5c3fbc5..ed93b7f 100644 --- a/internal/netxlite/resolver.go +++ b/internal/netxlite/resolver.go @@ -11,7 +11,7 @@ import ( "golang.org/x/net/idna" ) -// HTTPSSvc is the type returned for HTTPSSvc queries. +// HTTPSSvc is the type returned for HTTPS queries. type HTTPSSvc = dnsx.HTTPSSvc // Resolver performs domain name resolutions. @@ -28,26 +28,31 @@ type Resolver interface { // CloseIdleConnections closes idle connections, if any. CloseIdleConnections() - // LookupHTTPS issues a single HTTPS query for - // a domain without any retry mechanism whatsoever. + // LookupHTTPS issues an HTTPS query for a domain. LookupHTTPS( ctx context.Context, domain string) (*HTTPSSvc, error) } -// ErrNoDNSTransport indicates that the requested Resolver operation -// cannot be performed because we're using the "system" resolver. +// ErrNoDNSTransport is the error returned when you attempt to perform +// a DNS operation that requires a custom DNSTransport (e.g., DNSOverHTTPS) +// but you are using the "system" resolver instead. var ErrNoDNSTransport = errors.New("operation requires a DNS transport") -// NewResolverStdlib creates a new Resolver by combining -// WrapResolver with an internal "system" resolver type that -// adds extra functionality to net.Resolver. +// NewResolverStdlib creates a new Resolver by combining WrapResolver +// with an internal "system" resolver type. func NewResolverStdlib(logger Logger) Resolver { return WrapResolver(logger, &resolverSystem{}) } -// NewResolverUDP creates a new Resolver by combining -// WrapResolver with a SerialResolver attached to -// a DNSOverUDP transport. +// NewResolverUDP creates a new Resolver using DNS-over-UDP. +// +// Arguments: +// +// - logger is the logger to use +// +// - dialer is the dialer to create and connect UDP conns +// +// - address is the server address (e.g., 1.1.1.1:53) func NewResolverUDP(logger Logger, dialer Dialer, address string) Resolver { return WrapResolver(logger, NewSerialResolver( NewDNSOverUDP(dialer, address), @@ -68,6 +73,8 @@ func NewResolverUDP(logger Logger, dialer Dialer, address string) Resolver { // // 5. enforces reasonable timeouts ( // see https://github.com/ooni/probe/issues/1726). +// +// This is a low-level factory. Use only if out of alternatives. func WrapResolver(logger Logger, resolver Resolver) Resolver { return &resolverIDNA{ Resolver: &resolverLogger{ @@ -223,7 +230,9 @@ func (r *resolverShortCircuitIPAddr) LookupHost(ctx context.Context, hostname st return r.Resolver.LookupHost(ctx, hostname) } -// ErrNoResolver indicates you are using a dialer without a resolver. +// ErrNoResolver is the type of error returned by "without resolver" +// dialer when asked to dial for and endpoint containing a domain name, +// since they can only dial for endpoints containing IP addresses. var ErrNoResolver = errors.New("no configured resolver") // nullResolver is a resolver that is not capable of resolving diff --git a/internal/netxlite/serialresolver.go b/internal/netxlite/serialresolver.go index 5b8cab2..90f7723 100644 --- a/internal/netxlite/serialresolver.go +++ b/internal/netxlite/serialresolver.go @@ -9,16 +9,26 @@ import ( "github.com/ooni/probe-cli/v3/internal/atomicx" ) -// SerialResolver is a resolver that first issues an A query and then -// issues an AAAA query for the requested domain. +// SerialResolver uses a transport and sends performs a LookupHost +// operation in a serial fashion (query for A first, wait for response, +// then query for AAAA, and wait for response), hence its name. +// +// You should probably use NewSerialResolver to create a new instance. type SerialResolver struct { - Encoder DNSEncoder - Decoder DNSDecoder + // Encoder is the MANDATORY encoder to use. + Encoder DNSEncoder + + // Decoder is the MANDATORY decoder to use. + Decoder DNSDecoder + + // NumTimeouts is MANDATORY and counts the number of timeouts. NumTimeouts *atomicx.Int64 - Txp DNSTransport + + // Txp is the underlying DNS transport. + Txp DNSTransport } -// NewSerialResolver creates a new OONI Resolver instance. +// NewSerialResolver creates a new SerialResolver instance. func NewSerialResolver(t DNSTransport) *SerialResolver { return &SerialResolver{ Encoder: &DNSEncoderMiekg{}, @@ -33,22 +43,22 @@ func (r *SerialResolver) Transport() DNSTransport { return r.Txp } -// Network implements Resolver.Network +// Network returns the "network" of the underlying transport. func (r *SerialResolver) Network() string { return r.Txp.Network() } -// Address implements Resolver.Address +// Address returns the "address" of the underlying transport. func (r *SerialResolver) Address() string { return r.Txp.Address() } -// CloseIdleConnections closes idle connections. +// CloseIdleConnections closes idle connections, if any. func (r *SerialResolver) CloseIdleConnections() { r.Txp.CloseIdleConnections() } -// LookupHost implements Resolver.LookupHost. +// LookupHost performs an A lookup followed by an AAAA lookup for hostname. func (r *SerialResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) { var addrs []string addrsA, errA := r.lookupHostWithRetry(ctx, hostname, dns.TypeA) diff --git a/internal/netxlite/tls.go b/internal/netxlite/tls.go index f3deab6..01ea167 100644 --- a/internal/netxlite/tls.go +++ b/internal/netxlite/tls.go @@ -51,7 +51,10 @@ var ( } ) -// TLSVersionString returns a TLS version string. +// TLSVersionString returns a TLS version string. If value is zero, we +// return the empty string. If the value is unknown, we return +// `TLS_VERSION_UNKNOWN_ddd` where `ddd` is the numeric value passed +// to this function. func TLSVersionString(value uint16) string { if str, found := tlsVersionString[value]; found { return str @@ -59,7 +62,10 @@ func TLSVersionString(value uint16) string { return fmt.Sprintf("TLS_VERSION_UNKNOWN_%d", value) } -// TLSCipherSuiteString returns the TLS cipher suite as a string. +// TLSCipherSuiteString returns the TLS cipher suite as a string. If value +// is zero, we return the empty string. If we don't know the mapping from +// the value to a cipher suite name, we return `TLS_CIPHER_SUITE_UNKNOWN_ddd` +// where `ddd` is the numeric value passed to this function. func TLSCipherSuiteString(value uint16) string { if str, found := tlsCipherSuiteString[value]; found { return str @@ -67,8 +73,9 @@ func TLSCipherSuiteString(value uint16) string { return fmt.Sprintf("TLS_CIPHER_SUITE_UNKNOWN_%d", value) } -// NewDefaultCertPool returns a copy of the default x509 -// certificate pool that we bundle from Mozilla. +// NewDefaultCertPool returns the default x509 certificate pool +// that we bundle from Mozilla. It's safe to modify the returned +// value: every invocation returns a distinct *x509.CertPool instance. func NewDefaultCertPool() *x509.CertPool { pool := x509.NewCertPool() // Assumption: AppendCertsFromPEM cannot fail because we @@ -82,7 +89,9 @@ func NewDefaultCertPool() *x509.CertPool { var ErrInvalidTLSVersion = errors.New("invalid TLS version") // ConfigureTLSVersion configures the correct TLS version into -// the specified *tls.Config or returns an error. +// a *tls.Config or returns ErrInvalidTLSVersion. +// +// Recognized strings: TLSv1.3, TLSv1.2, TLSv1.1, TLSv1.0. func ConfigureTLSVersion(config *tls.Config, version string) error { switch version { case "TLSv1.3": @@ -106,7 +115,10 @@ func ConfigureTLSVersion(config *tls.Config, version string) error { } // TLSConn is the type of connection that oohttp expects from -// any library that implements TLS functionality. +// any library that implements TLS functionality. By using this +// kind of TLSConn we're able to use both the standard library +// and gitlab.com/yawning/utls.git to perform TLS operations. Note +// that the stdlib's tls.Conn implements this interface. type TLSConn = oohttp.TLSConn // Ensures that a tls.Conn implements the TLSConn interface. @@ -118,15 +130,24 @@ type TLSHandshaker interface { // the given config. This function DOES NOT take ownership of the connection // and it's your responsibility to close it on failure. // + // Recommended tlsConfig setup: + // + // - set ServerName to be the SNI; + // + // - set RootCAs to NewDefaultCertPool(); + // + // - set NextProtos to []string{"h2", "http/1.1"} for HTTPS + // and []string{"dot"} for DNS-over-TLS. + // // QUIRK: The returned connection will always implement the TLSConn interface // exposed by this package. A future version of this interface will instead // return directly a TLSConn to avoid unconditional castings. - Handshake(ctx context.Context, conn net.Conn, config *tls.Config) ( + Handshake(ctx context.Context, conn net.Conn, tlsConfig *tls.Config) ( net.Conn, tls.ConnectionState, error) } // NewTLSHandshakerStdlib creates a new TLS handshaker using the -// go standard library to create TLS connections. +// go standard library to manage TLS. // // The handshaker guarantees: // @@ -235,18 +256,17 @@ type TLSDialer interface { // CloseIdleConnections closes idle connections, if any. CloseIdleConnections() - // DialTLSContext dials a TLS connection. + // DialTLSContext dials a TLS connection. This method will always + // return to you a TLSConn, so you can always safely cast to TLSConn. DialTLSContext(ctx context.Context, network, address string) (net.Conn, error) } -// NewTLSDialer creates a new TLS dialer using the given dialer -// and TLS handshaker to establish TLS connections. +// NewTLSDialer creates a new TLS dialer using the given dialer and handshaker. func NewTLSDialer(dialer Dialer, handshaker TLSHandshaker) TLSDialer { return NewTLSDialerWithConfig(dialer, handshaker, &tls.Config{}) } -// NewTLSDialerWithConfig is like NewTLSDialer but takes an optional config -// parameter containing your desired TLS configuration. +// NewTLSDialerWithConfig is like NewTLSDialer with an optional config. func NewTLSDialerWithConfig(d Dialer, h TLSHandshaker, c *tls.Config) TLSDialer { return &tlsDialer{Config: c, Dialer: d, TLSHandshaker: h} } @@ -351,10 +371,11 @@ func (h *tlsHandshakerErrWrapper) Handshake( return tlsconn, state, nil } -// ErrNoTLSDialer indicates that no TLS dialer is configured. +// ErrNoTLSDialer is the type of error returned by "null" TLS dialers +// when you attempt to dial with them. var ErrNoTLSDialer = errors.New("no configured TLS dialer") -// NewNullTLSDialer returns a TLS dialer that always fails. +// NewNullTLSDialer returns a TLS dialer that always fails with ErrNoTLSDialer. func NewNullTLSDialer() TLSDialer { return &nullTLSDialer{} } diff --git a/internal/netxlite/utls.go b/internal/netxlite/utls.go index 995465c..0239782 100644 --- a/internal/netxlite/utls.go +++ b/internal/netxlite/utls.go @@ -9,8 +9,10 @@ import ( utls "gitlab.com/yawning/utls.git" ) -// NewTLSHandshakerUTLS creates a new TLS handshaker using the -// gitlab.com/yawning/utls library to create TLS conns. +// NewTLSHandshakerUTLS creates a new TLS handshaker using +// gitlab.com/yawning/utls for TLS. +// +// The id is the address of something like utls.HelloFirefox_55. // // The handshaker guarantees: // @@ -51,8 +53,7 @@ func newConnUTLS(clientHello *utls.ClientHelloID) func(conn net.Conn, config *tl // ErrUTLSHandshakePanic indicates that there was panic handshaking // when we were using the yawning/utls library for parroting. -// -// See https://github.com/ooni/probe/issues/1770 +// See https://github.com/ooni/probe/issues/1770 for more information. var ErrUTLSHandshakePanic = errors.New("utls: handshake panic") func (c *utlsConn) HandshakeContext(ctx context.Context) (err error) { diff --git a/internal/tutorial/netxlite/README.md b/internal/tutorial/netxlite/README.md index 6d18425..7cb7685 100644 --- a/internal/tutorial/netxlite/README.md +++ b/internal/tutorial/netxlite/README.md @@ -2,9 +2,34 @@ Netxlite is the underlying networking library we use in OONI. In most cases, network experiments do not use netxlite directly, rather -they use abstractions built on top of netxlite. Though, you need to -know about netxlite if you need to modify these abstractions. +they use abstractions built on top of netxlite (e.g., measurex). +Though, you need to know about netxlite if you need to modify +these abstractions. For this reason, this chapter shows the basic netxlite primitives that we use when writing higher-level measurement primitives. +We will start from simple primitives and we will combine them +together to reach to the point where we can perform GET requests +to websites using already existing TLS or QUIC connections. (The code +we will end up writing will look like a stripped down version of +the measurex library, for which there is a separate tutorial.) + +Index: + +- [chapter01](chapter01) shows how to establish TCP connections; + +- [chapter02](chapter02) covers TLS handshakes; + +- [chapter03](chapter03) discusses TLS parroting; + +- [chapter04](chapter04) shows how to establish QUIC sessions; + +- [chapter05](chapter05) is about the "system" DNS resolver; + +- [chapter06](chapter06) discusses custom DNS-over-UDP resolvers; + +- [chapter07](chapter07) shows how to perform an HTTP GET +using an already existing TLS connection to a website; + +- [chapter08](chapter08) is like chapter07 but for QUIC.