diff --git a/internal/netxlite/resolver.go b/internal/netxlite/resolver.go index 179e98a..5c3fbc5 100644 --- a/internal/netxlite/resolver.go +++ b/internal/netxlite/resolver.go @@ -45,6 +45,15 @@ func NewResolverStdlib(logger Logger) Resolver { return WrapResolver(logger, &resolverSystem{}) } +// NewResolverUDP creates a new Resolver by combining +// WrapResolver with a SerialResolver attached to +// a DNSOverUDP transport. +func NewResolverUDP(logger Logger, dialer Dialer, address string) Resolver { + return WrapResolver(logger, NewSerialResolver( + NewDNSOverUDP(dialer, address), + )) +} + // WrapResolver creates a new resolver that wraps an // existing resolver to add these properties: // diff --git a/internal/netxlite/resolver_test.go b/internal/netxlite/resolver_test.go index 1351465..0e4848e 100644 --- a/internal/netxlite/resolver_test.go +++ b/internal/netxlite/resolver_test.go @@ -26,6 +26,23 @@ func TestNewResolverSystem(t *testing.T) { _ = errWrapper.Resolver.(*resolverSystem) } +func TestNewResolverUDP(t *testing.T) { + d := NewDialerWithoutResolver(log.Log) + resolver := NewResolverUDP(log.Log, d, "1.1.1.1:53") + idna := resolver.(*resolverIDNA) + logger := idna.Resolver.(*resolverLogger) + if logger.Logger != log.Log { + t.Fatal("invalid logger") + } + shortCircuit := logger.Resolver.(*resolverShortCircuitIPAddr) + errWrapper := shortCircuit.Resolver.(*resolverErrWrapper) + serio := errWrapper.Resolver.(*SerialResolver) + txp := serio.Transport().(*DNSOverUDP) + if txp.Address() != "1.1.1.1:53" { + t.Fatal("invalid address") + } +} + func TestResolverSystem(t *testing.T) { t.Run("Network and Address", func(t *testing.T) { r := &resolverSystem{} diff --git a/internal/tutorial/generator/main.go b/internal/tutorial/generator/main.go index 1b82b2c..a7b02ab 100644 --- a/internal/tutorial/generator/main.go +++ b/internal/tutorial/generator/main.go @@ -90,6 +90,19 @@ func gentorsf() { gen(path.Join(prefix, "chapter04"), "torsf.go") } +// gennetxlite generates the netxlite chapters. +func gennetxlite() { + prefix := path.Join(".", "netxlite") + gen(path.Join(prefix, "chapter01"), "main.go") + gen(path.Join(prefix, "chapter02"), "main.go") + gen(path.Join(prefix, "chapter03"), "main.go") + gen(path.Join(prefix, "chapter04"), "main.go") + gen(path.Join(prefix, "chapter05"), "main.go") + gen(path.Join(prefix, "chapter06"), "main.go") + gen(path.Join(prefix, "chapter07"), "main.go") + gen(path.Join(prefix, "chapter08"), "main.go") +} func main() { gentorsf() + gennetxlite() } diff --git a/internal/tutorial/netxlite/README.md b/internal/tutorial/netxlite/README.md new file mode 100644 index 0000000..6d18425 --- /dev/null +++ b/internal/tutorial/netxlite/README.md @@ -0,0 +1,10 @@ +# Tutorial: using the netxlite networking library + +Netxlite is the underlying networking library we use in OONI. In +most cases, network experiments do not use netxlite directly, rather +they use abstractions built on top of netxlite. Though, you need to +know about netxlite if you need to modify these abstractions. + +For this reason, this chapter shows the basic netxlite primitives +that we use when writing higher-level measurement primitives. + diff --git a/internal/tutorial/netxlite/chapter01/README.md b/internal/tutorial/netxlite/chapter01/README.md new file mode 100644 index 0000000..492cebe --- /dev/null +++ b/internal/tutorial/netxlite/chapter01/README.md @@ -0,0 +1,166 @@ + +# Chapter I: establishing TCP connections + +In this chapter we will write together a `main.go` file that +uses netxlite to establish a new TCP connection. + +(This file is auto-generated from the corresponding source file, +so make sure you don't edit it manually.) + +## The main.go file + +We define `main.go` file using `package main`. + +```Go +package main + +import ( + "context" + "errors" + "flag" + "net" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +``` + +### Main function + +```Go +func main() { +``` + +We use apex/log and configure it to emit debug messages. This +setting will allow us to see netxlite emitted logs. + +```Go + log.SetLevel(log.DebugLevel) +``` + +We use the flags package to define command line options and we +parse the command line options with `flag.Parse`. + +```Go + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() +``` + +We use the standard Go idiom to set a timeout using a context. + +```Go + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() +``` + +The bulk of the logic has been factored into a `dialTCP` function. + +```Go + conn, err := dialTCP(ctx, *address) +``` + +If there is a failure we invoke a function that prints the +error that occurred and then calls `os.Exit(1)` + +```Go + if err != nil { + fatal(err) + } +``` + +Otherwise, we're tidy and close the opened connection. + +```Go + conn.Close() +} + +``` + +### Dialing for TCP + +We construct a netxlite.Dialer (i.e., a type similar to net.Dialer) +and we use it to dial the new connection. + +Note that the dialer we're constructing here is not attached to +a resolver. This means that, if `address` contains a domain name +rather than an IP address, the dial operation will fail. + +While it is possible in netxlite to construct a dialer using a +resolver, here we're focusing on the step-by-step measuring perspective +where we want to perform each operation independently. + +```Go +func dialTCP(ctx context.Context, address string) (net.Conn, error) { + d := netxlite.NewDialerWithoutResolver(log.Log) + return d.DialContext(ctx, "tcp", address) +} + +``` + +### Printing the error + +Fundamental netxlite types guarantee that they always return a +`*netxlite.ErrWrapper` type on error. This type is an `error` and +we can use `errors.As` to see its content: + +- the Failure field is the OONI error string as specified in +https://github.com/ooni/spec, and is also the string that +is emitted in case one calls `err.Error`; + +- Operation is the name of the operation that failed; + +- WrappedErr is the underlying error that occurred and has +been wrapped by netxlite. + +```Go +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +``` + +## Running the code + +### Vanilla run + +You can now run this code as follows: + +```bash +go run -race ./internal/tutorial/netxlite/chapter01 +``` + +You will see debug logs describing what is happening along with timing info. + +### Connection timeout + +```bash +go run -race ./internal/tutorial/netxlite/chapter01 -address 8.8.4.4:1 +``` + +should cause a connect timeout error. Try lowering the timout adding, e.g., +the `-timeout 5s` flag to the command line. + +### Connection refused + +```bash +go run -race ./internal/tutorial/netxlite/chapter01 -address '[::1]:1' +``` + +should give you a connection refused error in most cases. (We are quoting +the `::1` IPv6 address using `[` and `]` here.) + +## Conclusions + +We have seen how to use netxlite to establish a TCP connection. diff --git a/internal/tutorial/netxlite/chapter01/main.go b/internal/tutorial/netxlite/chapter01/main.go new file mode 100644 index 0000000..1249149 --- /dev/null +++ b/internal/tutorial/netxlite/chapter01/main.go @@ -0,0 +1,167 @@ +// -=-=- StartHere -=-=- +// +// # Chapter I: establishing TCP connections +// +// In this chapter we will write together a `main.go` file that +// uses netxlite to establish a new TCP connection. +// +// (This file is auto-generated from the corresponding source file, +// so make sure you don't edit it manually.) +// +// ## The main.go file +// +// We define `main.go` file using `package main`. +// +// ```Go +package main + +import ( + "context" + "errors" + "flag" + "net" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// ``` +// +// ### Main function +// +// ```Go +func main() { + // ``` + // + // We use apex/log and configure it to emit debug messages. This + // setting will allow us to see netxlite emitted logs. + // + // ```Go + log.SetLevel(log.DebugLevel) + // ``` + // + // We use the flags package to define command line options and we + // parse the command line options with `flag.Parse`. + // + // ```Go + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + // ``` + // + // We use the standard Go idiom to set a timeout using a context. + // + // ```Go + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + // ``` + // + // The bulk of the logic has been factored into a `dialTCP` function. + // + // ```Go + conn, err := dialTCP(ctx, *address) + // ``` + // + // If there is a failure we invoke a function that prints the + // error that occurred and then calls `os.Exit(1)` + // + // ```Go + if err != nil { + fatal(err) + } + // ``` + // + // Otherwise, we're tidy and close the opened connection. + // + // ```Go + conn.Close() +} + +// ``` +// +// ### Dialing for TCP +// +// We construct a netxlite.Dialer (i.e., a type similar to net.Dialer) +// and we use it to dial the new connection. +// +// Note that the dialer we're constructing here is not attached to +// a resolver. This means that, if `address` contains a domain name +// rather than an IP address, the dial operation will fail. +// +// While it is possible in netxlite to construct a dialer using a +// resolver, here we're focusing on the step-by-step measuring perspective +// where we want to perform each operation independently. +// +// ```Go +func dialTCP(ctx context.Context, address string) (net.Conn, error) { + d := netxlite.NewDialerWithoutResolver(log.Log) + return d.DialContext(ctx, "tcp", address) +} + +// ``` +// +// ### Printing the error +// +// Fundamental netxlite types guarantee that they always return a +// `*netxlite.ErrWrapper` type on error. This type is an `error` and +// we can use `errors.As` to see its content: +// +// - the Failure field is the OONI error string as specified in +// https://github.com/ooni/spec, and is also the string that +// is emitted in case one calls `err.Error`; +// +// - Operation is the name of the operation that failed; +// +// - WrappedErr is the underlying error that occurred and has +// been wrapped by netxlite. +// +// ```Go +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +// ``` +// +// ## Running the code +// +// ### Vanilla run +// +// You can now run this code as follows: +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter01 +// ``` +// +// You will see debug logs describing what is happening along with timing info. +// +// ### Connection timeout +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter01 -address 8.8.4.4:1 +// ``` +// +// should cause a connect timeout error. Try lowering the timout adding, e.g., +// the `-timeout 5s` flag to the command line. +// +// ### Connection refused +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter01 -address '[::1]:1' +// ``` +// +// should give you a connection refused error in most cases. (We are quoting +// the `::1` IPv6 address using `[` and `]` here.) +// +// ## Conclusions +// +// We have seen how to use netxlite to establish a TCP connection. diff --git a/internal/tutorial/netxlite/chapter02/README.md b/internal/tutorial/netxlite/chapter02/README.md new file mode 100644 index 0000000..97a54ec --- /dev/null +++ b/internal/tutorial/netxlite/chapter02/README.md @@ -0,0 +1,254 @@ + +# Chapter I: TLS handshakes + +In this chapter we will write together a `main.go` file that +uses netxlite to establish a new TCP connection and then performs +a TLS handshake using the established connection. + +(This file is auto-generated from the corresponding source file, +so make sure you don't edit it manually.) + +## The main.go file + +We define `main.go` file using `package main`. + +```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "net" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +``` + +### Main function + +```Go +func main() { +``` + +The beginning of main is just like in the previous chapter +except that here we also have a `-sni` flag. + +```Go + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() +``` + +We create a TLS config. In general you always want to specify +these three fields when you're performing handshakes: + +- `ServerName`, which controls the SNI + +- `NextProtos`, which controls the ALPN + +- `RootCAs`, which we are forcing here to be the +CA pool bundled with OONI (so we don't have to trust +the system-wide certificate store) + +```Go + tlsConfig := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h2", "http/1.1"}, + RootCAs: netxlite.NewDefaultCertPool(), + } +``` + +The logic to dial and handshake have been factored +into a function called `dialTLS`. + +```Go + conn, state, err := dialTLS(ctx, *address, tlsConfig) +``` + +If there is an error, we bail, like before. Otherwise we +print information about the established TLS connection, which +is returned by `dialTLS` and assigned to `state`. Finally, +like in the previous chapter, we close the connection. + +```Go + if err != nil { + fatal(err) + } + log.Infof("Conn type : %T", conn) + log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) + log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) + log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version)) + conn.Close() +} + +``` + +### Dialing and handshaking + + +The `dialTCP` function is exactly as in the previous chapter. +```Go + +func dialTCP(ctx context.Context, address string) (net.Conn, error) { + d := netxlite.NewDialerWithoutResolver(log.Log) + return d.DialContext(ctx, "tcp", address) +} + +``` + +The `handshakeTLS` function performs the handshake given a TCP +connection and a TLS config. This function creates a new handshaker +using the stdlib to manage TLS conns (we will see how to use +alternative TLS libraries in the next chapter). Then, once it +has constructed an handshaker, it invokes its `Handshake` method +to obtain a TLS conn (nil on failure), a TLS connection state +(empty on failure), and an error (nil on success). + +While the returned connection is a `net.Conn`, the `Handshake` +function guarantees that the returned connection is always +compatible with the `netxlite.TLSConn` interface. Basically +this interface is an extension of `net.Conn` that also +allows to perform TLS specific operations, such as handshaking +and obtaining the connection state. (We will see in a later +chapter why this guarantee helps when writing more complex code.) + +```Go + +func handshakeTLS(ctx context.Context, tcpConn net.Conn, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + th := netxlite.NewTLSHandshakerStdlib(log.Log) + return th.Handshake(ctx, tcpConn, config) +} + +``` + +Lastly, `dialTLS` combines `dialTCP` and `handshakeTLS` +together. The code you see here is a stripped down version +of the code in the `measurex` library that helps to +perform this dial+handshake operation in a single function call. + +```Go + +func dialTLS(ctx context.Context, address string, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + tcpConn, err := dialTCP(ctx, address) + if err != nil { + return nil, tls.ConnectionState{}, err + } + tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + if err != nil { + tcpConn.Close() + return nil, tls.ConnectionState{}, err + } + return tlsConn, state, nil +} + +``` + +### Printing the error + +This code did not change since the previous chapter. + +```Go + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +``` + +## Running the code + +### Vanilla run + +You can now run this code as follows: + +```bash +go run -race ./internal/tutorial/netxlite/chapter02 +``` + +You will see debug logs describing what is happening along with timing info. + +### Connect timeout + +```bash +go run -race ./internal/tutorial/netxlite/chapter02 -address 8.8.4.4:1 +``` + +should cause a connect timeout error. Try lowering the timout adding, e.g., +the `-timeout 5s` flag to the command line. + +### Connection refused + +```bash +go run -race ./internal/tutorial/netxlite/chapter02 -address '[::1]:1' +``` + +should give you a connection refused error in most cases. (We are quoting +the `::1` IPv6 address using `[` and `]` here.) + +### SNI mismatch + +```bash +go run -race ./internal/tutorial/netxlite/chapter02 -sni example.com +``` + +should give you a TLS invalid hostname error (for historical reasons +named `ssl_invalid_hostname`). + +### TLS handshake reset + +If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +and then run: + +```bash +sudo ./jafar -iptables-reset-keyword dns.google +``` + +Then run in another terminal + +```bash +go run ./internal/tutorial/netxlite/chapter02 +``` + +Then you can interrupt Jafar using ^C. + +### TLS handshake timeout + +If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +and then run: + +```bash +sudo ./jafar -iptables-drop-keyword dns.google +``` + +Then run in another terminal + +```bash +go run ./internal/tutorial/netxlite/chapter02 +``` + +Then you can interrupt Jafar using ^C. + +## Conclusions + +We have seen how to use netxlite to establish a TCP connection +and perform a TLS handshake using such a connection. diff --git a/internal/tutorial/netxlite/chapter02/main.go b/internal/tutorial/netxlite/chapter02/main.go new file mode 100644 index 0000000..f90d925 --- /dev/null +++ b/internal/tutorial/netxlite/chapter02/main.go @@ -0,0 +1,255 @@ +// -=-=- StartHere -=-=- +// +// # Chapter I: TLS handshakes +// +// In this chapter we will write together a `main.go` file that +// uses netxlite to establish a new TCP connection and then performs +// a TLS handshake using the established connection. +// +// (This file is auto-generated from the corresponding source file, +// so make sure you don't edit it manually.) +// +// ## The main.go file +// +// We define `main.go` file using `package main`. +// +// ```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "net" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// ``` +// +// ### Main function +// +// ```Go +func main() { + // ``` + // + // The beginning of main is just like in the previous chapter + // except that here we also have a `-sni` flag. + // + // ```Go + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + // ``` + // + // We create a TLS config. In general you always want to specify + // these three fields when you're performing handshakes: + // + // - `ServerName`, which controls the SNI + // + // - `NextProtos`, which controls the ALPN + // + // - `RootCAs`, which we are forcing here to be the + // CA pool bundled with OONI (so we don't have to trust + // the system-wide certificate store) + // + // ```Go + tlsConfig := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h2", "http/1.1"}, + RootCAs: netxlite.NewDefaultCertPool(), + } + // ``` + // + // The logic to dial and handshake have been factored + // into a function called `dialTLS`. + // + // ```Go + conn, state, err := dialTLS(ctx, *address, tlsConfig) + // ``` + // + // If there is an error, we bail, like before. Otherwise we + // print information about the established TLS connection, which + // is returned by `dialTLS` and assigned to `state`. Finally, + // like in the previous chapter, we close the connection. + // + // ```Go + if err != nil { + fatal(err) + } + log.Infof("Conn type : %T", conn) + log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) + log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) + log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version)) + conn.Close() +} + +// ``` +// +// ### Dialing and handshaking +// +// +// The `dialTCP` function is exactly as in the previous chapter. +// ```Go + +func dialTCP(ctx context.Context, address string) (net.Conn, error) { + d := netxlite.NewDialerWithoutResolver(log.Log) + return d.DialContext(ctx, "tcp", address) +} + +// ``` +// +// The `handshakeTLS` function performs the handshake given a TCP +// connection and a TLS config. This function creates a new handshaker +// using the stdlib to manage TLS conns (we will see how to use +// alternative TLS libraries in the next chapter). Then, once it +// has constructed an handshaker, it invokes its `Handshake` method +// to obtain a TLS conn (nil on failure), a TLS connection state +// (empty on failure), and an error (nil on success). +// +// While the returned connection is a `net.Conn`, the `Handshake` +// function guarantees that the returned connection is always +// compatible with the `netxlite.TLSConn` interface. Basically +// this interface is an extension of `net.Conn` that also +// allows to perform TLS specific operations, such as handshaking +// and obtaining the connection state. (We will see in a later +// chapter why this guarantee helps when writing more complex code.) +// +// ```Go + +func handshakeTLS(ctx context.Context, tcpConn net.Conn, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + th := netxlite.NewTLSHandshakerStdlib(log.Log) + return th.Handshake(ctx, tcpConn, config) +} + +// ``` +// +// Lastly, `dialTLS` combines `dialTCP` and `handshakeTLS` +// together. The code you see here is a stripped down version +// of the code in the `measurex` library that helps to +// perform this dial+handshake operation in a single function call. +// +// ```Go + +func dialTLS(ctx context.Context, address string, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + tcpConn, err := dialTCP(ctx, address) + if err != nil { + return nil, tls.ConnectionState{}, err + } + tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + if err != nil { + tcpConn.Close() + return nil, tls.ConnectionState{}, err + } + return tlsConn, state, nil +} + +// ``` +// +// ### Printing the error +// +// This code did not change since the previous chapter. +// +// ```Go + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +// ``` +// +// ## Running the code +// +// ### Vanilla run +// +// You can now run this code as follows: +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter02 +// ``` +// +// You will see debug logs describing what is happening along with timing info. +// +// ### Connect timeout +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter02 -address 8.8.4.4:1 +// ``` +// +// should cause a connect timeout error. Try lowering the timout adding, e.g., +// the `-timeout 5s` flag to the command line. +// +// ### Connection refused +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter02 -address '[::1]:1' +// ``` +// +// should give you a connection refused error in most cases. (We are quoting +// the `::1` IPv6 address using `[` and `]` here.) +// +// ### SNI mismatch +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter02 -sni example.com +// ``` +// +// should give you a TLS invalid hostname error (for historical reasons +// named `ssl_invalid_hostname`). +// +// ### TLS handshake reset +// +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// and then run: +// +// ```bash +// sudo ./jafar -iptables-reset-keyword dns.google +// ``` +// +// Then run in another terminal +// +// ```bash +// go run ./internal/tutorial/netxlite/chapter02 +// ``` +// +// Then you can interrupt Jafar using ^C. +// +// ### TLS handshake timeout +// +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// and then run: +// +// ```bash +// sudo ./jafar -iptables-drop-keyword dns.google +// ``` +// +// Then run in another terminal +// +// ```bash +// go run ./internal/tutorial/netxlite/chapter02 +// ``` +// +// Then you can interrupt Jafar using ^C. +// +// ## Conclusions +// +// We have seen how to use netxlite to establish a TCP connection +// and perform a TLS handshake using such a connection. diff --git a/internal/tutorial/netxlite/chapter03/README.md b/internal/tutorial/netxlite/chapter03/README.md new file mode 100644 index 0000000..fc45a0f --- /dev/null +++ b/internal/tutorial/netxlite/chapter03/README.md @@ -0,0 +1,197 @@ + +# Chapter I: TLS parroting + +In this chapter we will write together a `main.go` file that +uses netxlite to establish a new TCP connection and then performs +a TLS handshake using the established connection. + +Rather than using the Go standard library, like we did in the +previous chapter, we will use the `gitlab.com/yawning/utls.git` +library to customize the ClientHello to look like Firefox. + +(This file is auto-generated from the corresponding source file, +so make sure you don't edit it manually.) + +## The main.go file + +We define `main.go` file using `package main`. + +The beginning of the program is equal to the previous chapter, +so there is not much to say about it. + +```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "net" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" + utls "gitlab.com/yawning/utls.git" +) + +func main() { + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + tlsConfig := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h2", "http/1.1"}, + RootCAs: netxlite.NewDefaultCertPool(), + } + conn, state, err := dialTLS(ctx, *address, tlsConfig) + if err != nil { + fatal(err) + } + log.Infof("Conn type : %T", conn) + log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) + log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) + log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version)) + conn.Close() +} + +func dialTCP(ctx context.Context, address string) (net.Conn, error) { + d := netxlite.NewDialerWithoutResolver(log.Log) + return d.DialContext(ctx, "tcp", address) +} + +func handshakeTLS(ctx context.Context, tcpConn net.Conn, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { +``` + +The following line of code is where we diverge from the +previous chapter. Here we're creating a TLS handshaker +that uses `gitlab.com/yawning/utls.git` and sets the +ClientHello to look like Firefox 55. (This is also +know as TLS parroting because we're parroting what this +version of Firefox would do.) + +Note that, when you use parroting, some settings inside +the `tls.Config` (such as the ALPN) may be ignored +if they conflict with what the parroted browser would do. + +```Go + th := netxlite.NewTLSHandshakerUTLS(log.Log, &utls.HelloFirefox_55) +``` + +The rest of the program is exactly like the one in the +previous chapter, so we won't add further comments. + +```Go + return th.Handshake(ctx, tcpConn, config) +} + +func dialTLS(ctx context.Context, address string, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + tcpConn, err := dialTCP(ctx, address) + if err != nil { + return nil, tls.ConnectionState{}, err + } + tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + if err != nil { + tcpConn.Close() + return nil, tls.ConnectionState{}, err + } + return tlsConn, state, nil +} + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +``` + +## Running the code + +You can now run this code as follows: + +```bash +go run -race ./internal/tutorial/netxlite/chapter03 +``` + +You will see debug logs describing what is happening along with timing info. + +### Connect timeout + +```bash +go run -race ./internal/tutorial/netxlite/chapter03 -address 8.8.4.4:1 +``` + +should cause a connect timeout error. Try lowering the timout adding, e.g., +the `-timeout 5s` flag to the command line. + +### Connection refused + +```bash +go run -race ./internal/tutorial/netxlite/chapter03 -address '[::1]:1' +``` + +should give you a connection refused error in most cases. (We are quoting +the `::1` IPv6 address using `[` and `]` here.) + +### SNI mismatch + +```bash +go run -race ./internal/tutorial/netxlite/chapter03 -sni example.com +``` + +should give you a TLS invalid hostname error (for historical reasons +named `ssl_invalid_hostname`). + +### TLS handshake reset + +If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +and then run: + +```bash +sudo ./jafar -iptables-reset-keyword dns.google +``` + +Then run in another terminal + +```bash +go run ./internal/tutorial/netxlite/chapter03 +``` + +Then you can interrupt Jafar using ^C. + +### TLS handshake timeout + +If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +and then run: + +```bash +sudo ./jafar -iptables-drop-keyword dns.google +``` + +Then run in another terminal + +```bash +go run ./internal/tutorial/netxlite/chapter03 +``` + +Then you can interrupt Jafar using ^C. + +## Conclusions + +We have seen how to use netxlite to establish a TCP connection +and perform a TLS handshake using such a connection with a specific +configuration that parrots Firefox v55's ClientHello. diff --git a/internal/tutorial/netxlite/chapter03/main.go b/internal/tutorial/netxlite/chapter03/main.go new file mode 100644 index 0000000..010cb8f --- /dev/null +++ b/internal/tutorial/netxlite/chapter03/main.go @@ -0,0 +1,198 @@ +// -=-=- StartHere -=-=- +// +// # Chapter I: TLS parroting +// +// In this chapter we will write together a `main.go` file that +// uses netxlite to establish a new TCP connection and then performs +// a TLS handshake using the established connection. +// +// Rather than using the Go standard library, like we did in the +// previous chapter, we will use the `gitlab.com/yawning/utls.git` +// library to customize the ClientHello to look like Firefox. +// +// (This file is auto-generated from the corresponding source file, +// so make sure you don't edit it manually.) +// +// ## The main.go file +// +// We define `main.go` file using `package main`. +// +// The beginning of the program is equal to the previous chapter, +// so there is not much to say about it. +// +// ```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "net" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" + utls "gitlab.com/yawning/utls.git" +) + +func main() { + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + tlsConfig := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h2", "http/1.1"}, + RootCAs: netxlite.NewDefaultCertPool(), + } + conn, state, err := dialTLS(ctx, *address, tlsConfig) + if err != nil { + fatal(err) + } + log.Infof("Conn type : %T", conn) + log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) + log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) + log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version)) + conn.Close() +} + +func dialTCP(ctx context.Context, address string) (net.Conn, error) { + d := netxlite.NewDialerWithoutResolver(log.Log) + return d.DialContext(ctx, "tcp", address) +} + +func handshakeTLS(ctx context.Context, tcpConn net.Conn, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + // ``` + // + // The following line of code is where we diverge from the + // previous chapter. Here we're creating a TLS handshaker + // that uses `gitlab.com/yawning/utls.git` and sets the + // ClientHello to look like Firefox 55. (This is also + // know as TLS parroting because we're parroting what this + // version of Firefox would do.) + // + // Note that, when you use parroting, some settings inside + // the `tls.Config` (such as the ALPN) may be ignored + // if they conflict with what the parroted browser would do. + // + // ```Go + th := netxlite.NewTLSHandshakerUTLS(log.Log, &utls.HelloFirefox_55) + // ``` + // + // The rest of the program is exactly like the one in the + // previous chapter, so we won't add further comments. + // + // ```Go + return th.Handshake(ctx, tcpConn, config) +} + +func dialTLS(ctx context.Context, address string, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + tcpConn, err := dialTCP(ctx, address) + if err != nil { + return nil, tls.ConnectionState{}, err + } + tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + if err != nil { + tcpConn.Close() + return nil, tls.ConnectionState{}, err + } + return tlsConn, state, nil +} + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +// ``` +// +// ## Running the code +// +// You can now run this code as follows: +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter03 +// ``` +// +// You will see debug logs describing what is happening along with timing info. +// +// ### Connect timeout +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter03 -address 8.8.4.4:1 +// ``` +// +// should cause a connect timeout error. Try lowering the timout adding, e.g., +// the `-timeout 5s` flag to the command line. +// +// ### Connection refused +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter03 -address '[::1]:1' +// ``` +// +// should give you a connection refused error in most cases. (We are quoting +// the `::1` IPv6 address using `[` and `]` here.) +// +// ### SNI mismatch +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter03 -sni example.com +// ``` +// +// should give you a TLS invalid hostname error (for historical reasons +// named `ssl_invalid_hostname`). +// +// ### TLS handshake reset +// +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// and then run: +// +// ```bash +// sudo ./jafar -iptables-reset-keyword dns.google +// ``` +// +// Then run in another terminal +// +// ```bash +// go run ./internal/tutorial/netxlite/chapter03 +// ``` +// +// Then you can interrupt Jafar using ^C. +// +// ### TLS handshake timeout +// +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// and then run: +// +// ```bash +// sudo ./jafar -iptables-drop-keyword dns.google +// ``` +// +// Then run in another terminal +// +// ```bash +// go run ./internal/tutorial/netxlite/chapter03 +// ``` +// +// Then you can interrupt Jafar using ^C. +// +// ## Conclusions +// +// We have seen how to use netxlite to establish a TCP connection +// and perform a TLS handshake using such a connection with a specific +// configuration that parrots Firefox v55's ClientHello. diff --git a/internal/tutorial/netxlite/chapter04/README.md b/internal/tutorial/netxlite/chapter04/README.md new file mode 100644 index 0000000..53af8f2 --- /dev/null +++ b/internal/tutorial/netxlite/chapter04/README.md @@ -0,0 +1,161 @@ + +# Chapter I: Using QUIC + +In this chapter we will write together a `main.go` file that +uses netxlite to establish a new QUIC session with an UDP endpoint. + +Conceptually, this program is very similar to the ones presented +in chapters 2 and 3, except that here we use QUIC. + +(This file is auto-generated from the corresponding source file, +so make sure you don't edit it manually.) + +## The main.go file + +We define `main.go` file using `package main`. + +The beginning of the program is equal to the previous chapters, +so there is not much to say about it. + +```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "os" + "time" + + "github.com/apex/log" + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func main() { + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() +``` + +The main difference is that we set the ALPN correctly for +QUIC/HTTP3 by using `"h3"` here. + +```Go + config := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h3"}, + RootCAs: netxlite.NewDefaultCertPool(), + } +``` + +Also, where previously we called `dialTLS` now we call +a function with a similar API called `dialQUIC`. + +``` + sess, state, err := dialQUIC(ctx, *address, config) +``` + +The rest of the main function is pretty much the same. + +```Go + if err != nil { + fatal(err) + } + log.Infof("Sess type : %T", sess) + log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) + log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) + log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version)) + sess.CloseWithError(0, "") +} + +``` + +The dialQUIC function is new. We need to create a QUIC listener +and, using it, a QUICDialer. These two steps are separated so +higher level code can wrap the QUICDialer and collect stats on +the returned connections. Also, as previously, this dialer is +not attached to a resolver, so it will fail if provided a domain +name. The rationale for doing that is similar to before: we +are focusing on step-by-step measurements where each operation +is performed independently. (That is, we assume that before +the code written in this main we have already resolved the +domain name of interest using a resolver, which we will investigate +in the next two chapters.) + +```Go +func dialQUIC(ctx context.Context, address string, + config *tls.Config) (quic.EarlySession, tls.ConnectionState, error) { + ql := netxlite.NewQUICListener() + d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) + sess, err := d.DialContext(ctx, "udp", address, config, &quic.Config{}) + if err != nil { + return nil, tls.ConnectionState{}, err + } +``` + +The following line unwraps the connection state returned by +QUIC code to be of the same type of the ConnectionState that +we returned in the previous chapters. + +```Go + return sess, sess.ConnectionState().TLS.ConnectionState, nil +} + +``` + +The rest of the program is equal to the previous chapters. + +```Go +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +``` + +## Running the code + +### Vanilla run + +You can now run this code as follows: + +```bash +go run -race ./internal/tutorial/netxlite/chapter04 +``` + +You will see debug logs describing what is happening along with timing info. + +### QUIC handshake timeout + +```bash +go run -race ./internal/tutorial/netxlite/chapter04 -address 8.8.4.4:1 +``` + +should cause a QUIC timeout error. Try lowering the timout adding, e.g., +the `-timeout 5s` flag to the command line. + +### SNI mismatch + +```bash +go run -race ./internal/tutorial/netxlite/chapter04 -sni example.com +``` + +should give you a TLS error mentioning that the certificate is invalid. + +## Conclusions + +We have seen how to use netxlite to establish a QUIC session +with a remote UDP endpoint speaking QUIC. diff --git a/internal/tutorial/netxlite/chapter04/main.go b/internal/tutorial/netxlite/chapter04/main.go new file mode 100644 index 0000000..15b8f84 --- /dev/null +++ b/internal/tutorial/netxlite/chapter04/main.go @@ -0,0 +1,162 @@ +// -=-=- StartHere -=-=- +// +// # Chapter I: Using QUIC +// +// In this chapter we will write together a `main.go` file that +// uses netxlite to establish a new QUIC session with an UDP endpoint. +// +// Conceptually, this program is very similar to the ones presented +// in chapters 2 and 3, except that here we use QUIC. +// +// (This file is auto-generated from the corresponding source file, +// so make sure you don't edit it manually.) +// +// ## The main.go file +// +// We define `main.go` file using `package main`. +// +// The beginning of the program is equal to the previous chapters, +// so there is not much to say about it. +// +// ```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "os" + "time" + + "github.com/apex/log" + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func main() { + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + // ``` + // + // The main difference is that we set the ALPN correctly for + // QUIC/HTTP3 by using `"h3"` here. + // + // ```Go + config := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h3"}, + RootCAs: netxlite.NewDefaultCertPool(), + } + // ``` + // + // Also, where previously we called `dialTLS` now we call + // a function with a similar API called `dialQUIC`. + // + // ``` + sess, state, err := dialQUIC(ctx, *address, config) + // ``` + // + // The rest of the main function is pretty much the same. + // + // ```Go + if err != nil { + fatal(err) + } + log.Infof("Sess type : %T", sess) + log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) + log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) + log.Infof("TLS version : %s", netxlite.TLSVersionString(state.Version)) + sess.CloseWithError(0, "") +} + +// ``` +// +// The dialQUIC function is new. We need to create a QUIC listener +// and, using it, a QUICDialer. These two steps are separated so +// higher level code can wrap the QUICDialer and collect stats on +// the returned connections. Also, as previously, this dialer is +// not attached to a resolver, so it will fail if provided a domain +// name. The rationale for doing that is similar to before: we +// are focusing on step-by-step measurements where each operation +// is performed independently. (That is, we assume that before +// the code written in this main we have already resolved the +// domain name of interest using a resolver, which we will investigate +// in the next two chapters.) +// +// ```Go +func dialQUIC(ctx context.Context, address string, + config *tls.Config) (quic.EarlySession, tls.ConnectionState, error) { + ql := netxlite.NewQUICListener() + d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) + sess, err := d.DialContext(ctx, "udp", address, config, &quic.Config{}) + if err != nil { + return nil, tls.ConnectionState{}, err + } + // ``` + // + // The following line unwraps the connection state returned by + // QUIC code to be of the same type of the ConnectionState that + // we returned in the previous chapters. + // + // ```Go + return sess, sess.ConnectionState().TLS.ConnectionState, nil +} + +// ``` +// +// The rest of the program is equal to the previous chapters. +// +// ```Go +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +// ``` +// +// ## Running the code +// +// ### Vanilla run +// +// You can now run this code as follows: +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter04 +// ``` +// +// You will see debug logs describing what is happening along with timing info. +// +// ### QUIC handshake timeout +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter04 -address 8.8.4.4:1 +// ``` +// +// should cause a QUIC timeout error. Try lowering the timout adding, e.g., +// the `-timeout 5s` flag to the command line. +// +// ### SNI mismatch +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter04 -sni example.com +// ``` +// +// should give you a TLS error mentioning that the certificate is invalid. +// +// ## Conclusions +// +// We have seen how to use netxlite to establish a QUIC session +// with a remote UDP endpoint speaking QUIC. diff --git a/internal/tutorial/netxlite/chapter05/README.md b/internal/tutorial/netxlite/chapter05/README.md new file mode 100644 index 0000000..ef0a106 --- /dev/null +++ b/internal/tutorial/netxlite/chapter05/README.md @@ -0,0 +1,117 @@ + +# Chapter I: Using the "system" DNS resolver + +In this chapter we will write together a `main.go` file that +uses the "system" DNS resolver to lookup domain names. + +The "system" resolver is the one used by `getaddrinfo` on Unix. + +(This file is auto-generated from the corresponding source file, +so make sure you don't edit it manually.) + +## The main.go file + +We define `main.go` file using `package main`. + +```Go +package main + +import ( + "context" + "errors" + "flag" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func main() { +``` + +The beginning of the program is equal to the previous chapters, +so there is not much to say about it. + +```Go + log.SetLevel(log.DebugLevel) + hostname := flag.String("hostname", "dns.google", "Hostname to resolve") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() +``` + +We create a new resolver using the standard library to perform +domain name resolutions. Unless you're cross compiling, this +resolver will call the system resolver using a C API. On Unix +the called C API is `getaddrinfo`. + +The returned resolver implements an interface that is very +close to the API of the `net.Resolver` struct. + +```Go + reso := netxlite.NewResolverStdlib(log.Log) +``` + +We call `LookupHost` to map the hostname to IP addrs. The returned +value is either a list of addrs or an error. + +```Go + addrs, err := reso.LookupHost(ctx, *hostname) + if err != nil { + fatal(err) + } + log.Infof("resolver addrs: %+v", addrs) +} + +``` + +This function is exactly like it was in previous chapters. + +```Go +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +``` + +## Running the code + +### Vanilla run + +You can now run this code as follows: + +```bash +go run -race ./internal/tutorial/netxlite/chapter05 +``` + +You will see debug logs describing what is happening along with timing info. + +### NXDOMAIN error + +```bash +go run -race ./internal/tutorial/netxlite/chapter05 -hostname antani.ooni.io +``` + +should cause a `dns_nxdomain_error`, because the domain does not exist. + +### Timeout + +```bash +go run -race ./internal/tutorial/netxlite/chapter05 -timeout 10us +``` + +should cause a timeout error, because the timeout is ridicolously small. + +## Conclusions + +We have seen how to use the "system" DNS resolver. diff --git a/internal/tutorial/netxlite/chapter05/main.go b/internal/tutorial/netxlite/chapter05/main.go new file mode 100644 index 0000000..b4ff34c --- /dev/null +++ b/internal/tutorial/netxlite/chapter05/main.go @@ -0,0 +1,118 @@ +// -=-=- StartHere -=-=- +// +// # Chapter I: Using the "system" DNS resolver +// +// In this chapter we will write together a `main.go` file that +// uses the "system" DNS resolver to lookup domain names. +// +// The "system" resolver is the one used by `getaddrinfo` on Unix. +// +// (This file is auto-generated from the corresponding source file, +// so make sure you don't edit it manually.) +// +// ## The main.go file +// +// We define `main.go` file using `package main`. +// +// ```Go +package main + +import ( + "context" + "errors" + "flag" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func main() { + // ``` + // + // The beginning of the program is equal to the previous chapters, + // so there is not much to say about it. + // + // ```Go + log.SetLevel(log.DebugLevel) + hostname := flag.String("hostname", "dns.google", "Hostname to resolve") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + // ``` + // + // We create a new resolver using the standard library to perform + // domain name resolutions. Unless you're cross compiling, this + // resolver will call the system resolver using a C API. On Unix + // the called C API is `getaddrinfo`. + // + // The returned resolver implements an interface that is very + // close to the API of the `net.Resolver` struct. + // + // ```Go + reso := netxlite.NewResolverStdlib(log.Log) + // ``` + // + // We call `LookupHost` to map the hostname to IP addrs. The returned + // value is either a list of addrs or an error. + // + // ```Go + addrs, err := reso.LookupHost(ctx, *hostname) + if err != nil { + fatal(err) + } + log.Infof("resolver addrs: %+v", addrs) +} + +// ``` +// +// This function is exactly like it was in previous chapters. +// +// ```Go +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +// ``` +// +// ## Running the code +// +// ### Vanilla run +// +// You can now run this code as follows: +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter05 +// ``` +// +// You will see debug logs describing what is happening along with timing info. +// +// ### NXDOMAIN error +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter05 -hostname antani.ooni.io +// ``` +// +// should cause a `dns_nxdomain_error`, because the domain does not exist. +// +// ### Timeout +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter05 -timeout 10us +// ``` +// +// should cause a timeout error, because the timeout is ridicolously small. +// +// ## Conclusions +// +// We have seen how to use the "system" DNS resolver. diff --git a/internal/tutorial/netxlite/chapter06/README.md b/internal/tutorial/netxlite/chapter06/README.md new file mode 100644 index 0000000..bde30e1 --- /dev/null +++ b/internal/tutorial/netxlite/chapter06/README.md @@ -0,0 +1,121 @@ + +# Chapter I: Using a custom UDP resolver + +In this chapter we will write together a `main.go` file that +uses a custom UDP DNS resolver to lookup domain names. + +This program is very similar to the one in the previous chapter +except that we'll be configuring a custom resolver. + +(This file is auto-generated from the corresponding source file, +so make sure you don't edit it manually.) + +## The main.go file + +We define `main.go` file using `package main`. + +There's not much to say about the beginning of the program +since it is equal to the one in the previous chapter. + +```Go +package main + +import ( + "context" + "errors" + "flag" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func main() { + log.SetLevel(log.DebugLevel) + hostname := flag.String("hostname", "dns.google", "Hostname to resolve") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + serverAddr := flag.String("server-addr", "1.1.1.1:53", "DNS server address") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() +``` + +Here's where we start to diverge. We create a dialer without a resolver, +which is going to be used by the UDP resolver. + +```Go + dialer := netxlite.NewDialerWithoutResolver(log.Log) +``` + +Then, we create an UDP resolver. The arguments are the same as for +creating a system resolver, except that we also need to specify the +UDP endpoint address at which the server is listening. + +```Go + reso := netxlite.NewResolverUDP(log.Log, dialer, *serverAddr) +``` + +The API we invoke is the same as in the previous chapter, though, +and the rest of the program is equal to the one in the previous chapter. + +```Go + addrs, err := reso.LookupHost(ctx, *hostname) + if err != nil { + fatal(err) + } + log.Infof("resolver addrs: %+v", addrs) +} + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +``` + +## Running the code + +### Vanilla run + +You can now run this code as follows: + +```bash +go run -race ./internal/tutorial/netxlite/chapter06 +``` + +You will see debug logs describing what is happening along with timing info. + +### NXDOMAIN + +```bash +go run -race ./internal/tutorial/netxlite/chapter06 -hostname antani.ooni.io +``` + +should cause a `dns_nxdomain_error`, because the domain does not exist. + +### Timeout + +```bash +go run -race ./internal/tutorial/netxlite/chapter06 -timeout 10us +``` + +should cause a timeout error, because the timeout is ridicolously small. + +```bash +go run -race ./internal/tutorial/netxlite/chapter06 -server-addr 1.1.1.1:1 +``` + +should also cause a timeout, because 1.1.1.1:1 is not an endpoint +where a DNS-over-UDP resolver is listening. + +## Conclusions + +We have seen how to use a custom DNS-over-UDP resolver. diff --git a/internal/tutorial/netxlite/chapter06/main.go b/internal/tutorial/netxlite/chapter06/main.go new file mode 100644 index 0000000..65745df --- /dev/null +++ b/internal/tutorial/netxlite/chapter06/main.go @@ -0,0 +1,122 @@ +// -=-=- StartHere -=-=- +// +// # Chapter I: Using a custom UDP resolver +// +// In this chapter we will write together a `main.go` file that +// uses a custom UDP DNS resolver to lookup domain names. +// +// This program is very similar to the one in the previous chapter +// except that we'll be configuring a custom resolver. +// +// (This file is auto-generated from the corresponding source file, +// so make sure you don't edit it manually.) +// +// ## The main.go file +// +// We define `main.go` file using `package main`. +// +// There's not much to say about the beginning of the program +// since it is equal to the one in the previous chapter. +// +// ```Go +package main + +import ( + "context" + "errors" + "flag" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func main() { + log.SetLevel(log.DebugLevel) + hostname := flag.String("hostname", "dns.google", "Hostname to resolve") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + serverAddr := flag.String("server-addr", "1.1.1.1:53", "DNS server address") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + // ``` + // + // Here's where we start to diverge. We create a dialer without a resolver, + // which is going to be used by the UDP resolver. + // + // ```Go + dialer := netxlite.NewDialerWithoutResolver(log.Log) + // ``` + // + // Then, we create an UDP resolver. The arguments are the same as for + // creating a system resolver, except that we also need to specify the + // UDP endpoint address at which the server is listening. + // + // ```Go + reso := netxlite.NewResolverUDP(log.Log, dialer, *serverAddr) + // ``` + // + // The API we invoke is the same as in the previous chapter, though, + // and the rest of the program is equal to the one in the previous chapter. + // + // ```Go + addrs, err := reso.LookupHost(ctx, *hostname) + if err != nil { + fatal(err) + } + log.Infof("resolver addrs: %+v", addrs) +} + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +// ``` +// +// ## Running the code +// +// ### Vanilla run +// +// You can now run this code as follows: +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter06 +// ``` +// +// You will see debug logs describing what is happening along with timing info. +// +// ### NXDOMAIN +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter06 -hostname antani.ooni.io +// ``` +// +// should cause a `dns_nxdomain_error`, because the domain does not exist. +// +// ### Timeout +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter06 -timeout 10us +// ``` +// +// should cause a timeout error, because the timeout is ridicolously small. +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter06 -server-addr 1.1.1.1:1 +// ``` +// +// should also cause a timeout, because 1.1.1.1:1 is not an endpoint +// where a DNS-over-UDP resolver is listening. +// +// ## Conclusions +// +// We have seen how to use a custom DNS-over-UDP resolver. diff --git a/internal/tutorial/netxlite/chapter07/README.md b/internal/tutorial/netxlite/chapter07/README.md new file mode 100644 index 0000000..7e680a7 --- /dev/null +++ b/internal/tutorial/netxlite/chapter07/README.md @@ -0,0 +1,195 @@ + +# Chapter I: HTTP GET with TLS conn + +In this chapter we will write together a `main.go` file that +uses netxlite to establish a TLS connection to a remote endpoint +and then fetches a webpage from it using GET. + +This file is basically the same as the one used in chapter03 +with the small addition of the code to perform the GET. + +(This file is auto-generated from the corresponding source file, +so make sure you don't edit it manually.) + +## The main.go file + +We define `main.go` file using `package main`. + +The beginning of the program is equal to chapter03, +so there is not much to say about it. + +```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "net" + "net/http" + "net/url" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" + utls "gitlab.com/yawning/utls.git" +) + +func main() { + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + config := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h2", "http/1.1"}, + RootCAs: netxlite.NewDefaultCertPool(), + } + conn, _, err := dialTLS(ctx, *address, config) + if err != nil { + fatal(err) + } + log.Infof("Conn type : %T", conn) +``` + +This is where things diverge. We create an HTTP client +using a transport created with `netxlite.NewHTTPTransport`. + +This transport will have as TCP connections dialer a +"null" dialer that fails whenever you attempt to dial +(and we should not be dialing anything here since we +already have a TLS connection). + +It will also use as TLSDialer (the type that dials TLS +and, morally, combines `dialTCP` with `handshakeTLS`) one +that is "single use". What does this mean? Well, we +create such a TLSDialer using the connection we already +established. The first time the HTTP code dials for +TLS, the TLSDialer will return the connection we passed +to its constructor immediately. Every subsequent TLS +dial attempt will fail. + +The result is an HTTPTransport suitable for performing +a single request using the given TLS conn. + +(A similar construct allows to create an HTTPTransport that +uses a cleartext TCP connection. In the next chapter we'll +see how to do the same using QUIC.) + +```Go + clnt := &http.Client{Transport: netxlite.NewHTTPTransport( + log.Log, netxlite.NewNullDialer(), + netxlite.NewSingleUseTLSDialer(conn.(netxlite.TLSConn)), + )} +``` + +Once we have the proper transport and client, the rest of +the code is basically standard Go for fetching a webpage +using the GET method. + +```Go + log.Infof("Transport : %T", clnt.Transport) + defer clnt.CloseIdleConnections() + resp, err := clnt.Get( + (&url.URL{Scheme: "https", Host: *sni, Path: "/"}).String()) + if err != nil { + fatal(err) + } + log.Infof("Status code: %d", resp.StatusCode) + resp.Body.Close() +} + +``` + +We won't comment on the rest of the program because it is +exactly like what we've seen in chapter03. + +```Go + +func dialTCP(ctx context.Context, address string) (net.Conn, error) { + d := netxlite.NewDialerWithoutResolver(log.Log) + return d.DialContext(ctx, "tcp", address) +} + +func handshakeTLS(ctx context.Context, tcpConn net.Conn, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + th := netxlite.NewTLSHandshakerUTLS(log.Log, &utls.HelloFirefox_55) + return th.Handshake(ctx, tcpConn, config) +} + +func dialTLS(ctx context.Context, address string, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + tcpConn, err := dialTCP(ctx, address) + if err != nil { + return nil, tls.ConnectionState{}, err + } + tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + if err != nil { + tcpConn.Close() + return nil, tls.ConnectionState{}, err + } + return tlsConn, state, nil +} + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +``` + +## Running the code + +### Vanilla run + +You can now run this code as follows: + +```bash +go run -race ./internal/tutorial/netxlite/chapter07 +``` + +You will see debug logs describing what is happening along with timing info. + +### Connect timeout + +```bash +go run -race ./internal/tutorial/netxlite/chapter07 -address 8.8.4.4:1 +``` + +should cause a connect timeout error. Try lowering the timout adding, e.g., +the `-timeout 5s` flag to the command line. + +### Connection refused + +```bash +go run -race ./internal/tutorial/netxlite/chapter07 -address '[::1]:1' +``` + +should give you a connection refused error in most cases. (We are quoting +the `::1` IPv6 address using `[` and `]` here.) + +### SNI mismatch + +```bash +go run -race ./internal/tutorial/netxlite/chapter07 -sni example.com +``` + +should give you a TLS invalid hostname error (for historical reasons +named `ssl_invalid_hostname`). + +## Conclusions + +We have seen how to establish a TLS connection with a website +and then how to GET a webpage using such a connection. diff --git a/internal/tutorial/netxlite/chapter07/main.go b/internal/tutorial/netxlite/chapter07/main.go new file mode 100644 index 0000000..e726360 --- /dev/null +++ b/internal/tutorial/netxlite/chapter07/main.go @@ -0,0 +1,196 @@ +// -=-=- StartHere -=-=- +// +// # Chapter I: HTTP GET with TLS conn +// +// In this chapter we will write together a `main.go` file that +// uses netxlite to establish a TLS connection to a remote endpoint +// and then fetches a webpage from it using GET. +// +// This file is basically the same as the one used in chapter03 +// with the small addition of the code to perform the GET. +// +// (This file is auto-generated from the corresponding source file, +// so make sure you don't edit it manually.) +// +// ## The main.go file +// +// We define `main.go` file using `package main`. +// +// The beginning of the program is equal to chapter03, +// so there is not much to say about it. +// +// ```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "net" + "net/http" + "net/url" + "os" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/netxlite" + utls "gitlab.com/yawning/utls.git" +) + +func main() { + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + config := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h2", "http/1.1"}, + RootCAs: netxlite.NewDefaultCertPool(), + } + conn, _, err := dialTLS(ctx, *address, config) + if err != nil { + fatal(err) + } + log.Infof("Conn type : %T", conn) + // ``` + // + // This is where things diverge. We create an HTTP client + // using a transport created with `netxlite.NewHTTPTransport`. + // + // This transport will have as TCP connections dialer a + // "null" dialer that fails whenever you attempt to dial + // (and we should not be dialing anything here since we + // already have a TLS connection). + // + // It will also use as TLSDialer (the type that dials TLS + // and, morally, combines `dialTCP` with `handshakeTLS`) one + // that is "single use". What does this mean? Well, we + // create such a TLSDialer using the connection we already + // established. The first time the HTTP code dials for + // TLS, the TLSDialer will return the connection we passed + // to its constructor immediately. Every subsequent TLS + // dial attempt will fail. + // + // The result is an HTTPTransport suitable for performing + // a single request using the given TLS conn. + // + // (A similar construct allows to create an HTTPTransport that + // uses a cleartext TCP connection. In the next chapter we'll + // see how to do the same using QUIC.) + // + // ```Go + clnt := &http.Client{Transport: netxlite.NewHTTPTransport( + log.Log, netxlite.NewNullDialer(), + netxlite.NewSingleUseTLSDialer(conn.(netxlite.TLSConn)), + )} + // ``` + // + // Once we have the proper transport and client, the rest of + // the code is basically standard Go for fetching a webpage + // using the GET method. + // + // ```Go + log.Infof("Transport : %T", clnt.Transport) + defer clnt.CloseIdleConnections() + resp, err := clnt.Get( + (&url.URL{Scheme: "https", Host: *sni, Path: "/"}).String()) + if err != nil { + fatal(err) + } + log.Infof("Status code: %d", resp.StatusCode) + resp.Body.Close() +} + +// ``` +// +// We won't comment on the rest of the program because it is +// exactly like what we've seen in chapter03. +// +// ```Go + +func dialTCP(ctx context.Context, address string) (net.Conn, error) { + d := netxlite.NewDialerWithoutResolver(log.Log) + return d.DialContext(ctx, "tcp", address) +} + +func handshakeTLS(ctx context.Context, tcpConn net.Conn, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + th := netxlite.NewTLSHandshakerUTLS(log.Log, &utls.HelloFirefox_55) + return th.Handshake(ctx, tcpConn, config) +} + +func dialTLS(ctx context.Context, address string, + config *tls.Config) (net.Conn, tls.ConnectionState, error) { + tcpConn, err := dialTCP(ctx, address) + if err != nil { + return nil, tls.ConnectionState{}, err + } + tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + if err != nil { + tcpConn.Close() + return nil, tls.ConnectionState{}, err + } + return tlsConn, state, nil +} + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +// ``` +// +// ## Running the code +// +// ### Vanilla run +// +// You can now run this code as follows: +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter07 +// ``` +// +// You will see debug logs describing what is happening along with timing info. +// +// ### Connect timeout +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter07 -address 8.8.4.4:1 +// ``` +// +// should cause a connect timeout error. Try lowering the timout adding, e.g., +// the `-timeout 5s` flag to the command line. +// +// ### Connection refused +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter07 -address '[::1]:1' +// ``` +// +// should give you a connection refused error in most cases. (We are quoting +// the `::1` IPv6 address using `[` and `]` here.) +// +// ### SNI mismatch +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter07 -sni example.com +// ``` +// +// should give you a TLS invalid hostname error (for historical reasons +// named `ssl_invalid_hostname`). +// +// ## Conclusions +// +// We have seen how to establish a TLS connection with a website +// and then how to GET a webpage using such a connection. diff --git a/internal/tutorial/netxlite/chapter08/README.md b/internal/tutorial/netxlite/chapter08/README.md new file mode 100644 index 0000000..9d62580 --- /dev/null +++ b/internal/tutorial/netxlite/chapter08/README.md @@ -0,0 +1,162 @@ + +# Chapter I: HTTP GET with QUIC sess + +In this chapter we will write together a `main.go` file that +uses netxlite to establish a QUIC session to a remote endpoint +and then fetches a webpage from it using GET. + +This file is basically the same as the one used in chapter04 +with the small addition of the code to perform the GET. + +(This file is auto-generated from the corresponding source file, +so make sure you don't edit it manually.) + +## The main.go file + +We define `main.go` file using `package main`. + +The beginning of the program is equal to chapter04, +so there is not much to say about it. + +```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "net/http" + "net/url" + "os" + "time" + + "github.com/apex/log" + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func main() { + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + config := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h3"}, + RootCAs: netxlite.NewDefaultCertPool(), + } + sess, _, err := dialQUIC(ctx, *address, config) + if err != nil { + fatal(err) + } + log.Infof("Sess type : %T", sess) +``` + +This is where things diverge. We create an HTTP client +using a transport created with `netxlite.NewHTTP3Transport`. + +This transport will use a "single use" QUIC dialer. +What does this mean? Well, we create such a QUICDialer +using the session we already established. The first +time the HTTP code dials for QUIC, the QUICDialer will +return the session we passed to its constructor +immediately. Every subsequent QUIC dial attempt will fail. + +The result is an HTTPTransport suitable for performing +a single request using the given QUIC sess. + +(A similar construct allows to create an HTTPTransport that +uses a cleartext TCP connection. In the previous chapter we've +seen how to do the same using TLS conns.) + +```Go + clnt := &http.Client{Transport: netxlite.NewHTTP3Transport( + log.Log, netxlite.NewSingleUseQUICDialer(sess), &tls.Config{}, + )} +``` + +Once we have the proper transport and client, the rest of +the code is basically standard Go for fetching a webpage +using the GET method. + +```Go + log.Infof("Transport : %T", clnt.Transport) + defer clnt.CloseIdleConnections() + resp, err := clnt.Get( + (&url.URL{Scheme: "https", Host: *sni, Path: "/"}).String()) + if err != nil { + fatal(err) + } + log.Infof("Status code: %d", resp.StatusCode) + resp.Body.Close() +} + +``` + +We won't comment on the rest of the program because it is +exactly like what we've seen in chapter04. + +```Go + +func dialQUIC(ctx context.Context, address string, + config *tls.Config) (quic.EarlySession, tls.ConnectionState, error) { + ql := netxlite.NewQUICListener() + d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) + sess, err := d.DialContext(ctx, "udp", address, config, &quic.Config{}) + if err != nil { + return nil, tls.ConnectionState{}, err + } + return sess, sess.ConnectionState().TLS.ConnectionState, nil +} + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +``` + +## Running the code + +### Vanilla run + +You can now run this code as follows: + +```bash +go run -race ./internal/tutorial/netxlite/chapter08 +``` + +You will see debug logs describing what is happening along with timing info. + +### QUIC handshake timeout + +```bash +go run -race ./internal/tutorial/netxlite/chapter08 -address 8.8.4.4:1 +``` + +should cause a QUIC handshake timeout error. Try lowering the timout adding, e.g., +the `-timeout 5s` flag to the command line. + +### SNI mismatch + +```bash +go run -race ./internal/tutorial/netxlite/chapter08 -sni example.com +``` + +should give you an error mentioning the certificate is invalid. + +## Conclusions + +We have seen how to establish a QUIC session with a website +and then how to GET a webpage using such a session. diff --git a/internal/tutorial/netxlite/chapter08/main.go b/internal/tutorial/netxlite/chapter08/main.go new file mode 100644 index 0000000..136aaa9 --- /dev/null +++ b/internal/tutorial/netxlite/chapter08/main.go @@ -0,0 +1,163 @@ +// -=-=- StartHere -=-=- +// +// # Chapter I: HTTP GET with QUIC sess +// +// In this chapter we will write together a `main.go` file that +// uses netxlite to establish a QUIC session to a remote endpoint +// and then fetches a webpage from it using GET. +// +// This file is basically the same as the one used in chapter04 +// with the small addition of the code to perform the GET. +// +// (This file is auto-generated from the corresponding source file, +// so make sure you don't edit it manually.) +// +// ## The main.go file +// +// We define `main.go` file using `package main`. +// +// The beginning of the program is equal to chapter04, +// so there is not much to say about it. +// +// ```Go +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "net/http" + "net/url" + "os" + "time" + + "github.com/apex/log" + "github.com/lucas-clemente/quic-go" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func main() { + log.SetLevel(log.DebugLevel) + address := flag.String("address", "8.8.4.4:443", "Remote endpoint address") + sni := flag.String("sni", "dns.google", "SNI to use") + timeout := flag.Duration("timeout", 60*time.Second, "Timeout") + flag.Parse() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + config := &tls.Config{ + ServerName: *sni, + NextProtos: []string{"h3"}, + RootCAs: netxlite.NewDefaultCertPool(), + } + sess, _, err := dialQUIC(ctx, *address, config) + if err != nil { + fatal(err) + } + log.Infof("Sess type : %T", sess) + // ``` + // + // This is where things diverge. We create an HTTP client + // using a transport created with `netxlite.NewHTTP3Transport`. + // + // This transport will use a "single use" QUIC dialer. + // What does this mean? Well, we create such a QUICDialer + // using the session we already established. The first + // time the HTTP code dials for QUIC, the QUICDialer will + // return the session we passed to its constructor + // immediately. Every subsequent QUIC dial attempt will fail. + // + // The result is an HTTPTransport suitable for performing + // a single request using the given QUIC sess. + // + // (A similar construct allows to create an HTTPTransport that + // uses a cleartext TCP connection. In the previous chapter we've + // seen how to do the same using TLS conns.) + // + // ```Go + clnt := &http.Client{Transport: netxlite.NewHTTP3Transport( + log.Log, netxlite.NewSingleUseQUICDialer(sess), &tls.Config{}, + )} + // ``` + // + // Once we have the proper transport and client, the rest of + // the code is basically standard Go for fetching a webpage + // using the GET method. + // + // ```Go + log.Infof("Transport : %T", clnt.Transport) + defer clnt.CloseIdleConnections() + resp, err := clnt.Get( + (&url.URL{Scheme: "https", Host: *sni, Path: "/"}).String()) + if err != nil { + fatal(err) + } + log.Infof("Status code: %d", resp.StatusCode) + resp.Body.Close() +} + +// ``` +// +// We won't comment on the rest of the program because it is +// exactly like what we've seen in chapter04. +// +// ```Go + +func dialQUIC(ctx context.Context, address string, + config *tls.Config) (quic.EarlySession, tls.ConnectionState, error) { + ql := netxlite.NewQUICListener() + d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) + sess, err := d.DialContext(ctx, "udp", address, config, &quic.Config{}) + if err != nil { + return nil, tls.ConnectionState{}, err + } + return sess, sess.ConnectionState().TLS.ConnectionState, nil +} + +func fatal(err error) { + var ew *netxlite.ErrWrapper + if !errors.As(err, &ew) { + log.Fatal("cannot get ErrWrapper") + } + log.Warnf("error string : %s", err.Error()) + log.Warnf("OONI failure : %s", ew.Failure) + log.Warnf("failed operation: %s", ew.Operation) + log.Warnf("underlying error: %+v", ew.WrappedErr) + os.Exit(1) +} + +// ``` +// +// ## Running the code +// +// ### Vanilla run +// +// You can now run this code as follows: +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter08 +// ``` +// +// You will see debug logs describing what is happening along with timing info. +// +// ### QUIC handshake timeout +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter08 -address 8.8.4.4:1 +// ``` +// +// should cause a QUIC handshake timeout error. Try lowering the timout adding, e.g., +// the `-timeout 5s` flag to the command line. +// +// ### SNI mismatch +// +// ```bash +// go run -race ./internal/tutorial/netxlite/chapter08 -sni example.com +// ``` +// +// should give you an error mentioning the certificate is invalid. +// +// ## Conclusions +// +// We have seen how to establish a QUIC session with a website +// and then how to GET a webpage using such a session.