From 15da0f5344e721568be9b18282b96d435d04bf89 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 2 Jun 2022 22:25:37 +0200 Subject: [PATCH] cleanup(jafar): do not depend on netx and urlgetter (#792) There's no point in doing that. Also, once this change is merged, it becomes easier to cleanup/simplify netx. See https://github.com/ooni/probe/issues/2121 --- internal/cmd/jafar/README.md | 23 +-- .../cmd/jafar/httpproxy/httpproxy_test.go | 4 +- .../iptables/iptables_integration_test.go | 2 +- internal/cmd/jafar/main.go | 12 +- internal/cmd/jafar/resolver/resolver_test.go | 8 +- internal/cmd/jafar/tlsproxy/tlsproxy_test.go | 4 +- internal/cmd/jafar/uncensored/uncensored.go | 40 +--- .../cmd/jafar/uncensored/uncensored_test.go | 21 +- internal/netxlite/filtering/dns.go | 22 +- internal/netxlite/filtering/http.go | 159 +++++++++----- internal/netxlite/filtering/http_test.go | 194 +++++++++--------- internal/netxlite/filtering/tls.go | 30 ++- internal/netxlite/filtering/tls_test.go | 4 +- internal/netxlite/http.go | 9 + internal/netxlite/http_test.go | 21 ++ internal/netxlite/resolvercore.go | 10 + internal/netxlite/resolvercore_test.go | 17 ++ internal/tracex/http_test.go | 38 +--- script/nocopyreadall.bash | 2 +- 19 files changed, 345 insertions(+), 275 deletions(-) diff --git a/internal/cmd/jafar/README.md b/internal/cmd/jafar/README.md index 1fdd4a0..8b4722a 100644 --- a/internal/cmd/jafar/README.md +++ b/internal/cmd/jafar/README.md @@ -8,7 +8,7 @@ any system but it really only works on Linux. ## Building -We use Go >= 1.16. Jafar also needs the C library headers, +We use Go >= 1.18. Jafar also needs the C library headers, iptables installed, and root permissions. With Linux Alpine edge, you can compile Jafar with: @@ -198,24 +198,21 @@ the client Hello message will cause the TLS handshake to fail. ### uncensored ```bash - -uncensored-resolver-url string - URL of an hopefully uncensored resolver (default "dot://1.1.1.1:853") + -uncensored-resolver-doh string + URL of an hopefully uncensored DoH resolver (default "https://1.1.1.1/dns-query") ``` The HTTP, DNS, and TLS proxies need to resolve domain names. If you setup DNS censorship, they may be affected as well. To avoid this issue, we use a different -resolver for them, which by default is `dot://1.1.1.1:853`. You can change such -default by using the `-uncensored-resolver-url` command line flag. The input -URL is `://[:][/]`. Here are some examples: +resolver for them, which by default is the one shown above. You can change such +default by using the `-uncensored-resolver-doh` command line flag. The input +URL is an HTTPS URL pointing to a DoH server. Here are some examples: -* `system:///` uses the system resolver (i.e. `getaddrinfo`) -* `udp://8.8.8.8:53` uses DNS over UDP -* `tcp://8.8.8.8:53` used DNS over TCP -* `dot://8.8.8.8:853` uses DNS over TLS -* `https://dns.google/dns-query` uses DNS over HTTPS +* `https://dns.google/dns-query` +* `https://dns.quad9.net/dns-query` -So, for example, if you are using Jafar to censor `1.1.1.1:853`, then you -most likely want to use `-uncensored-resolver-url`. +So, for example, if you are using Jafar to censor `1.1.1.1:443`, then you +most likely want to use `-uncensored-resolver-doh`. ## Examples diff --git a/internal/cmd/jafar/httpproxy/httpproxy_test.go b/internal/cmd/jafar/httpproxy/httpproxy_test.go index d6e7f95..0132c4c 100644 --- a/internal/cmd/jafar/httpproxy/httpproxy_test.go +++ b/internal/cmd/jafar/httpproxy/httpproxy_test.go @@ -42,7 +42,7 @@ func TestLoop(t *testing.T) { } func TestListenError(t *testing.T) { - proxy := NewCensoringProxy([]string{""}, uncensored.DefaultClient) + proxy := NewCensoringProxy([]string{""}, uncensored.NewClient("https://1.1.1.1/dns-query")) server, addr, err := proxy.Start("8.8.8.8:80") if err == nil { t.Fatal("expected an error here") @@ -56,7 +56,7 @@ func TestListenError(t *testing.T) { } func newproxy(t *testing.T, blocked string) (*http.Server, net.Addr) { - proxy := NewCensoringProxy([]string{blocked}, uncensored.DefaultClient) + proxy := NewCensoringProxy([]string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query")) server, addr, err := proxy.Start("127.0.0.1:0") if err != nil { t.Fatal(err) diff --git a/internal/cmd/jafar/iptables/iptables_integration_test.go b/internal/cmd/jafar/iptables/iptables_integration_test.go index 52af27e..50f1722 100644 --- a/internal/cmd/jafar/iptables/iptables_integration_test.go +++ b/internal/cmd/jafar/iptables/iptables_integration_test.go @@ -240,7 +240,7 @@ func TestHijackDNS(t *testing.T) { } resolver := resolver.NewCensoringResolver( []string{"ooni.io"}, nil, nil, - uncensored.Must(uncensored.NewClient("dot://1.1.1.1:853")), + uncensored.NewClient("https://1.1.1.1/dns-query"), ) server, err := resolver.Start("127.0.0.1:0") if err != nil { diff --git a/internal/cmd/jafar/main.go b/internal/cmd/jafar/main.go index f0c0239..61f5285 100644 --- a/internal/cmd/jafar/main.go +++ b/internal/cmd/jafar/main.go @@ -61,7 +61,7 @@ var ( tlsProxyAddress *string tlsProxyBlock flagx.StringArray - uncensoredResolverURL *string + uncensoredResolverDoH *string ) func init() { @@ -167,9 +167,9 @@ func init() { ) // uncensored - uncensoredResolverURL = flag.String( - "uncensored-resolver-url", "dot://1.1.1.1:853", - "URL of an hopefully uncensored resolver", + uncensoredResolverDoH = flag.String( + "uncensored-resolver-doh", "https://1.1.1.1/dns-query", + "URL of an hopefully uncensored DoH resolver", ) } @@ -234,9 +234,7 @@ func tlsProxyStart(uncensored *uncensored.Client) net.Listener { } func newUncensoredClient() *uncensored.Client { - clnt, err := uncensored.NewClient(*uncensoredResolverURL) - runtimex.PanicOnError(err, "uncensored.NewClient failed") - return clnt + return uncensored.NewClient(*uncensoredResolverDoH) } func mustx(err error, message string, osExit func(int)) { diff --git a/internal/cmd/jafar/resolver/resolver_test.go b/internal/cmd/jafar/resolver/resolver_test.go index e4b97f2..f6f05b7 100644 --- a/internal/cmd/jafar/resolver/resolver_test.go +++ b/internal/cmd/jafar/resolver/resolver_test.go @@ -45,14 +45,14 @@ func TestLookupFailure(t *testing.T) { func TestFailureNoQuestion(t *testing.T) { resolver := NewCensoringResolver( - nil, nil, nil, uncensored.DefaultClient, + nil, nil, nil, uncensored.NewClient("https://1.1.1.1/dns-query"), ) resolver.ServeDNS(&fakeResponseWriter{t: t}, new(dns.Msg)) } func TestListenFailure(t *testing.T) { resolver := NewCensoringResolver( - nil, nil, nil, uncensored.DefaultClient, + nil, nil, nil, uncensored.NewClient("https://1.1.1.1/dns-query"), ) server, err := resolver.Start("8.8.8.8:53") if err == nil { @@ -66,9 +66,7 @@ func TestListenFailure(t *testing.T) { func newresolver(t *testing.T, blocked, hijacked, ignored []string) *dns.Server { resolver := NewCensoringResolver( blocked, hijacked, ignored, - // using faster dns because dot here causes miekg/dns's - // dns.Exchange to timeout and I don't want more complexity - uncensored.Must(uncensored.NewClient("system:///")), + uncensored.NewClient("https://1.1.1.1/dns-query"), ) server, err := resolver.Start("127.0.0.1:0") if err != nil { diff --git a/internal/cmd/jafar/tlsproxy/tlsproxy_test.go b/internal/cmd/jafar/tlsproxy/tlsproxy_test.go index 8c01d33..0b85c49 100644 --- a/internal/cmd/jafar/tlsproxy/tlsproxy_test.go +++ b/internal/cmd/jafar/tlsproxy/tlsproxy_test.go @@ -94,7 +94,7 @@ func TestFailWriteAfterConnect(t *testing.T) { func TestListenError(t *testing.T) { proxy := NewCensoringProxy( - []string{""}, uncensored.DefaultClient, + []string{""}, uncensored.NewClient("https://1.1.1.1/dns-query"), ) listener, err := proxy.Start("8.8.8.8:80") if err == nil { @@ -107,7 +107,7 @@ func TestListenError(t *testing.T) { func newproxy(t *testing.T, blocked string) net.Listener { proxy := NewCensoringProxy( - []string{blocked}, uncensored.DefaultClient, + []string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query"), ) listener, err := proxy.Start("127.0.0.1:0") if err != nil { diff --git a/internal/cmd/jafar/uncensored/uncensored.go b/internal/cmd/jafar/uncensored/uncensored.go index 21bfb4f..fcb51a7 100644 --- a/internal/cmd/jafar/uncensored/uncensored.go +++ b/internal/cmd/jafar/uncensored/uncensored.go @@ -9,10 +9,8 @@ import ( "net/http" "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/netxlite" ) // Client is DNS, HTTP, and TCP client. @@ -23,35 +21,15 @@ type Client struct { } // NewClient creates a new Client. -func NewClient(resolverURL string) (*Client, error) { - configuration, err := urlgetter.Configurer{ - Config: urlgetter.Config{ - ResolverURL: resolverURL, - }, - Logger: log.Log, - }.NewConfiguration() - if err != nil { - return nil, err - } +func NewClient(resolverURL string) *Client { + dnsClient := netxlite.NewParallelDNSOverHTTPSResolver(log.Log, resolverURL) return &Client{ - dnsClient: configuration.DNSClient, - httpTransport: netx.NewHTTPTransport(configuration.HTTPConfig), - dialer: netx.NewDialer(configuration.HTTPConfig), - }, nil + dnsClient: dnsClient, + httpTransport: netxlite.NewHTTPTransportWithResolver(log.Log, dnsClient), + dialer: netxlite.NewDialerWithResolver(log.Log, dnsClient), + } } -// Must panics if it's not possible to create a Client. Usually you should -// use it like `uncensored.Must(uncensored.NewClient(URL))`. -func Must(client *Client, err error) *Client { - runtimex.PanicOnError(err, "cannot create uncensored client") - return client -} - -// DefaultClient is the default client for DNS, HTTP, and TCP. -var DefaultClient = Must(NewClient("")) - -var _ model.Resolver = DefaultClient - // Address implements Resolver.Address func (c *Client) Address() string { return c.dnsClient.Address() @@ -77,15 +55,11 @@ func (c *Client) Network() string { return c.dnsClient.Network() } -var _ model.Dialer = DefaultClient - // DialContext implements Dialer.DialContext func (c *Client) DialContext(ctx context.Context, network, address string) (net.Conn, error) { return c.dialer.DialContext(ctx, network, address) } -var _ model.HTTPTransport = DefaultClient - // CloseIdleConnections implement HTTPRoundTripper.CloseIdleConnections func (c *Client) CloseIdleConnections() { c.dnsClient.CloseIdleConnections() diff --git a/internal/cmd/jafar/uncensored/uncensored_test.go b/internal/cmd/jafar/uncensored/uncensored_test.go index df0ac23..d03f229 100644 --- a/internal/cmd/jafar/uncensored/uncensored_test.go +++ b/internal/cmd/jafar/uncensored/uncensored_test.go @@ -10,16 +10,13 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) -func TestGood(t *testing.T) { - client, err := NewClient("dot://1.1.1.1:853") - if err != nil { - t.Fatal(err) - } +func TestNewClient(t *testing.T) { + client := NewClient("https://1.1.1.1/dns-query") defer client.CloseIdleConnections() - if client.Address() != "1.1.1.1:853" { + if client.Address() != "https://1.1.1.1/dns-query" { t.Fatal("invalid address") } - if client.Network() != "dot" { + if client.Network() != "doh" { t.Fatal("invalid network") } ctx := context.Background() @@ -64,13 +61,3 @@ func TestGood(t *testing.T) { t.Fatal("not the expected body") } } - -func TestNewClientFailure(t *testing.T) { - clnt, err := NewClient("antani:///") - if err == nil { - t.Fatal("expected an error here") - } - if clnt != nil { - t.Fatal("expected nil client here") - } -} diff --git a/internal/netxlite/filtering/dns.go b/internal/netxlite/filtering/dns.go index 3eede6d..02669ee 100644 --- a/internal/netxlite/filtering/dns.go +++ b/internal/netxlite/filtering/dns.go @@ -153,14 +153,28 @@ func (p *DNSServer) nxdomain(query *dns.Msg) *dns.Msg { } func (p *DNSServer) localHost(query *dns.Msg) *dns.Msg { - return p.compose(query, net.IPv6loopback, net.IPv4(127, 0, 0, 1)) + return dnsComposeResponse(query, net.IPv6loopback, net.IPv4(127, 0, 0, 1)) } func (p *DNSServer) empty(query *dns.Msg) *dns.Msg { - return p.compose(query) + return dnsComposeResponse(query) } -func (p *DNSServer) compose(query *dns.Msg, ips ...net.IP) *dns.Msg { +func dnsComposeQuery(domain string, qtype uint16) *dns.Msg { + question := dns.Question{ + Name: dns.Fqdn(domain), + Qtype: qtype, + Qclass: dns.ClassINET, + } + query := &dns.Msg{} + query.RecursionDesired = true + query.Question = make([]dns.Question, 1) + query.Question[0] = question + query.Id = dns.Id() + return query +} + +func dnsComposeResponse(query *dns.Msg, ips ...net.IP) *dns.Msg { runtimex.PanicIfTrue(len(query.Question) != 1, "expecting a single question") question := query.Question[0] reply := new(dns.Msg) @@ -205,5 +219,5 @@ func (p *DNSServer) cache(name string, query *dns.Msg) *dns.Msg { if len(ipAddrs) <= 0 { return p.nxdomain(query) } - return p.compose(query, ipAddrs...) + return dnsComposeResponse(query, ipAddrs...) } diff --git a/internal/netxlite/filtering/http.go b/internal/netxlite/filtering/http.go index 96b71ff..031a0ea 100644 --- a/internal/netxlite/filtering/http.go +++ b/internal/netxlite/filtering/http.go @@ -1,24 +1,23 @@ package filtering import ( + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "io" "net" "net/http" - "net/http/httputil" "net/url" + "github.com/google/martian/v3/mitm" + "github.com/miekg/dns" "github.com/ooni/probe-cli/v3/internal/runtimex" ) -// TODO(bassosimone): remove HTTPActionPass since we want integration tests -// to only run locally to make them much more predictable. - -// HTTPAction is an HTTP filtering action that this proxy should take. +// HTTPAction is an HTTP filtering action that this server should take. type HTTPAction string const ( - // HTTPActionPass passes the traffic to the destination. - HTTPActionPass = HTTPAction("pass") - // HTTPActionReset resets the connection. HTTPActionReset = HTTPAction("reset") @@ -30,25 +29,91 @@ const ( // HTTPAction451 causes the proxy to return a 451 error. HTTPAction451 = HTTPAction("451") + + // HTTPActionDoH causes the proxy to return a sensible reply + // with static IP addresses if the request is DoH. + HTTPActionDoH = HTTPAction("doh") ) -// HTTPProxy is a proxy that routes traffic depending on the -// host header and may implement filtering policies. -type HTTPProxy struct { - // OnIncomingHost is the MANDATORY hook called whenever we have - // successfully received an HTTP request. - OnIncomingHost func(host string) HTTPAction +// HTTPServer is a server that implements filtering policies. +type HTTPServer struct { + // action is the action to implement. + action HTTPAction + + // cert is the fake CA certificate. + cert *x509.Certificate + + // config is the config to generate certificates on the fly. + config *mitm.Config + + // privkey is the private key that signed the cert. + privkey *rsa.PrivateKey + + // server is the underlying server. + server *http.Server + + // url contains the server URL + url *url.URL } -// Start starts the proxy. -func (p *HTTPProxy) Start(address string) (net.Listener, error) { - listener, err := net.Listen("tcp", address) - if err != nil { - return nil, err +// NewHTTPServerCleartext creates a new HTTPServer using cleartext HTTP. +func NewHTTPServerCleartext(action HTTPAction) *HTTPServer { + return newHTTPOrHTTPSServer(action, false) +} + +// NewHTTPServerTLS creates a new HTTP server using HTTPS. +func NewHTTPServerTLS(action HTTPAction) *HTTPServer { + return newHTTPOrHTTPSServer(action, true) +} + +// Close closes the server ASAP. +func (p *HTTPServer) Close() error { + return p.server.Close() +} + +// URL returns the server's URL +func (p *HTTPServer) URL() *url.URL { + return p.url +} + +// TLSConfig returns a suitable base TLS config for the client. +func (p *HTTPServer) TLSConfig() *tls.Config { + config := &tls.Config{} + if p.cert != nil { + o := x509.NewCertPool() + o.AddCert(p.cert) + config.RootCAs = o } - server := &http.Server{Handler: p} - go server.Serve(listener) - return listener, nil + return config +} + +// newHTTPOrHTTPSServer is an internal factory for creating a new instance. +func newHTTPOrHTTPSServer(action HTTPAction, enableTLS bool) *HTTPServer { + listener, err := net.Listen("tcp", "127.0.0.1:0") + runtimex.PanicOnError(err, "net.Listen failed") + srv := &HTTPServer{ + action: action, + cert: nil, + config: nil, + privkey: nil, + server: nil, + url: &url.URL{ + Scheme: "", + Host: listener.Addr().String(), + }, + } + srv.server = &http.Server{Handler: srv} + switch enableTLS { + case false: + srv.url.Scheme = "http" + go srv.server.Serve(listener) + case true: + srv.url.Scheme = "https" + srv.cert, srv.privkey, srv.config = tlsConfigMITM() + srv.server.TLSConfig = srv.config.TLS() + go srv.server.ServeTLS(listener, "", "") // using server.TLSConfig + } + return srv } // HTTPBlockPage451 is the block page returned along with status 451 @@ -60,34 +125,22 @@ var HTTPBlockpage451 = []byte(` `) -const httpProxyProduct = "jafar/0.1.0" - // ServeHTTP serves HTTP requests -func (p *HTTPProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Implementation note: use Via header to detect in a loose way - // requests originated by us and directed to us. - if r.Header.Get("Via") == httpProxyProduct || r.Host == "" { - w.WriteHeader(http.StatusBadRequest) - return - } - p.handle(w, r) -} - -func (p *HTTPProxy) handle(w http.ResponseWriter, r *http.Request) { - switch policy := p.OnIncomingHost(r.Host); policy { - case HTTPActionPass: - p.proxy(w, r) +func (p *HTTPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch p.action { case HTTPActionReset, HTTPActionTimeout, HTTPActionEOF: - p.hijack(w, r, policy) + p.hijack(w, r, p.action) case HTTPAction451: w.WriteHeader(http.StatusUnavailableForLegalReasons) w.Write(HTTPBlockpage451) + case HTTPActionDoH: + httpServeDNSOverHTTPS(w, r) default: w.WriteHeader(http.StatusInternalServerError) } } -func (p *HTTPProxy) hijack(w http.ResponseWriter, r *http.Request, policy HTTPAction) { +func (p *HTTPServer) hijack(w http.ResponseWriter, r *http.Request, policy HTTPAction) { // Note: // // 1. we assume we can hihack the connection @@ -109,12 +162,22 @@ func (p *HTTPProxy) hijack(w http.ResponseWriter, r *http.Request, policy HTTPAc } } -func (p *HTTPProxy) proxy(w http.ResponseWriter, r *http.Request) { - r.Header.Add("Via", httpProxyProduct) // see ServeHTTP - proxy := httputil.NewSingleHostReverseProxy(&url.URL{ - Host: r.Host, - Scheme: "http", - }) - proxy.Transport = http.DefaultTransport - proxy.ServeHTTP(w, r) +func httpPanicToInternalServerError(w http.ResponseWriter) { + if r := recover(); r != nil { + w.WriteHeader(500) + } +} + +func httpServeDNSOverHTTPS(w http.ResponseWriter, r *http.Request) { + defer httpPanicToInternalServerError(w) + rawQuery, err := io.ReadAll(r.Body) + runtimex.PanicOnError(err, "io.ReadAll failed") + query := &dns.Msg{} + err = query.Unpack(rawQuery) + runtimex.PanicOnError(err, "query.Unpack failed") + runtimex.PanicIfTrue(query.Response, "is a response") + response := dnsComposeResponse(query, net.IPv4(8, 8, 8, 8), net.IPv4(8, 8, 4, 4)) + rawResponse, err := response.Pack() + runtimex.PanicOnError(err, "response.Pack failed") + w.Write(rawResponse) } diff --git a/internal/netxlite/filtering/http_test.go b/internal/netxlite/filtering/http_test.go index 3acb1e9..a2e2975 100644 --- a/internal/netxlite/filtering/http_test.go +++ b/internal/netxlite/filtering/http_test.go @@ -1,155 +1,134 @@ package filtering import ( + "bytes" "context" + "crypto/tls" "errors" - "net" + "io" "net/http" "net/url" - "strings" "testing" "time" - "github.com/apex/log" + "github.com/miekg/dns" + "github.com/ooni/probe-cli/v3/internal/model/mocks" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/ooni/probe-cli/v3/internal/runtimex" ) -func TestHTTPProxy(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - newproxy := func(action HTTPAction) (net.Listener, error) { - p := &HTTPProxy{ - OnIncomingHost: func(host string) HTTPAction { - return action - }, - } - return p.Start("127.0.0.1:0") - } +func TestHTTPServer(t *testing.T) { - httpGET := func(ctx context.Context, addr net.Addr, host string) (*http.Response, error) { - txp := netxlite.NewHTTPTransportStdlib(log.Log) - clnt := &http.Client{Transport: txp} - URL := &url.URL{ - Scheme: "http", - Host: addr.String(), - Path: "/", + httpGET := func(ctx context.Context, method string, URL *url.URL, host string, + config *tls.Config, requestBody []byte) (*http.Response, error) { + txp := &http.Transport{ + TLSClientConfig: config, } - req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil) + if config != nil { + config.ServerName = host + } + clnt := &http.Client{Transport: txp} + req, err := http.NewRequestWithContext( + ctx, method, URL.String(), bytes.NewReader(requestBody)) runtimex.PanicOnError(err, "http.NewRequest failed") req.Host = host return clnt.Do(req) } - t.Run("HTTPActionPass", func(t *testing.T) { - ctx := context.Background() - listener, err := newproxy(HTTPActionPass) - if err != nil { - t.Fatal(err) - } - resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it") - if err != nil { - t.Fatal(err) - } - if resp.StatusCode != 200 { - t.Fatal("unexpected status code", resp.StatusCode) - } - resp.Body.Close() - listener.Close() - }) - - t.Run("HTTPActionPass with self connect", func(t *testing.T) { - ctx := context.Background() - listener, err := newproxy(HTTPActionPass) - if err != nil { - t.Fatal(err) - } - resp, err := httpGET(ctx, listener.Addr(), listener.Addr().String()) - if err != nil { - t.Fatal(err) - } - if resp.StatusCode != 400 { - t.Fatal("unexpected status code", resp.StatusCode) - } - resp.Body.Close() - listener.Close() - }) - t.Run("HTTPActionReset", func(t *testing.T) { ctx := context.Background() - listener, err := newproxy(HTTPActionReset) - if err != nil { - t.Fatal(err) - } - resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it") - if err == nil || !strings.HasSuffix(err.Error(), netxlite.FailureConnectionReset) { + srvr := NewHTTPServerCleartext(HTTPActionReset) + resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil) + if netxlite.NewTopLevelGenericErrWrapper(err).Error() != netxlite.FailureConnectionReset { t.Fatal("unexpected err", err) } if resp != nil { t.Fatal("expected nil resp") } - listener.Close() + srvr.Close() }) t.Run("HTTPActionTimeout", func(t *testing.T) { ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) defer cancel() - listener, err := newproxy(HTTPActionTimeout) - if err != nil { - t.Fatal(err) - } - resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it") + srvr := NewHTTPServerCleartext(HTTPActionTimeout) + resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil) if !errors.Is(err, context.DeadlineExceeded) { t.Fatal("unexpected err", err) } if resp != nil { t.Fatal("expected nil resp") } - listener.Close() + srvr.Close() }) t.Run("HTTPActionEOF", func(t *testing.T) { ctx := context.Background() - listener, err := newproxy(HTTPActionEOF) - if err != nil { - t.Fatal(err) - } - resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it") - if err == nil || !strings.HasSuffix(err.Error(), netxlite.FailureEOFError) { + srvr := NewHTTPServerCleartext(HTTPActionEOF) + resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil) + if !errors.Is(err, io.EOF) { t.Fatal("unexpected err", err) } if resp != nil { t.Fatal("expected nil resp") } - listener.Close() + srvr.Close() }) t.Run("HTTPAction451", func(t *testing.T) { ctx := context.Background() - listener, err := newproxy(HTTPAction451) - if err != nil { - t.Fatal(err) - } - resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it") + srvr := NewHTTPServerCleartext(HTTPAction451) + resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil) if err != nil { t.Fatal(err) } if resp.StatusCode != 451 { t.Fatal("unexpected status code", resp.StatusCode) } + data, err := netxlite.ReadAllContext(ctx, resp.Body) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(HTTPBlockpage451, data) { + t.Fatal("unexpected data") + } resp.Body.Close() - listener.Close() + srvr.Close() + }) + + t.Run("HTTPActionDoH", func(t *testing.T) { + ctx := context.Background() + srvr := NewHTTPServerTLS(HTTPActionDoH) + query := dnsComposeQuery("nexa.polito.it", dns.TypeA) + rawQuery, err := query.Pack() + if err != nil { + t.Fatal(err) + } + resp, err := httpGET(ctx, "POST", srvr.URL(), "dns.google", srvr.TLSConfig(), rawQuery) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + t.Fatal("unexpected status code", resp.StatusCode) + } + data, err := netxlite.ReadAllContext(ctx, resp.Body) + if err != nil { + t.Fatal(err) + } + response := &dns.Msg{} + if err := response.Unpack(data); err != nil { + t.Fatal(err) + } + // It suffices to see it's a DNS response + resp.Body.Close() + srvr.Close() }) t.Run("unknown action", func(t *testing.T) { ctx := context.Background() - listener, err := newproxy("") - if err != nil { - t.Fatal(err) - } - resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it") + srvr := NewHTTPServerCleartext("") + resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil) if err != nil { t.Fatal(err) } @@ -157,17 +136,30 @@ func TestHTTPProxy(t *testing.T) { t.Fatal("unexpected status code", resp.StatusCode) } resp.Body.Close() - listener.Close() - }) - - t.Run("Start fails on an invalid address", func(t *testing.T) { - p := &HTTPProxy{} - listener, err := p.Start("127.0.0.1") - if err == nil || !strings.HasSuffix(err.Error(), "missing port in address") { - t.Fatal("unexpected err", err) - } - if listener != nil { - t.Fatal("expected nil listener") - } + srvr.Close() }) } + +type httpResponseWriter struct { + http.ResponseWriter + code int +} + +func (w *httpResponseWriter) WriteHeader(statusCode int) { + w.code = statusCode +} + +func TestHTTPServeDNSOverHTTPSPanic(t *testing.T) { + w := &httpResponseWriter{} + req := &http.Request{ + Body: io.NopCloser(&mocks.Reader{ + MockRead: func(b []byte) (int, error) { + return 0, io.ErrUnexpectedEOF + }, + }), + } + httpServeDNSOverHTTPS(w, req) + if w.code != 500 { + t.Fatal("did not intercept the panic") + } +} diff --git a/internal/netxlite/filtering/tls.go b/internal/netxlite/filtering/tls.go index abc5ddc..af6210a 100644 --- a/internal/netxlite/filtering/tls.go +++ b/internal/netxlite/filtering/tls.go @@ -66,14 +66,19 @@ type TLSServer struct { privkey *rsa.PrivateKey } -// NewTLSServer creates and starts a new TLSServer that executes -// the given action during the TLS handshake. -func NewTLSServer(action TLSAction) *TLSServer { - done := make(chan bool) +func tlsConfigMITM() (*x509.Certificate, *rsa.PrivateKey, *mitm.Config) { cert, privkey, err := mitm.NewAuthority("jafar", "OONI", 24*time.Hour) runtimex.PanicOnError(err, "mitm.NewAuthority failed") config, err := mitm.NewConfig(cert, privkey) runtimex.PanicOnError(err, "mitm.NewConfig failed") + return cert, privkey, config +} + +// NewTLSServer creates and starts a new TLSServer that executes +// the given action during the TLS handshake. +func NewTLSServer(action TLSAction) *TLSServer { + done := make(chan bool) + cert, privkey, config := tlsConfigMITM() listener, err := net.Listen("tcp", "127.0.0.1:0") runtimex.PanicOnError(err, "net.Listen failed") ctx, cancel := context.WithCancel(context.Background()) @@ -139,13 +144,8 @@ func (p *TLSServer) handle(ctx context.Context, tcpConn net.Conn) { GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { switch p.action { case TLSActionTimeout: - select { - case <-time.After(300 * time.Second): - return nil, errors.New("timing out the connection") - case <-ctx.Done(): - p.reset(tcpConn) - return nil, ctx.Err() - } + err := p.timeout(ctx, tcpConn) + return nil, err case TLSActionAlertInternalError: p.alert(tcpConn, tlsAlertInternalError) return nil, errors.New("already sent alert") @@ -170,6 +170,14 @@ func (p *TLSServer) handle(ctx context.Context, tcpConn net.Conn) { tlsConn.Close() } +func (p *TLSServer) timeout(ctx context.Context, tcpConn net.Conn) error { + ctx, cancel := context.WithTimeout(ctx, 300*time.Second) + defer cancel() + <-ctx.Done() + p.reset(tcpConn) + return ctx.Err() +} + func (p *TLSServer) reset(conn net.Conn) { if tc, good := conn.(*net.TCPConn); good { tc.SetLinger(0) diff --git a/internal/netxlite/filtering/tls_test.go b/internal/netxlite/filtering/tls_test.go index 066607a..29a5a31 100644 --- a/internal/netxlite/filtering/tls_test.go +++ b/internal/netxlite/filtering/tls_test.go @@ -116,7 +116,7 @@ func TestTLSServer(t *testing.T) { t.Fatal(err) } defer conn.Close() - data, err := io.ReadAll(conn) + data, err := netxlite.ReadAllContext(context.Background(), conn) if err != nil { t.Fatal(err) } @@ -134,7 +134,7 @@ func TestTLSServer(t *testing.T) { t.Fatal(err) } defer conn.Close() - data, err := io.ReadAll(conn) + data, err := netxlite.ReadAllContext(context.Background(), conn) if err != nil { t.Fatal(err) } diff --git a/internal/netxlite/http.go b/internal/netxlite/http.go index d553fa7..437b7a9 100644 --- a/internal/netxlite/http.go +++ b/internal/netxlite/http.go @@ -105,6 +105,15 @@ func (txp *httpTransportConnectionsCloser) CloseIdleConnections() { txp.TLSDialer.CloseIdleConnections() } +// NewHTTPTransportWithResolver creates a new HTTP transport using +// the stdlib for everything but the given resolver. +func NewHTTPTransportWithResolver(logger model.DebugLogger, reso model.Resolver) model.HTTPTransport { + dialer := NewDialerWithResolver(logger, reso) + thx := NewTLSHandshakerStdlib(logger) + tlsDialer := NewTLSDialer(dialer, thx) + return NewHTTPTransport(logger, dialer, tlsDialer) +} + // NewHTTPTransport combines NewOOHTTPBaseTransport and WrapHTTPTransport. // // This factory and NewHTTPTransportStdlib are the recommended diff --git a/internal/netxlite/http_test.go b/internal/netxlite/http_test.go index 873c6d1..56fe4b0 100644 --- a/internal/netxlite/http_test.go +++ b/internal/netxlite/http_test.go @@ -16,6 +16,27 @@ import ( "github.com/ooni/probe-cli/v3/internal/model/mocks" ) +func TestNewHTTPTransportWithResolver(t *testing.T) { + expected := errors.New("mocked error") + reso := &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, expected + }, + } + txp := NewHTTPTransportWithResolver(model.DiscardLogger, reso) + req, err := http.NewRequest("GET", "http://x.org", nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if !errors.Is(err, expected) { + t.Fatal("unexpected err") + } + if resp != nil { + t.Fatal("expected nil resp") + } +} + func TestHTTPTransportErrWrapper(t *testing.T) { t.Run("RoundTrip", func(t *testing.T) { t.Run("with failure", func(t *testing.T) { diff --git a/internal/netxlite/resolvercore.go b/internal/netxlite/resolvercore.go index 7640d99..b6149df 100644 --- a/internal/netxlite/resolvercore.go +++ b/internal/netxlite/resolvercore.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net" + "net/http" "strings" "time" @@ -30,6 +31,15 @@ func NewResolverStdlib(logger model.DebugLogger, wrappers ...model.DNSTransportW return WrapResolver(logger, newResolverSystem(wrappers...)) } +// NewParallelDNSOverHTTPSResolver creates a new DNS over HTTPS resolver +// that uses the standard library for all operations. This function constructs +// all the building blocks and calls WrapResolver on the returned resolver. +func NewParallelDNSOverHTTPSResolver(logger model.DebugLogger, URL string) model.Resolver { + client := &http.Client{Transport: NewHTTPTransportStdlib(logger)} + txp := WrapDNSTransport(NewUnwrappedDNSOverHTTPSTransport(client, URL)) + return WrapResolver(logger, NewUnwrappedParallelResolver(txp)) +} + func newResolverSystem(wrappers ...model.DNSTransportWrapper) *resolverSystem { return &resolverSystem{ t: WrapDNSTransport(&dnsOverGetaddrinfoTransport{}, wrappers...), diff --git a/internal/netxlite/resolvercore_test.go b/internal/netxlite/resolvercore_test.go index b4b0a8a..e5a9b1f 100644 --- a/internal/netxlite/resolvercore_test.go +++ b/internal/netxlite/resolvercore_test.go @@ -65,6 +65,23 @@ func TestNewParallelResolverUDP(t *testing.T) { } } +func TestNewParallelDNSOverHTTPSResolver(t *testing.T) { + resolver := NewParallelDNSOverHTTPSResolver(log.Log, "https://1.1.1.1/dns-query") + idna := resolver.(*resolverIDNA) + logger := idna.Resolver.(*resolverLogger) + if logger.Logger != log.Log { + t.Fatal("invalid logger") + } + shortCircuit := logger.Resolver.(*resolverShortCircuitIPAddr) + errWrapper := shortCircuit.Resolver.(*resolverErrWrapper) + para := errWrapper.Resolver.(*ParallelResolver) + txp := para.Transport().(*dnsTransportErrWrapper) + dnsTxp := txp.DNSTransport.(*DNSOverHTTPSTransport) + if dnsTxp.Address() != "https://1.1.1.1/dns-query" { + t.Fatal("invalid address") + } +} + func TestResolverSystem(t *testing.T) { t.Run("Network", func(t *testing.T) { expected := "antani" diff --git a/internal/tracex/http_test.go b/internal/tracex/http_test.go index 67db7c7..87b5f02 100644 --- a/internal/tracex/http_test.go +++ b/internal/tracex/http_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "net" "net/http" "net/url" "testing" @@ -52,23 +51,6 @@ func TestHTTPTransportSaver(t *testing.T) { }) t.Run("RoundTrip", func(t *testing.T) { - startServer := func(t *testing.T, action filtering.HTTPAction) (net.Listener, *url.URL) { - server := &filtering.HTTPProxy{ - OnIncomingHost: func(host string) filtering.HTTPAction { - return action - }, - } - listener, err := server.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - URL := &url.URL{ - Scheme: "http", - Host: listener.Addr().String(), - Path: "/", - } - return listener, URL - } measureHTTP := func(t *testing.T, URL *url.URL) (*http.Response, *Saver, error) { saver := &Saver{} @@ -141,9 +123,9 @@ func TestHTTPTransportSaver(t *testing.T) { } t.Run("on success", func(t *testing.T) { - listener, URL := startServer(t, filtering.HTTPAction451) - defer listener.Close() - resp, saver, err := measureHTTP(t, URL) + server := filtering.NewHTTPServerCleartext(filtering.HTTPAction451) + defer server.Close() + resp, saver, err := measureHTTP(t, server.URL()) if err != nil { t.Fatal(err) } @@ -155,8 +137,8 @@ func TestHTTPTransportSaver(t *testing.T) { if len(events) != 2 { t.Fatal("unexpected number of events") } - validateRequest(t, events[0], URL) - validateResponseSuccess(t, events[1], URL) + validateRequest(t, events[0], server.URL()) + validateResponseSuccess(t, events[1], server.URL()) data, err := netxlite.ReadAllContext(context.Background(), resp.Body) if err != nil { t.Fatal(err) @@ -193,9 +175,9 @@ func TestHTTPTransportSaver(t *testing.T) { } t.Run("on round trip failure", func(t *testing.T) { - listener, URL := startServer(t, filtering.HTTPActionReset) - defer listener.Close() - resp, saver, err := measureHTTP(t, URL) + server := filtering.NewHTTPServerCleartext(filtering.HTTPActionReset) + defer server.Close() + resp, saver, err := measureHTTP(t, server.URL()) if err == nil || err.Error() != "connection_reset" { t.Fatal("unexpected err", err) } @@ -206,8 +188,8 @@ func TestHTTPTransportSaver(t *testing.T) { if len(events) != 2 { t.Fatal("unexpected number of events") } - validateRequest(t, events[0], URL) - validateResponseFailure(t, events[1], URL) + validateRequest(t, events[0], server.URL()) + validateResponseFailure(t, events[1], server.URL()) }) // Sometimes useful for testing diff --git a/script/nocopyreadall.bash b/script/nocopyreadall.bash index 999a894..ae8fedc 100755 --- a/script/nocopyreadall.bash +++ b/script/nocopyreadall.bash @@ -7,7 +7,7 @@ for file in $(find . -type f -name \*.go); do # implement safer wrappers for these functions. continue fi - if [ "$file" = "./internal/netxlite/filtering/tls_test.go" ]; then + if [ "$file" = "./internal/netxlite/filtering/http.go" ]; then # We're allowed to use ReadAll and Copy in this file to # avoid depending on netxlite, so we can use filtering # inside of netxlite's own test suite.