cleanup(jafar): do not depend on netx and urlgetter (#792)

There's no point in doing that. Also, once this change is merged, it becomes easier to cleanup/simplify netx.

See https://github.com/ooni/probe/issues/2121

internal/cmd/jafar/README.md

@@ -8,7 +8,7 @@ any system but it really only works on Linux.
 
 ## Building
 
-We use Go >= 1.16. Jafar also needs the C library headers,
+We use Go >= 1.18. Jafar also needs the C library headers,
 iptables installed, and root permissions.
 
 With Linux Alpine edge, you can compile Jafar with:
@@ -198,24 +198,21 @@ the client Hello message will cause the TLS handshake to fail.
 ### uncensored
 
 ```bash
-  -uncensored-resolver-url string
+  -uncensored-resolver-doh string
-    	URL of an hopefully uncensored resolver (default "dot://1.1.1.1:853")
+     URL of an hopefully uncensored DoH resolver (default "https://1.1.1.1/dns-query")
 ```
 
 The HTTP, DNS, and TLS proxies need to resolve domain names. If you setup DNS
 censorship, they may be affected as well. To avoid this issue, we use a different
-resolver for them, which by default is `dot://1.1.1.1:853`. You can change such
+resolver for them, which by default is the one shown above. You can change such
-default by using the `-uncensored-resolver-url` command line flag. The input
+default by using the `-uncensored-resolver-doh` command line flag. The input
-URL is `<transport>://<domain>[:<port>][/<path>]`. Here are some examples:
+URL is an HTTPS URL pointing to a DoH server. Here are some examples:
 
-* `system:///` uses the system resolver (i.e. `getaddrinfo`)
+* `https://dns.google/dns-query`
-* `udp://8.8.8.8:53` uses DNS over UDP
+* `https://dns.quad9.net/dns-query`
-* `tcp://8.8.8.8:53` used DNS over TCP
-* `dot://8.8.8.8:853` uses DNS over TLS
-* `https://dns.google/dns-query` uses DNS over HTTPS
 
-So, for example, if you are using Jafar to censor `1.1.1.1:853`, then you
+So, for example, if you are using Jafar to censor `1.1.1.1:443`, then you
-most likely want to use `-uncensored-resolver-url`.
+most likely want to use `-uncensored-resolver-doh`.
 
 ## Examples
 

internal/cmd/jafar/httpproxy/httpproxy_test.go

@@ -42,7 +42,7 @@ func TestLoop(t *testing.T) {
 }
 
 func TestListenError(t *testing.T) {
-	proxy := NewCensoringProxy([]string{""}, uncensored.DefaultClient)
+	proxy := NewCensoringProxy([]string{""}, uncensored.NewClient("https://1.1.1.1/dns-query"))
 	server, addr, err := proxy.Start("8.8.8.8:80")
 	if err == nil {
 		t.Fatal("expected an error here")
@@ -56,7 +56,7 @@ func TestListenError(t *testing.T) {
 }
 
 func newproxy(t *testing.T, blocked string) (*http.Server, net.Addr) {
-	proxy := NewCensoringProxy([]string{blocked}, uncensored.DefaultClient)
+	proxy := NewCensoringProxy([]string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query"))
 	server, addr, err := proxy.Start("127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)

internal/cmd/jafar/iptables/iptables_integration_test.go

@@ -240,7 +240,7 @@ func TestHijackDNS(t *testing.T) {
 	}
 	resolver := resolver.NewCensoringResolver(
 		[]string{"ooni.io"}, nil, nil,
-		uncensored.Must(uncensored.NewClient("dot://1.1.1.1:853")),
+		uncensored.NewClient("https://1.1.1.1/dns-query"),
 	)
 	server, err := resolver.Start("127.0.0.1:0")
 	if err != nil {

internal/cmd/jafar/main.go

@@ -61,7 +61,7 @@ var (
 	tlsProxyAddress *string
 	tlsProxyBlock   flagx.StringArray
 
-	uncensoredResolverURL *string
+	uncensoredResolverDoH *string
 )
 
 func init() {
@@ -167,9 +167,9 @@ func init() {
 	)
 
 	// uncensored
-	uncensoredResolverURL = flag.String(
+	uncensoredResolverDoH = flag.String(
-		"uncensored-resolver-url", "dot://1.1.1.1:853",
+		"uncensored-resolver-doh", "https://1.1.1.1/dns-query",
-		"URL of an hopefully uncensored resolver",
+		"URL of an hopefully uncensored DoH resolver",
 	)
 }
 
@@ -234,9 +234,7 @@ func tlsProxyStart(uncensored *uncensored.Client) net.Listener {
 }
 
 func newUncensoredClient() *uncensored.Client {
-	clnt, err := uncensored.NewClient(*uncensoredResolverURL)
+	return uncensored.NewClient(*uncensoredResolverDoH)
-	runtimex.PanicOnError(err, "uncensored.NewClient failed")
-	return clnt
 }
 
 func mustx(err error, message string, osExit func(int)) {

internal/cmd/jafar/resolver/resolver_test.go

@@ -45,14 +45,14 @@ func TestLookupFailure(t *testing.T) {
 
 func TestFailureNoQuestion(t *testing.T) {
 	resolver := NewCensoringResolver(
-		nil, nil, nil, uncensored.DefaultClient,
+		nil, nil, nil, uncensored.NewClient("https://1.1.1.1/dns-query"),
 	)
 	resolver.ServeDNS(&fakeResponseWriter{t: t}, new(dns.Msg))
 }
 
 func TestListenFailure(t *testing.T) {
 	resolver := NewCensoringResolver(
-		nil, nil, nil, uncensored.DefaultClient,
+		nil, nil, nil, uncensored.NewClient("https://1.1.1.1/dns-query"),
 	)
 	server, err := resolver.Start("8.8.8.8:53")
 	if err == nil {
@@ -66,9 +66,7 @@ func TestListenFailure(t *testing.T) {
 func newresolver(t *testing.T, blocked, hijacked, ignored []string) *dns.Server {
 	resolver := NewCensoringResolver(
 		blocked, hijacked, ignored,
-		// using faster dns because dot here causes miekg/dns's
+		uncensored.NewClient("https://1.1.1.1/dns-query"),
-		// dns.Exchange to timeout and I don't want more complexity
-		uncensored.Must(uncensored.NewClient("system:///")),
 	)
 	server, err := resolver.Start("127.0.0.1:0")
 	if err != nil {

internal/cmd/jafar/tlsproxy/tlsproxy_test.go

@@ -94,7 +94,7 @@ func TestFailWriteAfterConnect(t *testing.T) {
 
 func TestListenError(t *testing.T) {
 	proxy := NewCensoringProxy(
-		[]string{""}, uncensored.DefaultClient,
+		[]string{""}, uncensored.NewClient("https://1.1.1.1/dns-query"),
 	)
 	listener, err := proxy.Start("8.8.8.8:80")
 	if err == nil {
@@ -107,7 +107,7 @@ func TestListenError(t *testing.T) {
 
 func newproxy(t *testing.T, blocked string) net.Listener {
 	proxy := NewCensoringProxy(
-		[]string{blocked}, uncensored.DefaultClient,
+		[]string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query"),
 	)
 	listener, err := proxy.Start("127.0.0.1:0")
 	if err != nil {

internal/cmd/jafar/uncensored/uncensored.go

@@ -9,10 +9,8 @@ import (
 	"net/http"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
-	"github.com/ooni/probe-cli/v3/internal/engine/netx"
 	"github.com/ooni/probe-cli/v3/internal/model"
-	"github.com/ooni/probe-cli/v3/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
 )
 
 // Client is DNS, HTTP, and TCP client.
@@ -23,35 +21,15 @@ type Client struct {
 }
 
 // NewClient creates a new Client.
-func NewClient(resolverURL string) (*Client, error) {
+func NewClient(resolverURL string) *Client {
-	configuration, err := urlgetter.Configurer{
+	dnsClient := netxlite.NewParallelDNSOverHTTPSResolver(log.Log, resolverURL)
-		Config: urlgetter.Config{
-			ResolverURL: resolverURL,
-		},
-		Logger: log.Log,
-	}.NewConfiguration()
-	if err != nil {
-		return nil, err
-	}
 	return &Client{
-		dnsClient:     configuration.DNSClient,
+		dnsClient:     dnsClient,
-		httpTransport: netx.NewHTTPTransport(configuration.HTTPConfig),
+		httpTransport: netxlite.NewHTTPTransportWithResolver(log.Log, dnsClient),
-		dialer:        netx.NewDialer(configuration.HTTPConfig),
+		dialer:        netxlite.NewDialerWithResolver(log.Log, dnsClient),
-	}, nil
+	}
 }
 
-// Must panics if it's not possible to create a Client. Usually you should
-// use it like `uncensored.Must(uncensored.NewClient(URL))`.
-func Must(client *Client, err error) *Client {
-	runtimex.PanicOnError(err, "cannot create uncensored client")
-	return client
-}
-
-// DefaultClient is the default client for DNS, HTTP, and TCP.
-var DefaultClient = Must(NewClient(""))
-
-var _ model.Resolver = DefaultClient
-
 // Address implements Resolver.Address
 func (c *Client) Address() string {
 	return c.dnsClient.Address()
@@ -77,15 +55,11 @@ func (c *Client) Network() string {
 	return c.dnsClient.Network()
 }
 
-var _ model.Dialer = DefaultClient
-
 // DialContext implements Dialer.DialContext
 func (c *Client) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
 	return c.dialer.DialContext(ctx, network, address)
 }
 
-var _ model.HTTPTransport = DefaultClient
-
 // CloseIdleConnections implement HTTPRoundTripper.CloseIdleConnections
 func (c *Client) CloseIdleConnections() {
 	c.dnsClient.CloseIdleConnections()

internal/cmd/jafar/uncensored/uncensored_test.go

@@ -10,16 +10,13 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
 )
 
-func TestGood(t *testing.T) {
+func TestNewClient(t *testing.T) {
-	client, err := NewClient("dot://1.1.1.1:853")
+	client := NewClient("https://1.1.1.1/dns-query")
-	if err != nil {
-		t.Fatal(err)
-	}
 	defer client.CloseIdleConnections()
-	if client.Address() != "1.1.1.1:853" {
+	if client.Address() != "https://1.1.1.1/dns-query" {
 		t.Fatal("invalid address")
 	}
-	if client.Network() != "dot" {
+	if client.Network() != "doh" {
 		t.Fatal("invalid network")
 	}
 	ctx := context.Background()
@@ -64,13 +61,3 @@ func TestGood(t *testing.T) {
 		t.Fatal("not the expected body")
 	}
 }
-
-func TestNewClientFailure(t *testing.T) {
-	clnt, err := NewClient("antani:///")
-	if err == nil {
-		t.Fatal("expected an error here")
-	}
-	if clnt != nil {
-		t.Fatal("expected nil client here")
-	}
-}

internal/netxlite/filtering/dns.go

@@ -153,14 +153,28 @@ func (p *DNSServer) nxdomain(query *dns.Msg) *dns.Msg {
 }
 
 func (p *DNSServer) localHost(query *dns.Msg) *dns.Msg {
-	return p.compose(query, net.IPv6loopback, net.IPv4(127, 0, 0, 1))
+	return dnsComposeResponse(query, net.IPv6loopback, net.IPv4(127, 0, 0, 1))
 }
 
 func (p *DNSServer) empty(query *dns.Msg) *dns.Msg {
-	return p.compose(query)
+	return dnsComposeResponse(query)
 }
 
-func (p *DNSServer) compose(query *dns.Msg, ips ...net.IP) *dns.Msg {
+func dnsComposeQuery(domain string, qtype uint16) *dns.Msg {
+	question := dns.Question{
+		Name:   dns.Fqdn(domain),
+		Qtype:  qtype,
+		Qclass: dns.ClassINET,
+	}
+	query := &dns.Msg{}
+	query.RecursionDesired = true
+	query.Question = make([]dns.Question, 1)
+	query.Question[0] = question
+	query.Id = dns.Id()
+	return query
+}
+
+func dnsComposeResponse(query *dns.Msg, ips ...net.IP) *dns.Msg {
 	runtimex.PanicIfTrue(len(query.Question) != 1, "expecting a single question")
 	question := query.Question[0]
 	reply := new(dns.Msg)
@@ -205,5 +219,5 @@ func (p *DNSServer) cache(name string, query *dns.Msg) *dns.Msg {
 	if len(ipAddrs) <= 0 {
 		return p.nxdomain(query)
 	}
-	return p.compose(query, ipAddrs...)
+	return dnsComposeResponse(query, ipAddrs...)
 }

internal/netxlite/filtering/http.go

@@ -1,24 +1,23 @@
 package filtering
 
 import (
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"io"
 	"net"
 	"net/http"
-	"net/http/httputil"
 	"net/url"
 
+	"github.com/google/martian/v3/mitm"
+	"github.com/miekg/dns"
 	"github.com/ooni/probe-cli/v3/internal/runtimex"
 )
 
-// TODO(bassosimone): remove HTTPActionPass since we want integration tests
+// HTTPAction is an HTTP filtering action that this server should take.
-// to only run locally to make them much more predictable.
-
-// HTTPAction is an HTTP filtering action that this proxy should take.
 type HTTPAction string
 
 const (
-	// HTTPActionPass passes the traffic to the destination.
-	HTTPActionPass = HTTPAction("pass")
-
 	// HTTPActionReset resets the connection.
 	HTTPActionReset = HTTPAction("reset")
 
@@ -30,25 +29,91 @@ const (
 
 	// HTTPAction451 causes the proxy to return a 451 error.
 	HTTPAction451 = HTTPAction("451")
+
+	// HTTPActionDoH causes the proxy to return a sensible reply
+	// with static IP addresses if the request is DoH.
+	HTTPActionDoH = HTTPAction("doh")
 )
 
-// HTTPProxy is a proxy that routes traffic depending on the
+// HTTPServer is a server that implements filtering policies.
-// host header and may implement filtering policies.
+type HTTPServer struct {
-type HTTPProxy struct {
+	// action is the action to implement.
-	// OnIncomingHost is the MANDATORY hook called whenever we have
+	action HTTPAction
-	// successfully received an HTTP request.
+
-	OnIncomingHost func(host string) HTTPAction
+	// cert is the fake CA certificate.
+	cert *x509.Certificate
+
+	// config is the config to generate certificates on the fly.
+	config *mitm.Config
+
+	// privkey is the private key that signed the cert.
+	privkey *rsa.PrivateKey
+
+	// server is the underlying server.
+	server *http.Server
+
+	// url contains the server URL
+	url *url.URL
 }
 
-// Start starts the proxy.
+// NewHTTPServerCleartext creates a new HTTPServer using cleartext HTTP.
-func (p *HTTPProxy) Start(address string) (net.Listener, error) {
+func NewHTTPServerCleartext(action HTTPAction) *HTTPServer {
-	listener, err := net.Listen("tcp", address)
+	return newHTTPOrHTTPSServer(action, false)
-	if err != nil {
+}
-		return nil, err
+
+// NewHTTPServerTLS creates a new HTTP server using HTTPS.
+func NewHTTPServerTLS(action HTTPAction) *HTTPServer {
+	return newHTTPOrHTTPSServer(action, true)
+}
+
+// Close closes the server ASAP.
+func (p *HTTPServer) Close() error {
+	return p.server.Close()
+}
+
+// URL returns the server's URL
+func (p *HTTPServer) URL() *url.URL {
+	return p.url
+}
+
+// TLSConfig returns a suitable base TLS config for the client.
+func (p *HTTPServer) TLSConfig() *tls.Config {
+	config := &tls.Config{}
+	if p.cert != nil {
+		o := x509.NewCertPool()
+		o.AddCert(p.cert)
+		config.RootCAs = o
 	}
-	server := &http.Server{Handler: p}
+	return config
-	go server.Serve(listener)
+}
-	return listener, nil
+
+// newHTTPOrHTTPSServer is an internal factory for creating a new instance.
+func newHTTPOrHTTPSServer(action HTTPAction, enableTLS bool) *HTTPServer {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	runtimex.PanicOnError(err, "net.Listen failed")
+	srv := &HTTPServer{
+		action:  action,
+		cert:    nil,
+		config:  nil,
+		privkey: nil,
+		server:  nil,
+		url: &url.URL{
+			Scheme: "",
+			Host:   listener.Addr().String(),
+		},
+	}
+	srv.server = &http.Server{Handler: srv}
+	switch enableTLS {
+	case false:
+		srv.url.Scheme = "http"
+		go srv.server.Serve(listener)
+	case true:
+		srv.url.Scheme = "https"
+		srv.cert, srv.privkey, srv.config = tlsConfigMITM()
+		srv.server.TLSConfig = srv.config.TLS()
+		go srv.server.ServeTLS(listener, "", "") // using server.TLSConfig
+	}
+	return srv
 }
 
 // HTTPBlockPage451 is the block page returned along with status 451
@@ -60,34 +125,22 @@ var HTTPBlockpage451 = []byte(`<html><head>
 </body></html>
 `)
 
-const httpProxyProduct = "jafar/0.1.0"
-
 // ServeHTTP serves HTTP requests
-func (p *HTTPProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+func (p *HTTPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// Implementation note: use Via header to detect in a loose way
+	switch p.action {
-	// requests originated by us and directed to us.
-	if r.Header.Get("Via") == httpProxyProduct || r.Host == "" {
-		w.WriteHeader(http.StatusBadRequest)
-		return
-	}
-	p.handle(w, r)
-}
-
-func (p *HTTPProxy) handle(w http.ResponseWriter, r *http.Request) {
-	switch policy := p.OnIncomingHost(r.Host); policy {
-	case HTTPActionPass:
-		p.proxy(w, r)
 	case HTTPActionReset, HTTPActionTimeout, HTTPActionEOF:
-		p.hijack(w, r, policy)
+		p.hijack(w, r, p.action)
 	case HTTPAction451:
 		w.WriteHeader(http.StatusUnavailableForLegalReasons)
 		w.Write(HTTPBlockpage451)
+	case HTTPActionDoH:
+		httpServeDNSOverHTTPS(w, r)
 	default:
 		w.WriteHeader(http.StatusInternalServerError)
 	}
 }
 
-func (p *HTTPProxy) hijack(w http.ResponseWriter, r *http.Request, policy HTTPAction) {
+func (p *HTTPServer) hijack(w http.ResponseWriter, r *http.Request, policy HTTPAction) {
 	// Note:
 	//
 	// 1. we assume we can hihack the connection
@@ -109,12 +162,22 @@ func (p *HTTPProxy) hijack(w http.ResponseWriter, r *http.Request, policy HTTPAc
 	}
 }
 
-func (p *HTTPProxy) proxy(w http.ResponseWriter, r *http.Request) {
+func httpPanicToInternalServerError(w http.ResponseWriter) {
-	r.Header.Add("Via", httpProxyProduct) // see ServeHTTP
+	if r := recover(); r != nil {
-	proxy := httputil.NewSingleHostReverseProxy(&url.URL{
+		w.WriteHeader(500)
-		Host:   r.Host,
+	}
-		Scheme: "http",
+}
-	})
+
-	proxy.Transport = http.DefaultTransport
+func httpServeDNSOverHTTPS(w http.ResponseWriter, r *http.Request) {
-	proxy.ServeHTTP(w, r)
+	defer httpPanicToInternalServerError(w)
+	rawQuery, err := io.ReadAll(r.Body)
+	runtimex.PanicOnError(err, "io.ReadAll failed")
+	query := &dns.Msg{}
+	err = query.Unpack(rawQuery)
+	runtimex.PanicOnError(err, "query.Unpack failed")
+	runtimex.PanicIfTrue(query.Response, "is a response")
+	response := dnsComposeResponse(query, net.IPv4(8, 8, 8, 8), net.IPv4(8, 8, 4, 4))
+	rawResponse, err := response.Pack()
+	runtimex.PanicOnError(err, "response.Pack failed")
+	w.Write(rawResponse)
 }

internal/netxlite/filtering/http_test.go

@@ -1,155 +1,134 @@
 package filtering
 
 import (
+	"bytes"
 	"context"
+	"crypto/tls"
 	"errors"
-	"net"
+	"io"
 	"net/http"
 	"net/url"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/apex/log"
+	"github.com/miekg/dns"
+	"github.com/ooni/probe-cli/v3/internal/model/mocks"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
 	"github.com/ooni/probe-cli/v3/internal/runtimex"
 )
 
-func TestHTTPProxy(t *testing.T) {
+func TestHTTPServer(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skip test in short mode")
-	}
-	newproxy := func(action HTTPAction) (net.Listener, error) {
-		p := &HTTPProxy{
-			OnIncomingHost: func(host string) HTTPAction {
-				return action
-			},
-		}
-		return p.Start("127.0.0.1:0")
-	}
 
-	httpGET := func(ctx context.Context, addr net.Addr, host string) (*http.Response, error) {
+	httpGET := func(ctx context.Context, method string, URL *url.URL, host string,
-		txp := netxlite.NewHTTPTransportStdlib(log.Log)
+		config *tls.Config, requestBody []byte) (*http.Response, error) {
-		clnt := &http.Client{Transport: txp}
+		txp := &http.Transport{
-		URL := &url.URL{
+			TLSClientConfig: config,
-			Scheme: "http",
-			Host:   addr.String(),
-			Path:   "/",
 		}
-		req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil)
+		if config != nil {
+			config.ServerName = host
+		}
+		clnt := &http.Client{Transport: txp}
+		req, err := http.NewRequestWithContext(
+			ctx, method, URL.String(), bytes.NewReader(requestBody))
 		runtimex.PanicOnError(err, "http.NewRequest failed")
 		req.Host = host
 		return clnt.Do(req)
 	}
 
-	t.Run("HTTPActionPass", func(t *testing.T) {
-		ctx := context.Background()
-		listener, err := newproxy(HTTPActionPass)
-		if err != nil {
-			t.Fatal(err)
-		}
-		resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it")
-		if err != nil {
-			t.Fatal(err)
-		}
-		if resp.StatusCode != 200 {
-			t.Fatal("unexpected status code", resp.StatusCode)
-		}
-		resp.Body.Close()
-		listener.Close()
-	})
-
-	t.Run("HTTPActionPass with self connect", func(t *testing.T) {
-		ctx := context.Background()
-		listener, err := newproxy(HTTPActionPass)
-		if err != nil {
-			t.Fatal(err)
-		}
-		resp, err := httpGET(ctx, listener.Addr(), listener.Addr().String())
-		if err != nil {
-			t.Fatal(err)
-		}
-		if resp.StatusCode != 400 {
-			t.Fatal("unexpected status code", resp.StatusCode)
-		}
-		resp.Body.Close()
-		listener.Close()
-	})
-
 	t.Run("HTTPActionReset", func(t *testing.T) {
 		ctx := context.Background()
-		listener, err := newproxy(HTTPActionReset)
+		srvr := NewHTTPServerCleartext(HTTPActionReset)
-		if err != nil {
+		resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil)
-			t.Fatal(err)
+		if netxlite.NewTopLevelGenericErrWrapper(err).Error() != netxlite.FailureConnectionReset {
-		}
-		resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it")
-		if err == nil || !strings.HasSuffix(err.Error(), netxlite.FailureConnectionReset) {
 			t.Fatal("unexpected err", err)
 		}
 		if resp != nil {
 			t.Fatal("expected nil resp")
 		}
-		listener.Close()
+		srvr.Close()
 	})
 
 	t.Run("HTTPActionTimeout", func(t *testing.T) {
 		ctx := context.Background()
-		ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
+		ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
 		defer cancel()
-		listener, err := newproxy(HTTPActionTimeout)
+		srvr := NewHTTPServerCleartext(HTTPActionTimeout)
-		if err != nil {
+		resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil)
-			t.Fatal(err)
-		}
-		resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it")
 		if !errors.Is(err, context.DeadlineExceeded) {
 			t.Fatal("unexpected err", err)
 		}
 		if resp != nil {
 			t.Fatal("expected nil resp")
 		}
-		listener.Close()
+		srvr.Close()
 	})
 
 	t.Run("HTTPActionEOF", func(t *testing.T) {
 		ctx := context.Background()
-		listener, err := newproxy(HTTPActionEOF)
+		srvr := NewHTTPServerCleartext(HTTPActionEOF)
-		if err != nil {
+		resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil)
-			t.Fatal(err)
+		if !errors.Is(err, io.EOF) {
-		}
-		resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it")
-		if err == nil || !strings.HasSuffix(err.Error(), netxlite.FailureEOFError) {
 			t.Fatal("unexpected err", err)
 		}
 		if resp != nil {
 			t.Fatal("expected nil resp")
 		}
-		listener.Close()
+		srvr.Close()
 	})
 
 	t.Run("HTTPAction451", func(t *testing.T) {
 		ctx := context.Background()
-		listener, err := newproxy(HTTPAction451)
+		srvr := NewHTTPServerCleartext(HTTPAction451)
-		if err != nil {
+		resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil)
-			t.Fatal(err)
-		}
-		resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it")
 		if err != nil {
 			t.Fatal(err)
 		}
 		if resp.StatusCode != 451 {
 			t.Fatal("unexpected status code", resp.StatusCode)
 		}
+		data, err := netxlite.ReadAllContext(ctx, resp.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(HTTPBlockpage451, data) {
+			t.Fatal("unexpected data")
+		}
 		resp.Body.Close()
-		listener.Close()
+		srvr.Close()
+	})
+
+	t.Run("HTTPActionDoH", func(t *testing.T) {
+		ctx := context.Background()
+		srvr := NewHTTPServerTLS(HTTPActionDoH)
+		query := dnsComposeQuery("nexa.polito.it", dns.TypeA)
+		rawQuery, err := query.Pack()
+		if err != nil {
+			t.Fatal(err)
+		}
+		resp, err := httpGET(ctx, "POST", srvr.URL(), "dns.google", srvr.TLSConfig(), rawQuery)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if resp.StatusCode != 200 {
+			t.Fatal("unexpected status code", resp.StatusCode)
+		}
+		data, err := netxlite.ReadAllContext(ctx, resp.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		response := &dns.Msg{}
+		if err := response.Unpack(data); err != nil {
+			t.Fatal(err)
+		}
+		// It suffices to see it's a DNS response
+		resp.Body.Close()
+		srvr.Close()
 	})
 
 	t.Run("unknown action", func(t *testing.T) {
 		ctx := context.Background()
-		listener, err := newproxy("")
+		srvr := NewHTTPServerCleartext("")
-		if err != nil {
+		resp, err := httpGET(ctx, "GET", srvr.URL(), "nexa.polito.it", srvr.TLSConfig(), nil)
-			t.Fatal(err)
-		}
-		resp, err := httpGET(ctx, listener.Addr(), "nexa.polito.it")
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -157,17 +136,30 @@ func TestHTTPProxy(t *testing.T) {
 			t.Fatal("unexpected status code", resp.StatusCode)
 		}
 		resp.Body.Close()
-		listener.Close()
+		srvr.Close()
-	})
-
-	t.Run("Start fails on an invalid address", func(t *testing.T) {
-		p := &HTTPProxy{}
-		listener, err := p.Start("127.0.0.1")
-		if err == nil || !strings.HasSuffix(err.Error(), "missing port in address") {
-			t.Fatal("unexpected err", err)
-		}
-		if listener != nil {
-			t.Fatal("expected nil listener")
-		}
 	})
 }
+
+type httpResponseWriter struct {
+	http.ResponseWriter
+	code int
+}
+
+func (w *httpResponseWriter) WriteHeader(statusCode int) {
+	w.code = statusCode
+}
+
+func TestHTTPServeDNSOverHTTPSPanic(t *testing.T) {
+	w := &httpResponseWriter{}
+	req := &http.Request{
+		Body: io.NopCloser(&mocks.Reader{
+			MockRead: func(b []byte) (int, error) {
+				return 0, io.ErrUnexpectedEOF
+			},
+		}),
+	}
+	httpServeDNSOverHTTPS(w, req)
+	if w.code != 500 {
+		t.Fatal("did not intercept the panic")
+	}
+}

internal/netxlite/filtering/tls.go

@@ -66,14 +66,19 @@ type TLSServer struct {
 	privkey *rsa.PrivateKey
 }
 
-// NewTLSServer creates and starts a new TLSServer that executes
+func tlsConfigMITM() (*x509.Certificate, *rsa.PrivateKey, *mitm.Config) {
-// the given action during the TLS handshake.
-func NewTLSServer(action TLSAction) *TLSServer {
-	done := make(chan bool)
 	cert, privkey, err := mitm.NewAuthority("jafar", "OONI", 24*time.Hour)
 	runtimex.PanicOnError(err, "mitm.NewAuthority failed")
 	config, err := mitm.NewConfig(cert, privkey)
 	runtimex.PanicOnError(err, "mitm.NewConfig failed")
+	return cert, privkey, config
+}
+
+// NewTLSServer creates and starts a new TLSServer that executes
+// the given action during the TLS handshake.
+func NewTLSServer(action TLSAction) *TLSServer {
+	done := make(chan bool)
+	cert, privkey, config := tlsConfigMITM()
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	runtimex.PanicOnError(err, "net.Listen failed")
 	ctx, cancel := context.WithCancel(context.Background())
@@ -139,13 +144,8 @@ func (p *TLSServer) handle(ctx context.Context, tcpConn net.Conn) {
 		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 			switch p.action {
 			case TLSActionTimeout:
-				select {
+				err := p.timeout(ctx, tcpConn)
-				case <-time.After(300 * time.Second):
+				return nil, err
-					return nil, errors.New("timing out the connection")
-				case <-ctx.Done():
-					p.reset(tcpConn)
-					return nil, ctx.Err()
-				}
 			case TLSActionAlertInternalError:
 				p.alert(tcpConn, tlsAlertInternalError)
 				return nil, errors.New("already sent alert")
@@ -170,6 +170,14 @@ func (p *TLSServer) handle(ctx context.Context, tcpConn net.Conn) {
 	tlsConn.Close()
 }
 
+func (p *TLSServer) timeout(ctx context.Context, tcpConn net.Conn) error {
+	ctx, cancel := context.WithTimeout(ctx, 300*time.Second)
+	defer cancel()
+	<-ctx.Done()
+	p.reset(tcpConn)
+	return ctx.Err()
+}
+
 func (p *TLSServer) reset(conn net.Conn) {
 	if tc, good := conn.(*net.TCPConn); good {
 		tc.SetLinger(0)

internal/netxlite/filtering/tls_test.go

@@ -116,7 +116,7 @@ func TestTLSServer(t *testing.T) {
 				t.Fatal(err)
 			}
 			defer conn.Close()
-			data, err := io.ReadAll(conn)
+			data, err := netxlite.ReadAllContext(context.Background(), conn)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -134,7 +134,7 @@ func TestTLSServer(t *testing.T) {
 				t.Fatal(err)
 			}
 			defer conn.Close()
-			data, err := io.ReadAll(conn)
+			data, err := netxlite.ReadAllContext(context.Background(), conn)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/netxlite/http.go

@@ -105,6 +105,15 @@ func (txp *httpTransportConnectionsCloser) CloseIdleConnections() {
 	txp.TLSDialer.CloseIdleConnections()
 }
 
+// NewHTTPTransportWithResolver creates a new HTTP transport using
+// the stdlib for everything but the given resolver.
+func NewHTTPTransportWithResolver(logger model.DebugLogger, reso model.Resolver) model.HTTPTransport {
+	dialer := NewDialerWithResolver(logger, reso)
+	thx := NewTLSHandshakerStdlib(logger)
+	tlsDialer := NewTLSDialer(dialer, thx)
+	return NewHTTPTransport(logger, dialer, tlsDialer)
+}
+
 // NewHTTPTransport combines NewOOHTTPBaseTransport and WrapHTTPTransport.
 //
 // This factory and NewHTTPTransportStdlib are the recommended

internal/netxlite/http_test.go

@@ -16,6 +16,27 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/model/mocks"
 )
 
+func TestNewHTTPTransportWithResolver(t *testing.T) {
+	expected := errors.New("mocked error")
+	reso := &mocks.Resolver{
+		MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
+			return nil, expected
+		},
+	}
+	txp := NewHTTPTransportWithResolver(model.DiscardLogger, reso)
+	req, err := http.NewRequest("GET", "http://x.org", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := txp.RoundTrip(req)
+	if !errors.Is(err, expected) {
+		t.Fatal("unexpected err")
+	}
+	if resp != nil {
+		t.Fatal("expected nil resp")
+	}
+}
+
 func TestHTTPTransportErrWrapper(t *testing.T) {
 	t.Run("RoundTrip", func(t *testing.T) {
 		t.Run("with failure", func(t *testing.T) {

internal/netxlite/resolvercore.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/http"
 	"strings"
 	"time"
 
@@ -30,6 +31,15 @@ func NewResolverStdlib(logger model.DebugLogger, wrappers ...model.DNSTransportW
 	return WrapResolver(logger, newResolverSystem(wrappers...))
 }
 
+// NewParallelDNSOverHTTPSResolver creates a new DNS over HTTPS resolver
+// that uses the standard library for all operations. This function constructs
+// all the building blocks and calls WrapResolver on the returned resolver.
+func NewParallelDNSOverHTTPSResolver(logger model.DebugLogger, URL string) model.Resolver {
+	client := &http.Client{Transport: NewHTTPTransportStdlib(logger)}
+	txp := WrapDNSTransport(NewUnwrappedDNSOverHTTPSTransport(client, URL))
+	return WrapResolver(logger, NewUnwrappedParallelResolver(txp))
+}
+
 func newResolverSystem(wrappers ...model.DNSTransportWrapper) *resolverSystem {
 	return &resolverSystem{
 		t: WrapDNSTransport(&dnsOverGetaddrinfoTransport{}, wrappers...),

internal/netxlite/resolvercore_test.go

@@ -65,6 +65,23 @@ func TestNewParallelResolverUDP(t *testing.T) {
 	}
 }
 
+func TestNewParallelDNSOverHTTPSResolver(t *testing.T) {
+	resolver := NewParallelDNSOverHTTPSResolver(log.Log, "https://1.1.1.1/dns-query")
+	idna := resolver.(*resolverIDNA)
+	logger := idna.Resolver.(*resolverLogger)
+	if logger.Logger != log.Log {
+		t.Fatal("invalid logger")
+	}
+	shortCircuit := logger.Resolver.(*resolverShortCircuitIPAddr)
+	errWrapper := shortCircuit.Resolver.(*resolverErrWrapper)
+	para := errWrapper.Resolver.(*ParallelResolver)
+	txp := para.Transport().(*dnsTransportErrWrapper)
+	dnsTxp := txp.DNSTransport.(*DNSOverHTTPSTransport)
+	if dnsTxp.Address() != "https://1.1.1.1/dns-query" {
+		t.Fatal("invalid address")
+	}
+}
+
 func TestResolverSystem(t *testing.T) {
 	t.Run("Network", func(t *testing.T) {
 		expected := "antani"

internal/tracex/http_test.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"testing"
@@ -52,23 +51,6 @@ func TestHTTPTransportSaver(t *testing.T) {
 	})
 
 	t.Run("RoundTrip", func(t *testing.T) {
-		startServer := func(t *testing.T, action filtering.HTTPAction) (net.Listener, *url.URL) {
-			server := &filtering.HTTPProxy{
-				OnIncomingHost: func(host string) filtering.HTTPAction {
-					return action
-				},
-			}
-			listener, err := server.Start("127.0.0.1:0")
-			if err != nil {
-				t.Fatal(err)
-			}
-			URL := &url.URL{
-				Scheme: "http",
-				Host:   listener.Addr().String(),
-				Path:   "/",
-			}
-			return listener, URL
-		}
 
 		measureHTTP := func(t *testing.T, URL *url.URL) (*http.Response, *Saver, error) {
 			saver := &Saver{}
@@ -141,9 +123,9 @@ func TestHTTPTransportSaver(t *testing.T) {
 		}
 
 		t.Run("on success", func(t *testing.T) {
-			listener, URL := startServer(t, filtering.HTTPAction451)
+			server := filtering.NewHTTPServerCleartext(filtering.HTTPAction451)
-			defer listener.Close()
+			defer server.Close()
-			resp, saver, err := measureHTTP(t, URL)
+			resp, saver, err := measureHTTP(t, server.URL())
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -155,8 +137,8 @@ func TestHTTPTransportSaver(t *testing.T) {
 			if len(events) != 2 {
 				t.Fatal("unexpected number of events")
 			}
-			validateRequest(t, events[0], URL)
+			validateRequest(t, events[0], server.URL())
-			validateResponseSuccess(t, events[1], URL)
+			validateResponseSuccess(t, events[1], server.URL())
 			data, err := netxlite.ReadAllContext(context.Background(), resp.Body)
 			if err != nil {
 				t.Fatal(err)
@@ -193,9 +175,9 @@ func TestHTTPTransportSaver(t *testing.T) {
 		}
 
 		t.Run("on round trip failure", func(t *testing.T) {
-			listener, URL := startServer(t, filtering.HTTPActionReset)
+			server := filtering.NewHTTPServerCleartext(filtering.HTTPActionReset)
-			defer listener.Close()
+			defer server.Close()
-			resp, saver, err := measureHTTP(t, URL)
+			resp, saver, err := measureHTTP(t, server.URL())
 			if err == nil || err.Error() != "connection_reset" {
 				t.Fatal("unexpected err", err)
 			}
@@ -206,8 +188,8 @@ func TestHTTPTransportSaver(t *testing.T) {
 			if len(events) != 2 {
 				t.Fatal("unexpected number of events")
 			}
-			validateRequest(t, events[0], URL)
+			validateRequest(t, events[0], server.URL())
-			validateResponseFailure(t, events[1], URL)
+			validateResponseFailure(t, events[1], server.URL())
 		})
 
 		// Sometimes useful for testing

script/nocopyreadall.bash

@@ -7,7 +7,7 @@ for file in $(find . -type f -name \*.go); do
 		# implement safer wrappers for these functions.
 		continue
 	fi
-	if [ "$file" = "./internal/netxlite/filtering/tls_test.go" ]; then
+	if [ "$file" = "./internal/netxlite/filtering/http.go" ]; then
 		# We're allowed to use ReadAll and Copy in this file to
 		# avoid depending on netxlite, so we can use filtering
 		# inside of netxlite's own test suite.