chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/experiment/urlgetter/runner.go

@@ -0,0 +1,129 @@
+package urlgetter
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
+)
+
+const httpRequestFailed = "http_request_failed"
+
+// ErrHTTPRequestFailed indicates that the HTTP request failed.
+var ErrHTTPRequestFailed = &errorx.ErrWrapper{
+	Failure:    httpRequestFailed,
+	Operation:  errorx.TopLevelOperation,
+	WrappedErr: errors.New(httpRequestFailed),
+}
+
+// The Runner job is to run a single measurement
+type Runner struct {
+	Config     Config
+	HTTPConfig netx.Config
+	Target     string
+}
+
+// Run runs a measurement and returns the measurement result
+func (r Runner) Run(ctx context.Context) error {
+	targetURL, err := url.Parse(r.Target)
+	if err != nil {
+		return fmt.Errorf("urlgetter: invalid target URL: %w", err)
+	}
+	switch targetURL.Scheme {
+	case "http", "https":
+		return r.httpGet(ctx, r.Target)
+	case "dnslookup":
+		return r.dnsLookup(ctx, targetURL.Hostname())
+	case "tlshandshake":
+		return r.tlsHandshake(ctx, targetURL.Host)
+	case "tcpconnect":
+		return r.tcpConnect(ctx, targetURL.Host)
+	default:
+		return errors.New("unknown targetURL scheme")
+	}
+}
+
+// MaybeUserAgent returns ua if ua is not empty. Otherwise it
+// returns httpheader.RandomUserAgent().
+func MaybeUserAgent(ua string) string {
+	if ua == "" {
+		ua = httpheader.UserAgent()
+	}
+	return ua
+}
+
+func (r Runner) httpGet(ctx context.Context, url string) error {
+	// Implementation note: empty Method implies using the GET method
+	req, err := http.NewRequest(r.Config.Method, url, nil)
+	runtimex.PanicOnError(err, "http.NewRequest failed")
+	req = req.WithContext(ctx)
+	req.Header.Set("Accept", httpheader.Accept())
+	req.Header.Set("Accept-Language", httpheader.AcceptLanguage())
+	req.Header.Set("User-Agent", MaybeUserAgent(r.Config.UserAgent))
+	if r.Config.HTTPHost != "" {
+		req.Host = r.Config.HTTPHost
+	}
+	// Implementation note: the following cookiejar accepts all cookies
+	// from all domains. As such, would not be safe for usage where cookies
+	// matter, but it's totally fine for performing measurements.
+	jar, err := cookiejar.New(nil)
+	runtimex.PanicOnError(err, "cookiejar.New failed")
+	httpClient := &http.Client{
+		Jar:       jar,
+		Transport: netx.NewHTTPTransport(r.HTTPConfig),
+	}
+	if r.Config.NoFollowRedirects {
+		httpClient.CheckRedirect = func(*http.Request, []*http.Request) error {
+			return http.ErrUseLastResponse
+		}
+	}
+	defer httpClient.CloseIdleConnections()
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if _, err = io.Copy(ioutil.Discard, resp.Body); err != nil {
+		return err
+	}
+	// Implementation note: we shall check for this error once we have read the
+	// whole body. Even though we discard the body, we want to know whether we
+	// see any error when reading the body before inspecting the HTTP status code.
+	if resp.StatusCode >= 400 && r.Config.FailOnHTTPError {
+		return ErrHTTPRequestFailed
+	}
+	return nil
+}
+
+func (r Runner) dnsLookup(ctx context.Context, hostname string) error {
+	resolver := netx.NewResolver(r.HTTPConfig)
+	_, err := resolver.LookupHost(ctx, hostname)
+	return err
+}
+
+func (r Runner) tlsHandshake(ctx context.Context, address string) error {
+	tlsDialer := netx.NewTLSDialer(r.HTTPConfig)
+	conn, err := tlsDialer.DialTLSContext(ctx, "tcp", address)
+	if conn != nil {
+		conn.Close()
+	}
+	return err
+}
+
+func (r Runner) tcpConnect(ctx context.Context, address string) error {
+	dialer := netx.NewDialer(r.HTTPConfig)
+	conn, err := dialer.DialContext(ctx, "tcp", address)
+	if conn != nil {
+		conn.Close()
+	}
+	return err
+}