chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/internal/httpx/fetch_test.go

@@ -0,0 +1,154 @@
+package httpx_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+)
+
+func TestFetchResourceIntegration(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    "http://facebook.com/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResource(ctx, "/robots.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) <= 0 {
+		t.Fatal("Did not expect an empty resource")
+	}
+}
+
+func TestFetchResourceExpiredContext(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	data, err := (httpx.Client{
+		BaseURL:    "http://facebook.com/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResource(ctx, "/robots.txt")
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
+
+func TestFetchResourceAndVerifyIntegration(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    "https://github.com/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResourceAndVerify(
+		ctx,
+		"/measurement-kit/generic-assets/releases/download/20190426155936/generic-assets-20190426155936.tar.gz",
+		"34d8a9c8ab30c242469482dc280be832d8a06b4400f8927604dd361bf979b795",
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) <= 0 {
+		t.Fatal("Did not expect an empty resource")
+	}
+}
+
+func TestFetchResourceInvalidURL(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    "http://\t/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResource(ctx, "/robots.txt")
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
+
+func TestFetchResource400(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(400)
+		},
+	))
+	defer server.Close()
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		Authorization: "foobar",
+		BaseURL:       server.URL,
+		HTTPClient:    http.DefaultClient,
+		Logger:        log.Log,
+		UserAgent:     "ooniprobe-engine/0.1.0",
+	}).FetchResource(ctx, "")
+	if err == nil || !strings.HasSuffix(err.Error(), "400 Bad Request") {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
+
+func TestFetchResourceAndVerify400(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(400)
+		},
+	))
+	defer server.Close()
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    server.URL,
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResourceAndVerify(ctx, "", "abcde")
+	if err == nil || !strings.HasSuffix(err.Error(), "400 Bad Request") {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
+
+func TestFetchResourceAndVerifyInvalidSHA256(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    "https://github.com/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResourceAndVerify(
+		ctx,
+		"/measurement-kit/generic-assets/releases/download/20190426155936/generic-assets-20190426155936.tar.gz",
+		"34d8a9ceeb30c242469482dc280be832d8a06b4400f8927604dd361bf979b795",
+	)
+	if err == nil || !strings.HasPrefix(err.Error(), "httpx: SHA256 mismatch:") {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}