chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/resources/resources.go

@@ -0,0 +1,104 @@
+// Package resources contains code to download resources.
+package resources
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
+)
+
+// Client is a client for fetching resources.
+type Client struct {
+	// HTTPClient is the HTTP client to use.
+	HTTPClient *http.Client
+
+	// Logger is the logger to use.
+	Logger model.Logger
+
+	// OSMkdirAll allows testing os.MkdirAll failures.
+	OSMkdirAll func(path string, perm os.FileMode) error
+
+	// UserAgent is the user agent to use.
+	UserAgent string
+
+	// WorkDir is the directory where to save resources.
+	WorkDir string
+}
+
+// Ensure ensures that resources are downloaded and current.
+func (c *Client) Ensure(ctx context.Context) error {
+	mkdirall := c.OSMkdirAll
+	if mkdirall == nil {
+		mkdirall = os.MkdirAll
+	}
+	if err := mkdirall(c.WorkDir, 0700); err != nil {
+		return err
+	}
+	for name, resource := range All {
+		if err := c.EnsureForSingleResource(
+			ctx, name, resource, func(real, expected string) bool {
+				return real == expected
+			},
+			gzip.NewReader, ioutil.ReadAll,
+		); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// EnsureForSingleResource ensures that a single resource
+// is downloaded and is current.
+func (c *Client) EnsureForSingleResource(
+	ctx context.Context, name string, resource ResourceInfo,
+	equal func(real, expected string) bool,
+	gzipNewReader func(r io.Reader) (*gzip.Reader, error),
+	ioutilReadAll func(r io.Reader) ([]byte, error),
+) error {
+	fullpath := filepath.Join(c.WorkDir, name)
+	data, err := ioutil.ReadFile(fullpath)
+	if err == nil {
+		sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
+		if equal(sha256sum, resource.SHA256) {
+			return nil
+		}
+		c.Logger.Debugf("resources: %s is outdated", fullpath)
+	} else {
+		c.Logger.Debugf("resources: can't read %s: %s", fullpath, err.Error())
+	}
+	data, err = (httpx.Client{
+		BaseURL:    BaseURL,
+		HTTPClient: c.HTTPClient,
+		Logger:     c.Logger,
+		UserAgent:  c.UserAgent,
+	}).FetchResourceAndVerify(ctx, resource.URLPath, resource.GzSHA256)
+	if err != nil {
+		return err
+	}
+	c.Logger.Debugf("resources: uncompress %s", fullpath)
+	gzreader, err := gzipNewReader(bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+	defer gzreader.Close()              // we already have a sha256 for it
+	data, err = ioutilReadAll(gzreader) // small file
+	if err != nil {
+		return err
+	}
+	sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
+	if equal(sha256sum, resource.SHA256) == false {
+		return fmt.Errorf("resources: %s sha256 mismatch", fullpath)
+	}
+	c.Logger.Debugf("resources: overwrite %s", fullpath)
+	return ioutil.WriteFile(fullpath, data, 0600)
+}