c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: merge probe-engine into probe-cli (#201)

Browse Source 
This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
2021-02-02 12:05:47 +01:00
committed by GitHub
parent b1ce300c8d
commit d57c78bc71
535 changed files with 66182 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/resources/README.md
+3
View File
@@ -0,0 +1,3 @@
# Package github.com/ooni/probe-engine/resources
This package contains code to download OONI resources.
internal/engine/resources/assets.go
+42
View File
@@ -0,0 +1,42 @@
package resources
const (
// Version contains the assets version.
Version = 20210129095811
// ASNDatabaseName is the ASN-DB file name
ASNDatabaseName = "asn.mmdb"
// CountryDatabaseName is country-DB file name
CountryDatabaseName = "country.mmdb"
// BaseURL is the asset's repository base URL
BaseURL = "https://github.com/"
)
// ResourceInfo contains information on a resource.
type ResourceInfo struct {
// URLPath is the resource's URL path.
URLPath string
// GzSHA256 is used to validate the downloaded file.
GzSHA256 string
// SHA256 is used to check whether the assets file
// stored locally is still up-to-date.
SHA256 string
}
// All contains info on all known assets.
var All = map[string]ResourceInfo{
"asn.mmdb": {
URLPath: "/ooni/probe-assets/releases/download/20210129095811/asn.mmdb.gz",
GzSHA256: "ef1759bf8b77128723436c4ec5a3d7f2e695fb5a959e741ba39012ced325132c",
SHA256: "0afa5afc48ba913933f17b11213c3044499c8338cf63b8f9af2778faa5875474",
},
"country.mmdb": {
URLPath: "/ooni/probe-assets/releases/download/20210129095811/country.mmdb.gz",
GzSHA256: "5d465224ab02242a8a79652161d2768e64dd91fc1ed840ca3d0746f4cd29a914",
SHA256: "b4aa1292d072d9b2631711e6d3ac69c1e89687b4d513d43a1c330a92b7345e4d",
},
}
internal/engine/resources/resources.go
+104
View File
@@ -0,0 +1,104 @@
// Package resources contains code to download resources.
package resources
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// Client is a client for fetching resources.
type Client struct {
// HTTPClient is the HTTP client to use.
HTTPClient *http.Client
// Logger is the logger to use.
Logger model.Logger
// OSMkdirAll allows testing os.MkdirAll failures.
OSMkdirAll func(path string, perm os.FileMode) error
// UserAgent is the user agent to use.
UserAgent string
// WorkDir is the directory where to save resources.
WorkDir string
}
// Ensure ensures that resources are downloaded and current.
func (c *Client) Ensure(ctx context.Context) error {
mkdirall := c.OSMkdirAll
if mkdirall == nil {
mkdirall = os.MkdirAll
}
if err := mkdirall(c.WorkDir, 0700); err != nil {
return err
}
for name, resource := range All {
if err := c.EnsureForSingleResource(
ctx, name, resource, func(real, expected string) bool {
return real == expected
},
gzip.NewReader, ioutil.ReadAll,
); err != nil {
return err
}
}
return nil
}
// EnsureForSingleResource ensures that a single resource
// is downloaded and is current.
func (c *Client) EnsureForSingleResource(
ctx context.Context, name string, resource ResourceInfo,
equal func(real, expected string) bool,
gzipNewReader func(r io.Reader) (*gzip.Reader, error),
ioutilReadAll func(r io.Reader) ([]byte, error),
) error {
fullpath := filepath.Join(c.WorkDir, name)
data, err := ioutil.ReadFile(fullpath)
if err == nil {
sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
if equal(sha256sum, resource.SHA256) {
return nil
}
c.Logger.Debugf("resources: %s is outdated", fullpath)
} else {
c.Logger.Debugf("resources: can't read %s: %s", fullpath, err.Error())
}
data, err = (httpx.Client{
BaseURL: BaseURL,
HTTPClient: c.HTTPClient,
Logger: c.Logger,
UserAgent: c.UserAgent,
}).FetchResourceAndVerify(ctx, resource.URLPath, resource.GzSHA256)
if err != nil {
return err
}
c.Logger.Debugf("resources: uncompress %s", fullpath)
gzreader, err := gzipNewReader(bytes.NewReader(data))
if err != nil {
return err
}
defer gzreader.Close() // we already have a sha256 for it
data, err = ioutilReadAll(gzreader) // small file
if err != nil {
return err
}
sha256sum := fmt.Sprintf("%x", sha256.Sum256(data))
if equal(sha256sum, resource.SHA256) == false {
return fmt.Errorf("resources: %s sha256 mismatch", fullpath)
}
c.Logger.Debugf("resources: overwrite %s", fullpath)
return ioutil.WriteFile(fullpath, data, 0600)
}
internal/engine/resources/resources_test.go
+180
View File
@@ -0,0 +1,180 @@
package resources_test
import (
"compress/gzip"
"context"
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/resources"
)
func TestEnsureMkdirAllFailure(t *testing.T) {
log.SetLevel(log.DebugLevel)
expected := errors.New("mocked error")
client := resources.Client{
HTTPClient: http.DefaultClient,
Logger: log.Log,
OSMkdirAll: func(string, os.FileMode) error {
return expected
},
UserAgent: "ooniprobe-engine/0.1.0",
WorkDir: "/foobar",
}
err := client.Ensure(context.Background())
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
}
func TestEnsure(t *testing.T) {
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
if err != nil {
t.Fatal(err)
}
client := resources.Client{
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "ooniprobe-engine/0.1.0",
WorkDir: tempdir,
}
err = client.Ensure(context.Background())
if err != nil {
t.Fatal(err)
}
// the second round should be idempotent
err = client.Ensure(context.Background())
if err != nil {
t.Fatal(err)
}
}
func TestEnsureFailure(t *testing.T) {
log.SetLevel(log.DebugLevel)
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
if err != nil {
t.Fatal(err)
}
client := resources.Client{
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "ooniprobe-engine/0.1.0",
WorkDir: tempdir,
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
err = client.Ensure(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
}
func TestEnsureFailAllComparisons(t *testing.T) {
log.SetLevel(log.DebugLevel)
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
if err != nil {
t.Fatal(err)
}
client := resources.Client{
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "ooniprobe-engine/0.1.0",
WorkDir: tempdir,
}
// run once to download the resource once
err = client.EnsureForSingleResource(
context.Background(), "ca-bundle.pem", resources.ResourceInfo{
URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
}, func(left, right string) bool {
return left == right
},
gzip.NewReader, ioutil.ReadAll,
)
if err != nil {
t.Fatal(err)
}
// re-run with broken comparison operator so that we should
// first redownload and then fail for invalid SHA256.
err = client.EnsureForSingleResource(
context.Background(), "ca-bundle.pem", resources.ResourceInfo{
URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
}, func(left, right string) bool {
return false // comparison for equality always fails
},
gzip.NewReader, ioutil.ReadAll,
)
if err == nil || !strings.HasSuffix(err.Error(), "sha256 mismatch") {
t.Fatal("not the error we expected")
}
}
func TestEnsureFailGzipNewReader(t *testing.T) {
log.SetLevel(log.DebugLevel)
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
if err != nil {
t.Fatal(err)
}
client := resources.Client{
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "ooniprobe-engine/0.1.0",
WorkDir: tempdir,
}
expected := errors.New("mocked error")
err = client.EnsureForSingleResource(
context.Background(), "ca-bundle.pem", resources.ResourceInfo{
URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
}, func(left, right string) bool {
return left == right
},
func(r io.Reader) (*gzip.Reader, error) {
return nil, expected
},
ioutil.ReadAll,
)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
}
func TestEnsureFailIoUtilReadAll(t *testing.T) {
log.SetLevel(log.DebugLevel)
tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test")
if err != nil {
t.Fatal(err)
}
client := resources.Client{
HTTPClient: http.DefaultClient,
Logger: log.Log,
UserAgent: "ooniprobe-engine/0.1.0",
WorkDir: tempdir,
}
expected := errors.New("mocked error")
err = client.EnsureForSingleResource(
context.Background(), "ca-bundle.pem", resources.ResourceInfo{
URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz",
GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab",
SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a",
}, func(left, right string) bool {
return left == right
},
gzip.NewReader, func(r io.Reader) ([]byte, error) {
return nil, expected
},
)
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
}