refactor: move more commands to internal/cmd (#207)

* refactor: move more commands to internal/cmd Part of https://github.com/ooni/probe/issues/1335. We would like all commands to be at the same level of engine rather than inside engine (now that we can do it). * fix: update .gitignore * refactor: also move jafar outside engine * We should be good now?
2021-02-03 12:23:15 +01:00
parent 6351d898d6
commit 4eeadd06a5
85 changed files with 72 additions and 65 deletions
@@ -0,0 +1,44 @@
+package httpx
+
+import (
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+type FakeTransport struct {
+	Err  error
+	Func func(*http.Request) (*http.Response, error)
+	Resp *http.Response
+}
+
+func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	time.Sleep(10 * time.Microsecond)
+	if txp.Func != nil {
+		return txp.Func(req)
+	}
+	if req.Body != nil {
+		ioutil.ReadAll(req.Body)
+		req.Body.Close()
+	}
+	if txp.Err != nil {
+		return nil, txp.Err
+	}
+	txp.Resp.Request = req // non thread safe but it doesn't matter
+	return txp.Resp, nil
+}
+
+func (txp FakeTransport) CloseIdleConnections() {}
+
+type FakeBody struct {
+	Err error
+}
+
+func (fb FakeBody) Read(p []byte) (int, error) {
+	time.Sleep(10 * time.Microsecond)
+	return 0, fb.Err
+}
+
+func (fb FakeBody) Close() error {
+	return nil
+}
@@ -0,0 +1,31 @@
+package httpx
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+)
+
+// FetchResource fetches the specified resource and returns it.
+func (c Client) FetchResource(ctx context.Context, URLPath string) ([]byte, error) {
+	request, err := c.NewRequest(ctx, "GET", URLPath, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+// FetchResourceAndVerify fetches and verifies a specific resource.
+func (c Client) FetchResourceAndVerify(ctx context.Context, URL, SHA256Sum string) ([]byte, error) {
+	c.Logger.Debugf("httpx: expected SHA256: %s", SHA256Sum)
+	data, err := c.FetchResource(ctx, URL)
+	if err != nil {
+		return nil, err
+	}
+	s := fmt.Sprintf("%x", sha256.Sum256(data))
+	c.Logger.Debugf("httpx: real SHA256: %s", s)
+	if SHA256Sum != s {
+		return nil, fmt.Errorf("httpx: SHA256 mismatch: got %s and expected %s", s, SHA256Sum)
+	}
+	return data, nil
+}
@@ -0,0 +1,154 @@
+package httpx_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
+)
+
+func TestFetchResourceIntegration(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    "http://facebook.com/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResource(ctx, "/robots.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) <= 0 {
+		t.Fatal("Did not expect an empty resource")
+	}
+}
+
+func TestFetchResourceExpiredContext(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	data, err := (httpx.Client{
+		BaseURL:    "http://facebook.com/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResource(ctx, "/robots.txt")
+	if !errors.Is(err, context.Canceled) {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
+
+func TestFetchResourceAndVerifyIntegration(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    "https://github.com/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResourceAndVerify(
+		ctx,
+		"/measurement-kit/generic-assets/releases/download/20190426155936/generic-assets-20190426155936.tar.gz",
+		"34d8a9c8ab30c242469482dc280be832d8a06b4400f8927604dd361bf979b795",
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) <= 0 {
+		t.Fatal("Did not expect an empty resource")
+	}
+}
+
+func TestFetchResourceInvalidURL(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    "http://\t/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResource(ctx, "/robots.txt")
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
+
+func TestFetchResource400(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(400)
+		},
+	))
+	defer server.Close()
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		Authorization: "foobar",
+		BaseURL:       server.URL,
+		HTTPClient:    http.DefaultClient,
+		Logger:        log.Log,
+		UserAgent:     "ooniprobe-engine/0.1.0",
+	}).FetchResource(ctx, "")
+	if err == nil || !strings.HasSuffix(err.Error(), "400 Bad Request") {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
+
+func TestFetchResourceAndVerify400(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(400)
+		},
+	))
+	defer server.Close()
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    server.URL,
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResourceAndVerify(ctx, "", "abcde")
+	if err == nil || !strings.HasSuffix(err.Error(), "400 Bad Request") {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
+
+func TestFetchResourceAndVerifyInvalidSHA256(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	ctx := context.Background()
+	data, err := (httpx.Client{
+		BaseURL:    "https://github.com/",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  "ooniprobe-engine/0.1.0",
+	}).FetchResourceAndVerify(
+		ctx,
+		"/measurement-kit/generic-assets/releases/download/20190426155936/generic-assets-20190426155936.tar.gz",
+		"34d8a9ceeb30c242469482dc280be832d8a06b4400f8927604dd361bf979b795",
+	)
+	if err == nil || !strings.HasPrefix(err.Error(), "httpx: SHA256 mismatch:") {
+		t.Fatal("not the error we expected")
+	}
+	if len(data) != 0 {
+		t.Fatal("expected an empty resource")
+	}
+}
@@ -0,0 +1,168 @@
+// Package httpx contains http extensions.
+package httpx
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
+)
+
+// Logger is the definition of Logger used by this package.
+type Logger interface {
+	Debugf(format string, v ...interface{})
+}
+
+// Client is an extended client.
+type Client struct {
+	// Accept contains the accept header.
+	Accept string
+
+	// Authorization contains the authorization header.
+	Authorization string
+
+	// BaseURL is the base URL of the API.
+	BaseURL string
+
+	// HTTPClient is the real http client to use.
+	HTTPClient *http.Client
+
+	// Host allows to set a specific host header. This is useful
+	// to implement, e.g., cloudfronting.
+	Host string
+
+	// Logger is the logger to use.
+	Logger Logger
+
+	// ProxyURL allows to force a proxy URL to fallback to a
+	// tunnel, e.g., Psiphon.
+	ProxyURL *url.URL
+
+	// UserAgent is the user agent to use.
+	UserAgent string
+}
+
+// NewRequestWithJSONBody creates a new request with a JSON body
+func (c Client) NewRequestWithJSONBody(
+	ctx context.Context, method, resourcePath string,
+	query url.Values, body interface{}) (*http.Request, error) {
+	data, err := json.Marshal(body)
+	if err != nil {
+		return nil, err
+	}
+	c.Logger.Debugf("httpx: request body: %d bytes", len(data))
+	request, err := c.NewRequest(
+		ctx, method, resourcePath, query, bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	if body != nil {
+		request.Header.Set("Content-Type", "application/json")
+	}
+	return request, nil
+}
+
+// NewRequest creates a new request.
+func (c Client) NewRequest(ctx context.Context, method, resourcePath string,
+	query url.Values, body io.Reader) (*http.Request, error) {
+	URL, err := url.Parse(c.BaseURL)
+	if err != nil {
+		return nil, err
+	}
+	URL.Path = resourcePath
+	if query != nil {
+		URL.RawQuery = query.Encode()
+	}
+	c.Logger.Debugf("httpx: method: %s", method)
+	c.Logger.Debugf("httpx: URL: %s", URL.String())
+	request, err := http.NewRequest(method, URL.String(), body)
+	if err != nil {
+		return nil, err
+	}
+	request.Host = c.Host // allow cloudfronting
+	if c.Authorization != "" {
+		request.Header.Set("Authorization", c.Authorization)
+	}
+	if c.Accept != "" {
+		request.Header.Set("Accept", c.Accept)
+	}
+	request.Header.Set("User-Agent", c.UserAgent)
+	// Implementation note: the following allows tunneling if c.ProxyURL
+	// is not nil. Because the proxy URL is set as part of each request
+	// generated using this function, every request that eventually needs
+	// to reconnect will always do so using the proxy.
+	ctx = dialer.WithProxyURL(ctx, c.ProxyURL)
+	return request.WithContext(ctx), nil
+}
+
+// Do performs the provided request and returns the response body or an error.
+func (c Client) Do(request *http.Request) ([]byte, error) {
+	response, err := c.HTTPClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode >= 400 {
+		return nil, fmt.Errorf("httpx: request failed: %s", response.Status)
+	}
+	return ioutil.ReadAll(response.Body)
+}
+
+// DoJSON performs the provided request and unmarshals the JSON response body
+// into the provided output variable.
+func (c Client) DoJSON(request *http.Request, output interface{}) error {
+	data, err := c.Do(request)
+	if err != nil {
+		return err
+	}
+	c.Logger.Debugf("httpx: response body: %d bytes", len(data))
+	return json.Unmarshal(data, output)
+}
+
+// GetJSON reads the JSON resource at resourcePath and unmarshals the
+// results into output. The request is bounded by the lifetime of the
+// context passed as argument. Returns the error that occurred.
+func (c Client) GetJSON(ctx context.Context, resourcePath string, output interface{}) error {
+	return c.GetJSONWithQuery(ctx, resourcePath, nil, output)
+}
+
+// GetJSONWithQuery is like GetJSON but also has a query.
+func (c Client) GetJSONWithQuery(
+	ctx context.Context, resourcePath string,
+	query url.Values, output interface{}) error {
+	request, err := c.NewRequest(ctx, "GET", resourcePath, query, nil)
+	if err != nil {
+		return err
+	}
+	return c.DoJSON(request, output)
+}
+
+// PostJSON creates a JSON subresource of the resource at resourcePath
+// using the JSON document at input and returning the result into the
+// JSON document at output. The request is bounded by the context's
+// lifetime. Returns the error that occurred.
+func (c Client) PostJSON(
+	ctx context.Context, resourcePath string, input, output interface{}) error {
+	request, err := c.NewRequestWithJSONBody(ctx, "POST", resourcePath, nil, input)
+	if err != nil {
+		return err
+	}
+	return c.DoJSON(request, output)
+}
+
+// PutJSON updates a JSON resource at a specific path and returns
+// the error that occurred and possibly an output document
+func (c Client) PutJSON(
+	ctx context.Context, resourcePath string, input, output interface{}) error {
+	request, err := c.NewRequestWithJSONBody(ctx, "PUT", resourcePath, nil, input)
+	if err != nil {
+		return err
+	}
+	return c.DoJSON(request, output)
+}
@@ -0,0 +1,316 @@
+package httpx_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
+)
+
+const userAgent = "miniooni/0.1.0-dev"
+
+func newClient() httpx.Client {
+	return httpx.Client{
+		BaseURL:    "https://httpbin.org",
+		HTTPClient: http.DefaultClient,
+		Logger:     log.Log,
+		UserAgent:  userAgent,
+	}
+}
+
+func TestNewRequestWithJSONBodyJSONMarshalFailure(t *testing.T) {
+	client := newClient()
+	req, err := client.NewRequestWithJSONBody(
+		context.Background(), "GET", "/", nil, make(chan interface{}),
+	)
+	if err == nil || !strings.HasPrefix(err.Error(), "json: unsupported type") {
+		t.Fatal("not the error we expected")
+	}
+	if req != nil {
+		t.Fatal("expected nil request here")
+	}
+}
+
+func TestNewRequestWithJSONBodyNewRequestFailure(t *testing.T) {
+	client := newClient()
+	client.BaseURL = "\t\t\t" // cause URL parse error
+	req, err := client.NewRequestWithJSONBody(
+		context.Background(), "GET", "/", nil, nil,
+	)
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+	if req != nil {
+		t.Fatal("expected nil request here")
+	}
+}
+
+func TestNewRequestWithQuery(t *testing.T) {
+	client := newClient()
+	q := url.Values{}
+	q.Add("antani", "mascetti")
+	q.Add("melandri", "conte")
+	req, err := client.NewRequest(
+		context.Background(), "GET", "/", q, nil,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL.Query().Get("antani") != "mascetti" {
+		t.Fatal("expected different query string here")
+	}
+	if req.URL.Query().Get("melandri") != "conte" {
+		t.Fatal("expected different query string here")
+	}
+}
+
+func TestNewRequestNewRequestFailure(t *testing.T) {
+	client := newClient()
+	req, err := client.NewRequest(
+		context.Background(), "\t\t\t", "/", nil, nil,
+	)
+	if err == nil || !strings.HasPrefix(err.Error(), "net/http: invalid method") {
+		t.Fatal("not the error we expected")
+	}
+	if req != nil {
+		t.Fatal("expected nil request here")
+	}
+}
+
+func TestNewRequestCloudfronting(t *testing.T) {
+	client := newClient()
+	client.Host = "www.x.org"
+	req, err := client.NewRequest(
+		context.Background(), "GET", "/", nil, nil,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.Host != client.Host {
+		t.Fatal("expected different req.Host here")
+	}
+}
+
+func TestNewRequestAcceptIsSet(t *testing.T) {
+	client := newClient()
+	client.Accept = "application/xml"
+	req, err := client.NewRequestWithJSONBody(
+		context.Background(), "GET", "/", nil, []string{},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.Header.Get("Accept") != "application/xml" {
+		t.Fatal("expected different Accept here")
+	}
+}
+
+func TestNewRequestContentTypeIsSet(t *testing.T) {
+	client := newClient()
+	req, err := client.NewRequestWithJSONBody(
+		context.Background(), "GET", "/", nil, []string{},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.Header.Get("Content-Type") != "application/json" {
+		t.Fatal("expected different Content-Type here")
+	}
+}
+
+func TestNewRequestAuthorizationHeader(t *testing.T) {
+	client := newClient()
+	client.Authorization = "deadbeef"
+	req, err := client.NewRequest(
+		context.Background(), "GET", "/", nil, nil,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.Header.Get("Authorization") != client.Authorization {
+		t.Fatal("expected different Authorization here")
+	}
+}
+
+func TestNewRequestUserAgentIsSet(t *testing.T) {
+	client := newClient()
+	req, err := client.NewRequest(
+		context.Background(), "GET", "/", nil, nil,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.Header.Get("User-Agent") != userAgent {
+		t.Fatal("expected different User-Agent here")
+	}
+}
+
+func TestNewRequestTunnelingIsPossible(t *testing.T) {
+	client := newClient()
+	client.ProxyURL = &url.URL{Scheme: "socks5", Host: "[::1]:54321"}
+	req, err := client.NewRequest(
+		context.Background(), "GET", "/", nil, nil,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cmp := cmp.Diff(dialer.ContextProxyURL(req.Context()), client.ProxyURL)
+	if cmp != "" {
+		t.Fatal(cmp)
+	}
+}
+
+func TestClientDoJSONClientDoFailure(t *testing.T) {
+	expected := errors.New("mocked error")
+	client := newClient()
+	client.HTTPClient = &http.Client{Transport: httpx.FakeTransport{
+		Err: expected,
+	}}
+	err := client.DoJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestClientDoJSONResponseNotSuccessful(t *testing.T) {
+	client := newClient()
+	client.HTTPClient = &http.Client{Transport: httpx.FakeTransport{
+		Resp: &http.Response{
+			StatusCode: 401,
+			Body:       httpx.FakeBody{},
+		},
+	}}
+	err := client.DoJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
+	if err == nil || !strings.HasPrefix(err.Error(), "httpx: request failed") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestClientDoJSONResponseReadingBodyError(t *testing.T) {
+	expected := errors.New("mocked error")
+	client := newClient()
+	client.HTTPClient = &http.Client{Transport: httpx.FakeTransport{
+		Resp: &http.Response{
+			StatusCode: 200,
+			Body: httpx.FakeBody{
+				Err: expected,
+			},
+		},
+	}}
+	err := client.DoJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
+	if !errors.Is(err, expected) {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestClientDoJSONResponseIsNotJSON(t *testing.T) {
+	client := newClient()
+	client.HTTPClient = &http.Client{Transport: httpx.FakeTransport{
+		Resp: &http.Response{
+			StatusCode: 200,
+			Body: httpx.FakeBody{
+				Err: io.EOF,
+			},
+		},
+	}}
+	err := client.DoJSON(&http.Request{URL: &url.URL{Scheme: "https", Host: "x.org"}}, nil)
+	if err == nil || err.Error() != "unexpected end of JSON input" {
+		t.Fatal("not the error we expected")
+	}
+}
+
+type httpbinheaders struct {
+	Headers map[string]string `json:"headers"`
+}
+
+func TestReadJSONSuccess(t *testing.T) {
+	var headers httpbinheaders
+	err := newClient().GetJSON(context.Background(), "/headers", &headers)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if headers.Headers["Host"] != "httpbin.org" {
+		t.Fatal("unexpected Host header")
+	}
+	if headers.Headers["User-Agent"] != "miniooni/0.1.0-dev" {
+		t.Fatal("unexpected Host header")
+	}
+}
+
+type httpbinpost struct {
+	Data string `json:"data"`
+}
+
+func TestCreateJSONSuccess(t *testing.T) {
+	headers := httpbinheaders{
+		Headers: map[string]string{
+			"Foo": "bar",
+		},
+	}
+	var response httpbinpost
+	err := newClient().PostJSON(context.Background(), "/post", &headers, &response)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.Data != `{"headers":{"Foo":"bar"}}` {
+		t.Fatal(response.Data)
+	}
+}
+
+type httpbinput struct {
+	Data string `json:"data"`
+}
+
+func TestUpdateJSONSuccess(t *testing.T) {
+	headers := httpbinheaders{
+		Headers: map[string]string{
+			"Foo": "bar",
+		},
+	}
+	var response httpbinpost
+	err := newClient().PutJSON(context.Background(), "/put", &headers, &response)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.Data != `{"headers":{"Foo":"bar"}}` {
+		t.Fatal(response.Data)
+	}
+}
+
+func TestReadJSONFailure(t *testing.T) {
+	var headers httpbinheaders
+	client := newClient()
+	client.BaseURL = "\t\t\t\t"
+	err := client.GetJSON(context.Background(), "/headers", &headers)
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestCreateJSONFailure(t *testing.T) {
+	var headers httpbinheaders
+	client := newClient()
+	client.BaseURL = "\t\t\t\t"
+	err := client.PostJSON(context.Background(), "/headers", &headers, &headers)
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+}
+
+func TestUpdateJSONFailure(t *testing.T) {
+	var headers httpbinheaders
+	client := newClient()
+	client.BaseURL = "\t\t\t\t"
+	err := client.PutJSON(context.Background(), "/headers", &headers, &headers)
+	if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
+		t.Fatal("not the error we expected")
+	}
+}