refactor: move more commands to internal/cmd (#207)

* refactor: move more commands to internal/cmd Part of https://github.com/ooni/probe/issues/1335. We would like all commands to be at the same level of engine rather than inside engine (now that we can do it). * fix: update .gitignore * refactor: also move jafar outside engine * We should be good now?
2021-02-03 12:23:15 +01:00
parent 6351d898d6
commit 4eeadd06a5
85 changed files with 72 additions and 65 deletions
@@ -0,0 +1,4 @@
+# oohelper
+
+This directory contains the source code of a simple client
+for the Web Connectivity test helper.
@@ -0,0 +1,138 @@
+package internal
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+	"github.com/ooni/probe-cli/v3/internal/engine/version"
+)
+
+type (
+	// CtrlResponse is the type of response returned by the test helper.
+	CtrlResponse = webconnectivity.ControlResponse
+
+	// ctrlRequest is the type of the request sent to the test helper.
+	ctrlRequest = webconnectivity.ControlRequest
+)
+
+// The following errors may be returned by this implementation.
+var (
+	ErrHTTPStatusCode          = errors.New("oohelper: http status code indicates failure")
+	ErrUnsupportedURLScheme    = errors.New("oohelper: unsupported URL scheme")
+	ErrUnsupportedExplicitPort = errors.New("oohelper: unsupported explicit port")
+	ErrEmptyURL                = errors.New("oohelper: empty server and/or target URL")
+	ErrInvalidURL              = errors.New("oohelper: cannot parse URL")
+	ErrCannotCreateRequest     = errors.New("oohelper: cannot create HTTP request")
+	ErrCannotParseJSONReply    = errors.New("oohelper: cannot parse JSON reply")
+)
+
+// OOClient is a client for the OONI Web Connectivity test helper.
+type OOClient struct {
+	// HTTPClient is the HTTP client to use.
+	HTTPClient *http.Client
+
+	// Resolver is the resolver to user.
+	Resolver netx.Resolver
+}
+
+// OOConfig contains configuration for the client.
+type OOConfig struct {
+	// ServerURL is the URL of the test helper server.
+	ServerURL string
+
+	// TargetURL is the URL that we want to measure.
+	TargetURL string
+}
+
+// MakeTCPEndpoints constructs the list of TCP endpoints to send
+// to the Web Connectivity test helper.
+func MakeTCPEndpoints(URL *url.URL, addrs []string) ([]string, error) {
+	var (
+		port string
+		out  []string
+	)
+	if URL.Host != URL.Hostname() {
+		return nil, ErrUnsupportedExplicitPort
+	}
+	switch URL.Scheme {
+	case "https":
+		port = "443"
+	case "http":
+		port = "80"
+	default:
+		return nil, ErrUnsupportedURLScheme
+	}
+	for _, addr := range addrs {
+		out = append(out, net.JoinHostPort(addr, port))
+	}
+	return out, nil
+}
+
+// Do sends a measurement request to the Web Connectivity test
+// helper and receives the corresponding response.
+func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, error) {
+	if config.TargetURL == "" || config.ServerURL == "" {
+		return nil, ErrEmptyURL
+	}
+	targetURL, err := url.Parse(config.TargetURL)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrInvalidURL, err.Error())
+	}
+	addrs, err := oo.Resolver.LookupHost(ctx, targetURL.Hostname())
+	if err != nil {
+		return nil, err
+	}
+	endpoints, err := MakeTCPEndpoints(targetURL, addrs)
+	if err != nil {
+		return nil, err
+	}
+	creq := ctrlRequest{
+		HTTPRequest: config.TargetURL,
+		HTTPRequestHeaders: map[string][]string{
+			"Accept":          {httpheader.Accept()},
+			"Accept-Language": {httpheader.AcceptLanguage()},
+			"User-Agent":      {httpheader.UserAgent()},
+		},
+		TCPConnect: endpoints,
+	}
+	data, err := json.Marshal(creq)
+	runtimex.PanicOnError(err, "oohelper: cannot marshal control request")
+	log.Debugf("out: %s", string(data))
+	req, err := http.NewRequestWithContext(ctx, "POST", config.ServerURL, bytes.NewReader(data))
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrCannotCreateRequest, err.Error())
+	}
+	req.Header.Add("user-agent", fmt.Sprintf(
+		"oohelper/%s ooniprobe-engine/%s", version.Version, version.Version,
+	))
+	req.Header.Add("content-type", "application/json")
+	resp, err := oo.HTTPClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, ErrHTTPStatusCode
+	}
+	data, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var cresp CtrlResponse
+	if err := json.Unmarshal(data, &cresp); err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrCannotParseJSONReply, err.Error())
+	}
+	return &cresp, nil
+}
@@ -0,0 +1,338 @@
+package internal_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/url"
+	"reflect"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal"
+)
+
+func TestMakeTCPEndpoints(t *testing.T) {
+	type args struct {
+		URL   *url.URL
+		addrs []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+		err  error
+	}{{
+		name: "with host != hostname",
+		args: args{URL: &url.URL{Host: "127.0.0.1:8080"}},
+		err:  internal.ErrUnsupportedExplicitPort,
+	}, {
+		name: "with unsupported URL scheme",
+		args: args{URL: &url.URL{Host: "127.0.0.1", Scheme: "imap"}},
+		err:  internal.ErrUnsupportedURLScheme,
+	}, {
+		name: "with http scheme",
+		args: args{
+			URL:   &url.URL{Host: "www.kernel.org", Scheme: "http"},
+			addrs: []string{"1.1.1.1", "2.2.2.2", "::1"},
+		},
+		want: []string{"1.1.1.1:80", "2.2.2.2:80", "[::1]:80"},
+	}, {
+		name: "with https scheme",
+		args: args{
+			URL:   &url.URL{Host: "www.kernel.org", Scheme: "https"},
+			addrs: []string{"1.1.1.1", "2.2.2.2", "::1"},
+		},
+		want: []string{"1.1.1.1:443", "2.2.2.2:443", "[::1]:443"},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := internal.MakeTCPEndpoints(tt.args.URL, tt.args.addrs)
+			if !errors.Is(err, tt.err) {
+				t.Errorf("MakeTCPEndpoints() error = %v, wantErr %v", err, tt.err)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("MakeTCPEndpoints() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestOOClientDoWithEmptyTargetURL(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{}
+	clnt := internal.OOClient{}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrEmptyURL) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithEmptyServerURL(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{TargetURL: "http://www.example.com"}
+	clnt := internal.OOClient{}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrEmptyURL) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithInvalidTargetURL(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{TargetURL: "\t", ServerURL: "https://wcth.ooni.io"}
+	clnt := internal.OOClient{}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrInvalidURL) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithResolverFailure(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverThatFails(),
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrNotFound) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithUnsupportedExplicitPort(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com:8080",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrUnsupportedExplicitPort) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithInvalidServerURL(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "\t",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrCannotCreateRequest) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithRoundTripError(t *testing.T) {
+	expected := errors.New("mocked error")
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{
+			Transport: internal.FakeTransport{Err: expected},
+		},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithInvalidStatusCode(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{Transport: internal.FakeTransport{
+			Resp: &http.Response{
+				StatusCode: 400,
+				Body:       &internal.FakeBody{},
+			},
+		}},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrHTTPStatusCode) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithBodyReadError(t *testing.T) {
+	expected := errors.New("mocked error")
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{Transport: internal.FakeTransport{
+			Resp: &http.Response{
+				StatusCode: 200,
+				Body: &internal.FakeBody{
+					Err: expected,
+				},
+			},
+		}},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, expected) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+func TestOOClientDoWithInvalidJSON(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{Transport: internal.FakeTransport{
+			Resp: &http.Response{
+				StatusCode: 200,
+				Body: &internal.FakeBody{
+					Data: []byte("{"),
+				},
+			},
+		}},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, internal.ErrCannotParseJSONReply) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp != nil {
+		t.Fatal("expected nil response")
+	}
+}
+
+const goodresponse = `{
+    "tcp_connect": {
+        "172.217.21.68:80": {
+            "status": true,
+            "failure": null
+        }
+    },
+    "http_request": {
+        "body_length": 207878,
+        "failure": null,
+        "title": "Google",
+        "headers": {
+            "Content-Type": "text/html"
+        },
+        "status_code": 200
+    },
+    "dns": {
+        "failure": null,
+        "addrs": [
+            "172.217.17.68"
+        ]
+    }
+}`
+
+func TestOOClientDoWithParseableJSON(t *testing.T) {
+	ctx := context.Background()
+	config := internal.OOConfig{
+		TargetURL: "http://www.example.com",
+		ServerURL: "https://wcth.ooni.io",
+	}
+	clnt := internal.OOClient{
+		Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
+		HTTPClient: &http.Client{Transport: internal.FakeTransport{
+			Resp: &http.Response{
+				StatusCode: 200,
+				Body: &internal.FakeBody{
+					Data: []byte(goodresponse),
+				},
+			},
+		}},
+	}
+	cresp, err := clnt.Do(ctx, config)
+	if !errors.Is(err, nil) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+	if cresp.DNS.Failure != nil {
+		t.Fatal("unexpected Failure value")
+	}
+	if len(cresp.DNS.Addrs) != 1 {
+		t.Fatal("unexpected number of DNS entries")
+	}
+	if cresp.DNS.Addrs[0] != "172.217.17.68" {
+		t.Fatal("unexpected DNS addrs [0]")
+	}
+	if cresp.HTTPRequest.BodyLength != 207878 {
+		t.Fatal("invalid http body length")
+	}
+	if cresp.HTTPRequest.Failure != nil {
+		t.Fatal("invalid http failure")
+	}
+	if cresp.HTTPRequest.Title != "Google" {
+		t.Fatal("invalid http title")
+	}
+	if len(cresp.HTTPRequest.Headers) != 1 {
+		t.Fatal("invalid http headers length")
+	}
+	if cresp.HTTPRequest.Headers["Content-Type"] != "text/html" {
+		t.Fatal("invalid http content-type header")
+	}
+	if cresp.HTTPRequest.StatusCode != 200 {
+		t.Fatal("invalid http status code")
+	}
+	if len(cresp.TCPConnect) != 1 {
+		t.Fatal("invalid tcp connect length")
+	}
+	entry, ok := cresp.TCPConnect["172.217.21.68:80"]
+	if !ok {
+		t.Fatal("cannot find expected TCP connect entry")
+	}
+	if entry.Status != true {
+		t.Fatal("unexpected TCP connect entry status")
+	}
+	if entry.Failure != nil {
+		t.Fatal("unexpected TCP connect entry failure value")
+	}
+}
@@ -0,0 +1,102 @@
+package internal
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+type FakeResolver struct {
+	NumFailures *atomicx.Int64
+	Err         error
+	Result      []string
+}
+
+func NewFakeResolverThatFails() FakeResolver {
+	return FakeResolver{NumFailures: atomicx.NewInt64(), Err: ErrNotFound}
+}
+
+func NewFakeResolverWithResult(r []string) FakeResolver {
+	return FakeResolver{NumFailures: atomicx.NewInt64(), Result: r}
+}
+
+var ErrNotFound = &net.DNSError{
+	Err: "no such host",
+}
+
+func (c FakeResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
+	time.Sleep(10 * time.Microsecond)
+	if c.Err != nil {
+		if c.NumFailures != nil {
+			c.NumFailures.Add(1)
+		}
+		return nil, c.Err
+	}
+	return c.Result, nil
+}
+
+func (c FakeResolver) Network() string {
+	return "fake"
+}
+
+func (c FakeResolver) Address() string {
+	return ""
+}
+
+var _ netx.Resolver = FakeResolver{}
+
+type FakeTransport struct {
+	Err  error
+	Func func(*http.Request) (*http.Response, error)
+	Resp *http.Response
+}
+
+func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	time.Sleep(10 * time.Microsecond)
+	if txp.Func != nil {
+		return txp.Func(req)
+	}
+	if req.Body != nil {
+		ioutil.ReadAll(req.Body)
+		req.Body.Close()
+	}
+	if txp.Err != nil {
+		return nil, txp.Err
+	}
+	txp.Resp.Request = req // non thread safe but it doesn't matter
+	return txp.Resp, nil
+}
+
+func (txp FakeTransport) CloseIdleConnections() {}
+
+var _ netx.HTTPRoundTripper = FakeTransport{}
+
+type FakeBody struct {
+	Data []byte
+	Err  error
+}
+
+func (fb *FakeBody) Read(p []byte) (int, error) {
+	time.Sleep(10 * time.Microsecond)
+	if fb.Err != nil {
+		return 0, fb.Err
+	}
+	if len(fb.Data) <= 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, fb.Data)
+	fb.Data = fb.Data[n:]
+	return n, nil
+}
+
+func (fb *FakeBody) Close() error {
+	return nil
+}
+
+var _ io.ReadCloser = &FakeBody{}
@@ -0,0 +1,48 @@
+// Command oohelper contains a simple command line
+// client for the Web Connectivity test helper.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal"
+	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/netx"
+)
+
+var (
+	ctx, cancel = context.WithCancel(context.Background())
+	debug       = flag.Bool("debug", false, "Toggle debug mode")
+	httpClient  *http.Client
+	resolver    netx.Resolver
+	server      = flag.String("server", "https://wcth.ooni.io/", "URL of the test helper")
+	target      = flag.String("target", "", "Target URL for the test helper")
+)
+
+func init() {
+	txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
+	httpClient = &http.Client{Transport: txp}
+	resolver = netx.NewResolver(netx.Config{Logger: log.Log})
+}
+
+func main() {
+	logmap := map[bool]log.Level{
+		true:  log.DebugLevel,
+		false: log.InfoLevel,
+	}
+	flag.Parse()
+	log.SetLevel(logmap[*debug])
+	clnt := internal.OOClient{HTTPClient: httpClient, Resolver: resolver}
+	config := internal.OOConfig{TargetURL: *target, ServerURL: *server}
+	defer cancel()
+	cresp, err := clnt.Do(ctx, config)
+	runtimex.PanicOnError(err, "client.Do failed")
+	data, err := json.MarshalIndent(cresp, "", "    ")
+	runtimex.PanicOnError(err, "json.MarshalIndent failed")
+	fmt.Printf("%s\n", string(data))
+}
@@ -0,0 +1,8 @@
+package main
+
+import "testing"
+
+func TestSmoke(t *testing.T) {
+	*target = "http://www.example.com"
+	main()
+}