refactor(mlablocate*): move from i/e/internal to internal (#385)

We've been flattening the package structure for some time now.

While there, add very basic examples.
This commit is contained in:
Simone Basso
2021-06-15 19:51:03 +02:00
committed by GitHub
parent d84cf5b69f
commit 85b16c8bd2
9 changed files with 40 additions and 2 deletions
+1 -1
View File
@@ -4,8 +4,8 @@ import (
"context"
"net/http"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mlablocate"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/mlablocate"
)
type locateDeps interface {
+1 -1
View File
@@ -11,10 +11,10 @@ import (
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mlablocatev2"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/humanize"
"github.com/ooni/probe-cli/v3/internal/mlablocatev2"
)
const (
@@ -1,109 +0,0 @@
// Package mlablocate contains a locate.measurementlab.net client
// implementing v1 of the locate API. This version of the API isn't
// suitable for requesting servers for ndt7. You should use the
// mlablocatev2 package for that.
package mlablocate
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"github.com/ooni/probe-cli/v3/internal/iox"
)
// Logger is the logger expected by this package.
type Logger interface {
// Debugf formats and emits a debug message.
Debugf(format string, v ...interface{})
}
// HTTPClient is anything that looks like an http.Client.
type HTTPClient interface {
// Do behaves like http.Client.Do.
Do(req *http.Request) (*http.Response, error)
}
// Client is a locate.measurementlab.net client. Please use the
// NewClient factory to construct a new instance of client, otherwise
// you MUST fill all the fields marked as MANDATORY.
type Client struct {
// HTTPClient is the MANDATORY http client to use.
HTTPClient HTTPClient
// Hostname is the MANDATORY hostname of the mlablocate API.
Hostname string
// Logger is the MANDATORY logger to use.
Logger Logger
// Scheme is the MANDATORY scheme to use (http or https).
Scheme string
// UserAgent is the MANDATORY user-agent to use.
UserAgent string
}
// NewClient creates a new locate.measurementlab.net client.
func NewClient(httpClient HTTPClient, logger Logger, userAgent string) *Client {
return &Client{
HTTPClient: httpClient,
Hostname: "locate.measurementlab.net",
Logger: logger,
Scheme: "https",
UserAgent: userAgent,
}
}
// Result is a result of a query to locate.measurementlab.net.
type Result struct {
// FQDN is the mlab server's FQDN.
FQDN string `json:"fqdn"`
// Site is the ID of the site where the server is.
Site string `json:"site"`
}
// Query performs a locate.measurementlab.net query. This function returns
// either valid result, on success, or an error, on failure.
// (Note thay you cannot use this API to query for ndt7 servers. You should
// use the mlablocatev2 API to obtain such servers.)
func (c *Client) Query(ctx context.Context, tool string) (Result, error) {
// TODO(bassosimone): this code should probably be
// refactored to use the httpx package.
URL := &url.URL{
Scheme: c.Scheme,
Host: c.Hostname,
Path: tool,
}
req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil)
if err != nil {
return Result{}, err
}
req.Header.Add("User-Agent", c.UserAgent)
c.Logger.Debugf("mlablocate: GET %s", URL.String())
resp, err := c.HTTPClient.Do(req)
if err != nil {
return Result{}, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return Result{}, fmt.Errorf("mlablocate: non-200 status code: %d", resp.StatusCode)
}
data, err := iox.ReadAllContext(ctx, resp.Body)
if err != nil {
return Result{}, err
}
c.Logger.Debugf("mlablocate: %s", string(data))
var result Result
if err := json.Unmarshal(data, &result); err != nil {
return Result{}, err
}
if result.FQDN == "" {
return Result{}, errors.New("mlablocate: returned empty FQDN")
}
return result, nil
}
@@ -1,205 +0,0 @@
package mlablocate
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"github.com/apex/log"
)
func TestSuccess(t *testing.T) {
client := NewClient(
http.DefaultClient,
log.Log,
"miniooni/0.1.0-dev",
)
result, err := client.Query(context.Background(), "neubot/dash")
if err != nil {
t.Fatal(err)
}
if result.FQDN == "" {
t.Fatal("unexpected empty fqdn")
}
}
func Test404Response(t *testing.T) {
client := NewClient(
http.DefaultClient,
log.Log,
"miniooni/0.1.0-dev",
)
result, err := client.Query(context.Background(), "nonexistent")
if err == nil || !strings.Contains(err.Error(), "mlablocate: non-200 status code") {
t.Fatal("not the error we expected")
}
if result.FQDN != "" {
t.Fatal("expected empty fqdn")
}
}
func TestNewRequestFailure(t *testing.T) {
client := NewClient(
http.DefaultClient,
log.Log,
"miniooni/0.1.0-dev",
)
client.Hostname = "\t"
result, err := client.Query(context.Background(), "nonexistent")
if err == nil || !strings.Contains(err.Error(), "invalid URL escape") {
t.Fatal("not the error we expected")
}
if result.FQDN != "" {
t.Fatal("expected empty fqdn")
}
}
func TestHTTPClientDoFailure(t *testing.T) {
client := NewClient(
http.DefaultClient,
log.Log,
"miniooni/0.1.0-dev",
)
expected := errors.New("mocked error")
client.HTTPClient = &http.Client{
Transport: &roundTripFails{Error: expected},
}
result, err := client.Query(context.Background(), "nonexistent")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if result.FQDN != "" {
t.Fatal("expected empty fqdn")
}
}
type roundTripFails struct {
Error error
}
func (txp *roundTripFails) RoundTrip(*http.Request) (*http.Response, error) {
return nil, txp.Error
}
func TestCannotReadBody(t *testing.T) {
client := NewClient(
http.DefaultClient,
log.Log,
"miniooni/0.1.0-dev",
)
expected := errors.New("mocked error")
client.HTTPClient = &http.Client{
Transport: &readingBodyFails{Error: expected},
}
result, err := client.Query(context.Background(), "nonexistent")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if result.FQDN != "" {
t.Fatal("expected empty fqdn")
}
}
type readingBodyFails struct {
Error error
}
func (txp *readingBodyFails) RoundTrip(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: 200,
Body: &readingBodyFailsBody{Error: txp.Error},
}, nil
}
type readingBodyFailsBody struct {
Error error
}
func (b *readingBodyFailsBody) Read(p []byte) (int, error) {
return 0, b.Error
}
func (b *readingBodyFailsBody) Close() error {
return nil
}
func TestInvalidJSON(t *testing.T) {
client := NewClient(
http.DefaultClient,
log.Log,
"miniooni/0.1.0-dev",
)
client.HTTPClient = &http.Client{
Transport: &invalidJSON{},
}
result, err := client.Query(context.Background(), "nonexistent")
if err == nil || !strings.Contains(err.Error(), "unexpected end of JSON input") {
t.Fatal("not the error we expected")
}
if result.FQDN != "" {
t.Fatal("expected empty fqdn")
}
}
type invalidJSON struct{}
func (txp *invalidJSON) RoundTrip(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: 200,
Body: &invalidJSONBody{},
}, nil
}
type invalidJSONBody struct{}
func (b *invalidJSONBody) Read(p []byte) (int, error) {
if len(p) < 1 {
return 0, errors.New("slice too short")
}
p[0] = '{'
return 1, io.EOF
}
func (b *invalidJSONBody) Close() error {
return nil
}
func TestEmptyFQDN(t *testing.T) {
client := NewClient(
http.DefaultClient,
log.Log,
"miniooni/0.1.0-dev",
)
client.HTTPClient = &http.Client{
Transport: &emptyFQDN{},
}
result, err := client.Query(context.Background(), "nonexistent")
if err == nil || !strings.HasSuffix(err.Error(), "returned empty FQDN") {
t.Fatal("not the error we expected")
}
if result.FQDN != "" {
t.Fatal("expected empty fqdn")
}
}
type emptyFQDN struct{}
func (txp *emptyFQDN) RoundTrip(*http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: 200,
Body: &emptyFQDNBody{},
}, nil
}
type emptyFQDNBody struct{}
func (b *emptyFQDNBody) Read(p []byte) (int, error) {
return copy(p, []byte(`{"fqdn":""}`)), io.EOF
}
func (b *emptyFQDNBody) Close() error {
return nil
}
@@ -1,46 +0,0 @@
package mlablocatev2
import (
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/iox"
)
type FakeTransport struct {
Err error
Func func(*http.Request) (*http.Response, error)
Resp *http.Response
}
func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond)
if txp.Func != nil {
return txp.Func(req)
}
if req.Body != nil {
iox.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if txp.Err != nil {
return nil, txp.Err
}
txp.Resp.Request = req // non thread safe but it doesn't matter
return txp.Resp, nil
}
func (txp FakeTransport) CloseIdleConnections() {}
type FakeBody struct {
Data []byte
Err error
}
func (fb FakeBody) Read(p []byte) (int, error) {
time.Sleep(10 * time.Microsecond)
return copy(p, fb.Data), fb.Err // simplifed but OK
}
func (fb FakeBody) Close() error {
return nil
}
@@ -1,193 +0,0 @@
// Package mlablocatev2 implements m-lab locate services API v2. This
// API currently only allows you to get servers for ndt7. Use the
// mlablocate package for all other m-lab tools.
package mlablocatev2
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"regexp"
"github.com/ooni/probe-cli/v3/internal/iox"
)
const (
// ndt7URLPath is the URL path to be used for ndt
ndt7URLPath = "v2/nearest/ndt/ndt7"
)
var (
// ErrRequestFailed indicates that the response is not "200 Ok"
ErrRequestFailed = errors.New("mlablocatev2: request failed")
// ErrEmptyResponse indicates that no hosts were returned
ErrEmptyResponse = errors.New("mlablocatev2: empty response")
)
// Logger is the logger expected by this package.
type Logger interface {
// Debugf formats and emits a debug message.
Debugf(format string, v ...interface{})
}
// HTTPClient is anything that looks like an http.Client.
type HTTPClient interface {
// Do behaves like http.Client.Do.
Do(req *http.Request) (*http.Response, error)
}
// Client is a client for v2 of the locate services. Please use the
// NewClient factory to construct a new instance of client, otherwise
// you MUST fill all the fields marked as MANDATORY.
type Client struct {
// HTTPClient is the MANDATORY http client to use
HTTPClient HTTPClient
// Hostname is the MANDATORY hostname of the mlablocate API.
Hostname string
// Logger is the MANDATORY logger to use.
Logger Logger
// Scheme is the MANDATORY scheme to use (http or https).
Scheme string
// UserAgent is the MANDATORY user-agent to use.
UserAgent string
}
// NewClient creates a client for v2 of the locate services.
func NewClient(httpClient HTTPClient, logger Logger, userAgent string) Client {
return Client{
HTTPClient: httpClient,
Hostname: "locate.measurementlab.net",
Logger: logger,
Scheme: "https",
UserAgent: userAgent,
}
}
// entryRecord describes one of the boxes returned by v2 of
// the locate service. It gives you the FQDN of the specific
// box along with URLs for each experiment phase. You MUST
// use the URLs directly because they contain access tokens.
type entryRecord struct {
Machine string `json:"machine"`
URLs map[string]string `json:"urls"`
}
var (
// siteRegexp is the regexp to extract the site from the
// machine name when the domain is a v2 domain.
//
// Example: mlab3-mil04.mlab-oti.measurement-lab.org.
siteRegexp = regexp.MustCompile(
`^(mlab[1-4]d?)-([a-z]{3}[0-9tc]{2})\.([a-z0-9-]{1,16})\.(measurement-lab\.org)$`)
)
// Site returns the site name. If it is not possible to determine
// the site name, we return the empty string.
func (er entryRecord) Site() string {
m := siteRegexp.FindAllStringSubmatch(er.Machine, -1)
if len(m) != 1 || len(m[0]) != 5 {
return ""
}
return m[0][2]
}
// resultRecord is a result of a query to locate.measurementlab.net.
type resultRecord struct {
Results []entryRecord `json:"results"`
}
// query performs a locate.measurementlab.net query
// using v2 of the locate protocol.
func (c Client) query(ctx context.Context, path string) (resultRecord, error) {
// TODO(bassosimone): this code should probably be
// refactored to use the httpx package.
URL := &url.URL{
Scheme: c.Scheme,
Host: c.Hostname,
Path: path,
}
req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil)
if err != nil {
return resultRecord{}, err
}
req.Header.Add("User-Agent", c.UserAgent)
c.Logger.Debugf("mlablocatev2: GET %s", URL.String())
resp, err := c.HTTPClient.Do(req)
if err != nil {
return resultRecord{}, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return resultRecord{}, fmt.Errorf("%w: %d", ErrRequestFailed, resp.StatusCode)
}
data, err := iox.ReadAllContext(ctx, resp.Body)
if err != nil {
return resultRecord{}, err
}
c.Logger.Debugf("mlablocatev2: %s", string(data))
var result resultRecord
if err := json.Unmarshal(data, &result); err != nil {
return resultRecord{}, err
}
return result, nil
}
// NDT7Result is the result of a v2 locate services query for ndt7.
type NDT7Result struct {
// Hostname is an informative field containing the hostname
// to which you're connected. Because there are access tokens,
// you cannot use this field directly.
Hostname string
// Site is an informative field containing the site
// to which the server belongs to.
Site string
// WSSDownloadURL is the WebSocket URL to be used for
// performing a download over HTTPS. Note that the URL
// typically includes the required access token.
WSSDownloadURL string
// WSSUploadURL is like WSSDownloadURL but for the upload.
WSSUploadURL string
}
// QueryNDT7 performs a v2 locate services query for ndt7.
func (c Client) QueryNDT7(ctx context.Context) ([]NDT7Result, error) {
out, err := c.query(ctx, ndt7URLPath)
if err != nil {
return nil, err
}
var result []NDT7Result
for _, entry := range out.Results {
r := NDT7Result{
WSSDownloadURL: entry.URLs["wss:///ndt/v7/download"],
WSSUploadURL: entry.URLs["wss:///ndt/v7/upload"],
}
if r.WSSDownloadURL == "" || r.WSSUploadURL == "" {
continue
}
// Implementation note: we extract the hostname from the
// download URL, under the assumption that the download and
// the upload URLs have the same hostname.
url, err := url.Parse(r.WSSDownloadURL)
if err != nil {
continue
}
r.Site = entry.Site()
r.Hostname = url.Hostname()
result = append(result, r)
}
if len(result) <= 0 {
return nil, ErrEmptyResponse
}
return result, nil
}
@@ -1,250 +0,0 @@
package mlablocatev2
import (
"context"
"errors"
"io"
"net/http"
"net/url"
"strings"
"testing"
"github.com/apex/log"
)
func TestSuccess(t *testing.T) {
// this test is ~0.5 s, so we can always run it
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
result, err := client.QueryNDT7(context.Background())
if err != nil {
t.Fatal(err)
}
if len(result) <= 0 {
t.Fatal("unexpected empty result")
}
for _, entry := range result {
if entry.Hostname == "" {
t.Fatal("expected non empty Hostname here")
}
if entry.Site == "" {
t.Fatal("expected non=-empty Site here")
}
if entry.WSSDownloadURL == "" {
t.Fatal("expected non-empty WSSDownloadURL here")
}
if _, err := url.Parse(entry.WSSDownloadURL); err != nil {
t.Fatal(err)
}
if entry.WSSUploadURL == "" {
t.Fatal("expected non-empty WSSUploadURL here")
}
if _, err := url.Parse(entry.WSSUploadURL); err != nil {
t.Fatal(err)
}
}
}
func Test404Response(t *testing.T) {
// this test is ~0.5 s, so we can always run it
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
result, err := client.query(context.Background(), "nonexistent")
if !errors.Is(err, ErrRequestFailed) {
t.Fatal("not the error we expected")
}
if result.Results != nil {
t.Fatal("expected empty results")
}
}
func TestNewRequestFailure(t *testing.T) {
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
client.Hostname = "\t"
result, err := client.query(context.Background(), "nonexistent")
if err == nil || !strings.Contains(err.Error(), "invalid URL escape") {
t.Fatal("not the error we expected")
}
if result.Results != nil {
t.Fatal("expected nil results")
}
}
func TestHTTPClientDoFailure(t *testing.T) {
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
expected := errors.New("mocked error")
client.HTTPClient = &http.Client{
Transport: FakeTransport{Err: expected},
}
result, err := client.query(context.Background(), "nonexistent")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if result.Results != nil {
t.Fatal("expected nil results")
}
}
func TestCannotReadBody(t *testing.T) {
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
expected := errors.New("mocked error")
client.HTTPClient = &http.Client{
Transport: FakeTransport{
Resp: &http.Response{
StatusCode: 200,
Body: FakeBody{
Err: expected,
},
},
},
}
result, err := client.query(context.Background(), "nonexistent")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if result.Results != nil {
t.Fatal("expected nil results")
}
}
func TestInvalidJSON(t *testing.T) {
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
client.HTTPClient = &http.Client{
Transport: FakeTransport{
Resp: &http.Response{
StatusCode: 200,
Body: FakeBody{
Err: io.EOF,
Data: []byte(`{`),
},
},
},
}
result, err := client.query(context.Background(), "nonexistent")
if err == nil || !strings.Contains(err.Error(), "unexpected end of JSON input") {
t.Fatal("not the error we expected")
}
if result.Results != nil {
t.Fatal("expected nil results")
}
}
func TestEmptyResponse(t *testing.T) {
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
client.HTTPClient = &http.Client{
Transport: FakeTransport{
Resp: &http.Response{
StatusCode: 200,
Body: FakeBody{
Err: io.EOF,
Data: []byte(`{}`),
},
},
},
}
result, err := client.QueryNDT7(context.Background())
if !errors.Is(err, ErrEmptyResponse) {
t.Fatal("not the error we expected")
}
if result != nil {
t.Fatal("expected nil results")
}
}
func TestNDT7QueryFails(t *testing.T) {
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
client.HTTPClient = &http.Client{
Transport: FakeTransport{
Resp: &http.Response{
StatusCode: 404,
Body: FakeBody{Err: io.EOF},
},
},
}
result, err := client.QueryNDT7(context.Background())
if !errors.Is(err, ErrRequestFailed) {
t.Fatal("not the error we expected")
}
if result != nil {
t.Fatal("expected nil results")
}
}
func TestNDT7InvalidURLs(t *testing.T) {
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
client.HTTPClient = &http.Client{
Transport: FakeTransport{
Resp: &http.Response{
StatusCode: 200,
Body: FakeBody{
Data: []byte(
`{"results":[{"machine":"mlab3-mil04.mlab-oti.measurement-lab.org","urls":{"wss:///ndt/v7/download":":","wss:///ndt/v7/upload":":"}}]}`),
Err: io.EOF,
},
},
},
}
result, err := client.QueryNDT7(context.Background())
if !errors.Is(err, ErrEmptyResponse) {
t.Fatal("not the error we expected")
}
if result != nil {
t.Fatal("expected nil results")
}
}
func TestNDT7EmptyURLs(t *testing.T) {
client := NewClient(http.DefaultClient, log.Log, "miniooni/0.1.0-dev")
client.HTTPClient = &http.Client{
Transport: FakeTransport{
Resp: &http.Response{
StatusCode: 200,
Body: FakeBody{
Data: []byte(
`{"results":[{"machine":"mlab3-mil04.mlab-oti.measurement-lab.org","urls":{"wss:///ndt/v7/download":"","wss:///ndt/v7/upload":""}}]}`),
Err: io.EOF,
},
},
},
}
result, err := client.QueryNDT7(context.Background())
if !errors.Is(err, ErrEmptyResponse) {
t.Fatal("not the error we expected")
}
if result != nil {
t.Fatal("expected nil results")
}
}
func TestEntryRecordSite(t *testing.T) {
type fields struct {
Machine string
URLs map[string]string
}
tests := []struct {
name string
fields fields
want string
}{{
name: "with invalid machine name",
fields: fields{
Machine: "ndt-iupui-mlab3-mil02.mlab-oti.measurement-lab.org",
},
want: "",
}, {
name: "with valid machine name",
fields: fields{
Machine: "mlab3-mil04.mlab-oti.measurement-lab.org",
},
want: "mil04",
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
er := entryRecord{
Machine: tt.fields.Machine,
URLs: tt.fields.URLs,
}
if got := er.Site(); got != tt.want {
t.Errorf("entryRecord.Site() = %v, want %v", got, tt.want)
}
})
}
}