Implement the show command (#53)

.travis.yml

@@ -1,17 +1,10 @@
-matrix:
-  include:
-  - os: linux
+os: linux
 dist: xenial
 language: minimal
 services:
 - docker
 env:
 - OS_NAME: linux
-  - os: osx
-    language: minimal
-    env:
-    - OS_NAME: macos
-    osx_image: xcode10.2
 script:
 - ./build.sh _travis-${TRAVIS_OS_NAME}
 - ./scripts/travis_test.sh

config/settings.go

@@ -42,6 +42,7 @@ type NettestConfig struct {
 // Websites test group
 type Websites struct {
 	EnabledCategories []string `json:"enabled_categories"`
+	Limit             int      `json:"limit"`
 }
 
 // NettestConfigs returns a list configured enabled tests for the group

go.mod

@@ -16,8 +16,7 @@ require (
 	github.com/lib/pq v1.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.2
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
-	github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632
-	github.com/oschwald/maxminddb-golang v1.3.1 // indirect
+	github.com/ooni/probe-engine v0.0.0-20190828175829-9e908f8e3a3a
 	github.com/pkg/errors v0.8.1
 	github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051
 	github.com/ziutek/mymysql v1.5.4 // indirect

go.sum

@@ -139,14 +139,12 @@ github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632 h1:58EgLL9v2729NkY0U/lixOAGr9UsvRH67use+iJvz+c=
-github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632/go.mod h1:iq63Xvw89LLS+2Q9vdqNpmuGQLntyomd9K5w49hU+Ks=
+github.com/ooni/probe-engine v0.0.0-20190828175829-9e908f8e3a3a h1:e3SiEPrbtsB76tPm+sFLI5dtKBd7sa+vjtdfvSMKLqk=
+github.com/ooni/probe-engine v0.0.0-20190828175829-9e908f8e3a3a/go.mod h1:zovuewNdJp8exKnBlpCgZXNu/SN+Cs4acj2KgGZOP9s=
 github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8=
 github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
-github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE=
-github.com/oschwald/maxminddb-golang v1.3.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
-github.com/oschwald/maxminddb-golang v1.3.1 h1:kPc5+ieL5CC/Zn0IaXJPxDFlUxKTQEU8QBTtmfQDAIo=
-github.com/oschwald/maxminddb-golang v1.3.1/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
+github.com/oschwald/maxminddb-golang v1.4.0 h1:5/rpmW41qrgSed4wK32rdznbkTSXHcraY2LOMJX4DMc=
+github.com/oschwald/maxminddb-golang v1.4.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

internal/cli/run/run.go

@@ -3,8 +3,6 @@ package run
 import (
 	"context"
 	"errors"
-	"fmt"
-	"strings"
 
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
@@ -50,20 +48,20 @@ func init() {
 	cmd := root.Command("run", "Run a test group or OONI Run link")
 
 	var nettestGroupNamesBlue []string
+	var ctx *ooni.Context
+	var network *database.Network
+
 	for name := range groups.NettestGroups {
 		nettestGroupNamesBlue = append(nettestGroupNamesBlue, color.BlueString(name))
 	}
 
-	nettestGroup := cmd.Arg("name",
-		fmt.Sprintf("the nettest group to run. Supported tests are: %s, or nothing to run them all",
-			strings.Join(nettestGroupNamesBlue, ", "))).String()
-
 	noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool()
 	collectorURL := cmd.Flag("collector-url", "Specify the address of a custom collector").String()
 	bouncerURL := cmd.Flag("bouncer-url", "Specify the address of a custom bouncer").String()
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
-		ctx, err := root.Init()
+		var err error
+		ctx, err = root.Init()
 		if err != nil {
 			log.Errorf("%s", err)
 			return err
@@ -91,7 +89,7 @@ func init() {
 			log.WithError(err).Error("Failed to lookup the location of the probe")
 			return err
 		}
-		network, err := database.CreateNetwork(ctx.DB, ctx.Session.Location)
+		network, err = database.CreateNetwork(ctx.DB, ctx.Session.Location)
 		if err != nil {
 			log.WithError(err).Error("Failed to create the network row")
 			return err
@@ -112,8 +110,27 @@ func init() {
 				return err
 			}
 		}
+		return nil
+	})
 
-		if *nettestGroup == "" {
+	websitesCmd := cmd.Command("websites", "")
+	websitesCmd.Action(func(_ *kingpin.ParseContext) error {
+		return runNettestGroup("websites", ctx, network)
+	})
+	imCmd := cmd.Command("im", "")
+	imCmd.Action(func(_ *kingpin.ParseContext) error {
+		return runNettestGroup("im", ctx, network)
+	})
+	performanceCmd := cmd.Command("performance", "")
+	performanceCmd.Action(func(_ *kingpin.ParseContext) error {
+		return runNettestGroup("performance", ctx, network)
+	})
+	middleboxCmd := cmd.Command("middlebox", "")
+	middleboxCmd.Action(func(_ *kingpin.ParseContext) error {
+		return runNettestGroup("middlebox", ctx, network)
+	})
+	allCmd := cmd.Command("all", "").Default()
+	allCmd.Action(func(_ *kingpin.ParseContext) error {
 		log.Infof("Running %s tests", color.BlueString("all"))
 		for tg := range groups.NettestGroups {
 			if err := runNettestGroup(tg, ctx, network); err != nil {
@@ -121,8 +138,5 @@ func init() {
 			}
 		}
 		return nil
-		}
-		log.Infof("Running %s", color.BlueString(*nettestGroup))
-		return runNettestGroup(*nettestGroup, ctx, network)
 	})
 }

internal/cli/show/show.go

@@ -4,19 +4,27 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/internal/cli/root"
+	"github.com/ooni/probe-cli/internal/database"
+	"github.com/ooni/probe-cli/internal/output"
 )
 
 func init() {
 	cmd := root.Command("show", "Show a specific measurement")
 
+	msmtID := cmd.Arg("id", "the id of the measurement to show").Int64()
+
 	cmd.Action(func(_ *kingpin.ParseContext) error {
-		_, err := root.Init()
+		ctx, err := root.Init()
 		if err != nil {
 			log.WithError(err).Error("failed to initialize root context")
 			return err
 		}
-		log.Error("this function is not implemented")
-
+		msmt, err := database.GetMeasurementJSON(ctx.DB, *msmtID)
+		if err != nil {
+			log.Errorf("error: %v", err)
+			return err
+		}
+		output.MeasurementJSON(msmt)
 		return nil
 	})
 }

internal/database/actions.go

@@ -3,6 +3,9 @@ package database
 import (
 	"database/sql"
 	"encoding/json"
+	"bufio"
+	"io"
+	"io/ioutil"
 	"os"
 	"reflect"
 	"time"
@@ -10,6 +13,7 @@ import (
 	"github.com/apex/log"
 	"github.com/ooni/probe-engine/model"
 	"github.com/ooni/probe-cli/utils"
+	"github.com/ooni/probe-cli/internal/util"
 	"github.com/pkg/errors"
 	db "upper.io/db.v3"
 	"upper.io/db.v3/lib/sqlbuilder"
@@ -38,7 +42,67 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR
 	return measurements, nil
 }
 
-// GetResultTestKeys returns a list of TestKeys for a given measurements
+// GetMeasurementJSON will a map[string]interface{} given a database and a measurementID
+func GetMeasurementJSON(sess sqlbuilder.Database, measurementID int64) (map[string]interface{}, error) {
+	var (
+		measurement MeasurementURLNetwork
+		msmtJSON map[string]interface{}
+	)
+
+	req := sess.Select(
+		db.Raw("urls.*"),
+		db.Raw("measurements.*"),
+	).From("measurements").
+		LeftJoin("urls").On("urls.url_id = measurements.url_id").
+		Where("measurements.measurement_id= ?", measurementID)
+
+	if err := req.One(&measurement); err != nil {
+		log.Errorf("failed to run query %s: %v", req.String(), err)
+		return nil, err
+	}
+	reportFilePath := measurement.Measurement.ReportFilePath
+	// If the url->url is NULL then we are dealing with a single entry
+	// measurement and all we have to do is read the file and return it.
+	if (measurement.URL.URL.Valid == false) {
+		b, err := ioutil.ReadFile(reportFilePath)
+		if err != nil {
+			return nil, err
+		}
+		if err := json.Unmarshal(b, &msmtJSON); err != nil {
+			return nil, err
+		}
+		return msmtJSON, nil
+	}
+
+	// When the URL is a string then we need to seek until we reach the
+	// measurement line in the file that matches the target input
+	url := measurement.URL.URL.String
+	file, err := os.Open(reportFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	reader := bufio.NewReader(file)
+
+	for {
+		line, err := util.ReadLine(reader)
+		if (err == io.EOF) {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		if err := json.Unmarshal([]byte(line), &msmtJSON); err != nil {
+			return nil, err
+		}
+		if (msmtJSON["input"].(string) == url) {
+			return msmtJSON, nil
+		}
+	}
+	return nil, errors.New("Could not find measurement")
+}
+
+// GetResultTestKeys returns a list of TestKeys for a given result
 func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error) {
 	res := sess.Collection("measurements").Find("result_id", resultID)
 	defer res.Close()
@@ -48,22 +112,20 @@ func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error)
 		tk   PerformanceTestKeys
 	)
 	for res.Next(&msmt) {
-		if msmt.TestName == "web_connectivity" {
-			break
-		}
 		// We only really care about performance keys.
 		// Note: since even in case of failure we still initialise an empty struct,
 		// it could be that these keys come out as initializes with the default
 		// values.
 		// XXX we may want to change this behaviour by adding `omitempty` to the
 		// struct definition.
-		if msmt.TestName == "ndt" || msmt.TestName == "dash" {
+		if msmt.TestName != "ndt" && msmt.TestName != "dash" {
+			return "{}", nil
+		}
 		if err := json.Unmarshal([]byte(msmt.TestKeys), &tk); err != nil {
 			log.WithError(err).Error("failed to parse testKeys")
 			return "{}", err
 		}
 	}
-	}
 	b, err := json.Marshal(tk)
 	if err != nil {
 		log.WithError(err).Error("failed to serialize testKeys")

internal/log/handlers/cli/cli.go

@@ -119,6 +119,8 @@ func (h *Handler) TypedLog(t string, e *log.Entry) error {
 		return logTable(h.Writer, e.Fields)
 	case "measurement_item":
 		return logMeasurementItem(h.Writer, e.Fields)
+	case "measurement_json":
+		return logMeasurementJSON(h.Writer, e.Fields)
 	case "measurement_summary":
 		return logMeasurementSummary(h.Writer, e.Fields)
 	case "result_item":

internal/log/handlers/cli/measurements.go

@@ -129,3 +129,14 @@ func logMeasurementSummary(w io.Writer, f log.Fields) error {
 
 	return nil
 }
+
+func logMeasurementJSON(w io.Writer, f log.Fields) error {
+	m := f.Get("measurement_json").(map[string]interface{})
+
+	json, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(w, string(json))
+	return nil
+}

internal/output/output.go

@@ -11,6 +11,14 @@ import (
 	"github.com/ooni/probe-cli/internal/util"
 )
 
+// MeasurementJSON prints the JSON of a measurement
+func MeasurementJSON(j map[string]interface{}) {
+	log.WithFields(log.Fields{
+		"type":       "measurement_json",
+		"measurement_json": j,
+	}).Info("Measurement JSON")
+}
+
 // Progress logs a progress type event
 func Progress(key string, perc float64, msg string) {
 	log.WithFields(log.Fields{

internal/util/util.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"bytes"
+	"bufio"
 	"fmt"
 	"os"
 	"regexp"
@@ -112,3 +113,18 @@ func WrapString(s string, lim uint) string {
 
 	return buf.String()
 }
+
+
+// ReadLine will read a single line from a bufio.Reader
+func ReadLine(r *bufio.Reader) (string, error) {
+	var (
+		isPrefix bool
+		err error
+		line, ln []byte
+	)
+	for isPrefix && err == nil {
+		line, isPrefix, err = r.ReadLine()
+		ln = append(ln, line...)
+	}
+	return string(ln), err
+}

nettests/websites/web_connectivity.go

@@ -10,11 +10,11 @@ import (
 	"github.com/ooni/probe-engine/orchestra/testlists"
 )
 
-func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
+func lookupURLs(ctl *nettests.Controller, limit int) ([]string, map[int64]int64, error) {
 	var urls []string
 	urlIDMap := make(map[int64]int64)
 	testlist, err := testlists.NewClient(ctl.Ctx.Session).Do(
-		context.Background(), ctl.Ctx.Session.ProbeCC(),
+		context.Background(), ctl.Ctx.Session.ProbeCC(), limit,
 	)
 	if err != nil {
 		return nil, nil, err
@@ -41,7 +41,7 @@ type WebConnectivity struct {
 
 // Run starts the test
 func (n WebConnectivity) Run(ctl *nettests.Controller) error {
-	urls, urlIDMap, err := lookupURLs(ctl)
+	urls, urlIDMap, err := lookupURLs(ctl, ctl.Ctx.Config.NettestGroups.Websites.Limit)
 	if err != nil {
 		return err
 	}

scripts/travis_test.sh

@@ -3,4 +3,4 @@
 set -ex
 
 ./dist/${OS_NAME}/amd64/ooni onboard --yes
-./dist/${OS_NAME}/amd64/ooni run -v --no-collector
+./dist/${OS_NAME}/amd64/ooni run --config testdata/testing-config.json -v --no-collector

testdata/testing-config.json

@@ -0,0 +1,64 @@
+{
+  "_": "This is your OONI Probe config file. See https://ooni.io/help/probe-cli for help",
+  "_version": 0,
+  "_informed_consent": true,
+  "_is_beta": true,
+  "auto_update": true,
+  "sharing": {
+    "include_ip": false,
+    "include_asn": true,
+    "include_country": true,
+    "include_gps": true,
+    "upload_results": true
+  },
+  "notifications": {
+    "enabled": true,
+    "notify_on_test_completion": true,
+    "notify_on_news": false
+  },
+  "automated_testing": {
+    "enabled": false,
+    "enabled_tests": [
+      "web-connectivity",
+      "facebook-messenger",
+      "whatsapp",
+      "telegram",
+      "dash",
+      "ndt",
+      "http-invalid-request-line",
+      "http-header-field-manipulation"
+    ],
+    "monthly_allowance": "300MB"
+  },
+  "test_settings": {
+    "websites": {
+      "enabled_categories": [],
+      "limit": 10
+    },
+    "instant_messaging": {
+      "enabled_tests": [
+        "facebook-messenger",
+        "whatsapp",
+        "telegram"
+      ]
+    },
+    "performance": {
+      "ndt_server": "auto",
+      "ndt_server_port": "auto",
+      "dash_server": "auto",
+      "dash_server_port": "auto"
+    },
+    "middlebox": {
+      "enabled_tests": [
+        "http-invalid-request-line",
+        "http-header-field-manipulation"
+      ]
+    }
+  },
+  "advanced": {
+    "use_domain_fronting": false,
+    "send_crash_reports": true,
+    "collector_url": "",
+    "bouncer_url": "https://bouncer.ooni.io"
+  }
+}