Merge pull request #13 from ooni/release/beta

WIP towards BETA

Gopkg.lock

@@ -29,16 +29,10 @@
   version = "v1.0.0"
 
 [[projects]]
-  branch = "master"
+  name = "github.com/certifi/gocertifi"
-  name = "github.com/aybabtme/rgbterm"
   packages = ["."]
-  revision = "cc83f3b3ce5911279513a46d6d3316d67bedaa54"
+  revision = "deb3ae2ef2610fde3330947281941c562861188b"
-
+  version = "2018.01.18"
-[[projects]]
-  branch = "master"
-  name = "github.com/beorn7/perks"
-  packages = ["quantile"]
-  revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
 
 [[projects]]
   name = "github.com/fatih/color"
@@ -47,10 +41,10 @@
   version = "v1.7.0"
 
 [[projects]]
-  name = "github.com/golang/protobuf"
+  branch = "master"
-  packages = ["proto"]
+  name = "github.com/getsentry/raven-go"
-  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
+  packages = ["."]
-  version = "v1.1.0"
+  revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa"
 
 [[projects]]
   branch = "master"
@@ -79,12 +73,6 @@
   revision = "323a32be5a2421b8c7087225079c6c900ec397cd"
   version = "v1.7.0"
 
-[[projects]]
-  name = "github.com/matttproud/golang_protobuf_extensions"
-  packages = ["pbutil"]
-  revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
-  version = "v1.0.1"
-
 [[projects]]
   branch = "master"
   name = "github.com/measurement-kit/go-measurement-kit"
@@ -97,12 +85,6 @@
   packages = ["."]
   revision = "9520e82c474b0a04dd04f8a40959027271bab992"
 
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66"
-
 [[projects]]
   name = "github.com/oschwald/geoip2-golang"
   packages = ["."]
@@ -121,40 +103,6 @@
   revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
   version = "v0.8.0"
 
-[[projects]]
-  name = "github.com/prometheus/client_golang"
-  packages = ["prometheus"]
-  revision = "c5b7fccd204277076155f10851dad72b76a49317"
-  version = "v0.8.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/client_model"
-  packages = ["go"]
-  revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/common"
-  packages = [
-    "expfmt",
-    "internal/bitbucket.org/ww/goautoneg",
-    "model",
-    "version"
-  ]
-  revision = "7600349dcfe1abd18d72d3a1770870d9800a7801"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/procfs"
-  packages = [
-    ".",
-    "internal/util",
-    "nfs",
-    "xfs"
-  ]
-  revision = "fe93d378a6b03758a2c1b65e86cf630bf78681c0"
-
 [[projects]]
   branch = "master"
   name = "github.com/rubenv/sql-migrate"
@@ -201,6 +149,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "95c3e971d63b97b0dc531f67d98401cfa9968b99aacf1eed73ce801bbaadb0cd"
+  inputs-digest = "b2f5c39222a1fb405e3f48d2ae3b4758757fe708e12dbd23743c19135e225579"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -61,3 +61,7 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"]
 [[constraint]]
   name = "github.com/oschwald/geoip2-golang"
   version = "1.2.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/getsentry/raven-go"

LICENSE.md

@@ -0,0 +1,26 @@
+Copyright 2018 Open Observatory of Network Interference (OONI), The Tor Project
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

cmd/ooni/main.go

@@ -8,18 +8,22 @@ import (
 	_ "github.com/ooni/probe-cli/internal/cli/info"
 	_ "github.com/ooni/probe-cli/internal/cli/list"
 	_ "github.com/ooni/probe-cli/internal/cli/nettest"
+	_ "github.com/ooni/probe-cli/internal/cli/onboard"
 	_ "github.com/ooni/probe-cli/internal/cli/run"
 	_ "github.com/ooni/probe-cli/internal/cli/show"
 	_ "github.com/ooni/probe-cli/internal/cli/upload"
 	_ "github.com/ooni/probe-cli/internal/cli/version"
 
 	"github.com/ooni/probe-cli/internal/cli/app"
+	"github.com/ooni/probe-cli/internal/crashreport"
 )
 
 func main() {
-	err := app.Run()
+	crashreport.CapturePanicAndWait(func() {
-	if err == nil {
+		err := app.Run()
-		return
+		if err == nil {
-	}
+			return
-	log.WithError(err).Fatal("main exit")
+		}
+		log.WithError(err).Fatal("main exit")
+	}, nil)
 }

config/advanced.go

@@ -1,7 +0,0 @@
-package config
-
-// Advanced settings
-type Advanced struct {
-	IncludeCountry    bool `json:"include_country"`
-	UseDomainFronting bool `json:"use_domain_fronting"`
-}

config/automated_testing.go

@@ -1,8 +0,0 @@
-package config
-
-// AutomatedTesting settings
-type AutomatedTesting struct {
-	Enabled          bool     `json:"enabled"`
-	EnabledTests     []string `json:"enabled_tests"`
-	MonthlyAllowance string   `json:"monthly_allowance"`
-}

config/notifications.go

@@ -1,8 +0,0 @@
-package config
-
-// Notifications settings
-type Notifications struct {
-	Enabled                bool `json:"enabled"`
-	NotifyOnTestCompletion bool `json:"notify_on_test_completion"`
-	NotifyOnNews           bool `json:"notify_on_news"`
-}

config/parser.go

@@ -0,0 +1,109 @@
+package config
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"sync"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/internal/crashreport"
+	"github.com/ooni/probe-cli/utils"
+	"github.com/pkg/errors"
+)
+
+// ReadConfig reads the configuration from the path
+func ReadConfig(path string) (*Config, error) {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := ParseConfig(b)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing config")
+	}
+	c.path = path
+	return c, err
+}
+
+// ParseConfig returns config from JSON bytes.
+func ParseConfig(b []byte) (*Config, error) {
+	var c Config
+
+	if err := json.Unmarshal(b, &c); err != nil {
+		return nil, errors.Wrap(err, "parsing json")
+	}
+
+	if err := c.Default(); err != nil {
+		return nil, errors.Wrap(err, "defaulting")
+	}
+
+	if err := c.Validate(); err != nil {
+		return nil, errors.Wrap(err, "validating")
+	}
+	if c.Advanced.SendCrashReports == false {
+		log.Info("Disabling crash reporting.")
+		crashreport.Disabled = true
+	}
+
+	return &c, nil
+}
+
+// Config for the OONI Probe installation
+type Config struct {
+	// Private settings
+	Comment         string `json:"_"`
+	Version         int64  `json:"_version"`
+	InformedConsent bool   `json:"_informed_consent"`
+	IsBeta          bool   `json:"_is_beta"` // This is a boolean flag used to indicate this installation of OONI Probe was a beta install. These installations will have their data deleted across releases.
+
+	AutoUpdate       bool             `json:"auto_update"`
+	Sharing          Sharing          `json:"sharing"`
+	Notifications    Notifications    `json:"notifications"`
+	AutomatedTesting AutomatedTesting `json:"automated_testing"`
+	NettestGroups    NettestGroups    `json:"test_settings"`
+	Advanced         Advanced         `json:"advanced"`
+
+	mutex sync.Mutex
+	path  string
+}
+
+// Write the config file in json to the path
+func (c *Config) Write() error {
+	c.Lock()
+	configJSON, _ := json.MarshalIndent(c, "", "  ")
+	if c.path == "" {
+		return errors.New("config file path is empty")
+	}
+	if err := ioutil.WriteFile(c.path, configJSON, 0644); err != nil {
+		return errors.Wrap(err, "writing config JSON")
+	}
+	c.Unlock()
+	return nil
+}
+
+// Lock acquires the write mutex
+func (c *Config) Lock() {
+	c.mutex.Lock()
+}
+
+// Unlock releases the write mutex
+func (c *Config) Unlock() {
+	c.mutex.Unlock()
+}
+
+// Default config settings
+func (c *Config) Default() error {
+	home, err := utils.GetOONIHome()
+	if err != nil {
+		return err
+	}
+
+	c.path = utils.ConfigPath(home)
+	return nil
+}
+
+// Validate the config file
+func (c *Config) Validate() error {
+	return nil
+}

config/parser_test.go

@@ -0,0 +1,19 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestParseConfig(t *testing.T) {
+	config, err := ReadConfig("testdata/valid-config.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(config.NettestGroups.Middlebox.EnabledTests) < 0 {
+		t.Error("at least one middlebox test should be enabled")
+	}
+	if config.Advanced.IncludeCountry == false {
+		t.Error("country should be included")
+	}
+}

config/settings.go

@@ -82,11 +82,10 @@ func (s *InstantMessaging) NettestConfigs() []NettestConfig {
 
 // Performance nettest group
 type Performance struct {
-	EnabledTests   []string `json:"enabled_tests"`
+	NDTServer      string `json:"ndt_server"`
-	NDTServer      string   `json:"ndt_server"`
+	NDTServerPort  string `json:"ndt_server_port"`
-	NDTServerPort  string   `json:"ndt_server_port"`
+	DashServer     string `json:"dash_server"`
-	DashServer     string   `json:"dash_server"`
+	DashServerPort string `json:"dash_server_port"`
-	DashServerPort string   `json:"dash_server_port"`
 }
 
 // Middlebox nettest group
@@ -101,3 +100,32 @@ type NettestGroups struct {
 	Performance      Performance      `json:"performance"`
 	Middlebox        Middlebox        `json:"middlebox"`
 }
+
+// Notifications settings
+type Notifications struct {
+	Enabled                bool `json:"enabled"`
+	NotifyOnTestCompletion bool `json:"notify_on_test_completion"`
+	NotifyOnNews           bool `json:"notify_on_news"`
+}
+
+// Sharing settings
+type Sharing struct {
+	IncludeIP     bool `json:"include_ip"`
+	IncludeASN    bool `json:"include_asn"`
+	IncludeGPS    bool `json:"include_gps"`
+	UploadResults bool `json:"upload_results"`
+}
+
+// Advanced settings
+type Advanced struct {
+	IncludeCountry    bool `json:"include_country"`
+	UseDomainFronting bool `json:"use_domain_fronting"`
+	SendCrashReports  bool `json:"send_crash_reports"`
+}
+
+// AutomatedTesting settings
+type AutomatedTesting struct {
+	Enabled          bool     `json:"enabled"`
+	EnabledTests     []string `json:"enabled_tests"`
+	MonthlyAllowance string   `json:"monthly_allowance"`
+}

config/sharing.go

@@ -1,10 +0,0 @@
-package config
-
-// Sharing settings
-type Sharing struct {
-	IncludeIP        bool `json:"include_ip"`
-	IncludeASN       bool `json:"include_asn"`
-	IncludeGPS       bool `json:"include_gps"`
-	UploadResults    bool `json:"upload_results"`
-	SendCrashReports bool `json:"send_crash_reports"`
-}

config/testdata/valid-config.json

@@ -0,0 +1,63 @@
+{
+  "_": "This is your OONI Probe config file. See https://ooni.io/help/probe-cli for help",
+  "_version": 0,
+  "_informed_consent": false,
+  "auto_update": true,
+  "sharing": {
+    "include_ip": false,
+    "include_asn": true,
+    "include_gps": true,
+    "upload_results": true
+  },
+  "notifications": {
+    "enabled": true,
+    "notify_on_test_completion": true,
+    "notify_on_news": false
+  },
+  "automated_testing": {
+    "enabled": false,
+    "enabled_tests": [
+      "web-connectivity",
+      "facebook-messenger",
+      "whatsapp",
+      "telegram",
+      "dash",
+      "ndt",
+      "http-invalid-request-line",
+      "http-header-field-manipulation"
+    ],
+    "monthly_allowance": "300MB"
+  },
+  "test_settings": {
+    "websites": {
+      "enabled_categories": []
+    },
+    "instant_messaging": {
+      "enabled_tests": [
+        "facebook-messenger",
+        "whatsapp",
+        "telegram"
+      ]
+    },
+    "performance": {
+      "enabled_tests": [
+        "ndt"
+      ],
+      "ndt_server": "auto",
+      "ndt_server_port": "auto",
+      "dash_server": "auto",
+      "dash_server_port": "auto"
+    },
+    "middlebox": {
+      "enabled_tests": [
+        "http-invalid-request-line",
+        "http-header-field-manipulation"
+      ]
+    }
+  },
+  "advanced": {
+    "include_country": true,
+    "use_domain_fronting": false,
+    "send_crash_reports": true
+  }
+}

data/default-config.json

@@ -1,12 +1,14 @@
 {
-  "_": "This is your OONI Probe config file. See https://ooni.io/help/ooniprobe-cli for help",
+  "_": "This is your OONI Probe config file. See https://ooni.io/help/probe-cli for help",
+  "_version": 0,
+  "_informed_consent": false,
+  "_is_beta": true,
   "auto_update": true,
   "sharing": {
     "include_ip": false,
     "include_asn": true,
     "include_gps": true,
-    "upload_results": true,
+    "upload_results": true
-    "send_crash_reports": true
   },
   "notifications": {
     "enabled": true,
@@ -56,8 +58,7 @@
   },
   "advanced": {
     "include_country": true,
-    "use_domain_fronting": true
+    "use_domain_fronting": false,
-  },
+    "send_crash_reports": true
-  "_config_version": "0.0.1",
+  }
-  "_informed_consent": true
 }

internal/bindata/bindata.go

@@ -80,26 +80,26 @@ func (fi bindataFileInfo) Sys() interface{} {
 }
 
 var _bindataDataDefaultconfigjson = []byte(
-	"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\xcd\x6e\xe3\x38\x0c\xc7\xef\x7d\x0a\x41\xe7\x3a\xcd\x62\x6f\x39" +
+	"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\x41\x6f\xdb\x3c\x0c\x86\xef\xf9\x15\x82\xce\x75\x53\xe0\xbb\xe5" +
-	"\xee\x6d\x0f\xbb\x1d\x60\xe6\x56\x14\x82\x62\xd1\x36\x31\x32\xa9\x11\xe9\x64\x82\x41\xdf\x7d\x20\x35\x89\xed\x7e" +
+	"\xf8\xdd\x76\xd8\x3a\x60\xbb\x15\x85\x20\x5b\xb4\x4d\x4c\x26\x35\x91\x4e\x16\x0c\xfd\xef\x83\xdc\x24\x56\xda\xae" +
-	"\x4d\x8f\xe2\x9f\xa2\xc8\x1f\x29\xfe\xba\x31\xc6\x3a\xbb\x33\xf6\xdb\x80\x62\x50\xcc\x89\xa7\x6c\xee\xef\xff\xff" +
+	"\xeb\xd1\xef\x4b\x53\xe2\x43\x52\xbf\x37\xc6\x58\x67\x77\xc6\x7e\x1f\x51\x0c\x8a\x39\xf2\x9c\xcd\xfd\xfd\x97\x4f" +
-	"\xd7\x7c\xc9\xbc\x07\xd3\x32\x75\xd8\x9b\x0e\x23\x6c\xcc\x57\x00\x33\xa8\x26\xd9\xdd\xdd\x31\x13\x6e\x90\xef\x06" +
+	"\xe6\x6b\xe6\x16\x4c\xc7\xd4\xe3\x60\x7a\x8c\x70\x6b\xbe\x01\x98\x51\x35\xc9\x6e\xbb\x65\x26\xbc\x45\xde\x8e\x10" +
-	"\x88\xa9\x1e\x52\xf1\x6f\xda\x88\xa6\xe3\x6c\x8a\xd9\xde\x96\xf0\x7e\x52\x76\x53\x0a\x5e\xc1\xee\x8c\xe6\x09\xaa" +
+	"\xd3\x36\x95\xd8\xa6\x8b\x68\x7a\xce\xa6\x48\xf6\x66\x49\xbd\x87\x2c\xc8\x64\x77\xe6\xee\x59\x40\xea\x39\x4f\x10" +
-	"\x59\x06\x9f\x91\x7a\xbb\x33\x25\x09\x63\x2c\x52\x1b\xa7\x00\x0e\x93\xdd\x99\xce\x47\xa9\x7e\x0b\xc1\x0b\x2d\x02" +
+	"\x5c\xc7\x24\x40\x6a\x77\xa6\xf7\x51\xe0\xe4\x8a\x6b\x41\xbd\xdd\x19\xcd\xf3\xb3\xe6\x67\x65\x37\xa7\xe0\x15\x6a" +
-	"\x2c\x84\x3e\xc9\x5a\x98\x52\x64\x1f\x5c\x06\x99\xa2\xbe\xd0\x04\x28\xb8\x36\x7b\x19\x5c\x86\xc4\xf9\xaa\xdf\x18" +
+	"\x59\x46\x9f\x91\x06\xbb\x33\xa5\x06\x63\x2c\x52\x17\xe7\x00\x0e\x53\x9d\xb2\x32\xbc\x50\x95\xa0\x32\x86\x24\xd7" +
-	"\xf3\x54\x33\x23\x56\xec\xb0\xf5\x8a\x4c\x32\xe7\x07\xe4\xf7\x11\xc2\x3a\x5a\xf5\x3d\x39\x26\xa7\x20\xea\x5a\x1e" +
+	"\xc6\x9c\x22\xfb\xe0\x32\xc8\x1c\xf5\xec\x6d\x8c\x79\x5a\x4e\x26\x56\xec\xb1\xf3\x8a\x4c\xb2\x9e\x0f\xe4\xdb\x08" +
-	"\x53\x84\x72\xf1\x3d\x37\x82\xa3\x5c\xea\xbb\xbe\x58\x10\x8d\x5e\x21\xd4\x28\x2b\x2a\xf3\xab\x4b\x24\x67\x6b\x75" +
+	"\xe1\x3a\xd3\x12\x7b\x74\x4c\x4e\x41\xd4\x75\x3c\xa5\x08\xfa\x0c\xe4\xcd\x30\x82\x83\x9c\xef\x7f\x39\xb1\x20\x98" +
-	"\x2f\xe1\x1e\xaa\xd9\x18\x7b\x84\x7d\xd3\x32\x11\xb4\x8a\x07\xd4\x93\xbd\xbd\x28\x9d\x6f\x61\xcf\xfc\xbd\x19\x41" +
+	"\xbc\x42\x58\xb2\x5c\x55\xbd\x9e\x5a\x97\x7c\x52\x97\xf0\x92\xee\x61\x91\x8d\xb1\x07\x68\x9b\x8e\x89\xa0\x53\xdc" +
-	"\x04\xa8\x87\x3c\x6b\xc7\xc1\xab\xf8\x94\x66\x8b\x42\x84\x3e\xfb\x71\xb6\x04\x2f\xc3\x7c\xa2\xa0\xf3\xa1\x8c\x44" +
+	"\xa3\x1e\xed\xcd\xd9\xe9\x7d\x07\x2d\xf3\x8f\x66\x02\x11\xa0\x01\xf2\xea\x1d\x46\xaf\xe2\x53\x5a\x15\x85\x08\x43" +
-	"\x83\x74\xf0\x11\x43\x93\xe1\xc7\x04\xa2\x4d\x44\x82\x17\x2e\x03\xf8\x00\xb9\xe9\x10\x62\x68\x46\x4f\x98\xa6\x58" +
+	"\xf6\xd3\xaa\x04\x2f\xe3\xfa\x45\x41\xd7\x8f\x32\x31\x0d\xd2\xde\x47\x0c\x4d\x86\x9f\x33\x88\x36\x11\x09\x5e\x84" +
-	"\x29\xdb\xea\xf6\x78\x2e\x6e\x64\xd2\x21\x9e\x9c\x8f\x91\x8f\x9e\xda\x32\x36\xf6\xef\xed\xf6\xbf\x7f\xec\x95\x58" +
+	"\x8c\xe0\x03\xe4\xa6\x47\x88\xa1\x99\x3c\x61\x9a\xe3\x42\xd9\x2e\x61\x8f\xa7\xe2\x26\x26\x1d\xe3\xd1\xf9\x18\xf9" +
-	"\xa5\x2d\xa0\x05\xd6\xa2\x47\x47\xd8\x0b\x2a\xcc\x96\x05\xab\xd6\x2b\xf4\x9c\xb1\xaa\x0f\x8f\x55\x7e\xba\x4e\x92" +
+	"\xe0\xa9\x2b\x63\x61\xff\xbb\xbb\xfb\xfc\xbf\xbd\x10\x5b\x68\x0b\x68\x81\x55\xf5\xe8\x00\xad\xa0\xc2\xaa\x54\xac" +
-	"\xa8\x27\x75\x85\x8d\xef\x97\x0d\xf8\x00\xf6\xc7\x50\xdf\xc2\xba\x04\x7b\x36\xad\xf3\x48\x90\x3b\xce\xe3\xb9\xe8" +
+	"\x3a\xaf\x30\x70\xc6\xc5\x7d\x78\x5c\xec\xa7\xcb\xa4\x88\x7a\x52\x57\xd8\xf8\xa1\x6e\xc0\x3b\xb0\xdf\x87\xfa\x16" +
-	"\xcf\x64\x50\x1a\x71\x09\xb5\xec\x8e\x13\xc8\x07\xc8\x05\x5d\x99\x2e\xfb\x86\xe6\xca\xf4\xbf\x76\x28\x8d\x7e\xf7" +
+	"\xd6\x1a\xec\x49\xba\xbe\x47\x82\x5c\xb6\xe7\x54\xf4\x47\x6e\x50\x1a\x71\x4e\x55\x77\xc7\x09\xe4\x3d\xe4\x82\xae" +
-	"\xf6\x42\x5c\x5f\x5f\x95\x31\x62\x08\x11\xf6\xfc\xf3\x93\x45\xfc\x79\x80\x3e\x39\x42\x57\x9e\xf3\xd7\x0a\x87\x42" +
+	"\x4c\x97\x7d\xc3\x73\x89\xb3\xbe\x0e\x28\x8d\xfe\xeb\xdf\x95\x79\xfd\xfb\x55\x19\x13\x86\x10\xa1\xe5\x5f\x1f\x2c" +
-	"\x33\xbc\xde\x33\x2d\x4f\xa4\xf9\xf4\x62\x73\x08\xb8\xc0\xa3\x47\x72\x5d\x66\x3a\xff\xc5\xd5\x7a\x70\xcf\x2b\xd1" +
+	"\xe2\xdf\x03\xf4\xc1\x11\xba\xf0\x5c\x57\x2b\xec\x0b\xcd\xf0\xfa\x1d\xe9\x78\x26\xcd\xc7\x17\x2f\x83\x80\x0b\x3c" +
-	"\x1d\x20\xcb\xf3\x47\xb7\xdb\xcd\x76\xf3\xd7\xf3\xb6\x73\x48\xa5\x83\x65\xde\x98\x04\x48\x2f\xd7\x9f\x6e\x7e\x07" +
+	"\x79\x24\xd7\x67\xa6\xd3\x2e\xd6\xab\x27\x40\xc1\x75\xb9\x70\xc8\x50\x10\xd4\xef\xc7\xe6\x69\xf3\x27\x00\x00\xff" +
-	"\x00\x00\xff\xff\x0f\x7e\x15\xb3\x6d\x05\x00\x00")
+	"\xff\x42\x02\xc0\xed\x72\x05\x00\x00")
 
 func bindataDataDefaultconfigjsonBytes() ([]byte, error) {
 	return bindataRead(

internal/cli/app/app.go

@@ -3,13 +3,13 @@ package app
 import (
 	"os"
 
+	ooni "github.com/ooni/probe-cli"
 	"github.com/ooni/probe-cli/internal/cli/root"
-	"github.com/ooni/probe-cli/internal/cli/version"
 )
 
 // Run the app. This is the main app entry point
 func Run() error {
-	root.Cmd.Version(version.Version)
+	root.Cmd.Version(ooni.Version)
 	_, err := root.Cmd.Parse(os.Args[1:])
 	return err
 }

internal/cli/geoip/geoip.go

@@ -1,9 +1,12 @@
 package geoip
 
 import (
+	"fmt"
+
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/internal/cli/root"
+	"github.com/ooni/probe-cli/internal/output"
 	"github.com/ooni/probe-cli/utils"
 )
 
@@ -13,12 +16,16 @@ func init() {
 	shouldUpdate := cmd.Flag("update", "Update the geoip database").Bool()
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
-		log.Info("geoip")
+		output.SectionTitle("GeoIP lookup")
 		ctx, err := root.Init()
 		if err != nil {
 			return err
 		}
 
+		if err = ctx.MaybeDownloadDataFiles(); err != nil {
+			log.WithError(err).Error("failed to download data files")
+		}
+
 		geoipPath := utils.GeoIPDir(ctx.Home)
 		if *shouldUpdate {
 			utils.DownloadGeoIPDatabaseFiles(geoipPath)
@@ -31,7 +38,8 @@ func init() {
 		}
 
 		log.WithFields(log.Fields{
-			"asn":          loc.ASN,
+			"type":         "table",
+			"asn":          fmt.Sprintf("AS%d", loc.ASN),
 			"network_name": loc.NetworkName,
 			"country_code": loc.CountryCode,
 			"ip":           loc.IP,

internal/cli/info/info.go

@@ -11,6 +11,7 @@ func init() {
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		log.Info("Info")
+		log.Error("this function is not implemented")
 		return nil
 	})
 }

internal/cli/list/list.go

@@ -1,6 +1,8 @@
 package list
 
 import (
+	"fmt"
+
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/internal/cli/root"
@@ -11,52 +13,78 @@ import (
 func init() {
 	cmd := root.Command("list", "List results")
 
+	resultID := cmd.Arg("id", "the id of the result to list measurements for").Int64()
+
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		ctx, err := root.Init()
 		if err != nil {
 			log.WithError(err).Error("failed to initialize root context")
 			return err
 		}
-		doneResults, incompleteResults, err := database.ListResults(ctx.DB)
+		if *resultID > 0 {
-		if err != nil {
+			measurements, err := database.ListMeasurements(ctx.DB, *resultID)
-			log.WithError(err).Error("failed to list results")
+			if err != nil {
-			return err
+				log.WithError(err).Error("failed to list measurements")
+				return err
+			}
+			for idx, msmt := range measurements {
+				fmt.Printf("%d: %v\n", idx, msmt)
+			}
+		} else {
+			doneResults, incompleteResults, err := database.ListResults(ctx.DB)
+			if err != nil {
+				log.WithError(err).Error("failed to list results")
+				return err
+			}
+
+			if len(incompleteResults) > 0 {
+				output.SectionTitle("Incomplete results")
+			}
+			for idx, result := range incompleteResults {
+				output.ResultItem(output.ResultItemData{
+					ID:            result.ID,
+					Index:         idx,
+					TotalCount:    len(incompleteResults),
+					Name:          result.Name,
+					StartTime:     result.StartTime,
+					NetworkName:   result.NetworkName,
+					Country:       result.Country,
+					ASN:           result.ASN,
+					Summary:       result.Summary,
+					Done:          result.Done,
+					DataUsageUp:   result.DataUsageUp,
+					DataUsageDown: result.DataUsageDown,
+				})
+			}
+
+			resultSummary := output.ResultSummaryData{}
+			netCount := make(map[string]int)
+			output.SectionTitle("Results")
+			for idx, result := range doneResults {
+				output.ResultItem(output.ResultItemData{
+					ID:            result.ID,
+					Index:         idx,
+					TotalCount:    len(doneResults),
+					Name:          result.Name,
+					StartTime:     result.StartTime,
+					NetworkName:   result.NetworkName,
+					Country:       result.Country,
+					ASN:           result.ASN,
+					Summary:       result.Summary,
+					Done:          result.Done,
+					DataUsageUp:   result.DataUsageUp,
+					DataUsageDown: result.DataUsageDown,
+				})
+				resultSummary.TotalTests++
+				netCount[result.ASN]++
+				resultSummary.TotalDataUsageUp += result.DataUsageUp
+				resultSummary.TotalDataUsageDown += result.DataUsageDown
+			}
+			resultSummary.TotalNetworks = int64(len(netCount))
+
+			output.ResultSummary(resultSummary)
 		}
 
-		log.Info("Results")
-		for idx, result := range doneResults {
-			output.ResultItem(output.ResultItemData{
-				ID:            result.ID,
-				Index:         idx,
-				TotalCount:    len(doneResults),
-				Name:          result.Name,
-				StartTime:     result.StartTime,
-				NetworkName:   result.NetworkName,
-				Country:       result.Country,
-				ASN:           result.ASN,
-				Summary:       result.Summary,
-				Done:          result.Done,
-				DataUsageUp:   result.DataUsageUp,
-				DataUsageDown: result.DataUsageDown,
-			})
-		}
-		log.Info("Incomplete results")
-		for idx, result := range incompleteResults {
-			output.ResultItem(output.ResultItemData{
-				ID:            result.ID,
-				Index:         idx,
-				TotalCount:    len(incompleteResults),
-				Name:          result.Name,
-				StartTime:     result.StartTime,
-				NetworkName:   result.NetworkName,
-				Country:       result.Country,
-				ASN:           result.ASN,
-				Summary:       result.Summary,
-				Done:          result.Done,
-				DataUsageUp:   result.DataUsageUp,
-				DataUsageDown: result.DataUsageDown,
-			})
-		}
 		return nil
 	})
 }

internal/cli/nettest/nettest.go

@@ -11,6 +11,7 @@ func init() {
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		log.Info("Nettest")
+		log.Error("this function is not implemented")
 		return nil
 	})
 }

internal/cli/onboard/onboard.go

@@ -0,0 +1,35 @@
+package onboard
+
+import (
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/internal/cli/root"
+	"github.com/ooni/probe-cli/internal/onboard"
+)
+
+func init() {
+	cmd := root.Command("onboard", "Starts the onboarding process")
+
+	yes := cmd.Flag("yes", "Answer yes to all the onboarding questions.").Bool()
+
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		ctx, err := root.Init()
+		if err != nil {
+			return err
+		}
+
+		if *yes == true {
+			ctx.Config.Lock()
+			ctx.Config.InformedConsent = true
+			ctx.Config.Unlock()
+
+			if err := ctx.Config.Write(); err != nil {
+				log.WithError(err).Error("failed to write config file")
+				return err
+			}
+			return nil
+		}
+
+		return onboard.Onboarding(ctx.Config)
+	})
+}

internal/cli/root/root.go

@@ -7,7 +7,6 @@ import (
 	"github.com/ooni/probe-cli/internal/log/handlers/batch"
 	"github.com/ooni/probe-cli/internal/log/handlers/cli"
 	"github.com/ooni/probe-cli/utils"
-	"github.com/prometheus/common/version"
 )
 
 // Cmd is the root command
@@ -33,7 +32,7 @@ func init() {
 		}
 		if *isVerbose {
 			log.SetLevel(log.DebugLevel)
-			log.Debugf("ooni version %s", version.Version)
+			log.Debugf("ooni version %s", ooni.Version)
 		}
 
 		Init = func() (*ooni.Context, error) {

internal/cli/run/run.go

@@ -4,10 +4,12 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
+	"github.com/fatih/color"
 	"github.com/ooni/probe-cli/internal/cli/root"
 	"github.com/ooni/probe-cli/internal/database"
 	"github.com/ooni/probe-cli/nettests"
@@ -18,7 +20,14 @@ import (
 func init() {
 	cmd := root.Command("run", "Run a test group or OONI Run link")
 
-	nettestGroup := cmd.Arg("name", "the nettest group to run").Required().String()
+	var nettestGroupNames []string
+	for name := range groups.NettestGroups {
+		nettestGroupNames = append(nettestGroupNames, color.BlueString(name))
+	}
+
+	nettestGroup := cmd.Arg("name",
+		fmt.Sprintf("the nettest group to run. Supported tests are: %s",
+			strings.Join(nettestGroupNames, ", "))).Required().String()
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		log.Infof("Starting %s", *nettestGroup)
@@ -27,6 +36,12 @@ func init() {
 			log.Errorf("%s", err)
 			return err
 		}
+
+		if err = ctx.MaybeOnboarding(); err != nil {
+			log.WithError(err).Error("failed to perform onboarding")
+			return err
+		}
+
 		group, ok := groups.NettestGroups[*nettestGroup]
 		if !ok {
 			log.Errorf("No test group named %s", *nettestGroup)

internal/cli/show/show.go

@@ -10,7 +10,13 @@ func init() {
 	cmd := root.Command("show", "Show a specific measurement")
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
-		log.Info("Show")
+		_, err := root.Init()
+		if err != nil {
+			log.WithError(err).Error("failed to initialize root context")
+			return err
+		}
+		log.Error("this function is not implemented")
+
 		return nil
 	})
 }

internal/cli/upload/upload.go

@@ -11,6 +11,7 @@ func init() {
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		log.Info("Uploading")
+		log.Error("this function is not implemented")
 		return nil
 	})
 }

internal/cli/version/version.go

@@ -4,15 +4,15 @@ import (
 	"fmt"
 
 	"github.com/alecthomas/kingpin"
+	ooni "github.com/ooni/probe-cli"
 	"github.com/ooni/probe-cli/internal/cli/root"
 )
 
-const Version = "3.0.0-dev.0"
 
 func init() {
 	cmd := root.Command("version", "Show version.")
 	cmd.Action(func(_ *kingpin.ParseContext) error {
-		fmt.Println(Version)
+		fmt.Println(ooni.Version)
 		return nil
 	})
 }

internal/colors/colors.go

@@ -1,40 +0,0 @@
-package colors
-
-import (
-	color "github.com/aybabtme/rgbterm"
-)
-
-// Gray string.
-func Gray(s string) string {
-	return color.FgString(s, 150, 150, 150)
-}
-
-// Blue string.
-func Blue(s string) string {
-	return color.FgString(s, 77, 173, 247)
-}
-
-// Cyan string.
-func Cyan(s string) string {
-	return color.FgString(s, 34, 184, 207)
-}
-
-// Green string.
-func Green(s string) string {
-	return color.FgString(s, 0, 200, 255)
-}
-
-// Red string.
-func Red(s string) string {
-	return color.FgString(s, 194, 37, 92)
-}
-
-// Yellow string.
-func Yellow(s string) string {
-	return color.FgString(s, 252, 196, 25)
-}
-
-// Purple string.
-func Purple(s string) string {
-	return color.FgString(s, 96, 97, 190)
-}

internal/crashreport/crashreport.go

@@ -0,0 +1,47 @@
+package crashreport
+
+import (
+	"github.com/getsentry/raven-go"
+)
+
+// Disabled flag is used to globally disable crash reporting and make all the
+// crash reporting logic a no-op.
+var Disabled = false
+
+// CapturePanic is a wrapper around raven.CapturePanic that becomes a noop if
+// `Disabled` is set to true.
+func CapturePanic(f func(), tags map[string]string) (interface{}, string) {
+	if Disabled == true {
+		return nil, ""
+	}
+	return raven.CapturePanic(f, tags)
+}
+
+// CapturePanicAndWait is a wrapper around raven.CapturePanicAndWait that becomes a noop if
+// `Disabled` is set to true.
+func CapturePanicAndWait(f func(), tags map[string]string) (interface{}, string) {
+	if Disabled == true {
+		return nil, ""
+	}
+	return raven.CapturePanicAndWait(f, tags)
+}
+
+// CaptureError is a wrapper around raven.CaptureError
+func CaptureError(err error, tags map[string]string) string {
+	if Disabled == true {
+		return ""
+	}
+	return raven.CaptureError(err, tags)
+}
+
+// CaptureErrorAndWait is a wrapper around raven.CaptureErrorAndWait
+func CaptureErrorAndWait(err error, tags map[string]string) string {
+	if Disabled == true {
+		return ""
+	}
+	return raven.CaptureErrorAndWait(err, tags)
+}
+
+func init() {
+	raven.SetDSN("https://cb4510e090f64382ac371040c19b2258:8448daeebfa643c289ef398f8645980b@sentry.io/1234954")
+}

internal/database/models.go

@@ -7,16 +7,11 @@ import (
 
 	"github.com/apex/log"
 	"github.com/jmoiron/sqlx"
+	"github.com/ooni/probe-cli/nettests/summary"
 	"github.com/ooni/probe-cli/utils"
 	"github.com/pkg/errors"
 )
 
-// ResultSummaryFunc is the function used to generate result summaries
-type ResultSummaryFunc func(SummaryMap) (string, error)
-
-// SummaryMap contains a mapping from test name to serialized summary for it
-type SummaryMap map[string][]string
-
 // UpdateOne will run the specified update query and check that it only affected one row
 func UpdateOne(db *sqlx.DB, query string, arg interface{}) error {
 	res, err := db.NamedExec(query, arg)
@@ -34,6 +29,42 @@ func UpdateOne(db *sqlx.DB, query string, arg interface{}) error {
 	return nil
 }
 
+// ListMeasurements given a result ID
+func ListMeasurements(db *sqlx.DB, resultID int64) ([]*Measurement, error) {
+	measurements := []*Measurement{}
+
+	rows, err := db.Query(`SELECT id, name,
+		start_time, runtime,
+		country,
+		asn,
+		summary,
+		input
+		FROM measurements
+		WHERE result_id = ?
+		ORDER BY start_time;`, resultID)
+	if err != nil {
+		return measurements, errors.Wrap(err, "failed to get measurement list")
+	}
+
+	for rows.Next() {
+		msmt := Measurement{}
+		err = rows.Scan(&msmt.ID, &msmt.Name,
+			&msmt.StartTime, &msmt.Runtime,
+			&msmt.CountryCode,
+			&msmt.ASN,
+			&msmt.Summary, &msmt.Input,
+			//&result.DataUsageUp, &result.DataUsageDown)
+		)
+		if err != nil {
+			log.WithError(err).Error("failed to fetch a row")
+			continue
+		}
+		measurements = append(measurements, &msmt)
+	}
+
+	return measurements, nil
+}
+
 // Measurement model
 type Measurement struct {
 	ID             int64     `db:"id"`
@@ -262,8 +293,8 @@ func ListResults(db *sqlx.DB) ([]*Result, []*Result, error) {
 
 // MakeSummaryMap return a mapping of test names to summaries for the given
 // result
-func MakeSummaryMap(db *sqlx.DB, r *Result) (SummaryMap, error) {
+func MakeSummaryMap(db *sqlx.DB, r *Result) (summary.SummaryMap, error) {
-	summaryMap := SummaryMap{}
+	summaryMap := summary.SummaryMap{}
 
 	msmts := []Measurement{}
 	// XXX maybe we only want to select some of the columns
@@ -283,7 +314,7 @@ func MakeSummaryMap(db *sqlx.DB, r *Result) (SummaryMap, error) {
 }
 
 // Finished marks the result as done and sets the runtime
-func (r *Result) Finished(db *sqlx.DB, makeSummary ResultSummaryFunc) error {
+func (r *Result) Finished(db *sqlx.DB, makeSummary summary.ResultSummaryFunc) error {
 	if r.Done == true || r.Runtime != 0 {
 		return errors.New("Result is already finished")
 	}

internal/log/handlers/cli/cli.go

@@ -4,12 +4,14 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/apex/log"
 	"github.com/fatih/color"
 	colorable "github.com/mattn/go-colorable"
+	"github.com/ooni/probe-cli/internal/util"
 )
 
 // Default handler outputting to stderr.
@@ -60,16 +62,62 @@ func New(w io.Writer) *Handler {
 	}
 }
 
+func logSectionTitle(w io.Writer, f log.Fields) error {
+	colWidth := 24
+
+	title := f.Get("title").(string)
+	fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth+2)+"┓\n")
+	fmt.Fprintf(w, "┃ %s ┃\n", util.RightPad(title, colWidth))
+	fmt.Fprintf(w, "┗"+strings.Repeat("━", colWidth+2)+"┛\n")
+	return nil
+}
+
+func logTable(w io.Writer, f log.Fields) error {
+	color := color.New(color.FgBlue)
+
+	names := f.Names()
+
+	var lines []string
+	colWidth := 0
+	for _, name := range names {
+		if name == "type" {
+			continue
+		}
+		line := fmt.Sprintf("%s: %s", color.Sprint(name), f.Get(name))
+		lineLength := util.EscapeAwareRuneCountInString(line)
+		lines = append(lines, line)
+		if colWidth < lineLength {
+			colWidth = lineLength
+		}
+	}
+
+	fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth+2)+"┓\n")
+	for _, line := range lines {
+		fmt.Fprintf(w, "┃ %s ┃\n",
+			util.RightPad(line, colWidth),
+		)
+	}
+	fmt.Fprintf(w, "┗"+strings.Repeat("━", colWidth+2)+"┛\n")
+	return nil
+}
+
 // TypedLog is used for handling special "typed" logs to the CLI
 func (h *Handler) TypedLog(t string, e *log.Entry) error {
 	switch t {
 	case "progress":
-		// XXX replace this with something more fancy like https://github.com/tj/go-progress
+		var err error
-		fmt.Fprintf(h.Writer, "%.1f%% [%s]: %s", e.Fields.Get("percentage").(float64)*100, e.Fields.Get("key"), e.Message)
+		s := fmt.Sprintf("%.2f%%: %-25s", e.Fields.Get("percentage").(float64)*100, e.Message)
+		fmt.Fprintf(h.Writer, s)
 		fmt.Fprintln(h.Writer)
-		return nil
+		return err
+	case "table":
+		return logTable(h.Writer, e.Fields)
 	case "result_item":
 		return logResultItem(h.Writer, e.Fields)
+	case "result_summary":
+		return logResultSummary(h.Writer, e.Fields)
+	case "section_title":
+		return logSectionTitle(h.Writer, e.Fields)
 	default:
 		return h.DefaultLog(e)
 	}
@@ -81,15 +129,15 @@ func (h *Handler) DefaultLog(e *log.Entry) error {
 	level := Strings[e.Level]
 	names := e.Fields.Names()
 
-	color.Fprintf(h.Writer, "%s %-25s", bold.Sprintf("%*s", h.Padding+1, level), e.Message)
+	s := color.Sprintf("%s %-25s", bold.Sprintf("%*s", h.Padding+1, level), e.Message)
-
 	for _, name := range names {
 		if name == "source" {
 			continue
 		}
-		fmt.Fprintf(h.Writer, " %s=%s", color.Sprint(name), e.Fields.Get(name))
+		s += fmt.Sprintf(" %s=%s", color.Sprint(name), e.Fields.Get(name))
 	}
 
+	fmt.Fprintf(h.Writer, s)
 	fmt.Fprintln(h.Writer)
 
 	return nil

internal/log/handlers/cli/progress/progress.go

@@ -0,0 +1,126 @@
+// Package progress provides a simple terminal progress bar.
+package progress
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"io"
+	"math"
+	"strings"
+)
+
+// Bar is a progress bar.
+type Bar struct {
+	StartDelimiter string  // StartDelimiter for the bar ("|").
+	EndDelimiter   string  // EndDelimiter for the bar ("|").
+	Filled         string  // Filled section representation ("█").
+	Empty          string  // Empty section representation ("░")
+	Total          float64 // Total value.
+	Width          int     // Width of the bar.
+
+	value float64
+	tmpl  *template.Template
+	text  string
+}
+
+// New returns a new bar with the given total.
+func New(total float64) *Bar {
+	b := &Bar{
+		StartDelimiter: "|",
+		EndDelimiter:   "|",
+		Filled:         "█",
+		Empty:          "░",
+		Total:          total,
+		Width:          60,
+	}
+
+	b.Template(`{{.Percent | printf "%3.0f"}}% {{.Bar}} {{.Text}}`)
+
+	return b
+}
+
+// NewInt returns a new bar with the given total.
+func NewInt(total int) *Bar {
+	return New(float64(total))
+}
+
+// Text sets the text value.
+func (b *Bar) Text(s string) {
+	b.text = s
+}
+
+// Value sets the value.
+func (b *Bar) Value(n float64) {
+	if n > b.Total {
+		panic("Bar update value cannot be greater than the total")
+	}
+	b.value = n
+}
+
+// ValueInt sets the value.
+func (b *Bar) ValueInt(n int) {
+	b.Value(float64(n))
+}
+
+// Percent returns the percentage
+func (b *Bar) percent() float64 {
+	return (b.value / b.Total) * 100
+}
+
+// Bar returns the progress bar string.
+func (b *Bar) bar() string {
+	p := b.value / b.Total
+	filled := math.Ceil(float64(b.Width) * p)
+	empty := math.Floor(float64(b.Width) - filled)
+	s := b.StartDelimiter
+	s += strings.Repeat(b.Filled, int(filled))
+	s += strings.Repeat(b.Empty, int(empty))
+	s += b.EndDelimiter
+	return s
+}
+
+// String returns the progress bar.
+func (b *Bar) String() string {
+	var buf bytes.Buffer
+
+	data := struct {
+		Value          float64
+		Total          float64
+		Percent        float64
+		StartDelimiter string
+		EndDelimiter   string
+		Bar            string
+		Text           string
+	}{
+		Value:          b.value,
+		Text:           b.text,
+		StartDelimiter: b.StartDelimiter,
+		EndDelimiter:   b.EndDelimiter,
+		Percent:        b.percent(),
+		Bar:            b.bar(),
+	}
+
+	if err := b.tmpl.Execute(&buf, data); err != nil {
+		panic(err)
+	}
+
+	return buf.String()
+}
+
+// WriteTo writes the progress bar to w.
+func (b *Bar) WriteTo(w io.Writer) (int64, error) {
+	s := fmt.Sprintf("\r   %s ", b.String())
+	_, err := io.WriteString(w, s)
+	return int64(len(s)), err
+}
+
+// Template for rendering. This method will panic if the template fails to parse.
+func (b *Bar) Template(s string) {
+	t, err := template.New("").Parse(s)
+	if err != nil {
+		panic(err)
+	}
+
+	b.tmpl = t
+}

internal/log/handlers/cli/results.go

@@ -8,38 +8,10 @@ import (
 	"time"
 
 	"github.com/apex/log"
+	"github.com/ooni/probe-cli/internal/util"
+	"github.com/ooni/probe-cli/nettests/summary"
 )
 
-func RightPad(str string, length int) string {
-	return str + strings.Repeat(" ", length-len(str))
-}
-
-// XXX Copy-pasta from nettest/groups
-// PerformanceSummary is the result summary for a performance test
-type PerformanceSummary struct {
-	Upload   int64
-	Download int64
-	Ping     float64
-	Bitrate  int64
-}
-
-// MiddleboxSummary is the summary for the middlebox tests
-type MiddleboxSummary struct {
-	Detected bool
-}
-
-// IMSummary is the summary for the im tests
-type IMSummary struct {
-	Tested  uint
-	Blocked uint
-}
-
-// WebsitesSummary is the summary for the websites test
-type WebsitesSummary struct {
-	Tested  uint
-	Blocked uint
-}
-
 func formatSpeed(speed int64) string {
 	if speed < 1000 {
 		return fmt.Sprintf("%d Kbit/s", speed)
@@ -54,7 +26,7 @@ func formatSpeed(speed int64) string {
 
 var summarizers = map[string]func(string) []string{
 	"websites": func(ss string) []string {
-		var summary WebsitesSummary
+		var summary summary.WebsitesSummary
 		if err := json.Unmarshal([]byte(ss), &summary); err != nil {
 			return nil
 		}
@@ -65,7 +37,7 @@ var summarizers = map[string]func(string) []string{
 		}
 	},
 	"performance": func(ss string) []string {
-		var summary PerformanceSummary
+		var summary summary.PerformanceSummary
 		if err := json.Unmarshal([]byte(ss), &summary); err != nil {
 			return nil
 		}
@@ -76,7 +48,7 @@ var summarizers = map[string]func(string) []string{
 		}
 	},
 	"im": func(ss string) []string {
-		var summary IMSummary
+		var summary summary.IMSummary
 		if err := json.Unmarshal([]byte(ss), &summary); err != nil {
 			return nil
 		}
@@ -87,7 +59,7 @@ var summarizers = map[string]func(string) []string{
 		}
 	},
 	"middlebox": func(ss string) []string {
-		var summary MiddleboxSummary
+		var summary summary.MiddleboxSummary
 		if err := json.Unmarshal([]byte(ss), &summary); err != nil {
 			return nil
 		}
@@ -122,21 +94,21 @@ func logResultItem(w io.Writer, f log.Fields) error {
 		fmt.Fprintf(w, "┢"+strings.Repeat("━", colWidth*2+2)+"┪\n")
 	}
 
-	firstRow := RightPad(fmt.Sprintf("#%d - %s", rID, startTime.Format(time.RFC822)), colWidth*2)
+	firstRow := util.RightPad(fmt.Sprintf("#%d - %s", rID, startTime.Format(time.RFC822)), colWidth*2)
 	fmt.Fprintf(w, "┃ "+firstRow+" ┃\n")
 	fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n")
 
 	summary := makeSummary(name, f.Get("summary").(string))
 
 	fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
-		RightPad(name, colWidth),
+		util.RightPad(name, colWidth),
-		RightPad(summary[0], colWidth)))
+		util.RightPad(summary[0], colWidth)))
 	fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
-		RightPad(networkName, colWidth),
+		util.RightPad(networkName, colWidth),
-		RightPad(summary[1], colWidth)))
+		util.RightPad(summary[1], colWidth)))
 	fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
-		RightPad(asn, colWidth),
+		util.RightPad(asn, colWidth),
-		RightPad(summary[2], colWidth)))
+		util.RightPad(summary[2], colWidth)))
 
 	if index == totalCount-1 {
 		fmt.Fprintf(w, "└┬──────────────┬──────────────┬──────────────┬")
@@ -145,3 +117,25 @@ func logResultItem(w io.Writer, f log.Fields) error {
 	}
 	return nil
 }
+
+func logResultSummary(w io.Writer, f log.Fields) error {
+
+	networks := f.Get("total_networks").(int64)
+	tests := f.Get("total_tests").(int64)
+	dataUp := f.Get("total_data_usage_up").(int64)
+	dataDown := f.Get("total_data_usage_down").(int64)
+	if tests == 0 {
+		fmt.Fprintf(w, "No results\n")
+		fmt.Fprintf(w, "Try running:\n")
+		fmt.Fprintf(w, "  ooni run websites\n")
+		return nil
+	}
+	//              └┬──────────────┬──────────────┬──────────────┬
+	fmt.Fprintf(w, " │ %s │ %s │ %s │\n",
+		util.RightPad(fmt.Sprintf("%d tests", tests), 12),
+		util.RightPad(fmt.Sprintf("%d nets", networks), 12),
+		util.RightPad(fmt.Sprintf("%d ⬆ %d ⬇", dataUp, dataDown), 12))
+	fmt.Fprintf(w, " └──────────────┴──────────────┴──────────────┘\n")
+
+	return nil
+}

internal/onboard/onboard.go

@@ -0,0 +1,141 @@
+package onboard
+
+import (
+	"fmt"
+
+	"github.com/apex/log"
+	"github.com/fatih/color"
+	"github.com/ooni/probe-cli/config"
+	"github.com/ooni/probe-cli/internal/output"
+	survey "gopkg.in/AlecAivazis/survey.v1"
+)
+
+func Onboarding(config *config.Config) error {
+	output.SectionTitle("What is OONI Probe?")
+
+	fmt.Println()
+	output.Paragraph("Your tool for detecting internet censorship!")
+	fmt.Println()
+	output.Paragraph("OONI Probe checks whether your provider blocks access to sites and services. Run OONI Probe to collect evidence of internet censorship and to measure your network performance.")
+	fmt.Println()
+	output.PressEnterToContinue("Press 'Enter' to continue...")
+
+	output.SectionTitle("Heads Up")
+	fmt.Println()
+	output.Bullet("Anyone monitoring your internet activity (such as your government or ISP) may be able to see that you are running OONI Probe.")
+	fmt.Println()
+	output.Bullet("The network data you will collect will automatically be published (unless you opt-out in the settings).")
+	fmt.Println()
+	output.Bullet("You may test objectionable sites.")
+	fmt.Println()
+	output.Bullet("Read the documentation to learn more.")
+	fmt.Println()
+	output.PressEnterToContinue("Press 'Enter' to continue...")
+
+	output.SectionTitle("Pop Quiz!")
+	output.Paragraph("")
+	answer := ""
+	quiz1 := &survey.Select{
+		Message: "Anyone monitoring my internet activity may be able to see that I am running OONI Probe.",
+		Options: []string{"true", "false"},
+		Default: "true",
+	}
+	survey.AskOne(quiz1, &answer, nil)
+	if answer != "true" {
+		output.Paragraph(color.RedString("Actually..."))
+		output.Paragraph("OONI Probe is not a privacy tool. Therefore, anyone monitoring your internet activity may be able to see which software you are running.")
+	} else {
+		output.Paragraph(color.BlueString("Good job!"))
+	}
+	answer = ""
+	quiz2 := &survey.Select{
+		Message: "The network data I will collect will automatically be published (unless I opt-out in the settings).",
+		Options: []string{"true", "false"},
+		Default: "true",
+	}
+	survey.AskOne(quiz2, &answer, nil)
+	if answer != "true" {
+		output.Paragraph(color.RedString("Actually..."))
+		output.Paragraph("The network data you will collect will automatically be published to increase transparency of internet censorship (unless you opt-out in the settings).")
+	} else {
+		output.Paragraph(color.BlueString("Well done!"))
+	}
+
+	changeDefaults := false
+	prompt := &survey.Confirm{
+		Message: "Do you want to change the default settings?",
+		Default: false,
+	}
+	survey.AskOne(prompt, &changeDefaults, nil)
+
+	settings := struct {
+		IncludeIP        bool
+		IncludeNetwork   bool
+		IncludeCountry   bool
+		UploadResults    bool
+		SendCrashReports bool
+	}{}
+	settings.IncludeIP = false
+	settings.IncludeNetwork = true
+	settings.IncludeCountry = true
+	settings.UploadResults = true
+	settings.SendCrashReports = true
+
+	if changeDefaults == true {
+		var qs = []*survey.Question{
+			{
+				Name:   "IncludeIP",
+				Prompt: &survey.Confirm{Message: "Should we include your IP?"},
+			},
+			{
+				Name: "IncludeNetwork",
+				Prompt: &survey.Confirm{
+					Message: "Can we include your network name?",
+					Default: true,
+				},
+			},
+			{
+				Name: "IncludeCountry",
+				Prompt: &survey.Confirm{
+					Message: "Can we include your country name?",
+					Default: true,
+				},
+			},
+			{
+				Name: "UploadResults",
+				Prompt: &survey.Confirm{
+					Message: "Can we upload your results?",
+					Default: true,
+				},
+			},
+			{
+				Name: "SendCrashReports",
+				Prompt: &survey.Confirm{
+					Message: "Can we send crash reports to OONI?",
+					Default: true,
+				},
+			},
+		}
+
+		err := survey.Ask(qs, &settings)
+		if err != nil {
+			log.WithError(err).Error("there was an error in parsing your responses")
+			return err
+		}
+	}
+
+	config.Lock()
+	config.InformedConsent = true
+	config.Advanced.IncludeCountry = settings.IncludeCountry
+	config.Advanced.SendCrashReports = settings.SendCrashReports
+	config.Sharing.IncludeIP = settings.IncludeIP
+	config.Sharing.IncludeASN = settings.IncludeNetwork
+	config.Sharing.UploadResults = settings.UploadResults
+	config.Unlock()
+
+	if err := config.Write(); err != nil {
+		log.WithError(err).Error("failed to write config file")
+		return err
+	}
+	return nil
+}

internal/output/output.go

@@ -1,9 +1,13 @@
 package output
 
 import (
+	"bufio"
+	"fmt"
+	"os"
 	"time"
 
 	"github.com/apex/log"
+	"github.com/ooni/probe-cli/internal/util"
 )
 
 // Progress logs a progress type event
@@ -51,3 +55,43 @@ func ResultItem(result ResultItemData) {
 		"total_count":     result.TotalCount,
 	}).Info("result item")
 }
+
+type ResultSummaryData struct {
+	TotalTests         int64
+	TotalDataUsageUp   int64
+	TotalDataUsageDown int64
+	TotalNetworks      int64
+}
+
+func ResultSummary(result ResultSummaryData) {
+	log.WithFields(log.Fields{
+		"type":                  "result_summary",
+		"total_tests":           result.TotalTests,
+		"total_data_usage_up":   result.TotalDataUsageUp,
+		"total_data_usage_down": result.TotalDataUsageDown,
+		"total_networks":        result.TotalNetworks,
+	}).Info("result summary")
+}
+
+// SectionTitle is the title of a section
+func SectionTitle(text string) {
+	log.WithFields(log.Fields{
+		"type":  "section_title",
+		"title": text,
+	}).Info(text)
+}
+
+func Paragraph(text string) {
+	const width = 80
+	fmt.Println(util.WrapString(text, width))
+}
+
+func Bullet(text string) {
+	const width = 80
+	fmt.Printf("• %s\n", util.WrapString(text, width))
+}
+
+func PressEnterToContinue(text string) {
+	fmt.Print(text)
+	bufio.NewReader(os.Stdin).ReadBytes('\n')
+}

internal/util/util.go

@@ -1,19 +1,110 @@
 package util
 
 import (
+	"bytes"
 	"fmt"
 	"os"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
 
-	"github.com/ooni/probe-cli/internal/colors"
+	"github.com/fatih/color"
 )
 
 // Log outputs a log message.
 func Log(msg string, v ...interface{}) {
-	fmt.Printf("     %s\n", colors.Purple(fmt.Sprintf(msg, v...)))
+	fmt.Printf("     %s\n", color.CyanString(msg, v...))
 }
 
 // Fatal error
 func Fatal(err error) {
-	fmt.Fprintf(os.Stderr, "\n     %s %s\n\n", colors.Red("Error:"), err)
+	fmt.Fprintf(os.Stderr, "\n     %s %s\n\n", color.RedString("Error:"), err)
 	os.Exit(1)
 }
+
+// Finds the ansi escape sequences (like colors)
+// Taken from: https://github.com/chalk/ansi-regex/blob/d9d806ecb45d899cf43408906a4440060c5c50e5/index.js
+var ansiEscapes = regexp.MustCompile(`[\x1B\x9B][[\]()#;?]*` +
+	`(?:(?:(?:[a-zA-Z\d]*(?:;[a-zA-Z\\d]*)*)?\x07)` +
+	`|(?:(?:\d{1,4}(?:;\d{0,4})*)?[\dA-PRZcf-ntqry=><~]))`)
+
+func EscapeAwareRuneCountInString(s string) int {
+	n := utf8.RuneCountInString(s)
+	for _, sm := range ansiEscapes.FindAllString(s, -1) {
+		n -= utf8.RuneCountInString(sm)
+	}
+	return n
+}
+
+func RightPad(str string, length int) string {
+	return str + strings.Repeat(" ", length-EscapeAwareRuneCountInString(str))
+}
+
+// WrapString wraps the given string within lim width in characters.
+//
+// Wrapping is currently naive and only happens at white-space. A future
+// version of the library will implement smarter wrapping. This means that
+// pathological cases can dramatically reach past the limit, such as a very
+// long word.
+// This is taken from: https://github.com/mitchellh/go-wordwrap/tree/f253961a26562056904822f2a52d4692347db1bd
+func WrapString(s string, lim uint) string {
+	// Initialize a buffer with a slightly larger size to account for breaks
+	init := make([]byte, 0, len(s))
+	buf := bytes.NewBuffer(init)
+
+	var current uint
+	var wordBuf, spaceBuf bytes.Buffer
+
+	for _, char := range s {
+		if char == '\n' {
+			if wordBuf.Len() == 0 {
+				if current+uint(spaceBuf.Len()) > lim {
+					current = 0
+				} else {
+					current += uint(spaceBuf.Len())
+					spaceBuf.WriteTo(buf)
+				}
+				spaceBuf.Reset()
+			} else {
+				current += uint(spaceBuf.Len() + wordBuf.Len())
+				spaceBuf.WriteTo(buf)
+				spaceBuf.Reset()
+				wordBuf.WriteTo(buf)
+				wordBuf.Reset()
+			}
+			buf.WriteRune(char)
+			current = 0
+		} else if unicode.IsSpace(char) {
+			if spaceBuf.Len() == 0 || wordBuf.Len() > 0 {
+				current += uint(spaceBuf.Len() + wordBuf.Len())
+				spaceBuf.WriteTo(buf)
+				spaceBuf.Reset()
+				wordBuf.WriteTo(buf)
+				wordBuf.Reset()
+			}
+
+			spaceBuf.WriteRune(char)
+		} else {
+
+			wordBuf.WriteRune(char)
+
+			if current+uint(spaceBuf.Len()+wordBuf.Len()) > lim && uint(wordBuf.Len()) < lim {
+				buf.WriteRune('\n')
+				current = 0
+				spaceBuf.Reset()
+			}
+		}
+	}
+
+	if wordBuf.Len() == 0 {
+		if current+uint(spaceBuf.Len()) <= lim {
+			spaceBuf.WriteTo(buf)
+		}
+	} else {
+		spaceBuf.WriteTo(buf)
+		wordBuf.WriteTo(buf)
+	}
+
+	return buf.String()
+}

internal/util/util_test.go

@@ -0,0 +1,18 @@
+package util
+
+import (
+	"testing"
+
+	"github.com/fatih/color"
+)
+
+func TestEscapeAwareRuneCountInString(t *testing.T) {
+	var bold = color.New(color.Bold)
+	var myColor = color.New(color.FgBlue)
+
+	s := myColor.Sprintf("•ABC%s%s", bold.Sprintf("DEF"), "\x1B[00;38;5;244m\x1B[m\x1B[00;38;5;33mGHI\x1B[0m")
+	count := EscapeAwareRuneCountInString(s)
+	if count != 10 {
+		t.Errorf("Count was incorrect, got: %d, want: %d.", count, 10)
+	}
+}

nettests/groups/groups.go

@@ -2,14 +2,13 @@ package groups
 
 import (
 	"encoding/json"
-	"fmt"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/internal/database"
 	"github.com/ooni/probe-cli/nettests"
 	"github.com/ooni/probe-cli/nettests/im"
 	"github.com/ooni/probe-cli/nettests/middlebox"
 	"github.com/ooni/probe-cli/nettests/performance"
+	"github.com/ooni/probe-cli/nettests/summary"
 	"github.com/ooni/probe-cli/nettests/websites"
 )
 
@@ -17,42 +16,7 @@ import (
 type NettestGroup struct {
 	Label    string
 	Nettests []nettests.Nettest
-	Summary  database.ResultSummaryFunc
+	Summary  summary.ResultSummaryFunc
-}
-
-// PerformanceSummary is the result summary for a performance test
-type PerformanceSummary struct {
-	Upload   int64
-	Download int64
-	Ping     float64
-	Bitrate  int64
-}
-
-// MiddleboxSummary is the summary for the middlebox tests
-type MiddleboxSummary struct {
-	Detected bool
-}
-
-// IMSummary is the summary for the im tests
-type IMSummary struct {
-	Tested  uint
-	Blocked uint
-}
-
-// WebsitesSummary is the summary for the websites test
-type WebsitesSummary struct {
-	Tested  uint
-	Blocked uint
-}
-
-func checkRequiredKeys(rk []string, m database.SummaryMap) error {
-	for _, key := range rk {
-		if _, ok := m[key]; ok {
-			continue
-		}
-		return fmt.Errorf("missing SummaryMap key '%s'", key)
-	}
-	return nil
 }
 
 // NettestGroups that can be run by the user
@@ -62,14 +26,14 @@ var NettestGroups = map[string]NettestGroup{
 		Nettests: []nettests.Nettest{
 			websites.WebConnectivity{},
 		},
-		Summary: func(m database.SummaryMap) (string, error) {
+		Summary: func(m summary.SummaryMap) (string, error) {
-			if err := checkRequiredKeys([]string{"WebConnectivity"}, m); err != nil {
+			if err := summary.CheckRequiredKeys([]string{"WebConnectivity"}, m); err != nil {
 				log.WithError(err).Error("missing keys")
 				return "", err
 			}
 
 			// XXX to generate this I need to create the summary map as a list
-			var summary WebsitesSummary
+			var summary summary.WebsitesSummary
 			summary.Tested = 0
 			summary.Blocked = 0
 			for _, msmtSummaryStr := range m["WebConnectivity"] {
@@ -98,8 +62,8 @@ var NettestGroups = map[string]NettestGroup{
 			performance.Dash{},
 			performance.NDT{},
 		},
-		Summary: func(m database.SummaryMap) (string, error) {
+		Summary: func(m summary.SummaryMap) (string, error) {
-			if err := checkRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil {
+			if err := summary.CheckRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil {
 				log.WithError(err).Error("missing keys")
 				return "", err
 			}
@@ -108,7 +72,7 @@ var NettestGroups = map[string]NettestGroup{
 				err         error
 				ndtSummary  performance.NDTSummary
 				dashSummary performance.DashSummary
-				summary     PerformanceSummary
+				summary     summary.PerformanceSummary
 			)
 			err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary)
 			if err != nil {
@@ -137,8 +101,8 @@ var NettestGroups = map[string]NettestGroup{
 			middlebox.HTTPInvalidRequestLine{},
 			middlebox.HTTPHeaderFieldManipulation{},
 		},
-		Summary: func(m database.SummaryMap) (string, error) {
+		Summary: func(m summary.SummaryMap) (string, error) {
-			if err := checkRequiredKeys([]string{"WebConnectivity"}, m); err != nil {
+			if err := summary.CheckRequiredKeys([]string{"HttpInvalidRequestLine", "HttpHeaderFieldManipulation"}, m); err != nil {
 				log.WithError(err).Error("missing keys")
 				return "", err
 			}
@@ -147,7 +111,7 @@ var NettestGroups = map[string]NettestGroup{
 				err         error
 				hhfmSummary middlebox.HTTPHeaderFieldManipulationSummary
 				hirlSummary middlebox.HTTPInvalidRequestLineSummary
-				summary     MiddleboxSummary
+				summary     summary.MiddleboxSummary
 			)
 			err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary)
 			if err != nil {
@@ -174,8 +138,8 @@ var NettestGroups = map[string]NettestGroup{
 			im.Telegram{},
 			im.WhatsApp{},
 		},
-		Summary: func(m database.SummaryMap) (string, error) {
+		Summary: func(m summary.SummaryMap) (string, error) {
-			if err := checkRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil {
+			if err := summary.CheckRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil {
 				log.WithError(err).Error("missing keys")
 				return "", err
 			}
@@ -184,7 +148,7 @@ var NettestGroups = map[string]NettestGroup{
 				waSummary im.WhatsAppSummary
 				tgSummary im.TelegramSummary
 				fbSummary im.FacebookMessengerSummary
-				summary   IMSummary
+				summary   summary.IMSummary
 			)
 			err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary)
 			if err != nil {

nettests/nettests.go

@@ -7,11 +7,9 @@ import (
 	"path/filepath"
 
 	"github.com/apex/log"
+	"github.com/fatih/color"
 	"github.com/measurement-kit/go-measurement-kit"
-	homedir "github.com/mitchellh/go-homedir"
 	ooni "github.com/ooni/probe-cli"
-	"github.com/ooni/probe-cli/internal/cli/version"
-	"github.com/ooni/probe-cli/internal/colors"
 	"github.com/ooni/probe-cli/internal/database"
 	"github.com/ooni/probe-cli/internal/output"
 	"github.com/ooni/probe-cli/utils"
@@ -76,11 +74,13 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 	caBundlePath := getCaBundlePath()
 	msmtPath := c.msmtPath
 
-	userHome, err := homedir.Dir()
+	userHome, err := utils.GetOONIHome()
 	if err != nil {
 		log.WithError(err).Error("failed to figure out the homedir")
 		return err
 	}
+	// Get the parent of it
+	userHome = filepath.Dir(userHome)
 
 	relPath, err := filepath.Rel(userHome, caBundlePath)
 	if err != nil {
@@ -121,7 +121,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 		DisableReportFile: false,
 		DisableCollector:  false,
 		SoftwareName:      "ooniprobe",
-		SoftwareVersion:   version.Version,
+		SoftwareVersion:   ooni.Version,
 
 		OutputPath:       msmtPath,
 		GeoIPCountryPath: geoIPCountryPath,
@@ -132,6 +132,8 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 	log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath)
 
 	nt.On("log", func(e mk.Event) {
+		log.Debugf(color.RedString(e.Key))
+
 		level := e.Value.LogLevel
 		msg := e.Value.Message
 
@@ -161,7 +163,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 	})
 
 	nt.On("status.geoip_lookup", func(e mk.Event) {
-		log.Debugf(colors.Red(e.Key))
+		log.Debugf(color.RedString(e.Key))
 
 		msmtTemplate.ASN = e.Value.ProbeASN
 		msmtTemplate.IP = e.Value.ProbeIP
@@ -169,7 +171,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 	})
 
 	nt.On("status.measurement_start", func(e mk.Event) {
-		log.Debugf(colors.Red(e.Key))
+		log.Debugf(color.RedString(e.Key))
 
 		idx := e.Value.Idx
 		msmt, err := database.CreateMeasurement(c.Ctx.DB, msmtTemplate, e.Value.Input)
@@ -181,29 +183,64 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 	})
 
 	nt.On("status.progress", func(e mk.Event) {
-		log.Debugf(colors.Red(e.Key))
+		log.Debugf(color.RedString(e.Key))
 		c.OnProgress(e.Value.Percentage, e.Value.Message)
 	})
 
 	nt.On("status.update.*", func(e mk.Event) {
-		log.Debugf(colors.Red(e.Key))
+		log.Debugf(color.RedString(e.Key))
+	})
+
+	// XXX should these be made into permanent failures?
+	nt.On("failure.asn_lookup", func(e mk.Event) {
+		log.Debugf(color.RedString(e.Key))
+		log.Debugf("%v", e.Value)
+	})
+	nt.On("failure.cc_lookup", func(e mk.Event) {
+		log.Debugf(color.RedString(e.Key))
+		log.Debugf("%v", e.Value)
+	})
+	nt.On("failure.ip_lookup", func(e mk.Event) {
+		log.Debugf(color.RedString(e.Key))
+		log.Debugf("%v", e.Value)
+	})
+
+	nt.On("failure.resolver_lookup", func(e mk.Event) {
+		log.Debugf(color.RedString(e.Key))
+		log.Debugf("%v", e.Value)
+	})
+
+	nt.On("failure.report_create", func(e mk.Event) {
+		log.Debugf(color.RedString(e.Key))
+		log.Debugf("%v", e.Value)
+	})
+
+	nt.On("failure.report_close", func(e mk.Event) {
+		log.Debugf(color.RedString(e.Key))
+		log.Debugf("%v", e.Value)
+	})
+
+	nt.On("failure.startup", func(e mk.Event) {
+		log.Debugf(color.RedString(e.Key))
+
+		c.msmts[e.Value.Idx].Failed(c.Ctx.DB, e.Value.Failure)
 	})
 
 	nt.On("failure.measurement", func(e mk.Event) {
-		log.Debugf(colors.Red(e.Key))
+		log.Debugf(color.RedString(e.Key))
 
 		c.msmts[e.Value.Idx].Failed(c.Ctx.DB, e.Value.Failure)
 	})
 
 	nt.On("failure.measurement_submission", func(e mk.Event) {
-		log.Debugf(colors.Red(e.Key))
+		log.Debugf(color.RedString(e.Key))
 
 		failure := e.Value.Failure
 		c.msmts[e.Value.Idx].UploadFailed(c.Ctx.DB, failure)
 	})
 
 	nt.On("status.measurement_submission", func(e mk.Event) {
-		log.Debugf(colors.Red(e.Key))
+		log.Debugf(color.RedString(e.Key))
 
 		if err := c.msmts[e.Value.Idx].UploadSucceeded(c.Ctx.DB); err != nil {
 			log.WithError(err).Error("failed to mark msmt as uploaded")
@@ -211,7 +248,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 	})
 
 	nt.On("status.measurement_done", func(e mk.Event) {
-		log.Debugf(colors.Red(e.Key))
+		log.Debugf(color.RedString(e.Key))
 
 		if err := c.msmts[e.Value.Idx].Done(c.Ctx.DB); err != nil {
 			log.WithError(err).Error("failed to mark msmt as done")
@@ -219,6 +256,8 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 	})
 
 	nt.On("measurement", func(e mk.Event) {
+		log.Debugf("status.end")
+
 		c.OnEntry(e.Value.Idx, e.Value.JSONStr)
 	})
 

nettests/summary/summary.go

@@ -0,0 +1,44 @@
+package summary
+
+import "fmt"
+
+// ResultSummaryFunc is the function used to generate result summaries
+type ResultSummaryFunc func(SummaryMap) (string, error)
+
+// SummaryMap contains a mapping from test name to serialized summary for it
+type SummaryMap map[string][]string
+
+// PerformanceSummary is the result summary for a performance test
+type PerformanceSummary struct {
+	Upload   int64
+	Download int64
+	Ping     float64
+	Bitrate  int64
+}
+
+// MiddleboxSummary is the summary for the middlebox tests
+type MiddleboxSummary struct {
+	Detected bool
+}
+
+// IMSummary is the summary for the im tests
+type IMSummary struct {
+	Tested  uint
+	Blocked uint
+}
+
+// WebsitesSummary is the summary for the websites test
+type WebsitesSummary struct {
+	Tested  uint
+	Blocked uint
+}
+
+func CheckRequiredKeys(rk []string, m SummaryMap) error {
+	for _, key := range rk {
+		if _, ok := m[key]; ok {
+			continue
+		}
+		return fmt.Errorf("missing SummaryMap key '%s'", key)
+	}
+	return nil
+}

nettests/websites/web_connectivity.go

@@ -29,6 +29,7 @@ func lookupURLs(ctl *nettests.Controller) ([]string, error) {
 		parsed = new(URLResponse)
 		urls   []string
 	)
+	// XXX pass in the configuration for category codes
 	reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
 		orchestrateBaseURL,
 		ctl.Ctx.Location.CountryCode)

ooni.go

@@ -1,41 +1,26 @@
 package ooni
 
 import (
-	"encoding/json"
 	"io/ioutil"
 	"os"
-	"path/filepath"
+	"path"
-	"sync"
 
 	"github.com/apex/log"
 	"github.com/jmoiron/sqlx"
 	"github.com/ooni/probe-cli/config"
+	"github.com/ooni/probe-cli/internal/bindata"
 	"github.com/ooni/probe-cli/internal/database"
 	"github.com/ooni/probe-cli/internal/legacy"
+	"github.com/ooni/probe-cli/internal/onboard"
 	"github.com/ooni/probe-cli/utils"
 	"github.com/pkg/errors"
 )
 
-// Onboarding process
+const Version = "3.0.0-dev.0"
-func Onboarding(c *Config) error {
-	log.Info("Onboarding starting")
-
-	// To prevent races we always must acquire the config file lock before
-	// changing it.
-	c.Lock()
-	c.InformedConsent = true
-	c.Unlock()
-
-	if err := c.Write(); err != nil {
-		log.Warnf("Failed to save informed consent: %v", err)
-		return err
-	}
-	return nil
-}
 
 // Context for OONI Probe
 type Context struct {
-	Config   *Config
+	Config   *config.Config
 	DB       *sqlx.DB
 	Location *utils.LocationInfo
 
@@ -58,8 +43,11 @@ func (c *Context) MaybeLocationLookup() error {
 func (c *Context) LocationLookup() error {
 	var err error
 
-	geoipDir := utils.GeoIPDir(c.Home)
+	if err = c.MaybeDownloadDataFiles(); err != nil {
+		log.WithError(err).Error("failed to download data files")
+	}
 
+	geoipDir := utils.GeoIPDir(c.Home)
 	c.Location, err = utils.GeoIPLookup(geoipDir)
 	if err != nil {
 		return err
@@ -68,6 +56,35 @@ func (c *Context) LocationLookup() error {
 	return nil
 }
 
+// MaybeOnboarding will run the onboarding process only if the informed consent
+// config option is set to false
+func (c *Context) MaybeOnboarding() error {
+	if c.Config.InformedConsent == false {
+		if err := onboard.Onboarding(c.Config); err != nil {
+			return errors.Wrap(err, "onboarding")
+		}
+	}
+	return nil
+}
+
+// MaybeDownloadDataFiles will download geoip data files if they are not present
+func (c *Context) MaybeDownloadDataFiles() error {
+	geoipDir := utils.GeoIPDir(c.Home)
+	if _, err := os.Stat(path.Join(geoipDir, "GeoLite2-Country.mmdb")); os.IsNotExist(err) {
+		log.Debugf("Downloading GeoIP database files")
+		if err := utils.DownloadGeoIPDatabaseFiles(geoipDir); err != nil {
+			return err
+		}
+	}
+	if _, err := os.Stat(path.Join(geoipDir, "GeoIP.dat")); os.IsNotExist(err) {
+		log.Debugf("Downloading legacy GeoIP database Files")
+		if err := utils.DownloadLegacyGeoIPDatabaseFiles(geoipDir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Init the OONI manager
 func (c *Context) Init() error {
 	var err error
@@ -82,22 +99,16 @@ func (c *Context) Init() error {
 
 	if c.configPath != "" {
 		log.Debugf("Reading config file from %s", c.configPath)
-		c.Config, err = ReadConfig(c.configPath)
+		c.Config, err = config.ReadConfig(c.configPath)
 	} else {
 		log.Debug("Reading default config file")
-		c.Config, err = ReadDefaultConfigPaths(c.Home)
+		c.Config, err = InitDefaultConfig(c.Home)
 	}
 	if err != nil {
 		return err
 	}
 
 	c.dbPath = utils.DBDir(c.Home, "main")
-	if c.Config.InformedConsent == false {
-		if err = Onboarding(c.Config); err != nil {
-			return errors.Wrap(err, "onboarding")
-		}
-	}
-
 	log.Debugf("Connecting to database sqlite3://%s", c.dbPath)
 	db, err := database.Connect(c.dbPath)
 	if err != nil {
@@ -118,158 +129,53 @@ func (c *Context) Init() error {
 func NewContext(configPath string, homePath string) *Context {
 	return &Context{
 		Home:       homePath,
-		Config:     &Config{},
+		Config:     &config.Config{},
 		configPath: configPath,
 	}
 }
 
-// Config for the OONI Probe installation
-type Config struct {
-	// Private settings
-	Comment         string `json:"_"`
-	ConfigVersion   string `json:"_config_version"`
-	InformedConsent bool   `json:"_informed_consent"`
-
-	AutoUpdate       bool                    `json:"auto_update"`
-	Sharing          config.Sharing          `json:"sharing"`
-	Notifications    config.Notifications    `json:"notifications"`
-	AutomatedTesting config.AutomatedTesting `json:"automated_testing"`
-	NettestGroups    config.NettestGroups    `json:"test_settings"`
-	Advanced         config.Advanced         `json:"advanced"`
-
-	mutex sync.Mutex
-	path  string
-}
-
-// Write the config file in json to the path
-func (c *Config) Write() error {
-	c.Lock()
-	configJSON, _ := json.MarshalIndent(c, "", "  ")
-	if c.path == "" {
-		return errors.New("config file path is empty")
-	}
-	if err := ioutil.WriteFile(c.path, configJSON, 0644); err != nil {
-		return errors.Wrap(err, "writing config JSON")
-	}
-	c.Unlock()
-	return nil
-}
-
-// Lock acquires the write mutex
-func (c *Config) Lock() {
-	c.mutex.Lock()
-}
-
-// Unlock releases the write mutex
-func (c *Config) Unlock() {
-	c.mutex.Unlock()
-}
-
-// Default config settings
-func (c *Config) Default() error {
-	home, err := utils.GetOONIHome()
-	if err != nil {
-		return err
-	}
-
-	c.path = filepath.Join(home, "config.json")
-	return nil
-}
-
-// Validate the config file
-func (c *Config) Validate() error {
-	return nil
-}
-
-// ParseConfig returns config from JSON bytes.
-func ParseConfig(b []byte) (*Config, error) {
-	c := &Config{}
-
-	if err := json.Unmarshal(b, c); err != nil {
-		return nil, errors.Wrap(err, "parsing json")
-	}
-
-	if err := c.Default(); err != nil {
-		return nil, errors.Wrap(err, "defaulting")
-	}
-
-	if err := c.Validate(); err != nil {
-		return nil, errors.Wrap(err, "validating")
-	}
-
-	return c, nil
-}
-
 // MaybeInitializeHome does the setup for a new OONI Home
 func MaybeInitializeHome(home string) error {
-	firstRun := false
 	for _, d := range utils.RequiredDirs(home) {
 		if _, e := os.Stat(d); e != nil {
-			firstRun = true
 			if err := os.MkdirAll(d, 0700); err != nil {
 				return err
 			}
 		}
 	}
-	if firstRun == true {
-		log.Info("This is the first time you are running OONI Probe. Downloading some files.")
-		geoipDir := utils.GeoIPDir(home)
-		if err := utils.DownloadGeoIPDatabaseFiles(geoipDir); err != nil {
-			return err
-		}
-		if err := utils.DownloadLegacyGeoIPDatabaseFiles(geoipDir); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
-// ReadConfig reads the configuration from the path
+// InitDefaultConfig reads the config from common locations or creates it if
-func ReadConfig(path string) (*Config, error) {
+// missing.
-	b, err := ioutil.ReadFile(path)
+func InitDefaultConfig(home string) (*config.Config, error) {
-
+	var (
-	if os.IsNotExist(err) {
+		err        error
-		c := &Config{}
+		c          *config.Config
-
+		configPath = utils.ConfigPath(home)
-		if err = c.Default(); err != nil {
+	)
-			return nil, errors.Wrap(err, "defaulting")
-		}
-
-		if err = c.Validate(); err != nil {
-			return nil, errors.Wrap(err, "validating")
-		}
-
-		return c, nil
-	}
 
+	c, err = config.ReadConfig(configPath)
 	if err != nil {
-		return nil, errors.Wrap(err, "reading file")
+		if os.IsNotExist(err) {
-	}
+			log.Debugf("writing default config to %s", configPath)
-
+			var data []byte
-	c, err := ParseConfig(b)
+			data, err = bindata.Asset("data/default-config.json")
-	if err != nil {
-		return nil, errors.Wrap(err, "parsing config")
-	}
-	c.path = path
-	return c, err
-}
-
-// ReadDefaultConfigPaths from common locations.
-func ReadDefaultConfigPaths(home string) (*Config, error) {
-	var paths = []string{
-		filepath.Join(home, "config.json"),
-	}
-	for _, path := range paths {
-		if _, err := os.Stat(path); err == nil {
-			c, err := ReadConfig(path)
 			if err != nil {
 				return nil, err
 			}
-			return c, nil
+			err = ioutil.WriteFile(
+				configPath,
+				data,
+				0644,
+			)
+			if err != nil {
+				return nil, err
+			}
+			return InitDefaultConfig(home)
 		}
+		return nil, err
 	}
 
-	// Run from the default config
+	return c, nil
-	return ReadConfig(paths[0])
 }

utils/paths.go

@@ -20,6 +20,11 @@ func RequiredDirs(home string) []string {
 	return requiredDirs
 }
 
+// ConfigPath returns the default path to the config file
+func ConfigPath(home string) string {
+	return filepath.Join(home, "config.json")
+}
+
 // GeoIPDir returns the geoip data dir for the given OONI Home
 func GeoIPDir(home string) string {
 	return filepath.Join(home, "geoip")