diff --git a/Gopkg.lock b/Gopkg.lock index f4938e4..3114935 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -29,16 +29,10 @@ version = "v1.0.0" [[projects]] - branch = "master" - name = "github.com/aybabtme/rgbterm" + name = "github.com/certifi/gocertifi" packages = ["."] - revision = "cc83f3b3ce5911279513a46d6d3316d67bedaa54" - -[[projects]] - branch = "master" - name = "github.com/beorn7/perks" - packages = ["quantile"] - revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + revision = "deb3ae2ef2610fde3330947281941c562861188b" + version = "2018.01.18" [[projects]] name = "github.com/fatih/color" @@ -47,10 +41,10 @@ version = "v1.7.0" [[projects]] - name = "github.com/golang/protobuf" - packages = ["proto"] - revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" - version = "v1.1.0" + branch = "master" + name = "github.com/getsentry/raven-go" + packages = ["."] + revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa" [[projects]] branch = "master" @@ -79,12 +73,6 @@ revision = "323a32be5a2421b8c7087225079c6c900ec397cd" version = "v1.7.0" -[[projects]] - name = "github.com/matttproud/golang_protobuf_extensions" - packages = ["pbutil"] - revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" - version = "v1.0.1" - [[projects]] branch = "master" name = "github.com/measurement-kit/go-measurement-kit" @@ -97,12 +85,6 @@ packages = ["."] revision = "9520e82c474b0a04dd04f8a40959027271bab992" -[[projects]] - branch = "master" - name = "github.com/mitchellh/go-homedir" - packages = ["."] - revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66" - [[projects]] name = "github.com/oschwald/geoip2-golang" packages = ["."] @@ -121,40 +103,6 @@ revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" -[[projects]] - name = "github.com/prometheus/client_golang" - packages = ["prometheus"] - revision = "c5b7fccd204277076155f10851dad72b76a49317" - version = "v0.8.0" - -[[projects]] - branch = "master" - name = "github.com/prometheus/client_model" - packages = ["go"] - revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" - -[[projects]] - branch = "master" - name = "github.com/prometheus/common" - packages = [ - "expfmt", - "internal/bitbucket.org/ww/goautoneg", - "model", - "version" - ] - revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" - -[[projects]] - branch = "master" - name = "github.com/prometheus/procfs" - packages = [ - ".", - "internal/util", - "nfs", - "xfs" - ] - revision = "fe93d378a6b03758a2c1b65e86cf630bf78681c0" - [[projects]] branch = "master" name = "github.com/rubenv/sql-migrate" @@ -201,6 +149,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "95c3e971d63b97b0dc531f67d98401cfa9968b99aacf1eed73ce801bbaadb0cd" + inputs-digest = "b2f5c39222a1fb405e3f48d2ae3b4758757fe708e12dbd23743c19135e225579" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 7a941ae..7bc1e5b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -61,3 +61,7 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"] [[constraint]] name = "github.com/oschwald/geoip2-golang" version = "1.2.1" + +[[constraint]] + branch = "master" + name = "github.com/getsentry/raven-go" diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..4cb6d13 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,26 @@ +Copyright 2018 Open Observatory of Network Interference (OONI), The Tor Project + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cmd/ooni/main.go b/cmd/ooni/main.go index e48b9b3..2b56450 100644 --- a/cmd/ooni/main.go +++ b/cmd/ooni/main.go @@ -8,18 +8,22 @@ import ( _ "github.com/ooni/probe-cli/internal/cli/info" _ "github.com/ooni/probe-cli/internal/cli/list" _ "github.com/ooni/probe-cli/internal/cli/nettest" + _ "github.com/ooni/probe-cli/internal/cli/onboard" _ "github.com/ooni/probe-cli/internal/cli/run" _ "github.com/ooni/probe-cli/internal/cli/show" _ "github.com/ooni/probe-cli/internal/cli/upload" _ "github.com/ooni/probe-cli/internal/cli/version" "github.com/ooni/probe-cli/internal/cli/app" + "github.com/ooni/probe-cli/internal/crashreport" ) func main() { - err := app.Run() - if err == nil { - return - } - log.WithError(err).Fatal("main exit") + crashreport.CapturePanicAndWait(func() { + err := app.Run() + if err == nil { + return + } + log.WithError(err).Fatal("main exit") + }, nil) } diff --git a/config/advanced.go b/config/advanced.go deleted file mode 100644 index 2add03c..0000000 --- a/config/advanced.go +++ /dev/null @@ -1,7 +0,0 @@ -package config - -// Advanced settings -type Advanced struct { - IncludeCountry bool `json:"include_country"` - UseDomainFronting bool `json:"use_domain_fronting"` -} diff --git a/config/automated_testing.go b/config/automated_testing.go deleted file mode 100644 index 56dc660..0000000 --- a/config/automated_testing.go +++ /dev/null @@ -1,8 +0,0 @@ -package config - -// AutomatedTesting settings -type AutomatedTesting struct { - Enabled bool `json:"enabled"` - EnabledTests []string `json:"enabled_tests"` - MonthlyAllowance string `json:"monthly_allowance"` -} diff --git a/config/notifications.go b/config/notifications.go deleted file mode 100644 index 7341ac3..0000000 --- a/config/notifications.go +++ /dev/null @@ -1,8 +0,0 @@ -package config - -// Notifications settings -type Notifications struct { - Enabled bool `json:"enabled"` - NotifyOnTestCompletion bool `json:"notify_on_test_completion"` - NotifyOnNews bool `json:"notify_on_news"` -} diff --git a/config/parser.go b/config/parser.go new file mode 100644 index 0000000..5b5e15e --- /dev/null +++ b/config/parser.go @@ -0,0 +1,109 @@ +package config + +import ( + "encoding/json" + "io/ioutil" + "sync" + + "github.com/apex/log" + "github.com/ooni/probe-cli/internal/crashreport" + "github.com/ooni/probe-cli/utils" + "github.com/pkg/errors" +) + +// ReadConfig reads the configuration from the path +func ReadConfig(path string) (*Config, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + c, err := ParseConfig(b) + if err != nil { + return nil, errors.Wrap(err, "parsing config") + } + c.path = path + return c, err +} + +// ParseConfig returns config from JSON bytes. +func ParseConfig(b []byte) (*Config, error) { + var c Config + + if err := json.Unmarshal(b, &c); err != nil { + return nil, errors.Wrap(err, "parsing json") + } + + if err := c.Default(); err != nil { + return nil, errors.Wrap(err, "defaulting") + } + + if err := c.Validate(); err != nil { + return nil, errors.Wrap(err, "validating") + } + if c.Advanced.SendCrashReports == false { + log.Info("Disabling crash reporting.") + crashreport.Disabled = true + } + + return &c, nil +} + +// Config for the OONI Probe installation +type Config struct { + // Private settings + Comment string `json:"_"` + Version int64 `json:"_version"` + InformedConsent bool `json:"_informed_consent"` + IsBeta bool `json:"_is_beta"` // This is a boolean flag used to indicate this installation of OONI Probe was a beta install. These installations will have their data deleted across releases. + + AutoUpdate bool `json:"auto_update"` + Sharing Sharing `json:"sharing"` + Notifications Notifications `json:"notifications"` + AutomatedTesting AutomatedTesting `json:"automated_testing"` + NettestGroups NettestGroups `json:"test_settings"` + Advanced Advanced `json:"advanced"` + + mutex sync.Mutex + path string +} + +// Write the config file in json to the path +func (c *Config) Write() error { + c.Lock() + configJSON, _ := json.MarshalIndent(c, "", " ") + if c.path == "" { + return errors.New("config file path is empty") + } + if err := ioutil.WriteFile(c.path, configJSON, 0644); err != nil { + return errors.Wrap(err, "writing config JSON") + } + c.Unlock() + return nil +} + +// Lock acquires the write mutex +func (c *Config) Lock() { + c.mutex.Lock() +} + +// Unlock releases the write mutex +func (c *Config) Unlock() { + c.mutex.Unlock() +} + +// Default config settings +func (c *Config) Default() error { + home, err := utils.GetOONIHome() + if err != nil { + return err + } + + c.path = utils.ConfigPath(home) + return nil +} + +// Validate the config file +func (c *Config) Validate() error { + return nil +} diff --git a/config/parser_test.go b/config/parser_test.go new file mode 100644 index 0000000..010dbe6 --- /dev/null +++ b/config/parser_test.go @@ -0,0 +1,19 @@ +package config + +import ( + "testing" +) + +func TestParseConfig(t *testing.T) { + config, err := ReadConfig("testdata/valid-config.json") + if err != nil { + t.Error(err) + } + + if len(config.NettestGroups.Middlebox.EnabledTests) < 0 { + t.Error("at least one middlebox test should be enabled") + } + if config.Advanced.IncludeCountry == false { + t.Error("country should be included") + } +} diff --git a/config/nettest_groups.go b/config/settings.go similarity index 65% rename from config/nettest_groups.go rename to config/settings.go index 9be7a6c..c8ddf06 100644 --- a/config/nettest_groups.go +++ b/config/settings.go @@ -82,11 +82,10 @@ func (s *InstantMessaging) NettestConfigs() []NettestConfig { // Performance nettest group type Performance struct { - EnabledTests []string `json:"enabled_tests"` - NDTServer string `json:"ndt_server"` - NDTServerPort string `json:"ndt_server_port"` - DashServer string `json:"dash_server"` - DashServerPort string `json:"dash_server_port"` + NDTServer string `json:"ndt_server"` + NDTServerPort string `json:"ndt_server_port"` + DashServer string `json:"dash_server"` + DashServerPort string `json:"dash_server_port"` } // Middlebox nettest group @@ -101,3 +100,32 @@ type NettestGroups struct { Performance Performance `json:"performance"` Middlebox Middlebox `json:"middlebox"` } + +// Notifications settings +type Notifications struct { + Enabled bool `json:"enabled"` + NotifyOnTestCompletion bool `json:"notify_on_test_completion"` + NotifyOnNews bool `json:"notify_on_news"` +} + +// Sharing settings +type Sharing struct { + IncludeIP bool `json:"include_ip"` + IncludeASN bool `json:"include_asn"` + IncludeGPS bool `json:"include_gps"` + UploadResults bool `json:"upload_results"` +} + +// Advanced settings +type Advanced struct { + IncludeCountry bool `json:"include_country"` + UseDomainFronting bool `json:"use_domain_fronting"` + SendCrashReports bool `json:"send_crash_reports"` +} + +// AutomatedTesting settings +type AutomatedTesting struct { + Enabled bool `json:"enabled"` + EnabledTests []string `json:"enabled_tests"` + MonthlyAllowance string `json:"monthly_allowance"` +} diff --git a/config/sharing.go b/config/sharing.go deleted file mode 100644 index 6aebf8d..0000000 --- a/config/sharing.go +++ /dev/null @@ -1,10 +0,0 @@ -package config - -// Sharing settings -type Sharing struct { - IncludeIP bool `json:"include_ip"` - IncludeASN bool `json:"include_asn"` - IncludeGPS bool `json:"include_gps"` - UploadResults bool `json:"upload_results"` - SendCrashReports bool `json:"send_crash_reports"` -} diff --git a/config/testdata/valid-config.json b/config/testdata/valid-config.json new file mode 100644 index 0000000..c6c3875 --- /dev/null +++ b/config/testdata/valid-config.json @@ -0,0 +1,63 @@ +{ + "_": "This is your OONI Probe config file. See https://ooni.io/help/probe-cli for help", + "_version": 0, + "_informed_consent": false, + "auto_update": true, + "sharing": { + "include_ip": false, + "include_asn": true, + "include_gps": true, + "upload_results": true + }, + "notifications": { + "enabled": true, + "notify_on_test_completion": true, + "notify_on_news": false + }, + "automated_testing": { + "enabled": false, + "enabled_tests": [ + "web-connectivity", + "facebook-messenger", + "whatsapp", + "telegram", + "dash", + "ndt", + "http-invalid-request-line", + "http-header-field-manipulation" + ], + "monthly_allowance": "300MB" + }, + "test_settings": { + "websites": { + "enabled_categories": [] + }, + "instant_messaging": { + "enabled_tests": [ + "facebook-messenger", + "whatsapp", + "telegram" + ] + }, + "performance": { + "enabled_tests": [ + "ndt" + ], + "ndt_server": "auto", + "ndt_server_port": "auto", + "dash_server": "auto", + "dash_server_port": "auto" + }, + "middlebox": { + "enabled_tests": [ + "http-invalid-request-line", + "http-header-field-manipulation" + ] + } + }, + "advanced": { + "include_country": true, + "use_domain_fronting": false, + "send_crash_reports": true + } +} diff --git a/data/default-config.json b/data/default-config.json index 1068d91..cd72e53 100644 --- a/data/default-config.json +++ b/data/default-config.json @@ -1,12 +1,14 @@ { - "_": "This is your OONI Probe config file. See https://ooni.io/help/ooniprobe-cli for help", + "_": "This is your OONI Probe config file. See https://ooni.io/help/probe-cli for help", + "_version": 0, + "_informed_consent": false, + "_is_beta": true, "auto_update": true, "sharing": { "include_ip": false, "include_asn": true, "include_gps": true, - "upload_results": true, - "send_crash_reports": true + "upload_results": true }, "notifications": { "enabled": true, @@ -56,8 +58,7 @@ }, "advanced": { "include_country": true, - "use_domain_fronting": true - }, - "_config_version": "0.0.1", - "_informed_consent": true + "use_domain_fronting": false, + "send_crash_reports": true + } } diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index 1935962..812dc69 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -80,26 +80,26 @@ func (fi bindataFileInfo) Sys() interface{} { } var _bindataDataDefaultconfigjson = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\xcd\x6e\xe3\x38\x0c\xc7\xef\x7d\x0a\x41\xe7\x3a\xcd\x62\x6f\x39" + - "\xee\x6d\x0f\xbb\x1d\x60\xe6\x56\x14\x82\x62\xd1\x36\x31\x32\xa9\x11\xe9\x64\x82\x41\xdf\x7d\x20\x35\x89\xed\x7e" + - "\x4d\x8f\xe2\x9f\xa2\xc8\x1f\x29\xfe\xba\x31\xc6\x3a\xbb\x33\xf6\xdb\x80\x62\x50\xcc\x89\xa7\x6c\xee\xef\xff\xff" + - "\xd7\x7c\xc9\xbc\x07\xd3\x32\x75\xd8\x9b\x0e\x23\x6c\xcc\x57\x00\x33\xa8\x26\xd9\xdd\xdd\x31\x13\x6e\x90\xef\x06" + - "\x88\xa9\x1e\x52\xf1\x6f\xda\x88\xa6\xe3\x6c\x8a\xd9\xde\x96\xf0\x7e\x52\x76\x53\x0a\x5e\xc1\xee\x8c\xe6\x09\xaa" + - "\x59\x06\x9f\x91\x7a\xbb\x33\x25\x09\x63\x2c\x52\x1b\xa7\x00\x0e\x93\xdd\x99\xce\x47\xa9\x7e\x0b\xc1\x0b\x2d\x02" + - "\x2c\x84\x3e\xc9\x5a\x98\x52\x64\x1f\x5c\x06\x99\xa2\xbe\xd0\x04\x28\xb8\x36\x7b\x19\x5c\x86\xc4\xf9\xaa\xdf\x18" + - "\xf3\x54\x33\x23\x56\xec\xb0\xf5\x8a\x4c\x32\xe7\x07\xe4\xf7\x11\xc2\x3a\x5a\xf5\x3d\x39\x26\xa7\x20\xea\x5a\x1e" + - "\x53\x84\x72\xf1\x3d\x37\x82\xa3\x5c\xea\xbb\xbe\x58\x10\x8d\x5e\x21\xd4\x28\x2b\x2a\xf3\xab\x4b\x24\x67\x6b\x75" + - "\x2f\xe1\x1e\xaa\xd9\x18\x7b\x84\x7d\xd3\x32\x11\xb4\x8a\x07\xd4\x93\xbd\xbd\x28\x9d\x6f\x61\xcf\xfc\xbd\x19\x41" + - "\x04\xa8\x87\x3c\x6b\xc7\xc1\xab\xf8\x94\x66\x8b\x42\x84\x3e\xfb\x71\xb6\x04\x2f\xc3\x7c\xa2\xa0\xf3\xa1\x8c\x44" + - "\x83\x74\xf0\x11\x43\x93\xe1\xc7\x04\xa2\x4d\x44\x82\x17\x2e\x03\xf8\x00\xb9\xe9\x10\x62\x68\x46\x4f\x98\xa6\x58" + - "\x29\xdb\xea\xf6\x78\x2e\x6e\x64\xd2\x21\x9e\x9c\x8f\x91\x8f\x9e\xda\x32\x36\xf6\xef\xed\xf6\xbf\x7f\xec\x95\x58" + - "\xa5\x2d\xa0\x05\xd6\xa2\x47\x47\xd8\x0b\x2a\xcc\x96\x05\xab\xd6\x2b\xf4\x9c\xb1\xaa\x0f\x8f\x55\x7e\xba\x4e\x92" + - "\xa8\x27\x75\x85\x8d\xef\x97\x0d\xf8\x00\xf6\xc7\x50\xdf\xc2\xba\x04\x7b\x36\xad\xf3\x48\x90\x3b\xce\xe3\xb9\xe8" + - "\xcf\x64\x50\x1a\x71\x09\xb5\xec\x8e\x13\xc8\x07\xc8\x05\x5d\x99\x2e\xfb\x86\xe6\xca\xf4\xbf\x76\x28\x8d\x7e\xf7" + - "\xf6\x42\x5c\x5f\x5f\x95\x31\x62\x08\x11\xf6\xfc\xf3\x93\x45\xfc\x79\x80\x3e\x39\x42\x57\x9e\xf3\xd7\x0a\x87\x42" + - "\x33\xbc\xde\x33\x2d\x4f\xa4\xf9\xf4\x62\x73\x08\xb8\xc0\xa3\x47\x72\x5d\x66\x3a\xff\xc5\xd5\x7a\x70\xcf\x2b\xd1" + - "\x1d\x20\xcb\xf3\x47\xb7\xdb\xcd\x76\xf3\xd7\xf3\xb6\x73\x48\xa5\x83\x65\xde\x98\x04\x48\x2f\xd7\x9f\x6e\x7e\x07" + - "\x00\x00\xff\xff\x0f\x7e\x15\xb3\x6d\x05\x00\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\x41\x6f\xdb\x3c\x0c\x86\xef\xf9\x15\x82\xce\x75\x53\xe0\xbb\xe5" + + "\xf8\xdd\x76\xd8\x3a\x60\xbb\x15\x85\x20\x5b\xb4\x4d\x4c\x26\x35\x91\x4e\x16\x0c\xfd\xef\x83\xdc\x24\x56\xda\xae" + + "\xeb\xd1\xef\x4b\x53\xe2\x43\x52\xbf\x37\xc6\x58\x67\x77\xc6\x7e\x1f\x51\x0c\x8a\x39\xf2\x9c\xcd\xfd\xfd\x97\x4f" + + "\xe6\x6b\xe6\x16\x4c\xc7\xd4\xe3\x60\x7a\x8c\x70\x6b\xbe\x01\x98\x51\x35\xc9\x6e\xbb\x65\x26\xbc\x45\xde\x8e\x10" + + "\xd3\x36\x95\xd8\xa6\x8b\x68\x7a\xce\xa6\x48\xf6\x66\x49\xbd\x87\x2c\xc8\x64\x77\xe6\xee\x59\x40\xea\x39\x4f\x10" + + "\x5c\xc7\x24\x40\x6a\x77\xa6\xf7\x51\xe0\xe4\x8a\x6b\x41\xbd\xdd\x19\xcd\xf3\xb3\xe6\x67\x65\x37\xa7\xe0\x15\x6a" + + "\x59\x46\x9f\x91\x06\xbb\x33\xa5\x06\x63\x2c\x52\x17\xe7\x00\x0e\x53\x9d\xb2\x32\xbc\x50\x95\xa0\x32\x86\x24\xd7" + + "\xc6\x9c\x22\xfb\xe0\x32\xc8\x1c\xf5\xec\x6d\x8c\x79\x5a\x4e\x26\x56\xec\xb1\xf3\x8a\x4c\xb2\x9e\x0f\xe4\xdb\x08" + + "\xe1\x3a\xd3\x12\x7b\x74\x4c\x4e\x41\xd4\x75\x3c\xa5\x08\xfa\x0c\xe4\xcd\x30\x82\x83\x9c\xef\x7f\x39\xb1\x20\x98" + + "\xbc\x42\x58\xb2\x5c\x55\xbd\x9e\x5a\x97\x7c\x52\x97\xf0\x92\xee\x61\x91\x8d\xb1\x07\x68\x9b\x8e\x89\xa0\x53\xdc" + + "\xa3\x1e\xed\xcd\xd9\xe9\x7d\x07\x2d\xf3\x8f\x66\x02\x11\xa0\x01\xf2\xea\x1d\x46\xaf\xe2\x53\x5a\x15\x85\x08\x43" + + "\xf6\xd3\xaa\x04\x2f\xe3\xfa\x45\x41\xd7\x8f\x32\x31\x0d\xd2\xde\x47\x0c\x4d\x86\x9f\x33\x88\x36\x11\x09\x5e\x84" + + "\x8c\xe0\x03\xe4\xa6\x47\x88\xa1\x99\x3c\x61\x9a\xe3\x42\xd9\x2e\x61\x8f\xa7\xe2\x26\x26\x1d\xe3\xd1\xf9\x18\xf9" + + "\xe0\xa9\x2b\x63\x61\xff\xbb\xbb\xfb\xfc\xbf\xbd\x10\x5b\x68\x0b\x68\x81\x55\xf5\xe8\x00\xad\xa0\xc2\xaa\x54\xac" + + "\x3a\xaf\x30\x70\xc6\xc5\x7d\x78\x5c\xec\xa7\xcb\xa4\x88\x7a\x52\x57\xd8\xf8\xa1\x6e\xc0\x3b\xb0\xdf\x87\xfa\x16" + + "\xd6\x1a\xec\x49\xba\xbe\x47\x82\x5c\xb6\xe7\x54\xf4\x47\x6e\x50\x1a\x71\x4e\x55\x77\xc7\x09\xe4\x3d\xe4\x82\xae" + + "\x4c\x97\x7d\xc3\x73\x89\xb3\xbe\x0e\x28\x8d\xfe\xeb\xdf\x95\x79\xfd\xfb\x55\x19\x13\x86\x10\xa1\xe5\x5f\x1f\x2c" + + "\xe2\xdf\x03\xf4\xc1\x11\xba\xf0\x5c\x57\x2b\xec\x0b\xcd\xf0\xfa\x1d\xe9\x78\x26\xcd\xc7\x17\x2f\x83\x80\x0b\x3c" + + "\x79\x24\xd7\x67\xa6\xd3\x2e\xd6\xab\x27\x40\xc1\x75\xb9\x70\xc8\x50\x10\xd4\xef\xc7\xe6\x69\xf3\x27\x00\x00\xff" + + "\xff\x42\x02\xc0\xed\x72\x05\x00\x00") func bindataDataDefaultconfigjsonBytes() ([]byte, error) { return bindataRead( diff --git a/internal/cli/app/app.go b/internal/cli/app/app.go index f84bfcc..539874c 100644 --- a/internal/cli/app/app.go +++ b/internal/cli/app/app.go @@ -3,13 +3,13 @@ package app import ( "os" + ooni "github.com/ooni/probe-cli" "github.com/ooni/probe-cli/internal/cli/root" - "github.com/ooni/probe-cli/internal/cli/version" ) // Run the app. This is the main app entry point func Run() error { - root.Cmd.Version(version.Version) + root.Cmd.Version(ooni.Version) _, err := root.Cmd.Parse(os.Args[1:]) return err } diff --git a/internal/cli/geoip/geoip.go b/internal/cli/geoip/geoip.go index 82a1945..0de3620 100644 --- a/internal/cli/geoip/geoip.go +++ b/internal/cli/geoip/geoip.go @@ -1,9 +1,12 @@ package geoip import ( + "fmt" + "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/ooni/probe-cli/internal/cli/root" + "github.com/ooni/probe-cli/internal/output" "github.com/ooni/probe-cli/utils" ) @@ -13,12 +16,16 @@ func init() { shouldUpdate := cmd.Flag("update", "Update the geoip database").Bool() cmd.Action(func(_ *kingpin.ParseContext) error { - log.Info("geoip") + output.SectionTitle("GeoIP lookup") ctx, err := root.Init() if err != nil { return err } + if err = ctx.MaybeDownloadDataFiles(); err != nil { + log.WithError(err).Error("failed to download data files") + } + geoipPath := utils.GeoIPDir(ctx.Home) if *shouldUpdate { utils.DownloadGeoIPDatabaseFiles(geoipPath) @@ -31,7 +38,8 @@ func init() { } log.WithFields(log.Fields{ - "asn": loc.ASN, + "type": "table", + "asn": fmt.Sprintf("AS%d", loc.ASN), "network_name": loc.NetworkName, "country_code": loc.CountryCode, "ip": loc.IP, diff --git a/internal/cli/info/info.go b/internal/cli/info/info.go index ef80127..5743dff 100644 --- a/internal/cli/info/info.go +++ b/internal/cli/info/info.go @@ -11,6 +11,7 @@ func init() { cmd.Action(func(_ *kingpin.ParseContext) error { log.Info("Info") + log.Error("this function is not implemented") return nil }) } diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 80e1395..b52882b 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -1,6 +1,8 @@ package list import ( + "fmt" + "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/ooni/probe-cli/internal/cli/root" @@ -11,52 +13,78 @@ import ( func init() { cmd := root.Command("list", "List results") + resultID := cmd.Arg("id", "the id of the result to list measurements for").Int64() + cmd.Action(func(_ *kingpin.ParseContext) error { ctx, err := root.Init() if err != nil { log.WithError(err).Error("failed to initialize root context") return err } - doneResults, incompleteResults, err := database.ListResults(ctx.DB) - if err != nil { - log.WithError(err).Error("failed to list results") - return err + if *resultID > 0 { + measurements, err := database.ListMeasurements(ctx.DB, *resultID) + if err != nil { + log.WithError(err).Error("failed to list measurements") + return err + } + for idx, msmt := range measurements { + fmt.Printf("%d: %v\n", idx, msmt) + } + } else { + doneResults, incompleteResults, err := database.ListResults(ctx.DB) + if err != nil { + log.WithError(err).Error("failed to list results") + return err + } + + if len(incompleteResults) > 0 { + output.SectionTitle("Incomplete results") + } + for idx, result := range incompleteResults { + output.ResultItem(output.ResultItemData{ + ID: result.ID, + Index: idx, + TotalCount: len(incompleteResults), + Name: result.Name, + StartTime: result.StartTime, + NetworkName: result.NetworkName, + Country: result.Country, + ASN: result.ASN, + Summary: result.Summary, + Done: result.Done, + DataUsageUp: result.DataUsageUp, + DataUsageDown: result.DataUsageDown, + }) + } + + resultSummary := output.ResultSummaryData{} + netCount := make(map[string]int) + output.SectionTitle("Results") + for idx, result := range doneResults { + output.ResultItem(output.ResultItemData{ + ID: result.ID, + Index: idx, + TotalCount: len(doneResults), + Name: result.Name, + StartTime: result.StartTime, + NetworkName: result.NetworkName, + Country: result.Country, + ASN: result.ASN, + Summary: result.Summary, + Done: result.Done, + DataUsageUp: result.DataUsageUp, + DataUsageDown: result.DataUsageDown, + }) + resultSummary.TotalTests++ + netCount[result.ASN]++ + resultSummary.TotalDataUsageUp += result.DataUsageUp + resultSummary.TotalDataUsageDown += result.DataUsageDown + } + resultSummary.TotalNetworks = int64(len(netCount)) + + output.ResultSummary(resultSummary) } - log.Info("Results") - for idx, result := range doneResults { - output.ResultItem(output.ResultItemData{ - ID: result.ID, - Index: idx, - TotalCount: len(doneResults), - Name: result.Name, - StartTime: result.StartTime, - NetworkName: result.NetworkName, - Country: result.Country, - ASN: result.ASN, - Summary: result.Summary, - Done: result.Done, - DataUsageUp: result.DataUsageUp, - DataUsageDown: result.DataUsageDown, - }) - } - log.Info("Incomplete results") - for idx, result := range incompleteResults { - output.ResultItem(output.ResultItemData{ - ID: result.ID, - Index: idx, - TotalCount: len(incompleteResults), - Name: result.Name, - StartTime: result.StartTime, - NetworkName: result.NetworkName, - Country: result.Country, - ASN: result.ASN, - Summary: result.Summary, - Done: result.Done, - DataUsageUp: result.DataUsageUp, - DataUsageDown: result.DataUsageDown, - }) - } return nil }) } diff --git a/internal/cli/nettest/nettest.go b/internal/cli/nettest/nettest.go index 39fc053..3517a66 100644 --- a/internal/cli/nettest/nettest.go +++ b/internal/cli/nettest/nettest.go @@ -11,6 +11,7 @@ func init() { cmd.Action(func(_ *kingpin.ParseContext) error { log.Info("Nettest") + log.Error("this function is not implemented") return nil }) } diff --git a/internal/cli/onboard/onboard.go b/internal/cli/onboard/onboard.go new file mode 100644 index 0000000..108672a --- /dev/null +++ b/internal/cli/onboard/onboard.go @@ -0,0 +1,35 @@ +package onboard + +import ( + "github.com/alecthomas/kingpin" + "github.com/apex/log" + "github.com/ooni/probe-cli/internal/cli/root" + "github.com/ooni/probe-cli/internal/onboard" +) + +func init() { + cmd := root.Command("onboard", "Starts the onboarding process") + + yes := cmd.Flag("yes", "Answer yes to all the onboarding questions.").Bool() + + cmd.Action(func(_ *kingpin.ParseContext) error { + ctx, err := root.Init() + if err != nil { + return err + } + + if *yes == true { + ctx.Config.Lock() + ctx.Config.InformedConsent = true + ctx.Config.Unlock() + + if err := ctx.Config.Write(); err != nil { + log.WithError(err).Error("failed to write config file") + return err + } + return nil + } + + return onboard.Onboarding(ctx.Config) + }) +} diff --git a/internal/cli/root/root.go b/internal/cli/root/root.go index 5777f10..c7749da 100644 --- a/internal/cli/root/root.go +++ b/internal/cli/root/root.go @@ -7,7 +7,6 @@ import ( "github.com/ooni/probe-cli/internal/log/handlers/batch" "github.com/ooni/probe-cli/internal/log/handlers/cli" "github.com/ooni/probe-cli/utils" - "github.com/prometheus/common/version" ) // Cmd is the root command @@ -33,7 +32,7 @@ func init() { } if *isVerbose { log.SetLevel(log.DebugLevel) - log.Debugf("ooni version %s", version.Version) + log.Debugf("ooni version %s", ooni.Version) } Init = func() (*ooni.Context, error) { diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index b6125bc..7c8a9aa 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -4,10 +4,12 @@ import ( "errors" "fmt" "path/filepath" + "strings" "time" "github.com/alecthomas/kingpin" "github.com/apex/log" + "github.com/fatih/color" "github.com/ooni/probe-cli/internal/cli/root" "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/nettests" @@ -18,7 +20,14 @@ import ( func init() { cmd := root.Command("run", "Run a test group or OONI Run link") - nettestGroup := cmd.Arg("name", "the nettest group to run").Required().String() + var nettestGroupNames []string + for name := range groups.NettestGroups { + nettestGroupNames = append(nettestGroupNames, color.BlueString(name)) + } + + nettestGroup := cmd.Arg("name", + fmt.Sprintf("the nettest group to run. Supported tests are: %s", + strings.Join(nettestGroupNames, ", "))).Required().String() cmd.Action(func(_ *kingpin.ParseContext) error { log.Infof("Starting %s", *nettestGroup) @@ -27,6 +36,12 @@ func init() { log.Errorf("%s", err) return err } + + if err = ctx.MaybeOnboarding(); err != nil { + log.WithError(err).Error("failed to perform onboarding") + return err + } + group, ok := groups.NettestGroups[*nettestGroup] if !ok { log.Errorf("No test group named %s", *nettestGroup) diff --git a/internal/cli/show/show.go b/internal/cli/show/show.go index de919e3..ca75897 100644 --- a/internal/cli/show/show.go +++ b/internal/cli/show/show.go @@ -10,7 +10,13 @@ func init() { cmd := root.Command("show", "Show a specific measurement") cmd.Action(func(_ *kingpin.ParseContext) error { - log.Info("Show") + _, err := root.Init() + if err != nil { + log.WithError(err).Error("failed to initialize root context") + return err + } + log.Error("this function is not implemented") + return nil }) } diff --git a/internal/cli/upload/upload.go b/internal/cli/upload/upload.go index 208af91..42a70f4 100644 --- a/internal/cli/upload/upload.go +++ b/internal/cli/upload/upload.go @@ -11,6 +11,7 @@ func init() { cmd.Action(func(_ *kingpin.ParseContext) error { log.Info("Uploading") + log.Error("this function is not implemented") return nil }) } diff --git a/internal/cli/version/version.go b/internal/cli/version/version.go index 3deceda..5e55be1 100644 --- a/internal/cli/version/version.go +++ b/internal/cli/version/version.go @@ -4,15 +4,15 @@ import ( "fmt" "github.com/alecthomas/kingpin" + ooni "github.com/ooni/probe-cli" "github.com/ooni/probe-cli/internal/cli/root" ) -const Version = "3.0.0-dev.0" func init() { cmd := root.Command("version", "Show version.") cmd.Action(func(_ *kingpin.ParseContext) error { - fmt.Println(Version) + fmt.Println(ooni.Version) return nil }) } diff --git a/internal/colors/colors.go b/internal/colors/colors.go deleted file mode 100644 index 09b0c58..0000000 --- a/internal/colors/colors.go +++ /dev/null @@ -1,40 +0,0 @@ -package colors - -import ( - color "github.com/aybabtme/rgbterm" -) - -// Gray string. -func Gray(s string) string { - return color.FgString(s, 150, 150, 150) -} - -// Blue string. -func Blue(s string) string { - return color.FgString(s, 77, 173, 247) -} - -// Cyan string. -func Cyan(s string) string { - return color.FgString(s, 34, 184, 207) -} - -// Green string. -func Green(s string) string { - return color.FgString(s, 0, 200, 255) -} - -// Red string. -func Red(s string) string { - return color.FgString(s, 194, 37, 92) -} - -// Yellow string. -func Yellow(s string) string { - return color.FgString(s, 252, 196, 25) -} - -// Purple string. -func Purple(s string) string { - return color.FgString(s, 96, 97, 190) -} diff --git a/internal/crashreport/crashreport.go b/internal/crashreport/crashreport.go new file mode 100644 index 0000000..21e2c1c --- /dev/null +++ b/internal/crashreport/crashreport.go @@ -0,0 +1,47 @@ +package crashreport + +import ( + "github.com/getsentry/raven-go" +) + +// Disabled flag is used to globally disable crash reporting and make all the +// crash reporting logic a no-op. +var Disabled = false + +// CapturePanic is a wrapper around raven.CapturePanic that becomes a noop if +// `Disabled` is set to true. +func CapturePanic(f func(), tags map[string]string) (interface{}, string) { + if Disabled == true { + return nil, "" + } + return raven.CapturePanic(f, tags) +} + +// CapturePanicAndWait is a wrapper around raven.CapturePanicAndWait that becomes a noop if +// `Disabled` is set to true. +func CapturePanicAndWait(f func(), tags map[string]string) (interface{}, string) { + if Disabled == true { + return nil, "" + } + return raven.CapturePanicAndWait(f, tags) +} + +// CaptureError is a wrapper around raven.CaptureError +func CaptureError(err error, tags map[string]string) string { + if Disabled == true { + return "" + } + return raven.CaptureError(err, tags) +} + +// CaptureErrorAndWait is a wrapper around raven.CaptureErrorAndWait +func CaptureErrorAndWait(err error, tags map[string]string) string { + if Disabled == true { + return "" + } + return raven.CaptureErrorAndWait(err, tags) +} + +func init() { + raven.SetDSN("https://cb4510e090f64382ac371040c19b2258:8448daeebfa643c289ef398f8645980b@sentry.io/1234954") +} diff --git a/internal/database/models.go b/internal/database/models.go index daff59b..689d806 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -7,16 +7,11 @@ import ( "github.com/apex/log" "github.com/jmoiron/sqlx" + "github.com/ooni/probe-cli/nettests/summary" "github.com/ooni/probe-cli/utils" "github.com/pkg/errors" ) -// ResultSummaryFunc is the function used to generate result summaries -type ResultSummaryFunc func(SummaryMap) (string, error) - -// SummaryMap contains a mapping from test name to serialized summary for it -type SummaryMap map[string][]string - // UpdateOne will run the specified update query and check that it only affected one row func UpdateOne(db *sqlx.DB, query string, arg interface{}) error { res, err := db.NamedExec(query, arg) @@ -34,6 +29,42 @@ func UpdateOne(db *sqlx.DB, query string, arg interface{}) error { return nil } +// ListMeasurements given a result ID +func ListMeasurements(db *sqlx.DB, resultID int64) ([]*Measurement, error) { + measurements := []*Measurement{} + + rows, err := db.Query(`SELECT id, name, + start_time, runtime, + country, + asn, + summary, + input + FROM measurements + WHERE result_id = ? + ORDER BY start_time;`, resultID) + if err != nil { + return measurements, errors.Wrap(err, "failed to get measurement list") + } + + for rows.Next() { + msmt := Measurement{} + err = rows.Scan(&msmt.ID, &msmt.Name, + &msmt.StartTime, &msmt.Runtime, + &msmt.CountryCode, + &msmt.ASN, + &msmt.Summary, &msmt.Input, + //&result.DataUsageUp, &result.DataUsageDown) + ) + if err != nil { + log.WithError(err).Error("failed to fetch a row") + continue + } + measurements = append(measurements, &msmt) + } + + return measurements, nil +} + // Measurement model type Measurement struct { ID int64 `db:"id"` @@ -262,8 +293,8 @@ func ListResults(db *sqlx.DB) ([]*Result, []*Result, error) { // MakeSummaryMap return a mapping of test names to summaries for the given // result -func MakeSummaryMap(db *sqlx.DB, r *Result) (SummaryMap, error) { - summaryMap := SummaryMap{} +func MakeSummaryMap(db *sqlx.DB, r *Result) (summary.SummaryMap, error) { + summaryMap := summary.SummaryMap{} msmts := []Measurement{} // XXX maybe we only want to select some of the columns @@ -283,7 +314,7 @@ func MakeSummaryMap(db *sqlx.DB, r *Result) (SummaryMap, error) { } // Finished marks the result as done and sets the runtime -func (r *Result) Finished(db *sqlx.DB, makeSummary ResultSummaryFunc) error { +func (r *Result) Finished(db *sqlx.DB, makeSummary summary.ResultSummaryFunc) error { if r.Done == true || r.Runtime != 0 { return errors.New("Result is already finished") } diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index fbb04cd..ca716d5 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -4,12 +4,14 @@ import ( "fmt" "io" "os" + "strings" "sync" "time" "github.com/apex/log" "github.com/fatih/color" colorable "github.com/mattn/go-colorable" + "github.com/ooni/probe-cli/internal/util" ) // Default handler outputting to stderr. @@ -60,16 +62,62 @@ func New(w io.Writer) *Handler { } } +func logSectionTitle(w io.Writer, f log.Fields) error { + colWidth := 24 + + title := f.Get("title").(string) + fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth+2)+"┓\n") + fmt.Fprintf(w, "┃ %s ┃\n", util.RightPad(title, colWidth)) + fmt.Fprintf(w, "┗"+strings.Repeat("━", colWidth+2)+"┛\n") + return nil +} + +func logTable(w io.Writer, f log.Fields) error { + color := color.New(color.FgBlue) + + names := f.Names() + + var lines []string + colWidth := 0 + for _, name := range names { + if name == "type" { + continue + } + line := fmt.Sprintf("%s: %s", color.Sprint(name), f.Get(name)) + lineLength := util.EscapeAwareRuneCountInString(line) + lines = append(lines, line) + if colWidth < lineLength { + colWidth = lineLength + } + } + + fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth+2)+"┓\n") + for _, line := range lines { + fmt.Fprintf(w, "┃ %s ┃\n", + util.RightPad(line, colWidth), + ) + } + fmt.Fprintf(w, "┗"+strings.Repeat("━", colWidth+2)+"┛\n") + return nil +} + // TypedLog is used for handling special "typed" logs to the CLI func (h *Handler) TypedLog(t string, e *log.Entry) error { switch t { case "progress": - // XXX replace this with something more fancy like https://github.com/tj/go-progress - fmt.Fprintf(h.Writer, "%.1f%% [%s]: %s", e.Fields.Get("percentage").(float64)*100, e.Fields.Get("key"), e.Message) + var err error + s := fmt.Sprintf("%.2f%%: %-25s", e.Fields.Get("percentage").(float64)*100, e.Message) + fmt.Fprintf(h.Writer, s) fmt.Fprintln(h.Writer) - return nil + return err + case "table": + return logTable(h.Writer, e.Fields) case "result_item": return logResultItem(h.Writer, e.Fields) + case "result_summary": + return logResultSummary(h.Writer, e.Fields) + case "section_title": + return logSectionTitle(h.Writer, e.Fields) default: return h.DefaultLog(e) } @@ -81,15 +129,15 @@ func (h *Handler) DefaultLog(e *log.Entry) error { level := Strings[e.Level] names := e.Fields.Names() - color.Fprintf(h.Writer, "%s %-25s", bold.Sprintf("%*s", h.Padding+1, level), e.Message) - + s := color.Sprintf("%s %-25s", bold.Sprintf("%*s", h.Padding+1, level), e.Message) for _, name := range names { if name == "source" { continue } - fmt.Fprintf(h.Writer, " %s=%s", color.Sprint(name), e.Fields.Get(name)) + s += fmt.Sprintf(" %s=%s", color.Sprint(name), e.Fields.Get(name)) } + fmt.Fprintf(h.Writer, s) fmt.Fprintln(h.Writer) return nil diff --git a/internal/log/handlers/cli/progress/progress.go b/internal/log/handlers/cli/progress/progress.go new file mode 100644 index 0000000..50887a5 --- /dev/null +++ b/internal/log/handlers/cli/progress/progress.go @@ -0,0 +1,126 @@ +// Package progress provides a simple terminal progress bar. +package progress + +import ( + "bytes" + "fmt" + "html/template" + "io" + "math" + "strings" +) + +// Bar is a progress bar. +type Bar struct { + StartDelimiter string // StartDelimiter for the bar ("|"). + EndDelimiter string // EndDelimiter for the bar ("|"). + Filled string // Filled section representation ("█"). + Empty string // Empty section representation ("░") + Total float64 // Total value. + Width int // Width of the bar. + + value float64 + tmpl *template.Template + text string +} + +// New returns a new bar with the given total. +func New(total float64) *Bar { + b := &Bar{ + StartDelimiter: "|", + EndDelimiter: "|", + Filled: "█", + Empty: "░", + Total: total, + Width: 60, + } + + b.Template(`{{.Percent | printf "%3.0f"}}% {{.Bar}} {{.Text}}`) + + return b +} + +// NewInt returns a new bar with the given total. +func NewInt(total int) *Bar { + return New(float64(total)) +} + +// Text sets the text value. +func (b *Bar) Text(s string) { + b.text = s +} + +// Value sets the value. +func (b *Bar) Value(n float64) { + if n > b.Total { + panic("Bar update value cannot be greater than the total") + } + b.value = n +} + +// ValueInt sets the value. +func (b *Bar) ValueInt(n int) { + b.Value(float64(n)) +} + +// Percent returns the percentage +func (b *Bar) percent() float64 { + return (b.value / b.Total) * 100 +} + +// Bar returns the progress bar string. +func (b *Bar) bar() string { + p := b.value / b.Total + filled := math.Ceil(float64(b.Width) * p) + empty := math.Floor(float64(b.Width) - filled) + s := b.StartDelimiter + s += strings.Repeat(b.Filled, int(filled)) + s += strings.Repeat(b.Empty, int(empty)) + s += b.EndDelimiter + return s +} + +// String returns the progress bar. +func (b *Bar) String() string { + var buf bytes.Buffer + + data := struct { + Value float64 + Total float64 + Percent float64 + StartDelimiter string + EndDelimiter string + Bar string + Text string + }{ + Value: b.value, + Text: b.text, + StartDelimiter: b.StartDelimiter, + EndDelimiter: b.EndDelimiter, + Percent: b.percent(), + Bar: b.bar(), + } + + if err := b.tmpl.Execute(&buf, data); err != nil { + panic(err) + } + + return buf.String() +} + +// WriteTo writes the progress bar to w. +func (b *Bar) WriteTo(w io.Writer) (int64, error) { + s := fmt.Sprintf("\r %s ", b.String()) + _, err := io.WriteString(w, s) + return int64(len(s)), err +} + +// Template for rendering. This method will panic if the template fails to parse. +func (b *Bar) Template(s string) { + t, err := template.New("").Parse(s) + if err != nil { + panic(err) + } + + b.tmpl = t +} diff --git a/internal/log/handlers/cli/result_item.go b/internal/log/handlers/cli/results.go similarity index 65% rename from internal/log/handlers/cli/result_item.go rename to internal/log/handlers/cli/results.go index 1f3ec8e..630145b 100644 --- a/internal/log/handlers/cli/result_item.go +++ b/internal/log/handlers/cli/results.go @@ -8,38 +8,10 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-cli/internal/util" + "github.com/ooni/probe-cli/nettests/summary" ) -func RightPad(str string, length int) string { - return str + strings.Repeat(" ", length-len(str)) -} - -// XXX Copy-pasta from nettest/groups -// PerformanceSummary is the result summary for a performance test -type PerformanceSummary struct { - Upload int64 - Download int64 - Ping float64 - Bitrate int64 -} - -// MiddleboxSummary is the summary for the middlebox tests -type MiddleboxSummary struct { - Detected bool -} - -// IMSummary is the summary for the im tests -type IMSummary struct { - Tested uint - Blocked uint -} - -// WebsitesSummary is the summary for the websites test -type WebsitesSummary struct { - Tested uint - Blocked uint -} - func formatSpeed(speed int64) string { if speed < 1000 { return fmt.Sprintf("%d Kbit/s", speed) @@ -54,7 +26,7 @@ func formatSpeed(speed int64) string { var summarizers = map[string]func(string) []string{ "websites": func(ss string) []string { - var summary WebsitesSummary + var summary summary.WebsitesSummary if err := json.Unmarshal([]byte(ss), &summary); err != nil { return nil } @@ -65,7 +37,7 @@ var summarizers = map[string]func(string) []string{ } }, "performance": func(ss string) []string { - var summary PerformanceSummary + var summary summary.PerformanceSummary if err := json.Unmarshal([]byte(ss), &summary); err != nil { return nil } @@ -76,7 +48,7 @@ var summarizers = map[string]func(string) []string{ } }, "im": func(ss string) []string { - var summary IMSummary + var summary summary.IMSummary if err := json.Unmarshal([]byte(ss), &summary); err != nil { return nil } @@ -87,7 +59,7 @@ var summarizers = map[string]func(string) []string{ } }, "middlebox": func(ss string) []string { - var summary MiddleboxSummary + var summary summary.MiddleboxSummary if err := json.Unmarshal([]byte(ss), &summary); err != nil { return nil } @@ -122,21 +94,21 @@ func logResultItem(w io.Writer, f log.Fields) error { fmt.Fprintf(w, "┢"+strings.Repeat("━", colWidth*2+2)+"┪\n") } - firstRow := RightPad(fmt.Sprintf("#%d - %s", rID, startTime.Format(time.RFC822)), colWidth*2) + firstRow := util.RightPad(fmt.Sprintf("#%d - %s", rID, startTime.Format(time.RFC822)), colWidth*2) fmt.Fprintf(w, "┃ "+firstRow+" ┃\n") fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n") summary := makeSummary(name, f.Get("summary").(string)) fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", - RightPad(name, colWidth), - RightPad(summary[0], colWidth))) + util.RightPad(name, colWidth), + util.RightPad(summary[0], colWidth))) fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", - RightPad(networkName, colWidth), - RightPad(summary[1], colWidth))) + util.RightPad(networkName, colWidth), + util.RightPad(summary[1], colWidth))) fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", - RightPad(asn, colWidth), - RightPad(summary[2], colWidth))) + util.RightPad(asn, colWidth), + util.RightPad(summary[2], colWidth))) if index == totalCount-1 { fmt.Fprintf(w, "└┬──────────────┬──────────────┬──────────────┬") @@ -145,3 +117,25 @@ func logResultItem(w io.Writer, f log.Fields) error { } return nil } + +func logResultSummary(w io.Writer, f log.Fields) error { + + networks := f.Get("total_networks").(int64) + tests := f.Get("total_tests").(int64) + dataUp := f.Get("total_data_usage_up").(int64) + dataDown := f.Get("total_data_usage_down").(int64) + if tests == 0 { + fmt.Fprintf(w, "No results\n") + fmt.Fprintf(w, "Try running:\n") + fmt.Fprintf(w, " ooni run websites\n") + return nil + } + // └┬──────────────┬──────────────┬──────────────┬ + fmt.Fprintf(w, " │ %s │ %s │ %s │\n", + util.RightPad(fmt.Sprintf("%d tests", tests), 12), + util.RightPad(fmt.Sprintf("%d nets", networks), 12), + util.RightPad(fmt.Sprintf("%d ⬆ %d ⬇", dataUp, dataDown), 12)) + fmt.Fprintf(w, " └──────────────┴──────────────┴──────────────┘\n") + + return nil +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go new file mode 100644 index 0000000..f34a9ff --- /dev/null +++ b/internal/onboard/onboard.go @@ -0,0 +1,141 @@ +package onboard + +import ( + "fmt" + + "github.com/apex/log" + "github.com/fatih/color" + "github.com/ooni/probe-cli/config" + "github.com/ooni/probe-cli/internal/output" + survey "gopkg.in/AlecAivazis/survey.v1" +) + +func Onboarding(config *config.Config) error { + output.SectionTitle("What is OONI Probe?") + + fmt.Println() + output.Paragraph("Your tool for detecting internet censorship!") + fmt.Println() + output.Paragraph("OONI Probe checks whether your provider blocks access to sites and services. Run OONI Probe to collect evidence of internet censorship and to measure your network performance.") + fmt.Println() + output.PressEnterToContinue("Press 'Enter' to continue...") + + output.SectionTitle("Heads Up") + fmt.Println() + output.Bullet("Anyone monitoring your internet activity (such as your government or ISP) may be able to see that you are running OONI Probe.") + fmt.Println() + output.Bullet("The network data you will collect will automatically be published (unless you opt-out in the settings).") + fmt.Println() + output.Bullet("You may test objectionable sites.") + fmt.Println() + output.Bullet("Read the documentation to learn more.") + fmt.Println() + output.PressEnterToContinue("Press 'Enter' to continue...") + + output.SectionTitle("Pop Quiz!") + output.Paragraph("") + answer := "" + quiz1 := &survey.Select{ + Message: "Anyone monitoring my internet activity may be able to see that I am running OONI Probe.", + Options: []string{"true", "false"}, + Default: "true", + } + survey.AskOne(quiz1, &answer, nil) + if answer != "true" { + output.Paragraph(color.RedString("Actually...")) + output.Paragraph("OONI Probe is not a privacy tool. Therefore, anyone monitoring your internet activity may be able to see which software you are running.") + } else { + output.Paragraph(color.BlueString("Good job!")) + } + answer = "" + quiz2 := &survey.Select{ + Message: "The network data I will collect will automatically be published (unless I opt-out in the settings).", + Options: []string{"true", "false"}, + Default: "true", + } + survey.AskOne(quiz2, &answer, nil) + if answer != "true" { + output.Paragraph(color.RedString("Actually...")) + output.Paragraph("The network data you will collect will automatically be published to increase transparency of internet censorship (unless you opt-out in the settings).") + } else { + output.Paragraph(color.BlueString("Well done!")) + } + + changeDefaults := false + prompt := &survey.Confirm{ + Message: "Do you want to change the default settings?", + Default: false, + } + survey.AskOne(prompt, &changeDefaults, nil) + + settings := struct { + IncludeIP bool + IncludeNetwork bool + IncludeCountry bool + UploadResults bool + SendCrashReports bool + }{} + settings.IncludeIP = false + settings.IncludeNetwork = true + settings.IncludeCountry = true + settings.UploadResults = true + settings.SendCrashReports = true + + if changeDefaults == true { + var qs = []*survey.Question{ + { + Name: "IncludeIP", + Prompt: &survey.Confirm{Message: "Should we include your IP?"}, + }, + { + Name: "IncludeNetwork", + Prompt: &survey.Confirm{ + Message: "Can we include your network name?", + Default: true, + }, + }, + { + Name: "IncludeCountry", + Prompt: &survey.Confirm{ + Message: "Can we include your country name?", + Default: true, + }, + }, + { + Name: "UploadResults", + Prompt: &survey.Confirm{ + Message: "Can we upload your results?", + Default: true, + }, + }, + { + Name: "SendCrashReports", + Prompt: &survey.Confirm{ + Message: "Can we send crash reports to OONI?", + Default: true, + }, + }, + } + + err := survey.Ask(qs, &settings) + if err != nil { + log.WithError(err).Error("there was an error in parsing your responses") + return err + } + } + + config.Lock() + config.InformedConsent = true + config.Advanced.IncludeCountry = settings.IncludeCountry + config.Advanced.SendCrashReports = settings.SendCrashReports + config.Sharing.IncludeIP = settings.IncludeIP + config.Sharing.IncludeASN = settings.IncludeNetwork + config.Sharing.UploadResults = settings.UploadResults + config.Unlock() + + if err := config.Write(); err != nil { + log.WithError(err).Error("failed to write config file") + return err + } + return nil +} diff --git a/internal/output/output.go b/internal/output/output.go index f658765..2758fd8 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -1,9 +1,13 @@ package output import ( + "bufio" + "fmt" + "os" "time" "github.com/apex/log" + "github.com/ooni/probe-cli/internal/util" ) // Progress logs a progress type event @@ -51,3 +55,43 @@ func ResultItem(result ResultItemData) { "total_count": result.TotalCount, }).Info("result item") } + +type ResultSummaryData struct { + TotalTests int64 + TotalDataUsageUp int64 + TotalDataUsageDown int64 + TotalNetworks int64 +} + +func ResultSummary(result ResultSummaryData) { + log.WithFields(log.Fields{ + "type": "result_summary", + "total_tests": result.TotalTests, + "total_data_usage_up": result.TotalDataUsageUp, + "total_data_usage_down": result.TotalDataUsageDown, + "total_networks": result.TotalNetworks, + }).Info("result summary") +} + +// SectionTitle is the title of a section +func SectionTitle(text string) { + log.WithFields(log.Fields{ + "type": "section_title", + "title": text, + }).Info(text) +} + +func Paragraph(text string) { + const width = 80 + fmt.Println(util.WrapString(text, width)) +} + +func Bullet(text string) { + const width = 80 + fmt.Printf("• %s\n", util.WrapString(text, width)) +} + +func PressEnterToContinue(text string) { + fmt.Print(text) + bufio.NewReader(os.Stdin).ReadBytes('\n') +} diff --git a/internal/util/util.go b/internal/util/util.go index 45358bb..c47d301 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -1,19 +1,110 @@ package util import ( + "bytes" "fmt" "os" + "regexp" + "strings" + "unicode" + "unicode/utf8" - "github.com/ooni/probe-cli/internal/colors" + "github.com/fatih/color" ) // Log outputs a log message. func Log(msg string, v ...interface{}) { - fmt.Printf(" %s\n", colors.Purple(fmt.Sprintf(msg, v...))) + fmt.Printf(" %s\n", color.CyanString(msg, v...)) } // Fatal error func Fatal(err error) { - fmt.Fprintf(os.Stderr, "\n %s %s\n\n", colors.Red("Error:"), err) + fmt.Fprintf(os.Stderr, "\n %s %s\n\n", color.RedString("Error:"), err) os.Exit(1) } + +// Finds the ansi escape sequences (like colors) +// Taken from: https://github.com/chalk/ansi-regex/blob/d9d806ecb45d899cf43408906a4440060c5c50e5/index.js +var ansiEscapes = regexp.MustCompile(`[\x1B\x9B][[\]()#;?]*` + + `(?:(?:(?:[a-zA-Z\d]*(?:;[a-zA-Z\\d]*)*)?\x07)` + + `|(?:(?:\d{1,4}(?:;\d{0,4})*)?[\dA-PRZcf-ntqry=><~]))`) + +func EscapeAwareRuneCountInString(s string) int { + n := utf8.RuneCountInString(s) + for _, sm := range ansiEscapes.FindAllString(s, -1) { + n -= utf8.RuneCountInString(sm) + } + return n +} + +func RightPad(str string, length int) string { + return str + strings.Repeat(" ", length-EscapeAwareRuneCountInString(str)) +} + +// WrapString wraps the given string within lim width in characters. +// +// Wrapping is currently naive and only happens at white-space. A future +// version of the library will implement smarter wrapping. This means that +// pathological cases can dramatically reach past the limit, such as a very +// long word. +// This is taken from: https://github.com/mitchellh/go-wordwrap/tree/f253961a26562056904822f2a52d4692347db1bd +func WrapString(s string, lim uint) string { + // Initialize a buffer with a slightly larger size to account for breaks + init := make([]byte, 0, len(s)) + buf := bytes.NewBuffer(init) + + var current uint + var wordBuf, spaceBuf bytes.Buffer + + for _, char := range s { + if char == '\n' { + if wordBuf.Len() == 0 { + if current+uint(spaceBuf.Len()) > lim { + current = 0 + } else { + current += uint(spaceBuf.Len()) + spaceBuf.WriteTo(buf) + } + spaceBuf.Reset() + } else { + current += uint(spaceBuf.Len() + wordBuf.Len()) + spaceBuf.WriteTo(buf) + spaceBuf.Reset() + wordBuf.WriteTo(buf) + wordBuf.Reset() + } + buf.WriteRune(char) + current = 0 + } else if unicode.IsSpace(char) { + if spaceBuf.Len() == 0 || wordBuf.Len() > 0 { + current += uint(spaceBuf.Len() + wordBuf.Len()) + spaceBuf.WriteTo(buf) + spaceBuf.Reset() + wordBuf.WriteTo(buf) + wordBuf.Reset() + } + + spaceBuf.WriteRune(char) + } else { + + wordBuf.WriteRune(char) + + if current+uint(spaceBuf.Len()+wordBuf.Len()) > lim && uint(wordBuf.Len()) < lim { + buf.WriteRune('\n') + current = 0 + spaceBuf.Reset() + } + } + } + + if wordBuf.Len() == 0 { + if current+uint(spaceBuf.Len()) <= lim { + spaceBuf.WriteTo(buf) + } + } else { + spaceBuf.WriteTo(buf) + wordBuf.WriteTo(buf) + } + + return buf.String() +} diff --git a/internal/util/util_test.go b/internal/util/util_test.go new file mode 100644 index 0000000..3a567c7 --- /dev/null +++ b/internal/util/util_test.go @@ -0,0 +1,18 @@ +package util + +import ( + "testing" + + "github.com/fatih/color" +) + +func TestEscapeAwareRuneCountInString(t *testing.T) { + var bold = color.New(color.Bold) + var myColor = color.New(color.FgBlue) + + s := myColor.Sprintf("•ABC%s%s", bold.Sprintf("DEF"), "\x1B[00;38;5;244m\x1B[m\x1B[00;38;5;33mGHI\x1B[0m") + count := EscapeAwareRuneCountInString(s) + if count != 10 { + t.Errorf("Count was incorrect, got: %d, want: %d.", count, 10) + } +} diff --git a/nettests/groups/groups.go b/nettests/groups/groups.go index b35e04d..8b68dac 100644 --- a/nettests/groups/groups.go +++ b/nettests/groups/groups.go @@ -2,14 +2,13 @@ package groups import ( "encoding/json" - "fmt" "github.com/apex/log" - "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/nettests" "github.com/ooni/probe-cli/nettests/im" "github.com/ooni/probe-cli/nettests/middlebox" "github.com/ooni/probe-cli/nettests/performance" + "github.com/ooni/probe-cli/nettests/summary" "github.com/ooni/probe-cli/nettests/websites" ) @@ -17,42 +16,7 @@ import ( type NettestGroup struct { Label string Nettests []nettests.Nettest - Summary database.ResultSummaryFunc -} - -// PerformanceSummary is the result summary for a performance test -type PerformanceSummary struct { - Upload int64 - Download int64 - Ping float64 - Bitrate int64 -} - -// MiddleboxSummary is the summary for the middlebox tests -type MiddleboxSummary struct { - Detected bool -} - -// IMSummary is the summary for the im tests -type IMSummary struct { - Tested uint - Blocked uint -} - -// WebsitesSummary is the summary for the websites test -type WebsitesSummary struct { - Tested uint - Blocked uint -} - -func checkRequiredKeys(rk []string, m database.SummaryMap) error { - for _, key := range rk { - if _, ok := m[key]; ok { - continue - } - return fmt.Errorf("missing SummaryMap key '%s'", key) - } - return nil + Summary summary.ResultSummaryFunc } // NettestGroups that can be run by the user @@ -62,14 +26,14 @@ var NettestGroups = map[string]NettestGroup{ Nettests: []nettests.Nettest{ websites.WebConnectivity{}, }, - Summary: func(m database.SummaryMap) (string, error) { - if err := checkRequiredKeys([]string{"WebConnectivity"}, m); err != nil { + Summary: func(m summary.SummaryMap) (string, error) { + if err := summary.CheckRequiredKeys([]string{"WebConnectivity"}, m); err != nil { log.WithError(err).Error("missing keys") return "", err } // XXX to generate this I need to create the summary map as a list - var summary WebsitesSummary + var summary summary.WebsitesSummary summary.Tested = 0 summary.Blocked = 0 for _, msmtSummaryStr := range m["WebConnectivity"] { @@ -98,8 +62,8 @@ var NettestGroups = map[string]NettestGroup{ performance.Dash{}, performance.NDT{}, }, - Summary: func(m database.SummaryMap) (string, error) { - if err := checkRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil { + Summary: func(m summary.SummaryMap) (string, error) { + if err := summary.CheckRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil { log.WithError(err).Error("missing keys") return "", err } @@ -108,7 +72,7 @@ var NettestGroups = map[string]NettestGroup{ err error ndtSummary performance.NDTSummary dashSummary performance.DashSummary - summary PerformanceSummary + summary summary.PerformanceSummary ) err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary) if err != nil { @@ -137,8 +101,8 @@ var NettestGroups = map[string]NettestGroup{ middlebox.HTTPInvalidRequestLine{}, middlebox.HTTPHeaderFieldManipulation{}, }, - Summary: func(m database.SummaryMap) (string, error) { - if err := checkRequiredKeys([]string{"WebConnectivity"}, m); err != nil { + Summary: func(m summary.SummaryMap) (string, error) { + if err := summary.CheckRequiredKeys([]string{"HttpInvalidRequestLine", "HttpHeaderFieldManipulation"}, m); err != nil { log.WithError(err).Error("missing keys") return "", err } @@ -147,7 +111,7 @@ var NettestGroups = map[string]NettestGroup{ err error hhfmSummary middlebox.HTTPHeaderFieldManipulationSummary hirlSummary middlebox.HTTPInvalidRequestLineSummary - summary MiddleboxSummary + summary summary.MiddleboxSummary ) err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary) if err != nil { @@ -174,8 +138,8 @@ var NettestGroups = map[string]NettestGroup{ im.Telegram{}, im.WhatsApp{}, }, - Summary: func(m database.SummaryMap) (string, error) { - if err := checkRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil { + Summary: func(m summary.SummaryMap) (string, error) { + if err := summary.CheckRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil { log.WithError(err).Error("missing keys") return "", err } @@ -184,7 +148,7 @@ var NettestGroups = map[string]NettestGroup{ waSummary im.WhatsAppSummary tgSummary im.TelegramSummary fbSummary im.FacebookMessengerSummary - summary IMSummary + summary summary.IMSummary ) err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary) if err != nil { diff --git a/nettests/nettests.go b/nettests/nettests.go index a944c44..0243515 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -7,11 +7,9 @@ import ( "path/filepath" "github.com/apex/log" + "github.com/fatih/color" "github.com/measurement-kit/go-measurement-kit" - homedir "github.com/mitchellh/go-homedir" ooni "github.com/ooni/probe-cli" - "github.com/ooni/probe-cli/internal/cli/version" - "github.com/ooni/probe-cli/internal/colors" "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/internal/output" "github.com/ooni/probe-cli/utils" @@ -76,11 +74,13 @@ func (c *Controller) Init(nt *mk.Nettest) error { caBundlePath := getCaBundlePath() msmtPath := c.msmtPath - userHome, err := homedir.Dir() + userHome, err := utils.GetOONIHome() if err != nil { log.WithError(err).Error("failed to figure out the homedir") return err } + // Get the parent of it + userHome = filepath.Dir(userHome) relPath, err := filepath.Rel(userHome, caBundlePath) if err != nil { @@ -121,7 +121,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { DisableReportFile: false, DisableCollector: false, SoftwareName: "ooniprobe", - SoftwareVersion: version.Version, + SoftwareVersion: ooni.Version, OutputPath: msmtPath, GeoIPCountryPath: geoIPCountryPath, @@ -132,6 +132,8 @@ func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath) nt.On("log", func(e mk.Event) { + log.Debugf(color.RedString(e.Key)) + level := e.Value.LogLevel msg := e.Value.Message @@ -161,7 +163,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { }) nt.On("status.geoip_lookup", func(e mk.Event) { - log.Debugf(colors.Red(e.Key)) + log.Debugf(color.RedString(e.Key)) msmtTemplate.ASN = e.Value.ProbeASN msmtTemplate.IP = e.Value.ProbeIP @@ -169,7 +171,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { }) nt.On("status.measurement_start", func(e mk.Event) { - log.Debugf(colors.Red(e.Key)) + log.Debugf(color.RedString(e.Key)) idx := e.Value.Idx msmt, err := database.CreateMeasurement(c.Ctx.DB, msmtTemplate, e.Value.Input) @@ -181,29 +183,64 @@ func (c *Controller) Init(nt *mk.Nettest) error { }) nt.On("status.progress", func(e mk.Event) { - log.Debugf(colors.Red(e.Key)) + log.Debugf(color.RedString(e.Key)) c.OnProgress(e.Value.Percentage, e.Value.Message) }) nt.On("status.update.*", func(e mk.Event) { - log.Debugf(colors.Red(e.Key)) + log.Debugf(color.RedString(e.Key)) + }) + + // XXX should these be made into permanent failures? + nt.On("failure.asn_lookup", func(e mk.Event) { + log.Debugf(color.RedString(e.Key)) + log.Debugf("%v", e.Value) + }) + nt.On("failure.cc_lookup", func(e mk.Event) { + log.Debugf(color.RedString(e.Key)) + log.Debugf("%v", e.Value) + }) + nt.On("failure.ip_lookup", func(e mk.Event) { + log.Debugf(color.RedString(e.Key)) + log.Debugf("%v", e.Value) + }) + + nt.On("failure.resolver_lookup", func(e mk.Event) { + log.Debugf(color.RedString(e.Key)) + log.Debugf("%v", e.Value) + }) + + nt.On("failure.report_create", func(e mk.Event) { + log.Debugf(color.RedString(e.Key)) + log.Debugf("%v", e.Value) + }) + + nt.On("failure.report_close", func(e mk.Event) { + log.Debugf(color.RedString(e.Key)) + log.Debugf("%v", e.Value) + }) + + nt.On("failure.startup", func(e mk.Event) { + log.Debugf(color.RedString(e.Key)) + + c.msmts[e.Value.Idx].Failed(c.Ctx.DB, e.Value.Failure) }) nt.On("failure.measurement", func(e mk.Event) { - log.Debugf(colors.Red(e.Key)) + log.Debugf(color.RedString(e.Key)) c.msmts[e.Value.Idx].Failed(c.Ctx.DB, e.Value.Failure) }) nt.On("failure.measurement_submission", func(e mk.Event) { - log.Debugf(colors.Red(e.Key)) + log.Debugf(color.RedString(e.Key)) failure := e.Value.Failure c.msmts[e.Value.Idx].UploadFailed(c.Ctx.DB, failure) }) nt.On("status.measurement_submission", func(e mk.Event) { - log.Debugf(colors.Red(e.Key)) + log.Debugf(color.RedString(e.Key)) if err := c.msmts[e.Value.Idx].UploadSucceeded(c.Ctx.DB); err != nil { log.WithError(err).Error("failed to mark msmt as uploaded") @@ -211,7 +248,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { }) nt.On("status.measurement_done", func(e mk.Event) { - log.Debugf(colors.Red(e.Key)) + log.Debugf(color.RedString(e.Key)) if err := c.msmts[e.Value.Idx].Done(c.Ctx.DB); err != nil { log.WithError(err).Error("failed to mark msmt as done") @@ -219,6 +256,8 @@ func (c *Controller) Init(nt *mk.Nettest) error { }) nt.On("measurement", func(e mk.Event) { + log.Debugf("status.end") + c.OnEntry(e.Value.Idx, e.Value.JSONStr) }) diff --git a/nettests/summary/summary.go b/nettests/summary/summary.go new file mode 100644 index 0000000..f225927 --- /dev/null +++ b/nettests/summary/summary.go @@ -0,0 +1,44 @@ +package summary + +import "fmt" + +// ResultSummaryFunc is the function used to generate result summaries +type ResultSummaryFunc func(SummaryMap) (string, error) + +// SummaryMap contains a mapping from test name to serialized summary for it +type SummaryMap map[string][]string + +// PerformanceSummary is the result summary for a performance test +type PerformanceSummary struct { + Upload int64 + Download int64 + Ping float64 + Bitrate int64 +} + +// MiddleboxSummary is the summary for the middlebox tests +type MiddleboxSummary struct { + Detected bool +} + +// IMSummary is the summary for the im tests +type IMSummary struct { + Tested uint + Blocked uint +} + +// WebsitesSummary is the summary for the websites test +type WebsitesSummary struct { + Tested uint + Blocked uint +} + +func CheckRequiredKeys(rk []string, m SummaryMap) error { + for _, key := range rk { + if _, ok := m[key]; ok { + continue + } + return fmt.Errorf("missing SummaryMap key '%s'", key) + } + return nil +} diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index ed81742..d149207 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -29,6 +29,7 @@ func lookupURLs(ctl *nettests.Controller) ([]string, error) { parsed = new(URLResponse) urls []string ) + // XXX pass in the configuration for category codes reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s", orchestrateBaseURL, ctl.Ctx.Location.CountryCode) diff --git a/ooni.go b/ooni.go index 051e051..b17c57c 100644 --- a/ooni.go +++ b/ooni.go @@ -1,41 +1,26 @@ package ooni import ( - "encoding/json" "io/ioutil" "os" - "path/filepath" - "sync" + "path" "github.com/apex/log" "github.com/jmoiron/sqlx" "github.com/ooni/probe-cli/config" + "github.com/ooni/probe-cli/internal/bindata" "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/internal/legacy" + "github.com/ooni/probe-cli/internal/onboard" "github.com/ooni/probe-cli/utils" "github.com/pkg/errors" ) -// Onboarding process -func Onboarding(c *Config) error { - log.Info("Onboarding starting") - - // To prevent races we always must acquire the config file lock before - // changing it. - c.Lock() - c.InformedConsent = true - c.Unlock() - - if err := c.Write(); err != nil { - log.Warnf("Failed to save informed consent: %v", err) - return err - } - return nil -} +const Version = "3.0.0-dev.0" // Context for OONI Probe type Context struct { - Config *Config + Config *config.Config DB *sqlx.DB Location *utils.LocationInfo @@ -58,8 +43,11 @@ func (c *Context) MaybeLocationLookup() error { func (c *Context) LocationLookup() error { var err error - geoipDir := utils.GeoIPDir(c.Home) + if err = c.MaybeDownloadDataFiles(); err != nil { + log.WithError(err).Error("failed to download data files") + } + geoipDir := utils.GeoIPDir(c.Home) c.Location, err = utils.GeoIPLookup(geoipDir) if err != nil { return err @@ -68,6 +56,35 @@ func (c *Context) LocationLookup() error { return nil } +// MaybeOnboarding will run the onboarding process only if the informed consent +// config option is set to false +func (c *Context) MaybeOnboarding() error { + if c.Config.InformedConsent == false { + if err := onboard.Onboarding(c.Config); err != nil { + return errors.Wrap(err, "onboarding") + } + } + return nil +} + +// MaybeDownloadDataFiles will download geoip data files if they are not present +func (c *Context) MaybeDownloadDataFiles() error { + geoipDir := utils.GeoIPDir(c.Home) + if _, err := os.Stat(path.Join(geoipDir, "GeoLite2-Country.mmdb")); os.IsNotExist(err) { + log.Debugf("Downloading GeoIP database files") + if err := utils.DownloadGeoIPDatabaseFiles(geoipDir); err != nil { + return err + } + } + if _, err := os.Stat(path.Join(geoipDir, "GeoIP.dat")); os.IsNotExist(err) { + log.Debugf("Downloading legacy GeoIP database Files") + if err := utils.DownloadLegacyGeoIPDatabaseFiles(geoipDir); err != nil { + return err + } + } + return nil +} + // Init the OONI manager func (c *Context) Init() error { var err error @@ -82,22 +99,16 @@ func (c *Context) Init() error { if c.configPath != "" { log.Debugf("Reading config file from %s", c.configPath) - c.Config, err = ReadConfig(c.configPath) + c.Config, err = config.ReadConfig(c.configPath) } else { log.Debug("Reading default config file") - c.Config, err = ReadDefaultConfigPaths(c.Home) + c.Config, err = InitDefaultConfig(c.Home) } if err != nil { return err } c.dbPath = utils.DBDir(c.Home, "main") - if c.Config.InformedConsent == false { - if err = Onboarding(c.Config); err != nil { - return errors.Wrap(err, "onboarding") - } - } - log.Debugf("Connecting to database sqlite3://%s", c.dbPath) db, err := database.Connect(c.dbPath) if err != nil { @@ -118,158 +129,53 @@ func (c *Context) Init() error { func NewContext(configPath string, homePath string) *Context { return &Context{ Home: homePath, - Config: &Config{}, + Config: &config.Config{}, configPath: configPath, } } -// Config for the OONI Probe installation -type Config struct { - // Private settings - Comment string `json:"_"` - ConfigVersion string `json:"_config_version"` - InformedConsent bool `json:"_informed_consent"` - - AutoUpdate bool `json:"auto_update"` - Sharing config.Sharing `json:"sharing"` - Notifications config.Notifications `json:"notifications"` - AutomatedTesting config.AutomatedTesting `json:"automated_testing"` - NettestGroups config.NettestGroups `json:"test_settings"` - Advanced config.Advanced `json:"advanced"` - - mutex sync.Mutex - path string -} - -// Write the config file in json to the path -func (c *Config) Write() error { - c.Lock() - configJSON, _ := json.MarshalIndent(c, "", " ") - if c.path == "" { - return errors.New("config file path is empty") - } - if err := ioutil.WriteFile(c.path, configJSON, 0644); err != nil { - return errors.Wrap(err, "writing config JSON") - } - c.Unlock() - return nil -} - -// Lock acquires the write mutex -func (c *Config) Lock() { - c.mutex.Lock() -} - -// Unlock releases the write mutex -func (c *Config) Unlock() { - c.mutex.Unlock() -} - -// Default config settings -func (c *Config) Default() error { - home, err := utils.GetOONIHome() - if err != nil { - return err - } - - c.path = filepath.Join(home, "config.json") - return nil -} - -// Validate the config file -func (c *Config) Validate() error { - return nil -} - -// ParseConfig returns config from JSON bytes. -func ParseConfig(b []byte) (*Config, error) { - c := &Config{} - - if err := json.Unmarshal(b, c); err != nil { - return nil, errors.Wrap(err, "parsing json") - } - - if err := c.Default(); err != nil { - return nil, errors.Wrap(err, "defaulting") - } - - if err := c.Validate(); err != nil { - return nil, errors.Wrap(err, "validating") - } - - return c, nil -} - // MaybeInitializeHome does the setup for a new OONI Home func MaybeInitializeHome(home string) error { - firstRun := false for _, d := range utils.RequiredDirs(home) { if _, e := os.Stat(d); e != nil { - firstRun = true if err := os.MkdirAll(d, 0700); err != nil { return err } } } - if firstRun == true { - log.Info("This is the first time you are running OONI Probe. Downloading some files.") - geoipDir := utils.GeoIPDir(home) - if err := utils.DownloadGeoIPDatabaseFiles(geoipDir); err != nil { - return err - } - if err := utils.DownloadLegacyGeoIPDatabaseFiles(geoipDir); err != nil { - return err - } - } - return nil } -// ReadConfig reads the configuration from the path -func ReadConfig(path string) (*Config, error) { - b, err := ioutil.ReadFile(path) - - if os.IsNotExist(err) { - c := &Config{} - - if err = c.Default(); err != nil { - return nil, errors.Wrap(err, "defaulting") - } - - if err = c.Validate(); err != nil { - return nil, errors.Wrap(err, "validating") - } - - return c, nil - } +// InitDefaultConfig reads the config from common locations or creates it if +// missing. +func InitDefaultConfig(home string) (*config.Config, error) { + var ( + err error + c *config.Config + configPath = utils.ConfigPath(home) + ) + c, err = config.ReadConfig(configPath) if err != nil { - return nil, errors.Wrap(err, "reading file") - } - - c, err := ParseConfig(b) - if err != nil { - return nil, errors.Wrap(err, "parsing config") - } - c.path = path - return c, err -} - -// ReadDefaultConfigPaths from common locations. -func ReadDefaultConfigPaths(home string) (*Config, error) { - var paths = []string{ - filepath.Join(home, "config.json"), - } - for _, path := range paths { - if _, err := os.Stat(path); err == nil { - c, err := ReadConfig(path) + if os.IsNotExist(err) { + log.Debugf("writing default config to %s", configPath) + var data []byte + data, err = bindata.Asset("data/default-config.json") if err != nil { return nil, err } - return c, nil + err = ioutil.WriteFile( + configPath, + data, + 0644, + ) + if err != nil { + return nil, err + } + return InitDefaultConfig(home) } + return nil, err } - // Run from the default config - return ReadConfig(paths[0]) + return c, nil } diff --git a/utils/paths.go b/utils/paths.go index 63ce079..c4be216 100644 --- a/utils/paths.go +++ b/utils/paths.go @@ -20,6 +20,11 @@ func RequiredDirs(home string) []string { return requiredDirs } +// ConfigPath returns the default path to the config file +func ConfigPath(home string) string { + return filepath.Join(home, "config.json") +} + // GeoIPDir returns the geoip data dir for the given OONI Home func GeoIPDir(home string) string { return filepath.Join(home, "geoip")