diff --git a/Gopkg.lock b/Gopkg.lock index 56b86e1..f4938e4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -38,19 +38,19 @@ branch = "master" name = "github.com/beorn7/perks" packages = ["quantile"] - revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] name = "github.com/fatih/color" packages = ["."] - revision = "507f6050b8568533fb3f5504de8e5205fa62a114" - version = "v1.6.0" + revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" + version = "v1.7.0" [[projects]] name = "github.com/golang/protobuf" packages = ["proto"] - revision = "925541529c1fa6821df4e44ce2723319eb2be768" - version = "v1.0.0" + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" [[projects]] branch = "master" @@ -59,7 +59,7 @@ ".", "reflectx" ] - revision = "05cef0741ade10ca668982355b3f3f0bcf0ff0a8" + revision = "2aeb6a910c2b94f2d5eb53d9895d80e27264ec41" [[projects]] name = "github.com/mattn/go-colorable" @@ -76,20 +76,20 @@ [[projects]] name = "github.com/mattn/go-sqlite3" packages = ["."] - revision = "6c771bb9887719704b210e87e934f08be014bdb1" - version = "v1.6.0" + revision = "323a32be5a2421b8c7087225079c6c900ec397cd" + version = "v1.7.0" [[projects]] name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] - revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" - version = "v1.0.0" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" [[projects]] branch = "master" name = "github.com/measurement-kit/go-measurement-kit" packages = ["."] - revision = "cbf1c976aeaa2906f4fda278fc3068f7bde63c47" + revision = "4fe2e61c300930aedc10713557b6e05f29631fc0" [[projects]] branch = "master" @@ -101,7 +101,7 @@ branch = "master" name = "github.com/mitchellh/go-homedir" packages = ["."] - revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" + revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66" [[projects]] name = "github.com/oschwald/geoip2-golang" @@ -142,7 +142,7 @@ "model", "version" ] - revision = "89604d197083d4781071d3c65855d24ecfb0a563" + revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" [[projects]] branch = "master" @@ -153,7 +153,7 @@ "nfs", "xfs" ] - revision = "282c8707aa210456a825798969cc27edda34992a" + revision = "fe93d378a6b03758a2c1b65e86cf630bf78681c0" [[projects]] branch = "master" @@ -162,7 +162,7 @@ ".", "sqlparse" ] - revision = "f33734611e84d5fe45f35eccf0174f4836af4542" + revision = "081fe17d19ff4e2dd9f5a0c1158e6bcf74da6906" [[projects]] name = "github.com/shuLhan/go-bindata" @@ -180,7 +180,7 @@ "unix", "windows" ] - revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" + revision = "c11f84a56e43e20a78cee75a7c034031ecf57d1f" [[projects]] name = "gopkg.in/AlecAivazis/survey.v1" @@ -189,8 +189,8 @@ "core", "terminal" ] - revision = "0aa8b6a162b391fe2d95648b7677d1d6ac2090a6" - version = "v1.4.1" + revision = "e752db451e07e09c7d7dc8cada807a44bdb0fd47" + version = "v1.5.3" [[projects]] name = "gopkg.in/gorp.v1" @@ -201,6 +201,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "46860a32f649dbb2e01b285c0d5581078ba4b28a21e21175d47ccb2725a9c9fb" + inputs-digest = "95c3e971d63b97b0dc531f67d98401cfa9968b99aacf1eed73ce801bbaadb0cd" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index a68ad73..7a941ae 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -42,10 +42,6 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"] name = "github.com/pkg/errors" version = "0.8.0" -[[constraint]] - branch = "master" - name = "github.com/mitchellh/go-homedir" - [[constraint]] name = "gopkg.in/AlecAivazis/survey.v1" version = "1.4.1" diff --git a/Makefile b/Makefile index 77309fe..4acf750 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,18 @@ GO ?= go build: - @echo "Building ./ooni" + @echo "Building dist/ooni" @$(GO) build -i -o dist/ooni cmd/ooni/main.go .PHONY: build -update-mk: - @echo "updating mk" - @dep ensure -update github.com/measurement-kit/go-measurement-kit -.PHONY: update-mk +build-windows: + @echo "Building dist/ooni.exe" + CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build -o dist/ooni.exe -x cmd/ooni/main.go + +update-mk-libs: + @echo "updating mk-libs" + @cd vendor/github.com/measurement-kit/go-measurement-kit && curl -L -o master.zip https://github.com/measurement-kit/golang-prebuilt/archive/master.zip && unzip master.zip && mv golang-prebuilt-master libs && rm master.zip # This is a hack to workaround: https://github.com/golang/dep/issues/1240 +.PHONY: update-mk-libs bindata: @$(GO) run vendor/github.com/shuLhan/go-bindata/go-bindata/*.go \ diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 6e65b0b..ebe2576 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -16,6 +16,9 @@ CREATE TABLE `results` ( `runtime` REAL, `summary` JSON, `done` TINYINT(1), + `country` VARCHAR(2), + `asn` VARCHAR(16), + `network_name` VARCHAR(255), `data_usage_up` INTEGER, `data_usage_down` INTEGER ); diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index 45ee45f..1935962 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -130,20 +130,20 @@ func bindataDataDefaultconfigjson() (*asset, error) { } var _bindataDataMigrations1createmsmtresultssql = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x93\x31\xef\xda\x30\x10\xc5\xf7\x7c\x8a\x1b\x41\x2d\x03\x95\xe8\xc2" + - "\x64\x92\x6b\x9b\x36\x38\xc8\x71\xaa\x32\x25\x56\x63\x90\xd5\xc4\x89\x1c\x5b\xa8\xdf\xbe\x32\x24\x14\x68\xa0\xeb" + - "\x7f\x7d\xbf\xbb\x67\x9f\xdf\x79\xb1\x80\x77\x8d\x3a\x1a\x61\x25\x44\xed\x49\x07\xb7\x42\x66\x85\x95\x8d\xd4\x76" + - "\x23\x8f\x4a\x07\x41\xc4\xd2\x1d\x70\xb2\x49\x10\x4a\x23\x7b\x57\xdb\xbe\x5c\xdf\xa9\x8d\x14\xbd\x33\xe7\x1e\x8f" + - "\xa6\xdd\x50\x57\xf7\x24\xef\x5e\x1e\x1b\x32\x24\x1c\x1f\x0f\x86\x59\x00\x00\x50\xaa\xaa\x84\x98\x72\xfc\x8c\x0c" + - "\x76\x2c\xde\x12\xb6\x87\x6f\xb8\x07\x92\xf3\x34\xa6\x21\xc3\x2d\x52\xfe\xfe\x52\xab\x45\x23\x4b\xf8\x4e\x58\xf8" + - "\x85\xb0\xd9\x87\xd5\x6a\x3e\x80\xde\x0a\x63\x0b\xab\x3c\x8e\x08\x47\x1e\x6f\x71\x40\xc6\xe9\x8b\xce\x90\x24\x63" + - "\xb9\x6b\x1a\x61\x7e\x97\xf0\x35\x4b\xe9\xa0\x55\xad\x96\x25\xf0\x98\xee\x63\xca\x67\xcb\xd1\xb9\x12\x56\x14\xae" + - "\x17\x47\x59\xb8\xee\x7a\xd3\x7f\x61\xd5\x9e\xf4\x15\x07\xf3\xf5\xe3\xd8\x77\x2f\xfb\xd6\x66\x57\xdd\xa4\xb3\xe8" + - "\xf5\x5f\x7d\xf9\x71\x94\x7f\xb6\x4e\x5b\xef\x70\x6d\x19\x89\x96\xf6\xd4\x9a\x5f\xc5\xab\xbb\x5a\xff\xca\xf8\x63" + - "\x1c\xeb\x20\x54\xed\xcc\x74\xb5\xeb\xea\x56\x54\xc5\xff\x4b\x64\x35\x91\x9c\x91\x5d\x6b\x6c\x71\x50\xf5\x74\xeb" + - "\xc0\x7d\x06\x13\x54\xe9\xce\xd9\x27\x7d\x7e\x81\x8b\xdb\xec\x18\x7e\x42\x86\x34\xc4\xec\x76\xbf\x7d\xbc\x73\x48" + - "\x29\x44\x98\x20\x47\xc8\x90\x03\xcd\x93\xc4\x4b\xf9\xce\x47\x05\x21\xc9\x42\x12\xe1\x79\x5f\x9e\xfe\xb5\x3f\x01" + - "\x00\x00\xff\xff\x29\xbd\x69\x4f\xe4\x03\x00\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x93\xcf\x8e\x9b\x30\x10\xc6\xef\x3c\xc5\x1c\x13\xb5\x7b\xd8\x4a\xdb" + + "\xcb\x9e\xbc\x30\x6d\x69\x89\x89\x8c\xa9\x9a\x13\xb6\x8a\x37\xb2\x0a\x06\x19\x5b\x51\xdf\xbe\x72\x02\x69\x92\x42" + + "\x73\xdd\xeb\xf7\x9b\x3f\xf6\x7c\x33\x0f\x0f\xf0\xae\xd5\x7b\x2b\x9d\x82\xa4\x3b\x98\xe8\x52\x28\x9c\x74\xaa\x55" + + "\xc6\xbd\xa8\xbd\x36\x51\x94\xb0\x7c\x0b\x9c\xbc\x64\x08\xc2\xaa\xc1\x37\x6e\x10\xcf\x57\x6a\xab\xe4\xe0\xed\x31" + + "\x27\xa0\xf9\x6a\x68\xea\x6b\x52\xf6\xff\x6d\x1b\x33\x24\x1c\x6f\x1b\xc3\x2a\x02\x00\x10\xba\x16\x90\x52\x8e\x9f" + + "\x91\xc1\x96\xa5\x1b\xc2\x76\xf0\x0d\x77\x40\x4a\x9e\xa7\x34\x66\xb8\x41\xca\xdf\x9f\x62\x8d\x6c\x95\x80\xef\x84" + + "\xc5\x5f\x08\x5b\x7d\x78\x7a\x5a\x8f\x60\x70\xd2\xba\xca\xe9\x80\x13\xc2\x91\xa7\x1b\x1c\x91\xf5\xe6\xa4\x33\x24" + + "\xd9\x14\xee\xdb\x56\xda\xdf\x02\xbe\x16\x39\x1d\xb5\xba\x33\x4a\x00\x4f\xe9\x2e\xa5\x7c\xf5\x38\x55\xfe\xd9\x79" + + "\xe3\x42\xe8\xb9\xeb\x44\xe4\x60\xfe\xaa\x8f\x1f\x27\xd9\x28\x77\xe8\xec\xaf\x6a\xf1\xad\xb5\x74\xb2\xf2\x83\xdc" + + "\xab\xca\xf7\xe7\xbf\xff\x0b\xeb\xee\x60\xce\x38\x5a\x3f\xdf\x0e\xf2\xca\xab\xb7\x36\x4d\xdd\xcf\x56\x5e\x98\xd9" + + "\xf2\x90\xef\x4e\x73\x08\xcb\x26\x80\xe3\x8f\xe9\x5b\xaf\x52\x37\xde\xce\x47\xfb\xbe\xe9\x64\x5d\xdd\x0f\x51\xf5" + + "\xcc\x2e\x58\xd5\x77\xd6\x55\xaf\xba\x99\x4f\x1d\x79\xf0\x60\x86\x6a\xd3\x7b\xb7\x90\x17\x4e\xa2\xba\xf4\x8e\xe1" + + "\x27\x64\x48\x63\x2c\x2e\x2f\x26\xd8\xbb\x86\x9c\x42\x82\x19\x72\x84\x02\x39\xd0\x32\xcb\x82\x54\x6e\x83\x55\x10" + + "\x93\x22\x26\x09\x1e\xf7\x65\xf1\x7a\xff\x04\x00\x00\xff\xff\xdf\xf0\xa4\xca\x36\x04\x00\x00") func bindataDataMigrations1createmsmtresultssqlBytes() ([]byte, error) { return bindataRead( diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index f8ed972..80e1395 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -4,13 +4,59 @@ import ( "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/ooni/probe-cli/internal/cli/root" + "github.com/ooni/probe-cli/internal/database" + "github.com/ooni/probe-cli/internal/output" ) func init() { - cmd := root.Command("list", "List measurements") + cmd := root.Command("list", "List results") cmd.Action(func(_ *kingpin.ParseContext) error { - log.Info("Listing") + ctx, err := root.Init() + if err != nil { + log.WithError(err).Error("failed to initialize root context") + return err + } + doneResults, incompleteResults, err := database.ListResults(ctx.DB) + if err != nil { + log.WithError(err).Error("failed to list results") + return err + } + + log.Info("Results") + for idx, result := range doneResults { + output.ResultItem(output.ResultItemData{ + ID: result.ID, + Index: idx, + TotalCount: len(doneResults), + Name: result.Name, + StartTime: result.StartTime, + NetworkName: result.NetworkName, + Country: result.Country, + ASN: result.ASN, + Summary: result.Summary, + Done: result.Done, + DataUsageUp: result.DataUsageUp, + DataUsageDown: result.DataUsageDown, + }) + } + log.Info("Incomplete results") + for idx, result := range incompleteResults { + output.ResultItem(output.ResultItemData{ + ID: result.ID, + Index: idx, + TotalCount: len(incompleteResults), + Name: result.Name, + StartTime: result.StartTime, + NetworkName: result.NetworkName, + Country: result.Country, + ASN: result.ASN, + Summary: result.Summary, + Done: result.Done, + DataUsageUp: result.DataUsageUp, + DataUsageDown: result.DataUsageDown, + }) + } return nil }) } diff --git a/internal/cli/root/root.go b/internal/cli/root/root.go index 870c39d..5777f10 100644 --- a/internal/cli/root/root.go +++ b/internal/cli/root/root.go @@ -6,6 +6,7 @@ import ( ooni "github.com/ooni/probe-cli" "github.com/ooni/probe-cli/internal/log/handlers/batch" "github.com/ooni/probe-cli/internal/log/handlers/cli" + "github.com/ooni/probe-cli/utils" "github.com/prometheus/common/version" ) @@ -15,7 +16,7 @@ var Cmd = kingpin.New("ooni", "") // Command is syntax sugar for defining sub-commands var Command = Cmd.Command -// Init should be called by all subcommand that care to have a ooni.OONI instance +// Init should be called by all subcommand that care to have a ooni.Context instance var Init func() (*ooni.Context, error) func init() { @@ -38,7 +39,7 @@ func init() { Init = func() (*ooni.Context, error) { var err error - homePath, err := ooni.GetOONIHome() + homePath, err := utils.GetOONIHome() if err != nil { return nil, err } diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 0b0d87d..b6125bc 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -12,6 +12,7 @@ import ( "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/nettests" "github.com/ooni/probe-cli/nettests/groups" + "github.com/ooni/probe-cli/utils" ) func init() { @@ -40,8 +41,11 @@ func init() { } result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{ - Name: *nettestGroup, - StartTime: time.Now().UTC(), + Name: *nettestGroup, + StartTime: time.Now().UTC(), + Country: ctx.Location.CountryCode, + NetworkName: ctx.Location.NetworkName, + ASN: fmt.Sprintf("%d", ctx.Location.ASN), }) if err != nil { log.Errorf("DB result error: %s", err) @@ -52,7 +56,7 @@ func init() { log.Debugf("Running test %T", nt) msmtPath := filepath.Join(ctx.TempDir, fmt.Sprintf("msmt-%T-%s.jsonl", nt, - time.Now().UTC().Format(time.RFC3339Nano))) + time.Now().UTC().Format(utils.ResultTimestamp))) ctl := nettests.NewController(nt, ctx, result, msmtPath) if err = nt.Run(ctl); err != nil { diff --git a/internal/database/models.go b/internal/database/models.go index 0e5dfab..daff59b 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -190,6 +190,9 @@ type Result struct { ID int64 `db:"id"` Name string `db:"name"` StartTime time.Time `db:"start_time"` + Country string `db:"country"` + ASN string `db:"asn"` + NetworkName string `db:"network_name"` Runtime float64 `db:"runtime"` // Runtime is expressed in fractional seconds Summary string `db:"summary"` // XXX this should be JSON Done bool `db:"done"` @@ -198,6 +201,65 @@ type Result struct { MeasurementDir string `db:"measurement_dir"` } +// ListResults return the list of results +func ListResults(db *sqlx.DB) ([]*Result, []*Result, error) { + doneResults := []*Result{} + incompleteResults := []*Result{} + + rows, err := db.Query(`SELECT id, name, + start_time, runtime, + network_name, country, + asn, + summary, done + FROM results + WHERE done = 1 + ORDER BY start_time;`) + if err != nil { + return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") + } + + for rows.Next() { + result := Result{} + err = rows.Scan(&result.ID, &result.Name, + &result.StartTime, &result.Runtime, + &result.NetworkName, &result.Country, + &result.ASN, + &result.Summary, &result.Done, + //&result.DataUsageUp, &result.DataUsageDown) + ) + if err != nil { + log.WithError(err).Error("failed to fetch a row") + continue + } + doneResults = append(doneResults, &result) + } + + rows, err = db.Query(`SELECT + id, name, + start_time, + network_name, country, + asn + FROM results + WHERE done != 1 + ORDER BY start_time;`) + if err != nil { + return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") + } + + for rows.Next() { + result := Result{Done: false} + err = rows.Scan(&result.ID, &result.Name, &result.StartTime, + &result.NetworkName, &result.Country, + &result.ASN) + if err != nil { + log.WithError(err).Error("failed to fetch a row") + continue + } + incompleteResults = append(incompleteResults, &result) + } + return doneResults, incompleteResults, nil +} + // MakeSummaryMap return a mapping of test names to summaries for the given // result func MakeSummaryMap(db *sqlx.DB, r *Result) (SummaryMap, error) { @@ -258,8 +320,8 @@ func CreateResult(db *sqlx.DB, homePath string, r Result) (*Result, error) { } r.MeasurementDir = p res, err := db.NamedExec(`INSERT INTO results - (name, start_time) - VALUES (:name,:start_time)`, + (name, start_time, country, network_name, asn) + VALUES (:name,:start_time,:country,:network_name,:asn)`, r) if err != nil { return nil, errors.Wrap(err, "creating result") diff --git a/internal/legacy/legacy.go b/internal/legacy/legacy.go index 1877ddc..27dac4c 100644 --- a/internal/legacy/legacy.go +++ b/internal/legacy/legacy.go @@ -5,13 +5,13 @@ import ( "os" "path/filepath" - homedir "github.com/mitchellh/go-homedir" + "github.com/ooni/probe-cli/utils/homedir" "github.com/pkg/errors" "gopkg.in/AlecAivazis/survey.v1" ) // HomePath returns the path to the OONI Home -func HomePath() (string, error) { +func homePath() (string, error) { home, err := homedir.Dir() if err != nil { return "", err @@ -20,8 +20,11 @@ func HomePath() (string, error) { } // HomeExists returns true if a legacy home exists -func HomeExists() (bool, error) { - home, err := HomePath() +func homeExists() (bool, error) { + home, err := homePath() + if err == homedir.ErrNoHomeDir { + return false, nil + } if err != nil { return false, err } @@ -33,7 +36,7 @@ func HomeExists() (bool, error) { } // BackupHome the legacy home directory -func BackupHome() error { +func backupHome() error { home, err := homedir.Dir() if err != nil { return errors.Wrap(err, "backing up home") @@ -48,14 +51,14 @@ func BackupHome() error { // MaybeMigrateHome prompts the user if we should backup the legacy home func MaybeMigrateHome() error { - exists, err := HomeExists() + exists, err := homeExists() if err != nil { return err } if !exists { return nil } - home, err := HomePath() + home, err := homePath() if err != nil { return err } @@ -72,7 +75,7 @@ func MaybeMigrateHome() error { } } else { logf("Backing up ~/.ooni to ~/.ooni-legacy") - if err := BackupHome(); err != nil { + if err := backupHome(); err != nil { return err } } diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index 7e137ba..fbb04cd 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -68,6 +68,8 @@ func (h *Handler) TypedLog(t string, e *log.Entry) error { fmt.Fprintf(h.Writer, "%.1f%% [%s]: %s", e.Fields.Get("percentage").(float64)*100, e.Fields.Get("key"), e.Message) fmt.Fprintln(h.Writer) return nil + case "result_item": + return logResultItem(h.Writer, e.Fields) default: return h.DefaultLog(e) } diff --git a/internal/log/handlers/cli/result_item.go b/internal/log/handlers/cli/result_item.go new file mode 100644 index 0000000..1f3ec8e --- /dev/null +++ b/internal/log/handlers/cli/result_item.go @@ -0,0 +1,147 @@ +package cli + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "time" + + "github.com/apex/log" +) + +func RightPad(str string, length int) string { + return str + strings.Repeat(" ", length-len(str)) +} + +// XXX Copy-pasta from nettest/groups +// PerformanceSummary is the result summary for a performance test +type PerformanceSummary struct { + Upload int64 + Download int64 + Ping float64 + Bitrate int64 +} + +// MiddleboxSummary is the summary for the middlebox tests +type MiddleboxSummary struct { + Detected bool +} + +// IMSummary is the summary for the im tests +type IMSummary struct { + Tested uint + Blocked uint +} + +// WebsitesSummary is the summary for the websites test +type WebsitesSummary struct { + Tested uint + Blocked uint +} + +func formatSpeed(speed int64) string { + if speed < 1000 { + return fmt.Sprintf("%d Kbit/s", speed) + } else if speed < 1000*1000 { + return fmt.Sprintf("%.2f Mbit/s", float32(speed)/1000) + } else if speed < 1000*1000*1000 { + return fmt.Sprintf("%.2f Gbit/s", float32(speed)/(1000*1000)) + } + // WTF, you crazy? + return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000)) +} + +var summarizers = map[string]func(string) []string{ + "websites": func(ss string) []string { + var summary WebsitesSummary + if err := json.Unmarshal([]byte(ss), &summary); err != nil { + return nil + } + return []string{ + fmt.Sprintf("%d tested", summary.Tested), + fmt.Sprintf("%d blocked", summary.Blocked), + "", + } + }, + "performance": func(ss string) []string { + var summary PerformanceSummary + if err := json.Unmarshal([]byte(ss), &summary); err != nil { + return nil + } + return []string{ + fmt.Sprintf("Download: %s", formatSpeed(summary.Download)), + fmt.Sprintf("Upload: %s", formatSpeed(summary.Upload)), + fmt.Sprintf("Ping: %.2fms", summary.Ping), + } + }, + "im": func(ss string) []string { + var summary IMSummary + if err := json.Unmarshal([]byte(ss), &summary); err != nil { + return nil + } + return []string{ + fmt.Sprintf("%d tested", summary.Tested), + fmt.Sprintf("%d blocked", summary.Blocked), + "", + } + }, + "middlebox": func(ss string) []string { + var summary MiddleboxSummary + if err := json.Unmarshal([]byte(ss), &summary); err != nil { + return nil + } + return []string{ + fmt.Sprintf("Detected: %v", summary.Detected), + "", + "", + } + }, +} + +func makeSummary(name string, ss string) []string { + return summarizers[name](ss) +} + +func logResultItem(w io.Writer, f log.Fields) error { + colWidth := 24 + + rID := f.Get("id").(int64) + name := f.Get("name").(string) + startTime := f.Get("start_time").(time.Time) + networkName := f.Get("network_name").(string) + asn := fmt.Sprintf("AS %s", f.Get("asn").(string)) + //runtime := f.Get("runtime").(float64) + //dataUsageUp := f.Get("dataUsageUp").(int64) + //dataUsageDown := f.Get("dataUsageDown").(int64) + index := f.Get("index").(int) + totalCount := f.Get("total_count").(int) + if index == 0 { + fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth*2+2)+"┓\n") + } else { + fmt.Fprintf(w, "┢"+strings.Repeat("━", colWidth*2+2)+"┪\n") + } + + firstRow := RightPad(fmt.Sprintf("#%d - %s", rID, startTime.Format(time.RFC822)), colWidth*2) + fmt.Fprintf(w, "┃ "+firstRow+" ┃\n") + fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n") + + summary := makeSummary(name, f.Get("summary").(string)) + + fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", + RightPad(name, colWidth), + RightPad(summary[0], colWidth))) + fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", + RightPad(networkName, colWidth), + RightPad(summary[1], colWidth))) + fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", + RightPad(asn, colWidth), + RightPad(summary[2], colWidth))) + + if index == totalCount-1 { + fmt.Fprintf(w, "└┬──────────────┬──────────────┬──────────────┬") + fmt.Fprintf(w, strings.Repeat("─", colWidth*2-44)) + fmt.Fprintf(w, "┘\n") + } + return nil +} diff --git a/internal/output/output.go b/internal/output/output.go index bf76672..f658765 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -1,6 +1,8 @@ package output import ( + "time" + "github.com/apex/log" ) @@ -12,3 +14,40 @@ func Progress(key string, perc float64, msg string) { "percentage": perc, }).Info(msg) } + +// ResultItemData is the metadata about a result +type ResultItemData struct { + ID int64 + Name string + StartTime time.Time + Summary string + Runtime float64 + Country string + NetworkName string + ASN string + Done bool + DataUsageDown int64 + DataUsageUp int64 + Index int + TotalCount int +} + +// ResultItem logs a progress type event +func ResultItem(result ResultItemData) { + log.WithFields(log.Fields{ + "type": "result_item", + "id": result.ID, + "name": result.Name, + "start_time": result.StartTime, + "summary": result.Summary, + "country": result.Country, + "network_name": result.NetworkName, + "asn": result.ASN, + "runtime": result.Runtime, + "done": result.Done, + "data_usage_down": result.DataUsageDown, + "data_usage_up": result.DataUsageUp, + "index": result.Index, + "total_count": result.TotalCount, + }).Info("result item") +} diff --git a/nettests/groups/groups.go b/nettests/groups/groups.go index 9aceb1d..b35e04d 100644 --- a/nettests/groups/groups.go +++ b/nettests/groups/groups.go @@ -2,6 +2,7 @@ package groups import ( "encoding/json" + "fmt" "github.com/apex/log" "github.com/ooni/probe-cli/internal/database" @@ -44,6 +45,16 @@ type WebsitesSummary struct { Blocked uint } +func checkRequiredKeys(rk []string, m database.SummaryMap) error { + for _, key := range rk { + if _, ok := m[key]; ok { + continue + } + return fmt.Errorf("missing SummaryMap key '%s'", key) + } + return nil +} + // NettestGroups that can be run by the user var NettestGroups = map[string]NettestGroup{ "websites": NettestGroup{ @@ -52,6 +63,11 @@ var NettestGroups = map[string]NettestGroup{ websites.WebConnectivity{}, }, Summary: func(m database.SummaryMap) (string, error) { + if err := checkRequiredKeys([]string{"WebConnectivity"}, m); err != nil { + log.WithError(err).Error("missing keys") + return "", err + } + // XXX to generate this I need to create the summary map as a list var summary WebsitesSummary summary.Tested = 0 @@ -83,6 +99,11 @@ var NettestGroups = map[string]NettestGroup{ performance.NDT{}, }, Summary: func(m database.SummaryMap) (string, error) { + if err := checkRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil { + log.WithError(err).Error("missing keys") + return "", err + } + var ( err error ndtSummary performance.NDTSummary @@ -117,6 +138,11 @@ var NettestGroups = map[string]NettestGroup{ middlebox.HTTPHeaderFieldManipulation{}, }, Summary: func(m database.SummaryMap) (string, error) { + if err := checkRequiredKeys([]string{"WebConnectivity"}, m); err != nil { + log.WithError(err).Error("missing keys") + return "", err + } + var ( err error hhfmSummary middlebox.HTTPHeaderFieldManipulationSummary @@ -149,6 +175,10 @@ var NettestGroups = map[string]NettestGroup{ im.WhatsApp{}, }, Summary: func(m database.SummaryMap) (string, error) { + if err := checkRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil { + log.WithError(err).Error("missing keys") + return "", err + } var ( err error waSummary im.WhatsAppSummary diff --git a/nettests/nettests.go b/nettests/nettests.go index ed3a0cd..a944c44 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -3,10 +3,12 @@ package nettests import ( "encoding/json" "fmt" + "os" "path/filepath" "github.com/apex/log" "github.com/measurement-kit/go-measurement-kit" + homedir "github.com/mitchellh/go-homedir" ooni "github.com/ooni/probe-cli" "github.com/ooni/probe-cli/internal/cli/version" "github.com/ooni/probe-cli/internal/colors" @@ -42,6 +44,14 @@ type Controller struct { msmtPath string // XXX maybe we can drop this and just use a temporary file } +func getCaBundlePath() string { + path := os.Getenv("SSL_CERT_FILE") + if path != "" { + return path + } + return "/etc/ssl/cert.pem" +} + // Init should be called once to initialise the nettest func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("Init: %v", nt) @@ -59,26 +69,67 @@ func (c *Controller) Init(nt *mk.Nettest) error { ReportFilePath: c.msmtPath, } - log.Debugf("OutputPath: %s", c.msmtPath) + // This is to workaround homedirs having UTF-8 characters in them. + // See: https://github.com/measurement-kit/measurement-kit/issues/1635 + geoIPCountryPath := filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIP.dat") + geoIPASNPath := filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIPASNum.dat") + caBundlePath := getCaBundlePath() + msmtPath := c.msmtPath + + userHome, err := homedir.Dir() + if err != nil { + log.WithError(err).Error("failed to figure out the homedir") + return err + } + + relPath, err := filepath.Rel(userHome, caBundlePath) + if err != nil { + log.WithError(err).Error("caBundlePath is not relative to the users home") + } else { + caBundlePath = relPath + } + relPath, err = filepath.Rel(userHome, geoIPASNPath) + if err != nil { + log.WithError(err).Error("geoIPASNPath is not relative to the users home") + } else { + geoIPASNPath = relPath + } + relPath, err = filepath.Rel(userHome, geoIPCountryPath) + if err != nil { + log.WithError(err).Error("geoIPCountryPath is not relative to the users home") + } else { + geoIPCountryPath = relPath + } + + log.Debugf("Chdir to: %s", userHome) + if err := os.Chdir(userHome); err != nil { + log.WithError(err).Errorf("failed to chdir to %s", userHome) + return err + } + + log.Debugf("OutputPath: %s", msmtPath) nt.Options = mk.NettestOptions{ IncludeIP: c.Ctx.Config.Sharing.IncludeIP, IncludeASN: c.Ctx.Config.Sharing.IncludeASN, IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry, + LogLevel: "INFO", ProbeCC: c.Ctx.Location.CountryCode, ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN), ProbeIP: c.Ctx.Location.IP, - DisableCollector: false, - SoftwareName: "ooniprobe", - SoftwareVersion: version.Version, + DisableReportFile: false, + DisableCollector: false, + SoftwareName: "ooniprobe", + SoftwareVersion: version.Version, - // XXX - GeoIPCountryPath: filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIP.dat"), - GeoIPASNPath: filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIPASNum.dat"), - OutputPath: c.msmtPath, - CaBundlePath: "/etc/ssl/cert.pem", + OutputPath: msmtPath, + GeoIPCountryPath: geoIPCountryPath, + GeoIPASNPath: geoIPASNPath, + CaBundlePath: caBundlePath, } + log.Debugf("GeoIPASNPath: %s", nt.Options.GeoIPASNPath) + log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath) nt.On("log", func(e mk.Event) { level := e.Value.LogLevel @@ -117,7 +168,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { msmtTemplate.CountryCode = e.Value.ProbeCC }) - nt.On("status.measurement_started", func(e mk.Event) { + nt.On("status.measurement_start", func(e mk.Event) { log.Debugf(colors.Red(e.Key)) idx := e.Value.Idx @@ -151,7 +202,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { c.msmts[e.Value.Idx].UploadFailed(c.Ctx.DB, failure) }) - nt.On("status.measurement_uploaded", func(e mk.Event) { + nt.On("status.measurement_submission", func(e mk.Event) { log.Debugf(colors.Red(e.Key)) if err := c.msmts[e.Value.Idx].UploadSucceeded(c.Ctx.DB); err != nil { diff --git a/ooni.go b/ooni.go index a134a9d..051e051 100644 --- a/ooni.go +++ b/ooni.go @@ -9,7 +9,6 @@ import ( "github.com/apex/log" "github.com/jmoiron/sqlx" - homedir "github.com/mitchellh/go-homedir" "github.com/ooni/probe-cli/config" "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/internal/legacy" @@ -124,17 +123,6 @@ func NewContext(configPath string, homePath string) *Context { } } -// GetOONIHome returns the path to the OONI Home -func GetOONIHome() (string, error) { - home, err := homedir.Dir() - if err != nil { - return "", err - } - - path := filepath.Join(home, ".ooni") - return path, nil -} - // Config for the OONI Probe installation type Config struct { // Private settings @@ -179,7 +167,7 @@ func (c *Config) Unlock() { // Default config settings func (c *Config) Default() error { - home, err := GetOONIHome() + home, err := utils.GetOONIHome() if err != nil { return err } diff --git a/utils/homedir/homedir.go b/utils/homedir/homedir.go new file mode 100644 index 0000000..72c4aba --- /dev/null +++ b/utils/homedir/homedir.go @@ -0,0 +1,211 @@ +package homedir + +// Stolen from: https://github.com/puma/puma-dev/blob/master/homedir/homedir.go + +import ( + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" +) + +// DisableCache will disable caching of the home directory. Caching is enabled +// by default. +var DisableCache bool + +var homedirCache string +var cacheLock sync.RWMutex + +// ErrNoHomeDir when no home dir could be found +var ErrNoHomeDir = errors.New("no home directory available") + +// Dir returns the home directory for the executing user. +// +// This uses an OS-specific method for discovering the home directory. +// An error is returned if a home directory cannot be detected. +func Dir() (string, error) { + if !DisableCache { + cacheLock.RLock() + cached := homedirCache + cacheLock.RUnlock() + if cached != "" { + return cached, nil + } + } + + cacheLock.Lock() + defer cacheLock.Unlock() + + var result string + var err error + switch runtime.GOOS { + case "windows": + result, err = dirWindows() + case "darwin": + result, err = dirDarwin() + default: + // Unix-like system, so just assume Unix + result, err = dirUnix() + } + + if err != nil { + return "", err + } + + homedirCache = result + return result, nil +} + +// Expand expands the path to include the home directory if the path +// is prefixed with `~`. If it isn't prefixed with `~`, the path is +// returned as-is. +func Expand(path string) (string, error) { + if len(path) == 0 { + return path, nil + } + + if path[0] != '~' { + return path, nil + } + + if len(path) > 1 && path[1] != '/' && path[1] != '\\' { + return "", errors.New("cannot expand user-specific home dir") + } + + dir, err := Dir() + if err != nil { + return "", err + } + + return filepath.Join(dir, path[1:]), nil +} + +func MustExpand(path string) string { + str, err := Expand(path) + if err != nil { + panic(err) + } + + return str +} + +func dirDarwin() (string, error) { + // First prefer the HOME environmental variable + if home := os.Getenv("HOME"); home != "" { + return home, nil + } + + var stdout bytes.Buffer + + // If that fails, try OS specific commands + cmd := exec.Command("sh", "-c", `dscl -q . -read /Users/"$(whoami)" NFSHomeDirectory | sed 's/^[^ ]*: //'`) + cmd.Stdout = &stdout + if err := cmd.Run(); err == nil { + result := strings.TrimSpace(stdout.String()) + if result != "" { + return result, nil + } + } + + // try the shell + stdout.Reset() + cmd = exec.Command("sh", "-c", "cd && pwd") + cmd.Stdout = &stdout + if err := cmd.Run(); err == nil { + result := strings.TrimSpace(stdout.String()) + if result != "" { + return result, nil + } + } + + // try to figure out the user and check the default location + stdout.Reset() + cmd = exec.Command("whoami") + cmd.Stdout = &stdout + if err := cmd.Run(); err == nil { + user := strings.TrimSpace(stdout.String()) + + path := "/Users/" + user + + stat, err := os.Stat(path) + if err == nil && stat.IsDir() { + return path, nil + } + } + + return "", ErrNoHomeDir +} + +func dirUnix() (string, error) { + // First prefer the HOME environmental variable + if home := os.Getenv("HOME"); home != "" { + return home, nil + } + + var stdout bytes.Buffer + + // If that fails, try OS specific commands + cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid())) + cmd.Stdout = &stdout + if err := cmd.Run(); err == nil { + if passwd := strings.TrimSpace(stdout.String()); passwd != "" { + // username:password:uid:gid:gecos:home:shell + passwdParts := strings.SplitN(passwd, ":", 7) + if len(passwdParts) > 5 { + return passwdParts[5], nil + } + } + } + + // If all else fails, try the shell + stdout.Reset() + cmd = exec.Command("sh", "-c", "cd && pwd") + cmd.Stdout = &stdout + if err := cmd.Run(); err == nil { + result := strings.TrimSpace(stdout.String()) + if result == "" { + return "", errors.New("blank output when reading home directory") + } + } + + // try to figure out the user and check the default location + stdout.Reset() + cmd = exec.Command("whoami") + cmd.Stdout = &stdout + if err := cmd.Run(); err == nil { + user := strings.TrimSpace(stdout.String()) + + path := "/home/" + user + + stat, err := os.Stat(path) + if err == nil && stat.IsDir() { + return path, nil + } + } + + return "", ErrNoHomeDir +} + +func dirWindows() (string, error) { + // First prefer the HOME environmental variable + if home := os.Getenv("HOME"); home != "" { + return home, nil + } + + drive := os.Getenv("HOMEDRIVE") + path := os.Getenv("HOMEPATH") + home := drive + path + if drive == "" || path == "" { + home = os.Getenv("USERPROFILE") + } + if home == "" { + return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank") + } + + return home, nil +} diff --git a/utils/paths.go b/utils/paths.go index 42cf979..63ce079 100644 --- a/utils/paths.go +++ b/utils/paths.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "time" + + "github.com/ooni/probe-cli/utils/homedir" ) // RequiredDirs returns the required ooni home directories @@ -28,10 +30,13 @@ func DBDir(home string, name string) string { return filepath.Join(home, "db", fmt.Sprintf("%s.sqlite3", name)) } +// ResultTimestamp is a windows friendly timestamp +const ResultTimestamp = "2006-01-02T150405.999999999Z0700" + // MakeResultsDir creates and returns a directory for the result func MakeResultsDir(home string, name string, ts time.Time) (string, error) { p := filepath.Join(home, "msmts", - fmt.Sprintf("%s-%s", name, ts.Format(time.RFC3339Nano))) + fmt.Sprintf("%s-%s", name, ts.Format(ResultTimestamp))) // If the path already exists, this is a problem. It should not clash, because // we are using nanosecond precision for the starttime. @@ -44,3 +49,18 @@ func MakeResultsDir(home string, name string, ts time.Time) (string, error) { } return p, nil } + +// GetOONIHome returns the path to the OONI Home +func GetOONIHome() (string, error) { + if ooniHome := os.Getenv("OONI_HOME"); ooniHome != "" { + return ooniHome, nil + } + + home, err := homedir.Dir() + if err != nil { + return "", err + } + + path := filepath.Join(home, ".ooni") + return path, nil +}