diff --git a/.travis.yml b/.travis.yml index 1a5d4ef..7bedfbf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,10 @@ -matrix: - include: - - os: linux - dist: xenial - language: minimal - services: - - docker - env: - - OS_NAME: linux - - os: osx - language: minimal - env: - - OS_NAME: macos - osx_image: xcode10.2 +os: linux +dist: xenial +language: minimal +services: +- docker +env: +- OS_NAME: linux script: - ./build.sh _travis-${TRAVIS_OS_NAME} - ./scripts/travis_test.sh diff --git a/config/settings.go b/config/settings.go index f10e28f..5edbaee 100644 --- a/config/settings.go +++ b/config/settings.go @@ -42,6 +42,7 @@ type NettestConfig struct { // Websites test group type Websites struct { EnabledCategories []string `json:"enabled_categories"` + Limit int `json:"limit"` } // NettestConfigs returns a list configured enabled tests for the group diff --git a/go.mod b/go.mod index 03f3321..0677cf6 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,7 @@ require ( github.com/lib/pq v1.1.1 // indirect github.com/mattn/go-colorable v0.1.2 github.com/mattn/go-sqlite3 v1.10.0 // indirect - github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632 - github.com/oschwald/maxminddb-golang v1.3.1 // indirect + github.com/ooni/probe-engine v0.0.0-20190828175829-9e908f8e3a3a github.com/pkg/errors v0.8.1 github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051 github.com/ziutek/mymysql v1.5.4 // indirect diff --git a/go.sum b/go.sum index 33e717a..7bd4bbc 100644 --- a/go.sum +++ b/go.sum @@ -139,14 +139,12 @@ github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632 h1:58EgLL9v2729NkY0U/lixOAGr9UsvRH67use+iJvz+c= -github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632/go.mod h1:iq63Xvw89LLS+2Q9vdqNpmuGQLntyomd9K5w49hU+Ks= +github.com/ooni/probe-engine v0.0.0-20190828175829-9e908f8e3a3a h1:e3SiEPrbtsB76tPm+sFLI5dtKBd7sa+vjtdfvSMKLqk= +github.com/ooni/probe-engine v0.0.0-20190828175829-9e908f8e3a3a/go.mod h1:zovuewNdJp8exKnBlpCgZXNu/SN+Cs4acj2KgGZOP9s= github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8= github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE= -github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE= -github.com/oschwald/maxminddb-golang v1.3.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= -github.com/oschwald/maxminddb-golang v1.3.1 h1:kPc5+ieL5CC/Zn0IaXJPxDFlUxKTQEU8QBTtmfQDAIo= -github.com/oschwald/maxminddb-golang v1.3.1/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= +github.com/oschwald/maxminddb-golang v1.4.0 h1:5/rpmW41qrgSed4wK32rdznbkTSXHcraY2LOMJX4DMc= +github.com/oschwald/maxminddb-golang v1.4.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 7c7bd08..d327714 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -3,8 +3,6 @@ package run import ( "context" "errors" - "fmt" - "strings" "github.com/alecthomas/kingpin" "github.com/apex/log" @@ -50,20 +48,20 @@ func init() { cmd := root.Command("run", "Run a test group or OONI Run link") var nettestGroupNamesBlue []string + var ctx *ooni.Context + var network *database.Network + for name := range groups.NettestGroups { nettestGroupNamesBlue = append(nettestGroupNamesBlue, color.BlueString(name)) } - nettestGroup := cmd.Arg("name", - fmt.Sprintf("the nettest group to run. Supported tests are: %s, or nothing to run them all", - strings.Join(nettestGroupNamesBlue, ", "))).String() - noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool() collectorURL := cmd.Flag("collector-url", "Specify the address of a custom collector").String() bouncerURL := cmd.Flag("bouncer-url", "Specify the address of a custom bouncer").String() cmd.Action(func(_ *kingpin.ParseContext) error { - ctx, err := root.Init() + var err error + ctx, err = root.Init() if err != nil { log.Errorf("%s", err) return err @@ -91,7 +89,7 @@ func init() { log.WithError(err).Error("Failed to lookup the location of the probe") return err } - network, err := database.CreateNetwork(ctx.DB, ctx.Session.Location) + network, err = database.CreateNetwork(ctx.DB, ctx.Session.Location) if err != nil { log.WithError(err).Error("Failed to create the network row") return err @@ -112,17 +110,33 @@ func init() { return err } } + return nil + }) - if *nettestGroup == "" { - log.Infof("Running %s tests", color.BlueString("all")) - for tg := range groups.NettestGroups { - if err := runNettestGroup(tg, ctx, network); err != nil { - log.WithError(err).Errorf("failed to run %s", tg) - } + websitesCmd := cmd.Command("websites", "") + websitesCmd.Action(func(_ *kingpin.ParseContext) error { + return runNettestGroup("websites", ctx, network) + }) + imCmd := cmd.Command("im", "") + imCmd.Action(func(_ *kingpin.ParseContext) error { + return runNettestGroup("im", ctx, network) + }) + performanceCmd := cmd.Command("performance", "") + performanceCmd.Action(func(_ *kingpin.ParseContext) error { + return runNettestGroup("performance", ctx, network) + }) + middleboxCmd := cmd.Command("middlebox", "") + middleboxCmd.Action(func(_ *kingpin.ParseContext) error { + return runNettestGroup("middlebox", ctx, network) + }) + allCmd := cmd.Command("all", "").Default() + allCmd.Action(func(_ *kingpin.ParseContext) error { + log.Infof("Running %s tests", color.BlueString("all")) + for tg := range groups.NettestGroups { + if err := runNettestGroup(tg, ctx, network); err != nil { + log.WithError(err).Errorf("failed to run %s", tg) } - return nil } - log.Infof("Running %s", color.BlueString(*nettestGroup)) - return runNettestGroup(*nettestGroup, ctx, network) + return nil }) } diff --git a/internal/cli/show/show.go b/internal/cli/show/show.go index ca75897..a94ec35 100644 --- a/internal/cli/show/show.go +++ b/internal/cli/show/show.go @@ -4,19 +4,27 @@ import ( "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/ooni/probe-cli/internal/cli/root" + "github.com/ooni/probe-cli/internal/database" + "github.com/ooni/probe-cli/internal/output" ) func init() { cmd := root.Command("show", "Show a specific measurement") + msmtID := cmd.Arg("id", "the id of the measurement to show").Int64() + cmd.Action(func(_ *kingpin.ParseContext) error { - _, err := root.Init() + ctx, err := root.Init() if err != nil { log.WithError(err).Error("failed to initialize root context") return err } - log.Error("this function is not implemented") - + msmt, err := database.GetMeasurementJSON(ctx.DB, *msmtID) + if err != nil { + log.Errorf("error: %v", err) + return err + } + output.MeasurementJSON(msmt) return nil }) } diff --git a/internal/database/actions.go b/internal/database/actions.go index 7ebccf7..4930ad0 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -3,6 +3,9 @@ package database import ( "database/sql" "encoding/json" + "bufio" + "io" + "io/ioutil" "os" "reflect" "time" @@ -10,6 +13,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-engine/model" "github.com/ooni/probe-cli/utils" + "github.com/ooni/probe-cli/internal/util" "github.com/pkg/errors" db "upper.io/db.v3" "upper.io/db.v3/lib/sqlbuilder" @@ -38,7 +42,67 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR return measurements, nil } -// GetResultTestKeys returns a list of TestKeys for a given measurements +// GetMeasurementJSON will a map[string]interface{} given a database and a measurementID +func GetMeasurementJSON(sess sqlbuilder.Database, measurementID int64) (map[string]interface{}, error) { + var ( + measurement MeasurementURLNetwork + msmtJSON map[string]interface{} + ) + + req := sess.Select( + db.Raw("urls.*"), + db.Raw("measurements.*"), + ).From("measurements"). + LeftJoin("urls").On("urls.url_id = measurements.url_id"). + Where("measurements.measurement_id= ?", measurementID) + + if err := req.One(&measurement); err != nil { + log.Errorf("failed to run query %s: %v", req.String(), err) + return nil, err + } + reportFilePath := measurement.Measurement.ReportFilePath + // If the url->url is NULL then we are dealing with a single entry + // measurement and all we have to do is read the file and return it. + if (measurement.URL.URL.Valid == false) { + b, err := ioutil.ReadFile(reportFilePath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(b, &msmtJSON); err != nil { + return nil, err + } + return msmtJSON, nil + } + + // When the URL is a string then we need to seek until we reach the + // measurement line in the file that matches the target input + url := measurement.URL.URL.String + file, err := os.Open(reportFilePath) + if err != nil { + return nil, err + } + defer file.Close() + + reader := bufio.NewReader(file) + + for { + line, err := util.ReadLine(reader) + if (err == io.EOF) { + break + } else if err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(line), &msmtJSON); err != nil { + return nil, err + } + if (msmtJSON["input"].(string) == url) { + return msmtJSON, nil + } + } + return nil, errors.New("Could not find measurement") +} + +// GetResultTestKeys returns a list of TestKeys for a given result func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error) { res := sess.Collection("measurements").Find("result_id", resultID) defer res.Close() @@ -48,20 +112,18 @@ func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error) tk PerformanceTestKeys ) for res.Next(&msmt) { - if msmt.TestName == "web_connectivity" { - break - } // We only really care about performance keys. // Note: since even in case of failure we still initialise an empty struct, // it could be that these keys come out as initializes with the default // values. // XXX we may want to change this behaviour by adding `omitempty` to the // struct definition. - if msmt.TestName == "ndt" || msmt.TestName == "dash" { - if err := json.Unmarshal([]byte(msmt.TestKeys), &tk); err != nil { - log.WithError(err).Error("failed to parse testKeys") - return "{}", err - } + if msmt.TestName != "ndt" && msmt.TestName != "dash" { + return "{}", nil + } + if err := json.Unmarshal([]byte(msmt.TestKeys), &tk); err != nil { + log.WithError(err).Error("failed to parse testKeys") + return "{}", err } } b, err := json.Marshal(tk) diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index b826db5..eeaa94f 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -119,6 +119,8 @@ func (h *Handler) TypedLog(t string, e *log.Entry) error { return logTable(h.Writer, e.Fields) case "measurement_item": return logMeasurementItem(h.Writer, e.Fields) + case "measurement_json": + return logMeasurementJSON(h.Writer, e.Fields) case "measurement_summary": return logMeasurementSummary(h.Writer, e.Fields) case "result_item": diff --git a/internal/log/handlers/cli/measurements.go b/internal/log/handlers/cli/measurements.go index 85dca51..a0ed0c7 100644 --- a/internal/log/handlers/cli/measurements.go +++ b/internal/log/handlers/cli/measurements.go @@ -129,3 +129,14 @@ func logMeasurementSummary(w io.Writer, f log.Fields) error { return nil } + +func logMeasurementJSON(w io.Writer, f log.Fields) error { + m := f.Get("measurement_json").(map[string]interface{}) + + json, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + fmt.Fprintf(w, string(json)) + return nil +} diff --git a/internal/output/output.go b/internal/output/output.go index 0c55225..d97964d 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -11,6 +11,14 @@ import ( "github.com/ooni/probe-cli/internal/util" ) +// MeasurementJSON prints the JSON of a measurement +func MeasurementJSON(j map[string]interface{}) { + log.WithFields(log.Fields{ + "type": "measurement_json", + "measurement_json": j, + }).Info("Measurement JSON") +} + // Progress logs a progress type event func Progress(key string, perc float64, msg string) { log.WithFields(log.Fields{ diff --git a/internal/util/util.go b/internal/util/util.go index 7d64ccf..9b9300a 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -2,6 +2,7 @@ package util import ( "bytes" + "bufio" "fmt" "os" "regexp" @@ -112,3 +113,18 @@ func WrapString(s string, lim uint) string { return buf.String() } + + +// ReadLine will read a single line from a bufio.Reader +func ReadLine(r *bufio.Reader) (string, error) { + var ( + isPrefix bool + err error + line, ln []byte + ) + for isPrefix && err == nil { + line, isPrefix, err = r.ReadLine() + ln = append(ln, line...) + } + return string(ln), err +} diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index 1291777..ed543c7 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -10,11 +10,11 @@ import ( "github.com/ooni/probe-engine/orchestra/testlists" ) -func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { +func lookupURLs(ctl *nettests.Controller, limit int) ([]string, map[int64]int64, error) { var urls []string urlIDMap := make(map[int64]int64) testlist, err := testlists.NewClient(ctl.Ctx.Session).Do( - context.Background(), ctl.Ctx.Session.ProbeCC(), + context.Background(), ctl.Ctx.Session.ProbeCC(), limit, ) if err != nil { return nil, nil, err @@ -41,7 +41,7 @@ type WebConnectivity struct { // Run starts the test func (n WebConnectivity) Run(ctl *nettests.Controller) error { - urls, urlIDMap, err := lookupURLs(ctl) + urls, urlIDMap, err := lookupURLs(ctl, ctl.Ctx.Config.NettestGroups.Websites.Limit) if err != nil { return err } diff --git a/scripts/travis_test.sh b/scripts/travis_test.sh index 3b2a19b..efb13a6 100755 --- a/scripts/travis_test.sh +++ b/scripts/travis_test.sh @@ -3,4 +3,4 @@ set -ex ./dist/${OS_NAME}/amd64/ooni onboard --yes -./dist/${OS_NAME}/amd64/ooni run -v --no-collector +./dist/${OS_NAME}/amd64/ooni run --config testdata/testing-config.json -v --no-collector diff --git a/testdata/testing-config.json b/testdata/testing-config.json new file mode 100644 index 0000000..749d58f --- /dev/null +++ b/testdata/testing-config.json @@ -0,0 +1,64 @@ +{ + "_": "This is your OONI Probe config file. See https://ooni.io/help/probe-cli for help", + "_version": 0, + "_informed_consent": true, + "_is_beta": true, + "auto_update": true, + "sharing": { + "include_ip": false, + "include_asn": true, + "include_country": true, + "include_gps": true, + "upload_results": true + }, + "notifications": { + "enabled": true, + "notify_on_test_completion": true, + "notify_on_news": false + }, + "automated_testing": { + "enabled": false, + "enabled_tests": [ + "web-connectivity", + "facebook-messenger", + "whatsapp", + "telegram", + "dash", + "ndt", + "http-invalid-request-line", + "http-header-field-manipulation" + ], + "monthly_allowance": "300MB" + }, + "test_settings": { + "websites": { + "enabled_categories": [], + "limit": 10 + }, + "instant_messaging": { + "enabled_tests": [ + "facebook-messenger", + "whatsapp", + "telegram" + ] + }, + "performance": { + "ndt_server": "auto", + "ndt_server_port": "auto", + "dash_server": "auto", + "dash_server_port": "auto" + }, + "middlebox": { + "enabled_tests": [ + "http-invalid-request-line", + "http-header-field-manipulation" + ] + } + }, + "advanced": { + "use_domain_fronting": false, + "send_crash_reports": true, + "collector_url": "", + "bouncer_url": "https://bouncer.ooni.io" + } +}