2018-02-07 19:02:18 +01:00
|
|
|
package run
|
|
|
|
|
|
|
|
import (
|
2018-03-22 15:22:29 +01:00
|
|
|
"errors"
|
2018-02-13 17:11:22 +01:00
|
|
|
|
2018-02-07 19:02:18 +01:00
|
|
|
"github.com/alecthomas/kingpin"
|
2018-02-12 16:45:13 +01:00
|
|
|
"github.com/apex/log"
|
2018-06-29 16:50:05 +02:00
|
|
|
"github.com/fatih/color"
|
2018-09-27 18:38:37 +02:00
|
|
|
ooni "github.com/ooni/probe-cli"
|
2018-05-03 14:59:55 +02:00
|
|
|
"github.com/ooni/probe-cli/internal/cli/root"
|
|
|
|
"github.com/ooni/probe-cli/internal/database"
|
|
|
|
"github.com/ooni/probe-cli/nettests"
|
|
|
|
"github.com/ooni/probe-cli/nettests/groups"
|
2018-02-07 19:02:18 +01:00
|
|
|
)
|
|
|
|
|
2018-09-27 18:38:37 +02:00
|
|
|
func runNettestGroup(tg string, ctx *ooni.Context, network *database.Network) error {
|
|
|
|
group, ok := groups.NettestGroups[tg]
|
|
|
|
if !ok {
|
|
|
|
log.Errorf("No test group named %s", tg)
|
|
|
|
return errors.New("invalid test group name")
|
|
|
|
}
|
|
|
|
log.Debugf("Running test group %s", group.Label)
|
|
|
|
|
|
|
|
result, err := database.CreateResult(ctx.DB, ctx.Home, tg, network.ID)
|
|
|
|
if err != nil {
|
|
|
|
log.Errorf("DB result error: %s", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, nt := range group.Nettests {
|
|
|
|
log.Debugf("Running test %T", nt)
|
|
|
|
ctl := nettests.NewController(nt, ctx, result)
|
|
|
|
ctl.SetNettestIndex(i, len(group.Nettests))
|
|
|
|
if err = nt.Run(ctl); err != nil {
|
|
|
|
log.WithError(err).Errorf("Failed to run %s", group.Label)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = result.Finished(ctx.DB); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-02-07 19:02:18 +01:00
|
|
|
func init() {
|
|
|
|
cmd := root.Command("run", "Run a test group or OONI Run link")
|
|
|
|
|
2018-09-27 18:38:37 +02:00
|
|
|
var nettestGroupNamesBlue []string
|
2019-10-02 18:23:14 +02:00
|
|
|
var ctx *ooni.Context
|
|
|
|
var network *database.Network
|
|
|
|
|
2018-06-22 14:55:00 +02:00
|
|
|
for name := range groups.NettestGroups {
|
2018-09-27 18:38:37 +02:00
|
|
|
nettestGroupNamesBlue = append(nettestGroupNamesBlue, color.BlueString(name))
|
2018-06-22 14:55:00 +02:00
|
|
|
}
|
|
|
|
|
2018-09-17 11:51:54 +02:00
|
|
|
noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool()
|
2018-09-21 18:54:44 +02:00
|
|
|
collectorURL := cmd.Flag("collector-url", "Specify the address of a custom collector").String()
|
|
|
|
bouncerURL := cmd.Flag("bouncer-url", "Specify the address of a custom bouncer").String()
|
2018-09-17 11:51:54 +02:00
|
|
|
|
2018-02-07 19:02:18 +01:00
|
|
|
cmd.Action(func(_ *kingpin.ParseContext) error {
|
2019-10-02 18:23:14 +02:00
|
|
|
var err error
|
|
|
|
ctx, err = root.Init()
|
2018-02-12 16:45:13 +01:00
|
|
|
if err != nil {
|
|
|
|
log.Errorf("%s", err)
|
|
|
|
return err
|
|
|
|
}
|
2018-06-22 11:01:15 +02:00
|
|
|
|
|
|
|
if err = ctx.MaybeOnboarding(); err != nil {
|
|
|
|
log.WithError(err).Error("failed to perform onboarding")
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-09-17 11:51:54 +02:00
|
|
|
if *noCollector == true {
|
|
|
|
ctx.Config.Sharing.UploadResults = false
|
|
|
|
}
|
2018-09-21 18:54:44 +02:00
|
|
|
if *collectorURL != "" {
|
|
|
|
ctx.Config.Advanced.CollectorURL = *collectorURL
|
|
|
|
}
|
|
|
|
if *bouncerURL != "" {
|
|
|
|
ctx.Config.Advanced.BouncerURL = *bouncerURL
|
|
|
|
}
|
2019-05-23 16:38:46 +02:00
|
|
|
log.Debugf("Using collector: %s", ctx.Config.Advanced.CollectorURL)
|
|
|
|
log.Debugf("Using bouncer: %s", ctx.Config.Advanced.CollectorURL)
|
2018-09-17 11:51:54 +02:00
|
|
|
|
2018-03-23 12:41:06 +01:00
|
|
|
err = ctx.MaybeLocationLookup()
|
|
|
|
if err != nil {
|
|
|
|
log.WithError(err).Error("Failed to lookup the location of the probe")
|
|
|
|
return err
|
|
|
|
}
|
2019-10-03 09:43:25 +02:00
|
|
|
network, err = database.CreateNetwork(ctx.DB, ctx.Session)
|
2018-09-06 16:13:04 +02:00
|
|
|
if err != nil {
|
|
|
|
log.WithError(err).Error("Failed to create the network row")
|
2018-02-13 17:11:22 +01:00
|
|
|
return err
|
|
|
|
}
|
Integrate further with ooni/probe-engine: episode two (#46)
* utils/geoip.go: use github.com/ooni/probe-engine
Let's start using the engine by rewriting utils/geoip.go to
be just a thin wrapper around the engine functionality.
* Ready for review
* Checkpoint: the im tests are converted
Still have some doubts with respect to the variables that
are passed to MK via probe-engine. Will double check.
* fix(i/c/r/run.go): write the correct logic
* nettests: one more comment and also fix a format string
* Tweak previous
* progress
* Fix doofus
* better comment
* XXX => actionable comment
* Add glue to simplify test keys management
Making the concept of measurement more abstract in the engine is
not feasible because, when submitting a measurement, we need to
modify it to update the report ID and the measurement ID. Therefore,
returning a serialized measurement is not a good idea. We will
keep using a model.Measurement in the engine.
Changing model.Measurement.TestKeys's type from a `interface{}`
pointing to a well defined data structure to `map[string]interface{}`
is a regression because means that we are moving from code that
has a clear and defined structure to code that is more complicated
to parse and validate. Since we're already suffering havily from
the lack of a good schema, I'm not going to make the situation
worst by worsening the engine. At least for ndt7 and psiphon, we
now have a good schema and I don't want to lose that.
However, the current code in this repository is expecting the
test keys to be a `map[string]interface{}`. This choice was
dictated by the fact that we receive a JSON from Measurement Kit
and by the fact that there's not a clear schema.
To solve this tension, in this commit I am going to write glue
adapter code that makes sure that the TestKeys of a Measurement
are converted to `map[string]interface{}`. This will be done
using a type cast where possible and JSON serialization and parsing
otherwise. In a perfect world, glue is not a good idea, but in a
real world it may actually be useful.
When all tests in the engine will have a clear Go data structure,
we'll then remove the glue and just cast to the proper data
structure from `interface{}` where required.
* nettests/performance: use probe-engine
* go.{mod,sum}: upgrade to latest probe-engine
* nettests/middlebox: use ooni/probe-engine
* Update to the latest probe-engine
* web_connectivity: rewrite to use probe-engine
* Cosmetic change suggested by @hellais
* nettests/nettests.go: remove unused code
* nettests/nettests.go: fix progress
* nettests/nettests.go: remove go-measurement-kit code
* We don't depend on go-measurement-kit anymore
* Improve non-verbose output where possible
See also: https://github.com/measurement-kit/measurement-kit/issues/1856
* Make web_connectivity output pleasant
* Update to the latest probe-engine
* nettests/nettests.go: honour sharing settings
* Update to the latest probe-engine
* Use log.WithFields for probe-engine
* Update go.mod go.sum
* Revert "Update go.mod go.sum"
This reverts commit 5ecd38d8236f4a4e9b77ddb8e8a0d1e3cdd4b818.
* Revert "Revert "Update go.mod go.sum""
This reverts commit 6114b31eca98826112032776bd0feff02d763ecd.
* Upgrade ooni/probe-engine
* Unset GOPATH before running go build commands
* Dockefile: fix linux build by using latest
* Update to the latest ooni/probe-engine
```
go get -u github.com/ooni/probe-engine
go mod tidy
```
* Repair build
2019-08-15 18:08:43 +02:00
|
|
|
if ctx.Config.Advanced.BouncerURL != "" {
|
|
|
|
ctx.Session.AddAvailableHTTPSBouncer(ctx.Config.Advanced.BouncerURL)
|
|
|
|
}
|
2019-10-28 14:05:05 +01:00
|
|
|
if ctx.Config.Sharing.UploadResults && ctx.Config.Advanced.CollectorURL != "" {
|
|
|
|
ctx.Session.AddAvailableHTTPSCollector(ctx.Config.Advanced.CollectorURL)
|
Integrate further with ooni/probe-engine: episode two (#46)
* utils/geoip.go: use github.com/ooni/probe-engine
Let's start using the engine by rewriting utils/geoip.go to
be just a thin wrapper around the engine functionality.
* Ready for review
* Checkpoint: the im tests are converted
Still have some doubts with respect to the variables that
are passed to MK via probe-engine. Will double check.
* fix(i/c/r/run.go): write the correct logic
* nettests: one more comment and also fix a format string
* Tweak previous
* progress
* Fix doofus
* better comment
* XXX => actionable comment
* Add glue to simplify test keys management
Making the concept of measurement more abstract in the engine is
not feasible because, when submitting a measurement, we need to
modify it to update the report ID and the measurement ID. Therefore,
returning a serialized measurement is not a good idea. We will
keep using a model.Measurement in the engine.
Changing model.Measurement.TestKeys's type from a `interface{}`
pointing to a well defined data structure to `map[string]interface{}`
is a regression because means that we are moving from code that
has a clear and defined structure to code that is more complicated
to parse and validate. Since we're already suffering havily from
the lack of a good schema, I'm not going to make the situation
worst by worsening the engine. At least for ndt7 and psiphon, we
now have a good schema and I don't want to lose that.
However, the current code in this repository is expecting the
test keys to be a `map[string]interface{}`. This choice was
dictated by the fact that we receive a JSON from Measurement Kit
and by the fact that there's not a clear schema.
To solve this tension, in this commit I am going to write glue
adapter code that makes sure that the TestKeys of a Measurement
are converted to `map[string]interface{}`. This will be done
using a type cast where possible and JSON serialization and parsing
otherwise. In a perfect world, glue is not a good idea, but in a
real world it may actually be useful.
When all tests in the engine will have a clear Go data structure,
we'll then remove the glue and just cast to the proper data
structure from `interface{}` where required.
* nettests/performance: use probe-engine
* go.{mod,sum}: upgrade to latest probe-engine
* nettests/middlebox: use ooni/probe-engine
* Update to the latest probe-engine
* web_connectivity: rewrite to use probe-engine
* Cosmetic change suggested by @hellais
* nettests/nettests.go: remove unused code
* nettests/nettests.go: fix progress
* nettests/nettests.go: remove go-measurement-kit code
* We don't depend on go-measurement-kit anymore
* Improve non-verbose output where possible
See also: https://github.com/measurement-kit/measurement-kit/issues/1856
* Make web_connectivity output pleasant
* Update to the latest probe-engine
* nettests/nettests.go: honour sharing settings
* Update to the latest probe-engine
* Use log.WithFields for probe-engine
* Update go.mod go.sum
* Revert "Update go.mod go.sum"
This reverts commit 5ecd38d8236f4a4e9b77ddb8e8a0d1e3cdd4b818.
* Revert "Revert "Update go.mod go.sum""
This reverts commit 6114b31eca98826112032776bd0feff02d763ecd.
* Upgrade ooni/probe-engine
* Unset GOPATH before running go build commands
* Dockefile: fix linux build by using latest
* Update to the latest ooni/probe-engine
```
go get -u github.com/ooni/probe-engine
go mod tidy
```
* Repair build
2019-08-15 18:08:43 +02:00
|
|
|
}
|
2019-10-28 14:05:05 +01:00
|
|
|
if err := ctx.Session.MaybeLookupBackends(); err != nil {
|
|
|
|
log.WithError(err).Warn("Failed to discover OONI backends")
|
|
|
|
return err
|
Integrate further with ooni/probe-engine: episode two (#46)
* utils/geoip.go: use github.com/ooni/probe-engine
Let's start using the engine by rewriting utils/geoip.go to
be just a thin wrapper around the engine functionality.
* Ready for review
* Checkpoint: the im tests are converted
Still have some doubts with respect to the variables that
are passed to MK via probe-engine. Will double check.
* fix(i/c/r/run.go): write the correct logic
* nettests: one more comment and also fix a format string
* Tweak previous
* progress
* Fix doofus
* better comment
* XXX => actionable comment
* Add glue to simplify test keys management
Making the concept of measurement more abstract in the engine is
not feasible because, when submitting a measurement, we need to
modify it to update the report ID and the measurement ID. Therefore,
returning a serialized measurement is not a good idea. We will
keep using a model.Measurement in the engine.
Changing model.Measurement.TestKeys's type from a `interface{}`
pointing to a well defined data structure to `map[string]interface{}`
is a regression because means that we are moving from code that
has a clear and defined structure to code that is more complicated
to parse and validate. Since we're already suffering havily from
the lack of a good schema, I'm not going to make the situation
worst by worsening the engine. At least for ndt7 and psiphon, we
now have a good schema and I don't want to lose that.
However, the current code in this repository is expecting the
test keys to be a `map[string]interface{}`. This choice was
dictated by the fact that we receive a JSON from Measurement Kit
and by the fact that there's not a clear schema.
To solve this tension, in this commit I am going to write glue
adapter code that makes sure that the TestKeys of a Measurement
are converted to `map[string]interface{}`. This will be done
using a type cast where possible and JSON serialization and parsing
otherwise. In a perfect world, glue is not a good idea, but in a
real world it may actually be useful.
When all tests in the engine will have a clear Go data structure,
we'll then remove the glue and just cast to the proper data
structure from `interface{}` where required.
* nettests/performance: use probe-engine
* go.{mod,sum}: upgrade to latest probe-engine
* nettests/middlebox: use ooni/probe-engine
* Update to the latest probe-engine
* web_connectivity: rewrite to use probe-engine
* Cosmetic change suggested by @hellais
* nettests/nettests.go: remove unused code
* nettests/nettests.go: fix progress
* nettests/nettests.go: remove go-measurement-kit code
* We don't depend on go-measurement-kit anymore
* Improve non-verbose output where possible
See also: https://github.com/measurement-kit/measurement-kit/issues/1856
* Make web_connectivity output pleasant
* Update to the latest probe-engine
* nettests/nettests.go: honour sharing settings
* Update to the latest probe-engine
* Use log.WithFields for probe-engine
* Update go.mod go.sum
* Revert "Update go.mod go.sum"
This reverts commit 5ecd38d8236f4a4e9b77ddb8e8a0d1e3cdd4b818.
* Revert "Revert "Update go.mod go.sum""
This reverts commit 6114b31eca98826112032776bd0feff02d763ecd.
* Upgrade ooni/probe-engine
* Unset GOPATH before running go build commands
* Dockefile: fix linux build by using latest
* Update to the latest ooni/probe-engine
```
go get -u github.com/ooni/probe-engine
go mod tidy
```
* Repair build
2019-08-15 18:08:43 +02:00
|
|
|
}
|
2019-10-28 14:05:05 +01:00
|
|
|
// Make sure we share what the user wants us to share.
|
|
|
|
ctx.Session.SetIncludeProbeIP(ctx.Config.Sharing.IncludeIP)
|
|
|
|
ctx.Session.SetIncludeProbeASN(ctx.Config.Sharing.IncludeASN)
|
|
|
|
ctx.Session.SetIncludeProbeCC(ctx.Config.Sharing.IncludeCountry)
|
2019-10-02 18:23:14 +02:00
|
|
|
return nil
|
|
|
|
})
|
2018-02-13 17:11:22 +01:00
|
|
|
|
2019-10-02 18:23:14 +02:00
|
|
|
websitesCmd := cmd.Command("websites", "")
|
|
|
|
websitesCmd.Action(func(_ *kingpin.ParseContext) error {
|
|
|
|
return runNettestGroup("websites", ctx, network)
|
|
|
|
})
|
|
|
|
imCmd := cmd.Command("im", "")
|
|
|
|
imCmd.Action(func(_ *kingpin.ParseContext) error {
|
|
|
|
return runNettestGroup("im", ctx, network)
|
|
|
|
})
|
|
|
|
performanceCmd := cmd.Command("performance", "")
|
|
|
|
performanceCmd.Action(func(_ *kingpin.ParseContext) error {
|
|
|
|
return runNettestGroup("performance", ctx, network)
|
|
|
|
})
|
|
|
|
middleboxCmd := cmd.Command("middlebox", "")
|
|
|
|
middleboxCmd.Action(func(_ *kingpin.ParseContext) error {
|
|
|
|
return runNettestGroup("middlebox", ctx, network)
|
|
|
|
})
|
|
|
|
allCmd := cmd.Command("all", "").Default()
|
|
|
|
allCmd.Action(func(_ *kingpin.ParseContext) error {
|
|
|
|
log.Infof("Running %s tests", color.BlueString("all"))
|
|
|
|
for tg := range groups.NettestGroups {
|
|
|
|
if err := runNettestGroup(tg, ctx, network); err != nil {
|
|
|
|
log.WithError(err).Errorf("failed to run %s", tg)
|
2018-03-08 13:46:21 +01:00
|
|
|
}
|
2018-02-13 17:11:22 +01:00
|
|
|
}
|
2019-10-02 18:23:14 +02:00
|
|
|
return nil
|
2018-02-07 19:02:18 +01:00
|
|
|
})
|
|
|
|
}
|