2018-02-07 19:02:18 +01:00
|
|
|
package ooni
|
|
|
|
|
|
|
|
import (
|
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
|
|
|
|
2018-02-12 16:45:13 +01:00
|
|
|
"github.com/apex/log"
|
2018-05-03 14:59:55 +02:00
|
|
|
"github.com/ooni/probe-cli/config"
|
2018-06-25 17:49:17 +02:00
|
|
|
"github.com/ooni/probe-cli/internal/bindata"
|
2018-05-03 14:59:55 +02:00
|
|
|
"github.com/ooni/probe-cli/internal/database"
|
Integrate further with ooni/probe-engine: episode two (#46)
* utils/geoip.go: use github.com/ooni/probe-engine
Let's start using the engine by rewriting utils/geoip.go to
be just a thin wrapper around the engine functionality.
* Ready for review
* Checkpoint: the im tests are converted
Still have some doubts with respect to the variables that
are passed to MK via probe-engine. Will double check.
* fix(i/c/r/run.go): write the correct logic
* nettests: one more comment and also fix a format string
* Tweak previous
* progress
* Fix doofus
* better comment
* XXX => actionable comment
* Add glue to simplify test keys management
Making the concept of measurement more abstract in the engine is
not feasible because, when submitting a measurement, we need to
modify it to update the report ID and the measurement ID. Therefore,
returning a serialized measurement is not a good idea. We will
keep using a model.Measurement in the engine.
Changing model.Measurement.TestKeys's type from a `interface{}`
pointing to a well defined data structure to `map[string]interface{}`
is a regression because means that we are moving from code that
has a clear and defined structure to code that is more complicated
to parse and validate. Since we're already suffering havily from
the lack of a good schema, I'm not going to make the situation
worst by worsening the engine. At least for ndt7 and psiphon, we
now have a good schema and I don't want to lose that.
However, the current code in this repository is expecting the
test keys to be a `map[string]interface{}`. This choice was
dictated by the fact that we receive a JSON from Measurement Kit
and by the fact that there's not a clear schema.
To solve this tension, in this commit I am going to write glue
adapter code that makes sure that the TestKeys of a Measurement
are converted to `map[string]interface{}`. This will be done
using a type cast where possible and JSON serialization and parsing
otherwise. In a perfect world, glue is not a good idea, but in a
real world it may actually be useful.
When all tests in the engine will have a clear Go data structure,
we'll then remove the glue and just cast to the proper data
structure from `interface{}` where required.
* nettests/performance: use probe-engine
* go.{mod,sum}: upgrade to latest probe-engine
* nettests/middlebox: use ooni/probe-engine
* Update to the latest probe-engine
* web_connectivity: rewrite to use probe-engine
* Cosmetic change suggested by @hellais
* nettests/nettests.go: remove unused code
* nettests/nettests.go: fix progress
* nettests/nettests.go: remove go-measurement-kit code
* We don't depend on go-measurement-kit anymore
* Improve non-verbose output where possible
See also: https://github.com/measurement-kit/measurement-kit/issues/1856
* Make web_connectivity output pleasant
* Update to the latest probe-engine
* nettests/nettests.go: honour sharing settings
* Update to the latest probe-engine
* Use log.WithFields for probe-engine
* Update go.mod go.sum
* Revert "Update go.mod go.sum"
This reverts commit 5ecd38d8236f4a4e9b77ddb8e8a0d1e3cdd4b818.
* Revert "Revert "Update go.mod go.sum""
This reverts commit 6114b31eca98826112032776bd0feff02d763ecd.
* Upgrade ooni/probe-engine
* Unset GOPATH before running go build commands
* Dockefile: fix linux build by using latest
* Update to the latest ooni/probe-engine
```
go get -u github.com/ooni/probe-engine
go mod tidy
```
* Repair build
2019-08-15 18:08:43 +02:00
|
|
|
"github.com/ooni/probe-cli/internal/enginex"
|
2018-05-03 14:59:55 +02:00
|
|
|
"github.com/ooni/probe-cli/internal/legacy"
|
2018-06-25 17:14:29 +02:00
|
|
|
"github.com/ooni/probe-cli/internal/onboard"
|
2018-05-03 14:59:55 +02:00
|
|
|
"github.com/ooni/probe-cli/utils"
|
2019-05-23 16:38:46 +02:00
|
|
|
"github.com/ooni/probe-cli/version"
|
2019-10-28 14:05:05 +01:00
|
|
|
engine "github.com/ooni/probe-engine"
|
2018-02-07 19:02:18 +01:00
|
|
|
"github.com/pkg/errors"
|
2018-09-05 18:40:37 +02:00
|
|
|
"upper.io/db.v3/lib/sqlbuilder"
|
2018-02-07 19:02:18 +01:00
|
|
|
)
|
|
|
|
|
2018-02-12 17:29:03 +01:00
|
|
|
// Context for OONI Probe
|
|
|
|
type Context struct {
|
Integrate further with ooni/probe-engine: episode two (#46)
* utils/geoip.go: use github.com/ooni/probe-engine
Let's start using the engine by rewriting utils/geoip.go to
be just a thin wrapper around the engine functionality.
* Ready for review
* Checkpoint: the im tests are converted
Still have some doubts with respect to the variables that
are passed to MK via probe-engine. Will double check.
* fix(i/c/r/run.go): write the correct logic
* nettests: one more comment and also fix a format string
* Tweak previous
* progress
* Fix doofus
* better comment
* XXX => actionable comment
* Add glue to simplify test keys management
Making the concept of measurement more abstract in the engine is
not feasible because, when submitting a measurement, we need to
modify it to update the report ID and the measurement ID. Therefore,
returning a serialized measurement is not a good idea. We will
keep using a model.Measurement in the engine.
Changing model.Measurement.TestKeys's type from a `interface{}`
pointing to a well defined data structure to `map[string]interface{}`
is a regression because means that we are moving from code that
has a clear and defined structure to code that is more complicated
to parse and validate. Since we're already suffering havily from
the lack of a good schema, I'm not going to make the situation
worst by worsening the engine. At least for ndt7 and psiphon, we
now have a good schema and I don't want to lose that.
However, the current code in this repository is expecting the
test keys to be a `map[string]interface{}`. This choice was
dictated by the fact that we receive a JSON from Measurement Kit
and by the fact that there's not a clear schema.
To solve this tension, in this commit I am going to write glue
adapter code that makes sure that the TestKeys of a Measurement
are converted to `map[string]interface{}`. This will be done
using a type cast where possible and JSON serialization and parsing
otherwise. In a perfect world, glue is not a good idea, but in a
real world it may actually be useful.
When all tests in the engine will have a clear Go data structure,
we'll then remove the glue and just cast to the proper data
structure from `interface{}` where required.
* nettests/performance: use probe-engine
* go.{mod,sum}: upgrade to latest probe-engine
* nettests/middlebox: use ooni/probe-engine
* Update to the latest probe-engine
* web_connectivity: rewrite to use probe-engine
* Cosmetic change suggested by @hellais
* nettests/nettests.go: remove unused code
* nettests/nettests.go: fix progress
* nettests/nettests.go: remove go-measurement-kit code
* We don't depend on go-measurement-kit anymore
* Improve non-verbose output where possible
See also: https://github.com/measurement-kit/measurement-kit/issues/1856
* Make web_connectivity output pleasant
* Update to the latest probe-engine
* nettests/nettests.go: honour sharing settings
* Update to the latest probe-engine
* Use log.WithFields for probe-engine
* Update go.mod go.sum
* Revert "Update go.mod go.sum"
This reverts commit 5ecd38d8236f4a4e9b77ddb8e8a0d1e3cdd4b818.
* Revert "Revert "Update go.mod go.sum""
This reverts commit 6114b31eca98826112032776bd0feff02d763ecd.
* Upgrade ooni/probe-engine
* Unset GOPATH before running go build commands
* Dockefile: fix linux build by using latest
* Update to the latest ooni/probe-engine
```
go get -u github.com/ooni/probe-engine
go mod tidy
```
* Repair build
2019-08-15 18:08:43 +02:00
|
|
|
Config *config.Config
|
|
|
|
DB sqlbuilder.Database
|
|
|
|
IsBatch bool
|
2019-10-28 14:05:05 +01:00
|
|
|
Session *engine.Session
|
2018-03-23 12:41:06 +01:00
|
|
|
|
|
|
|
Home string
|
|
|
|
TempDir string
|
2018-03-23 12:10:14 +01:00
|
|
|
|
|
|
|
dbPath string
|
|
|
|
configPath string
|
2018-02-12 16:45:13 +01:00
|
|
|
}
|
|
|
|
|
2018-03-23 12:41:06 +01:00
|
|
|
// MaybeLocationLookup will lookup the location of the user unless it's already cached
|
|
|
|
func (c *Context) MaybeLocationLookup() error {
|
2019-10-28 14:05:05 +01:00
|
|
|
return c.Session.MaybeLookupLocation()
|
2018-03-23 12:41:06 +01:00
|
|
|
}
|
|
|
|
|
2018-06-22 11:01:15 +02:00
|
|
|
// MaybeOnboarding will run the onboarding process only if the informed consent
|
|
|
|
// config option is set to false
|
|
|
|
func (c *Context) MaybeOnboarding() error {
|
|
|
|
if c.Config.InformedConsent == false {
|
2018-09-17 16:16:56 +02:00
|
|
|
if c.IsBatch == true {
|
|
|
|
return errors.New("cannot run onboarding in batch mode")
|
|
|
|
}
|
2018-06-25 17:14:29 +02:00
|
|
|
if err := onboard.Onboarding(c.Config); err != nil {
|
2018-06-22 11:01:15 +02:00
|
|
|
return errors.Wrap(err, "onboarding")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-02-12 16:45:13 +01:00
|
|
|
// Init the OONI manager
|
2018-02-12 17:29:03 +01:00
|
|
|
func (c *Context) Init() error {
|
2018-03-23 12:10:14 +01:00
|
|
|
var err error
|
|
|
|
|
|
|
|
if err = legacy.MaybeMigrateHome(); err != nil {
|
2018-02-12 16:45:13 +01:00
|
|
|
return errors.Wrap(err, "migrating home")
|
|
|
|
}
|
2018-03-23 12:10:14 +01:00
|
|
|
|
2018-03-23 15:38:25 +01:00
|
|
|
if err = MaybeInitializeHome(c.Home); err != nil {
|
2018-03-23 12:10:14 +01:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if c.configPath != "" {
|
|
|
|
log.Debugf("Reading config file from %s", c.configPath)
|
2018-06-22 10:29:47 +02:00
|
|
|
c.Config, err = config.ReadConfig(c.configPath)
|
2018-03-23 12:10:14 +01:00
|
|
|
} else {
|
|
|
|
log.Debug("Reading default config file")
|
2018-06-25 17:49:17 +02:00
|
|
|
c.Config, err = InitDefaultConfig(c.Home)
|
2018-03-23 12:10:14 +01:00
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-11-14 17:58:31 +01:00
|
|
|
if err = c.Config.MaybeMigrate(); err != nil {
|
|
|
|
return errors.Wrap(err, "migrating config")
|
|
|
|
}
|
2018-03-23 12:10:14 +01:00
|
|
|
|
2018-03-27 15:09:34 +02:00
|
|
|
c.dbPath = utils.DBDir(c.Home, "main")
|
2018-03-23 12:10:14 +01:00
|
|
|
log.Debugf("Connecting to database sqlite3://%s", c.dbPath)
|
|
|
|
db, err := database.Connect(c.dbPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.DB = db
|
|
|
|
|
2018-03-19 13:20:42 +01:00
|
|
|
tempDir, err := ioutil.TempDir("", "ooni")
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "creating TempDir")
|
|
|
|
}
|
|
|
|
c.TempDir = tempDir
|
|
|
|
|
2019-10-28 14:05:05 +01:00
|
|
|
sess, err := engine.NewSession(engine.SessionConfig{
|
|
|
|
Logger: enginex.Logger,
|
|
|
|
SoftwareName: "ooniprobe-desktop",
|
|
|
|
SoftwareVersion: version.Version,
|
|
|
|
AssetsDir: utils.AssetsDir(c.Home),
|
|
|
|
TempDir: c.TempDir,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.Session = sess
|
|
|
|
|
2018-02-12 16:45:13 +01:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-05-23 16:38:46 +02:00
|
|
|
// NewContext creates a new context instance.
|
2018-03-23 12:10:14 +01:00
|
|
|
func NewContext(configPath string, homePath string) *Context {
|
2018-02-12 17:29:03 +01:00
|
|
|
return &Context{
|
2018-03-23 12:10:14 +01:00
|
|
|
Home: homePath,
|
2018-06-22 10:29:47 +02:00
|
|
|
Config: &config.Config{},
|
2018-03-23 12:10:14 +01:00
|
|
|
configPath: configPath,
|
2018-02-12 16:45:13 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-23 15:38:25 +01:00
|
|
|
// MaybeInitializeHome does the setup for a new OONI Home
|
|
|
|
func MaybeInitializeHome(home string) error {
|
2018-03-27 15:09:34 +02:00
|
|
|
for _, d := range utils.RequiredDirs(home) {
|
|
|
|
if _, e := os.Stat(d); e != nil {
|
|
|
|
if err := os.MkdirAll(d, 0700); err != nil {
|
2018-03-23 12:10:14 +01:00
|
|
|
return err
|
2018-03-19 13:20:42 +01:00
|
|
|
}
|
2018-03-08 20:46:31 +01:00
|
|
|
}
|
|
|
|
}
|
2018-03-23 12:10:14 +01:00
|
|
|
return nil
|
2018-03-08 20:46:31 +01:00
|
|
|
}
|
|
|
|
|
2018-06-25 17:49:17 +02:00
|
|
|
// InitDefaultConfig reads the config from common locations or creates it if
|
|
|
|
// missing.
|
|
|
|
func InitDefaultConfig(home string) (*config.Config, error) {
|
|
|
|
var (
|
|
|
|
err error
|
|
|
|
c *config.Config
|
|
|
|
configPath = utils.ConfigPath(home)
|
|
|
|
)
|
|
|
|
|
|
|
|
c, err = config.ReadConfig(configPath)
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
log.Debugf("writing default config to %s", configPath)
|
|
|
|
var data []byte
|
|
|
|
data, err = bindata.Asset("data/default-config.json")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
err = ioutil.WriteFile(
|
|
|
|
configPath,
|
|
|
|
data,
|
|
|
|
0644,
|
|
|
|
)
|
2018-02-12 16:45:13 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-06-25 17:49:17 +02:00
|
|
|
return InitDefaultConfig(home)
|
2018-02-12 16:45:13 +01:00
|
|
|
}
|
2018-06-25 17:49:17 +02:00
|
|
|
return nil, err
|
2018-02-12 16:45:13 +01:00
|
|
|
}
|
2018-03-08 11:25:40 +01:00
|
|
|
|
2018-06-25 17:49:17 +02:00
|
|
|
return c, nil
|
2018-02-07 19:02:18 +01:00
|
|
|
}
|