diff --git a/Gopkg.lock b/Gopkg.lock index 9edafb4..459507d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -37,6 +37,27 @@ packages = ["."] revision = "cc83f3b3ce5911279513a46d6d3316d67bedaa54" +[[projects]] + branch = "master" + name = "github.com/beorn7/perks" + packages = ["quantile"] + revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9" + +[[projects]] + name = "github.com/golang/protobuf" + packages = ["proto"] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/jmoiron/sqlx" + packages = [ + ".", + "reflectx" + ] + revision = "05cef0741ade10ca668982355b3f3f0bcf0ff0a8" + [[projects]] name = "github.com/mattn/go-colorable" packages = ["."] @@ -49,6 +70,18 @@ revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" +[[projects]] + name = "github.com/mattn/go-sqlite3" + packages = ["."] + revision = "6c771bb9887719704b210e87e934f08be014bdb1" + version = "v1.6.0" + +[[projects]] + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" + version = "v1.0.0" + [[projects]] branch = "master" name = "github.com/mgutz/ansi" @@ -67,6 +100,49 @@ revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" +[[projects]] + name = "github.com/prometheus/client_golang" + packages = ["prometheus"] + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" + +[[projects]] + branch = "master" + name = "github.com/prometheus/client_model" + packages = ["go"] + revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" + +[[projects]] + branch = "master" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model", + "version" + ] + revision = "89604d197083d4781071d3c65855d24ecfb0a563" + +[[projects]] + branch = "master" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs" + ] + revision = "282c8707aa210456a825798969cc27edda34992a" + +[[projects]] + branch = "master" + name = "github.com/rubenv/sql-migrate" + packages = [ + ".", + "sqlparse" + ] + revision = "f33734611e84d5fe45f35eccf0174f4836af4542" + [[projects]] branch = "master" name = "golang.org/x/sys" @@ -83,9 +159,15 @@ revision = "0aa8b6a162b391fe2d95648b7677d1d6ac2090a6" version = "v1.4.1" +[[projects]] + name = "gopkg.in/gorp.v1" + packages = ["."] + revision = "c87af80f3cc5036b55b83d77171e156791085e2e" + version = "v1.7.1" + [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "b014c3dfbccece13286bfddc98c8ec74007629647cd54df8d9af9d94985cdb17" + inputs-digest = "d1ff03c816b12576a88c6ab2c3ffab44bf0fd40a67d765965f06fd9ac9e58d99" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index 405c7eb..9e2100e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,18 @@ +GO_BINDATA_VERSION = $(shell go-bindata --version | cut -d ' ' -f2 | head -n 1 || echo "missing") +REQ_GO_BINDATA_VERSION = 3.2.0 GO ?= go build: @echo "Building ./ooni" @$(GO) build -o ooni cmd/ooni/main.go .PHONY: build + +bindata: +ifneq ($(GO_BINDATA_VERSION),$(REQ_GO_BINDATA_VERSION)) + go get -u github.com/shuLhan/go-bindata/...; +endif + @go-bindata \ + -nometadata \ + -o internal/bindata/bindata.go -pkg bindata \ + data; +.PHONY: bindata diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql new file mode 100644 index 0000000..dc58c40 --- /dev/null +++ b/data/migrations/1_create_msmt_results.sql @@ -0,0 +1,46 @@ +-- +migrate Down +-- +migrate StatementBegin + +DROP TABLE `results`; +DROP TABLE `measurements`; + +-- +migrate StatementEnd + +-- +migrate Up +-- +migrate StatementBegin + +CREATE TABLE `results` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `name` VARCHAR(255), + `startTime` DATETIME, + `endTime` DATETIME, + `summary` JSON, + `done` TINYINT(1), + `dataUsageUp` INTEGER, + `dataUsageDown` INTEGER, + `createdAt` DATETIME NOT NULL, + `updatedAt` DATETIME NOT NULL +); + +CREATE TABLE `measurements` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `name` VARCHAR(255), + `startTime` DATETIME, + `endTime` DATETIME, + `summary` JSON, + `ip` VARCHAR(255), + `asn` INTEGER, + `country` VARCHAR(2), + `networkName` VARCHAR(255), + `state` TEXT, + `failure` VARCHAR(255), + `reportFile` VARCHAR(255), + `reportId` VARCHAR(255), + `input` VARCHAR(255), + `measurementId` VARCHAR(255), + `createdAt` DATETIME NOT NULL, + `updatedAt` DATETIME NOT NULL, + `resultId` INTEGER REFERENCES `results` (`id`) ON DELETE SET NULL ON UPDATE CASCADE +); + +-- +migrate StatementEnd diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go new file mode 100644 index 0000000..f45dfc4 --- /dev/null +++ b/internal/bindata/bindata.go @@ -0,0 +1,263 @@ +// Code generated by go-bindata. DO NOT EDIT. +// sources: +// data/default-config.json + +package bindata + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info fileInfoEx +} + +type fileInfoEx interface { + os.FileInfo + MD5Checksum() string +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time + md5checksum string +} + +func (fi bindataFileInfo) Name() string { + return fi.name +} +func (fi bindataFileInfo) Size() int64 { + return fi.size +} +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} +func (fi bindataFileInfo) MD5Checksum() string { + return fi.md5checksum +} +func (fi bindataFileInfo) IsDir() bool { + return false +} +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _dataDefaultConfigJson = []byte( + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\xcd\x6e\xe3\x38\x0c\xc7\xef\x7d\x0a\x41\xe7\x3a\xcd\x62\x6f\x39" + + "\xee\x6d\x0f\xbb\x1d\x60\xe6\x56\x14\x82\x62\xd1\x36\x31\x32\xa9\x11\xe9\x64\x82\x41\xdf\x7d\x20\x35\x89\xed\x7e" + + "\x4d\x8f\xe2\x9f\xa2\xc8\x1f\x29\xfe\xba\x31\xc6\x3a\xbb\x33\xf6\xdb\x80\x62\x50\xcc\x89\xa7\x6c\xee\xef\xff\xff" + + "\xd7\x7c\xc9\xbc\x07\xd3\x32\x75\xd8\x9b\x0e\x23\x6c\xcc\x57\x00\x33\xa8\x26\xd9\xdd\xdd\x31\x13\x6e\x90\xef\x06" + + "\x88\xa9\x1e\x52\xf1\x6f\xda\x88\xa6\xe3\x6c\x8a\xd9\xde\x96\xf0\x7e\x52\x76\x53\x0a\x5e\xc1\xee\x8c\xe6\x09\xaa" + + "\x59\x06\x9f\x91\x7a\xbb\x33\x25\x09\x63\x2c\x52\x1b\xa7\x00\x0e\x93\xdd\x99\xce\x47\xa9\x7e\x0b\xc1\x0b\x2d\x02" + + "\x2c\x84\x3e\xc9\x5a\x98\x52\x64\x1f\x5c\x06\x99\xa2\xbe\xd0\x04\x28\xb8\x36\x7b\x19\x5c\x86\xc4\xf9\xaa\xdf\x18" + + "\xf3\x54\x33\x23\x56\xec\xb0\xf5\x8a\x4c\x32\xe7\x07\xe4\xf7\x11\xc2\x3a\x5a\xf5\x3d\x39\x26\xa7\x20\xea\x5a\x1e" + + "\x53\x84\x72\xf1\x3d\x37\x82\xa3\x5c\xea\xbb\xbe\x58\x10\x8d\x5e\x21\xd4\x28\x2b\x2a\xf3\xab\x4b\x24\x67\x6b\x75" + + "\x2f\xe1\x1e\xaa\xd9\x18\x7b\x84\x7d\xd3\x32\x11\xb4\x8a\x07\xd4\x93\xbd\xbd\x28\x9d\x6f\x61\xcf\xfc\xbd\x19\x41" + + "\x04\xa8\x87\x3c\x6b\xc7\xc1\xab\xf8\x94\x66\x8b\x42\x84\x3e\xfb\x71\xb6\x04\x2f\xc3\x7c\xa2\xa0\xf3\xa1\x8c\x44" + + "\x83\x74\xf0\x11\x43\x93\xe1\xc7\x04\xa2\x4d\x44\x82\x17\x2e\x03\xf8\x00\xb9\xe9\x10\x62\x68\x46\x4f\x98\xa6\x58" + + "\x29\xdb\xea\xf6\x78\x2e\x6e\x64\xd2\x21\x9e\x9c\x8f\x91\x8f\x9e\xda\x32\x36\xf6\xef\xed\xf6\xbf\x7f\xec\x95\x58" + + "\xa5\x2d\xa0\x05\xd6\xa2\x47\x47\xd8\x0b\x2a\xcc\x96\x05\xab\xd6\x2b\xf4\x9c\xb1\xaa\x0f\x8f\x55\x7e\xba\x4e\x92" + + "\xa8\x27\x75\x85\x8d\xef\x97\x0d\xf8\x00\xf6\xc7\x50\xdf\xc2\xba\x04\x7b\x36\xad\xf3\x48\x90\x3b\xce\xe3\xb9\xe8" + + "\xcf\x64\x50\x1a\x71\x09\xb5\xec\x8e\x13\xc8\x07\xc8\x05\x5d\x99\x2e\xfb\x86\xe6\xca\xf4\xbf\x76\x28\x8d\x7e\xf7" + + "\xf6\x42\x5c\x5f\x5f\x95\x31\x62\x08\x11\xf6\xfc\xf3\x93\x45\xfc\x79\x80\x3e\x39\x42\x57\x9e\xf3\xd7\x0a\x87\x42" + + "\x33\xbc\xde\x33\x2d\x4f\xa4\xf9\xf4\x62\x73\x08\xb8\xc0\xa3\x47\x72\x5d\x66\x3a\xff\xc5\xd5\x7a\x70\xcf\x2b\xd1" + + "\x1d\x20\xcb\xf3\x47\xb7\xdb\xcd\x76\xf3\xd7\xf3\xb6\x73\x48\xa5\x83\x65\xde\x98\x04\x48\x2f\xd7\x9f\x6e\x7e\x07" + + "\x00\x00\xff\xff\x0f\x7e\x15\xb3\x6d\x05\x00\x00") + +func dataDefaultConfigJsonBytes() ([]byte, error) { + return bindataRead( + _dataDefaultConfigJson, + "data/default-config.json", + ) +} + +func dataDefaultConfigJson() (*asset, error) { + bytes, err := dataDefaultConfigJsonBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "data/default-config.json", size: 0, md5checksum: "", mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist} +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +// nolint: deadcode +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist} +} + +// AssetNames returns the names of the assets. +// nolint: deadcode +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "data/default-config.json": dataDefaultConfigJson, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist} + } + } + } + if node.Func != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist} + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} + +var _bintree = &bintree{nil, map[string]*bintree{ + "data": {nil, map[string]*bintree{ + "default-config.json": {dataDefaultConfigJson, map[string]*bintree{}}, + }}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} diff --git a/internal/cli/root/root.go b/internal/cli/root/root.go index 47d0aaa..e9a36c2 100644 --- a/internal/cli/root/root.go +++ b/internal/cli/root/root.go @@ -5,6 +5,7 @@ import ( "github.com/apex/log" "github.com/apex/log/handlers/cli" ooni "github.com/openobservatory/gooni" + "github.com/openobservatory/gooni/internal/database" "github.com/prometheus/common/version" ) @@ -15,7 +16,7 @@ var Cmd = kingpin.New("ooni", "") var Command = Cmd.Command // Init should be called by all subcommand that care to have a ooni.OONI instance -var Init func() (*ooni.Config, *ooni.OONI, error) +var Init func() (*ooni.Config, *ooni.Context, error) func init() { configPath := Cmd.Flag("config", "Set a custom config file path").Short('c').String() @@ -28,19 +29,33 @@ func init() { log.Debugf("ooni version %s", version.Version) } - Init = func() (*ooni.Config, *ooni.OONI, error) { + Init = func() (*ooni.Config, *ooni.Context, error) { var c *ooni.Config var err error if *configPath != "" { + log.Debugf("Reading config file from %s", *configPath) c, err = ooni.ReadConfig(*configPath) } else { + log.Debug("Reading default config file") c, err = ooni.ReadDefaultConfigPaths() } if err != nil { return nil, nil, err } - o := ooni.New(c) + + dbPath, err := DefaultDatabasePath() + if err != nil { + return nil, nil, err + } + + log.Debugf("Connecting to database sqlite3://%s", dbPath) + db, err := database.Connect(dbPath) + if err != nil { + return nil, nil, err + } + + o := ooni.New(c, db) o.Init() return c, o, nil } diff --git a/internal/database/database.go b/internal/database/database.go new file mode 100644 index 0000000..341e43f --- /dev/null +++ b/internal/database/database.go @@ -0,0 +1,44 @@ +package database + +import ( + "path/filepath" + + "github.com/apex/log" + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + "github.com/openobservatory/gooni/internal/bindata" + "github.com/pkg/errors" + migrate "github.com/rubenv/sql-migrate" +) + +// RunMigrations runs the database migrations +func RunMigrations(db *sqlx.DB) error { + migrations := &migrate.AssetMigrationSource{ + Asset: bindata.Asset, + AssetDir: bindata.AssetDir, + Dir: "data/migrations", + } + n, err := migrate.Exec(db.DB, "sqlite3", migrations, migrate.Up) + if err != nil { + return err + } + log.Debugf("performed %d migrations", n) + return nil +} + +func Connect(path string) (*sqlx.DB, error) { + db, err := sqlx.Connect("sqlite3", path) + if err != nil { + return nil, err + } + // XXX RunMigrations(db) + return db, nil +} + +func DefaultDatabasePath() (string, error) { + home, err := GetOONIHome() + if err != nil { + return errors.Wrap(err, "default database path") + } + return filepath.Join(home, "db", "main.db"), nil +} diff --git a/internal/database/models.go b/internal/database/models.go new file mode 100644 index 0000000..78d1c6f --- /dev/null +++ b/internal/database/models.go @@ -0,0 +1,34 @@ +package database + +import "time" + +// Measurement model +type Measurement struct { + ID int `db:"id"` + Name string `db:"name"` + StartTime time.Time `db:"startTime"` + EndTime time.Time `db:"endTime"` + Summary string `db:"summary"` // XXX this should be JSON + ASN int `db:"asn"` + IP string `db:"ip"` + CountryCode string `db:"country"` + State string `db:"state"` + Failure string `db:"failure"` + ReportFilePath string `db:"reportFile"` + ReportID string `db:"reportId"` + Input string `db:"input"` + MeasurementID string `db:"measurementId"` + ResultID string `db:"resultId"` +} + +// Result model +type Result struct { + ID int `db:"id"` + Name int `db:"name"` + StartTime time.Time `db:"startTime"` + EndTime time.Time `db:"endTime"` + Summary string `db:"summary"` // XXX this should be JSON + Done bool `db:"done"` + DataUsageUp int `db:"dataUsageUp"` + DataUsageDown int `db:"dataUsageDown"` +} diff --git a/ooni.go b/ooni.go index 8d434fd..c9772ec 100644 --- a/ooni.go +++ b/ooni.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/apex/log" + "github.com/jmoiron/sqlx" homedir "github.com/mitchellh/go-homedir" "github.com/openobservatory/gooni/config" "github.com/openobservatory/gooni/internal/legacy" @@ -30,28 +31,30 @@ func Onboarding(c *Config) error { return nil } -// OONI manager. -type OONI struct { +// Context for OONI Probe +type Context struct { config *Config + db *sqlx.DB } // Init the OONI manager -func (o *OONI) Init() error { +func (c *Context) Init() error { if err := legacy.MaybeMigrateHome(); err != nil { return errors.Wrap(err, "migrating home") } - if o.config.InformedConsent == false { - if err := Onboarding(o.config); err != nil { + if c.config.InformedConsent == false { + if err := Onboarding(c.config); err != nil { return errors.Wrap(err, "onboarding") } } return nil } -// New OONI manager instance. -func New(c *Config) *OONI { - return &OONI{ +// New Context instance. +func New(c *Config, d *sqlx.DB) *Context { + return &Context{ config: c, + db: d, } }