From e7ee54436e7ea889d14341c09176c74f8edd9ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 13 Feb 2018 18:11:22 +0200 Subject: [PATCH] Get some data into the database --- data/migrations/1_create_msmt_results.sql | 26 +++---- internal/bindata/bindata.go | 27 ++++--- internal/cli/run/run.go | 30 +++++++- internal/database/database.go | 12 ++- internal/database/models.go | 93 +++++++++++++++++++---- nettests/groups/groups.go | 14 ---- 6 files changed, 137 insertions(+), 65 deletions(-) diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index dc58c40..dbe2d39 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -12,35 +12,31 @@ DROP TABLE `measurements`; CREATE TABLE `results` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), - `startTime` DATETIME, - `endTime` DATETIME, + `start_time` DATETIME, + `end_time` DATETIME, `summary` JSON, `done` TINYINT(1), - `dataUsageUp` INTEGER, - `dataUsageDown` INTEGER, - `createdAt` DATETIME NOT NULL, - `updatedAt` DATETIME NOT NULL + `data_usage_up` INTEGER, + `data_usage_down` INTEGER ); CREATE TABLE `measurements` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), - `startTime` DATETIME, - `endTime` DATETIME, + `start_time` DATETIME, + `end_time` DATETIME, `summary` JSON, `ip` VARCHAR(255), `asn` INTEGER, `country` VARCHAR(2), - `networkName` VARCHAR(255), + `network_name` VARCHAR(255), `state` TEXT, `failure` VARCHAR(255), - `reportFile` VARCHAR(255), - `reportId` VARCHAR(255), + `report_file` VARCHAR(255), + `report_id` VARCHAR(255), `input` VARCHAR(255), - `measurementId` VARCHAR(255), - `createdAt` DATETIME NOT NULL, - `updatedAt` DATETIME NOT NULL, - `resultId` INTEGER REFERENCES `results` (`id`) ON DELETE SET NULL ON UPDATE CASCADE + `measurement_id` VARCHAR(255), + `result_id` INTEGER REFERENCES `results` (`id`) ON DELETE SET NULL ON UPDATE CASCADE ); -- +migrate StatementEnd diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index 5b334d2..68822e5 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -118,20 +118,19 @@ func dataDefaultConfigJson() (*asset, error) { } var _dataMigrations1_create_msmt_resultsSql = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x93\xcf\x8e\xda\x30\x10\xc6\xef\x79\x8a\x39\x82\x5a\x0e\xad\xc4\x89" + - "\x93\x49\x86\xd6\x6d\x70\x90\xe3\x54\xe5\x16\xab\x76\x91\x55\xe2\x44\x8e\x23\xd4\xb7\x5f\x19\x02\x1b\xb2\x81\xcb" + - "\x5e\xf6\x3a\xbf\x6f\xfe\x78\x3e\xcf\x62\x01\x9f\x2a\x73\x70\xd2\x6b\x48\xea\x93\x8d\x86\x81\xdc\x4b\xaf\x2b\x6d" + - "\xfd\x5a\x1f\x8c\x8d\xa2\x84\x67\x3b\x10\x64\x9d\x22\x94\x4e\xb7\xdd\xd1\xb7\xe5\xea\x2e\x5a\x69\xd9\x76\xee\x9c" + - "\x13\xd0\x74\x35\xb4\xea\x9e\x14\xcd\xd3\xb6\x31\x47\x22\x70\xdc\x18\x66\x11\x00\x40\x69\x54\x09\x94\x09\xfc\x86" + - "\x1c\x76\x9c\x6e\x09\xdf\xc3\x4f\xdc\x03\x29\x44\x46\x59\xcc\x71\x8b\x4c\x7c\xbe\x68\xad\xac\x74\x09\xbf\x08\x8f" + - "\xbf\x13\x3e\xfb\xba\x5c\xce\x7b\xd0\x7a\xe9\xbc\x30\x81\x26\x44\xa0\xa0\x5b\xec\x89\xb6\x6a\x32\xde\x76\x55\x25" + - "\xdd\xff\x12\x7e\xe4\x19\xeb\x63\xaa\xb6\xba\x04\x41\xd9\x9e\x32\x31\xfb\x72\x2d\xae\xa4\x97\x45\x2b\x0f\xba\x68" + - "\x6e\xa3\x8e\x51\xd8\xfd\x18\xfe\x71\x5a\x7a\xad\x88\x7f\x6d\x0e\x2c\x13\xc0\x8a\x34\xed\x25\x5d\xa3\x1e\x4a\xa2" + - "\xf9\x6a\xbc\xbb\x3b\x7b\x3e\xe0\x02\x4d\x33\x59\x5c\xb6\x6f\x77\x53\x77\xd6\x87\xec\x9b\xfc\x2a\xb6\xda\x9f\x6a" + - "\xf7\x8f\x3d\x99\xd4\x07\x97\xf0\xf7\xf5\x51\x7f\xa5\x39\x76\x6e\x5a\xed\x74\x53\x3b\xbf\x31\xc7\x67\x98\xaa\x49" + - "\x68\x6c\xd3\xf9\x49\x32\xf0\xe1\x41\xee\x3b\xbd\xbf\x8d\x17\x6e\x85\x0e\x0c\xe6\xb8\x41\x8e\x2c\xc6\x7c\x78\x49" + - "\xe1\x0f\xcc\x21\x63\x90\x60\x8a\x02\x21\xc7\x4b\x99\x10\x2a\x76\xa1\x3a\xc4\x24\x8f\x49\x82\xe7\x4f\xf5\xf0\xaa" + - "\x5f\x02\x00\x00\xff\xff\xe7\x43\x45\x75\x4e\x04\x00\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x93\x4f\x6f\xf2\x30\x0c\xc6\xef\xfd\x14\x3e\x82\xde\x97\xc3\x26\x71" + + "\xe2\x14\x5a\x6f\xeb\x56\x52\x94\xa6\xd3\x38\xb5\xd1\x1a\x50\x34\x9a\x56\x69\x22\xb4\x6f\x3f\x85\x7f\x2a\xac\x70" + + "\xde\xd5\xbf\xc7\x8e\xfd\xd8\x99\x4c\xe0\x5f\xad\x36\x46\x58\x09\x51\xb3\xd3\x41\x3f\x90\x59\x61\x65\x2d\xb5\x9d" + + "\xcb\x8d\xd2\x41\x10\xb1\x74\x09\x9c\xcc\x13\x84\xd2\xc8\xce\x6d\x6d\x57\xce\x2e\xa2\xb5\x14\x9d\x33\xfb\x1c\x8f" + + "\x86\xab\xa1\xae\x2e\x49\xde\xde\x7d\x36\x64\x48\x38\x5e\x3f\x0c\xa3\x00\x00\xa0\x54\x55\x09\x31\xe5\xf8\x8c\x0c" + + "\x96\x2c\x5e\x10\xb6\x82\x37\x5c\x01\xc9\x79\x1a\xd3\x90\xe1\x02\x29\xff\x7f\xd0\x6a\x51\xcb\x12\xde\x09\x0b\x5f" + + "\x08\x1b\x3d\x4e\xa7\xe3\x23\xe8\xac\x30\xb6\xb0\xca\xe3\x88\x70\xe4\xf1\x02\x8f\x48\xea\x6a\x18\x74\xae\xae\x85" + + "\xf9\x2e\xe1\x35\x4b\xe9\x31\x56\x35\x5a\x96\xc0\x63\xba\x8a\x29\x1f\x3d\x9c\xca\x57\xc2\x8a\xc2\x75\x62\x23\x0b" + + "\xd7\x9e\xdb\xfd\x0d\xab\x66\xa7\xcf\x38\x18\xcf\xae\x67\xbf\xb0\xf7\x4f\x1a\xa0\xda\xc1\xf2\xa2\xd3\xd7\x63\x7f" + + "\x36\x4e\x5b\x9f\x7d\x96\x9f\xc4\x5a\xda\x5d\x63\xbe\x8a\x7b\xcd\x5a\x6f\x33\x7e\x9c\xe6\x5a\x0b\xb5\x75\x66\x58" + + "\x6d\x64\xdb\x18\x5b\xac\xd5\xf6\x2e\xf7\x36\x0e\x50\xa5\x5b\x67\x07\x49\x6f\x19\xb7\x92\x0f\xb7\x5a\xf4\x37\xc4" + + "\xf0\x09\x19\xd2\x10\xb3\xfe\x29\xfb\x25\x8e\x21\xa5\x10\x61\x82\x1c\x21\x43\x0e\x34\x4f\x12\x1f\xca\x97\xde\x77" + + "\x08\x49\x16\x92\x08\xf7\x57\x71\xf3\x5b\xfd\x04\x00\x00\xff\xff\xc4\x16\x0a\x20\xcf\x03\x00\x00") func dataMigrations1_create_msmt_resultsSqlBytes() ([]byte, error) { return bindataRead( diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 557e4a7..b18196b 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -1,10 +1,14 @@ package run import ( + "time" + "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/openobservatory/gooni/internal/cli/root" + "github.com/openobservatory/gooni/internal/database" "github.com/openobservatory/gooni/internal/util" + "github.com/openobservatory/gooni/nettests" "github.com/openobservatory/gooni/nettests/groups" ) @@ -15,15 +19,33 @@ func init() { cmd.Action(func(_ *kingpin.ParseContext) error { util.Log("Starting %s", *nettestGroup) - config, ooni, err := root.Init() + _, ctx, err := root.Init() if err != nil { log.Errorf("%s", err) return err } - log.Infof("%s", config) - log.Infof("%s", ooni) + group := groups.NettestGroups[*nettestGroup] + log.Debugf("Running test group %s", group.Label) - groups.Run(*nettestGroup, ooni) + result, err := database.CreateResult(ctx.DB, database.Result{ + Name: *nettestGroup, + StartTime: time.Now().UTC(), // XXX get this from MK + }) + if err != nil { + log.Errorf("%s", err) + return err + } + + for _, nt := range group.Nettests { + ctl := nettests.NewController(ctx) + nt.Run(ctl) + // XXX + // 1. Generate the summary + // 2. Link the measurement to the Result (this should probably happen in + // the nettest class) + // 3. Update the summary of the result and the other metadata in the db + } + result.Update(ctx.DB) return nil }) } diff --git a/internal/database/database.go b/internal/database/database.go index 8336206..84abc07 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -5,7 +5,7 @@ import ( "github.com/apex/log" "github.com/jmoiron/sqlx" - _ "github.com/mattn/go-sqlite3" + _ "github.com/mattn/go-sqlite3" // this is needed to load the sqlite3 driver ooni "github.com/openobservatory/gooni" "github.com/openobservatory/gooni/internal/bindata" "github.com/pkg/errors" @@ -14,6 +14,7 @@ import ( // RunMigrations runs the database migrations func RunMigrations(db *sqlx.DB) error { + log.Debugf("running migrations") migrations := &migrate.AssetMigrationSource{ Asset: bindata.Asset, AssetDir: bindata.AssetDir, @@ -27,19 +28,24 @@ func RunMigrations(db *sqlx.DB) error { return nil } +// Connect to the database func Connect(path string) (*sqlx.DB, error) { db, err := sqlx.Connect("sqlite3", path) if err != nil { return nil, err } - // XXX RunMigrations(db) + err = RunMigrations(db) + if err != nil { + return nil, err + } return db, nil } +// DefaultDatabasePath for the main database func DefaultDatabasePath() (string, error) { home, err := ooni.GetOONIHome() if err != nil { return "", errors.Wrap(err, "default database path") } - return filepath.Join(home, "db", "main.db"), nil + return filepath.Join(home, "db", "main.sqlite3"), nil } diff --git a/internal/database/models.go b/internal/database/models.go index 78d1c6f..e9afcd2 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -1,34 +1,97 @@ package database -import "time" +import ( + "time" + + "github.com/apex/log" + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" +) // Measurement model type Measurement struct { - ID int `db:"id"` + ID int64 `db:"id"` Name string `db:"name"` - StartTime time.Time `db:"startTime"` - EndTime time.Time `db:"endTime"` + StartTime time.Time `db:"start_time"` + EndTime time.Time `db:"end_time"` Summary string `db:"summary"` // XXX this should be JSON - ASN int `db:"asn"` + ASN int64 `db:"asn"` IP string `db:"ip"` CountryCode string `db:"country"` State string `db:"state"` Failure string `db:"failure"` - ReportFilePath string `db:"reportFile"` - ReportID string `db:"reportId"` + ReportFilePath string `db:"report_file"` + ReportID string `db:"report_id"` Input string `db:"input"` - MeasurementID string `db:"measurementId"` - ResultID string `db:"resultId"` + MeasurementID string `db:"measurement_id"` + ResultID string `db:"result_id"` +} + +// CreateMeasurement writes the measurement to the database a returns a pointer +// to the Measurement +func CreateMeasurement(db *sqlx.DB, m Measurement) (*Measurement, error) { + res, err := db.NamedExec(`INSERT INTO measurements + (name, start_time, + summary, asn, ip, country, + state, failure, report_file, + report_id, input, measurement_id, + result_id) + VALUES (:name,:start_time, + :summary,:asn,:ip,:country, + :state,:failure,:report_file, + :report_id,:input,:measurement_id, + :result_id)`, + m) + if err != nil { + return nil, errors.Wrap(err, "creating measurement") + } + id, err := res.LastInsertId() + if err != nil { + return nil, errors.Wrap(err, "creating measurement") + } + m.ID = id + return &m, nil +} + +// Update the measurement in the database +func (r Measurement) Update(db *sqlx.DB) error { + // XXX implement me + return nil } // Result model type Result struct { - ID int `db:"id"` - Name int `db:"name"` - StartTime time.Time `db:"startTime"` - EndTime time.Time `db:"endTime"` + ID int64 `db:"id"` + Name string `db:"name"` + StartTime time.Time `db:"start_time"` + EndTime time.Time `db:"end_time"` Summary string `db:"summary"` // XXX this should be JSON Done bool `db:"done"` - DataUsageUp int `db:"dataUsageUp"` - DataUsageDown int `db:"dataUsageDown"` + DataUsageUp int64 `db:"data_usage_up"` + DataUsageDown int64 `db:"data_usage_down"` +} + +// Update the Result in the database +func (r Result) Update(db *sqlx.DB) error { + // XXX implement me + return nil +} + +// CreateResult writes the Result to the database a returns a pointer +// to the Result +func CreateResult(db *sqlx.DB, r Result) (*Result, error) { + log.Debugf("Creating result %s", r) + res, err := db.NamedExec(`INSERT INTO results + (name, start_time) + VALUES (:name,:start_time)`, + r) + if err != nil { + return nil, errors.Wrap(err, "creating result") + } + id, err := res.LastInsertId() + if err != nil { + return nil, errors.Wrap(err, "creating measurement") + } + r.ID = id + return &r, nil } diff --git a/nettests/groups/groups.go b/nettests/groups/groups.go index 663e21f..033bd62 100644 --- a/nettests/groups/groups.go +++ b/nettests/groups/groups.go @@ -1,8 +1,6 @@ package groups import ( - "github.com/apex/log" - ooni "github.com/openobservatory/gooni" "github.com/openobservatory/gooni/nettests" "github.com/openobservatory/gooni/nettests/performance" "github.com/openobservatory/gooni/nettests/websites" @@ -31,15 +29,3 @@ var NettestGroups = map[string]nettests.NettestGroup{ Nettests: []nettests.Nettest{}, }, } - -// Run runs a specific test group -func Run(name string, ctx *ooni.Context) error { - group := NettestGroups[name] - log.Debugf("Running test group %s", group.Label) - - for _, nt := range group.Nettests { - ctl := nettests.NewController(ctx) - nt.Run(ctl) - } - return nil -}