Get some data into the database

data/migrations/1_create_msmt_results.sql

@@ -12,35 +12,31 @@ DROP TABLE `measurements`;
 CREATE TABLE `results` (
     `id` INTEGER PRIMARY KEY AUTOINCREMENT,
     `name` VARCHAR(255),
-    `startTime` DATETIME,
+    `start_time` DATETIME,
-    `endTime` DATETIME,
+    `end_time` DATETIME,
     `summary` JSON,
     `done` TINYINT(1),
-    `dataUsageUp` INTEGER,
+    `data_usage_up` INTEGER,
-    `dataUsageDown` INTEGER,
+    `data_usage_down` INTEGER
-    `createdAt` DATETIME NOT NULL,
-    `updatedAt` DATETIME NOT NULL
 );
 
 CREATE TABLE `measurements` (
     `id` INTEGER PRIMARY KEY AUTOINCREMENT,
     `name` VARCHAR(255),
-    `startTime` DATETIME,
+    `start_time` DATETIME,
-    `endTime` DATETIME,
+    `end_time` DATETIME,
     `summary` JSON,
     `ip` VARCHAR(255),
     `asn` INTEGER,
     `country` VARCHAR(2),
-    `networkName` VARCHAR(255),
+    `network_name` VARCHAR(255),
     `state` TEXT,
     `failure` VARCHAR(255),
-    `reportFile` VARCHAR(255),
+    `report_file` VARCHAR(255),
-    `reportId` VARCHAR(255),
+    `report_id` VARCHAR(255),
     `input` VARCHAR(255),
-    `measurementId` VARCHAR(255),
+    `measurement_id` VARCHAR(255),
-    `createdAt` DATETIME NOT NULL,
+    `result_id` INTEGER REFERENCES `results` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
-    `updatedAt` DATETIME NOT NULL,
-    `resultId` INTEGER REFERENCES `results` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
 );
 
 -- +migrate StatementEnd

internal/bindata/bindata.go

@@ -118,20 +118,19 @@ func dataDefaultConfigJson() (*asset, error) {
 }
 
 var _dataMigrations1_create_msmt_resultsSql = []byte(
-	"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x93\xcf\x8e\xda\x30\x10\xc6\xef\x79\x8a\x39\x82\x5a\x0e\xad\xc4\x89" +
+	"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x93\x4f\x6f\xf2\x30\x0c\xc6\xef\xfd\x14\x3e\x82\xde\x97\xc3\x26\x71" +
-	"\x93\x49\x86\xd6\x6d\x70\x90\xe3\x54\xe5\x16\xab\x76\x91\x55\xe2\x44\x8e\x23\xd4\xb7\x5f\x19\x02\x1b\xb2\x81\xcb" +
+	"\xe2\x14\x5a\x6f\xeb\x56\x52\x94\xa6\xd3\x38\xb5\xd1\x1a\x50\x34\x9a\x56\x69\x22\xb4\x6f\x3f\x85\x7f\x2a\xac\x70" +
-	"\x5e\xf6\x3a\xbf\x6f\xfe\x78\x3e\xcf\x62\x01\x9f\x2a\x73\x70\xd2\x6b\x48\xea\x93\x8d\x86\x81\xdc\x4b\xaf\x2b\x6d" +
+	"\xde\xd5\xbf\xc7\x8e\xfd\xd8\x99\x4c\xe0\x5f\xad\x36\x46\x58\x09\x51\xb3\xd3\x41\x3f\x90\x59\x61\x65\x2d\xb5\x9d" +
-	"\xfd\x5a\x1f\x8c\x8d\xa2\x84\x67\x3b\x10\x64\x9d\x22\x94\x4e\xb7\xdd\xd1\xb7\xe5\xea\x2e\x5a\x69\xd9\x76\xee\x9c" +
+	"\xcb\x8d\xd2\x41\x10\xb1\x74\x09\x9c\xcc\x13\x84\xd2\xc8\xce\x6d\x6d\x57\xce\x2e\xa2\xb5\x14\x9d\x33\xfb\x1c\x8f" +
-	"\x13\xd0\x74\x35\xb4\xea\x9e\x14\xcd\xd3\xb6\x31\x47\x22\x70\xdc\x18\x66\x11\x00\x40\x69\x54\x09\x94\x09\xfc\x86" +
+	"\x86\xab\xa1\xae\x2e\x49\xde\xde\x7d\x36\x64\x48\x38\x5e\x3f\x0c\xa3\x00\x00\xa0\x54\x55\x09\x31\xe5\xf8\x8c\x0c" +
-	"\x1c\x76\x9c\x6e\x09\xdf\xc3\x4f\xdc\x03\x29\x44\x46\x59\xcc\x71\x8b\x4c\x7c\xbe\x68\xad\xac\x74\x09\xbf\x08\x8f" +
+	"\x96\x2c\x5e\x10\xb6\x82\x37\x5c\x01\xc9\x79\x1a\xd3\x90\xe1\x02\x29\xff\x7f\xd0\x6a\x51\xcb\x12\xde\x09\x0b\x5f" +
-	"\xbf\x13\x3e\xfb\xba\x5c\xce\x7b\xd0\x7a\xe9\xbc\x30\x81\x26\x44\xa0\xa0\x5b\xec\x89\xb6\x6a\x32\xde\x76\x55\x25" +
+	"\x08\x1b\x3d\x4e\xa7\xe3\x23\xe8\xac\x30\xb6\xb0\xca\xe3\x88\x70\xe4\xf1\x02\x8f\x48\xea\x6a\x18\x74\xae\xae\x85" +
-	"\xdd\xff\x12\x7e\xe4\x19\xeb\x63\xaa\xb6\xba\x04\x41\xd9\x9e\x32\x31\xfb\x72\x2d\xae\xa4\x97\x45\x2b\x0f\xba\x68" +
+	"\xf9\x2e\xe1\x35\x4b\xe9\x31\x56\x35\x5a\x96\xc0\x63\xba\x8a\x29\x1f\x3d\x9c\xca\x57\xc2\x8a\xc2\x75\x62\x23\x0b" +
-	"\x6e\xa3\x8e\x51\xd8\xfd\x18\xfe\x71\x5a\x7a\xad\x88\x7f\x6d\x0e\x2c\x13\xc0\x8a\x34\xed\x25\x5d\xa3\x1e\x4a\xa2" +
+	"\xd7\x9e\xdb\xfd\x0d\xab\x66\xa7\xcf\x38\x18\xcf\xae\x67\xbf\xb0\xf7\x4f\x1a\xa0\xda\xc1\xf2\xa2\xd3\xd7\x63\x7f" +
-	"\xf9\x6a\xbc\xbb\x3b\x7b\x3e\xe0\x02\x4d\x33\x59\x5c\xb6\x6f\x77\x53\x77\xd6\x87\xec\x9b\xfc\x2a\xb6\xda\x9f\x6a" +
+	"\x36\x4e\x5b\x9f\x7d\x96\x9f\xc4\x5a\xda\x5d\x63\xbe\x8a\x7b\xcd\x5a\x6f\x33\x7e\x9c\xe6\x5a\x0b\xb5\x75\x66\x58" +
-	"\xf7\x8f\x3d\x99\xd4\x07\x97\xf0\xf7\xf5\x51\x7f\xa5\x39\x76\x6e\x5a\xed\x74\x53\x3b\xbf\x31\xc7\x67\x98\xaa\x49" +
+	"\x6d\x64\xdb\x18\x5b\xac\xd5\xf6\x2e\xf7\x36\x0e\x50\xa5\x5b\x67\x07\x49\x6f\x19\xb7\x92\x0f\xb7\x5a\xf4\x37\xc4" +
-	"\x68\x6c\xd3\xf9\x49\x32\xf0\xe1\x41\xee\x3b\xbd\xbf\x8d\x17\x6e\x85\x0e\x0c\xe6\xb8\x41\x8e\x2c\xc6\x7c\x78\x49" +
+	"\xf0\x09\x19\xd2\x10\xb3\xfe\x29\xfb\x25\x8e\x21\xa5\x10\x61\x82\x1c\x21\x43\x0e\x34\x4f\x12\x1f\xca\x97\xde\x77" +
-	"\xe1\x0f\xcc\x21\x63\x90\x60\x8a\x02\x21\xc7\x4b\x99\x10\x2a\x76\xa1\x3a\xc4\x24\x8f\x49\x82\xe7\x4f\xf5\xf0\xaa" +
+	"\x08\x49\x16\x92\x08\xf7\x57\x71\xf3\x5b\xfd\x04\x00\x00\xff\xff\xc4\x16\x0a\x20\xcf\x03\x00\x00")
-	"\x5f\x02\x00\x00\xff\xff\xe7\x43\x45\x75\x4e\x04\x00\x00")
 
 func dataMigrations1_create_msmt_resultsSqlBytes() ([]byte, error) {
 	return bindataRead(

internal/cli/run/run.go

@@ -1,10 +1,14 @@
 package run
 
 import (
+	"time"
+
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/openobservatory/gooni/internal/cli/root"
+	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/util"
+	"github.com/openobservatory/gooni/nettests"
 	"github.com/openobservatory/gooni/nettests/groups"
 )
 
@@ -15,15 +19,33 @@ func init() {
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		util.Log("Starting %s", *nettestGroup)
-		config, ooni, err := root.Init()
+		_, ctx, err := root.Init()
 		if err != nil {
 			log.Errorf("%s", err)
 			return err
 		}
-		log.Infof("%s", config)
+		group := groups.NettestGroups[*nettestGroup]
-		log.Infof("%s", ooni)
+		log.Debugf("Running test group %s", group.Label)
 
-		groups.Run(*nettestGroup, ooni)
+		result, err := database.CreateResult(ctx.DB, database.Result{
+			Name:      *nettestGroup,
+			StartTime: time.Now().UTC(), // XXX get this from MK
+		})
+		if err != nil {
+			log.Errorf("%s", err)
+			return err
+		}
+
+		for _, nt := range group.Nettests {
+			ctl := nettests.NewController(ctx)
+			nt.Run(ctl)
+			// XXX
+			// 1. Generate the summary
+			// 2. Link the measurement to the Result (this should probably happen in
+			// the nettest class)
+			// 3. Update the summary of the result and the other metadata in the db
+		}
+		result.Update(ctx.DB)
 		return nil
 	})
 }

internal/database/database.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/apex/log"
 	"github.com/jmoiron/sqlx"
-	_ "github.com/mattn/go-sqlite3"
+	_ "github.com/mattn/go-sqlite3" // this is needed to load the sqlite3 driver
 	ooni "github.com/openobservatory/gooni"
 	"github.com/openobservatory/gooni/internal/bindata"
 	"github.com/pkg/errors"
@@ -14,6 +14,7 @@ import (
 
 // RunMigrations runs the database migrations
 func RunMigrations(db *sqlx.DB) error {
+	log.Debugf("running migrations")
 	migrations := &migrate.AssetMigrationSource{
 		Asset:    bindata.Asset,
 		AssetDir: bindata.AssetDir,
@@ -27,19 +28,24 @@ func RunMigrations(db *sqlx.DB) error {
 	return nil
 }
 
+// Connect to the database
 func Connect(path string) (*sqlx.DB, error) {
 	db, err := sqlx.Connect("sqlite3", path)
 	if err != nil {
 		return nil, err
 	}
-	// XXX RunMigrations(db)
+	err = RunMigrations(db)
+	if err != nil {
+		return nil, err
+	}
 	return db, nil
 }
 
+// DefaultDatabasePath for the main database
 func DefaultDatabasePath() (string, error) {
 	home, err := ooni.GetOONIHome()
 	if err != nil {
 		return "", errors.Wrap(err, "default database path")
 	}
-	return filepath.Join(home, "db", "main.db"), nil
+	return filepath.Join(home, "db", "main.sqlite3"), nil
 }

internal/database/models.go

@@ -1,34 +1,97 @@
 package database
 
-import "time"
+import (
+	"time"
+
+	"github.com/apex/log"
+	"github.com/jmoiron/sqlx"
+	"github.com/pkg/errors"
+)
 
 // Measurement model
 type Measurement struct {
-	ID             int       `db:"id"`
+	ID             int64     `db:"id"`
 	Name           string    `db:"name"`
-	StartTime      time.Time `db:"startTime"`
+	StartTime      time.Time `db:"start_time"`
-	EndTime        time.Time `db:"endTime"`
+	EndTime        time.Time `db:"end_time"`
 	Summary        string    `db:"summary"` // XXX this should be JSON
-	ASN            int       `db:"asn"`
+	ASN            int64     `db:"asn"`
 	IP             string    `db:"ip"`
 	CountryCode    string    `db:"country"`
 	State          string    `db:"state"`
 	Failure        string    `db:"failure"`
-	ReportFilePath string    `db:"reportFile"`
+	ReportFilePath string    `db:"report_file"`
-	ReportID       string    `db:"reportId"`
+	ReportID       string    `db:"report_id"`
 	Input          string    `db:"input"`
-	MeasurementID  string    `db:"measurementId"`
+	MeasurementID  string    `db:"measurement_id"`
-	ResultID       string    `db:"resultId"`
+	ResultID       string    `db:"result_id"`
+}
+
+// CreateMeasurement writes the measurement to the database a returns a pointer
+// to the Measurement
+func CreateMeasurement(db *sqlx.DB, m Measurement) (*Measurement, error) {
+	res, err := db.NamedExec(`INSERT INTO measurements
+		(name, start_time,
+			summary, asn, ip, country,
+			state, failure, report_file,
+			report_id, input, measurement_id,
+			result_id)
+		VALUES (:name,:start_time,
+			:summary,:asn,:ip,:country,
+			:state,:failure,:report_file,
+			:report_id,:input,:measurement_id,
+			:result_id)`,
+		m)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating measurement")
+	}
+	id, err := res.LastInsertId()
+	if err != nil {
+		return nil, errors.Wrap(err, "creating measurement")
+	}
+	m.ID = id
+	return &m, nil
+}
+
+// Update the measurement in the database
+func (r Measurement) Update(db *sqlx.DB) error {
+	// XXX implement me
+	return nil
 }
 
 // Result model
 type Result struct {
-	ID            int       `db:"id"`
+	ID            int64     `db:"id"`
-	Name          int       `db:"name"`
+	Name          string    `db:"name"`
-	StartTime     time.Time `db:"startTime"`
+	StartTime     time.Time `db:"start_time"`
-	EndTime       time.Time `db:"endTime"`
+	EndTime       time.Time `db:"end_time"`
 	Summary       string    `db:"summary"` // XXX this should be JSON
 	Done          bool      `db:"done"`
-	DataUsageUp   int       `db:"dataUsageUp"`
+	DataUsageUp   int64     `db:"data_usage_up"`
-	DataUsageDown int       `db:"dataUsageDown"`
+	DataUsageDown int64     `db:"data_usage_down"`
+}
+
+// Update the Result in the database
+func (r Result) Update(db *sqlx.DB) error {
+	// XXX implement me
+	return nil
+}
+
+// CreateResult writes the Result to the database a returns a pointer
+// to the Result
+func CreateResult(db *sqlx.DB, r Result) (*Result, error) {
+	log.Debugf("Creating result %s", r)
+	res, err := db.NamedExec(`INSERT INTO results
+		(name, start_time)
+		VALUES (:name,:start_time)`,
+		r)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating result")
+	}
+	id, err := res.LastInsertId()
+	if err != nil {
+		return nil, errors.Wrap(err, "creating measurement")
+	}
+	r.ID = id
+	return &r, nil
 }

nettests/groups/groups.go

@@ -1,8 +1,6 @@
 package groups
 
 import (
-	"github.com/apex/log"
-	ooni "github.com/openobservatory/gooni"
 	"github.com/openobservatory/gooni/nettests"
 	"github.com/openobservatory/gooni/nettests/performance"
 	"github.com/openobservatory/gooni/nettests/websites"
@@ -31,15 +29,3 @@ var NettestGroups = map[string]nettests.NettestGroup{
 		Nettests: []nettests.Nettest{},
 	},
 }
-
-// Run runs a specific test group
-func Run(name string, ctx *ooni.Context) error {
-	group := NettestGroups[name]
-	log.Debugf("Running test group %s", group.Label)
-
-	for _, nt := range group.Nettests {
-		ctl := nettests.NewController(ctx)
-		nt.Run(ctl)
-	}
-	return nil
-}