c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update the measurement, network and url creation to the new schema

Browse Source
This commit is contained in:
Arturo Filastò
2018-09-07 12:55:27 +02:00
parent 71ed0e969f
commit 35bd334cfc
6 changed files with 138 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/cli/run/run.go
+4 -21
View File
@@ -3,9 +3,7 @@ package run
import (
"errors"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
@@ -14,7 +12,6 @@ import (
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/nettests"
"github.com/ooni/probe-cli/nettests/groups"
"github.com/ooni/probe-cli/utils"
)
func init() {
@@ -55,24 +52,13 @@ func init() {
return err
}
network := database.Network{
ASN: ctx.Location.ASN,
CountryCode: ctx.Location.CountryCode,
NetworkName: ctx.Location.NetworkName,
IP: ctx.Location.IP,
}
newID, err := ctx.DB.Collection("networks").Insert(network)
network, err := database.CreateNetwork(ctx.DB, ctx.Location)
if err != nil {
log.WithError(err).Error("Failed to create the network row")
return nil
}
network.ID = newID.(int64)
result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{
TestGroupName: *nettestGroup,
StartTime: time.Now().UTC(),
NetworkID: network.ID,
})
result, err := database.CreateResult(ctx.DB, ctx.Home, *nettestGroup, network.ID)
if err != nil {
log.Errorf("DB result error: %s", err)
return err
@@ -80,16 +66,13 @@ func init() {
for _, nt := range group.Nettests {
log.Debugf("Running test %T", nt)
msmtPath := filepath.Join(ctx.TempDir,
fmt.Sprintf("msmt-%T-%s.jsonl", nt,
time.Now().UTC().Format(utils.ResultTimestamp)))
ctl := nettests.NewController(nt, ctx, result, msmtPath)
ctl := nettests.NewController(nt, ctx, result)
if err = nt.Run(ctl); err != nil {
log.WithError(err).Errorf("Failed to run %s", group.Label)
return err
}
}
if err = result.Finished(ctx.DB, group.Summary); err != nil {
return err
}
internal/database/actions.go
+47 -22
View File
@@ -1,6 +1,7 @@
package database
import (
"database/sql"
"time"
"github.com/apex/log"
@@ -117,41 +118,65 @@ func ListResults(db sqlbuilder.Database) ([]*Result, []*Result, error) {
// CreateMeasurement writes the measurement to the database a returns a pointer
// to the Measurement
func CreateMeasurement(sess sqlbuilder.Database, m Measurement, i string) (*Measurement, error) {
col := sess.Collection("measurements")
func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testName string, resultID int64, reportFilePath string, urlID sql.NullInt64) (*Measurement, error) {
msmt := Measurement{
ReportID: reportID,
TestName: testName,
ResultID: resultID,
ReportFilePath: reportFilePath,
URLID: urlID,
// XXX Do we want to have this be part of something else?
StartTime: time.Now().UTC(),
TestKeys: "",
}
// XXX Do we want to have this be part of something else?
m.StartTime = time.Now().UTC()
m.TestKeys = ""
// XXX insert also the URL and stuff
//m.Input = i
//m.State = "active"
newID, err := col.Insert(m)
newID, err := sess.Collection("measurements").Insert(msmt)
if err != nil {
return nil, errors.Wrap(err, "creating measurement")
}
m.ID = newID.(int64)
return &m, nil
msmt.ID = newID.(int64)
return &msmt, nil
}
// CreateResult writes the Result to the database a returns a pointer
// to the Result
func CreateResult(sess sqlbuilder.Database, homePath string, r Result) (*Result, error) {
log.Debugf("Creating result %v", r)
func CreateResult(sess sqlbuilder.Database, homePath string, testGroupName string, networkID int64) (*Result, error) {
startTime := time.Now().UTC()
col := sess.Collection("results")
p, err := utils.MakeResultsDir(homePath, r.TestGroupName, r.StartTime)
p, err := utils.MakeResultsDir(homePath, testGroupName, startTime)
if err != nil {
return nil, err
}
r.MeasurementDir = p
newID, err := col.Insert(r)
result := Result{
TestGroupName: testGroupName,
StartTime: startTime,
NetworkID: networkID,
}
result.MeasurementDir = p
log.Debugf("Creating result %v", result)
newID, err := sess.Collection("results").Insert(result)
if err != nil {
return nil, errors.Wrap(err, "creating result")
}
r.ID = newID.(int64)
return &r, nil
result.ID = newID.(int64)
return &result, nil
}
// CreateNetwork will create a new network in the network table
func CreateNetwork(sess sqlbuilder.Database, location *utils.LocationInfo) (*Network, error) {
network := Network{
ASN: location.ASN,
CountryCode: location.CountryCode,
NetworkName: location.NetworkName,
IP: location.IP,
}
newID, err := sess.Collection("networks").Insert(network)
if err != nil {
return nil, err
}
network.ID = newID.(int64)
return &network, nil
}
internal/database/actions_test.go
+8 -12
View File
@@ -5,7 +5,6 @@ import (
"io/ioutil"
"os"
"testing"
"time"
)
func TestMeasurementWorkflow(t *testing.T) {
@@ -25,21 +24,18 @@ func TestMeasurementWorkflow(t *testing.T) {
if err != nil {
t.Error(err)
}
result, err := CreateResult(sess, tmpdir, Result{
TestGroupName: "websites",
StartTime: time.Now().UTC(),
})
result, err := CreateResult(sess, tmpdir, "websites", 0)
if err != nil {
t.Fatal(err)
}
msmtTemplate := Measurement{
ReportID: sql.NullString{String: "", Valid: false},
TestName: "antani",
ResultID: result.ID,
ReportFilePath: tmpdir,
}
m1, err := CreateMeasurement(sess, msmtTemplate, "")
reportID := sql.NullString{String: "", Valid: false}
testName := "antani"
resultID := result.ID
reportFilePath := tmpdir
urlID := sql.NullInt64{Int64: 0, Valid: false}
m1, err := CreateMeasurement(sess, reportID, testName, resultID, reportFilePath, urlID)
if err != nil {
t.Fatal(err)
}
internal/database/models.go
+2 -2
View File
@@ -23,7 +23,7 @@ type Network struct {
// URL represents URLs from the testing lists
type URL struct {
ID int64 `db:"id"`
URL int64 `db:"url"`
URL string `db:"url"`
CategoryCode string `db:"category_code"`
CountryCode string `db:"country_code"`
}
@@ -42,7 +42,7 @@ type Measurement struct {
UploadFailureMsg sql.NullString `db:"upload_failure_msg,omitempty"`
IsRerun bool `db:"is_rerun"`
ReportID sql.NullString `db:"report_id,omitempty"`
URLID string `db:"url_id"` // Used to reference URL
URLID sql.NullInt64 `db:"url_id,omitempty"` // Used to reference URL
MeasurementID sql.NullInt64 `db:"measurement_id,omitempty"`
IsAnomaly sql.NullBool `db:"is_anomaly,omitempty"`
// FIXME we likely want to support JSON. See: https://github.com/upper/db/issues/462