fix: import path should be github.com/ooni/probe-cli/v3 (#200)

See https://github.com/ooni/probe/issues/1335#issuecomment-771499511
2021-02-02 10:32:46 +01:00
parent faa9308b1e
commit b1ce300c8d
68 changed files with 86 additions and 85 deletions
@@ -0,0 +1,355 @@
+package database
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"reflect"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
+	"github.com/pkg/errors"
+	db "upper.io/db.v3"
+	"upper.io/db.v3/lib/sqlbuilder"
+)
+
+// ListMeasurements given a result ID
+func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementURLNetwork, error) {
+	measurements := []MeasurementURLNetwork{}
+	req := sess.Select(
+		db.Raw("networks.*"),
+		db.Raw("urls.*"),
+		db.Raw("measurements.*"),
+		db.Raw("results.*"),
+	).From("results").
+		Join("measurements").On("results.result_id = measurements.result_id").
+		Join("networks").On("results.network_id = networks.network_id").
+		LeftJoin("urls").On("urls.url_id = measurements.url_id").
+		OrderBy("measurements.measurement_start_time").
+		Where("results.result_id = ?", resultID)
+	if err := req.All(&measurements); err != nil {
+		log.Errorf("failed to run query %s: %v", req.String(), err)
+		return measurements, err
+	}
+	return measurements, nil
+}
+
+// GetMeasurementJSON returns a map[string]interface{} given a database and a measurementID
+func GetMeasurementJSON(sess sqlbuilder.Database, measurementID int64) (map[string]interface{}, error) {
+	var (
+		measurement MeasurementURLNetwork
+		msmtJSON    map[string]interface{}
+	)
+	req := sess.Select(
+		db.Raw("urls.*"),
+		db.Raw("measurements.*"),
+	).From("measurements").
+		LeftJoin("urls").On("urls.url_id = measurements.url_id").
+		Where("measurements.measurement_id= ?", measurementID)
+	if err := req.One(&measurement); err != nil {
+		log.Errorf("failed to run query %s: %v", req.String(), err)
+		return nil, err
+	}
+	if measurement.IsUploaded {
+		// TODO(bassosimone): this should be a function exposed by probe-engine
+		reportID := measurement.Measurement.ReportID.String
+		measurementURL := &url.URL{
+			Scheme: "https",
+			Host:   "api.ooni.io",
+			Path:   "/api/v1/raw_measurement",
+		}
+		query := url.Values{}
+		query.Add("report_id", reportID)
+		if measurement.URL.URL.Valid == true {
+			query.Add("input", measurement.URL.URL.String)
+		}
+		measurementURL.RawQuery = query.Encode()
+		log.Debugf("using %s", measurementURL.String())
+		resp, err := http.Get(measurementURL.String())
+		if err != nil {
+			log.Errorf("failed to fetch the measurement %s %s", reportID, measurement.URL.URL.String)
+			return nil, err
+		}
+		defer resp.Body.Close()
+		if err := json.NewDecoder(resp.Body).Decode(&msmtJSON); err != nil {
+			log.Error("failed to unmarshal the measurement_json")
+			return nil, err
+		}
+		return msmtJSON, nil
+	}
+	// MeasurementFilePath might be NULL because the measurement from a
+	// 3.0.0-beta install
+	if measurement.Measurement.MeasurementFilePath.Valid == false {
+		log.Error("invalid measurement_file_path")
+		log.Error("backup your OONI_HOME and run `ooniprobe reset`")
+		return nil, errors.New("cannot access measurement file")
+	}
+	measurementFilePath := measurement.Measurement.MeasurementFilePath.String
+	b, err := ioutil.ReadFile(measurementFilePath)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(b, &msmtJSON); err != nil {
+		log.Error("failed to unmarshal the measurement_json")
+		log.Error("backup your OONI_HOME and run `ooniprobe reset`")
+		return nil, err
+	}
+	return msmtJSON, nil
+}
+
+// GetResultTestKeys returns a list of TestKeys for a given result
+func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error) {
+	res := sess.Collection("measurements").Find("result_id", resultID)
+	defer res.Close()
+
+	var (
+		msmt Measurement
+		tk   PerformanceTestKeys
+	)
+	for res.Next(&msmt) {
+		// We only really care about performance keys.
+		// Note: since even in case of failure we still initialise an empty struct,
+		// it could be that these keys come out as initializes with the default
+		// values.
+		// XXX we may want to change this behaviour by adding `omitempty` to the
+		// struct definition.
+		if msmt.TestName != "ndt" && msmt.TestName != "dash" {
+			return "{}", nil
+		}
+		if err := json.Unmarshal([]byte(msmt.TestKeys), &tk); err != nil {
+			log.WithError(err).Error("failed to parse testKeys")
+			return "{}", err
+		}
+	}
+	b, err := json.Marshal(tk)
+	if err != nil {
+		log.WithError(err).Error("failed to serialize testKeys")
+		return "{}", err
+	}
+	return string(b), nil
+}
+
+// GetMeasurementCounts returns the number of anomalous and total measurement for a given result
+func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) {
+	var (
+		totalCount uint64
+		anmlyCount uint64
+		err        error
+	)
+	col := sess.Collection("measurements")
+
+	// XXX these two queries can be done with a single query
+	totalCount, err = col.Find("result_id", resultID).
+		Count()
+	if err != nil {
+		log.WithError(err).Error("failed to get total count")
+		return totalCount, anmlyCount, err
+	}
+
+	anmlyCount, err = col.Find("result_id", resultID).
+		And(db.Cond{"is_anomaly": true}).Count()
+	if err != nil {
+		log.WithError(err).Error("failed to get anmly count")
+		return totalCount, anmlyCount, err
+	}
+
+	log.Debugf("counts: %d, %d, %d", resultID, totalCount, anmlyCount)
+	return totalCount, anmlyCount, err
+}
+
+// ListResults return the list of results
+func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) {
+	doneResults := []ResultNetwork{}
+	incompleteResults := []ResultNetwork{}
+	req := sess.Select(
+		db.Raw("networks.*"),
+		db.Raw("results.*"),
+	).From("results").
+		Join("networks").On("results.network_id = networks.network_id").
+		OrderBy("results.result_start_time")
+	if err := req.Where("result_is_done = true").All(&doneResults); err != nil {
+		return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
+	}
+	if err := req.Where("result_is_done = false").All(&incompleteResults); err != nil {
+		return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
+	}
+	return doneResults, incompleteResults, nil
+}
+
+// DeleteResult will delete a particular result and the relative measurement on
+// disk.
+func DeleteResult(sess sqlbuilder.Database, resultID int64) error {
+	var result Result
+	res := sess.Collection("results").Find("result_id", resultID)
+	if err := res.One(&result); err != nil {
+		if err == db.ErrNoMoreRows {
+			return err
+		}
+		log.WithError(err).Error("error in obtaining the result")
+		return err
+	}
+	if err := res.Delete(); err != nil {
+		log.WithError(err).Error("failed to delete the result directory")
+		return err
+	}
+
+	os.RemoveAll(result.MeasurementDir)
+	return nil
+}
+
+// CreateMeasurement writes the measurement to the database a returns a pointer
+// to the Measurement
+func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testName string, measurementDir string, idx int, resultID int64, urlID sql.NullInt64) (*Measurement, error) {
+	// TODO we should look into generating this file path in a more robust way.
+	// If there are two identical test_names in the same test group there is
+	// going to be a clash of test_name
+	msmtFilePath := filepath.Join(measurementDir, fmt.Sprintf("msmt-%s-%d.json", testName, idx))
+	msmt := Measurement{
+		ReportID:            reportID,
+		TestName:            testName,
+		ResultID:            resultID,
+		MeasurementFilePath: sql.NullString{String: msmtFilePath, Valid: true},
+		URLID:               urlID,
+		IsFailed:            false,
+		IsDone:              false,
+		// XXX Do we want to have this be part of something else?
+		StartTime: time.Now().UTC(),
+		TestKeys:  "",
+	}
+
+	newID, err := sess.Collection("measurements").Insert(msmt)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating measurement")
+	}
+	msmt.ID = newID.(int64)
+	return &msmt, nil
+}
+
+// CreateResult writes the Result to the database a returns a pointer
+// to the Result
+func CreateResult(sess sqlbuilder.Database, homePath string, testGroupName string, networkID int64) (*Result, error) {
+	startTime := time.Now().UTC()
+
+	p, err := utils.MakeResultsDir(homePath, testGroupName, startTime)
+	if err != nil {
+		return nil, err
+	}
+
+	result := Result{
+		TestGroupName: testGroupName,
+		StartTime:     startTime,
+		NetworkID:     networkID,
+	}
+	result.MeasurementDir = p
+	log.Debugf("Creating result %v", result)
+
+	newID, err := sess.Collection("results").Insert(result)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating result")
+	}
+	result.ID = newID.(int64)
+	return &result, nil
+}
+
+// CreateNetwork will create a new network in the network table
+func CreateNetwork(sess sqlbuilder.Database, loc enginex.LocationProvider) (*Network, error) {
+	network := Network{
+		ASN:         loc.ProbeASN(),
+		CountryCode: loc.ProbeCC(),
+		NetworkName: loc.ProbeNetworkName(),
+		// On desktop we consider it to always be wifi
+		NetworkType: "wifi",
+		IP:          loc.ProbeIP(),
+	}
+	newID, err := sess.Collection("networks").Insert(network)
+	if err != nil {
+		return nil, err
+	}
+
+	network.ID = newID.(int64)
+	return &network, nil
+}
+
+// CreateOrUpdateURL will create a new URL entry to the urls table if it doesn't
+// exists, otherwise it will update the category code of the one already in
+// there.
+func CreateOrUpdateURL(sess sqlbuilder.Database, urlStr string, categoryCode string, countryCode string) (int64, error) {
+	var url URL
+
+	tx, err := sess.NewTx(nil)
+	if err != nil {
+		log.WithError(err).Error("failed to create transaction")
+		return 0, err
+	}
+	res := tx.Collection("urls").Find(
+		db.Cond{"url": urlStr, "url_country_code": countryCode},
+	)
+	err = res.One(&url)
+
+	if err == db.ErrNoMoreRows {
+		url = URL{
+			URL:          sql.NullString{String: urlStr, Valid: true},
+			CategoryCode: sql.NullString{String: categoryCode, Valid: true},
+			CountryCode:  sql.NullString{String: countryCode, Valid: true},
+		}
+		newID, insErr := tx.Collection("urls").Insert(url)
+		if insErr != nil {
+			log.Error("Failed to insert into the URLs table")
+			return 0, insErr
+		}
+		url.ID = sql.NullInt64{Int64: newID.(int64), Valid: true}
+	} else if err != nil {
+		log.WithError(err).Error("Failed to get single result")
+		return 0, err
+	} else {
+		url.CategoryCode = sql.NullString{String: categoryCode, Valid: true}
+		res.Update(url)
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		log.WithError(err).Error("Failed to write to the URL table")
+		return 0, err
+	}
+
+	log.Debugf("returning url %d", url.ID.Int64)
+
+	return url.ID.Int64, nil
+}
+
+// AddTestKeys writes the summary to the measurement
+func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error {
+	var (
+		isAnomaly      bool
+		isAnomalyValid bool
+	)
+	tkBytes, err := json.Marshal(tk)
+	if err != nil {
+		log.WithError(err).Error("failed to serialize summary")
+	}
+
+	// This is necessary so that we can extract from the the opaque testKeys just
+	// the IsAnomaly field of bool type.
+	// Maybe generics are not so bad after-all, heh golang?
+	isAnomalyValue := reflect.ValueOf(tk).FieldByName("IsAnomaly")
+	if isAnomalyValue.IsValid() == true && isAnomalyValue.Kind() == reflect.Bool {
+		isAnomaly = isAnomalyValue.Bool()
+		isAnomalyValid = true
+	}
+	msmt.TestKeys = string(tkBytes)
+	msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: isAnomalyValid}
+
+	err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
+	if err != nil {
+		log.WithError(err).Error("failed to update measurement")
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}
@@ -0,0 +1,363 @@
+package database
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	db "upper.io/db.v3"
+)
+
+type locationInfo struct {
+	asn         uint
+	countryCode string
+	ip          string
+	networkName string
+	resolverIP  string
+}
+
+func (lp *locationInfo) ProbeASN() uint {
+	return lp.asn
+}
+
+func (lp *locationInfo) ProbeASNString() string {
+	return fmt.Sprintf("AS%d", lp.asn)
+}
+
+func (lp *locationInfo) ProbeCC() string {
+	return lp.countryCode
+}
+
+func (lp *locationInfo) ProbeIP() string {
+	return lp.ip
+}
+
+func (lp *locationInfo) ProbeNetworkName() string {
+	return lp.networkName
+}
+
+func (lp *locationInfo) ResolverIP() string {
+	return lp.resolverIP
+}
+
+func TestMeasurementWorkflow(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	tmpdir, err := ioutil.TempDir("", "oonitest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	location := locationInfo{
+		asn:         0,
+		countryCode: "IT",
+		networkName: "Unknown",
+	}
+	network, err := CreateNetwork(sess, &location)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reportID := sql.NullString{String: "", Valid: false}
+	testName := "antani"
+	resultID := result.ID
+	msmtFilePath := tmpdir
+	urlID := sql.NullInt64{Int64: 0, Valid: false}
+
+	m1, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var m2 Measurement
+	err = sess.Collection("measurements").Find("measurement_id", m1.ID).One(&m2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m2.ResultID != m1.ResultID {
+		t.Error("result_id mismatch")
+	}
+
+	done, incomplete, err := ListResults(sess)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(incomplete) != 1 {
+		t.Error("there should be 1 incomplete measurement")
+	}
+	if len(done) != 0 {
+		t.Error("there should be 0 done measurements")
+	}
+
+	msmts, err := ListMeasurements(sess, resultID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if msmts[0].Network.NetworkType != "wifi" {
+		t.Error("network_type should be wifi")
+	}
+}
+
+func TestDeleteResult(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	tmpdir, err := ioutil.TempDir("", "oonitest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	location := locationInfo{
+		asn:         0,
+		countryCode: "IT",
+		networkName: "Unknown",
+	}
+	network, err := CreateNetwork(sess, &location)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reportID := sql.NullString{String: "", Valid: false}
+	testName := "antani"
+	resultID := result.ID
+	msmtFilePath := tmpdir
+	urlID := sql.NullInt64{Int64: 0, Valid: false}
+
+	m1, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var m2 Measurement
+	err = sess.Collection("measurements").Find("measurement_id", m1.ID).One(&m2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m2.ResultID != m1.ResultID {
+		t.Error("result_id mismatch")
+	}
+
+	err = DeleteResult(sess, resultID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	totalResults, err := sess.Collection("results").Find().Count()
+	if err != nil {
+		t.Fatal(err)
+	}
+	totalMeasurements, err := sess.Collection("measurements").Find().Count()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if totalResults != 0 {
+		t.Fatal("results should be zero")
+	}
+	if totalMeasurements != 0 {
+		t.Fatal("measurements should be zero")
+	}
+
+	err = DeleteResult(sess, 20)
+	if err != db.ErrNoMoreRows {
+		t.Fatal(err)
+	}
+}
+
+func TestNetworkCreate(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	l1 := locationInfo{
+		asn:         2,
+		countryCode: "IT",
+		networkName: "Antaninet",
+	}
+
+	l2 := locationInfo{
+		asn:         3,
+		countryCode: "IT",
+		networkName: "Fufnet",
+	}
+
+	_, err = CreateNetwork(sess, &l1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = CreateNetwork(sess, &l2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
+func TestURLCreation(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID1, err := CreateOrUpdateURL(sess, "https://google.com", "GMB", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID2, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID3, err := CreateOrUpdateURL(sess, "https://facebook.com", "GRP", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID4, err := CreateOrUpdateURL(sess, "https://facebook.com", "GMP", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newID5, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if newID2 != newID1 {
+		t.Error("inserting the same URL with different category code should produce the same result")
+	}
+
+	if newID3 == newID1 {
+		t.Error("inserting different URL should produce different ids")
+	}
+
+	if newID4 != newID3 {
+		t.Error("inserting the same URL with different category code should produce the same result")
+	}
+
+	if newID5 != newID1 {
+		t.Error("the ID of google should still be the same")
+	}
+}
+
+func TestPerformanceTestKeys(t *testing.T) {
+	var tk PerformanceTestKeys
+
+	ndtS := "{\"download\":100.0,\"upload\":20.0,\"ping\":2.2}"
+	dashS := "{\"median_bitrate\":102.0}"
+	if err := json.Unmarshal([]byte(ndtS), &tk); err != nil {
+		t.Fatal("failed to parse ndtS")
+	}
+	if err := json.Unmarshal([]byte(dashS), &tk); err != nil {
+		t.Fatal("failed to parse dashS")
+	}
+	if tk.Bitrate != 102.0 {
+		t.Fatalf("error Bitrate %f", tk.Bitrate)
+	}
+	if tk.Download != 100.0 {
+		t.Fatalf("error Download %f", tk.Download)
+	}
+}
+
+func TestGetMeasurementJSON(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	tmpdir, err := ioutil.TempDir("", "oonitest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	location := locationInfo{
+		asn:         0,
+		countryCode: "IT",
+		networkName: "Unknown",
+	}
+	network, err := CreateNetwork(sess, &location)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reportID := sql.NullString{String: "20210111T085144Z_ndt_RU_3216_n1_qMVnP0PTX7ObUSmD", Valid: true}
+	testName := "antani"
+	resultID := result.ID
+	msmtFilePath := tmpdir
+	urlID := sql.NullInt64{Int64: 0, Valid: false}
+
+	msmt, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	msmt.IsUploaded = true
+	err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tk, err := GetMeasurementJSON(sess, msmt.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk["probe_asn"] != "AS3216" {
+		t.Error("inconsistent measurement downloaded")
+	}
+}
@@ -0,0 +1,47 @@
+package database
+
+import (
+	"database/sql"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/bindata"
+	migrate "github.com/rubenv/sql-migrate"
+	"upper.io/db.v3/lib/sqlbuilder"
+	"upper.io/db.v3/sqlite"
+)
+
+// RunMigrations runs the database migrations
+func RunMigrations(db *sql.DB) error {
+	log.Debugf("running migrations")
+	migrations := &migrate.AssetMigrationSource{
+		Asset:    bindata.Asset,
+		AssetDir: bindata.AssetDir,
+		Dir:      "data/migrations",
+	}
+	n, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up)
+	if err != nil {
+		return err
+	}
+	log.Debugf("performed %d migrations", n)
+	return nil
+}
+
+// Connect to the database
+func Connect(path string) (db sqlbuilder.Database, err error) {
+	settings := sqlite.ConnectionURL{
+		Database: path,
+		Options:  map[string]string{"_foreign_keys": "1"},
+	}
+	sess, err := sqlite.Open(settings)
+	if err != nil {
+		log.WithError(err).Error("failed to open the DB")
+		return nil, err
+	}
+
+	err = RunMigrations(sess.Driver().(*sql.DB))
+	if err != nil {
+		log.WithError(err).Error("failed to run DB migration")
+		return nil, err
+	}
+	return sess, err
+}
@@ -0,0 +1,32 @@
+package database
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/apex/log"
+)
+
+func TestConnect(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Error(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Error(err)
+	}
+
+	colls, err := sess.Collections()
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(colls) < 1 {
+		log.Fatal("missing tables")
+	}
+
+}
@@ -0,0 +1,150 @@
+package database
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/pkg/errors"
+	"upper.io/db.v3/lib/sqlbuilder"
+)
+
+// ResultNetwork is used to represent the structure made from the JOIN
+// between the results and networks tables.
+type ResultNetwork struct {
+	Result  `db:",inline"`
+	Network `db:",inline"`
+}
+
+// MeasurementURLNetwork is used for the JOIN between Measurement and URL
+type MeasurementURLNetwork struct {
+	Measurement `db:",inline"`
+	Network     `db:",inline"`
+	Result      `db:",inline"`
+	URL         `db:",inline"`
+}
+
+// Network represents a network tested by the user
+type Network struct {
+	ID          int64  `db:"network_id,omitempty"`
+	NetworkName string `db:"network_name"`
+	NetworkType string `db:"network_type"`
+	IP          string `db:"ip"`
+	ASN         uint   `db:"asn"`
+	CountryCode string `db:"network_country_code"`
+}
+
+// URL represents URLs from the testing lists
+type URL struct {
+	ID           sql.NullInt64  `db:"url_id,omitempty"`
+	URL          sql.NullString `db:"url"`
+	CategoryCode sql.NullString `db:"category_code"`
+	CountryCode  sql.NullString `db:"url_country_code"`
+}
+
+// Measurement model
+type Measurement struct {
+	ID               int64          `db:"measurement_id,omitempty"`
+	TestName         string         `db:"test_name"`
+	StartTime        time.Time      `db:"measurement_start_time"`
+	Runtime          float64        `db:"measurement_runtime"` // Fractional number of seconds
+	IsDone           bool           `db:"measurement_is_done"`
+	IsUploaded       bool           `db:"measurement_is_uploaded"`
+	IsFailed         bool           `db:"measurement_is_failed"`
+	FailureMsg       sql.NullString `db:"measurement_failure_msg,omitempty"`
+	IsUploadFailed   bool           `db:"measurement_is_upload_failed"`
+	UploadFailureMsg sql.NullString `db:"measurement_upload_failure_msg,omitempty"`
+	IsRerun          bool           `db:"measurement_is_rerun"`
+	ReportID         sql.NullString `db:"report_id,omitempty"`
+	URLID            sql.NullInt64  `db:"url_id,omitempty"` // Used to reference URL
+	MeasurementID    sql.NullInt64  `db:"collector_measurement_id,omitempty"`
+	IsAnomaly        sql.NullBool   `db:"is_anomaly,omitempty"`
+	// FIXME we likely want to support JSON. See: https://github.com/upper/db/issues/462
+	TestKeys            string         `db:"test_keys"`
+	ResultID            int64          `db:"result_id"`
+	ReportFilePath      sql.NullString `db:"report_file_path,omitempty"`
+	MeasurementFilePath sql.NullString `db:"measurement_file_path,omitempty"`
+}
+
+// Result model
+type Result struct {
+	ID             int64     `db:"result_id,omitempty"`
+	TestGroupName  string    `db:"test_group_name"`
+	StartTime      time.Time `db:"result_start_time"`
+	NetworkID      int64     `db:"network_id"`     // Used to include a Network
+	Runtime        float64   `db:"result_runtime"` // Runtime is expressed in fractional seconds
+	IsViewed       bool      `db:"result_is_viewed"`
+	IsDone         bool      `db:"result_is_done"`
+	DataUsageUp    float64   `db:"result_data_usage_up"`
+	DataUsageDown  float64   `db:"result_data_usage_down"`
+	MeasurementDir string    `db:"measurement_dir"`
+}
+
+// PerformanceTestKeys is the result summary for a performance test
+type PerformanceTestKeys struct {
+	Upload   float64 `json:"upload"`
+	Download float64 `json:"download"`
+	Ping     float64 `json:"ping"`
+	Bitrate  float64 `json:"median_bitrate"`
+}
+
+// Finished marks the result as done and sets the runtime
+func (r *Result) Finished(sess sqlbuilder.Database) error {
+	if r.IsDone == true || r.Runtime != 0 {
+		return errors.New("Result is already finished")
+	}
+	r.Runtime = time.Now().UTC().Sub(r.StartTime).Seconds()
+	r.IsDone = true
+
+	err := sess.Collection("results").Find("result_id", r.ID).Update(r)
+	if err != nil {
+		return errors.Wrap(err, "updating finished result")
+	}
+	return nil
+}
+
+// Failed writes the error string to the measurement
+func (m *Measurement) Failed(sess sqlbuilder.Database, failure string) error {
+	m.FailureMsg = sql.NullString{String: failure, Valid: true}
+	m.IsFailed = true
+	err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
+	if err != nil {
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}
+
+// Done marks the measurement as completed
+func (m *Measurement) Done(sess sqlbuilder.Database) error {
+	runtime := time.Now().UTC().Sub(m.StartTime)
+	m.Runtime = runtime.Seconds()
+	m.IsDone = true
+
+	err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
+	if err != nil {
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}
+
+// UploadFailed writes the error string for the upload failure to the measurement
+func (m *Measurement) UploadFailed(sess sqlbuilder.Database, failure string) error {
+	m.UploadFailureMsg = sql.NullString{String: failure, Valid: true}
+	m.IsUploaded = false
+
+	err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
+	if err != nil {
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}
+
+// UploadSucceeded writes the error string for the upload failure to the measurement
+func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error {
+	m.IsUploaded = true
+
+	err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
+	if err != nil {
+		return errors.Wrap(err, "updating measurement")
+	}
+	return nil
+}