Fix List* workflows and add unittests for them

.travis.yml

@@ -6,7 +6,7 @@ go:
 - 1.x
 
 install:
-- go get -u github.com/golang/dep/...
+- make install-dev-deps
 - dep ensure
 - make update-mk-libs
 

Makefile

@@ -1,5 +1,10 @@
 GO ?= go
 
+install-dev-deps:
+	@$(GO) get -u github.com/golang/dep/...
+	@$(GO) get golang.org/x/tools/cmd/cover
+	@$(GO) get github.com/mattn/goveralls
+
 build:
 	@echo "Building dist/ooni"
 	@$(GO) build -i -o dist/ooni cmd/ooni/main.go

internal/database/actions.go

@@ -7,111 +7,54 @@ import (
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/utils"
 	"github.com/pkg/errors"
+	db "upper.io/db.v3"
 	"upper.io/db.v3/lib/sqlbuilder"
 )
 
 // ListMeasurements given a result ID
-func ListMeasurements(db sqlbuilder.Database, resultID int64) ([]*Measurement, error) {
+func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementURLNetwork, error) {
-	measurements := []*Measurement{}
+	measurements := []MeasurementURLNetwork{}
 
-	/*
+	req := sess.Select(
-		FIXME
+		"networks.id as network_id",
-		rows, err := db.Query(`SELECT id, name,
+		"results.id as result_id",
-			start_time, runtime,
+		"urls.id as url_id",
-			country,
+		db.Raw("networks.*"),
-			asn,
+		db.Raw("urls.*"),
-			summary,
+		db.Raw("measurements.*"),
-			input
+	).From("results").
-			FROM measurements
+		Join("measurements").On("results.id = measurements.result_id").
-			WHERE result_id = ?
+		Join("networks").On("results.network_id = networks.id").
-			ORDER BY start_time;`, resultID)
+		LeftJoin("urls").On("urls.id = measurements.url_id").
-		if err != nil {
+		OrderBy("measurements.start_time").
-			return measurements, errors.Wrap(err, "failed to get measurement list")
+		Where("results.id = ?", resultID)
-		}
-
-			for rows.Next() {
-				msmt := Measurement{}
-				err = rows.Scan(&msmt.ID, &msmt.Name,
-					&msmt.StartTime, &msmt.Runtime,
-					&msmt.CountryCode,
-					&msmt.ASN,
-					&msmt.Summary, &msmt.Input,
-					//&result.DataUsageUp, &result.DataUsageDown)
-				)
-				if err != nil {
-					log.WithError(err).Error("failed to fetch a row")
-					continue
-				}
-				measurements = append(measurements, &msmt)
-			}
-	*/
 
+	if err := req.All(&measurements); err != nil {
+		log.Errorf("failed to run query %s: %v", req.String(), err)
+		return measurements, err
+	}
 	return measurements, nil
 }
 
 // ListResults return the list of results
-func ListResults(db sqlbuilder.Database) ([]*Result, []*Result, error) {
+func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) {
-	doneResults := []*Result{}
+	doneResults := []ResultNetwork{}
-	incompleteResults := []*Result{}
+	incompleteResults := []ResultNetwork{}
 
-	/*
+	req := sess.Select(
-		FIXME
+		"networks.id AS network_id",
-		rows, err := db.Query(`SELECT id, name,
+		db.Raw("results.*"),
-			start_time, runtime,
+		db.Raw("networks.*"),
-			network_name, country,
+	).From("results").
-			asn,
+		Join("networks").On("results.network_id = networks.id").
-			summary, done
+		OrderBy("results.start_time")
-			FROM results
-			WHERE done = 1
-			ORDER BY start_time;`)
-		if err != nil {
-			return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
-		}
-			for rows.Next() {
-				result := Result{}
-				err = rows.Scan(&result.ID, &result.Name,
-					&result.StartTime, &result.Runtime,
-					&result.NetworkName, &result.Country,
-					&result.ASN,
-					&result.Summary, &result.Done,
-					//&result.DataUsageUp, &result.DataUsageDown)
-				)
-				if err != nil {
-					log.WithError(err).Error("failed to fetch a row")
-					continue
-				}
-				doneResults = append(doneResults, &result)
-			}
-	*/
 
-	/*
+	if err := req.Where("is_done = true").All(&doneResults); err != nil {
-			FIXME
+		return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
-		rows, err := db.Query(`SELECT
+	}
-			id, name,
+	if err := req.Where("is_done = false").All(&incompleteResults); err != nil {
-			start_time,
+		return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
-			network_name, country,
+	}
-			asn
-			FROM results
-			WHERE done != 1
-			ORDER BY start_time;`)
-		if err != nil {
-			return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
-		}
-	*/
-
-	/*
-		for rows.Next() {
-			result := Result{Done: false}
-			err = rows.Scan(&result.ID, &result.Name, &result.StartTime,
-				&result.NetworkName, &result.Country,
-				&result.ASN)
-			if err != nil {
-				log.WithError(err).Error("failed to fetch a row")
-				continue
-			}
-			incompleteResults = append(incompleteResults, &result)
-		}
-	*/
 
 	return doneResults, incompleteResults, nil
 }

internal/database/actions_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/apex/log"
 	"github.com/ooni/probe-cli/utils"
 )
 
@@ -14,7 +15,8 @@ func TestMeasurementWorkflow(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.Remove(tmpfile.Name())
+	log.Infof("%s", tmpfile.Name())
+	//defer os.Remove(tmpfile.Name())
 
 	tmpdir, err := ioutil.TempDir("", "oonitest")
 	if err != nil {
@@ -62,4 +64,23 @@ func TestMeasurementWorkflow(t *testing.T) {
 		t.Error("result_id mismatch")
 	}
 
+	done, incomplete, err := ListResults(sess)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(incomplete) != 1 {
+		t.Error("there should be 1 incomplete measurement")
+	}
+	if len(done) != 0 {
+		t.Error("there should be 0 done measurements")
+	}
+
+	msmts, err := ListMeasurements(sess, resultID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if msmts[0].Network.NetworkType != "wifi" {
+		t.Error("network_type should be wifi")
+	}
 }

internal/database/models.go

@@ -11,6 +11,21 @@ import (
 	"upper.io/db.v3/lib/sqlbuilder"
 )
 
+// ResultNetwork is used to represent the structure made from the JOIN
+// between the results and networks tables.
+type ResultNetwork struct {
+	Result  `db:",inline"`
+	Network `db:",inline"`
+}
+
+// MeasurementURLNetwork is used for the JOIN between Measurement and URL
+type MeasurementURLNetwork struct {
+	Measurement `db:",inline"`
+	Network     `db:",inline"`
+	NetworkID   int64 `db:"network_id"`
+	URL         `db:",inline"`
+}
+
 // Network represents a network tested by the user
 type Network struct {
 	ID          int64  `db:"id"`
@@ -23,10 +38,10 @@ type Network struct {
 
 // URL represents URLs from the testing lists
 type URL struct {
-	ID           int64  `db:"id"`
+	ID           sql.NullInt64  `db:"id"`
-	URL          string `db:"url"`
+	URL          sql.NullString `db:"url"`
-	CategoryCode string `db:"category_code"`
+	CategoryCode sql.NullString `db:"category_code"`
-	CountryCode  string `db:"country_code"`
+	CountryCode  sql.NullString `db:"country_code"`
 }
 
 // Measurement model