Implement writing using the new single measurement pattern

internal/database/actions.go

@@ -1,12 +1,12 @@
 package database
 
 import (
-	"bufio"
 	"database/sql"
 	"encoding/json"
-	"io"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"reflect"
 	"time"
 
@@ -55,44 +55,17 @@ func GetMeasurementJSON(sess sqlbuilder.Database, measurementID int64) (map[stri
 		log.Errorf("failed to run query %s: %v", req.String(), err)
 		return nil, err
 	}
-	reportFilePath := measurement.Measurement.ReportFilePath
-	// If the url->url is NULL then we are dealing with a single entry
-	// measurement and all we have to do is read the file and return it.
-	if measurement.URL.URL.Valid == false {
-		b, err := ioutil.ReadFile(reportFilePath)
-		if err != nil {
-			return nil, err
-		}
-		if err := json.Unmarshal(b, &msmtJSON); err != nil {
-			return nil, err
-		}
-		return msmtJSON, nil
-	}
-	// When the URL is a string then we need to seek until we reach the
-	// measurement line in the file that matches the target input
-	url := measurement.URL.URL.String
-	file, err := os.Open(reportFilePath)
+	measurementFilePath := measurement.Measurement.MeasurementFilePath.String
+	// TODO handle the case in which we have MeasurementFilePath == NULL because
+	// it's a beta measurement
+	b, err := ioutil.ReadFile(measurementFilePath)
 	if err != nil {
 		return nil, err
 	}
-	defer file.Close()
-	reader := bufio.NewReader(file)
-	for {
-		line, err := reader.ReadString('\n')
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, err
-		}
-		if err := json.Unmarshal([]byte(line), &msmtJSON); err != nil {
-			return nil, err
-		}
-		if msmtJSON["input"].(string) == url {
-			return msmtJSON, nil
-		}
+	if err := json.Unmarshal(b, &msmtJSON); err != nil {
+		return nil, err
 	}
-	return nil, errors.New("Could not find measurement")
+	return msmtJSON, nil
 }
 
 // GetResultTestKeys returns a list of TestKeys for a given result
@@ -197,15 +170,16 @@ func DeleteResult(sess sqlbuilder.Database, resultID int64) error {
 
 // CreateMeasurement writes the measurement to the database a returns a pointer
 // to the Measurement
-func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testName string, resultID int64, reportFilePath string, urlID sql.NullInt64) (*Measurement, error) {
+func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testName string, measurementDir string, idx int, resultID int64, urlID sql.NullInt64) (*Measurement, error) {
+	msmtFilePath := filepath.Join(measurementDir, fmt.Sprintf("msmt-%d.json", idx))
 	msmt := Measurement{
-		ReportID:       reportID,
-		TestName:       testName,
-		ResultID:       resultID,
-		ReportFilePath: reportFilePath,
-		URLID:          urlID,
-		IsFailed:       false,
-		IsDone:         false,
+		ReportID:            reportID,
+		TestName:            testName,
+		ResultID:            resultID,
+		MeasurementFilePath: sql.NullString{String: msmtFilePath, Valid: true},
+		URLID:               urlID,
+		IsFailed:            false,
+		IsDone:              false,
 		// XXX Do we want to have this be part of something else?
 		StartTime: time.Now().UTC(),
 		TestKeys:  "",

internal/database/models.go

@@ -2,11 +2,8 @@ package database
 
 import (
 	"database/sql"
-	"os"
-	"path/filepath"
 	"time"
 
-	"github.com/ooni/probe-cli/utils/shutil"
 	"github.com/pkg/errors"
 	"upper.io/db.v3/lib/sqlbuilder"
 )
@@ -62,9 +59,10 @@ type Measurement struct {
 	MeasurementID    sql.NullInt64  `db:"collector_measurement_id,omitempty"`
 	IsAnomaly        sql.NullBool   `db:"is_anomaly,omitempty"`
 	// FIXME we likely want to support JSON. See: https://github.com/upper/db/issues/462
-	TestKeys       string `db:"test_keys"`
-	ResultID       int64  `db:"result_id"`
-	ReportFilePath string `db:"report_file_path"`
+	TestKeys            string         `db:"test_keys"`
+	ResultID            int64          `db:"result_id"`
+	ReportFilePath      sql.NullString `db:"report_file_path,omitempty"`
+	MeasurementFilePath sql.NullString `db:"measurement_file_path,omitempty"`
 }
 
 // Result model
@@ -150,32 +148,3 @@ func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error {
 	}
 	return nil
 }
-
-// AddToResult adds a measurement to a result
-func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error {
-	var err error
-
-	m.ResultID = result.ID
-	finalPath := filepath.Join(result.MeasurementDir,
-		filepath.Base(m.ReportFilePath))
-
-	// If the finalPath already exists, it means it has already been moved there.
-	// This happens in multi input reports
-	if _, err = os.Stat(finalPath); os.IsNotExist(err) {
-		err := shutil.CopyFile(m.ReportFilePath, finalPath, false)
-		if err != nil {
-			return errors.Wrap(err, "copying report file")
-		}
-		err = os.Remove(m.ReportFilePath)
-		if err != nil {
-			return errors.Wrap(err, "deleting report file")
-		}
-	}
-	m.ReportFilePath = finalPath
-
-	err = sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
-	if err != nil {
-		return errors.Wrap(err, "updating measurement")
-	}
-	return nil
-}