Implement support for not writing to disk and fetching measurements f… (#189)

* Implement support for not writing to disk and fetching measurements from the API

* Handle case of input not being set

* Comment about exposing raw_measurement in probe-engine

* Add basic test for GetMeasurementJSON

* Update internal/database/actions.go

Co-authored-by: Simone Basso <bassosimone@gmail.com>

internal/database/actions.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -55,6 +56,25 @@ func GetMeasurementJSON(sess sqlbuilder.Database, measurementID int64) (map[stri
 		log.Errorf("failed to run query %s: %v", req.String(), err)
 		return nil, err
 	}
+	if measurement.IsUploaded {
+		// TODO(bassosimone): this should be a function exposed by probe-engine
+		reportID := measurement.Measurement.ReportID.String
+		measurementURL := fmt.Sprintf("https://api.ooni.io/api/v1/raw_measurement?report_id=%s", reportID)
+		if measurement.URL.URL.Valid == true {
+			measurementURL += "&input=" + measurement.URL.URL.String
+		}
+		resp, err := http.Get(measurementURL)
+		if err != nil {
+			log.Errorf("failed to fetch the measurement %s %s", reportID, measurement.URL.URL.String)
+			return nil, err
+		}
+		defer resp.Body.Close()
+		if err := json.NewDecoder(resp.Body).Decode(&msmtJSON); err != nil {
+			log.Error("failed to unmarshal the measurement_json")
+			return nil, err
+		}
+		return msmtJSON, nil
+	}
 	// MeasurementFilePath might be NULL because the measurement from a
 	// 3.0.0-beta install
 	if measurement.Measurement.MeasurementFilePath.Valid == false {

internal/database/actions_test.go

@@ -303,3 +303,61 @@ func TestPerformanceTestKeys(t *testing.T) {
 		t.Fatalf("error Download %f", tk.Download)
 	}
 }
+
+func TestGetMeasurementJSON(t *testing.T) {
+	tmpfile, err := ioutil.TempFile("", "dbtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	tmpdir, err := ioutil.TempDir("", "oonitest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	sess, err := Connect(tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	location := locationInfo{
+		asn:         0,
+		countryCode: "IT",
+		networkName: "Unknown",
+	}
+	network, err := CreateNetwork(sess, &location)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result, err := CreateResult(sess, tmpdir, "websites", network.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reportID := sql.NullString{String: "20210111T085144Z_ndt_RU_3216_n1_qMVnP0PTX7ObUSmD", Valid: true}
+	testName := "antani"
+	resultID := result.ID
+	msmtFilePath := tmpdir
+	urlID := sql.NullInt64{Int64: 0, Valid: false}
+
+	msmt, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	msmt.IsUploaded = true
+	err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tk, err := GetMeasurementJSON(sess, msmt.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk["probe_asn"] != "AS3216" {
+		t.Error("inconsistent measurement downloaded")
+	}
+}

internal/nettests/nettests.go

@@ -149,6 +149,7 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err
 			// undertsand what went wrong (censorship? bug? anomaly?).
 		}
 
+		saveToDisk := true
 		if c.Probe.Config().Sharing.UploadResults {
 			// Implementation note: SubmitMeasurement will fail here if we did fail
 			// to open the report but we still want to continue. There will be a
@@ -160,12 +161,18 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err
 				}
 			} else if err := c.msmts[idx64].UploadSucceeded(c.Probe.DB()); err != nil {
 				return errors.Wrap(err, "failed to mark upload as succeeded")
+			} else {
+				// Everything went OK, don't save to disk
+				saveToDisk = false
+			}
+		}
+		// We only save the measurement to disk if we failed to upload the measurement
+		if saveToDisk == true {
+			if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil {
+				return errors.Wrap(err, "failed to save measurement on disk")
 			}
 		}
 
-		if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil {
-			return errors.Wrap(err, "failed to save measurement on disk")
-		}
 		if err := c.msmts[idx64].Done(c.Probe.DB()); err != nil {
 			return errors.Wrap(err, "failed to mark measurement as done")
 		}