From af4cbd1846d01cf36380fcd715a29675134952b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 11 Jan 2021 19:59:20 +0100 Subject: [PATCH] =?UTF-8?q?Implement=20support=20for=20not=20writing=20to?= =?UTF-8?q?=20disk=20and=20fetching=20measurements=20f=E2=80=A6=20(#189)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Implement support for not writing to disk and fetching measurements from the API * Handle case of input not being set * Comment about exposing raw_measurement in probe-engine * Add basic test for GetMeasurementJSON * Update internal/database/actions.go Co-authored-by: Simone Basso --- internal/database/actions.go | 20 +++++++++++ internal/database/actions_test.go | 58 +++++++++++++++++++++++++++++++ internal/nettests/nettests.go | 13 +++++-- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/internal/database/actions.go b/internal/database/actions.go index 0cb0818..a82eeed 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/http" "os" "path/filepath" "reflect" @@ -55,6 +56,25 @@ func GetMeasurementJSON(sess sqlbuilder.Database, measurementID int64) (map[stri log.Errorf("failed to run query %s: %v", req.String(), err) return nil, err } + if measurement.IsUploaded { + // TODO(bassosimone): this should be a function exposed by probe-engine + reportID := measurement.Measurement.ReportID.String + measurementURL := fmt.Sprintf("https://api.ooni.io/api/v1/raw_measurement?report_id=%s", reportID) + if measurement.URL.URL.Valid == true { + measurementURL += "&input=" + measurement.URL.URL.String + } + resp, err := http.Get(measurementURL) + if err != nil { + log.Errorf("failed to fetch the measurement %s %s", reportID, measurement.URL.URL.String) + return nil, err + } + defer resp.Body.Close() + if err := json.NewDecoder(resp.Body).Decode(&msmtJSON); err != nil { + log.Error("failed to unmarshal the measurement_json") + return nil, err + } + return msmtJSON, nil + } // MeasurementFilePath might be NULL because the measurement from a // 3.0.0-beta install if measurement.Measurement.MeasurementFilePath.Valid == false { diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index a2b7760..902a100 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -303,3 +303,61 @@ func TestPerformanceTestKeys(t *testing.T) { t.Fatalf("error Download %f", tk.Download) } } + +func TestGetMeasurementJSON(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dbtest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + tmpdir, err := ioutil.TempDir("", "oonitest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + sess, err := Connect(tmpfile.Name()) + if err != nil { + t.Fatal(err) + } + + location := locationInfo{ + asn: 0, + countryCode: "IT", + networkName: "Unknown", + } + network, err := CreateNetwork(sess, &location) + if err != nil { + t.Fatal(err) + } + + result, err := CreateResult(sess, tmpdir, "websites", network.ID) + if err != nil { + t.Fatal(err) + } + + reportID := sql.NullString{String: "20210111T085144Z_ndt_RU_3216_n1_qMVnP0PTX7ObUSmD", Valid: true} + testName := "antani" + resultID := result.ID + msmtFilePath := tmpdir + urlID := sql.NullInt64{Int64: 0, Valid: false} + + msmt, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID) + if err != nil { + t.Fatal(err) + } + msmt.IsUploaded = true + err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt) + if err != nil { + t.Fatal(err) + } + + tk, err := GetMeasurementJSON(sess, msmt.ID) + if err != nil { + t.Fatal(err) + } + if tk["probe_asn"] != "AS3216" { + t.Error("inconsistent measurement downloaded") + } +} diff --git a/internal/nettests/nettests.go b/internal/nettests/nettests.go index 225ca08..132810e 100644 --- a/internal/nettests/nettests.go +++ b/internal/nettests/nettests.go @@ -149,6 +149,7 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err // undertsand what went wrong (censorship? bug? anomaly?). } + saveToDisk := true if c.Probe.Config().Sharing.UploadResults { // Implementation note: SubmitMeasurement will fail here if we did fail // to open the report but we still want to continue. There will be a @@ -160,12 +161,18 @@ func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) err } } else if err := c.msmts[idx64].UploadSucceeded(c.Probe.DB()); err != nil { return errors.Wrap(err, "failed to mark upload as succeeded") + } else { + // Everything went OK, don't save to disk + saveToDisk = false + } + } + // We only save the measurement to disk if we failed to upload the measurement + if saveToDisk == true { + if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil { + return errors.Wrap(err, "failed to save measurement on disk") } } - if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil { - return errors.Wrap(err, "failed to save measurement on disk") - } if err := c.msmts[idx64].Done(c.Probe.DB()); err != nil { return errors.Wrap(err, "failed to mark measurement as done") }