Update the measurement, network and url creation to the new schema

nettests/websites/web_connectivity.go

@@ -6,7 +6,9 @@ import (
 	"io/ioutil"
 	"net/http"
 
+	"github.com/apex/log"
 	"github.com/measurement-kit/go-measurement-kit"
+	"github.com/ooni/probe-cli/internal/database"
 	"github.com/ooni/probe-cli/nettests"
 	"github.com/pkg/errors"
 )
@@ -14,6 +16,7 @@ import (
 // URLInfo contains the URL and the citizenlab category code for that URL
 type URLInfo struct {
 	URL          string `json:"url"`
+	CountryCode  string `json:"country_code"`
 	CategoryCode string `json:"category_code"`
 }
 
@@ -24,10 +27,11 @@ type URLResponse struct {
 
 const orchestrateBaseURL = "https://events.proteus.test.ooni.io"
 
-func lookupURLs(ctl *nettests.Controller) ([]string, error) {
+func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
 	var (
-		parsed = new(URLResponse)
-		urls   []string
+		parsed   = new(URLResponse)
+		urls     []string
+		urlIDMap map[int64]int64
 	)
 	// XXX pass in the configuration for category codes
 	reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
@@ -36,22 +40,58 @@ func lookupURLs(ctl *nettests.Controller) ([]string, error) {
 
 	resp, err := http.Get(reqURL)
 	if err != nil {
-		return urls, errors.Wrap(err, "failed to perform request")
+		return urls, urlIDMap, errors.Wrap(err, "failed to perform request")
 	}
 
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return urls, errors.Wrap(err, "failed to read response body")
+		return urls, urlIDMap, errors.Wrap(err, "failed to read response body")
 	}
 	err = json.Unmarshal([]byte(body), &parsed)
 	if err != nil {
-		return urls, errors.Wrap(err, "failed to parse json")
+		return urls, urlIDMap, errors.Wrap(err, "failed to parse json")
 	}
 
-	for _, url := range parsed.Results {
+	for idx, url := range parsed.Results {
+		var urlID int64
+
+		res, err := ctl.Ctx.DB.Update("urls").Set(
+			"url", url.URL,
+			"category_code", url.CategoryCode,
+			"country_code", url.CountryCode,
+		).Where("url = ? AND country_code = ?", url.URL, url.CountryCode).Exec()
+
+		if err != nil {
+			log.Error("Failed to write to the URL table")
+		} else {
+			affected, err := res.RowsAffected()
+
+			if err != nil {
+				log.Error("Failed to get affected row count")
+			} else if affected == 0 {
+				newID, err := ctl.Ctx.DB.Collection("urls").Insert(
+					database.URL{
+						URL:          url.URL,
+						CategoryCode: url.CategoryCode,
+						CountryCode:  url.CountryCode,
+					})
+				if err != nil {
+					log.Error("Failed to insert into the URLs table")
+				}
+				urlID = newID.(int64)
+			} else {
+				lastID, err := res.LastInsertId()
+				if err != nil {
+					log.Error("failed to get URL ID")
+				}
+				urlID = lastID
+			}
+		}
+
+		urlIDMap[int64(idx)] = urlID
 		urls = append(urls, url.URL)
 	}
-	return urls, nil
+	return urls, urlIDMap, nil
 }
 
 // WebConnectivity test implementation
@@ -63,10 +103,11 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error {
 	nt := mk.NewNettest("WebConnectivity")
 	ctl.Init(nt)
 
-	urls, err := lookupURLs(ctl)
+	urls, urlIDMap, err := lookupURLs(ctl)
 	if err != nil {
 		return err
 	}
+	ctl.SetInputIdxMap(urlIDMap)
 	nt.Options.Inputs = urls
 
 	return nt.Run()