Get web_connectivity to work with dynamic URL fetching

nettests/nettests.go

@@ -43,6 +43,8 @@ type Controller struct {
 // Init should be called once to initialise the nettest
 func (c *Controller) Init(nt *mk.Nettest) error {
 	log.Debugf("Init: %v", nt)
+	c.Ctx.LocationLookup()
+
 	c.msmts = make(map[int64]*database.Measurement)
 
 	msmtTemplate := database.Measurement{
@@ -57,9 +59,14 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 
 	log.Debugf("OutputPath: %s", c.msmtPath)
 	nt.Options = mk.NettestOptions{
-		IncludeIP:        c.Ctx.Config.Sharing.IncludeIP,
-		IncludeASN:       c.Ctx.Config.Sharing.IncludeASN,
-		IncludeCountry:   c.Ctx.Config.Advanced.IncludeCountry,
+		IncludeIP:      c.Ctx.Config.Sharing.IncludeIP,
+		IncludeASN:     c.Ctx.Config.Sharing.IncludeASN,
+		IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry,
+
+		ProbeCC:  c.Ctx.Location.CountryCode,
+		ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN),
+		ProbeIP:  c.Ctx.Location.IP,
+
 		DisableCollector: false,
 		SoftwareName:     "ooniprobe",
 		SoftwareVersion:  version.Version,

nettests/websites/web_connectivity.go

@@ -1,10 +1,56 @@
 package websites
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
 	"github.com/measurement-kit/go-measurement-kit"
 	"github.com/openobservatory/gooni/nettests"
+	"github.com/pkg/errors"
 )
 
+type URLInfo struct {
+	URL          string `json:"url"`
+	CategoryCode string `json:"category_code"`
+}
+
+type URLResponse struct {
+	Results []URLInfo `json:"results"`
+}
+
+const orchestrateBaseURL = "https://events.proteus.test.ooni.io"
+
+func lookupURLs(ctl *nettests.Controller) ([]string, error) {
+	var (
+		parsed = new(URLResponse)
+		urls   []string
+	)
+	reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
+		orchestrateBaseURL,
+		ctl.Ctx.Location.CountryCode)
+
+	resp, err := http.Get(reqURL)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to perform request")
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to read response body")
+	}
+	err = json.Unmarshal([]byte(body), &parsed)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to parse json")
+	}
+
+	for _, url := range parsed.Results {
+		urls = append(urls, url.URL)
+	}
+	return urls, nil
+}
+
 // WebConnectivity test implementation
 type WebConnectivity struct {
 }
@@ -13,6 +59,13 @@ type WebConnectivity struct {
 func (n WebConnectivity) Run(ctl *nettests.Controller) error {
 	nt := mk.NewNettest("WebConnectivity")
 	ctl.Init(nt)
+
+	urls, err := lookupURLs(ctl)
+	if err != nil {
+		return err
+	}
+	nt.Options.Inputs = urls
+
 	return nt.Run()
 }