Get web_connectivity to work with dynamic URL fetching

Gopkg.lock

@@ -89,7 +89,7 @@
   branch = "master"
   name = "github.com/measurement-kit/go-measurement-kit"
   packages = ["."]
-  revision = "6ae2401a8e498a90ccdd3edbda1841add079b70e"
+  revision = "cbf1c976aeaa2906f4fda278fc3068f7bde63c47"
 
 [[projects]]
   branch = "master"

Makefile

@@ -5,6 +5,11 @@ build:
 	@$(GO) build -i -o dist/ooni cmd/ooni/main.go
 .PHONY: build
 
+update-mk:
+	@echo "updating mk"
+	@dep ensure -update github.com/measurement-kit/go-measurement-kit
+.PHONY: update-mk
+
 bindata:
 	@$(GO) run vendor/github.com/shuLhan/go-bindata/go-bindata/*.go \
 		-nometadata	\

internal/cli/run/run.go

@@ -33,6 +33,12 @@ func init() {
 		}
 		log.Debugf("Running test group %s", group.Label)
 
+		err = ctx.MaybeLocationLookup()
+		if err != nil {
+			log.WithError(err).Error("Failed to lookup the location of the probe")
+			return err
+		}
+
 		result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{
 			Name:      *nettestGroup,
 			StartTime: time.Now().UTC(),

nettests/nettests.go

@@ -43,6 +43,8 @@ type Controller struct {
 // Init should be called once to initialise the nettest
 func (c *Controller) Init(nt *mk.Nettest) error {
 	log.Debugf("Init: %v", nt)
+	c.Ctx.LocationLookup()
+
 	c.msmts = make(map[int64]*database.Measurement)
 
 	msmtTemplate := database.Measurement{
@@ -57,9 +59,14 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 
 	log.Debugf("OutputPath: %s", c.msmtPath)
 	nt.Options = mk.NettestOptions{
-		IncludeIP:        c.Ctx.Config.Sharing.IncludeIP,
+		IncludeIP:      c.Ctx.Config.Sharing.IncludeIP,
-		IncludeASN:       c.Ctx.Config.Sharing.IncludeASN,
+		IncludeASN:     c.Ctx.Config.Sharing.IncludeASN,
-		IncludeCountry:   c.Ctx.Config.Advanced.IncludeCountry,
+		IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry,
+
+		ProbeCC:  c.Ctx.Location.CountryCode,
+		ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN),
+		ProbeIP:  c.Ctx.Location.IP,
+
 		DisableCollector: false,
 		SoftwareName:     "ooniprobe",
 		SoftwareVersion:  version.Version,

nettests/websites/web_connectivity.go

@@ -1,10 +1,56 @@
 package websites
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
 	"github.com/measurement-kit/go-measurement-kit"
 	"github.com/openobservatory/gooni/nettests"
+	"github.com/pkg/errors"
 )
 
+type URLInfo struct {
+	URL          string `json:"url"`
+	CategoryCode string `json:"category_code"`
+}
+
+type URLResponse struct {
+	Results []URLInfo `json:"results"`
+}
+
+const orchestrateBaseURL = "https://events.proteus.test.ooni.io"
+
+func lookupURLs(ctl *nettests.Controller) ([]string, error) {
+	var (
+		parsed = new(URLResponse)
+		urls   []string
+	)
+	reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
+		orchestrateBaseURL,
+		ctl.Ctx.Location.CountryCode)
+
+	resp, err := http.Get(reqURL)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to perform request")
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to read response body")
+	}
+	err = json.Unmarshal([]byte(body), &parsed)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to parse json")
+	}
+
+	for _, url := range parsed.Results {
+		urls = append(urls, url.URL)
+	}
+	return urls, nil
+}
+
 // WebConnectivity test implementation
 type WebConnectivity struct {
 }
@@ -13,6 +59,13 @@ type WebConnectivity struct {
 func (n WebConnectivity) Run(ctl *nettests.Controller) error {
 	nt := mk.NewNettest("WebConnectivity")
 	ctl.Init(nt)
+
+	urls, err := lookupURLs(ctl)
+	if err != nil {
+		return err
+	}
+	nt.Options.Inputs = urls
+
 	return nt.Run()
 }
 

ooni.go

@@ -13,6 +13,7 @@ import (
 	"github.com/openobservatory/gooni/config"
 	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/legacy"
+	"github.com/openobservatory/gooni/utils"
 	"github.com/pkg/errors"
 )
 
@@ -35,15 +36,39 @@ func Onboarding(c *Config) error {
 
 // Context for OONI Probe
 type Context struct {
-	Config *Config
+	Config   *Config
-	DB     *sqlx.DB
+	DB       *sqlx.DB
+	Location *utils.LocationInfo
+
+	Home    string
+	TempDir string
 
-	Home       string
-	TempDir    string
 	dbPath     string
 	configPath string
 }
 
+// MaybeLocationLookup will lookup the location of the user unless it's already cached
+func (c *Context) MaybeLocationLookup() error {
+	if c.Location == nil {
+		return c.LocationLookup()
+	}
+	return nil
+}
+
+// LocationLookup lookup the location of the user via geoip
+func (c *Context) LocationLookup() error {
+	var err error
+
+	dbPath := filepath.Join(c.Home, "geoip")
+
+	c.Location, err = utils.GeoIPLookup(dbPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Init the OONI manager
 func (c *Context) Init() error {
 	var err error