diff --git a/go.mod b/go.mod index 52bc6dc..b8f266a 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/mattn/go-colorable v0.1.8 github.com/mattn/go-sqlite3 v1.14.5 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/ooni/probe-engine v0.20.1-0.20201126115133-286613b74e6c + github.com/ooni/probe-engine v0.20.1-0.20201130132023-3049779878bf github.com/pkg/errors v0.9.1 github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351 github.com/sirupsen/logrus v1.7.0 // indirect diff --git a/go.sum b/go.sum index eb3e2bf..4896d8e 100644 --- a/go.sum +++ b/go.sum @@ -350,8 +350,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/ooni/probe-engine v0.20.1-0.20201126115133-286613b74e6c h1:+4uKh4ljd5Lui16QAKSKLw4Ld2NACDnuW7lxnUDZ8n8= -github.com/ooni/probe-engine v0.20.1-0.20201126115133-286613b74e6c/go.mod h1:58nNKsvU/jPjsQ8OZJFNu06l6yRJYYWfNE1JwGmbRwc= +github.com/ooni/probe-engine v0.20.1-0.20201130132023-3049779878bf h1:xPhWI1hp1WeKBxca5rX3fyoICZ5JiweFv7fUnx4R2Mg= +github.com/ooni/probe-engine v0.20.1-0.20201130132023-3049779878bf/go.mod h1:58nNKsvU/jPjsQ8OZJFNu06l6yRJYYWfNE1JwGmbRwc= github.com/ooni/psiphon v0.1.0 h1:fUcDpzZSd2McI5GomHtwusQllYPQoq3ETM3GMfQxCQQ= github.com/ooni/psiphon v0.1.0/go.mod h1:p+l7SBAGTH3bw9ALcLvlrz6ry+isIM2f51Y5yOhgmTQ= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= diff --git a/internal/nettests/web_connectivity.go b/internal/nettests/web_connectivity.go index 32dbc59..7f33e2d 100644 --- a/internal/nettests/web_connectivity.go +++ b/internal/nettests/web_connectivity.go @@ -1,22 +1,27 @@ package nettests import ( + "context" + "github.com/apex/log" "github.com/ooni/probe-cli/internal/database" engine "github.com/ooni/probe-engine" ) func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) { + inputloader := engine.NewInputLoader(engine.InputLoaderConfig{ + InputPolicy: engine.InputRequired, + Session: ctl.Session, + URLCategories: categories, + URLLimit: limit, + }) + testlist, err := inputloader.Load(context.Background()) var urls []string urlIDMap := make(map[int64]int64) - testlist, err := ctl.Session.QueryTestListsURLs(&engine.TestListsURLsConfig{ - Limit: limit, - Categories: categories, - }) if err != nil { return nil, nil, err } - for idx, url := range testlist.Result { + for idx, url := range testlist { log.Debugf("Going over URL %d", idx) urlID, err := database.CreateOrUpdateURL( ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode,