refactor(webconnectivity): use engine.InputLoader (#179)

This diff pins to ooni/probe-engine@3049779878
and starts using the recently introduced probe-engine APIs.

Namely, here, we use the InputLoader for loading URLs.

I've confirmed manually everything is still working as intended.

Part of https://github.com/ooni/probe/issues/1283.

(In particular, the InputLoader is the abstraction allowing us to load
input from several sources, including command line flags and external
files.)
This commit is contained in:
Simone Basso
2020-11-30 15:39:25 +01:00
committed by GitHub
parent 2fc160b202
commit a324e94973
3 changed files with 13 additions and 8 deletions
+10 -5
View File
@@ -1,22 +1,27 @@
package nettests
import (
"context"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/database"
engine "github.com/ooni/probe-engine"
)
func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
inputloader := engine.NewInputLoader(engine.InputLoaderConfig{
InputPolicy: engine.InputRequired,
Session: ctl.Session,
URLCategories: categories,
URLLimit: limit,
})
testlist, err := inputloader.Load(context.Background())
var urls []string
urlIDMap := make(map[int64]int64)
testlist, err := ctl.Session.QueryTestListsURLs(&engine.TestListsURLsConfig{
Limit: limit,
Categories: categories,
})
if err != nil {
return nil, nil, err
}
for idx, url := range testlist.Result {
for idx, url := range testlist {
log.Debugf("Going over URL %d", idx)
urlID, err := database.CreateOrUpdateURL(
ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode,