c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(webconnectivity): use engine.InputLoader (#179)

Browse Source 
This diff pins to ooni/probe-engine@3049779878
and starts using the recently introduced probe-engine APIs.

Namely, here, we use the InputLoader for loading URLs.

I've confirmed manually everything is still working as intended.

Part of https://github.com/ooni/probe/issues/1283.

(In particular, the InputLoader is the abstraction allowing us to load
input from several sources, including command line flags and external
files.)
This commit is contained in:
Simone Basso 2020-11-30 15:39:25 +01:00 committed by GitHub
parent 2fc160b202
commit a324e94973
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -13,7 +13,7 @@ require (
github.com/mattn/go-colorable v0.1.8
github.com/mattn/go-sqlite3 v1.14.5 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/ooni/probe-engine v0.20.1-0.20201126115133-286613b74e6c
github.com/ooni/probe-engine v0.20.1-0.20201130132023-3049779878bf
github.com/pkg/errors v0.9.1
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
github.com/sirupsen/logrus v1.7.0 // indirect

4
go.sum
View File

@ -350,8 +350,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/ooni/probe-engine v0.20.1-0.20201126115133-286613b74e6c h1:+4uKh4ljd5Lui16QAKSKLw4Ld2NACDnuW7lxnUDZ8n8=
github.com/ooni/probe-engine v0.20.1-0.20201126115133-286613b74e6c/go.mod h1:58nNKsvU/jPjsQ8OZJFNu06l6yRJYYWfNE1JwGmbRwc=
github.com/ooni/probe-engine v0.20.1-0.20201130132023-3049779878bf h1:xPhWI1hp1WeKBxca5rX3fyoICZ5JiweFv7fUnx4R2Mg=
github.com/ooni/probe-engine v0.20.1-0.20201130132023-3049779878bf/go.mod h1:58nNKsvU/jPjsQ8OZJFNu06l6yRJYYWfNE1JwGmbRwc=
github.com/ooni/psiphon v0.1.0 h1:fUcDpzZSd2McI5GomHtwusQllYPQoq3ETM3GMfQxCQQ=
github.com/ooni/psiphon v0.1.0/go.mod h1:p+l7SBAGTH3bw9ALcLvlrz6ry+isIM2f51Y5yOhgmTQ=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=

15
internal/nettests/web_connectivity.go
View File

@ -1,22 +1,27 @@
package nettests
import (
"context"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/database"
engine "github.com/ooni/probe-engine"
)
func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
inputloader := engine.NewInputLoader(engine.InputLoaderConfig{
InputPolicy: engine.InputRequired,
Session: ctl.Session,
URLCategories: categories,
URLLimit: limit,
})
testlist, err := inputloader.Load(context.Background())
var urls []string
urlIDMap := make(map[int64]int64)
testlist, err := ctl.Session.QueryTestListsURLs(&engine.TestListsURLsConfig{
Limit: limit,
Categories: categories,
})
if err != nil {
return nil, nil, err
}
for idx, url := range testlist.Result {
for idx, url := range testlist {
log.Debugf("Going over URL %d", idx)
urlID, err := database.CreateOrUpdateURL(
ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode,