cfd3b8f9b2
This diff introduces the possibility of specifying --input-file file multiple times to force ooniprobe to read inputs from file. Like we do for miniooni, the file shall contain a single entry per line and this entry should be a URL for websites. Likewise, one can specify --input URL multiple times. This implementation is a very simple, initial implementation and there is a bunch of changes I'd like to add on top of it. And also perhaps a bunch of cleanups. I've chosen to expose these flag _only_ for websites for now. Part of https://github.com/ooni/probe/issues/1283.
62 lines
1.7 KiB
Go
62 lines
1.7 KiB
Go
package nettests
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/apex/log"
|
|
"github.com/ooni/probe-cli/internal/database"
|
|
engine "github.com/ooni/probe-engine"
|
|
)
|
|
|
|
func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
|
|
inputloader := engine.NewInputLoader(engine.InputLoaderConfig{
|
|
InputPolicy: engine.InputRequired,
|
|
Session: ctl.Session,
|
|
SourceFiles: ctl.InputFiles,
|
|
StaticInputs: ctl.Inputs,
|
|
URLCategories: categories,
|
|
URLLimit: limit,
|
|
})
|
|
testlist, err := inputloader.Load(context.Background())
|
|
var urls []string
|
|
urlIDMap := make(map[int64]int64)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for idx, url := range testlist {
|
|
log.Debugf("Going over URL %d", idx)
|
|
urlID, err := database.CreateOrUpdateURL(
|
|
ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode,
|
|
)
|
|
if err != nil {
|
|
log.Error("failed to add to the URL table")
|
|
return nil, nil, err
|
|
}
|
|
log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
|
|
urlIDMap[int64(idx)] = urlID
|
|
urls = append(urls, url.URL)
|
|
}
|
|
return urls, urlIDMap, nil
|
|
}
|
|
|
|
// WebConnectivity test implementation
|
|
type WebConnectivity struct {
|
|
}
|
|
|
|
// Run starts the test
|
|
func (n WebConnectivity) Run(ctl *Controller) error {
|
|
log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
|
|
urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesURLLimit, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctl.SetInputIdxMap(urlIDMap)
|
|
builder, err := ctl.Session.NewExperimentBuilder(
|
|
"web_connectivity",
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return ctl.Run(builder, urls)
|
|
}
|