feat(inputloader): use check-in to fetch URLs (#267)

* ongoing work

* reduce diff with master

* feat(inputloader): use the check-in API

Part of https://github.com/ooni/probe/issues/1299

* fix: better naming for a variable

* chore: add more tests

* fix: add one more TODO

cmd/ooniprobe/internal/cli/run/run.go

@@ -1,8 +1,6 @@
 package run
 
 import (
-	"runtime"
-
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/fatih/color"
@@ -77,14 +75,6 @@ func init() {
 
 	unattendedCmd := cmd.Command("unattended", "")
 	unattendedCmd.Action(func(_ *kingpin.ParseContext) error {
-		// Until we have enabled the check-in API we're called every
-		// hour on darwin and we need to self throttle.
-		// TODO(bassosimone): switch to check-in and remove this hack.
-		switch runtime.GOOS {
-		case "darwin", "windows":
-			const veryFew = 10
-			probe.Config().Nettests.WebsitesURLLimit = veryFew
-		}
 		return functionalRun(func(name string, gr nettests.Group) bool {
 			return gr.UnattendedOK == true
 		})

cmd/ooniprobe/internal/nettests/web_connectivity.go

@@ -6,16 +6,29 @@ import (
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
 	engine "github.com/ooni/probe-cli/v3/internal/engine"
+	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 
+// TODO(bassosimone): we should remove the limit argument and
+// we should also remove it from the config.
+
+// TODO(bassosimone): we should propagate the kind of run
+// to here such that we get the right runType.
+
+// TODO(bassosimone): we are breaking the use case in which
+// someone choose the number of URLs explicitly via the config.
+
 func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
 	inputloader := engine.NewInputLoader(engine.InputLoaderConfig{
-		InputPolicy:   engine.InputOrQueryBackend,
-		Session:       ctl.Session,
-		SourceFiles:   ctl.InputFiles,
-		StaticInputs:  ctl.Inputs,
-		URLCategories: categories,
-		URLLimit:      limit,
+		CheckInConfig: &model.CheckInConfig{
+			WebConnectivity: model.CheckInConfigWebConnectivity{
+				CategoryCodes: categories,
+			},
+		},
+		InputPolicy:  engine.InputOrQueryBackend,
+		Session:      ctl.Session,
+		SourceFiles:  ctl.InputFiles,
+		StaticInputs: ctl.Inputs,
 	})
 	testlist, err := inputloader.Load(context.Background())
 	var urls []string