fix(E2E): ensure miniooni.bash is WAI (#972)

This diff re-enables `E2E/miniooni.bash`. To make it working properly, we
needed to figure out which were the right cloudfronts to use.

I looked into the configuration and determined that both cloudfronts
should be used because they basically map to the same host.

I also determined it was backwards to test a mixture of prod and testing
APIs, and probably also flaky. So, I  choose to only test the prod.

Additionally, I added support for testing all supported tunnels.

Closes https://github.com/ooni/probe/issues/2336

E2E/miniooni.bash

@@ -6,21 +6,46 @@
 #
 # Note: using --tunnel=psiphon assumes that we have been compiling
 # miniooni with builtin support for psiphon.
+#
+# Note about cloudfront: as of 2022-10-08, dvp6h0xblpcqp.cloudfront.net
+# and dkyhjv0wpi2dk.cloudfront.net front distinct aliases of the
+# same backend host (backend-fsn.ooni.org). We can use either addr
+# and the result should be the same. So, let us test that.
 
 set -euxo pipefail
 
-backends=()
+rm -f E2E/o.jsonl
-backends+=( "https://api.ooni.io" )
-backends+=( "https://dvp6h0xblpcqp.cloudfront.net" )
-backends+=( "https://ams-pg-test.ooni.org" )
 
 miniooni="${1:-./miniooni}"
-for ps in ${backends[@]}; do
-    opt="-o E2E/o.jsonl --probe-services=$ps"
-    $miniooni --yes $opt -i http://mail.google.com web_connectivity
-done
 
-$miniooni --tunnel=psiphon --yes -i http://mail.google.com web_connectivity
+$miniooni --yes -o E2E/o.jsonl \
-$miniooni --tunnel=tor --yes -i http://mail.google.com web_connectivity
+	--probe-services=https://api.ooni.io/ \
+	--tunnel=none \
+	web_connectivity -i https://mail.google.com/robots.txt
 
-#go run ./internal/cmd/e2epostprocess -expected 5  # TODO(bassosimone): fix this
+$miniooni --yes -o E2E/o.jsonl \
+	--probe-services=https://dvp6h0xblpcqp.cloudfront.net/ \
+	--tunnel=none \
+	web_connectivity -i https://mail.google.com/robots.txt
+
+$miniooni --yes -o E2E/o.jsonl \
+	--probe-services=https://dkyhjv0wpi2dk.cloudfront.net/ \
+	--tunnel=none \
+	web_connectivity -i https://mail.google.com/robots.txt
+
+$miniooni --yes -o E2E/o.jsonl \
+	--probe-services=https://api.ooni.io/ \
+	--tunnel=tor \
+	web_connectivity -i https://mail.google.com/robots.txt
+
+$miniooni --yes -o E2E/o.jsonl \
+	--probe-services=https://api.ooni.io/ \
+	--tunnel=psiphon \
+	web_connectivity -i https://mail.google.com/robots.txt
+
+$miniooni --yes -o E2E/o.jsonl \
+	--probe-services=https://api.ooni.io/ \
+	--tunnel=torsf \
+	web_connectivity -i https://mail.google.com/robots.txt
+
+go run ./internal/cmd/e2epostprocess -expected 6 -backend https://api.ooni.io/

internal/cmd/apitool/main.go

@@ -29,7 +29,7 @@ func newclient() probeservices.Client {
 	ua := fmt.Sprintf("apitool/%s ooniprobe-engine/%s", version.Version, version.Version)
 	return probeservices.Client{
 		APIClientTemplate: httpx.APIClientTemplate{
-			BaseURL:    "https://api.ooni.io/",
+			BaseURL:    *backend,
 			HTTPClient: &http.Client{Transport: txp},
 			Logger:     log.Log,
 			UserAgent:  ua,
@@ -50,6 +50,7 @@ func fatalOnError(err error, message string) {
 }
 
 var (
+	backend  = flag.String("backend", "https://api.ooni.io/", "Backend to use")
 	debug    = flag.Bool("v", false, "Enable verbose mode")
 	input    = flag.String("input", "", "Input of the measurement")
 	mode     = flag.String("mode", "", "One of: check, meta, raw")

internal/cmd/e2epostprocess/main.go

@@ -27,6 +27,7 @@ type Measurement struct {
 }
 
 func main() {
+	backend := flag.String("backend", "https://api.ooni.io/", "Backend to use")
 	expected := flag.Int("expected", 0, "Expected number of measurement files")
 	flag.Parse()
 	if *expected <= 0 {
@@ -61,6 +62,7 @@ func main() {
 			options := []string{
 				"run",
 				"./internal/cmd/apitool",
+				"-backend", *backend,
 				"-mode", "meta",
 				"-report-id", entry.ReportID,
 			}

internal/engine/session.go

@@ -193,6 +193,8 @@ func NewSession(ctx context.Context, config SessionConfig) (*Session, error) {
 			config.Logger.Infof("tunnel '%s' running...", proxyURL.Scheme)
 			sess.tunnel = tunnel
 			proxyURL = tunnel.SOCKS5ProxyURL()
+		case "none":
+			proxyURL = nil // explicit way of saying we don't want to use a tunnel
 		}
 	}
 	sess.proxyURL = proxyURL