refactor: reorganize and to end and smoke testing (#348)

1. we can merge the e2eminiooni.yml test into the miniooni.yml test
so to reduce the number of tests we run;

2. ideally we would like the smoketest.sh test to evolve and also
check whether we can fetch the measurements we submitted, so start
moving this script into the `./E2E` folder, add a note saying we
would like to do that, and direct all the tests to run this script
at its new location and with its new name (`ooniprobe.sh`).

With these two changes, it's fine to remove the ooniprobe2debian.yml
test in ooni/e2etesting because we're moving its functionality to this
repository. (We mentioned the need to do this move in a previous TODO
comment at the top of such a script.)

Work part of https://github.com/ooni/probe/issues/1468

.github/workflows/e2eminiooni.yml

@@ -1,15 +0,0 @@
-# e2eminiooni runs end-to-end tests using miniooni
-name: "e2eminiooni"
-on:
-  push:
-    branches:
-      - "master"
-jobs:
-  test:
-    runs-on: "ubuntu-20.04"
-    steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.16"
-      - uses: actions/checkout@v2
-      - run: ./E2E/miniooni.bash

.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/386
-      - run: ./smoketest.sh ./CLI/linux/386/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/linux/386/ooniprobe
       - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
       - run: |
           for deb in *.deb; do
@@ -27,7 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64
-      - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/linux/amd64/ooniprobe
       - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
       - run: |
           for deb in *.deb; do
@@ -45,7 +45,7 @@ jobs:
       - run: sudo apt-get update -q
       - run: sudo apt-get install -y qemu-user-static
       - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm
-      - run: ./smoketest.sh ./CLI/linux/arm/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/linux/arm/ooniprobe
       - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
       - run: |
           for deb in *.deb; do
@@ -64,7 +64,7 @@ jobs:
       - run: sudo apt-get update -q
       - run: sudo apt-get install -y qemu-user-static
       - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64
-      - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/linux/arm64/ooniprobe
       - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
       - run: |
           for deb in *.deb; do

.github/workflows/macos.yml

@@ -13,4 +13,4 @@ jobs:
           go-version: "1.16.4"
       - uses: actions/checkout@v2
       - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/darwin/amd64/ooniprobe
-      - run: ./smoketest.sh ./CLI/darwin/amd64/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/darwin/amd64/ooniprobe

.github/workflows/miniooni.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/miniooni
 
-      - run: ./CLI/linux/amd64/miniooni --yes -nNi https://example.com web_connectivity
+      - run: ./E2E/miniooni.bash ./CLI/linux/amd64/miniooni
 
       - uses: actions/upload-artifact@v1
         with:

.github/workflows/windows.yml

@@ -26,4 +26,4 @@ jobs:
       - uses: actions/download-artifact@v2
         with:
           name: ooniprobe.exe
-      - run: bash.exe ./smoketest.sh ./ooniprobe.exe
+      - run: bash.exe ./E2E/ooniprobe.sh ./ooniprobe.exe

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+/*.asc
 /*.deb
 /*.jsonl
 /*.tar.gz

E2E/miniooni.bash

@@ -1,16 +1,21 @@
 #!/bin/bash
+#
+# This script checks whether we're able to submit measurements to
+# different backends using miniooni. It fails if we cannot find in
+# the specific backend the measurement we submitted.
+#
 set -e
-go build -v ./internal/cmd/miniooni
-probeservices=()
-probeservices+=( "https://ps1.ooni.io" )
-probeservices+=( "https://dvp6h0xblpcqp.cloudfront.net" )
-probeservices+=( "https://ams-pg-test.ooni.org" )
-for ps in ${probeservices[@]}; do
+backends=()
+backends+=( "https://ps1.ooni.io" )
+backends+=( "https://dvp6h0xblpcqp.cloudfront.net" )
+backends+=( "https://ams-pg-test.ooni.org" )
+miniooni="${1:-./miniooni}"
+for ps in ${backends[@]}; do
     opt="-o E2E/o.jsonl --probe-services=$ps"
     set -x
-    ./miniooni --yes $opt -i http://mail.google.com web_connectivity
-    ./miniooni --yes $opt tor
-    ./miniooni --yes $opt psiphon
+    $miniooni --yes $opt -i http://mail.google.com web_connectivity
+    $miniooni --yes $opt tor
+    $miniooni --yes $opt psiphon
     set +x
 done
 set -x

E2E/ooniprobe.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# This test for now uses --no-collector and we just ensure that the OONI
+# instance is not exploding. We are confident that, if miniooni submits
+# measurements, also ooniprobe should be able to do that. However, it would
+# actually be nice if someone could enhance this script to also make sure
+# that we can actually fetch the measurements we submit.
+#
+set -ex
+if [ "$#" != 1 ]; then
+    echo "Usage: $0 <binary>" 1>&2
+    exit 1
+fi
+$1 onboard --yes
+# Important! DO NOT run performance from CI b/c it will overload m-lab servers
+$1 run websites --config cmd/ooniprobe/testdata/testing-config.json --no-collector

smoketest.sh

@@ -1,9 +0,0 @@
-#!/bin/sh
-set -ex
-if [ "$#" != 1 ]; then
-    echo "Usage: $0 <binary>" 1>&2
-    exit 1
-fi
-$1 onboard --yes
-# Important! DO NOT run performance from CI b/c it will overload m-lab servers
-$1 run websites --config cmd/ooniprobe/testdata/testing-config.json --no-collector