diff --git a/.github/workflows/e2eminiooni.yml b/.github/workflows/e2eminiooni.yml
deleted file mode 100644
index dc06539..0000000
--- a/.github/workflows/e2eminiooni.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-# e2eminiooni runs end-to-end tests using miniooni
-name: "e2eminiooni"
-on:
-  push:
-    branches:
-      - "master"
-jobs:
-  test:
-    runs-on: "ubuntu-20.04"
-    steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.16"
-      - uses: actions/checkout@v2
-      - run: ./E2E/miniooni.bash
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 7c4a431..38994bf 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/386
-      - run: ./smoketest.sh ./CLI/linux/386/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/linux/386/ooniprobe
       - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
       - run: |
           for deb in *.deb; do
@@ -27,7 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64
-      - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/linux/amd64/ooniprobe
       - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
       - run: |
           for deb in *.deb; do
@@ -45,7 +45,7 @@ jobs:
       - run: sudo apt-get update -q
       - run: sudo apt-get install -y qemu-user-static
       - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm
-      - run: ./smoketest.sh ./CLI/linux/arm/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/linux/arm/ooniprobe
       - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
       - run: |
           for deb in *.deb; do
@@ -64,7 +64,7 @@ jobs:
       - run: sudo apt-get update -q
       - run: sudo apt-get install -y qemu-user-static
       - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64
-      - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/linux/arm64/ooniprobe
       - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
       - run: |
           for deb in *.deb; do
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index d32830e..fd66466 100644
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -13,4 +13,4 @@ jobs:
           go-version: "1.16.4"
       - uses: actions/checkout@v2
       - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/darwin/amd64/ooniprobe
-      - run: ./smoketest.sh ./CLI/darwin/amd64/ooniprobe
+      - run: ./E2E/ooniprobe.sh ./CLI/darwin/amd64/ooniprobe
diff --git a/.github/workflows/miniooni.yml b/.github/workflows/miniooni.yml
index 4aa187c..c44f2d6 100644
--- a/.github/workflows/miniooni.yml
+++ b/.github/workflows/miniooni.yml
@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/miniooni
 
-      - run: ./CLI/linux/amd64/miniooni --yes -nNi https://example.com web_connectivity
+      - run: ./E2E/miniooni.bash ./CLI/linux/amd64/miniooni
 
       - uses: actions/upload-artifact@v1
         with:
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 51055a4..d2fb303 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -26,4 +26,4 @@ jobs:
       - uses: actions/download-artifact@v2
         with:
           name: ooniprobe.exe
-      - run: bash.exe ./smoketest.sh ./ooniprobe.exe
+      - run: bash.exe ./E2E/ooniprobe.sh ./ooniprobe.exe
diff --git a/.gitignore b/.gitignore
index fe853c7..fc96fb6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 .DS_Store
+/*.asc
 /*.deb
 /*.jsonl
 /*.tar.gz
diff --git a/E2E/miniooni.bash b/E2E/miniooni.bash
index 36b7ce5..8ea4b35 100755
--- a/E2E/miniooni.bash
+++ b/E2E/miniooni.bash
@@ -1,16 +1,21 @@
 #!/bin/bash
+#
+# This script checks whether we're able to submit measurements to
+# different backends using miniooni. It fails if we cannot find in
+# the specific backend the measurement we submitted.
+#
 set -e
-go build -v ./internal/cmd/miniooni
-probeservices=()
-probeservices+=( "https://ps1.ooni.io" )
-probeservices+=( "https://dvp6h0xblpcqp.cloudfront.net" )
-probeservices+=( "https://ams-pg-test.ooni.org" )
-for ps in ${probeservices[@]}; do
+backends=()
+backends+=( "https://ps1.ooni.io" )
+backends+=( "https://dvp6h0xblpcqp.cloudfront.net" )
+backends+=( "https://ams-pg-test.ooni.org" )
+miniooni="${1:-./miniooni}"
+for ps in ${backends[@]}; do
     opt="-o E2E/o.jsonl --probe-services=$ps"
     set -x
-    ./miniooni --yes $opt -i http://mail.google.com web_connectivity
-    ./miniooni --yes $opt tor
-    ./miniooni --yes $opt psiphon
+    $miniooni --yes $opt -i http://mail.google.com web_connectivity
+    $miniooni --yes $opt tor
+    $miniooni --yes $opt psiphon
     set +x
 done
 set -x
diff --git a/E2E/ooniprobe.sh b/E2E/ooniprobe.sh
new file mode 100755
index 0000000..1dc1475
--- /dev/null
+++ b/E2E/ooniprobe.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# This test for now uses --no-collector and we just ensure that the OONI
+# instance is not exploding. We are confident that, if miniooni submits
+# measurements, also ooniprobe should be able to do that. However, it would
+# actually be nice if someone could enhance this script to also make sure
+# that we can actually fetch the measurements we submit.
+#
+set -ex
+if [ "$#" != 1 ]; then
+    echo "Usage: $0 <binary>" 1>&2
+    exit 1
+fi
+$1 onboard --yes
+# Important! DO NOT run performance from CI b/c it will overload m-lab servers
+$1 run websites --config cmd/ooniprobe/testdata/testing-config.json --no-collector
diff --git a/smoketest.sh b/smoketest.sh
deleted file mode 100755
index aca6972..0000000
--- a/smoketest.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-set -ex
-if [ "$#" != 1 ]; then
-    echo "Usage: $0 <binary>" 1>&2
-    exit 1
-fi
-$1 onboard --yes
-# Important! DO NOT run performance from CI b/c it will overload m-lab servers
-$1 run websites --config cmd/ooniprobe/testdata/testing-config.json --no-collector