From 2389c7cab3ecb1bc9ece24d5c1eae4d705dabf17 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 7 Sep 2022 14:55:36 +0200 Subject: [PATCH] QA: refactor, cleanup, fix, and simplify (#938) While working on https://github.com/ooni/probe/issues/2130, and specifically on the action item related to making sure all workflows are green, I was confronted with the complexity of the QA directory. There's plenty of cleaning up and simplifying there. The original intent was to A/B test `miniooni` and `measurement_kit` to ensure they were behaving the same. We don't have this need anymore. Rather, it seems the QA scripts have grown large and flaky, to the point that I am always tempted to ignore them. The underlying censorship engine, jafar, has also not been developed for quite some time. So, the first step towards improve the QA infrastructure seems to be humble and acknowledge that we cannot realistically maintain these checks using jafar as a backend for so many experiments. Let us focus on our most important experiment, Web Connectivity, and let us keep QA checks for it. Additionally, let us simplify and cleanup QA as much as possible, though without introducing radical changes. The end result is a QA for Web Connectivity that seems reasonable and runs in six minutes. --- .../{qawebconnectivity.yml => qa.yml} | 8 +- .github/workflows/qafbmessenger.yml | 14 - .github/workflows/qahhfm.yml | 14 - .github/workflows/qahirl.yml | 14 - .github/workflows/qatelegram.yml | 14 - .github/workflows/qawhatsapp.yml | 14 - .vscode/settings.json | 3 +- QA/.gitignore | 1 + QA/Dockerfile | 2 - QA/README.md | 36 +-- QA/common.py | 4 - QA/dockermain.sh | 21 ++ QA/fbmessenger.py | 286 ------------------ QA/hhfm.py | 66 ---- QA/hirl.py | 67 ---- QA/probeasn.py | 88 ------ QA/pyrun.sh | 6 - QA/rundocker.bash | 17 +- QA/telegram.py | 219 -------------- QA/whatsapp.py | 235 -------------- 20 files changed, 46 insertions(+), 1083 deletions(-) rename .github/workflows/{qawebconnectivity.yml => qa.yml} (69%) delete mode 100644 .github/workflows/qafbmessenger.yml delete mode 100644 .github/workflows/qahhfm.yml delete mode 100644 .github/workflows/qahirl.yml delete mode 100644 .github/workflows/qatelegram.yml delete mode 100644 .github/workflows/qawhatsapp.yml delete mode 100644 QA/Dockerfile create mode 100755 QA/dockermain.sh delete mode 100755 QA/fbmessenger.py delete mode 100755 QA/hhfm.py delete mode 100755 QA/hirl.py delete mode 100755 QA/probeasn.py delete mode 100755 QA/pyrun.sh delete mode 100755 QA/telegram.py delete mode 100755 QA/whatsapp.py diff --git a/.github/workflows/qawebconnectivity.yml b/.github/workflows/qa.yml similarity index 69% rename from .github/workflows/qawebconnectivity.yml rename to .github/workflows/qa.yml index 163cb6f..6f8c1f2 100644 --- a/.github/workflows/qawebconnectivity.yml +++ b/.github/workflows/qa.yml @@ -1,14 +1,16 @@ -# Runs QA checks for the webconnectivity experiment -name: "qawebconnectivity" +# Runs quality assurance checks +name: "qa" on: push: branches: - "release/**" - "fullbuild" + - "qabuild" jobs: - test: + test_webconnectivity: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - run: ./QA/rundocker.bash "webconnectivity" + diff --git a/.github/workflows/qafbmessenger.yml b/.github/workflows/qafbmessenger.yml deleted file mode 100644 index df6efce..0000000 --- a/.github/workflows/qafbmessenger.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the fbmessenger experiment -name: "qafbmessenger" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "fbmessenger" diff --git a/.github/workflows/qahhfm.yml b/.github/workflows/qahhfm.yml deleted file mode 100644 index d9a7642..0000000 --- a/.github/workflows/qahhfm.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the hhfm experiment -name: "qahhfm" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "hhfm" diff --git a/.github/workflows/qahirl.yml b/.github/workflows/qahirl.yml deleted file mode 100644 index 284d790..0000000 --- a/.github/workflows/qahirl.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the hirl experiment -name: "qahirl" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "hirl" diff --git a/.github/workflows/qatelegram.yml b/.github/workflows/qatelegram.yml deleted file mode 100644 index 627b7a2..0000000 --- a/.github/workflows/qatelegram.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the telegram experiment -name: "qatelegram" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "telegram" diff --git a/.github/workflows/qawhatsapp.yml b/.github/workflows/qawhatsapp.yml deleted file mode 100644 index fa667be..0000000 --- a/.github/workflows/qawhatsapp.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the whatsapp experiment -name: "qawhatsapp" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "whatsapp" diff --git a/.vscode/settings.json b/.vscode/settings.json index c57221a..f4c2c50 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,8 @@ "python.formatting.provider": "black", "gopls": { "build.directoryFilters": [ - "-GOCACHE" + "-GOCACHE", + "-GOPATH" ] } } diff --git a/QA/.gitignore b/QA/.gitignore index 4d78627..50dab8e 100644 --- a/QA/.gitignore +++ b/QA/.gitignore @@ -1,3 +1,4 @@ +/Dockerfile /GOPATH /GOCACHE /__pycache__ diff --git a/QA/Dockerfile b/QA/Dockerfile deleted file mode 100644 index ebba07d..0000000 --- a/QA/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM golang:1.17-alpine -RUN apk add go git musl-dev iptables tmux bind-tools curl sudo python3 diff --git a/QA/README.md b/QA/README.md index 3b4807f..3422bff 100644 --- a/QA/README.md +++ b/QA/README.md @@ -1,34 +1,7 @@ # Quality Assurance scripts This directory contains quality assurance scripts that use Jafar to -ensure that OONI implementations behave. These scripts take on the -command line as argument the path to a binary with a OONI Probe v2.x -like command line interface. We do not care about full compatibility -but rather about having enough similar flags that running these tools -in parallel is not too much of a burden for us. - -Tools with this shallow-compatible CLI are: - -1. `github.com/ooni/probe-legacy` -2. `github.com/measurement-kit/measurement-kit/src/measurement_kit` -3. `github.com/ooni/probe-engine/internal/cmd/miniooni` - -## Run QA on a Linux system - -These scripts assume you're on a Linux system with `iptables`, `bash`, -`python3`, and possibly a bunch of other tools installed. - -To start the QA script, run this command: - -```bash -sudo ./QA/$nettest.py $ooni_exe -``` - -where `$nettest` is the nettest name (e.g. `telegram`) and `$ooni_exe` -is the OONI Probe v2.x compatible binary to test. - -The Python script needs to run as root. Note however that sudo will also -be used to run `$ooni_exe` with the privileges of the `nobody` user. +ensure that OONI implementations behave. These scripts work with miniooni. ## Run QA using a docker container @@ -38,17 +11,14 @@ Run test in a suitable Docker container using: ./QA/rundocker.sh $nettest ``` -Note that this will run a `--privileged` docker container. This will -eventually run the Python script you would run on Linux. - -For now, the docker scripts only perform QA of `miniooni`. +Note that this will run a `--privileged` docker container. ## Diagnosing issues The Python script that performs the QA runs a specific OONI test under different failure conditions and stops at the first unexpected value found in the resulting JSONL report. You can infer what went wrong by reading -the output of the `$ooni_exe` command itself, which should be above the point +the output of the `miniooni` command itself, which should be above the point where the Python script stopped, as well as by inspecting the JSONL file on disk. By convention such file is named `$nettest.jsonl` and only contains the result of the last run of `$nettest`. diff --git a/QA/common.py b/QA/common.py index 0c64959..656cb00 100644 --- a/QA/common.py +++ b/QA/common.py @@ -3,13 +3,9 @@ import contextlib import json import os -import shlex import shutil import socket import subprocess -import sys -import time -import urllib.parse def execute(args): diff --git a/QA/dockermain.sh b/QA/dockermain.sh new file mode 100755 index 0000000..fab293d --- /dev/null +++ b/QA/dockermain.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -euxo pipefail + +# required because the container is running as root +git config --global --add safe.directory /jafar + +# TODO(bassosimone): investigate why using CGO_ENABLED=1 is such +# that all DNS lookups return `dns_nxdomain_error` +export CGO_ENABLED=0 + +# TODO(bassosimone): because this script runs as root, it's not +# possible to save the caching directories in github actions but +# doing that would making re-executing these scripts faster. +export GOPATH=/jafar/QA/GOPATH +export GOCACHE=/jafar/QA/GOCACHE + +go build -v ./internal/cmd/miniooni + +go build -v ./internal/cmd/jafar + +sudo ./QA/$1.py ./miniooni diff --git a/QA/fbmessenger.py b/QA/fbmessenger.py deleted file mode 100755 index 33162bb..0000000 --- a/QA/fbmessenger.py +++ /dev/null @@ -1,286 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/fbmessenger.py - main QA script for fbmessenger - - This script performs a bunch of fbmessenger tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -services = { - "stun": "stun.fbsbx.com", - "b_api": "b-api.facebook.com", - "b_graph": "b-graph.facebook.com", - "edge": "edge-mqtt.facebook.com", - "external_cdn": "external.xx.fbcdn.net", - "scontent_cdn": "scontent.xx.fbcdn.net", - "star": "star.c10r.facebook.com", -} - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni( - ooni_exe, outfile, "facebook_messenger", tag, args - ) - assert tk["requests"] is None - if tk["tcp_connect"] is not None: - assert isinstance(tk["tcp_connect"], list) - assert len(tk["tcp_connect"]) > 0 - for entry in tk["tcp_connect"]: - assert isinstance(entry, dict) - assert isinstance(entry["ip"], str) - assert isinstance(entry["port"], int) - assert isinstance(entry["status"], dict) - failure = entry["status"]["failure"] - success = entry["status"]["success"] - assert isinstance(failure, str) or failure is None - assert isinstance(success, bool) - return tk - - -def helper_for_blocking_services_via_dns(service): - """ Helper for hijacking a service via dns """ - args = [] - args.append("-iptables-hijack-dns-to") - args.append("127.0.0.1:53") - args.append("-dns-proxy-block") - args.append(service) - return args - - -def helper_for_hijacking_services_via_dns(service): - """ Helper for hijacking a service via dns """ - args = [] - args.append("-iptables-hijack-dns-to") - args.append("127.0.0.1:53") - args.append("-dns-proxy-hijack") - args.append(service) - return args - - -def helper_for_blocking_services_via_tcp(service): - """ Helper for blocking a service via tcp """ - args = [] - args.append("-iptables-reset-ip") - args.append(service) - return args - - -def fbmessenger_dns_hijacked_for_all(ooni_exe, outfile): - """ Test case where everything we measure is DNS hijacked """ - args = [] - for _, value in services.items(): - args.extend(helper_for_hijacking_services_via_dns(value)) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_dns_hijacked_for_all", args, - ) - assert tk["facebook_b_api_dns_consistent"] == False - assert tk["facebook_b_api_reachable"] == None - assert tk["facebook_b_graph_dns_consistent"] == False - assert tk["facebook_b_graph_reachable"] == None - assert tk["facebook_edge_dns_consistent"] == False - assert tk["facebook_edge_reachable"] == None - assert tk["facebook_external_cdn_dns_consistent"] == False - assert tk["facebook_external_cdn_reachable"] == None - assert tk["facebook_scontent_cdn_dns_consistent"] == False - assert tk["facebook_scontent_cdn_reachable"] == None - assert tk["facebook_star_dns_consistent"] == False - assert tk["facebook_star_reachable"] == None - assert tk["facebook_stun_dns_consistent"] == False - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == False - - -def fbmessenger_dns_hijacked_for_some(ooni_exe, outfile): - """ Test case where some endpoints are DNS hijacked """ - args = [] - args.extend(helper_for_hijacking_services_via_dns(services["star"])) - args.extend(helper_for_hijacking_services_via_dns(services["edge"])) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_dns_hijacked_for_some", args, - ) - assert tk["facebook_b_api_dns_consistent"] == True - assert tk["facebook_b_api_reachable"] == True - assert tk["facebook_b_graph_dns_consistent"] == True - assert tk["facebook_b_graph_reachable"] == True - assert tk["facebook_edge_dns_consistent"] == False - assert tk["facebook_edge_reachable"] == None - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == True - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == True - assert tk["facebook_star_dns_consistent"] == False - assert tk["facebook_star_reachable"] == None - assert tk["facebook_stun_dns_consistent"] == True - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == False - - -def fbmessenger_dns_blocked_for_all(ooni_exe, outfile): - """ Test case where everything we measure is DNS blocked """ - args = [] - for _, value in services.items(): - args.extend(helper_for_blocking_services_via_dns(value)) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_dns_blocked_for_all", args, - ) - assert tk["facebook_b_api_dns_consistent"] == False - assert tk["facebook_b_api_reachable"] == None - assert tk["facebook_b_graph_dns_consistent"] == False - assert tk["facebook_b_graph_reachable"] == None - assert tk["facebook_edge_dns_consistent"] == False - assert tk["facebook_edge_reachable"] == None - assert tk["facebook_external_cdn_dns_consistent"] == False - assert tk["facebook_external_cdn_reachable"] == None - assert tk["facebook_scontent_cdn_dns_consistent"] == False - assert tk["facebook_scontent_cdn_reachable"] == None - assert tk["facebook_star_dns_consistent"] == False - assert tk["facebook_star_reachable"] == None - assert tk["facebook_stun_dns_consistent"] == False - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == False - - -def fbmessenger_dns_blocked_for_some(ooni_exe, outfile): - """ Test case where some endpoints are DNS blocked """ - args = [] - args.extend(helper_for_blocking_services_via_dns(services["b_graph"])) - args.extend(helper_for_blocking_services_via_dns(services["stun"])) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_dns_blocked_for_some", args, - ) - assert tk["facebook_b_api_dns_consistent"] == True - assert tk["facebook_b_api_reachable"] == True - assert tk["facebook_b_graph_dns_consistent"] == False - assert tk["facebook_b_graph_reachable"] == None - assert tk["facebook_edge_dns_consistent"] == True - assert tk["facebook_edge_reachable"] == True - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == True - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == True - assert tk["facebook_star_dns_consistent"] == True - assert tk["facebook_star_reachable"] == True - assert tk["facebook_stun_dns_consistent"] == False - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == False - - -def fbmessenger_tcp_blocked_for_all(ooni_exe, outfile): - """ Test case where everything we measure is TCP blocked """ - args = [] - for _, value in services.items(): - args.extend(helper_for_blocking_services_via_tcp(value)) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_tcp_blocked_for_all", args, - ) - assert tk["facebook_b_api_dns_consistent"] == True - assert tk["facebook_b_api_reachable"] == False - assert tk["facebook_b_graph_dns_consistent"] == True - assert tk["facebook_b_graph_reachable"] == False - assert tk["facebook_edge_dns_consistent"] == True - assert tk["facebook_edge_reachable"] == False - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == False - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == False - assert tk["facebook_star_dns_consistent"] == True - assert tk["facebook_star_reachable"] == False - assert tk["facebook_stun_dns_consistent"] == True - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == False - assert tk["facebook_tcp_blocking"] == True - - -def fbmessenger_tcp_blocked_for_some(ooni_exe, outfile): - """ Test case where only some endpoints are TCP blocked """ - args = [] - args.extend(helper_for_blocking_services_via_tcp(services["edge"])) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_tcp_blocked_for_some", args, - ) - assert tk["facebook_b_api_dns_consistent"] == True - assert tk["facebook_b_api_reachable"] == True - assert tk["facebook_b_graph_dns_consistent"] == True - assert tk["facebook_b_graph_reachable"] == True - assert tk["facebook_edge_dns_consistent"] == True - assert tk["facebook_edge_reachable"] == False - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == True - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == True - assert tk["facebook_star_dns_consistent"] == True - assert tk["facebook_star_reachable"] == True - assert tk["facebook_stun_dns_consistent"] == True - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == False - assert tk["facebook_tcp_blocking"] == True - - -def fbmessenger_mixed_results(ooni_exe, outfile): - """ Test case where only some endpoints are TCP blocked """ - args = [] - args.extend(helper_for_blocking_services_via_tcp(services["edge"])) - args.extend(helper_for_blocking_services_via_dns(services["b_api"])) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_tcp_blocked_for_some", args, - ) - assert tk["facebook_b_api_dns_consistent"] == False - assert tk["facebook_b_api_reachable"] == None - assert tk["facebook_b_graph_dns_consistent"] == True - assert tk["facebook_b_graph_reachable"] == True - assert tk["facebook_edge_dns_consistent"] == True - assert tk["facebook_edge_reachable"] == False - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == True - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == True - assert tk["facebook_star_dns_consistent"] == True - assert tk["facebook_star_reachable"] == True - assert tk["facebook_stun_dns_consistent"] == True - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == True - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "fbmessenger.jsonl" - ooni_exe = sys.argv[1] - tests = [ - fbmessenger_dns_hijacked_for_all, - fbmessenger_dns_hijacked_for_some, - fbmessenger_dns_blocked_for_all, - fbmessenger_dns_blocked_for_some, - fbmessenger_tcp_blocked_for_all, - fbmessenger_tcp_blocked_for_some, - fbmessenger_mixed_results, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/QA/hhfm.py b/QA/hhfm.py deleted file mode 100755 index 75047f5..0000000 --- a/QA/hhfm.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/hhfm.py - main QA script for hhfm - - This script performs a bunch of hhfm tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni( - ooni_exe, outfile, "http_header_field_manipulation", tag, args - ) - # TODO(bassosimone): what checks to put here? - return tk - - -def hhfm_transparent_proxy(ooni_exe, outfile): - """ Test case where we're passing through a transparent proxy """ - args = ["-iptables-hijack-http-to", "127.0.0.1:80"] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "hhfm_transparent_proxy", args, - ) - # The proxy sees a domain that does not make any sense and does not - # otherwise know where to connect to. Hence the most likely result is - # a `dns_nxdomain_error` with total tampering. - assert tk["tampering"]["header_field_name"] == False - assert tk["tampering"]["header_field_number"] == False - assert tk["tampering"]["header_field_value"] == False - assert tk["tampering"]["header_name_capitalization"] == False - assert tk["tampering"]["header_name_diff"] == [] - assert tk["tampering"]["request_line_capitalization"] == False - assert tk["tampering"]["total"] == True - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "hhfm.jsonl" - ooni_exe = sys.argv[1] - tests = [ - hhfm_transparent_proxy, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/QA/hirl.py b/QA/hirl.py deleted file mode 100755 index 5257754..0000000 --- a/QA/hirl.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/hirl.py - main QA script for hirl - - This script performs a bunch of hirl tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni( - ooni_exe, outfile, "http_invalid_request_line", tag, args - ) - # TODO(bassosimone): what checks to put here? - return tk - - -def hirl_transparent_proxy(ooni_exe, outfile): - """ Test case where we're passing through a transparent proxy """ - args = ["-iptables-hijack-http-to", "127.0.0.1:80"] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "hirl_transparent_proxy", args, - ) - count = 0 - for entry in tk["failure_list"]: - if entry is None: - count += 1 - elif entry == "eof_error": - count += 1e03 - else: - count += 1e06 - assert count == 3002 - assert tk["tampering_list"] == [True, True, True, True, True] - assert tk["tampering"] == True - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "hirl.jsonl" - ooni_exe = sys.argv[1] - tests = [ - hirl_transparent_proxy, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/QA/probeasn.py b/QA/probeasn.py deleted file mode 100755 index f86438d..0000000 --- a/QA/probeasn.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/probeasn.py - QA script for the -g miniooni option. """ - -import contextlib -import json -import os -import shlex -import shutil -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -def execute_miniooni(ooni_exe, outfile, arguments): - """ Executes miniooni and returns the whole measurement. """ - if "miniooni" not in ooni_exe: - return None - tmpoutfile = "/tmp/{}".format(outfile) - with contextlib.suppress(FileNotFoundError): - os.remove(tmpoutfile) # just in case - cmdline = [ - ooni_exe, - arguments, - "-o", - tmpoutfile, - "--home", - "/tmp", - "example", - ] - print("exec: {}".format(cmdline)) - common.execute(cmdline) - shutil.copy(tmpoutfile, outfile) - result = common.read_result(outfile) - assert isinstance(result, dict) - assert isinstance(result["test_keys"], dict) - return result - - -def probeasn_without_g_option(ooni_exe, outfile): - """ Test case where we're not passing to miniooni the -g option """ - m = execute_miniooni(ooni_exe, outfile, "-n") - if m is None: - return - assert m["probe_cc"] != "ZZ" - assert m["probe_ip"] == "127.0.0.1" - assert m["probe_asn"] != "AS0" - assert m["probe_network_name"] != "" - assert m["resolver_ip"] == "127.0.0.2" - assert m["resolver_asn"] != "AS0" - assert m["resolver_network_name"] != "" - - -def probeasn_with_g_option(ooni_exe, outfile): - """ Test case where we're passing the -g option """ - m = execute_miniooni(ooni_exe, outfile, "-gn") - if m is None: - return - assert m["probe_cc"] != "ZZ" - assert m["probe_ip"] == "127.0.0.1" - assert m["probe_asn"] == "AS0" - assert m["probe_network_name"] == "" - assert m["resolver_ip"] == "127.0.0.2" - assert m["resolver_asn"] == "AS0" - assert m["resolver_network_name"] == "" - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "probeasn.jsonl" - ooni_exe = sys.argv[1] - tests = [ - probeasn_with_g_option, - probeasn_without_g_option, - ] - for test in tests: - test(ooni_exe, outfile) - - -if __name__ == "__main__": - main() diff --git a/QA/pyrun.sh b/QA/pyrun.sh deleted file mode 100755 index 76f3f28..0000000 --- a/QA/pyrun.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -set -ex -export GOPATH=/jafar/QA/GOPATH GOCACHE=/jafar/QA/GOCACHE GO111MODULE=on -go build -v ./internal/cmd/miniooni -go build -v ./internal/cmd/jafar -sudo ./QA/$1.py ./miniooni diff --git a/QA/rundocker.bash b/QA/rundocker.bash index 8ce7320..77ec540 100755 --- a/QA/rundocker.bash +++ b/QA/rundocker.bash @@ -1,5 +1,16 @@ -#!/bin/sh -set -ex +#!/bin/bash + +set -euxo pipefail + DOCKER=${DOCKER:-docker} + +GOVERSION=$(cat GOVERSION) + +cat > QA/Dockerfile << EOF +FROM golang:$GOVERSION-alpine +RUN apk add gcc go git musl-dev iptables tmux bind-tools curl sudo python3 +EOF + $DOCKER build -t jafar-qa ./QA/ -$DOCKER run --privileged -v`pwd`:/jafar -w/jafar jafar-qa ./QA/pyrun.sh "$@" + +$DOCKER run --privileged -v$(pwd):/jafar -w/jafar jafar-qa ./QA/dockermain.sh "$@" diff --git a/QA/telegram.py b/QA/telegram.py deleted file mode 100755 index 1facdf2..0000000 --- a/QA/telegram.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/telegram.py - main QA script for telegram - - This script performs a bunch of telegram tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -ALL_POP_IPS = ( - "149.154.175.50", - "149.154.167.51", - "149.154.175.100", - "149.154.167.91", - "149.154.171.5", - "95.161.76.100", -) - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni(ooni_exe, outfile, "telegram", tag, args) - assert isinstance(tk["requests"], list) - assert len(tk["requests"]) > 0 - for entry in tk["requests"]: - assert isinstance(entry, dict) - failure = entry["failure"] - assert isinstance(failure, str) or failure is None - assert isinstance(entry["request"], dict) - req = entry["request"] - common.check_maybe_binary_value(req["body"]) - assert isinstance(req["headers"], dict) - for key, value in req["headers"].items(): - assert isinstance(key, str) - common.check_maybe_binary_value(value) - assert isinstance(req["method"], str) - assert isinstance(entry["response"], dict) - resp = entry["response"] - common.check_maybe_binary_value(resp["body"]) - assert isinstance(resp["code"], int) - if resp["headers"] is not None: - for key, value in resp["headers"].items(): - assert isinstance(key, str) - common.check_maybe_binary_value(value) - assert isinstance(tk["tcp_connect"], list) - assert len(tk["tcp_connect"]) > 0 - for entry in tk["tcp_connect"]: - assert isinstance(entry, dict) - assert isinstance(entry["ip"], str) - assert isinstance(entry["port"], int) - assert isinstance(entry["status"], dict) - failure = entry["status"]["failure"] - success = entry["status"]["success"] - assert isinstance(failure, str) or failure is None - assert isinstance(success, bool) - return tk - - -def args_for_blocking_all_pop_ips(): - """ Returns the arguments useful for blocking all POPs IPs """ - args = [] - for ip in ALL_POP_IPS: - args.append("-iptables-reset-ip") - args.append(ip) - return args - - -def args_for_blocking_web_telegram_org_http(): - """ Returns arguments for blocking web.telegram.org over http """ - return ["-iptables-reset-keyword", "Host: web.telegram.org"] - - -def args_for_blocking_web_telegram_org_https(): - """ Returns arguments for blocking web.telegram.org over https """ - # - # 00 00 - # 00 15 - # 00 13 - # 00 - # 00 10 - # 77 65 ... 67 web.telegram.org - # - return [ - "-iptables-reset-keyword-hex", - "|00 00 00 15 00 13 00 00 10 77 65 62 2e 74 65 6c 65 67 72 61 6d 2e 6f 72 67|", - ] - - -def telegram_block_everything(ooni_exe, outfile): - """ Test case where everything we measure is blocked """ - args = [] - args.extend(args_for_blocking_all_pop_ips()) - args.extend(args_for_blocking_web_telegram_org_https()) - args.extend(args_for_blocking_web_telegram_org_http()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_block_everything", args, - ) - assert tk["telegram_tcp_blocking"] == True - assert tk["telegram_http_blocking"] == True - assert tk["telegram_web_failure"] == "connection_reset" - assert tk["telegram_web_status"] == "blocked" - - -def telegram_tcp_blocking_all(ooni_exe, outfile): - """ Test case where all POPs are TCP/IP blocked """ - args = args_for_blocking_all_pop_ips() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_tcp_blocking_all", args - ) - assert tk["telegram_tcp_blocking"] == True - assert tk["telegram_http_blocking"] == True - assert tk["telegram_web_failure"] == None - assert tk["telegram_web_status"] == "ok" - - -def telegram_tcp_blocking_some(ooni_exe, outfile): - """ Test case where some POPs are TCP/IP blocked """ - args = [ - "-iptables-reset-ip", - ALL_POP_IPS[0], - ] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_tcp_blocking_some", args - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == False - assert tk["telegram_web_failure"] == None - assert tk["telegram_web_status"] == "ok" - - -def telegram_http_blocking_all(ooni_exe, outfile): - """ Test case where all POPs are HTTP blocked """ - args = [] - for ip in ALL_POP_IPS: - args.append("-iptables-reset-keyword") - args.append(ip) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_http_blocking_all", args, - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == True - assert tk["telegram_web_failure"] == None - assert tk["telegram_web_status"] == "ok" - - -def telegram_http_blocking_some(ooni_exe, outfile): - """ Test case where some POPs are HTTP blocked """ - args = [ - "-iptables-reset-keyword", - ALL_POP_IPS[0], - ] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_http_blocking_some", args, - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == False - assert tk["telegram_web_failure"] == None - assert tk["telegram_web_status"] == "ok" - - -def telegram_web_failure_http(ooni_exe, outfile): - """ Test case where the web HTTP endpoint is blocked """ - args = args_for_blocking_web_telegram_org_http() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_web_failure_http", args, - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == False - assert tk["telegram_web_failure"] == "connection_reset" - assert tk["telegram_web_status"] == "blocked" - - -def telegram_web_failure_https(ooni_exe, outfile): - """ Test case where the web HTTPS endpoint is blocked """ - args = args_for_blocking_web_telegram_org_https() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_web_failure_https", args, - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == False - assert tk["telegram_web_failure"] == "connection_reset" - assert tk["telegram_web_status"] == "blocked" - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "telegram.jsonl" - ooni_exe = sys.argv[1] - tests = [ - telegram_block_everything, - telegram_tcp_blocking_all, - telegram_tcp_blocking_some, - telegram_http_blocking_all, - telegram_http_blocking_some, - telegram_web_failure_http, - telegram_web_failure_https, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/QA/whatsapp.py b/QA/whatsapp.py deleted file mode 100755 index 35b2dd7..0000000 --- a/QA/whatsapp.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/whatsapp.py - main QA script for whatsapp - - This script performs a bunch of whatsapp tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni(ooni_exe, outfile, "whatsapp", tag, args) - assert isinstance(tk["requests"], list) - assert len(tk["requests"]) > 0 - for entry in tk["requests"]: - assert isinstance(entry, dict) - failure = entry["failure"] - assert isinstance(failure, str) or failure is None - assert isinstance(entry["request"], dict) - req = entry["request"] - common.check_maybe_binary_value(req["body"]) - assert isinstance(req["headers"], dict) - for key, value in req["headers"].items(): - assert isinstance(key, str) - common.check_maybe_binary_value(value) - assert isinstance(req["method"], str) - assert isinstance(entry["response"], dict) - resp = entry["response"] - common.check_maybe_binary_value(resp["body"]) - assert isinstance(resp["code"], int) - if resp["headers"] is not None: - for key, value in resp["headers"].items(): - assert isinstance(key, str) - common.check_maybe_binary_value(value) - assert isinstance(tk["tcp_connect"], list) - assert len(tk["tcp_connect"]) > 0 - for entry in tk["tcp_connect"]: - assert isinstance(entry, dict) - assert isinstance(entry["ip"], str) - assert isinstance(entry["port"], int) - assert isinstance(entry["status"], dict) - failure = entry["status"]["failure"] - success = entry["status"]["success"] - assert isinstance(failure, str) or failure is None - assert isinstance(success, bool) - return tk - - -def helper_for_blocking_endpoints(start, stop): - """ Helper function for generating args for blocking endpoints """ - args = [] - for num in range(start, stop): - args.append("-iptables-reset-ip") - args.append("e{}.whatsapp.net".format(num)) - return args - - -def args_for_blocking_all_endpoints(): - """ Returns the arguments useful for blocking all endpoints """ - return helper_for_blocking_endpoints(1, 17) - - -def args_for_blocking_some_endpoints(): - """ Returns the arguments useful for blocking some endpoints """ - # Implementation note: apparently all the endpoints are now using just - # four IP addresses, hence here we block some endpoints via DNS. - # - # TODO(bassosimone): this fact calls for creating an issue for making - # the whatsapp experiment implementation more efficient. - args = [] - args.append("-iptables-hijack-dns-to") - args.append("127.0.0.1:53") - for n in range(1, 7): - args.append("-dns-proxy-block") - args.append("e{}.whatsapp.net".format(n)) - return args - - -def args_for_blocking_v_whatsapp_net_https(): - """ Returns arguments for blocking v.whatsapp.net over https """ - # - # 00 00 - # 00 13 - # 00 11 - # 00 - # 00 0e - # 76 2e ... 74 v.whatsapp.net - # - return [ - "-iptables-reset-keyword-hex", - "|00 00 00 13 00 11 00 00 0e 76 2e 77 68 61 74 73 61 70 70 2e 6e 65 74|", - ] - - -def args_for_blocking_web_whatsapp_com_http(): - """ Returns arguments for blocking web.whatsapp.com over http """ - return ["-iptables-reset-keyword", "Host: web.whatsapp.com"] - - -def args_for_blocking_web_whatsapp_com_https(): - """ Returns arguments for blocking web.whatsapp.com over https """ - # - # 00 00 - # 00 15 - # 00 13 - # 00 - # 00 10 - # 77 65 ... 6d web.whatsapp.com - # - return [ - "-iptables-reset-keyword-hex", - "|00 00 00 15 00 13 00 00 10 77 65 62 2e 77 68 61 74 73 61 70 70 2e 63 6f 6d|", - ] - - -def whatsapp_block_everything(ooni_exe, outfile): - """ Test case where everything we measure is blocked """ - args = [] - args.extend(args_for_blocking_all_endpoints()) - args.extend(args_for_blocking_v_whatsapp_net_https()) - args.extend(args_for_blocking_web_whatsapp_com_https()) - args.extend(args_for_blocking_web_whatsapp_com_http()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_everything", args, - ) - assert tk["registration_server_failure"] == "connection_reset" - assert tk["registration_server_status"] == "blocked" - assert tk["whatsapp_endpoints_status"] == "blocked" - assert tk["whatsapp_web_failure"] == "connection_reset" - assert tk["whatsapp_web_status"] == "blocked" - - -def whatsapp_block_all_endpoints(ooni_exe, outfile): - """ Test case where we only block whatsapp endpoints """ - args = args_for_blocking_all_endpoints() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_all_endpoints", args - ) - assert tk["registration_server_failure"] == None - assert tk["registration_server_status"] == "ok" - assert tk["whatsapp_endpoints_status"] == "blocked" - assert tk["whatsapp_web_failure"] == None - assert tk["whatsapp_web_status"] == "ok" - - -def whatsapp_block_some_endpoints(ooni_exe, outfile): - """ Test case where we block some whatsapp endpoints """ - args = args_for_blocking_some_endpoints() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_some_endpoints", args - ) - assert tk["registration_server_failure"] == None - assert tk["registration_server_status"] == "ok" - assert tk["whatsapp_endpoints_status"] == "ok" - assert tk["whatsapp_web_failure"] == None - assert tk["whatsapp_web_status"] == "ok" - - -def whatsapp_block_registration_server(ooni_exe, outfile): - """ Test case where we block the registration server """ - args = [] - args.extend(args_for_blocking_v_whatsapp_net_https()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_registration_server", args, - ) - assert tk["registration_server_failure"] == "connection_reset" - assert tk["registration_server_status"] == "blocked" - assert tk["whatsapp_endpoints_status"] == "ok" - assert tk["whatsapp_web_failure"] == None - assert tk["whatsapp_web_status"] == "ok" - - -def whatsapp_block_web_http(ooni_exe, outfile): - """ Test case where we block the HTTP web chat """ - args = [] - args.extend(args_for_blocking_web_whatsapp_com_http()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_web_http", args, - ) - assert tk["registration_server_failure"] == None - assert tk["registration_server_status"] == "ok" - assert tk["whatsapp_endpoints_status"] == "ok" - assert tk["whatsapp_web_failure"] == "connection_reset" - assert tk["whatsapp_web_status"] == "blocked" - - -def whatsapp_block_web_https(ooni_exe, outfile): - """ Test case where we block the HTTPS web chat """ - args = [] - args.extend(args_for_blocking_web_whatsapp_com_https()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_web_https", args, - ) - assert tk["registration_server_failure"] == None - assert tk["registration_server_status"] == "ok" - assert tk["whatsapp_endpoints_status"] == "ok" - assert tk["whatsapp_web_failure"] == "connection_reset" - assert tk["whatsapp_web_status"] == "blocked" - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "whatsapp.jsonl" - ooni_exe = sys.argv[1] - tests = [ - whatsapp_block_everything, - whatsapp_block_all_endpoints, - whatsapp_block_some_endpoints, - whatsapp_block_registration_server, - whatsapp_block_web_http, - whatsapp_block_web_https, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main()