diff --git a/.github/workflows/qawebconnectivity.yml b/.github/workflows/qa.yml similarity index 69% rename from .github/workflows/qawebconnectivity.yml rename to .github/workflows/qa.yml index 163cb6f..6f8c1f2 100644 --- a/.github/workflows/qawebconnectivity.yml +++ b/.github/workflows/qa.yml @@ -1,14 +1,16 @@ -# Runs QA checks for the webconnectivity experiment -name: "qawebconnectivity" +# Runs quality assurance checks +name: "qa" on: push: branches: - "release/**" - "fullbuild" + - "qabuild" jobs: - test: + test_webconnectivity: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - run: ./QA/rundocker.bash "webconnectivity" + diff --git a/.github/workflows/qafbmessenger.yml b/.github/workflows/qafbmessenger.yml deleted file mode 100644 index df6efce..0000000 --- a/.github/workflows/qafbmessenger.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the fbmessenger experiment -name: "qafbmessenger" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "fbmessenger" diff --git a/.github/workflows/qahhfm.yml b/.github/workflows/qahhfm.yml deleted file mode 100644 index d9a7642..0000000 --- a/.github/workflows/qahhfm.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the hhfm experiment -name: "qahhfm" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "hhfm" diff --git a/.github/workflows/qahirl.yml b/.github/workflows/qahirl.yml deleted file mode 100644 index 284d790..0000000 --- a/.github/workflows/qahirl.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the hirl experiment -name: "qahirl" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "hirl" diff --git a/.github/workflows/qatelegram.yml b/.github/workflows/qatelegram.yml deleted file mode 100644 index 627b7a2..0000000 --- a/.github/workflows/qatelegram.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the telegram experiment -name: "qatelegram" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "telegram" diff --git a/.github/workflows/qawhatsapp.yml b/.github/workflows/qawhatsapp.yml deleted file mode 100644 index fa667be..0000000 --- a/.github/workflows/qawhatsapp.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Runs QA checks for the whatsapp experiment -name: "qawhatsapp" -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - run: ./QA/rundocker.bash "whatsapp" diff --git a/.vscode/settings.json b/.vscode/settings.json index c57221a..f4c2c50 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,8 @@ "python.formatting.provider": "black", "gopls": { "build.directoryFilters": [ - "-GOCACHE" + "-GOCACHE", + "-GOPATH" ] } } diff --git a/QA/.gitignore b/QA/.gitignore index 4d78627..50dab8e 100644 --- a/QA/.gitignore +++ b/QA/.gitignore @@ -1,3 +1,4 @@ +/Dockerfile /GOPATH /GOCACHE /__pycache__ diff --git a/QA/Dockerfile b/QA/Dockerfile deleted file mode 100644 index ebba07d..0000000 --- a/QA/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM golang:1.17-alpine -RUN apk add go git musl-dev iptables tmux bind-tools curl sudo python3 diff --git a/QA/README.md b/QA/README.md index 3b4807f..3422bff 100644 --- a/QA/README.md +++ b/QA/README.md @@ -1,34 +1,7 @@ # Quality Assurance scripts This directory contains quality assurance scripts that use Jafar to -ensure that OONI implementations behave. These scripts take on the -command line as argument the path to a binary with a OONI Probe v2.x -like command line interface. We do not care about full compatibility -but rather about having enough similar flags that running these tools -in parallel is not too much of a burden for us. - -Tools with this shallow-compatible CLI are: - -1. `github.com/ooni/probe-legacy` -2. `github.com/measurement-kit/measurement-kit/src/measurement_kit` -3. `github.com/ooni/probe-engine/internal/cmd/miniooni` - -## Run QA on a Linux system - -These scripts assume you're on a Linux system with `iptables`, `bash`, -`python3`, and possibly a bunch of other tools installed. - -To start the QA script, run this command: - -```bash -sudo ./QA/$nettest.py $ooni_exe -``` - -where `$nettest` is the nettest name (e.g. `telegram`) and `$ooni_exe` -is the OONI Probe v2.x compatible binary to test. - -The Python script needs to run as root. Note however that sudo will also -be used to run `$ooni_exe` with the privileges of the `nobody` user. +ensure that OONI implementations behave. These scripts work with miniooni. ## Run QA using a docker container @@ -38,17 +11,14 @@ Run test in a suitable Docker container using: ./QA/rundocker.sh $nettest ``` -Note that this will run a `--privileged` docker container. This will -eventually run the Python script you would run on Linux. - -For now, the docker scripts only perform QA of `miniooni`. +Note that this will run a `--privileged` docker container. ## Diagnosing issues The Python script that performs the QA runs a specific OONI test under different failure conditions and stops at the first unexpected value found in the resulting JSONL report. You can infer what went wrong by reading -the output of the `$ooni_exe` command itself, which should be above the point +the output of the `miniooni` command itself, which should be above the point where the Python script stopped, as well as by inspecting the JSONL file on disk. By convention such file is named `$nettest.jsonl` and only contains the result of the last run of `$nettest`. diff --git a/QA/common.py b/QA/common.py index 0c64959..656cb00 100644 --- a/QA/common.py +++ b/QA/common.py @@ -3,13 +3,9 @@ import contextlib import json import os -import shlex import shutil import socket import subprocess -import sys -import time -import urllib.parse def execute(args): diff --git a/QA/dockermain.sh b/QA/dockermain.sh new file mode 100755 index 0000000..fab293d --- /dev/null +++ b/QA/dockermain.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -euxo pipefail + +# required because the container is running as root +git config --global --add safe.directory /jafar + +# TODO(bassosimone): investigate why using CGO_ENABLED=1 is such +# that all DNS lookups return `dns_nxdomain_error` +export CGO_ENABLED=0 + +# TODO(bassosimone): because this script runs as root, it's not +# possible to save the caching directories in github actions but +# doing that would making re-executing these scripts faster. +export GOPATH=/jafar/QA/GOPATH +export GOCACHE=/jafar/QA/GOCACHE + +go build -v ./internal/cmd/miniooni + +go build -v ./internal/cmd/jafar + +sudo ./QA/$1.py ./miniooni diff --git a/QA/fbmessenger.py b/QA/fbmessenger.py deleted file mode 100755 index 33162bb..0000000 --- a/QA/fbmessenger.py +++ /dev/null @@ -1,286 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/fbmessenger.py - main QA script for fbmessenger - - This script performs a bunch of fbmessenger tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -services = { - "stun": "stun.fbsbx.com", - "b_api": "b-api.facebook.com", - "b_graph": "b-graph.facebook.com", - "edge": "edge-mqtt.facebook.com", - "external_cdn": "external.xx.fbcdn.net", - "scontent_cdn": "scontent.xx.fbcdn.net", - "star": "star.c10r.facebook.com", -} - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni( - ooni_exe, outfile, "facebook_messenger", tag, args - ) - assert tk["requests"] is None - if tk["tcp_connect"] is not None: - assert isinstance(tk["tcp_connect"], list) - assert len(tk["tcp_connect"]) > 0 - for entry in tk["tcp_connect"]: - assert isinstance(entry, dict) - assert isinstance(entry["ip"], str) - assert isinstance(entry["port"], int) - assert isinstance(entry["status"], dict) - failure = entry["status"]["failure"] - success = entry["status"]["success"] - assert isinstance(failure, str) or failure is None - assert isinstance(success, bool) - return tk - - -def helper_for_blocking_services_via_dns(service): - """ Helper for hijacking a service via dns """ - args = [] - args.append("-iptables-hijack-dns-to") - args.append("127.0.0.1:53") - args.append("-dns-proxy-block") - args.append(service) - return args - - -def helper_for_hijacking_services_via_dns(service): - """ Helper for hijacking a service via dns """ - args = [] - args.append("-iptables-hijack-dns-to") - args.append("127.0.0.1:53") - args.append("-dns-proxy-hijack") - args.append(service) - return args - - -def helper_for_blocking_services_via_tcp(service): - """ Helper for blocking a service via tcp """ - args = [] - args.append("-iptables-reset-ip") - args.append(service) - return args - - -def fbmessenger_dns_hijacked_for_all(ooni_exe, outfile): - """ Test case where everything we measure is DNS hijacked """ - args = [] - for _, value in services.items(): - args.extend(helper_for_hijacking_services_via_dns(value)) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_dns_hijacked_for_all", args, - ) - assert tk["facebook_b_api_dns_consistent"] == False - assert tk["facebook_b_api_reachable"] == None - assert tk["facebook_b_graph_dns_consistent"] == False - assert tk["facebook_b_graph_reachable"] == None - assert tk["facebook_edge_dns_consistent"] == False - assert tk["facebook_edge_reachable"] == None - assert tk["facebook_external_cdn_dns_consistent"] == False - assert tk["facebook_external_cdn_reachable"] == None - assert tk["facebook_scontent_cdn_dns_consistent"] == False - assert tk["facebook_scontent_cdn_reachable"] == None - assert tk["facebook_star_dns_consistent"] == False - assert tk["facebook_star_reachable"] == None - assert tk["facebook_stun_dns_consistent"] == False - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == False - - -def fbmessenger_dns_hijacked_for_some(ooni_exe, outfile): - """ Test case where some endpoints are DNS hijacked """ - args = [] - args.extend(helper_for_hijacking_services_via_dns(services["star"])) - args.extend(helper_for_hijacking_services_via_dns(services["edge"])) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_dns_hijacked_for_some", args, - ) - assert tk["facebook_b_api_dns_consistent"] == True - assert tk["facebook_b_api_reachable"] == True - assert tk["facebook_b_graph_dns_consistent"] == True - assert tk["facebook_b_graph_reachable"] == True - assert tk["facebook_edge_dns_consistent"] == False - assert tk["facebook_edge_reachable"] == None - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == True - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == True - assert tk["facebook_star_dns_consistent"] == False - assert tk["facebook_star_reachable"] == None - assert tk["facebook_stun_dns_consistent"] == True - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == False - - -def fbmessenger_dns_blocked_for_all(ooni_exe, outfile): - """ Test case where everything we measure is DNS blocked """ - args = [] - for _, value in services.items(): - args.extend(helper_for_blocking_services_via_dns(value)) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_dns_blocked_for_all", args, - ) - assert tk["facebook_b_api_dns_consistent"] == False - assert tk["facebook_b_api_reachable"] == None - assert tk["facebook_b_graph_dns_consistent"] == False - assert tk["facebook_b_graph_reachable"] == None - assert tk["facebook_edge_dns_consistent"] == False - assert tk["facebook_edge_reachable"] == None - assert tk["facebook_external_cdn_dns_consistent"] == False - assert tk["facebook_external_cdn_reachable"] == None - assert tk["facebook_scontent_cdn_dns_consistent"] == False - assert tk["facebook_scontent_cdn_reachable"] == None - assert tk["facebook_star_dns_consistent"] == False - assert tk["facebook_star_reachable"] == None - assert tk["facebook_stun_dns_consistent"] == False - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == False - - -def fbmessenger_dns_blocked_for_some(ooni_exe, outfile): - """ Test case where some endpoints are DNS blocked """ - args = [] - args.extend(helper_for_blocking_services_via_dns(services["b_graph"])) - args.extend(helper_for_blocking_services_via_dns(services["stun"])) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_dns_blocked_for_some", args, - ) - assert tk["facebook_b_api_dns_consistent"] == True - assert tk["facebook_b_api_reachable"] == True - assert tk["facebook_b_graph_dns_consistent"] == False - assert tk["facebook_b_graph_reachable"] == None - assert tk["facebook_edge_dns_consistent"] == True - assert tk["facebook_edge_reachable"] == True - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == True - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == True - assert tk["facebook_star_dns_consistent"] == True - assert tk["facebook_star_reachable"] == True - assert tk["facebook_stun_dns_consistent"] == False - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == False - - -def fbmessenger_tcp_blocked_for_all(ooni_exe, outfile): - """ Test case where everything we measure is TCP blocked """ - args = [] - for _, value in services.items(): - args.extend(helper_for_blocking_services_via_tcp(value)) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_tcp_blocked_for_all", args, - ) - assert tk["facebook_b_api_dns_consistent"] == True - assert tk["facebook_b_api_reachable"] == False - assert tk["facebook_b_graph_dns_consistent"] == True - assert tk["facebook_b_graph_reachable"] == False - assert tk["facebook_edge_dns_consistent"] == True - assert tk["facebook_edge_reachable"] == False - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == False - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == False - assert tk["facebook_star_dns_consistent"] == True - assert tk["facebook_star_reachable"] == False - assert tk["facebook_stun_dns_consistent"] == True - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == False - assert tk["facebook_tcp_blocking"] == True - - -def fbmessenger_tcp_blocked_for_some(ooni_exe, outfile): - """ Test case where only some endpoints are TCP blocked """ - args = [] - args.extend(helper_for_blocking_services_via_tcp(services["edge"])) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_tcp_blocked_for_some", args, - ) - assert tk["facebook_b_api_dns_consistent"] == True - assert tk["facebook_b_api_reachable"] == True - assert tk["facebook_b_graph_dns_consistent"] == True - assert tk["facebook_b_graph_reachable"] == True - assert tk["facebook_edge_dns_consistent"] == True - assert tk["facebook_edge_reachable"] == False - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == True - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == True - assert tk["facebook_star_dns_consistent"] == True - assert tk["facebook_star_reachable"] == True - assert tk["facebook_stun_dns_consistent"] == True - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == False - assert tk["facebook_tcp_blocking"] == True - - -def fbmessenger_mixed_results(ooni_exe, outfile): - """ Test case where only some endpoints are TCP blocked """ - args = [] - args.extend(helper_for_blocking_services_via_tcp(services["edge"])) - args.extend(helper_for_blocking_services_via_dns(services["b_api"])) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "fbmessenger_tcp_blocked_for_some", args, - ) - assert tk["facebook_b_api_dns_consistent"] == False - assert tk["facebook_b_api_reachable"] == None - assert tk["facebook_b_graph_dns_consistent"] == True - assert tk["facebook_b_graph_reachable"] == True - assert tk["facebook_edge_dns_consistent"] == True - assert tk["facebook_edge_reachable"] == False - assert tk["facebook_external_cdn_dns_consistent"] == True - assert tk["facebook_external_cdn_reachable"] == True - assert tk["facebook_scontent_cdn_dns_consistent"] == True - assert tk["facebook_scontent_cdn_reachable"] == True - assert tk["facebook_star_dns_consistent"] == True - assert tk["facebook_star_reachable"] == True - assert tk["facebook_stun_dns_consistent"] == True - assert tk["facebook_stun_reachable"] == None - assert tk["facebook_dns_blocking"] == True - assert tk["facebook_tcp_blocking"] == True - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "fbmessenger.jsonl" - ooni_exe = sys.argv[1] - tests = [ - fbmessenger_dns_hijacked_for_all, - fbmessenger_dns_hijacked_for_some, - fbmessenger_dns_blocked_for_all, - fbmessenger_dns_blocked_for_some, - fbmessenger_tcp_blocked_for_all, - fbmessenger_tcp_blocked_for_some, - fbmessenger_mixed_results, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/QA/hhfm.py b/QA/hhfm.py deleted file mode 100755 index 75047f5..0000000 --- a/QA/hhfm.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/hhfm.py - main QA script for hhfm - - This script performs a bunch of hhfm tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni( - ooni_exe, outfile, "http_header_field_manipulation", tag, args - ) - # TODO(bassosimone): what checks to put here? - return tk - - -def hhfm_transparent_proxy(ooni_exe, outfile): - """ Test case where we're passing through a transparent proxy """ - args = ["-iptables-hijack-http-to", "127.0.0.1:80"] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "hhfm_transparent_proxy", args, - ) - # The proxy sees a domain that does not make any sense and does not - # otherwise know where to connect to. Hence the most likely result is - # a `dns_nxdomain_error` with total tampering. - assert tk["tampering"]["header_field_name"] == False - assert tk["tampering"]["header_field_number"] == False - assert tk["tampering"]["header_field_value"] == False - assert tk["tampering"]["header_name_capitalization"] == False - assert tk["tampering"]["header_name_diff"] == [] - assert tk["tampering"]["request_line_capitalization"] == False - assert tk["tampering"]["total"] == True - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "hhfm.jsonl" - ooni_exe = sys.argv[1] - tests = [ - hhfm_transparent_proxy, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/QA/hirl.py b/QA/hirl.py deleted file mode 100755 index 5257754..0000000 --- a/QA/hirl.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/hirl.py - main QA script for hirl - - This script performs a bunch of hirl tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni( - ooni_exe, outfile, "http_invalid_request_line", tag, args - ) - # TODO(bassosimone): what checks to put here? - return tk - - -def hirl_transparent_proxy(ooni_exe, outfile): - """ Test case where we're passing through a transparent proxy """ - args = ["-iptables-hijack-http-to", "127.0.0.1:80"] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "hirl_transparent_proxy", args, - ) - count = 0 - for entry in tk["failure_list"]: - if entry is None: - count += 1 - elif entry == "eof_error": - count += 1e03 - else: - count += 1e06 - assert count == 3002 - assert tk["tampering_list"] == [True, True, True, True, True] - assert tk["tampering"] == True - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "hirl.jsonl" - ooni_exe = sys.argv[1] - tests = [ - hirl_transparent_proxy, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/QA/probeasn.py b/QA/probeasn.py deleted file mode 100755 index f86438d..0000000 --- a/QA/probeasn.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/probeasn.py - QA script for the -g miniooni option. """ - -import contextlib -import json -import os -import shlex -import shutil -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -def execute_miniooni(ooni_exe, outfile, arguments): - """ Executes miniooni and returns the whole measurement. """ - if "miniooni" not in ooni_exe: - return None - tmpoutfile = "/tmp/{}".format(outfile) - with contextlib.suppress(FileNotFoundError): - os.remove(tmpoutfile) # just in case - cmdline = [ - ooni_exe, - arguments, - "-o", - tmpoutfile, - "--home", - "/tmp", - "example", - ] - print("exec: {}".format(cmdline)) - common.execute(cmdline) - shutil.copy(tmpoutfile, outfile) - result = common.read_result(outfile) - assert isinstance(result, dict) - assert isinstance(result["test_keys"], dict) - return result - - -def probeasn_without_g_option(ooni_exe, outfile): - """ Test case where we're not passing to miniooni the -g option """ - m = execute_miniooni(ooni_exe, outfile, "-n") - if m is None: - return - assert m["probe_cc"] != "ZZ" - assert m["probe_ip"] == "127.0.0.1" - assert m["probe_asn"] != "AS0" - assert m["probe_network_name"] != "" - assert m["resolver_ip"] == "127.0.0.2" - assert m["resolver_asn"] != "AS0" - assert m["resolver_network_name"] != "" - - -def probeasn_with_g_option(ooni_exe, outfile): - """ Test case where we're passing the -g option """ - m = execute_miniooni(ooni_exe, outfile, "-gn") - if m is None: - return - assert m["probe_cc"] != "ZZ" - assert m["probe_ip"] == "127.0.0.1" - assert m["probe_asn"] == "AS0" - assert m["probe_network_name"] == "" - assert m["resolver_ip"] == "127.0.0.2" - assert m["resolver_asn"] == "AS0" - assert m["resolver_network_name"] == "" - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "probeasn.jsonl" - ooni_exe = sys.argv[1] - tests = [ - probeasn_with_g_option, - probeasn_without_g_option, - ] - for test in tests: - test(ooni_exe, outfile) - - -if __name__ == "__main__": - main() diff --git a/QA/pyrun.sh b/QA/pyrun.sh deleted file mode 100755 index 76f3f28..0000000 --- a/QA/pyrun.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -set -ex -export GOPATH=/jafar/QA/GOPATH GOCACHE=/jafar/QA/GOCACHE GO111MODULE=on -go build -v ./internal/cmd/miniooni -go build -v ./internal/cmd/jafar -sudo ./QA/$1.py ./miniooni diff --git a/QA/rundocker.bash b/QA/rundocker.bash index 8ce7320..77ec540 100755 --- a/QA/rundocker.bash +++ b/QA/rundocker.bash @@ -1,5 +1,16 @@ -#!/bin/sh -set -ex +#!/bin/bash + +set -euxo pipefail + DOCKER=${DOCKER:-docker} + +GOVERSION=$(cat GOVERSION) + +cat > QA/Dockerfile << EOF +FROM golang:$GOVERSION-alpine +RUN apk add gcc go git musl-dev iptables tmux bind-tools curl sudo python3 +EOF + $DOCKER build -t jafar-qa ./QA/ -$DOCKER run --privileged -v`pwd`:/jafar -w/jafar jafar-qa ./QA/pyrun.sh "$@" + +$DOCKER run --privileged -v$(pwd):/jafar -w/jafar jafar-qa ./QA/dockermain.sh "$@" diff --git a/QA/telegram.py b/QA/telegram.py deleted file mode 100755 index 1facdf2..0000000 --- a/QA/telegram.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/telegram.py - main QA script for telegram - - This script performs a bunch of telegram tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -ALL_POP_IPS = ( - "149.154.175.50", - "149.154.167.51", - "149.154.175.100", - "149.154.167.91", - "149.154.171.5", - "95.161.76.100", -) - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni(ooni_exe, outfile, "telegram", tag, args) - assert isinstance(tk["requests"], list) - assert len(tk["requests"]) > 0 - for entry in tk["requests"]: - assert isinstance(entry, dict) - failure = entry["failure"] - assert isinstance(failure, str) or failure is None - assert isinstance(entry["request"], dict) - req = entry["request"] - common.check_maybe_binary_value(req["body"]) - assert isinstance(req["headers"], dict) - for key, value in req["headers"].items(): - assert isinstance(key, str) - common.check_maybe_binary_value(value) - assert isinstance(req["method"], str) - assert isinstance(entry["response"], dict) - resp = entry["response"] - common.check_maybe_binary_value(resp["body"]) - assert isinstance(resp["code"], int) - if resp["headers"] is not None: - for key, value in resp["headers"].items(): - assert isinstance(key, str) - common.check_maybe_binary_value(value) - assert isinstance(tk["tcp_connect"], list) - assert len(tk["tcp_connect"]) > 0 - for entry in tk["tcp_connect"]: - assert isinstance(entry, dict) - assert isinstance(entry["ip"], str) - assert isinstance(entry["port"], int) - assert isinstance(entry["status"], dict) - failure = entry["status"]["failure"] - success = entry["status"]["success"] - assert isinstance(failure, str) or failure is None - assert isinstance(success, bool) - return tk - - -def args_for_blocking_all_pop_ips(): - """ Returns the arguments useful for blocking all POPs IPs """ - args = [] - for ip in ALL_POP_IPS: - args.append("-iptables-reset-ip") - args.append(ip) - return args - - -def args_for_blocking_web_telegram_org_http(): - """ Returns arguments for blocking web.telegram.org over http """ - return ["-iptables-reset-keyword", "Host: web.telegram.org"] - - -def args_for_blocking_web_telegram_org_https(): - """ Returns arguments for blocking web.telegram.org over https """ - # - # 00 00 - # 00 15 - # 00 13 - # 00 - # 00 10 - # 77 65 ... 67 web.telegram.org - # - return [ - "-iptables-reset-keyword-hex", - "|00 00 00 15 00 13 00 00 10 77 65 62 2e 74 65 6c 65 67 72 61 6d 2e 6f 72 67|", - ] - - -def telegram_block_everything(ooni_exe, outfile): - """ Test case where everything we measure is blocked """ - args = [] - args.extend(args_for_blocking_all_pop_ips()) - args.extend(args_for_blocking_web_telegram_org_https()) - args.extend(args_for_blocking_web_telegram_org_http()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_block_everything", args, - ) - assert tk["telegram_tcp_blocking"] == True - assert tk["telegram_http_blocking"] == True - assert tk["telegram_web_failure"] == "connection_reset" - assert tk["telegram_web_status"] == "blocked" - - -def telegram_tcp_blocking_all(ooni_exe, outfile): - """ Test case where all POPs are TCP/IP blocked """ - args = args_for_blocking_all_pop_ips() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_tcp_blocking_all", args - ) - assert tk["telegram_tcp_blocking"] == True - assert tk["telegram_http_blocking"] == True - assert tk["telegram_web_failure"] == None - assert tk["telegram_web_status"] == "ok" - - -def telegram_tcp_blocking_some(ooni_exe, outfile): - """ Test case where some POPs are TCP/IP blocked """ - args = [ - "-iptables-reset-ip", - ALL_POP_IPS[0], - ] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_tcp_blocking_some", args - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == False - assert tk["telegram_web_failure"] == None - assert tk["telegram_web_status"] == "ok" - - -def telegram_http_blocking_all(ooni_exe, outfile): - """ Test case where all POPs are HTTP blocked """ - args = [] - for ip in ALL_POP_IPS: - args.append("-iptables-reset-keyword") - args.append(ip) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_http_blocking_all", args, - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == True - assert tk["telegram_web_failure"] == None - assert tk["telegram_web_status"] == "ok" - - -def telegram_http_blocking_some(ooni_exe, outfile): - """ Test case where some POPs are HTTP blocked """ - args = [ - "-iptables-reset-keyword", - ALL_POP_IPS[0], - ] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_http_blocking_some", args, - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == False - assert tk["telegram_web_failure"] == None - assert tk["telegram_web_status"] == "ok" - - -def telegram_web_failure_http(ooni_exe, outfile): - """ Test case where the web HTTP endpoint is blocked """ - args = args_for_blocking_web_telegram_org_http() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_web_failure_http", args, - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == False - assert tk["telegram_web_failure"] == "connection_reset" - assert tk["telegram_web_status"] == "blocked" - - -def telegram_web_failure_https(ooni_exe, outfile): - """ Test case where the web HTTPS endpoint is blocked """ - args = args_for_blocking_web_telegram_org_https() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "telegram_web_failure_https", args, - ) - assert tk["telegram_tcp_blocking"] == False - assert tk["telegram_http_blocking"] == False - assert tk["telegram_web_failure"] == "connection_reset" - assert tk["telegram_web_status"] == "blocked" - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "telegram.jsonl" - ooni_exe = sys.argv[1] - tests = [ - telegram_block_everything, - telegram_tcp_blocking_all, - telegram_tcp_blocking_some, - telegram_http_blocking_all, - telegram_http_blocking_some, - telegram_web_failure_http, - telegram_web_failure_https, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/QA/whatsapp.py b/QA/whatsapp.py deleted file mode 100755 index 35b2dd7..0000000 --- a/QA/whatsapp.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/whatsapp.py - main QA script for whatsapp - - This script performs a bunch of whatsapp tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import contextlib -import json -import os -import shlex -import socket -import subprocess -import sys -import time -import urllib.parse - -sys.path.insert(0, ".") -import common - - -def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args): - """ Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid. """ - tk = common.execute_jafar_and_miniooni(ooni_exe, outfile, "whatsapp", tag, args) - assert isinstance(tk["requests"], list) - assert len(tk["requests"]) > 0 - for entry in tk["requests"]: - assert isinstance(entry, dict) - failure = entry["failure"] - assert isinstance(failure, str) or failure is None - assert isinstance(entry["request"], dict) - req = entry["request"] - common.check_maybe_binary_value(req["body"]) - assert isinstance(req["headers"], dict) - for key, value in req["headers"].items(): - assert isinstance(key, str) - common.check_maybe_binary_value(value) - assert isinstance(req["method"], str) - assert isinstance(entry["response"], dict) - resp = entry["response"] - common.check_maybe_binary_value(resp["body"]) - assert isinstance(resp["code"], int) - if resp["headers"] is not None: - for key, value in resp["headers"].items(): - assert isinstance(key, str) - common.check_maybe_binary_value(value) - assert isinstance(tk["tcp_connect"], list) - assert len(tk["tcp_connect"]) > 0 - for entry in tk["tcp_connect"]: - assert isinstance(entry, dict) - assert isinstance(entry["ip"], str) - assert isinstance(entry["port"], int) - assert isinstance(entry["status"], dict) - failure = entry["status"]["failure"] - success = entry["status"]["success"] - assert isinstance(failure, str) or failure is None - assert isinstance(success, bool) - return tk - - -def helper_for_blocking_endpoints(start, stop): - """ Helper function for generating args for blocking endpoints """ - args = [] - for num in range(start, stop): - args.append("-iptables-reset-ip") - args.append("e{}.whatsapp.net".format(num)) - return args - - -def args_for_blocking_all_endpoints(): - """ Returns the arguments useful for blocking all endpoints """ - return helper_for_blocking_endpoints(1, 17) - - -def args_for_blocking_some_endpoints(): - """ Returns the arguments useful for blocking some endpoints """ - # Implementation note: apparently all the endpoints are now using just - # four IP addresses, hence here we block some endpoints via DNS. - # - # TODO(bassosimone): this fact calls for creating an issue for making - # the whatsapp experiment implementation more efficient. - args = [] - args.append("-iptables-hijack-dns-to") - args.append("127.0.0.1:53") - for n in range(1, 7): - args.append("-dns-proxy-block") - args.append("e{}.whatsapp.net".format(n)) - return args - - -def args_for_blocking_v_whatsapp_net_https(): - """ Returns arguments for blocking v.whatsapp.net over https """ - # - # 00 00 - # 00 13 - # 00 11 - # 00 - # 00 0e - # 76 2e ... 74 v.whatsapp.net - # - return [ - "-iptables-reset-keyword-hex", - "|00 00 00 13 00 11 00 00 0e 76 2e 77 68 61 74 73 61 70 70 2e 6e 65 74|", - ] - - -def args_for_blocking_web_whatsapp_com_http(): - """ Returns arguments for blocking web.whatsapp.com over http """ - return ["-iptables-reset-keyword", "Host: web.whatsapp.com"] - - -def args_for_blocking_web_whatsapp_com_https(): - """ Returns arguments for blocking web.whatsapp.com over https """ - # - # 00 00 - # 00 15 - # 00 13 - # 00 - # 00 10 - # 77 65 ... 6d web.whatsapp.com - # - return [ - "-iptables-reset-keyword-hex", - "|00 00 00 15 00 13 00 00 10 77 65 62 2e 77 68 61 74 73 61 70 70 2e 63 6f 6d|", - ] - - -def whatsapp_block_everything(ooni_exe, outfile): - """ Test case where everything we measure is blocked """ - args = [] - args.extend(args_for_blocking_all_endpoints()) - args.extend(args_for_blocking_v_whatsapp_net_https()) - args.extend(args_for_blocking_web_whatsapp_com_https()) - args.extend(args_for_blocking_web_whatsapp_com_http()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_everything", args, - ) - assert tk["registration_server_failure"] == "connection_reset" - assert tk["registration_server_status"] == "blocked" - assert tk["whatsapp_endpoints_status"] == "blocked" - assert tk["whatsapp_web_failure"] == "connection_reset" - assert tk["whatsapp_web_status"] == "blocked" - - -def whatsapp_block_all_endpoints(ooni_exe, outfile): - """ Test case where we only block whatsapp endpoints """ - args = args_for_blocking_all_endpoints() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_all_endpoints", args - ) - assert tk["registration_server_failure"] == None - assert tk["registration_server_status"] == "ok" - assert tk["whatsapp_endpoints_status"] == "blocked" - assert tk["whatsapp_web_failure"] == None - assert tk["whatsapp_web_status"] == "ok" - - -def whatsapp_block_some_endpoints(ooni_exe, outfile): - """ Test case where we block some whatsapp endpoints """ - args = args_for_blocking_some_endpoints() - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_some_endpoints", args - ) - assert tk["registration_server_failure"] == None - assert tk["registration_server_status"] == "ok" - assert tk["whatsapp_endpoints_status"] == "ok" - assert tk["whatsapp_web_failure"] == None - assert tk["whatsapp_web_status"] == "ok" - - -def whatsapp_block_registration_server(ooni_exe, outfile): - """ Test case where we block the registration server """ - args = [] - args.extend(args_for_blocking_v_whatsapp_net_https()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_registration_server", args, - ) - assert tk["registration_server_failure"] == "connection_reset" - assert tk["registration_server_status"] == "blocked" - assert tk["whatsapp_endpoints_status"] == "ok" - assert tk["whatsapp_web_failure"] == None - assert tk["whatsapp_web_status"] == "ok" - - -def whatsapp_block_web_http(ooni_exe, outfile): - """ Test case where we block the HTTP web chat """ - args = [] - args.extend(args_for_blocking_web_whatsapp_com_http()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_web_http", args, - ) - assert tk["registration_server_failure"] == None - assert tk["registration_server_status"] == "ok" - assert tk["whatsapp_endpoints_status"] == "ok" - assert tk["whatsapp_web_failure"] == "connection_reset" - assert tk["whatsapp_web_status"] == "blocked" - - -def whatsapp_block_web_https(ooni_exe, outfile): - """ Test case where we block the HTTPS web chat """ - args = [] - args.extend(args_for_blocking_web_whatsapp_com_https()) - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, "whatsapp_block_web_https", args, - ) - assert tk["registration_server_failure"] == None - assert tk["registration_server_status"] == "ok" - assert tk["whatsapp_endpoints_status"] == "ok" - assert tk["whatsapp_web_failure"] == "connection_reset" - assert tk["whatsapp_web_status"] == "blocked" - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "whatsapp.jsonl" - ooni_exe = sys.argv[1] - tests = [ - whatsapp_block_everything, - whatsapp_block_all_endpoints, - whatsapp_block_some_endpoints, - whatsapp_block_registration_server, - whatsapp_block_web_http, - whatsapp_block_web_https, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main()