feat(QA): add test cases for websteps vs webconnectivity (#583)

This pull request introduces a set of Node.js scripts for performing A/B comparison of websteps and webconnectivity as described in https://github.com/ooni/probe/issues/1805. Rather than using Jafar, I ended up using `miniooni`'s `--censor` command line flag introduced in [v3.12.0-alpha.1](https://github.com/ooni/probe-cli/releases/tag/v3.12.0-alpha.1). The main reason for doing so is that it's simpler to run tests without requiring root access and Linux _and_ Docker (e.g., I did not develop part of this diff using Linux). Additionally, I choose to use Node.js rather than extending the existing Python framework for QA, because I found Node.js easier when working with JSON data.

.eslintrc.json

@@ -0,0 +1,13 @@
+{
+    "env": {
+        "node": true,
+        "es2021": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 13,
+        "sourceType": "module"
+    },
+    "rules": {
+    }
+}

.gitignore

@@ -22,4 +22,5 @@
 /ptxclient.exe
 /*.tar.gz
 /testdata/gotmp
+/tmp-*
 /*.zip

QA/index.mjs

@@ -0,0 +1,106 @@
+// This script performs QA checks.
+
+import { runTestCase } from "./lib/runner.mjs"
+import { testCases as webTestCases } from "./lib/web.mjs"
+
+// testCases lists all the test cases.
+//
+// A test case is an object with the structure described
+// by the documentation of runTestCase in ./lib/runner.mjs.
+const testCases = [
+    ...webTestCases,
+]
+
+// checkForDuplicateTestCases ensures there are no duplicate names.
+function checkForDuplicateTestCases() {
+    let dups = {}
+    for (let i = 0; i < testCases.length; i++) {
+        const testCase = testCases[i]
+        if (dups[testCase.name] !== undefined) {
+            console.log(`fatal: duplicate test case name: ${testCase.name}`)
+            process.exit(1)
+        }
+        dups[testCase.name] = true
+    }
+}
+
+// runAllTestCases runs all the available test cases.
+function runAllTestCases() {
+    let result = true
+    for (let i = 0; i < testCases.length; i++) {
+        result = result && runTestCase(testCases[i])
+    }
+    return result
+}
+
+// makeTestCasesMap creates a map from the test case name
+// to the test case definition.
+function makeTestCasesMap() {
+    var map = {}
+    for (let i = 0; i < testCases.length; i++) {
+        const testCase = testCases[i]
+        map[testCase.name] = testCase
+    }
+    return map
+}
+
+// commandRun implements the run command.
+function commandRun(args) {
+    const bailOnFailure = (result) => {
+        if (!result) {
+            console.log("some checks failed (see above logs)")
+            process.exit(1)
+        }
+    }
+    if (args.length < 1) {
+        bailOnFailure(runAllTestCases())
+        return
+    }
+    let result = true
+    const map = makeTestCasesMap()
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i]
+        const testCase = map[arg]
+        if (testCase === undefined) {
+            console.log(`unknown test case: ${arg}`)
+            process.exit(1)
+        }
+        result = result && runTestCase(testCase)
+    }
+    bailOnFailure(result)
+}
+
+// commandList implements the list command.
+function commandList() {
+    for (let i = 0; i < testCases.length; i++) {
+        const testCase = testCases[i]
+        console.log(`${testCase.name}: ${testCase.description}`)
+    }
+}
+
+// main is the main function.
+function main() {
+    const usageAndExit = (exitcode) => {
+        console.log("usage: node ./QA/index.mjs list")
+        console.log("usage: node ./QA/index.mjs run [test_case_name...]")
+        process.exit(exitcode)
+    }
+    if (process.argv.length < 3) {
+        usageAndExit(0)
+    }
+    checkForDuplicateTestCases()
+    const command = process.argv[2]
+    switch (command) {
+        case "list":
+            commandList()
+            break
+        case "run":
+            commandRun(process.argv.slice(3))
+            break
+        default:
+            console.log(`unknown command: ${command}`)
+            usageAndExit(1)
+    }
+}
+
+main()

QA/lib/analysis.mjs

@@ -0,0 +1,196 @@
+// This file contains code for analysing results.
+
+import { type } from "os"
+
+// checkAnnotations checks whether we have annotations.
+function checkAnnotations(report) {
+    let result = true
+
+    const isObject = typeof (report.annotations) === "object"
+    console.log(`checking whether annotations is an object... ${isObject}`)
+    result = result && isObject
+
+    const hasArchitecture = typeof (report.annotations.architecture) === "string"
+    console.log(`checking whether annotations contains architecture... ${hasArchitecture}`)
+    result = result && hasArchitecture
+
+    const hasEngineName = typeof (report.annotations.engine_name) === "string"
+    console.log(`checking whether annotations contains engine_name... ${hasEngineName}`)
+    result = result && hasEngineName
+
+    const hasEngineVersion = typeof (report.annotations.engine_version) === "string"
+    console.log(`checking whether annotations contains engine_version... ${hasEngineVersion}`)
+    result = result && hasEngineVersion
+
+    const hasPlatform = typeof (report.annotations.platform) === "string"
+    console.log(`checking whether annotations contains platform... ${hasPlatform}`)
+    result = result && hasPlatform
+
+    return result
+}
+
+// checkDataFormatVersion checks whether we have data_format_version.
+function checkDataFormatVersion(report) {
+    const result = report.data_format_version === "0.2.0"
+    console.log(`checking whether we have the right data format version... ${result}`)
+    return result
+}
+
+// checkExtensions ensures that extensions exists and has the right type.
+function checkExtensions(report) {
+    const result = typeof (report.extensions) === "object"
+    console.log(`checking whether report.extensions is an object... ${result}`)
+    // Quirk: some experiments (e.g. web_connectivity) don't include
+    // an extensions object, and we don't want to fail here.
+    return true
+}
+
+// checkInput ensures that the input is correct.
+function checkInput(testCase, report) {
+    const result = testCase.input === report.input
+    console.log(`checking whether input is correct... ${result}`)
+    return result
+}
+
+// checkExperimentName ensures that the experimentName is correctly
+// set in the output report file.
+function checkExperimentName(name, report) {
+    const result = report.test_name === name
+    console.log(`checking whether the experiment name is correct... ${result}`)
+    return result
+}
+
+// checkStartTime ensures that a given start time is in the correct format.
+function checkStartTime(report, key) {
+    const value = report[key]
+    let result = typeof (value) === "string"
+    console.log(`checking whether ${key} is a string... ${result}`)
+    result = result && value.match(/^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$/) !== null
+    console.log(`checking whether ${key} matches the date regexp... ${result}`)
+    return result
+}
+
+// checkASN ensures that an ASN field is correct.
+function checkASN(report, key) {
+    const value = report[key]
+    let result = typeof (value) === "string"
+    console.log(`checking whether ${key} is a string... ${result}`)
+    result = result && value.match(/^AS[0-9]+$/) !== null
+    console.log(`checking whether ${key} matches the ASN regexp... ${result}`)
+    return result
+}
+
+// checkProbeCC ensures that the probe_cc field is correct.
+function checkProbeCC(report) {
+    const value = report["probe_cc"]
+    let result = typeof (value) === "string"
+    console.log(`checking whether probe_cc is a string... ${result}`)
+    result = result && value.match(/^[A-Z]{2}$/) !== null
+    console.log(`checking whether probe_cc matches the CC regexp... ${result}`)
+    return result
+
+}
+
+// checkProbeIP ensures that the probe_ip field is correct.
+function checkProbeIP(report) {
+    const result = report.probe_ip === "127.0.0.1"
+    console.log(`checking whether probe_ip is correct... ${result}`)
+    return result
+}
+
+// checkReportID ensures that the report_id field is correct.
+function checkReportID(report) {
+    const result = report.report_id === "" // note: we don't submit
+    console.log(`checking whether report_id is correct... ${result}`)
+    return result
+}
+
+// checkString ensures that a field is a string.
+function checkString(report, key) {
+    const value = report[key]
+    const result = typeof(value) === "string"
+    console.log(`checking whether ${key} is a string... ${result}`)
+    return result
+}
+
+// checkNetworkName ensures that an xxx_network_name field is correct.
+function checkNetworkName(report, key) {
+    return checkString(report, key)
+}
+
+// checkResolverIP ensures that the resolver_ip field is correct.
+function checkResolverIP(report) {
+    return checkString(report, "resolver_ip")
+}
+
+// checkSoftwareName ensures that the software_name field is correct.
+function checkSoftwareName(report) {
+    return checkString(report, "software_name")
+}
+
+// checkSoftwareVersion ensures that the software_version field is correct.
+function checkSoftwareVersion(report) {
+    return checkString(report, "software_version")
+}
+
+// checkTestRuntime ensures that the test_runtime field is correct.
+function checkTestRuntime(report) {
+    const result = typeof(report.test_runtime) === "number"
+    console.log(`checking whether test_runtime is a number... ${result}`)
+    return result
+}
+
+// checkTestVersion ensures that the test_version field is correct.
+function checkTestVersion(report) {
+    return checkString(report, "test_version")
+}
+
+// checkTestKeys ensures that test_keys is an object.
+function checkTestKeys(report) {
+    const result = typeof(report.test_keys) === "object"
+    console.log(`checking whether test_keys is an object... ${result}`)
+    return result
+}
+
+// checkMeasurement is a function that invokes a standard set of checks
+// to validate the top-level test keys of a result.
+//
+// This function helps to implement per-experiment checkers.
+//
+// Arguments:
+//
+// - testCase is the current test case
+//
+// - name is the name of the current experiment
+//
+// - report is the JSON measurement
+//
+// - extraChecks is the optional callback to perform extra checks
+// that takes in input the report's test keys.
+export function checkMeasurement(testCase, name, report, extraChecks) {
+    let result = true
+    result = result && checkAnnotations(report)
+    result = result && checkDataFormatVersion(report)
+    result = result && checkExtensions(report)
+    result = result && checkInput(testCase, report)
+    result = result && checkStartTime(report, "measurement_start_time")
+    result = result && checkASN(report, "probe_asn")
+    result = result && checkProbeCC(report)
+    result = result && checkProbeIP(report)
+    result = result && checkNetworkName(report, "probe_network_name")
+    result = result && checkReportID(report)
+    result = result && checkASN(report, "resolver_asn")
+    result = result && checkResolverIP(report)
+    result = result && checkNetworkName(report, "resolver_network_name")
+    result = result && checkSoftwareName(report)
+    result = result && checkSoftwareVersion(report)
+    result = result && checkExperimentName(name, report)
+    result = result && checkTestRuntime(report)
+    result = result && checkStartTime(report, "test_start_time")
+    result = result && checkTestVersion(report)
+    result = result && checkTestKeys(report)
+    if (typeof extraChecks === "function") {
+        result = result && extraChecks(report.test_keys)
+    }
+    return result
+}

QA/lib/blocking.mjs

@@ -0,0 +1,17 @@
+// This file contains helpers for describing blocking rules.
+
+// hijackPopularDNSServers returns an object containing the rules
+// for hijacking popular DNS servers with `miniooni --censor`.
+export function hijackPopularDNSServers() {
+    return {
+        // cloudflare
+        "1.1.1.1:53/udp": "hijack-dns",
+        "1.0.0.1:53/udp": "hijack-dns",
+        // google
+        "8.8.8.8:53/udp": "hijack-dns",
+        "8.8.4.4:53/udp": "hijack-dns",
+        // quad9
+        "9.9.9.9:53/udp": "hijack-dns",
+        "9.9.9.10:53/udp": "hijack-dns",
+    }
+}

QA/lib/runner.mjs

@@ -0,0 +1,92 @@
+// This file contains code for running test cases.
+
+import child_process from "child_process"
+import crypto from "crypto"
+import fs from "fs"
+import path from "path"
+
+// tempFile returns the name of a temporary file. This function is
+// not as secure as using mktemp but it does not matter in this
+// context. We just need to create file names in the local directory
+// with enough entropy that every run has a different name.
+//
+// See https://stackoverflow.com/questions/7055061 for an insightful
+// discussion about creating temporary files in node.
+function tempFile(suffix) {
+    return path.join(`tmp-${crypto.randomBytes(16).toString('hex')}.${suffix}`)
+}
+
+// exec executes a command. This function throws on failure. The stdout
+// and stderr should be console.log-ed once the command returns.
+function exec(command) {
+    console.log(`+ ${command}`)
+    child_process.execSync(command)
+}
+
+// writeCensorJsonFile writes a censor.json file using a file name
+// containing random characters and returns the file name.
+function writeCensorJsonFile(testCase) {
+    const fileName = tempFile("json")
+    fs.writeFileSync(fileName, JSON.stringify(testCase.blocking))
+    return fileName
+}
+
+// readReportFile reads and parses the report file, thus returning
+// the JSON object contained inside the report file.
+function readReportFile(reportFile) {
+    const data = fs.readFileSync(reportFile, { "encoding": "utf-8" })
+    return JSON.parse(data)
+}
+
+// runExperiment runs the given test case with the given experiment.
+function runExperiment(testCase, experiment, checker) {
+    console.log(`## running: ${testCase.name}.${experiment}`)
+    console.log("")
+    const censorJson = writeCensorJsonFile(testCase)
+    const reportJson = tempFile("json")
+    // Note: using -n because we don't want to submit QA checks.
+    exec(`./miniooni -n --censor ${censorJson} -o ${reportJson} -i ${testCase.input} ${experiment}`)
+    console.log("")
+    // TODO(bassosimone): support multiple entries per file
+    const report = readReportFile(reportJson)
+    const analysisResult = checker(testCase, experiment, report)
+    console.log("")
+    console.log("")
+    if (analysisResult !== true && analysisResult !== false) {
+       console.log("the analysis function returned neither true nor false")
+       process.exit(1)
+    }
+    return analysisResult
+}
+
+// recompileMiniooni recompiles miniooni if needed.
+function recompileMiniooni() {
+    exec("go build -v ./internal/cmd/miniooni")
+}
+
+// runTestCase runs the given test case.
+//
+// A test case is an object with the following fields:
+//
+// - name (string): the name of the test case;
+//
+// - description (string): a description of the test case;
+//
+// - input (string): the input to pass to the experiment;
+//
+// - blocking (object): a blocking specification (i.e., the
+// serialization of a filtering.TProxyConfig struct);
+//
+// - experiments (object): the keys are names of nettests
+// to run and the values are functions taking as their
+// unique argument the experiment's test_keys.
+export function runTestCase(testCase) {
+    recompileMiniooni()
+    console.log("")
+    console.log(`# running: ${testCase.name}`)
+    let result = true
+    for (const [name, checker] of Object.entries(testCase.experiments)) {
+        result = result && runExperiment(testCase, name, checker)
+    }
+    return result
+}

QA/lib/web.mjs

@@ -0,0 +1,641 @@
+// This file contains test cases for web nettests.
+
+import { checkMeasurement } from "./analysis.mjs"
+
+// webConnectivityCheckTopLevel checks the top-level keys
+// of the web connectivity experiment agains a template
+// object. Returns true if they match, false on mismatch.
+function webConnectivityCheckTopLevel(tk, template) {
+    let result = true
+    for (const [key, value] of Object.entries(template)) {
+        const check = tk[key] === value
+        console.log(`checking whether ${key}'s value is ${value}... ${check}`)
+        result = result && check
+    }
+    return result
+}
+
+// Here we export all the test cases. Please, see the documentation
+// of runner.runTestCase for a description of what is a test case.
+export const testCases = [
+
+    //
+    // DNS checks
+    //
+    // We start with checks where _only_ the system resolver fails.
+    //
+
+    {
+        name: "web_dns_system_nxdomain",
+        description: "the system resolver returns NXDOMAIN",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Domains: {
+                "nexa.polito.it": "nxdomain"
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": "dns_nxdomain_error",
+                        "dns_consistency": "inconsistent",
+                        "control_failure": null,
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "dns",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_dns_system_refused",
+        description: "the system resolver returns REFUSED",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Domains: {
+                "nexa.polito.it": "refused"
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": "dns_refused_error",
+                        "dns_consistency": "inconsistent",
+                        "control_failure": null,
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": null,
+                        "blocking": null, // TODO(bassosimone): this is clearly a bug
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_dns_system_localhost",
+        description: "the system resolver returns localhost",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Domains: {
+                "nexa.polito.it": "localhost",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    // TODO(bassosimone): Web Connectivity does not correctly handle this case
+                    // but, still, correctly sets blocking as "dns"
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "inconsistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "connection_refused",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "dns",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_dns_system_bogon_not_localhost",
+        description: "the system resolver returns a bogon that is not localhost",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            DNSCache: {
+                "nexa.polito.it": ["10.0.0.1"],
+            },
+            Domains: {
+                "nexa.polito.it": "cache",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    // TODO(bassosimone): Web Connectivity does not correctly handle this case
+                    // but, still, correctly sets blocking as "dns"
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "inconsistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "generic_timeout_error",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "dns",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_dns_system_no_answer",
+        description: "the system resolver returns an empty answer",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Domains: {
+                "nexa.polito.it": "no-answer",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": "dns_no_answer",
+                        "dns_consistency": "inconsistent",
+                        "control_failure": null,
+                        "http_experiment_failure": null,
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": null,
+                        "blocking": null, // TODO(bassosimone): this is clearly a bug
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_dns_system_timeout",
+        description: "the system resolver times out",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Domains: {
+                "nexa.polito.it": "timeout",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": "generic_timeout_error",
+                        "dns_consistency": "inconsistent",
+                        "control_failure": null,
+                        "http_experiment_failure": null,
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": null,
+                        "blocking": null, // TODO(bassosimone): this is clearly a bug
+                    })
+                    return result
+                })
+            },
+        },
+
+    },
+
+    // TODO(bassosimone): here we should insert more checks where not only the system
+    // resolver is blocked but also other resolvers are.
+
+    //
+    // TCP connect
+    //
+    // This section contains TCP connect failures.
+    //
+
+    {
+        name: "web_tcp_connect_timeout",
+        description: "timeout when connecting to the IP address",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Endpoints: {
+                "130.192.16.171:443/tcp": "tcp-drop-syn",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "generic_timeout_error",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "tcp_ip",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_tcp_connect_refused",
+        description: "connection refused when connecting to the IP address",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Endpoints: {
+                "130.192.16.171:443/tcp": "tcp-reject-syn",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "connection_refused",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "tcp_ip",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    //
+    // TLS handshake
+    //
+    // This section contains TLS handshake failures.
+    //
+
+    {
+        name: "web_tls_handshake_timeout",
+        description: "timeout when performing the TLS handshake",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Endpoints: {
+                "130.192.16.171:443/tcp": "drop-data",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "generic_timeout_error",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "http-failure",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_tls_handshake_reset",
+        description: "reset when performing the TLS handshake",
+        input: "https://nexa.polito.it/",
+        blocking: {
+            Endpoints: {
+                "130.192.16.171:443/tcp": "hijack-tls",
+            },
+            SNIs: {
+                "nexa.polito.it": "reset",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "connection_reset",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "http-failure",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    //
+    // QUIC
+    //
+
+    {
+        name: "web_quic_handshake_timeout",
+        description: "timeout when performing the QUIC handshake",
+        input: "https://dns.google/",
+        blocking: {
+            Endpoints: {
+                "8.8.8.8:443/udp": "drop-data",
+                "8.8.4.4:443/udp": "drop-data",
+                "[2001:4860:4860::8888]:443/udp": "drop-data",
+                "[2001:4860:4860::8844]:443/udp": "drop-data",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+        },
+    },
+
+    //
+    // Cleartext HTTP
+    //
+
+    {
+        name: "web_http_reset",
+        description: "reset when performing the HTTP round trip",
+        input: "http://nexa.polito.it/",
+        blocking: {
+            Endpoints: {
+                "130.192.16.171:80/tcp": "hijack-http",
+            },
+            Hosts: {
+                "nexa.polito.it": "reset",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "connection_reset",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "http-failure",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_http_timeout",
+        description: "timeout when performing the HTTP round trip",
+        input: "http://nexa.polito.it/",
+        blocking: {
+            Endpoints: {
+                "130.192.16.171:80/tcp": "hijack-http",
+            },
+            Hosts: {
+                "nexa.polito.it": "timeout",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "generic_timeout_error",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "http-failure",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    {
+        name: "web_http_451",
+        description: "451 when performing the HTTP round trip",
+        input: "http://nexa.polito.it/",
+        blocking: {
+            Endpoints: {
+                "130.192.16.171:80/tcp": "hijack-http",
+            },
+            Hosts: {
+                "nexa.polito.it": "451",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    // TODO(bassosimone): there is no easy way to check for the body
+                    // proportion robustly because it's a float.
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": null,
+                        "body_length_match": false,
+                        "status_code_match": false,
+                        "headers_match": false,
+                        "title_match": false,
+                        "accessible": false,
+                        "blocking": "http-diff",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    //
+    // More complex scenarios
+    //
+
+    // In this scenario the second IP address for the domain fails
+    // with reset. Web Connectivity sees that but overall says it's
+    // all good because the good IP happens to be the first. We'll
+    // see what changes if we swap the IPs in the next scenario.
+    {
+        name: "web_tcp_second_ip_connection_reset",
+        description: "the second IP returned by DNS fails with connection reset",
+        input: "https://dns.google/",
+        blocking: {
+            DNSCache: {
+                "dns.google": ["8.8.4.4", "8.8.8.8"],
+            },
+            Domains: {
+                "dns.google": "cache",
+            },
+            Endpoints: {
+                "8.8.8.8:443/tcp": "hijack-tls",
+            },
+            SNIs: {
+                "dns.google": "reset",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": null,
+                        "body_length_match": true,
+                        "body_proportion": 1,
+                        "status_code_match": true,
+                        "headers_match": true,
+                        "title_match": true,
+                        "accessible": true,
+                        "blocking": false,
+                    })
+                    return result
+                })
+            },
+        },
+    },
+
+    // This scenario is like the previous one except that we swap
+    // the IP addresses and now Web Connectivity says failure.
+    {
+        name: "web_tcp_first_ip_connection_reset",
+        description: "the first IP returned by DNS fails with connection reset",
+        input: "https://dns.google/",
+        blocking: {
+            DNSCache: {
+                "dns.google": ["8.8.4.4", "8.8.8.8"],
+            },
+            Domains: {
+                "dns.google": "cache",
+            },
+            Endpoints: {
+                "8.8.4.4:443/tcp": "hijack-tls",
+            },
+            SNIs: {
+                "dns.google": "reset",
+            },
+        },
+        experiments: {
+            websteps: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report)
+            },
+            web_connectivity: (testCase, name, report) => {
+                return checkMeasurement(testCase, name, report, (tk) => {
+                    let result = true
+                    result = result && webConnectivityCheckTopLevel(tk, {
+                        "dns_experiment_failure": null,
+                        "dns_consistency": "consistent",
+                        "control_failure": null,
+                        "http_experiment_failure": "connection_reset",
+                        "body_length_match": null,
+                        "body_proportion": 0,
+                        "status_code_match": null,
+                        "headers_match": null,
+                        "title_match": null,
+                        "accessible": false,
+                        "blocking": "http-failure",
+                    })
+                    return result
+                })
+            },
+        },
+    },
+]