feat(QA): add test cases for websteps vs webconnectivity (#583)
This pull request introduces a set of Node.js scripts for performing A/B comparison of websteps and webconnectivity as described in https://github.com/ooni/probe/issues/1805. Rather than using Jafar, I ended up using `miniooni`'s `--censor` command line flag introduced in [v3.12.0-alpha.1](https://github.com/ooni/probe-cli/releases/tag/v3.12.0-alpha.1). The main reason for doing so is that it's simpler to run tests without requiring root access and Linux _and_ Docker (e.g., I did not develop part of this diff using Linux). Additionally, I choose to use Node.js rather than extending the existing Python framework for QA, because I found Node.js easier when working with JSON data.
This commit is contained in:
parent
be89878dd4
commit
da34cfe6c9
7 changed files with 1066 additions and 0 deletions
106
QA/index.mjs
Normal file
106
QA/index.mjs
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
// This script performs QA checks.
|
||||
|
||||
import { runTestCase } from "./lib/runner.mjs"
|
||||
import { testCases as webTestCases } from "./lib/web.mjs"
|
||||
|
||||
// testCases lists all the test cases.
|
||||
//
|
||||
// A test case is an object with the structure described
|
||||
// by the documentation of runTestCase in ./lib/runner.mjs.
|
||||
const testCases = [
|
||||
...webTestCases,
|
||||
]
|
||||
|
||||
// checkForDuplicateTestCases ensures there are no duplicate names.
|
||||
function checkForDuplicateTestCases() {
|
||||
let dups = {}
|
||||
for (let i = 0; i < testCases.length; i++) {
|
||||
const testCase = testCases[i]
|
||||
if (dups[testCase.name] !== undefined) {
|
||||
console.log(`fatal: duplicate test case name: ${testCase.name}`)
|
||||
process.exit(1)
|
||||
}
|
||||
dups[testCase.name] = true
|
||||
}
|
||||
}
|
||||
|
||||
// runAllTestCases runs all the available test cases.
|
||||
function runAllTestCases() {
|
||||
let result = true
|
||||
for (let i = 0; i < testCases.length; i++) {
|
||||
result = result && runTestCase(testCases[i])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// makeTestCasesMap creates a map from the test case name
|
||||
// to the test case definition.
|
||||
function makeTestCasesMap() {
|
||||
var map = {}
|
||||
for (let i = 0; i < testCases.length; i++) {
|
||||
const testCase = testCases[i]
|
||||
map[testCase.name] = testCase
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
// commandRun implements the run command.
|
||||
function commandRun(args) {
|
||||
const bailOnFailure = (result) => {
|
||||
if (!result) {
|
||||
console.log("some checks failed (see above logs)")
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
if (args.length < 1) {
|
||||
bailOnFailure(runAllTestCases())
|
||||
return
|
||||
}
|
||||
let result = true
|
||||
const map = makeTestCasesMap()
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]
|
||||
const testCase = map[arg]
|
||||
if (testCase === undefined) {
|
||||
console.log(`unknown test case: ${arg}`)
|
||||
process.exit(1)
|
||||
}
|
||||
result = result && runTestCase(testCase)
|
||||
}
|
||||
bailOnFailure(result)
|
||||
}
|
||||
|
||||
// commandList implements the list command.
|
||||
function commandList() {
|
||||
for (let i = 0; i < testCases.length; i++) {
|
||||
const testCase = testCases[i]
|
||||
console.log(`${testCase.name}: ${testCase.description}`)
|
||||
}
|
||||
}
|
||||
|
||||
// main is the main function.
|
||||
function main() {
|
||||
const usageAndExit = (exitcode) => {
|
||||
console.log("usage: node ./QA/index.mjs list")
|
||||
console.log("usage: node ./QA/index.mjs run [test_case_name...]")
|
||||
process.exit(exitcode)
|
||||
}
|
||||
if (process.argv.length < 3) {
|
||||
usageAndExit(0)
|
||||
}
|
||||
checkForDuplicateTestCases()
|
||||
const command = process.argv[2]
|
||||
switch (command) {
|
||||
case "list":
|
||||
commandList()
|
||||
break
|
||||
case "run":
|
||||
commandRun(process.argv.slice(3))
|
||||
break
|
||||
default:
|
||||
console.log(`unknown command: ${command}`)
|
||||
usageAndExit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
Loading…
Reference in a new issue