da34cfe6c9
This pull request introduces a set of Node.js scripts for performing A/B comparison of websteps and webconnectivity as described in https://github.com/ooni/probe/issues/1805. Rather than using Jafar, I ended up using `miniooni`'s `--censor` command line flag introduced in [v3.12.0-alpha.1](https://github.com/ooni/probe-cli/releases/tag/v3.12.0-alpha.1). The main reason for doing so is that it's simpler to run tests without requiring root access and Linux _and_ Docker (e.g., I did not develop part of this diff using Linux). Additionally, I choose to use Node.js rather than extending the existing Python framework for QA, because I found Node.js easier when working with JSON data.
93 lines
3.3 KiB
JavaScript
93 lines
3.3 KiB
JavaScript
// This file contains code for running test cases.
|
|
|
|
import child_process from "child_process"
|
|
import crypto from "crypto"
|
|
import fs from "fs"
|
|
import path from "path"
|
|
|
|
// tempFile returns the name of a temporary file. This function is
|
|
// not as secure as using mktemp but it does not matter in this
|
|
// context. We just need to create file names in the local directory
|
|
// with enough entropy that every run has a different name.
|
|
//
|
|
// See https://stackoverflow.com/questions/7055061 for an insightful
|
|
// discussion about creating temporary files in node.
|
|
function tempFile(suffix) {
|
|
return path.join(`tmp-${crypto.randomBytes(16).toString('hex')}.${suffix}`)
|
|
}
|
|
|
|
// exec executes a command. This function throws on failure. The stdout
|
|
// and stderr should be console.log-ed once the command returns.
|
|
function exec(command) {
|
|
console.log(`+ ${command}`)
|
|
child_process.execSync(command)
|
|
}
|
|
|
|
// writeCensorJsonFile writes a censor.json file using a file name
|
|
// containing random characters and returns the file name.
|
|
function writeCensorJsonFile(testCase) {
|
|
const fileName = tempFile("json")
|
|
fs.writeFileSync(fileName, JSON.stringify(testCase.blocking))
|
|
return fileName
|
|
}
|
|
|
|
// readReportFile reads and parses the report file, thus returning
|
|
// the JSON object contained inside the report file.
|
|
function readReportFile(reportFile) {
|
|
const data = fs.readFileSync(reportFile, { "encoding": "utf-8" })
|
|
return JSON.parse(data)
|
|
}
|
|
|
|
// runExperiment runs the given test case with the given experiment.
|
|
function runExperiment(testCase, experiment, checker) {
|
|
console.log(`## running: ${testCase.name}.${experiment}`)
|
|
console.log("")
|
|
const censorJson = writeCensorJsonFile(testCase)
|
|
const reportJson = tempFile("json")
|
|
// Note: using -n because we don't want to submit QA checks.
|
|
exec(`./miniooni -n --censor ${censorJson} -o ${reportJson} -i ${testCase.input} ${experiment}`)
|
|
console.log("")
|
|
// TODO(bassosimone): support multiple entries per file
|
|
const report = readReportFile(reportJson)
|
|
const analysisResult = checker(testCase, experiment, report)
|
|
console.log("")
|
|
console.log("")
|
|
if (analysisResult !== true && analysisResult !== false) {
|
|
console.log("the analysis function returned neither true nor false")
|
|
process.exit(1)
|
|
}
|
|
return analysisResult
|
|
}
|
|
|
|
// recompileMiniooni recompiles miniooni if needed.
|
|
function recompileMiniooni() {
|
|
exec("go build -v ./internal/cmd/miniooni")
|
|
}
|
|
|
|
// runTestCase runs the given test case.
|
|
//
|
|
// A test case is an object with the following fields:
|
|
//
|
|
// - name (string): the name of the test case;
|
|
//
|
|
// - description (string): a description of the test case;
|
|
//
|
|
// - input (string): the input to pass to the experiment;
|
|
//
|
|
// - blocking (object): a blocking specification (i.e., the
|
|
// serialization of a filtering.TProxyConfig struct);
|
|
//
|
|
// - experiments (object): the keys are names of nettests
|
|
// to run and the values are functions taking as their
|
|
// unique argument the experiment's test_keys.
|
|
export function runTestCase(testCase) {
|
|
recompileMiniooni()
|
|
console.log("")
|
|
console.log(`# running: ${testCase.name}`)
|
|
let result = true
|
|
for (const [name, checker] of Object.entries(testCase.experiments)) {
|
|
result = result && runExperiment(testCase, name, checker)
|
|
}
|
|
return result
|
|
}
|