c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Spring cleanup: remove unused/unneded code (#761)

Browse Source 
* cleanup: remove the archival package

See https://github.com/ooni/probe/issues/2116

* cleanup: remove websteps fall 2021 edition

See https://github.com/ooni/probe/issues/2116

* cleanup: remove JavaScript based testing framework

https://github.com/ooni/probe/issues/2116

* cleanup: remove the unused ooapi package

See https://github.com/ooni/probe/issues/2116
This commit is contained in:
Simone Basso 2022-05-25 13:21:39 +02:00 committed by GitHub
parent 8b0815efab
commit 7a0a156aec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
89 changed files with 2 additions and 18567 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
QA/index.mjs
View File

@ -1,106 +0,0 @@
// This script performs QA checks.
import { runTestCase } from "./lib/runner.mjs"
import { testCases as webTestCases } from "./lib/web.mjs"
// testCases lists all the test cases.
//
// A test case is an object with the structure described
// by the documentation of runTestCase in ./lib/runner.mjs.
const testCases = [
...webTestCases,
]
// checkForDuplicateTestCases ensures there are no duplicate names.
function checkForDuplicateTestCases() {
let dups = {}
for (let i = 0; i < testCases.length; i++) {
const testCase = testCases[i]
if (dups[testCase.name] !== undefined) {
console.log(`fatal: duplicate test case name: ${testCase.name}`)
process.exit(1)
}
dups[testCase.name] = true
}
}
// runAllTestCases runs all the available test cases.
function runAllTestCases() {
let result = true
for (let i = 0; i < testCases.length; i++) {
result = result && runTestCase(testCases[i])
}
return result
}
// makeTestCasesMap creates a map from the test case name
// to the test case definition.
function makeTestCasesMap() {
var map = {}
for (let i = 0; i < testCases.length; i++) {
const testCase = testCases[i]
map[testCase.name] = testCase
}
return map
}
// commandRun implements the run command.
function commandRun(args) {
const bailOnFailure = (result) => {
if (!result) {
console.log("some checks failed (see above logs)")
process.exit(1)
}
}
if (args.length < 1) {
bailOnFailure(runAllTestCases())
return
}
let result = true
const map = makeTestCasesMap()
for (let i = 0; i < args.length; i++) {
const arg = args[i]
const testCase = map[arg]
if (testCase === undefined) {
console.log(`unknown test case: ${arg}`)
process.exit(1)
}
result = result && runTestCase(testCase)
}
bailOnFailure(result)
}
// commandList implements the list command.
function commandList() {
for (let i = 0; i < testCases.length; i++) {
const testCase = testCases[i]
console.log(`${testCase.name}: ${testCase.description}`)
}
}
// main is the main function.
function main() {
const usageAndExit = (exitcode) => {
console.log("usage: node ./QA/index.mjs list")
console.log("usage: node ./QA/index.mjs run [test_case_name...]")
process.exit(exitcode)
}
if (process.argv.length < 3) {
usageAndExit(0)
}
checkForDuplicateTestCases()
const command = process.argv[2]
switch (command) {
case "list":
commandList()
break
case "run":
commandRun(process.argv.slice(3))
break
default:
console.log(`unknown command: ${command}`)
usageAndExit(1)
}
}
main()

196
QA/lib/analysis.mjs
View File

@ -1,196 +0,0 @@
// This file contains code for analysing results.
import { type } from "os"
// checkAnnotations checks whether we have annotations.
function checkAnnotations(report) {
let result = true
const isObject = typeof (report.annotations) === "object"
console.log(`checking whether annotations is an object... ${isObject}`)
result = result && isObject
const hasArchitecture = typeof (report.annotations.architecture) === "string"
console.log(`checking whether annotations contains architecture... ${hasArchitecture}`)
result = result && hasArchitecture
const hasEngineName = typeof (report.annotations.engine_name) === "string"
console.log(`checking whether annotations contains engine_name... ${hasEngineName}`)
result = result && hasEngineName
const hasEngineVersion = typeof (report.annotations.engine_version) === "string"
console.log(`checking whether annotations contains engine_version... ${hasEngineVersion}`)
result = result && hasEngineVersion
const hasPlatform = typeof (report.annotations.platform) === "string"
console.log(`checking whether annotations contains platform... ${hasPlatform}`)
result = result && hasPlatform
return result
}
// checkDataFormatVersion checks whether we have data_format_version.
function checkDataFormatVersion(report) {
const result = report.data_format_version === "0.2.0"
console.log(`checking whether we have the right data format version... ${result}`)
return result
}
// checkExtensions ensures that extensions exists and has the right type.
function checkExtensions(report) {
const result = typeof (report.extensions) === "object"
console.log(`checking whether report.extensions is an object... ${result}`)
// Quirk: some experiments (e.g. web_connectivity) don't include
// an extensions object, and we don't want to fail here.
return true
}
// checkInput ensures that the input is correct.
function checkInput(testCase, report) {
const result = testCase.input === report.input
console.log(`checking whether input is correct... ${result}`)
return result
}
// checkExperimentName ensures that the experimentName is correctly
// set in the output report file.
function checkExperimentName(name, report) {
const result = report.test_name === name
console.log(`checking whether the experiment name is correct... ${result}`)
return result
}
// checkStartTime ensures that a given start time is in the correct format.
function checkStartTime(report, key) {
const value = report[key]
let result = typeof (value) === "string"
console.log(`checking whether ${key} is a string... ${result}`)
result = result && value.match(/^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$/) !== null
console.log(`checking whether ${key} matches the date regexp... ${result}`)
return result
}
// checkASN ensures that an ASN field is correct.
function checkASN(report, key) {
const value = report[key]
let result = typeof (value) === "string"
console.log(`checking whether ${key} is a string... ${result}`)
result = result && value.match(/^AS[0-9]+$/) !== null
console.log(`checking whether ${key} matches the ASN regexp... ${result}`)
return result
}
// checkProbeCC ensures that the probe_cc field is correct.
function checkProbeCC(report) {
const value = report["probe_cc"]
let result = typeof (value) === "string"
console.log(`checking whether probe_cc is a string... ${result}`)
result = result && value.match(/^[A-Z]{2}$/) !== null
console.log(`checking whether probe_cc matches the CC regexp... ${result}`)
return result
}
// checkProbeIP ensures that the probe_ip field is correct.
function checkProbeIP(report) {
const result = report.probe_ip === "127.0.0.1"
console.log(`checking whether probe_ip is correct... ${result}`)
return result
}
// checkReportID ensures that the report_id field is correct.
function checkReportID(report) {
const result = report.report_id === "" // note: we don't submit
console.log(`checking whether report_id is correct... ${result}`)
return result
}
// checkString ensures that a field is a string.
function checkString(report, key) {
const value = report[key]
const result = typeof(value) === "string"
console.log(`checking whether ${key} is a string... ${result}`)
return result
}
// checkNetworkName ensures that an xxx_network_name field is correct.
function checkNetworkName(report, key) {
return checkString(report, key)
}
// checkResolverIP ensures that the resolver_ip field is correct.
function checkResolverIP(report) {
return checkString(report, "resolver_ip")
}
// checkSoftwareName ensures that the software_name field is correct.
function checkSoftwareName(report) {
return checkString(report, "software_name")
}
// checkSoftwareVersion ensures that the software_version field is correct.
function checkSoftwareVersion(report) {
return checkString(report, "software_version")
}
// checkTestRuntime ensures that the test_runtime field is correct.
function checkTestRuntime(report) {
const result = typeof(report.test_runtime) === "number"
console.log(`checking whether test_runtime is a number... ${result}`)
return result
}
// checkTestVersion ensures that the test_version field is correct.
function checkTestVersion(report) {
return checkString(report, "test_version")
}
// checkTestKeys ensures that test_keys is an object.
function checkTestKeys(report) {
const result = typeof(report.test_keys) === "object"
console.log(`checking whether test_keys is an object... ${result}`)
return result
}
// checkMeasurement is a function that invokes a standard set of checks
// to validate the top-level test keys of a result.
//
// This function helps to implement per-experiment checkers.
//
// Arguments:
//
// - testCase is the current test case
//
// - name is the name of the current experiment
//
// - report is the JSON measurement
//
// - extraChecks is the optional callback to perform extra checks
// that takes in input the report's test keys.
export function checkMeasurement(testCase, name, report, extraChecks) {
let result = true
result = result && checkAnnotations(report)
result = result && checkDataFormatVersion(report)
result = result && checkExtensions(report)
result = result && checkInput(testCase, report)
result = result && checkStartTime(report, "measurement_start_time")
result = result && checkASN(report, "probe_asn")
result = result && checkProbeCC(report)
result = result && checkProbeIP(report)
result = result && checkNetworkName(report, "probe_network_name")
result = result && checkReportID(report)
result = result && checkASN(report, "resolver_asn")
result = result && checkResolverIP(report)
result = result && checkNetworkName(report, "resolver_network_name")
result = result && checkSoftwareName(report)
result = result && checkSoftwareVersion(report)
result = result && checkExperimentName(name, report)
result = result && checkTestRuntime(report)
result = result && checkStartTime(report, "test_start_time")
result = result && checkTestVersion(report)
result = result && checkTestKeys(report)
if (typeof extraChecks === "function") {
result = result && extraChecks(report.test_keys)
}
return result
}

17
QA/lib/blocking.mjs
View File

@ -1,17 +0,0 @@
// This file contains helpers for describing blocking rules.
// hijackPopularDNSServers returns an object containing the rules
// for hijacking popular DNS servers with `miniooni --censor`.
export function hijackPopularDNSServers() {
return {
// cloudflare
"1.1.1.1:53/udp": "hijack-dns",
"1.0.0.1:53/udp": "hijack-dns",
// google
"8.8.8.8:53/udp": "hijack-dns",
"8.8.4.4:53/udp": "hijack-dns",
// quad9
"9.9.9.9:53/udp": "hijack-dns",
"9.9.9.10:53/udp": "hijack-dns",
}
}

92
QA/lib/runner.mjs
View File

@ -1,92 +0,0 @@
// This file contains code for running test cases.
import child_process from "child_process"
import crypto from "crypto"
import fs from "fs"
import path from "path"
// tempFile returns the name of a temporary file. This function is
// not as secure as using mktemp but it does not matter in this
// context. We just need to create file names in the local directory
// with enough entropy that every run has a different name.
//
// See https://stackoverflow.com/questions/7055061 for an insightful
// discussion about creating temporary files in node.
function tempFile(suffix) {
return path.join(`tmp-${crypto.randomBytes(16).toString('hex')}.${suffix}`)
}
// exec executes a command. This function throws on failure. The stdout
// and stderr should be console.log-ed once the command returns.
function exec(command) {
console.log(`+ ${command}`)
child_process.execSync(command)
}
// writeCensorJsonFile writes a censor.json file using a file name
// containing random characters and returns the file name.
function writeCensorJsonFile(testCase) {
const fileName = tempFile("json")
fs.writeFileSync(fileName, JSON.stringify(testCase.blocking))
return fileName
}
// readReportFile reads and parses the report file, thus returning
// the JSON object contained inside the report file.
function readReportFile(reportFile) {
const data = fs.readFileSync(reportFile, { "encoding": "utf-8" })
return JSON.parse(data)
}
// runExperiment runs the given test case with the given experiment.
function runExperiment(testCase, experiment, checker) {
console.log(`## running: ${testCase.name}.${experiment}`)
console.log("")
const censorJson = writeCensorJsonFile(testCase)
const reportJson = tempFile("json")
// Note: using -n because we don't want to submit QA checks.
exec(`./miniooni -n --censor ${censorJson} -o ${reportJson} -i ${testCase.input} ${experiment}`)
console.log("")
// TODO(bassosimone): support multiple entries per file
const report = readReportFile(reportJson)
const analysisResult = checker(testCase, experiment, report)
console.log("")
console.log("")
if (analysisResult !== true && analysisResult !== false) {
console.log("the analysis function returned neither true nor false")
process.exit(1)
}
return analysisResult
}
// recompileMiniooni recompiles miniooni if needed.
function recompileMiniooni() {
exec("go build -v ./internal/cmd/miniooni")
}
// runTestCase runs the given test case.
//
// A test case is an object with the following fields:
//
// - name (string): the name of the test case;
//
// - description (string): a description of the test case;
//
// - input (string): the input to pass to the experiment;
//
// - blocking (object): a blocking specification (i.e., the
// serialization of a filtering.TProxyConfig struct);
//
// - experiments (object): the keys are names of nettests
// to run and the values are functions taking as their
// unique argument the experiment's test_keys.
export function runTestCase(testCase) {
recompileMiniooni()
console.log("")
console.log(`# running: ${testCase.name}`)
let result = true
for (const [name, checker] of Object.entries(testCase.experiments)) {
result = result && runExperiment(testCase, name, checker)
}
return result
}

641
QA/lib/web.mjs
View File

@ -1,641 +0,0 @@
// This file contains test cases for web nettests.
import { checkMeasurement } from "./analysis.mjs"
// webConnectivityCheckTopLevel checks the top-level keys
// of the web connectivity experiment agains a template
// object. Returns true if they match, false on mismatch.
function webConnectivityCheckTopLevel(tk, template) {
let result = true
for (const [key, value] of Object.entries(template)) {
const check = tk[key] === value
console.log(`checking whether ${key}'s value is ${value}... ${check}`)
result = result && check
}
return result
}
// Here we export all the test cases. Please, see the documentation
// of runner.runTestCase for a description of what is a test case.
export const testCases = [
//
// DNS checks
//
// We start with checks where _only_ the system resolver fails.
//
{
name: "web_dns_system_nxdomain",
description: "the system resolver returns NXDOMAIN",
input: "https://nexa.polito.it/",
blocking: {
Domains: {
"nexa.polito.it": "nxdomain"
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": "dns_nxdomain_error",
"dns_consistency": "inconsistent",
"control_failure": null,
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "dns",
})
return result
})
},
},
},
{
name: "web_dns_system_refused",
description: "the system resolver returns REFUSED",
input: "https://nexa.polito.it/",
blocking: {
Domains: {
"nexa.polito.it": "refused"
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": "dns_refused_error",
"dns_consistency": "inconsistent",
"control_failure": null,
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": null,
"blocking": null, // TODO(bassosimone): this is clearly a bug
})
return result
})
},
},
},
{
name: "web_dns_system_localhost",
description: "the system resolver returns localhost",
input: "https://nexa.polito.it/",
blocking: {
Domains: {
"nexa.polito.it": "localhost",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
// TODO(bassosimone): Web Connectivity does not correctly handle this case
// but, still, correctly sets blocking as "dns"
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "inconsistent",
"control_failure": null,
"http_experiment_failure": "connection_refused",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "dns",
})
return result
})
},
},
},
{
name: "web_dns_system_bogon_not_localhost",
description: "the system resolver returns a bogon that is not localhost",
input: "https://nexa.polito.it/",
blocking: {
DNSCache: {
"nexa.polito.it": ["10.0.0.1"],
},
Domains: {
"nexa.polito.it": "cache",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
// TODO(bassosimone): Web Connectivity does not correctly handle this case
// but, still, correctly sets blocking as "dns"
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "inconsistent",
"control_failure": null,
"http_experiment_failure": "generic_timeout_error",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "dns",
})
return result
})
},
},
},
{
name: "web_dns_system_no_answer",
description: "the system resolver returns an empty answer",
input: "https://nexa.polito.it/",
blocking: {
Domains: {
"nexa.polito.it": "no-answer",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": "dns_no_answer",
"dns_consistency": "inconsistent",
"control_failure": null,
"http_experiment_failure": null,
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": null,
"blocking": null, // TODO(bassosimone): this is clearly a bug
})
return result
})
},
},
},
{
name: "web_dns_system_timeout",
description: "the system resolver times out",
input: "https://nexa.polito.it/",
blocking: {
Domains: {
"nexa.polito.it": "timeout",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": "generic_timeout_error",
"dns_consistency": "inconsistent",
"control_failure": null,
"http_experiment_failure": null,
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": null,
"blocking": null, // TODO(bassosimone): this is clearly a bug
})
return result
})
},
},
},
// TODO(bassosimone): here we should insert more checks where not only the system
// resolver is blocked but also other resolvers are.
//
// TCP connect
//
// This section contains TCP connect failures.
//
{
name: "web_tcp_connect_timeout",
description: "timeout when connecting to the IP address",
input: "https://nexa.polito.it/",
blocking: {
Endpoints: {
"130.192.16.171:443/tcp": "tcp-drop-syn",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": "generic_timeout_error",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "tcp_ip",
})
return result
})
},
},
},
{
name: "web_tcp_connect_refused",
description: "connection refused when connecting to the IP address",
input: "https://nexa.polito.it/",
blocking: {
Endpoints: {
"130.192.16.171:443/tcp": "tcp-reject-syn",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": "connection_refused",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "tcp_ip",
})
return result
})
},
},
},
//
// TLS handshake
//
// This section contains TLS handshake failures.
//
{
name: "web_tls_handshake_timeout",
description: "timeout when performing the TLS handshake",
input: "https://nexa.polito.it/",
blocking: {
Endpoints: {
"130.192.16.171:443/tcp": "drop-data",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": "generic_timeout_error",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "http-failure",
})
return result
})
},
},
},
{
name: "web_tls_handshake_reset",
description: "reset when performing the TLS handshake",
input: "https://nexa.polito.it/",
blocking: {
Endpoints: {
"130.192.16.171:443/tcp": "hijack-tls",
},
SNIs: {
"nexa.polito.it": "reset",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": "connection_reset",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "http-failure",
})
return result
})
},
},
},
//
// QUIC
//
{
name: "web_quic_handshake_timeout",
description: "timeout when performing the QUIC handshake",
input: "https://dns.google/",
blocking: {
Endpoints: {
"8.8.8.8:443/udp": "drop-data",
"8.8.4.4:443/udp": "drop-data",
"[2001:4860:4860::8888]:443/udp": "drop-data",
"[2001:4860:4860::8844]:443/udp": "drop-data",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
},
},
//
// Cleartext HTTP
//
{
name: "web_http_reset",
description: "reset when performing the HTTP round trip",
input: "http://nexa.polito.it/",
blocking: {
Endpoints: {
"130.192.16.171:80/tcp": "hijack-http",
},
Hosts: {
"nexa.polito.it": "reset",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": "connection_reset",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "http-failure",
})
return result
})
},
},
},
{
name: "web_http_timeout",
description: "timeout when performing the HTTP round trip",
input: "http://nexa.polito.it/",
blocking: {
Endpoints: {
"130.192.16.171:80/tcp": "hijack-http",
},
Hosts: {
"nexa.polito.it": "timeout",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": "generic_timeout_error",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "http-failure",
})
return result
})
},
},
},
{
name: "web_http_451",
description: "451 when performing the HTTP round trip",
input: "http://nexa.polito.it/",
blocking: {
Endpoints: {
"130.192.16.171:80/tcp": "hijack-http",
},
Hosts: {
"nexa.polito.it": "451",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
// TODO(bassosimone): there is no easy way to check for the body
// proportion robustly because it's a float.
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": null,
"body_length_match": false,
"status_code_match": false,
"headers_match": false,
"title_match": false,
"accessible": false,
"blocking": "http-diff",
})
return result
})
},
},
},
//
// More complex scenarios
//
// In this scenario the second IP address for the domain fails
// with reset. Web Connectivity sees that but overall says it's
// all good because the good IP happens to be the first. We'll
// see what changes if we swap the IPs in the next scenario.
{
name: "web_tcp_second_ip_connection_reset",
description: "the second IP returned by DNS fails with connection reset",
input: "https://dns.google/",
blocking: {
DNSCache: {
"dns.google": ["8.8.4.4", "8.8.8.8"],
},
Domains: {
"dns.google": "cache",
},
Endpoints: {
"8.8.8.8:443/tcp": "hijack-tls",
},
SNIs: {
"dns.google": "reset",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": null,
"body_length_match": true,
"body_proportion": 1,
"status_code_match": true,
"headers_match": true,
"title_match": true,
"accessible": true,
"blocking": false,
})
return result
})
},
},
},
// This scenario is like the previous one except that we swap
// the IP addresses and now Web Connectivity says failure.
{
name: "web_tcp_first_ip_connection_reset",
description: "the first IP returned by DNS fails with connection reset",
input: "https://dns.google/",
blocking: {
DNSCache: {
"dns.google": ["8.8.4.4", "8.8.8.8"],
},
Domains: {
"dns.google": "cache",
},
Endpoints: {
"8.8.4.4:443/tcp": "hijack-tls",
},
SNIs: {
"dns.google": "reset",
},
},
experiments: {
websteps: (testCase, name, report) => {
return checkMeasurement(testCase, name, report)
},
web_connectivity: (testCase, name, report) => {
return checkMeasurement(testCase, name, report, (tk) => {
let result = true
result = result && webConnectivityCheckTopLevel(tk, {
"dns_experiment_failure": null,
"dns_consistency": "consistent",
"control_failure": null,
"http_experiment_failure": "connection_reset",
"body_length_match": null,
"body_proportion": 0,
"status_code_match": null,
"headers_match": null,
"title_match": null,
"accessible": false,
"blocking": "http-failure",
})
return result
})
},
},
},
]

2
go.mod
View File

@ -20,7 +20,6 @@ require (
github.com/hexops/gotextdiff v1.0.3 github.com/hexops/gotextdiff v1.0.3
github.com/iancoleman/strcase v0.2.0 github.com/iancoleman/strcase v0.2.0
github.com/lucas-clemente/quic-go v0.27.0 github.com/lucas-clemente/quic-go v0.27.0
github.com/marten-seemann/qtls-go1-18 v0.1.1
github.com/mattn/go-colorable v0.1.12 github.com/mattn/go-colorable v0.1.12
github.com/miekg/dns v1.1.49 github.com/miekg/dns v1.1.49
github.com/mitchellh/go-wordwrap v1.0.1 github.com/mitchellh/go-wordwrap v1.0.1
@ -73,6 +72,7 @@ require (
github.com/marten-seemann/qpack v0.2.1 // indirect github.com/marten-seemann/qpack v0.2.1 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-sqlite3 v1.14.13 // indirect github.com/mattn/go-sqlite3 v1.14.13 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect

86
internal/archival/dialer.go
View File

@ -1,86 +0,0 @@
package archival
//
// Saves dial and net.Conn events
//
import (
"context"
"net"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// NetworkEvent contains a network event. This kind of events
// are generated by Dialer, QUICDialer, Conn, QUICConn.
type NetworkEvent struct {
Count int
Failure error
Finished time.Time
Network string
Operation string
RemoteAddr string
Started time.Time
}
// DialContext dials with the given dialer with the given arguments
// and stores the dial result inside of this saver.
func (s *Saver) DialContext(ctx context.Context,
dialer model.Dialer, network, address string) (net.Conn, error) {
started := time.Now()
conn, err := dialer.DialContext(ctx, network, address)
s.appendNetworkEvent(&NetworkEvent{
Count: 0,
Failure: err,
Finished: time.Now(),
Network: network,
Operation: netxlite.ConnectOperation,
RemoteAddr: address,
Started: started,
})
return conn, err
}
// Read reads from the given conn and stores the results in the saver.
func (s *Saver) Read(conn net.Conn, buf []byte) (int, error) {
network := conn.RemoteAddr().Network()
remoteAddr := conn.RemoteAddr().String()
started := time.Now()
count, err := conn.Read(buf)
s.appendNetworkEvent(&NetworkEvent{
Count: count,
Failure: err,
Finished: time.Now(),
Network: network,
Operation: netxlite.ReadOperation,
RemoteAddr: remoteAddr,
Started: started,
})
return count, err
}
// Write writes to the given conn and stores the results into the saver.
func (s *Saver) Write(conn net.Conn, buf []byte) (int, error) {
network := conn.RemoteAddr().Network()
remoteAddr := conn.RemoteAddr().String()
started := time.Now()
count, err := conn.Write(buf)
s.appendNetworkEvent(&NetworkEvent{
Count: count,
Failure: err,
Finished: time.Now(),
Network: network,
Operation: netxlite.WriteOperation,
RemoteAddr: remoteAddr,
Started: started,
})
return count, err
}
func (s *Saver) appendNetworkEvent(ev *NetworkEvent) {
s.mu.Lock()
s.trace.Network = append(s.trace.Network, ev)
s.mu.Unlock()
}

281
internal/archival/dialer_test.go
View File

@ -1,281 +0,0 @@
package archival
import (
"context"
"errors"
"io"
"net"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestSaverDialContext(t *testing.T) {
// newConn creates a new connection with the desired properties.
newConn := func(address string) net.Conn {
return &mocks.Conn{
MockRemoteAddr: func() net.Addr {
return &mocks.Addr{
MockString: func() string {
return address
},
}
},
MockClose: func() error {
return nil
},
}
}
// newDialer creates a dialer for testing.
newDialer := func(conn net.Conn, err error) model.Dialer {
return &mocks.Dialer{
MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
time.Sleep(1 * time.Microsecond)
return conn, err
},
}
}
t.Run("on success", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
dialer := newDialer(newConn(mockedEndpoint), nil)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: 0,
ExpectedErr: nil,
ExpectedNetwork: "tcp",
ExpectedOp: netxlite.ConnectOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
ctx := context.Background()
conn, err := saver.DialContext(ctx, dialer, "tcp", mockedEndpoint)
if err != nil {
t.Fatal(err)
}
if conn == nil {
t.Fatal("expected non-nil conn")
}
conn.Close()
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
dialer := newDialer(nil, mockedError)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: 0,
ExpectedErr: mockedError,
ExpectedNetwork: "tcp",
ExpectedOp: netxlite.ConnectOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
ctx := context.Background()
conn, err := saver.DialContext(ctx, dialer, "tcp", mockedEndpoint)
if !errors.Is(err, mockedError) {
t.Fatal("unexpected err", err)
}
if conn != nil {
t.Fatal("expected nil conn")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
}
func TestSaverRead(t *testing.T) {
// newConn is a helper function for creating a new connection.
newConn := func(endpoint string, numBytes int, err error) net.Conn {
return &mocks.Conn{
MockRead: func(b []byte) (int, error) {
time.Sleep(time.Microsecond)
return numBytes, err
},
MockRemoteAddr: func() net.Addr {
return &mocks.Addr{
MockString: func() string {
return endpoint
},
MockNetwork: func() string {
return "tcp"
},
}
},
}
}
t.Run("on success", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
const mockedNumBytes = 128
conn := newConn(mockedEndpoint, mockedNumBytes, nil)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: mockedNumBytes,
ExpectedErr: nil,
ExpectedNetwork: "tcp",
ExpectedOp: netxlite.ReadOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
buf := make([]byte, 1024)
count, err := saver.Read(conn, buf)
if err != nil {
t.Fatal(err)
}
if count != mockedNumBytes {
t.Fatal("unexpected count")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
conn := newConn(mockedEndpoint, 0, mockedError)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: 0,
ExpectedErr: mockedError,
ExpectedNetwork: "tcp",
ExpectedOp: netxlite.ReadOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
buf := make([]byte, 1024)
count, err := saver.Read(conn, buf)
if !errors.Is(err, mockedError) {
t.Fatal("unexpected err", err)
}
if count != 0 {
t.Fatal("unexpected count")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
}
func TestSaverWrite(t *testing.T) {
// newConn is a helper function for creating a new connection.
newConn := func(endpoint string, numBytes int, err error) net.Conn {
return &mocks.Conn{
MockWrite: func(b []byte) (int, error) {
time.Sleep(time.Microsecond)
return numBytes, err
},
MockRemoteAddr: func() net.Addr {
return &mocks.Addr{
MockString: func() string {
return endpoint
},
MockNetwork: func() string {
return "tcp"
},
}
},
}
}
t.Run("on success", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
const mockedNumBytes = 128
conn := newConn(mockedEndpoint, mockedNumBytes, nil)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: mockedNumBytes,
ExpectedErr: nil,
ExpectedNetwork: "tcp",
ExpectedOp: netxlite.WriteOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
buf := make([]byte, 1024)
count, err := saver.Write(conn, buf)
if err != nil {
t.Fatal(err)
}
if count != mockedNumBytes {
t.Fatal("unexpected count")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
conn := newConn(mockedEndpoint, 0, mockedError)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: 0,
ExpectedErr: mockedError,
ExpectedNetwork: "tcp",
ExpectedOp: netxlite.WriteOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
buf := make([]byte, 1024)
count, err := saver.Write(conn, buf)
if !errors.Is(err, mockedError) {
t.Fatal("unexpected err", err)
}
if count != 0 {
t.Fatal("unexpected count")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
}
// SingleNetworkEventValidator expects to find a single
// network event inside of the saver and ensures that such
// an event contains the required field values.
type SingleNetworkEventValidator struct {
ExpectedCount int
ExpectedErr error
ExpectedNetwork string
ExpectedOp string
ExpectedEpnt string
Saver *Saver
}
func (v *SingleNetworkEventValidator) Validate() error {
trace := v.Saver.MoveOutTrace()
if len(trace.Network) != 1 {
return errors.New("expected to see a single .Network event")
}
entry := trace.Network[0]
if entry.Count != v.ExpectedCount {
return errors.New("expected to see a different .Count")
}
if !errors.Is(entry.Failure, v.ExpectedErr) {
return errors.New("unexpected .Failure")
}
if !entry.Finished.After(entry.Started) {
return errors.New(".Finished should be after .Started")
}
if entry.Network != v.ExpectedNetwork {
return errors.New("invalid value for .Network")
}
if entry.Operation != v.ExpectedOp {
return errors.New("invalid value for .Operation")
}
if entry.RemoteAddr != v.ExpectedEpnt {
return errors.New("unexpected value for .RemoteAddr")
}
return nil
}

4
internal/archival/doc.go
View File

@ -1,4 +0,0 @@
// Package implements a Saver type that saves network, TCP, DNS,
// and TLS events. Given a Saver, you can export a Trace. Given a
// Trace you can obtain data in the OONI archival data format.
package archival

107
internal/archival/http.go
View File

@ -1,107 +0,0 @@
package archival
//
// Saves HTTP events
//
import (
"bytes"
"io"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// HTTPRoundTripEvent contains an HTTP round trip.
type HTTPRoundTripEvent struct {
Failure error
Finished time.Time
Method string
RequestHeaders http.Header
ResponseBody []byte
ResponseBodyIsTruncated bool
ResponseBodyLength int64
ResponseHeaders http.Header
Started time.Time
StatusCode int64
Transport string
URL string
}
// HTTPRoundTrip performs the round trip with the given transport and
// the given arguments and saves the results into the saver.
//
// The maxBodySnapshotSize argument controls the maximum size of the
// body snapshot that we collect along with the HTTP round trip.
func (s *Saver) HTTPRoundTrip(
txp model.HTTPTransport, maxBodySnapshotSize int64,
req *http.Request) (*http.Response, error) {
started := time.Now()
resp, err := txp.RoundTrip(req)
rt := &HTTPRoundTripEvent{
Failure: nil, // set later
Finished: time.Time{}, // set later
Method: req.Method,
RequestHeaders: s.cloneRequestHeaders(req),
ResponseBody: nil, // set later
ResponseBodyIsTruncated: false,
ResponseBodyLength: 0,
ResponseHeaders: nil, // set later
Started: started,
StatusCode: 0, // set later
Transport: txp.Network(),
URL: req.URL.String(),
}
if err != nil {
rt.Finished = time.Now()
rt.Failure = err
s.appendHTTPRoundTripEvent(rt)
return nil, err
}
rt.StatusCode = int64(resp.StatusCode)
rt.ResponseHeaders = resp.Header.Clone()
r := io.LimitReader(resp.Body, maxBodySnapshotSize)
body, err := netxlite.ReadAllContext(req.Context(), r)
if err != nil {
rt.Finished = time.Now()
rt.Failure = err
s.appendHTTPRoundTripEvent(rt)
return nil, err
}
resp.Body = &archivalHTTPTransportBody{ // allow for reading again the whole body
Reader: io.MultiReader(bytes.NewReader(body), resp.Body),
Closer: resp.Body,
}
rt.ResponseBody = body
rt.ResponseBodyLength = int64(len(body))
rt.ResponseBodyIsTruncated = int64(len(body)) >= maxBodySnapshotSize
rt.Finished = time.Now()
s.appendHTTPRoundTripEvent(rt)
return resp, nil
}
// cloneRequestHeaders ensure we include the Host header among the saved
// headers, which is what OONI should do, even though the Go transport is
// such that this header is added later when we're sending the request.
func (s *Saver) cloneRequestHeaders(req *http.Request) http.Header {
header := req.Header.Clone()
if req.Host != "" {
header.Set("Host", req.Host)
} else {
header.Set("Host", req.URL.Host)
}
return header
}
type archivalHTTPTransportBody struct {
io.Reader
io.Closer
}
func (s *Saver) appendHTTPRoundTripEvent(ev *HTTPRoundTripEvent) {
s.mu.Lock()
s.trace.HTTPRoundTrip = append(s.trace.HTTPRoundTrip, ev)
s.mu.Unlock()
}

354
internal/archival/http_test.go
View File

@ -1,354 +0,0 @@
package archival
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func TestSaverHTTPRoundTrip(t *testing.T) {
// newHTTPTransport creates a new HTTP transport for testing.
newHTTPTransport := func(resp *http.Response, err error) model.HTTPTransport {
return &mocks.HTTPTransport{
MockRoundTrip: func(req *http.Request) (*http.Response, error) {
return resp, err
},
MockNetwork: func() string {
return "tcp"
},
}
}
// successFlowWithBody is a successful test case with possible body truncation.
successFlowWithBody := func(realBody []byte, maxBodySize int64) error {
// truncate the expected body if required
expectedBody := realBody
truncated := false
if int64(len(realBody)) > maxBodySize {
expectedBody = realBody[:maxBodySize]
truncated = true
}
// construct the saver and the validator
saver := NewSaver()
v := &SingleHTTPRoundTripValidator{
ExpectFailure: nil,
ExpectMethod: "GET",
ExpectRequestHeaders: map[string][]string{
"Host": {"127.0.0.1:8080"},
"User-Agent": {"antani/1.0"},
"X-Client-IP": {"130.192.91.211"},
},
ExpectResponseBody: expectedBody,
ExpectResponseBodyIsTruncated: truncated,
ExpectResponseBodyLength: int64(len(expectedBody)),
ExpectResponseHeaders: map[string][]string{
"Server": {"antani/1.0"},
"Content-Type": {"text/plain"},
},
ExpectStatusCode: 200,
ExpectTransport: "tcp",
ExpectURL: "http://127.0.0.1:8080/antani",
RealResponseBody: realBody,
Saver: saver,
}
// construct transport and perform the HTTP round trip
txp := newHTTPTransport(v.NewHTTPResponse(), nil)
resp, err := saver.HTTPRoundTrip(txp, maxBodySize, v.NewHTTPRequest())
if err != nil {
return err
}
if resp == nil {
return errors.New("expected non-nil resp")
}
// ensure that we can still read the _full_ response body
ctx := context.Background()
data, err := netxlite.ReadAllContext(ctx, resp.Body)
if err != nil {
return err
}
if diff := cmp.Diff(realBody, data); diff != "" {
return errors.New(diff)
}
// validate the content of the trace
return v.Validate()
}
t.Run("on success without truncation", func(t *testing.T) {
realBody := []byte("0xdeadbeef")
const maxBodySize = 1 << 20
err := successFlowWithBody(realBody, maxBodySize)
if err != nil {
t.Fatal(err)
}
})
t.Run("on success with truncation", func(t *testing.T) {
realBody := []byte("0xdeadbeef")
const maxBodySize = 4
err := successFlowWithBody(realBody, maxBodySize)
if err != nil {
t.Fatal(err)
}
})
t.Run("on failure during round trip", func(t *testing.T) {
expectedError := netxlite.NewTopLevelGenericErrWrapper(netxlite.ECONNRESET)
const maxBodySize = 1 << 20
saver := NewSaver()
v := &SingleHTTPRoundTripValidator{
ExpectFailure: expectedError,
ExpectMethod: "GET",
ExpectRequestHeaders: map[string][]string{
"Host": {"127.0.0.1:8080"},
"User-Agent": {"antani/1.0"},
"X-Client-IP": {"130.192.91.211"},
},
ExpectResponseBody: nil,
ExpectResponseBodyIsTruncated: false,
ExpectResponseBodyLength: 0,
ExpectResponseHeaders: nil,
ExpectStatusCode: 0,
ExpectTransport: "tcp",
ExpectURL: "http://127.0.0.1:8080/antani",
RealResponseBody: nil,
Saver: saver,
}
txp := newHTTPTransport(nil, expectedError)
resp, err := saver.HTTPRoundTrip(txp, maxBodySize, v.NewHTTPRequest())
if !errors.Is(err, expectedError) {
t.Fatal(err)
}
if resp != nil {
t.Fatal("expected nil resp")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure reading body", func(t *testing.T) {
expectedError := netxlite.NewTopLevelGenericErrWrapper(netxlite.ECONNRESET)
const maxBodySize = 1 << 20
saver := NewSaver()
v := &SingleHTTPRoundTripValidator{
ExpectFailure: expectedError,
ExpectMethod: "GET",
ExpectRequestHeaders: map[string][]string{
"Host": {"127.0.0.1:8080"},
"User-Agent": {"antani/1.0"},
"X-Client-IP": {"130.192.91.211"},
},
ExpectResponseBody: nil,
ExpectResponseBodyIsTruncated: false,
ExpectResponseBodyLength: 0,
ExpectResponseHeaders: map[string][]string{
"Server": {"antani/1.0"},
"Content-Type": {"text/plain"},
},
ExpectStatusCode: 200,
ExpectTransport: "tcp",
ExpectURL: "http://127.0.0.1:8080/antani",
RealResponseBody: nil,
Saver: saver,
}
resp := v.NewHTTPResponse()
// Hack the body so it returns a connection reset error
// after some useful piece of data. We do not see any
// body in the response or in the trace. We may possibly
// want to include all the body we could read into the
// trace in the future, but for now it seems fine to do
// exactly what the previous code was doing.
resp.Body = io.NopCloser(io.MultiReader(
bytes.NewReader([]byte("0xdeadbeef")),
&mocks.Reader{
MockRead: func(b []byte) (int, error) {
return 0, expectedError
},
},
))
txp := newHTTPTransport(resp, nil)
resp, err := saver.HTTPRoundTrip(txp, maxBodySize, v.NewHTTPRequest())
if !errors.Is(err, expectedError) {
t.Fatal("unexpected err", err)
}
if resp != nil {
t.Fatal("expected nil resp")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("cloneRequestHeaders", func(t *testing.T) {
// doWithRequest is an helper function that creates a suitable
// round tripper and returns trace.HTTPRoundTrip[0] for inspection
doWithRequest := func(req *http.Request) (*HTTPRoundTripEvent, error) {
expect := errors.New("mocked err")
txp := newHTTPTransport(nil, expect)
saver := NewSaver()
const maxBodySize = 1 << 20 // irrelevant
resp, err := saver.HTTPRoundTrip(txp, maxBodySize, req)
if !errors.Is(err, expect) {
return nil, fmt.Errorf("unexpected error: %w", err)
}
if resp != nil {
return nil, errors.New("expected nil resp")
}
trace := saver.MoveOutTrace()
if len(trace.HTTPRoundTrip) != 1 {
return nil, errors.New("expected exactly one HTTPRoundTrip")
}
return trace.HTTPRoundTrip[0], nil
}
t.Run("with req.URL.Host", func(t *testing.T) {
req, err := http.NewRequest("GET", "https://x.org/", nil)
if err != nil {
t.Fatal(err)
}
ev, err := doWithRequest(req)
if err != nil {
t.Fatal(err)
}
if ev.RequestHeaders.Get("Host") != "x.org" {
t.Fatal("unexpected request host")
}
})
t.Run("with req.Host", func(t *testing.T) {
req, err := http.NewRequest("GET", "https://x.org/", nil)
if err != nil {
t.Fatal(err)
}
req.Host = "google.com"
ev, err := doWithRequest(req)
if err != nil {
t.Fatal(err)
}
if ev.RequestHeaders.Get("Host") != "google.com" {
t.Fatal("unexpected request host")
}
})
})
}
type SingleHTTPRoundTripValidator struct {
ExpectFailure error
ExpectMethod string
ExpectRequestHeaders http.Header
ExpectResponseBody []byte
ExpectResponseBodyIsTruncated bool
ExpectResponseBodyLength int64
ExpectResponseHeaders http.Header
ExpectStatusCode int64
ExpectTransport string
ExpectURL string
RealResponseBody []byte
Saver *Saver
}
func (v *SingleHTTPRoundTripValidator) NewHTTPRequest() *http.Request {
parsedURL, err := url.Parse(v.ExpectURL)
runtimex.PanicOnError(err, "url.Parse should not fail here")
// The saving code clones the headers and adds the host header, which
// Go would instead add later. So, a realistic mock should not include
// such an header inside of the http.Request.
clonedHeaders := v.ExpectRequestHeaders.Clone()
clonedHeaders.Del("Host")
return &http.Request{
Method: v.ExpectMethod,
URL: parsedURL,
Proto: "",
ProtoMajor: 0,
ProtoMinor: 0,
Header: clonedHeaders,
Body: nil,
GetBody: nil,
ContentLength: 0,
TransferEncoding: nil,
Close: false,
Host: "",
Form: nil,
PostForm: nil,
MultipartForm: nil,
Trailer: nil,
RemoteAddr: "",
RequestURI: "",
TLS: nil,
Cancel: nil,
Response: nil,
}
}
func (v *SingleHTTPRoundTripValidator) NewHTTPResponse() *http.Response {
body := io.NopCloser(bytes.NewReader(v.RealResponseBody))
return &http.Response{
Status: http.StatusText(int(v.ExpectStatusCode)),
StatusCode: int(v.ExpectStatusCode),
Proto: "",
ProtoMajor: 0,
ProtoMinor: 0,
Header: v.ExpectResponseHeaders,
Body: body,
ContentLength: 0,
TransferEncoding: nil,
Close: false,
Uncompressed: false,
Trailer: nil,
Request: nil,
TLS: nil,
}
}
func (v *SingleHTTPRoundTripValidator) Validate() error {
trace := v.Saver.MoveOutTrace()
if len(trace.HTTPRoundTrip) != 1 {
return errors.New("expected to see one event")
}
entry := trace.HTTPRoundTrip[0]
if !errors.Is(entry.Failure, v.ExpectFailure) {
return errors.New("unexpected .Failure")
}
if !entry.Finished.After(entry.Started) {
return errors.New(".Finished is not after .Started")
}
if entry.Method != v.ExpectMethod {
return errors.New("unexpected .Method")
}
if diff := cmp.Diff(v.ExpectRequestHeaders, entry.RequestHeaders); diff != "" {
return errors.New(diff)
}
if diff := cmp.Diff(v.ExpectResponseBody, entry.ResponseBody); diff != "" {
return errors.New(diff)
}
if entry.ResponseBodyIsTruncated != v.ExpectResponseBodyIsTruncated {
return errors.New("unexpected .ResponseBodyIsTruncated")
}
if entry.ResponseBodyLength != v.ExpectResponseBodyLength {
return errors.New("unexpected .ResponseBodyLength")
}
if diff := cmp.Diff(v.ExpectResponseHeaders, entry.ResponseHeaders); diff != "" {
return errors.New(diff)
}
if entry.StatusCode != v.ExpectStatusCode {
return errors.New("unexpected .StatusCode")
}
if entry.Transport != v.ExpectTransport {
return errors.New("unexpected .Transport")
}
if entry.URL != v.ExpectURL {
return errors.New("unexpected .URL")
}
return nil
}

91
internal/archival/quic.go
View File

@ -1,91 +0,0 @@
package archival
//
// Saves QUIC events.
//
import (
"context"
"crypto/tls"
"net"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// WriteTo performs WriteTo with the given pconn and saves the
// operation's results inside the saver.
func (s *Saver) WriteTo(pconn model.UDPLikeConn, buf []byte, addr net.Addr) (int, error) {
started := time.Now()
count, err := pconn.WriteTo(buf, addr)
s.appendNetworkEvent(&NetworkEvent{
Count: count,
Failure: err,
Finished: time.Now(),
Network: addr.Network(),
Operation: netxlite.WriteToOperation,
RemoteAddr: addr.String(),
Started: started,
})
return count, err
}
// ReadFrom performs ReadFrom with the given pconn and saves the
// operation's results inside the saver.
func (s *Saver) ReadFrom(pconn model.UDPLikeConn, buf []byte) (int, net.Addr, error) {
started := time.Now()
count, addr, err := pconn.ReadFrom(buf)
s.appendNetworkEvent(&NetworkEvent{
Count: count,
Failure: err,
Finished: time.Now(),
Network: "udp", // must be always set even on failure
Operation: netxlite.ReadFromOperation,
RemoteAddr: s.safeAddrString(addr),
Started: started,
})
return count, addr, err
}
func (s *Saver) safeAddrString(addr net.Addr) (out string) {
if addr != nil {
out = addr.String()
}
return
}
// QUICDialContext dials a QUIC session using the given dialer
// and saves the results inside of the saver.
func (s *Saver) QUICDialContext(ctx context.Context, dialer model.QUICDialer,
network, address string, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) {
started := time.Now()
var state tls.ConnectionState
sess, err := dialer.DialContext(ctx, network, address, tlsConfig, quicConfig)
if err == nil {
<-sess.HandshakeComplete().Done() // robustness (the dialer already does that)
state = sess.ConnectionState().TLS.ConnectionState
}
s.appendQUICHandshake(&QUICTLSHandshakeEvent{
ALPN: tlsConfig.NextProtos,
CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite),
Failure: err,
Finished: time.Now(),
NegotiatedProto: state.NegotiatedProtocol,
Network: "quic",
PeerCerts: s.tlsPeerCerts(err, &state),
RemoteAddr: address,
SNI: tlsConfig.ServerName,
SkipVerify: tlsConfig.InsecureSkipVerify,
Started: started,
TLSVersion: netxlite.TLSVersionString(state.Version),
})
return sess, err
}
func (s *Saver) appendQUICHandshake(ev *QUICTLSHandshakeEvent) {
s.mu.Lock()
s.trace.QUICHandshake = append(s.trace.QUICHandshake, ev)
s.mu.Unlock()
}

420
internal/archival/quic_test.go
View File

@ -1,420 +0,0 @@
package archival
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"io"
"net"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/lucas-clemente/quic-go"
"github.com/marten-seemann/qtls-go1-18" // it's annoying to depend on that
"github.com/ooni/probe-cli/v3/internal/fakefill"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestSaverWriteTo(t *testing.T) {
// newAddr creates an new net.Addr for testing.
newAddr := func(endpoint string) net.Addr {
return &mocks.Addr{
MockString: func() string {
return endpoint
},
MockNetwork: func() string {
return "udp"
},
}
}
// newConn is a helper function for creating a new connection.
newConn := func(numBytes int, err error) model.UDPLikeConn {
return &mocks.UDPLikeConn{
MockWriteTo: func(p []byte, addr net.Addr) (int, error) {
time.Sleep(time.Microsecond)
return numBytes, err
},
}
}
t.Run("on success", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
const mockedNumBytes = 128
addr := newAddr(mockedEndpoint)
conn := newConn(mockedNumBytes, nil)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: mockedNumBytes,
ExpectedErr: nil,
ExpectedNetwork: "udp",
ExpectedOp: netxlite.WriteToOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
buf := make([]byte, 1024)
count, err := saver.WriteTo(conn, buf, addr)
if err != nil {
t.Fatal(err)
}
if count != mockedNumBytes {
t.Fatal("invalid count")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
addr := newAddr(mockedEndpoint)
conn := newConn(0, mockedError)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: 0,
ExpectedErr: mockedError,
ExpectedNetwork: "udp",
ExpectedOp: netxlite.WriteToOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
buf := make([]byte, 1024)
count, err := saver.WriteTo(conn, buf, addr)
if !errors.Is(err, mockedError) {
t.Fatal("unexpected err", err)
}
if count != 0 {
t.Fatal("invalid count")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
}
func TestSaverReadFrom(t *testing.T) {
// newAddr creates an new net.Addr for testing.
newAddr := func(endpoint string) net.Addr {
return &mocks.Addr{
MockString: func() string {
return endpoint
},
MockNetwork: func() string {
return "udp"
},
}
}
// newConn is a helper function for creating a new connection.
newConn := func(numBytes int, addr net.Addr, err error) model.UDPLikeConn {
return &mocks.UDPLikeConn{
MockReadFrom: func(p []byte) (int, net.Addr, error) {
time.Sleep(time.Microsecond)
return numBytes, addr, err
},
}
}
t.Run("on success", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
const mockedNumBytes = 128
expectedAddr := newAddr(mockedEndpoint)
conn := newConn(mockedNumBytes, expectedAddr, nil)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: mockedNumBytes,
ExpectedErr: nil,
ExpectedNetwork: "udp",
ExpectedOp: netxlite.ReadFromOperation,
ExpectedEpnt: mockedEndpoint,
Saver: saver,
}
buf := make([]byte, 1024)
count, addr, err := saver.ReadFrom(conn, buf)
if err != nil {
t.Fatal(err)
}
if expectedAddr.Network() != addr.Network() {
t.Fatal("invalid addr.Network")
}
if expectedAddr.String() != addr.String() {
t.Fatal("invalid addr.String")
}
if count != mockedNumBytes {
t.Fatal("invalid count")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure", func(t *testing.T) {
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
conn := newConn(0, nil, mockedError)
saver := NewSaver()
v := &SingleNetworkEventValidator{
ExpectedCount: 0,
ExpectedErr: mockedError,
ExpectedNetwork: "udp",
ExpectedOp: netxlite.ReadFromOperation,
ExpectedEpnt: "",
Saver: saver,
}
buf := make([]byte, 1024)
count, addr, err := saver.ReadFrom(conn, buf)
if !errors.Is(err, mockedError) {
t.Fatal(err)
}
if addr != nil {
t.Fatal("invalid addr")
}
if count != 0 {
t.Fatal("invalid count")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
}
func TestSaverQUICDialContext(t *testing.T) {
// newQUICDialer creates a new QUICDialer for testing.
newQUICDialer := func(qconn quic.EarlyConnection, err error) model.QUICDialer {
return &mocks.QUICDialer{
MockDialContext: func(
ctx context.Context, network, address string, tlsConfig *tls.Config,
quicConfig *quic.Config) (quic.EarlyConnection, error) {
time.Sleep(time.Microsecond)
return qconn, err
},
}
}
// newQUICConnection creates a new quic.EarlyConnection for testing.
newQUICConnection := func(handshakeComplete context.Context, state tls.ConnectionState) quic.EarlyConnection {
return &mocks.QUICEarlyConnection{
MockHandshakeComplete: func() context.Context {
return handshakeComplete
},
MockConnectionState: func() quic.ConnectionState {
return quic.ConnectionState{
TLS: qtls.ConnectionStateWith0RTT{
ConnectionState: state,
},
}
},
MockCloseWithError: func(code quic.ApplicationErrorCode, reason string) error {
return nil
},
}
}
t.Run("on success", func(t *testing.T) {
handshakeCtx := context.Background()
handshakeCtx, handshakeCancel := context.WithCancel(handshakeCtx)
handshakeCancel() // simulate a completed handshake
const expectedNetwork = "udp"
const mockedEndpoint = "8.8.4.4:443"
saver := NewSaver()
var peerCerts [][]byte
ff := &fakefill.Filler{}
ff.Fill(&peerCerts)
if len(peerCerts) < 1 {
t.Fatal("did not fill peerCerts")
}
v := &SingleQUICTLSHandshakeValidator{
ExpectedALPN: []string{"h3"},
ExpectedSNI: "dns.google",
ExpectedSkipVerify: true,
//
ExpectedCipherSuite: tls.TLS_AES_128_GCM_SHA256,
ExpectedNegotiatedProtocol: "h3",
ExpectedPeerCerts: peerCerts,
ExpectedVersion: tls.VersionTLS13,
//
ExpectedNetwork: "quic",
ExpectedRemoteAddr: mockedEndpoint,
//
QUICConfig: &quic.Config{},
//
ExpectedFailure: nil,
Saver: saver,
}
qconn := newQUICConnection(handshakeCtx, v.NewTLSConnectionState())
dialer := newQUICDialer(qconn, nil)
ctx := context.Background()
qconn, err := saver.QUICDialContext(ctx, dialer, expectedNetwork,
mockedEndpoint, v.NewTLSConfig(), v.QUICConfig)
if err != nil {
t.Fatal(err)
}
if qconn == nil {
t.Fatal("expected nil qconn")
}
qconn.CloseWithError(0, "")
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on other error", func(t *testing.T) {
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
const expectedNetwork = "udp"
const mockedEndpoint = "8.8.4.4:443"
saver := NewSaver()
v := &SingleQUICTLSHandshakeValidator{
ExpectedALPN: []string{"h3"},
ExpectedSNI: "dns.google",
ExpectedSkipVerify: true,
//
ExpectedCipherSuite: 0,
ExpectedNegotiatedProtocol: "",
ExpectedPeerCerts: nil,
ExpectedVersion: 0,
//
ExpectedNetwork: "quic",
ExpectedRemoteAddr: mockedEndpoint,
//
QUICConfig: &quic.Config{},
//
ExpectedFailure: mockedError,
Saver: saver,
}
dialer := newQUICDialer(nil, mockedError)
ctx := context.Background()
qconn, err := saver.QUICDialContext(ctx, dialer, expectedNetwork,
mockedEndpoint, v.NewTLSConfig(), v.QUICConfig)
if !errors.Is(err, mockedError) {
t.Fatal("unexpected error")
}
if qconn != nil {
t.Fatal("expected nil connection")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
// TODO(bassosimone): here we're not testing the case in which
// the certificate is invalid for the required SNI.
//
// We need first to figure out whether this is what happens
// when we validate for QUIC in such cases. If that's the case
// indeed, then we can write the tests.
t.Run("on x509.HostnameError", func(t *testing.T) {
t.Skip("test not implemented")
})
t.Run("on x509.UnknownAuthorityError", func(t *testing.T) {
t.Skip("test not implemented")
})
t.Run("on x509.CertificateInvalidError", func(t *testing.T) {
t.Skip("test not implemented")
})
}
type SingleQUICTLSHandshakeValidator struct {
// related to the tls.Config
ExpectedALPN []string
ExpectedSNI string
ExpectedSkipVerify bool
// related to the tls.ConnectionState
ExpectedCipherSuite uint16
ExpectedNegotiatedProtocol string
ExpectedPeerCerts [][]byte
ExpectedVersion uint16
// related to the mocked conn (TLS) / dial params (QUIC)
ExpectedNetwork string
ExpectedRemoteAddr string
// tells us whether we're using QUIC
QUICConfig *quic.Config
// other fields
ExpectedFailure error
Saver *Saver
}
func (v *SingleQUICTLSHandshakeValidator) NewTLSConfig() *tls.Config {
return &tls.Config{
NextProtos: v.ExpectedALPN,
ServerName: v.ExpectedSNI,
InsecureSkipVerify: v.ExpectedSkipVerify,
}
}
func (v *SingleQUICTLSHandshakeValidator) NewTLSConnectionState() tls.ConnectionState {
var state tls.ConnectionState
if v.ExpectedCipherSuite != 0 {
state.CipherSuite = v.ExpectedCipherSuite
}
if v.ExpectedNegotiatedProtocol != "" {
state.NegotiatedProtocol = v.ExpectedNegotiatedProtocol
}
for _, cert := range v.ExpectedPeerCerts {
state.PeerCertificates = append(state.PeerCertificates, &x509.Certificate{
Raw: cert,
})
}
if v.ExpectedVersion != 0 {
state.Version = v.ExpectedVersion
}
return state
}
func (v *SingleQUICTLSHandshakeValidator) Validate() error {
trace := v.Saver.MoveOutTrace()
var entries []*QUICTLSHandshakeEvent
if v.QUICConfig != nil {
entries = trace.QUICHandshake
} else {
entries = trace.TLSHandshake
}
if len(entries) != 1 {
return errors.New("expected to see a single entry")
}
entry := entries[0]
if diff := cmp.Diff(entry.ALPN, v.ExpectedALPN); diff != "" {
return errors.New(diff)
}
if entry.CipherSuite != netxlite.TLSCipherSuiteString(v.ExpectedCipherSuite) {
return errors.New("unexpected .CipherSuite")
}
if !errors.Is(entry.Failure, v.ExpectedFailure) {
return errors.New("unexpected .Failure")
}
if !entry.Finished.After(entry.Started) {
return errors.New(".Finished is not after .Started")
}
if entry.NegotiatedProto != v.ExpectedNegotiatedProtocol {
return errors.New("unexpected .NegotiatedProto")
}
if entry.Network != v.ExpectedNetwork {
return errors.New("unexpected .Network")
}
if diff := cmp.Diff(entry.PeerCerts, v.ExpectedPeerCerts); diff != "" {
return errors.New("unexpected .PeerCerts")
}
if entry.RemoteAddr != v.ExpectedRemoteAddr {
return errors.New("unexpected .RemoteAddr")
}
if entry.SNI != v.ExpectedSNI {
return errors.New("unexpected .ServerName")
}
if entry.SkipVerify != v.ExpectedSkipVerify {
return errors.New("unexpected .SkipVerify")
}
if entry.TLSVersion != netxlite.TLSVersionString(v.ExpectedVersion) {
return errors.New("unexpected .Version")
}
return nil
}

123
internal/archival/resolver.go
View File

@ -1,123 +0,0 @@
package archival
//
// Saves DNS lookup events
//
import (
"context"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
)
// DNSLookupEvent contains the results of a DNS lookup.
type DNSLookupEvent struct {
ALPNs []string
Addresses []string
Domain string
Failure error
Finished time.Time
LookupType string
ResolverAddress string
ResolverNetwork string
Started time.Time
}
// LookupHost performs a host lookup with the given resolver
// and saves the results into the saver.
func (s *Saver) LookupHost(ctx context.Context, reso model.Resolver, domain string) ([]string, error) {
started := time.Now()
addrs, err := reso.LookupHost(ctx, domain)
s.appendLookupHostEvent(&DNSLookupEvent{
ALPNs: nil,
Addresses: addrs,
Domain: domain,
Failure: err,
Finished: time.Now(),
LookupType: "getaddrinfo",
ResolverAddress: reso.Address(),
ResolverNetwork: reso.Network(),
Started: started,
})
return addrs, err
}
func (s *Saver) appendLookupHostEvent(ev *DNSLookupEvent) {
s.mu.Lock()
s.trace.DNSLookupHost = append(s.trace.DNSLookupHost, ev)
s.mu.Unlock()
}
// LookupHTTPS performs an HTTPSSvc-record lookup using the given
// resolver and saves the results into the saver.
func (s *Saver) LookupHTTPS(ctx context.Context, reso model.Resolver, domain string) (*model.HTTPSSvc, error) {
started := time.Now()
https, err := reso.LookupHTTPS(ctx, domain)
s.appendLookupHTTPSEvent(&DNSLookupEvent{
ALPNs: s.safeALPNs(https),
Addresses: s.safeAddresses(https),
Domain: domain,
Failure: err,
Finished: time.Now(),
LookupType: "https",
ResolverAddress: reso.Address(),
ResolverNetwork: reso.Network(),
Started: started,
})
return https, err
}
func (s *Saver) appendLookupHTTPSEvent(ev *DNSLookupEvent) {
s.mu.Lock()
s.trace.DNSLookupHTTPS = append(s.trace.DNSLookupHTTPS, ev)
s.mu.Unlock()
}
func (s *Saver) safeALPNs(https *model.HTTPSSvc) (out []string) {
if https != nil {
out = https.ALPN
}
return
}
func (s *Saver) safeAddresses(https *model.HTTPSSvc) (out []string) {
if https != nil {
out = append(out, https.IPv4...)
out = append(out, https.IPv6...)
}
return
}
// DNSRoundTripEvent contains the result of a DNS round trip.
type DNSRoundTripEvent struct {
Address string
Failure error
Finished time.Time
Network string
Query []byte
Reply []byte
Started time.Time
}
// DNSRoundTrip implements ArchivalSaver.DNSRoundTrip.
func (s *Saver) DNSRoundTrip(ctx context.Context, txp model.DNSTransport, query []byte) ([]byte, error) {
started := time.Now()
reply, err := txp.RoundTrip(ctx, query)
s.appendDNSRoundTripEvent(&DNSRoundTripEvent{
Address: txp.Address(),
Failure: err,
Finished: time.Now(),
Network: txp.Network(),
Query: query,
Reply: reply,
Started: started,
})
return reply, err
}
func (s *Saver) appendDNSRoundTripEvent(ev *DNSRoundTripEvent) {
s.mu.Lock()
s.trace.DNSRoundTrip = append(s.trace.DNSRoundTrip, ev)
s.mu.Unlock()
}

347
internal/archival/resolver_test.go
View File

@ -1,347 +0,0 @@
package archival
import (
"context"
"errors"
"io"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/fakefill"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestSaverLookupHost(t *testing.T) {
// newResolver helps to create a new resolver.
newResolver := func(addrs []string, err error) model.Resolver {
return &mocks.Resolver{
MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
return addrs, err
},
MockAddress: func() string {
return "8.8.8.8:53"
},
MockNetwork: func() string {
return "udp"
},
}
}
t.Run("on success", func(t *testing.T) {
const domain = "dns.google"
expectAddrs := []string{"8.8.8.8", "8.8.4.4"}
saver := NewSaver()
v := &SingleDNSLookupValidator{
ExpectALPNs: nil,
ExpectAddrs: expectAddrs,
ExpectDomain: domain,
ExpectLookupType: "getaddrinfo",
ExpectFailure: nil,
ExpectResolverAddress: "8.8.8.8:53",
ExpectResolverNetwork: "udp",
Saver: saver,
}
reso := newResolver(expectAddrs, nil)
ctx := context.Background()
addrs, err := saver.LookupHost(ctx, reso, domain)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(expectAddrs, addrs); diff != "" {
t.Fatal(diff)
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure", func(t *testing.T) {
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
const domain = "dns.google"
saver := NewSaver()
v := &SingleDNSLookupValidator{
ExpectALPNs: nil,
ExpectAddrs: nil,
ExpectDomain: domain,
ExpectLookupType: "getaddrinfo",
ExpectFailure: mockedError,
ExpectResolverAddress: "8.8.8.8:53",
ExpectResolverNetwork: "udp",
Saver: saver,
}
reso := newResolver(nil, mockedError)
ctx := context.Background()
addrs, err := saver.LookupHost(ctx, reso, domain)
if !errors.Is(err, mockedError) {
t.Fatal("invalid err", err)
}
if len(addrs) != 0 {
t.Fatal("invalid addrs")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
}
func TestSaverLookupHTTPS(t *testing.T) {
// newResolver helps to create a new resolver.
newResolver := func(alpns, ipv4, ipv6 []string, err error) model.Resolver {
return &mocks.Resolver{
MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) {
if alpns == nil && ipv4 == nil && ipv6 == nil {
return nil, err
}
return &model.HTTPSSvc{
ALPN: alpns,
IPv4: ipv4,
IPv6: ipv6,
}, err
},
MockAddress: func() string {
return "8.8.8.8:53"
},
MockNetwork: func() string {
return "udp"
},
}
}
t.Run("on success", func(t *testing.T) {
const domain = "dns.google"
expectALPN := []string{"h3", "h2", "http/1.1"}
expectA := []string{"8.8.8.8", "8.8.4.4"}
expectAAAA := []string{"2001:4860:4860::8844"}
expectAddrs := append(expectA, expectAAAA...)
saver := NewSaver()
v := &SingleDNSLookupValidator{
ExpectALPNs: expectALPN,
ExpectAddrs: expectAddrs,
ExpectDomain: domain,
ExpectLookupType: "https",
ExpectFailure: nil,
ExpectResolverAddress: "8.8.8.8:53",
ExpectResolverNetwork: "udp",
Saver: saver,
}
reso := newResolver(expectALPN, expectA, expectAAAA, nil)
ctx := context.Background()
https, err := saver.LookupHTTPS(ctx, reso, domain)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(expectALPN, https.ALPN); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(expectA, https.IPv4); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(expectAAAA, https.IPv6); diff != "" {
t.Fatal(diff)
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure", func(t *testing.T) {
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
const domain = "dns.google"
saver := NewSaver()
v := &SingleDNSLookupValidator{
ExpectALPNs: nil,
ExpectAddrs: nil,
ExpectDomain: domain,
ExpectLookupType: "https",
ExpectFailure: mockedError,
ExpectResolverAddress: "8.8.8.8:53",
ExpectResolverNetwork: "udp",
Saver: saver,
}
reso := newResolver(nil, nil, nil, mockedError)
ctx := context.Background()
https, err := saver.LookupHTTPS(ctx, reso, domain)
if !errors.Is(err, mockedError) {
t.Fatal("unexpected err", err)
}
if https != nil {
t.Fatal("expected nil https")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
}
type SingleDNSLookupValidator struct {
ExpectALPNs []string
ExpectAddrs []string
ExpectDomain string
ExpectLookupType string
ExpectFailure error
ExpectResolverAddress string
ExpectResolverNetwork string
Saver *Saver
}
func (v *SingleDNSLookupValidator) Validate() error {
trace := v.Saver.MoveOutTrace()
var entries []*DNSLookupEvent
switch v.ExpectLookupType {
case "getaddrinfo":
entries = trace.DNSLookupHost
case "https":
entries = trace.DNSLookupHTTPS
default:
return errors.New("invalid v.ExpectLookupType")
}
if len(entries) != 1 {
return errors.New("expected a single entry")
}
entry := entries[0]
if diff := cmp.Diff(v.ExpectALPNs, entry.ALPNs); diff != "" {
return errors.New(diff)
}
if diff := cmp.Diff(v.ExpectAddrs, entry.Addresses); diff != "" {
return errors.New(diff)
}
if v.ExpectDomain != entry.Domain {
return errors.New("invalid .Domain value")
}
if !errors.Is(entry.Failure, v.ExpectFailure) {
return errors.New("invalid .Failure value")
}
if !entry.Finished.After(entry.Started) {
return errors.New(".Finished is not after .Started")
}
if entry.ResolverAddress != v.ExpectResolverAddress {
return errors.New("invalid .ResolverAddress value")
}
if entry.ResolverNetwork != v.ExpectResolverNetwork {
return errors.New("invalid .ResolverNetwork value")
}
return nil
}
func TestSaverDNSRoundTrip(t *testing.T) {
// generateQueryAndReply generates a fake query and reply.
generateQueryAndReply := func() (query, reply []byte, err error) {
ff := &fakefill.Filler{}
ff.Fill(&query)
ff.Fill(&reply)
if len(query) < 1 || len(reply) < 1 {
return nil, nil, errors.New("did not generate query or reply")
}
return query, reply, nil
}
// newDNSTransport creates a suitable DNSTransport.
newDNSTransport := func(reply []byte, err error) model.DNSTransport {
return &mocks.DNSTransport{
MockRoundTrip: func(ctx context.Context, query []byte) ([]byte, error) {
return reply, err
},
MockNetwork: func() string {
return "udp"
},
MockAddress: func() string {
return "8.8.8.8:53"
},
}
}
t.Run("on success", func(t *testing.T) {
query, expectedReply, err := generateQueryAndReply()
if err != nil {
t.Fatal(err)
}
saver := NewSaver()
v := &SingleDNSRoundTripValidator{
ExpectAddress: "8.8.8.8:53",
ExpectFailure: nil,
ExpectNetwork: "udp",
ExpectQuery: query,
ExpectReply: expectedReply,
Saver: saver,
}
ctx := context.Background()
txp := newDNSTransport(expectedReply, nil)
reply, err := saver.DNSRoundTrip(ctx, txp, query)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(expectedReply, reply); diff != "" {
t.Fatal(diff)
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
t.Run("on failure", func(t *testing.T) {
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
query, _, err := generateQueryAndReply()
if err != nil {
t.Fatal(err)
}
saver := NewSaver()
v := &SingleDNSRoundTripValidator{
ExpectAddress: "8.8.8.8:53",
ExpectFailure: mockedError,
ExpectNetwork: "udp",
ExpectQuery: query,
ExpectReply: nil,
Saver: saver,
}
ctx := context.Background()
txp := newDNSTransport(nil, mockedError)
reply, err := saver.DNSRoundTrip(ctx, txp, query)
if !errors.Is(err, mockedError) {
t.Fatal(err)
}
if len(reply) != 0 {
t.Fatal("unexpected reply")
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
}
type SingleDNSRoundTripValidator struct {
ExpectAddress string
ExpectFailure error
ExpectNetwork string
ExpectQuery []byte
ExpectReply []byte
Saver *Saver
}
func (v *SingleDNSRoundTripValidator) Validate() error {
trace := v.Saver.MoveOutTrace()
if len(trace.DNSRoundTrip) != 1 {
return errors.New("expected a single entry")
}
entry := trace.DNSRoundTrip[0]
if v.ExpectAddress != entry.Address {
return errors.New("invalid .Address")
}
if !errors.Is(entry.Failure, v.ExpectFailure) {
return errors.New("invalid .Failure value")
}
if !entry.Finished.After(entry.Started) {
return errors.New(".Finished is not after .Started")
}
if v.ExpectNetwork != entry.Network {
return errors.New("invalid .Network value")
}
if diff := cmp.Diff(v.ExpectQuery, entry.Query); diff != "" {
return errors.New(diff)
}
if diff := cmp.Diff(v.ExpectReply, entry.Reply); diff != "" {
return errors.New(diff)
}
return nil
}

40
internal/archival/saver.go
View File

@ -1,40 +0,0 @@
package archival
//
// Saver implementation
//
import (
"sync"
)
// Saver allows to save network, DNS, QUIC, TLS, HTTP events.
//
// You MUST use NewSaver to create a new instance.
type Saver struct {
// mu provides mutual exclusion.
mu sync.Mutex
// trace is the current trace.
trace *Trace
}
// NewSaver creates a new Saver instance.
//
// You MUST use this function to create a Saver.
func NewSaver() *Saver {
return &Saver{
mu: sync.Mutex{},
trace: &Trace{},
}
}
// MoveOutTrace moves the current trace out of the saver and
// creates a new empty trace inside it.
func (as *Saver) MoveOutTrace() *Trace {
as.mu.Lock()
t := as.trace
as.trace = &Trace{}
as.mu.Unlock()
return t
}

37
internal/archival/saver_test.go
View File

@ -1,37 +0,0 @@
package archival
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/fakefill"
)
func TestSaverNewSaver(t *testing.T) {
saver := NewSaver()
if saver.trace == nil {
t.Fatal("expected non-nil trace here")
}
}
func TestSaverMoveOutTrace(t *testing.T) {
saver := NewSaver()
var ev DNSRoundTripEvent
ff := &fakefill.Filler{}
ff.Fill(&ev)
if len(ev.Query) < 1 {
t.Fatal("did not fill") // be sure
}
saver.appendDNSRoundTripEvent(&ev)
trace := saver.MoveOutTrace()
if len(saver.trace.DNSRoundTrip) != 0 {
t.Fatal("expected zero length")
}
if len(trace.DNSRoundTrip) != 1 {
t.Fatal("expected one entry")
}
entry := trace.DNSRoundTrip[0]
if diff := cmp.Diff(&ev, entry); diff != "" {
t.Fatal(diff)
}
}

89
internal/archival/tls.go
View File

@ -1,89 +0,0 @@
package archival
//
// Saves TLS events
//
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"net"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// QUICTLSHandshakeEvent contains a QUIC or TLS handshake event.
type QUICTLSHandshakeEvent struct {
ALPN []string
CipherSuite string
Failure error
Finished time.Time
NegotiatedProto string
Network string
PeerCerts [][]byte
RemoteAddr string
SNI string
SkipVerify bool
Started time.Time
TLSVersion string
}
// TLSHandshake performs a TLS handshake with the given handshaker
// and saves the results into the saver.
func (s *Saver) TLSHandshake(ctx context.Context, thx model.TLSHandshaker,
conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) {
network := conn.RemoteAddr().Network()
remoteAddr := conn.RemoteAddr().String()
started := time.Now()
tconn, state, err := thx.Handshake(ctx, conn, config)
// Implementation note: state is an empty ConnectionState on failure
// so it's safe to access its fields also in that case
s.appendTLSHandshake(&QUICTLSHandshakeEvent{
ALPN: config.NextProtos,
CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite),
Failure: err,
Finished: time.Now(),
NegotiatedProto: state.NegotiatedProtocol,
Network: network,
PeerCerts: s.tlsPeerCerts(err, &state),
RemoteAddr: remoteAddr,
SNI: config.ServerName,
SkipVerify: config.InsecureSkipVerify,
Started: started,
TLSVersion: netxlite.TLSVersionString(state.Version),
})
return tconn, state, err
}
func (s *Saver) appendTLSHandshake(ev *QUICTLSHandshakeEvent) {
s.mu.Lock()
s.trace.TLSHandshake = append(s.trace.TLSHandshake, ev)
s.mu.Unlock()
}
func (s *Saver) tlsPeerCerts(err error, state *tls.ConnectionState) (out [][]byte) {
var x509HostnameError x509.HostnameError
if errors.As(err, &x509HostnameError) {
// Test case: https://wrong.host.badssl.com/
return [][]byte{x509HostnameError.Certificate.Raw}
}
var x509UnknownAuthorityError x509.UnknownAuthorityError
if errors.As(err, &x509UnknownAuthorityError) {
// Test case: https://self-signed.badssl.com/. This error has
// never been among the ones returned by MK.
return [][]byte{x509UnknownAuthorityError.Cert.Raw}
}
var x509CertificateInvalidError x509.CertificateInvalidError
if errors.As(err, &x509CertificateInvalidError) {
// Test case: https://expired.badssl.com/
return [][]byte{x509CertificateInvalidError.Cert.Raw}
}
for _, cert := range state.PeerCertificates {
out = append(out, cert.Raw)
}
return
}

178
internal/archival/tls_test.go
View File

@ -1,178 +0,0 @@
package archival
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"net"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/ooni/probe-cli/v3/internal/fakefill"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
func TestSaverTLSHandshake(t *testing.T) {
// newTLSHandshaker helps with building a TLS handshaker
newTLSHandshaker := func(tlsConn net.Conn, state tls.ConnectionState, err error) model.TLSHandshaker {
return &mocks.TLSHandshaker{
MockHandshake: func(ctx context.Context, tcpConn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) {
time.Sleep(1 * time.Microsecond)
return tlsConn, state, err
},
}
}
// newTCPConn creates a suitable net.Conn
newTCPConn := func(address string) net.Conn {
return &mocks.Conn{
MockRemoteAddr: func() net.Addr {
return &mocks.Addr{
MockString: func() string {
return address
},
MockNetwork: func() string {
return "tcp"
},
}
},
MockClose: func() error {
return nil
},
}
}
t.Run("on success", func(t *testing.T) {
const mockedEndpoint = "8.8.4.4:443"
var certs [][]byte
ff := &fakefill.Filler{}
ff.Fill(&certs)
if len(certs) < 1 {
t.Fatal("did not fill certs")
}
saver := NewSaver()
v := &SingleQUICTLSHandshakeValidator{
ExpectedALPN: []string{"h2", "http/1.1"},
ExpectedSNI: "dns.google",
ExpectedSkipVerify: true,
//
ExpectedCipherSuite: tls.TLS_AES_128_GCM_SHA256,
ExpectedNegotiatedProtocol: "h2",
ExpectedPeerCerts: certs,
ExpectedVersion: tls.VersionTLS12,
//
ExpectedNetwork: "tcp",
ExpectedRemoteAddr: mockedEndpoint,
//
QUICConfig: nil, // this is not QUIC
ExpectedFailure: nil,
Saver: saver,
}
expectedState := v.NewTLSConnectionState()
thx := newTLSHandshaker(newTCPConn(mockedEndpoint), expectedState, nil)
ctx := context.Background()
tcpConn := newTCPConn(mockedEndpoint)
conn, state, err := saver.TLSHandshake(ctx, thx, tcpConn, v.NewTLSConfig())
if conn == nil {
t.Fatal("expected non-nil conn")
}
conn.Close()
if diff := cmp.Diff(expectedState, state, cmpopts.IgnoreUnexported(tls.ConnectionState{})); diff != "" {
t.Fatal(diff)
}
if err != nil {
t.Fatal(err)
}
if err := v.Validate(); err != nil {
t.Fatal(err)
}
})
// failureFlow is the flow we run on failure.
failureFlow := func(mockedError error, peerCerts [][]byte) error {
const mockedEndpoint = "8.8.4.4:443"
saver := NewSaver()
v := &SingleQUICTLSHandshakeValidator{
ExpectedALPN: []string{"h2", "http/1.1"},
ExpectedSNI: "dns.google",
ExpectedSkipVerify: true,
//
ExpectedCipherSuite: 0,
ExpectedNegotiatedProtocol: "",
ExpectedPeerCerts: peerCerts,
ExpectedVersion: 0,
//
ExpectedNetwork: "tcp",
ExpectedRemoteAddr: mockedEndpoint,
//
QUICConfig: nil, // this is not QUIC
ExpectedFailure: mockedError,
Saver: saver,
}
expectedState := v.NewTLSConnectionState()
thx := newTLSHandshaker(nil, expectedState, mockedError)
ctx := context.Background()
tcpConn := newTCPConn(mockedEndpoint)
conn, state, err := saver.TLSHandshake(ctx, thx, tcpConn, v.NewTLSConfig())
if conn != nil {
return errors.New("expected nil conn")
}
if diff := cmp.Diff(expectedState, state, cmpopts.IgnoreUnexported(tls.ConnectionState{})); diff != "" {
return errors.New(diff)
}
if !errors.Is(err, mockedError) {
return fmt.Errorf("unexpected err: %w", err)
}
return v.Validate()
}
t.Run("on generic failure", func(t *testing.T) {
mockedError := netxlite.NewTopLevelGenericErrWrapper(io.EOF)
if err := failureFlow(mockedError, nil); err != nil {
t.Fatal(err)
}
})
t.Run("on x509.HostnameError", func(t *testing.T) {
var certificate []byte
ff := &fakefill.Filler{}
ff.Fill(&certificate)
mockedError := x509.HostnameError{
Certificate: &x509.Certificate{Raw: certificate},
}
if err := failureFlow(mockedError, [][]byte{certificate}); err != nil {
t.Fatal(err)
}
})
t.Run("on x509.UnknownAuthorityError", func(t *testing.T) {
var certificate []byte
ff := &fakefill.Filler{}
ff.Fill(&certificate)
mockedError := x509.UnknownAuthorityError{
Cert: &x509.Certificate{Raw: certificate},
}
if err := failureFlow(mockedError, [][]byte{certificate}); err != nil {
t.Fatal(err)
}
})
t.Run("on x509.CertificateInvalidError", func(t *testing.T) {
var certificate []byte
ff := &fakefill.Filler{}
ff.Fill(&certificate)
mockedError := x509.CertificateInvalidError{
Cert: &x509.Certificate{Raw: certificate},
}
if err := failureFlow(mockedError, [][]byte{certificate}); err != nil {
t.Fatal(err)
}
})
}

283
internal/archival/trace.go
View File

@ -1,283 +0,0 @@
package archival
import (
"net"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
//
// Trace implementation
//
// Trace contains the events.
type Trace struct {
// DNSLookupHTTPS contains DNSLookupHTTPS events.
DNSLookupHTTPS []*DNSLookupEvent
// DNSLookupHost contains DNSLookupHost events.
DNSLookupHost []*DNSLookupEvent
// DNSRoundTrip contains DNSRoundTrip events.
DNSRoundTrip []*DNSRoundTripEvent
// HTTPRoundTrip contains HTTPRoundTrip round trip events.
HTTPRoundTrip []*HTTPRoundTripEvent
// Network contains network events.
Network []*NetworkEvent
// QUICHandshake contains QUICHandshake handshake events.
QUICHandshake []*QUICTLSHandshakeEvent
// TLSHandshake contains TLSHandshake handshake events.
TLSHandshake []*QUICTLSHandshakeEvent
}
func (t *Trace) newFailure(err error) (out *string) {
if err != nil {
s := err.Error()
out = &s
}
return
}
//
// TCP connect
//
// NewArchivalTCPConnectResultList builds a TCP connect list in the OONI archival
// data format out of the results saved inside the trace.
func (t *Trace) NewArchivalTCPConnectResultList(begin time.Time) (out []model.ArchivalTCPConnectResult) {
for _, ev := range t.Network {
if ev.Operation != netxlite.ConnectOperation || ev.Network != "tcp" {
continue
}
// We assume Go is passing us legit data structures
ip, sport, _ := net.SplitHostPort(ev.RemoteAddr)
iport, _ := strconv.Atoi(sport)
out = append(out, model.ArchivalTCPConnectResult{
IP: ip,
Port: iport,
Status: model.ArchivalTCPConnectStatus{
Blocked: nil, // Web Connectivity only, depends on the control
Failure: t.newFailure(ev.Failure),
Success: ev.Failure == nil,
},
T: ev.Finished.Sub(begin).Seconds(),
})
}
return
}
//
// HTTP
//
// NewArchivalHTTPRequestResultList builds an HTTP requests list in the OONI
// archival data format out of the results saved inside the trace.
//
// This function will sort the emitted list of requests such that the last
// request that happened in time is the first one to be emitted. If the
// measurement code performs related requests sequentially (which is a kinda a
// given because you cannot follow a redirect before reading the previous request),
// then the result is sorted how the OONI pipeline expects it to be.
func (t *Trace) NewArchivalHTTPRequestResultList(begin time.Time) (out []model.ArchivalHTTPRequestResult) {
for _, ev := range t.HTTPRoundTrip {
out = append(out, model.ArchivalHTTPRequestResult{
Failure: t.newFailure(ev.Failure),
Request: model.ArchivalHTTPRequest{
Body: model.ArchivalMaybeBinaryData{},
BodyIsTruncated: false,
HeadersList: t.newHTTPHeadersList(ev.RequestHeaders),
Headers: t.newHTTPHeadersMap(ev.RequestHeaders),
Method: ev.Method,
Tor: model.ArchivalHTTPTor{},
Transport: ev.Transport,
URL: ev.URL,
},
Response: model.ArchivalHTTPResponse{
Body: model.ArchivalMaybeBinaryData{
Value: string(ev.ResponseBody),
},
BodyIsTruncated: ev.ResponseBodyIsTruncated,
Code: ev.StatusCode,
HeadersList: t.newHTTPHeadersList(ev.ResponseHeaders),
Headers: t.newHTTPHeadersMap(ev.ResponseHeaders),
Locations: ev.ResponseHeaders.Values("Location"), // safe with nil headers
},
T: ev.Finished.Sub(begin).Seconds(),
})
}
// Implementation note: historically OONI has always added
// the _last_ measurement in _first_ position. This has only
// been relevant for sequentially performed requests. For
// this purpose it feels okay to use T as the sorting key,
// since it's the time when we exited RoundTrip().
sort.Slice(out, func(i, j int) bool {
return out[i].T > out[j].T
})
return
}
func (t *Trace) newHTTPHeadersList(source http.Header) (out []model.ArchivalHTTPHeader) {
for key, values := range source {
for _, value := range values {
out = append(out, model.ArchivalHTTPHeader{
Key: key,
Value: model.ArchivalMaybeBinaryData{
Value: value,
},
})
}
}
// Implementation note: we need to sort the keys to have
// stable testing since map iteration is random.
sort.Slice(out, func(i, j int) bool {
return out[i].Key < out[j].Key
})
return
}
func (t *Trace) newHTTPHeadersMap(source http.Header) (out map[string]model.ArchivalMaybeBinaryData) {
for key, values := range source {
for index, value := range values {
if index > 0 {
break // only the first entry
}
if out == nil {
out = make(map[string]model.ArchivalMaybeBinaryData)
}
out[key] = model.ArchivalMaybeBinaryData{Value: value}
}
}
return
}
//
// DNS
//
// NewArchivalDNSLookupResultList builds a DNS lookups list in the OONI
// archival data format out of the results saved inside the trace.
func (t *Trace) NewArchivalDNSLookupResultList(begin time.Time) (out []model.ArchivalDNSLookupResult) {
for _, ev := range t.DNSLookupHost {
out = append(out, model.ArchivalDNSLookupResult{
Answers: t.gatherA(ev.Addresses),
Engine: ev.ResolverNetwork,
Failure: t.newFailure(ev.Failure),
Hostname: ev.Domain,
QueryType: "A",
ResolverHostname: nil, // legacy
ResolverPort: nil, // legacy
ResolverAddress: ev.ResolverAddress,
T: ev.Finished.Sub(begin).Seconds(),
})
aaaa := t.gatherAAAA(ev.Addresses)
if len(aaaa) <= 0 && ev.Failure == nil {
// We don't have any AAAA results. Historically we do not
// create a record for AAAA with no results when A succeeded
continue
}
out = append(out, model.ArchivalDNSLookupResult{
Answers: aaaa,
Engine: ev.ResolverNetwork,
Failure: t.newFailure(ev.Failure),
Hostname: ev.Domain,
QueryType: "AAAA",
ResolverHostname: nil, // legacy
ResolverPort: nil, // legacy
ResolverAddress: ev.ResolverAddress,
T: ev.Finished.Sub(begin).Seconds(),
})
}
return
}
func (t *Trace) gatherA(addrs []string) (out []model.ArchivalDNSAnswer) {
for _, addr := range addrs {
if strings.Contains(addr, ":") {
continue // it's AAAA so we need to skip it
}
answer := model.ArchivalDNSAnswer{AnswerType: "A"}
asn, org, _ := geolocate.LookupASN(addr)
answer.ASN = int64(asn)
answer.ASOrgName = org
answer.IPv4 = addr
out = append(out, answer)
}
return
}
func (t *Trace) gatherAAAA(addrs []string) (out []model.ArchivalDNSAnswer) {
for _, addr := range addrs {
if !strings.Contains(addr, ":") {
continue // it's A so we need to skip it
}
answer := model.ArchivalDNSAnswer{AnswerType: "AAAA"}
asn, org, _ := geolocate.LookupASN(addr)
answer.ASN = int64(asn)
answer.ASOrgName = org
answer.IPv6 = addr
out = append(out, answer)
}
return
}
//
// NetworkEvents
//
// NewArchivalNetworkEventList builds a network events list in the OONI
// archival data format out of the results saved inside the trace.
func (t *Trace) NewArchivalNetworkEventList(begin time.Time) (out []model.ArchivalNetworkEvent) {
for _, ev := range t.Network {
out = append(out, model.ArchivalNetworkEvent{
Address: ev.RemoteAddr,
Failure: t.newFailure(ev.Failure),
NumBytes: int64(ev.Count),
Operation: ev.Operation,
Proto: ev.Network,
T: ev.Finished.Sub(begin).Seconds(),
Tags: nil,
})
}
return
}
//
// TLS handshake
//
// NewArchivalTLSHandshakeResultList builds a TLS handshakes list in the OONI
// archival data format out of the results saved inside the trace.
func (t *Trace) NewArchivalTLSHandshakeResultList(begin time.Time) (out []model.ArchivalTLSOrQUICHandshakeResult) {
for _, ev := range t.TLSHandshake {
out = append(out, model.ArchivalTLSOrQUICHandshakeResult{
CipherSuite: ev.CipherSuite,
Failure: t.newFailure(ev.Failure),
NegotiatedProtocol: ev.NegotiatedProto,
NoTLSVerify: ev.SkipVerify,
PeerCertificates: t.makePeerCerts(ev.PeerCerts),
ServerName: ev.SNI,
T: ev.Finished.Sub(begin).Seconds(),
Tags: nil,
TLSVersion: ev.TLSVersion,
})
}
return
}
func (t *Trace) makePeerCerts(in [][]byte) (out []model.ArchivalMaybeBinaryData) {
for _, v := range in {
out = append(out, model.ArchivalMaybeBinaryData{Value: string(v)})
}
return
}

921
internal/archival/trace_test.go
View File

@ -1,921 +0,0 @@
package archival
import (
"errors"
"io"
"net/http"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// traceTime generates a time that is based off a fixed beginning
// in time, so we can easily compare times.
func traceTime(d int64) time.Time {
t := time.Date(2021, 01, 13, 14, 21, 59, 0, time.UTC)
return t.Add(time.Duration(d) * time.Millisecond)
}
// deltaSinceTraceTime computes the delta since the original
// trace time expressed in floating point seconds.
func deltaSinceTraceTime(d int64) float64 {
return (time.Duration(d) * time.Millisecond).Seconds()
}
// failureFromString converts a string to a failure.
func failureFromString(failure string) *string {
return &failure
}
func TestTraceNewArchivalTCPConnectResultList(t *testing.T) {
type fields struct {
DNSLookupHTTPS []*DNSLookupEvent
DNSLookupHost []*DNSLookupEvent
DNSRoundTrip []*DNSRoundTripEvent
HTTPRoundTrip []*HTTPRoundTripEvent
Network []*NetworkEvent
QUICHandshake []*QUICTLSHandshakeEvent
TLSHandshake []*QUICTLSHandshakeEvent
}
type args struct {
begin time.Time
}
tests := []struct {
name string
fields fields
args args
wantOut []model.ArchivalTCPConnectResult
}{{
name: "with empty trace",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: nil,
}, {
name: "we ignore I/O operations",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{{
Count: 1024,
Failure: nil,
Finished: traceTime(2),
Network: "tcp",
Operation: netxlite.WriteOperation,
RemoteAddr: "8.8.8.8:443",
Started: traceTime(1),
}, {
Count: 4096,
Failure: nil,
Finished: traceTime(4),
Network: "tcp",
Operation: netxlite.ReadOperation,
RemoteAddr: "8.8.8.8:443",
Started: traceTime(3),
}},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: nil,
}, {
name: "we ignore UDP connect",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{{
Count: 0,
Failure: nil,
Finished: traceTime(2),
Network: "udp",
Operation: netxlite.ConnectOperation,
RemoteAddr: "8.8.8.8:53",
Started: traceTime(1),
}},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: nil,
}, {
name: "with TCP connect success",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{{
Count: 0,
Failure: nil,
Finished: traceTime(2),
Network: "tcp",
Operation: netxlite.ConnectOperation,
RemoteAddr: "8.8.8.8:443",
Started: traceTime(1),
}},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalTCPConnectResult{{
IP: "8.8.8.8",
Port: 443,
Status: model.ArchivalTCPConnectStatus{
Blocked: nil,
Failure: nil,
Success: true,
},
T: deltaSinceTraceTime(2),
}},
}, {
name: "with TCP connect failure",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{{
Count: 0,
Failure: netxlite.NewTopLevelGenericErrWrapper(netxlite.ECONNREFUSED),
Finished: traceTime(2),
Network: "tcp",
Operation: netxlite.ConnectOperation,
RemoteAddr: "8.8.8.8:443",
Started: traceTime(1),
}},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalTCPConnectResult{{
IP: "8.8.8.8",
Port: 443,
Status: model.ArchivalTCPConnectStatus{
Blocked: nil,
Failure: failureFromString(netxlite.FailureConnectionRefused),
Success: false,
},
T: deltaSinceTraceTime(2),
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tr := &Trace{
DNSLookupHTTPS: tt.fields.DNSLookupHTTPS,
DNSLookupHost: tt.fields.DNSLookupHost,
DNSRoundTrip: tt.fields.DNSRoundTrip,
HTTPRoundTrip: tt.fields.HTTPRoundTrip,
Network: tt.fields.Network,
QUICHandshake: tt.fields.QUICHandshake,
TLSHandshake: tt.fields.TLSHandshake,
}
gotOut := tr.NewArchivalTCPConnectResultList(tt.args.begin)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestTraceNewArchivalHTTPRequestResultList(t *testing.T) {
type fields struct {
DNSLookupHTTPS []*DNSLookupEvent
DNSLookupHost []*DNSLookupEvent
DNSRoundTrip []*DNSRoundTripEvent
HTTPRoundTrip []*HTTPRoundTripEvent
Network []*NetworkEvent
QUICHandshake []*QUICTLSHandshakeEvent
TLSHandshake []*QUICTLSHandshakeEvent
}
type args struct {
begin time.Time
}
tests := []struct {
name string
fields fields
args args
wantOut []model.ArchivalHTTPRequestResult
}{{
name: "with empty trace",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: nil,
}, {
name: "with failure",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{{
Failure: netxlite.NewTopLevelGenericErrWrapper(netxlite.ECONNRESET),
Finished: traceTime(2),
Method: "GET",
RequestHeaders: http.Header{
"Accept": {"*/*"},
"X-Cookie": {"A", "B", "C"},
},
ResponseBody: nil,
ResponseBodyIsTruncated: false,
ResponseBodyLength: 0,
ResponseHeaders: nil,
Started: traceTime(1),
StatusCode: 0,
Transport: "tcp",
URL: "http://x.org/",
}},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalHTTPRequestResult{{
Failure: failureFromString(netxlite.FailureConnectionReset),
Request: model.ArchivalHTTPRequest{
Body: model.ArchivalMaybeBinaryData{},
BodyIsTruncated: false,
HeadersList: []model.ArchivalHTTPHeader{{
Key: "Accept",
Value: model.ArchivalMaybeBinaryData{
Value: "*/*",
},
}, {
Key: "X-Cookie",
Value: model.ArchivalMaybeBinaryData{
Value: "A",
},
}, {
Key: "X-Cookie",
Value: model.ArchivalMaybeBinaryData{
Value: "B",
},
}, {
Key: "X-Cookie",
Value: model.ArchivalMaybeBinaryData{
Value: "C",
},
}},
Headers: map[string]model.ArchivalMaybeBinaryData{
"Accept": {Value: "*/*"},
"X-Cookie": {Value: "A"},
},
Method: "GET",
Tor: model.ArchivalHTTPTor{},
Transport: "tcp",
URL: "http://x.org/",
},
Response: model.ArchivalHTTPResponse{
Body: model.ArchivalMaybeBinaryData{},
BodyIsTruncated: false,
Code: 0,
HeadersList: nil,
Headers: nil,
Locations: nil,
},
T: deltaSinceTraceTime(2),
}},
}, {
name: "with success",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{{
Failure: nil,
Finished: traceTime(2),
Method: "GET",
RequestHeaders: http.Header{
"Accept": {"*/*"},
"X-Cookie": {"A", "B", "C"},
},
ResponseBody: []byte("0xdeadbeef"),
ResponseBodyIsTruncated: true,
ResponseBodyLength: 10,
ResponseHeaders: http.Header{
"Server": {"antani/1.0"},
"X-Cookie-Reply": {"C", "D", "F"},
"Location": {"https://x.org/", "https://x.org/robots.txt"},
},
Started: traceTime(1),
StatusCode: 302,
Transport: "tcp",
URL: "http://x.org/",
}},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalHTTPRequestResult{{
Failure: nil,
Request: model.ArchivalHTTPRequest{
Body: model.ArchivalMaybeBinaryData{},
BodyIsTruncated: false,
HeadersList: []model.ArchivalHTTPHeader{{
Key: "Accept",
Value: model.ArchivalMaybeBinaryData{
Value: "*/*",
},
}, {
Key: "X-Cookie",
Value: model.ArchivalMaybeBinaryData{
Value: "A",
},
}, {
Key: "X-Cookie",
Value: model.ArchivalMaybeBinaryData{
Value: "B",
},
}, {
Key: "X-Cookie",
Value: model.ArchivalMaybeBinaryData{
Value: "C",
},
}},
Headers: map[string]model.ArchivalMaybeBinaryData{
"Accept": {Value: "*/*"},
"X-Cookie": {Value: "A"},
},
Method: "GET",
Tor: model.ArchivalHTTPTor{},
Transport: "tcp",
URL: "http://x.org/",
},
Response: model.ArchivalHTTPResponse{
Body: model.ArchivalMaybeBinaryData{
Value: "0xdeadbeef",
},
BodyIsTruncated: true,
Code: 302,
HeadersList: []model.ArchivalHTTPHeader{{
Key: "Location",
Value: model.ArchivalMaybeBinaryData{
Value: "https://x.org/",
},
}, {
Key: "Location",
Value: model.ArchivalMaybeBinaryData{
Value: "https://x.org/robots.txt",
},
}, {
Key: "Server",
Value: model.ArchivalMaybeBinaryData{
Value: "antani/1.0",
},
}, {
Key: "X-Cookie-Reply",
Value: model.ArchivalMaybeBinaryData{
Value: "C",
},
}, {
Key: "X-Cookie-Reply",
Value: model.ArchivalMaybeBinaryData{
Value: "D",
},
}, {
Key: "X-Cookie-Reply",
Value: model.ArchivalMaybeBinaryData{
Value: "F",
},
}},
Headers: map[string]model.ArchivalMaybeBinaryData{
"Server": {Value: "antani/1.0"},
"X-Cookie-Reply": {Value: "C"},
"Location": {Value: "https://x.org/"},
},
Locations: []string{
"https://x.org/",
"https://x.org/robots.txt",
},
},
T: deltaSinceTraceTime(2),
}},
}, {
name: "The result is sorted by the value of T",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{{
Failure: nil,
Finished: traceTime(3),
Method: "",
RequestHeaders: map[string][]string{},
ResponseBody: []byte{},
ResponseBodyIsTruncated: false,
ResponseBodyLength: 0,
ResponseHeaders: map[string][]string{},
Started: time.Time{},
StatusCode: 0,
Transport: "",
URL: "",
}, {
Failure: nil,
Finished: traceTime(2),
Method: "",
RequestHeaders: map[string][]string{},
ResponseBody: []byte{},
ResponseBodyIsTruncated: false,
ResponseBodyLength: 0,
ResponseHeaders: map[string][]string{},
Started: time.Time{},
StatusCode: 0,
Transport: "",
URL: "",
}, {
Failure: nil,
Finished: traceTime(5),
Method: "",
RequestHeaders: map[string][]string{},
ResponseBody: []byte{},
ResponseBodyIsTruncated: false,
ResponseBodyLength: 0,
ResponseHeaders: map[string][]string{},
Started: time.Time{},
StatusCode: 0,
Transport: "",
URL: "",
}},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalHTTPRequestResult{{
Failure: nil,
Request: model.ArchivalHTTPRequest{},
Response: model.ArchivalHTTPResponse{},
T: deltaSinceTraceTime(5),
}, {
Failure: nil,
Request: model.ArchivalHTTPRequest{},
Response: model.ArchivalHTTPResponse{},
T: deltaSinceTraceTime(3),
}, {
Failure: nil,
Request: model.ArchivalHTTPRequest{},
Response: model.ArchivalHTTPResponse{},
T: deltaSinceTraceTime(2),
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tr := &Trace{
DNSLookupHTTPS: tt.fields.DNSLookupHTTPS,
DNSLookupHost: tt.fields.DNSLookupHost,
DNSRoundTrip: tt.fields.DNSRoundTrip,
HTTPRoundTrip: tt.fields.HTTPRoundTrip,
Network: tt.fields.Network,
QUICHandshake: tt.fields.QUICHandshake,
TLSHandshake: tt.fields.TLSHandshake,
}
gotOut := tr.NewArchivalHTTPRequestResultList(tt.args.begin)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestTraceNewArchivalDNSLookupResultList(t *testing.T) {
type fields struct {
DNSLookupHTTPS []*DNSLookupEvent
DNSLookupHost []*DNSLookupEvent
DNSRoundTrip []*DNSRoundTripEvent
HTTPRoundTrip []*HTTPRoundTripEvent
Network []*NetworkEvent
QUICHandshake []*QUICTLSHandshakeEvent
TLSHandshake []*QUICTLSHandshakeEvent
}
type args struct {
begin time.Time
}
tests := []struct {
name string
fields fields
args args
wantOut []model.ArchivalDNSLookupResult
}{{
name: "with empty trace",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: nil,
}, {
name: "with NXDOMAIN failure",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{{
ALPNs: nil,
Addresses: nil,
Domain: "example.com",
Failure: netxlite.NewTopLevelGenericErrWrapper(errors.New(netxlite.DNSNoSuchHostSuffix)),
Finished: traceTime(2),
LookupType: "", // not processed
ResolverAddress: "8.8.8.8:53",
ResolverNetwork: "udp",
Started: traceTime(1),
}},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalDNSLookupResult{{
Answers: nil,
Engine: "udp",
Failure: failureFromString(netxlite.FailureDNSNXDOMAINError),
Hostname: "example.com",
QueryType: "A",
ResolverHostname: nil,
ResolverPort: nil,
ResolverAddress: "8.8.8.8:53",
T: deltaSinceTraceTime(2),
}, {
Answers: nil,
Engine: "udp",
Failure: failureFromString(netxlite.FailureDNSNXDOMAINError),
Hostname: "example.com",
QueryType: "AAAA",
ResolverHostname: nil,
ResolverPort: nil,
ResolverAddress: "8.8.8.8:53",
T: deltaSinceTraceTime(2),
}},
}, {
name: "with success for A and AAAA",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{{
ALPNs: nil,
Addresses: []string{
"8.8.8.8", "8.8.4.4", "2001:4860:4860::8844",
},
Domain: "dns.google",
Failure: nil,
Finished: traceTime(2),
LookupType: "", // not processed
ResolverAddress: "8.8.8.8:53",
ResolverNetwork: "udp",
Started: traceTime(1),
}},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalDNSLookupResult{{
Answers: []model.ArchivalDNSAnswer{{
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
Hostname: "",
IPv4: "8.8.8.8",
IPv6: "",
TTL: nil,
}, {
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
Hostname: "",
IPv4: "8.8.4.4",
IPv6: "",
TTL: nil,
}},
Engine: "udp",
Failure: nil,
Hostname: "dns.google",
QueryType: "A",
ResolverHostname: nil,
ResolverPort: nil,
ResolverAddress: "8.8.8.8:53",
T: deltaSinceTraceTime(2),
}, {
Answers: []model.ArchivalDNSAnswer{{
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "AAAA",
Hostname: "",
IPv4: "",
IPv6: "2001:4860:4860::8844",
TTL: nil,
}},
Engine: "udp",
Failure: nil,
Hostname: "dns.google",
QueryType: "AAAA",
ResolverHostname: nil,
ResolverPort: nil,
ResolverAddress: "8.8.8.8:53",
T: deltaSinceTraceTime(2),
}},
}, {
name: "when a domain has no AAAA addresses",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{{
ALPNs: nil,
Addresses: []string{
"8.8.8.8", "8.8.4.4",
},
Domain: "dns.google",
Failure: nil,
Finished: traceTime(2),
LookupType: "", // not processed
ResolverAddress: "8.8.8.8:53",
ResolverNetwork: "udp",
Started: traceTime(1),
}},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalDNSLookupResult{{
Answers: []model.ArchivalDNSAnswer{{
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
Hostname: "",
IPv4: "8.8.8.8",
IPv6: "",
TTL: nil,
}, {
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
Hostname: "",
IPv4: "8.8.4.4",
IPv6: "",
TTL: nil,
}},
Engine: "udp",
Failure: nil,
Hostname: "dns.google",
QueryType: "A",
ResolverHostname: nil,
ResolverPort: nil,
ResolverAddress: "8.8.8.8:53",
T: deltaSinceTraceTime(2),
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tr := &Trace{
DNSLookupHTTPS: tt.fields.DNSLookupHTTPS,
DNSLookupHost: tt.fields.DNSLookupHost,
DNSRoundTrip: tt.fields.DNSRoundTrip,
HTTPRoundTrip: tt.fields.HTTPRoundTrip,
Network: tt.fields.Network,
QUICHandshake: tt.fields.QUICHandshake,
TLSHandshake: tt.fields.TLSHandshake,
}
gotOut := tr.NewArchivalDNSLookupResultList(tt.args.begin)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestTraceNewArchivalNetworkEventList(t *testing.T) {
type fields struct {
DNSLookupHTTPS []*DNSLookupEvent
DNSLookupHost []*DNSLookupEvent
DNSRoundTrip []*DNSRoundTripEvent
HTTPRoundTrip []*HTTPRoundTripEvent
Network []*NetworkEvent
QUICHandshake []*QUICTLSHandshakeEvent
TLSHandshake []*QUICTLSHandshakeEvent
}
type args struct {
begin time.Time
}
tests := []struct {
name string
fields fields
args args
wantOut []model.ArchivalNetworkEvent
}{{
name: "with empty trace",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: nil,
}, {
name: "we fill all the fields we should be filling",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{{
Count: 1234,
Failure: netxlite.NewTopLevelGenericErrWrapper(io.EOF),
Finished: traceTime(2),
Network: "tcp",
Operation: netxlite.ReadOperation,
RemoteAddr: "8.8.8.8:443",
Started: traceTime(1),
}},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalNetworkEvent{{
Address: "8.8.8.8:443",
Failure: failureFromString(netxlite.FailureEOFError),
NumBytes: 1234,
Operation: netxlite.ReadOperation,
Proto: "tcp",
T: deltaSinceTraceTime(2),
Tags: nil,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tr := &Trace{
DNSLookupHTTPS: tt.fields.DNSLookupHTTPS,
DNSLookupHost: tt.fields.DNSLookupHost,
DNSRoundTrip: tt.fields.DNSRoundTrip,
HTTPRoundTrip: tt.fields.HTTPRoundTrip,
Network: tt.fields.Network,
QUICHandshake: tt.fields.QUICHandshake,
TLSHandshake: tt.fields.TLSHandshake,
}
gotOut := tr.NewArchivalNetworkEventList(tt.args.begin)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestTraceNewArchivalTLSHandshakeResultList(t *testing.T) {
type fields struct {
DNSLookupHTTPS []*DNSLookupEvent
DNSLookupHost []*DNSLookupEvent
DNSRoundTrip []*DNSRoundTripEvent
HTTPRoundTrip []*HTTPRoundTripEvent
Network []*NetworkEvent
QUICHandshake []*QUICTLSHandshakeEvent
TLSHandshake []*QUICTLSHandshakeEvent
}
type args struct {
begin time.Time
}
tests := []struct {
name string
fields fields
args args
wantOut []model.ArchivalTLSOrQUICHandshakeResult
}{{
name: "with empty trace",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{},
},
args: args{
begin: traceTime(0),
},
wantOut: nil,
}, {
name: "we fill all the fields we should be filling",
fields: fields{
DNSLookupHTTPS: []*DNSLookupEvent{},
DNSLookupHost: []*DNSLookupEvent{},
DNSRoundTrip: []*DNSRoundTripEvent{},
HTTPRoundTrip: []*HTTPRoundTripEvent{},
Network: []*NetworkEvent{},
QUICHandshake: []*QUICTLSHandshakeEvent{},
TLSHandshake: []*QUICTLSHandshakeEvent{{
ALPN: []string{"h2", "http/1.1"},
CipherSuite: "TLS_AES_128_GCM_SHA256",
Failure: netxlite.NewTopLevelGenericErrWrapper(io.EOF),
Finished: traceTime(2),
NegotiatedProto: "h2",
Network: "tcp",
PeerCerts: [][]byte{
[]byte("deadbeef"),
[]byte("xox"),
},
RemoteAddr: "8.8.8.8:443",
SNI: "dns.google",
SkipVerify: true,
Started: traceTime(1),
TLSVersion: "TLSv1.3",
}},
},
args: args{
begin: traceTime(0),
},
wantOut: []model.ArchivalTLSOrQUICHandshakeResult{{
CipherSuite: "TLS_AES_128_GCM_SHA256",
Failure: failureFromString(netxlite.FailureEOFError),
NegotiatedProtocol: "h2",
NoTLSVerify: true,
PeerCertificates: []model.ArchivalMaybeBinaryData{{
Value: "deadbeef",
}, {
Value: "xox",
}},
ServerName: "dns.google",
T: deltaSinceTraceTime(2),
Tags: nil,
TLSVersion: "TLSv1.3",
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tr := &Trace{
DNSLookupHTTPS: tt.fields.DNSLookupHTTPS,
DNSLookupHost: tt.fields.DNSLookupHost,
DNSRoundTrip: tt.fields.DNSRoundTrip,
HTTPRoundTrip: tt.fields.HTTPRoundTrip,
Network: tt.fields.Network,
QUICHandshake: tt.fields.QUICHandshake,
TLSHandshake: tt.fields.TLSHandshake,
}
gotOut := tr.NewArchivalTLSHandshakeResultList(tt.args.begin)
if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" {
t.Fatal(diff)
}
})
}
}

19
internal/cmd/miniooni/libminiooni.go
View File

@ -20,9 +20,6 @@ import (
"github.com/ooni/probe-cli/v3/internal/kvstore" "github.com/ooni/probe-cli/v3/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/legacy/assetsdir" "github.com/ooni/probe-cli/v3/internal/legacy/assetsdir"
"github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/netxlite/filtering"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version" "github.com/ooni/probe-cli/v3/internal/version"
"github.com/pborman/getopt/v2" "github.com/pborman/getopt/v2"
) )
@ -30,7 +27,6 @@ import (
// Options contains the options you can set from the CLI. // Options contains the options you can set from the CLI.
type Options struct { type Options struct {
Annotations []string Annotations []string
Censor string
ExtraOptions []string ExtraOptions []string
HomeDir string HomeDir string
Inputs []string Inputs []string
@ -65,10 +61,6 @@ func init() {
getopt.FlagLong( getopt.FlagLong(
&globalOptions.Annotations, "annotation", 'A', "Add annotaton", "KEY=VALUE", &globalOptions.Annotations, "annotation", 'A', "Add annotaton", "KEY=VALUE",
) )
getopt.FlagLong(
&globalOptions.Censor, "censor", 0,
"Specifies censorship rules to apply for QA purposes", "FILE",
)
getopt.FlagLong( getopt.FlagLong(
&globalOptions.ExtraOptions, "option", 'O', &globalOptions.ExtraOptions, "option", 'O',
"Pass an option to the experiment", "KEY=VALUE", "Pass an option to the experiment", "KEY=VALUE",
@ -316,17 +308,6 @@ func MainWithConfiguration(experimentName string, currentOptions Options) {
} }
log.Log = logger log.Log = logger
if currentOptions.Censor != "" {
config, err := filtering.NewTProxyConfig(currentOptions.Censor)
runtimex.PanicOnError(err, "cannot parse --censor file as JSON")
tproxy, err := filtering.NewTProxy(config, log.Log)
runtimex.PanicOnError(err, "cannot create tproxy instance")
defer tproxy.Close()
netxlite.TProxy = tproxy
log.Infof("miniooni: disabling submission with --censor to avoid pulluting OONI data")
currentOptions.NoCollector = true
}
//Mon Jan 2 15:04:05 -0700 MST 2006 //Mon Jan 2 15:04:05 -0700 MST 2006
log.Infof("Current time: %s", time.Now().Format("2006-01-02 15:04:05 MST")) log.Infof("Current time: %s", time.Now().Format("2006-01-02 15:04:05 MST"))

27
internal/cmd/oohelper/oohelper.go
View File

@ -11,9 +11,7 @@ import (
"github.com/apex/log" "github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal" "github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx"
"github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/measurex"
"github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/runtimex"
) )
@ -25,7 +23,6 @@ var (
resolver model.Resolver resolver model.Resolver
server = flag.String("server", "", "URL of the test helper") server = flag.String("server", "", "URL of the test helper")
target = flag.String("target", "", "Target URL for the test helper") target = flag.String("target", "", "Target URL for the test helper")
fwebsteps = flag.Bool("websteps", false, "Use the websteps TH")
) )
func newhttpclient() *http.Client { func newhttpclient() *http.Client {
@ -54,34 +51,12 @@ func main() {
} }
flag.Parse() flag.Parse()
log.SetLevel(logmap[*debug]) log.SetLevel(logmap[*debug])
apimap := map[bool]func() interface{}{ cresp := wcth()
false: wcth,
true: webstepsth,
}
cresp := apimap[*fwebsteps]()
data, err := json.MarshalIndent(cresp, "", " ") data, err := json.MarshalIndent(cresp, "", " ")
runtimex.PanicOnError(err, "json.MarshalIndent failed") runtimex.PanicOnError(err, "json.MarshalIndent failed")
fmt.Printf("%s\n", string(data)) fmt.Printf("%s\n", string(data))
} }
func webstepsth() interface{} {
serverURL := *server
if serverURL == "" {
serverURL = "https://1.th.ooni.org/api/v1/websteps"
}
clnt := &webstepsx.THClient{
DNServers: []*measurex.ResolverInfo{{
Network: "udp",
Address: "8.8.4.4:53",
}},
HTTPClient: httpClient,
ServerURL: serverURL,
}
cresp, err := clnt.Run(ctx, *target)
runtimex.PanicOnError(err, "client.Run failed")
return cresp
}
func wcth() interface{} { func wcth() interface{} {
serverURL := *server serverURL := *server
if serverURL == "" { if serverURL == "" {

2
internal/cmd/oohelperd/oohelperd.go
View File

@ -10,7 +10,6 @@ import (
"github.com/apex/log" "github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/webconnectivity" "github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx"
"github.com/ooni/probe-cli/v3/internal/engine/netx" "github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/runtimex"
@ -58,7 +57,6 @@ func main() {
func testableMain() { func testableMain() {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.Handle("/api/v1/websteps", &webstepsx.THHandler{})
mux.Handle("/", webconnectivity.Handler{ mux.Handle("/", webconnectivity.Handler{
Client: httpx, Client: httpx,
Dialer: dialer, Dialer: dialer,

13
internal/engine/allexperiments.go
View File

@ -29,7 +29,6 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator" "github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp"
) )
@ -352,18 +351,6 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
} }
}, },
"websteps": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, webstepsx.NewExperimentMeasurer(
*config.(*webstepsx.Config),
))
},
config: &webstepsx.Config{},
inputPolicy: InputOrQueryBackend,
}
},
"whatsapp": func(session *Session) *ExperimentBuilder { "whatsapp": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{ return &ExperimentBuilder{
build: func(config interface{}) *Experiment { build: func(config interface{}) *Experiment {

6
internal/engine/experiment/webstepsx/doc.go
View File

@ -1,6 +0,0 @@
// Package webstepsx contains a websteps implementation
// based on the internal/measurex package.
//
// This implementation does not follow any existing spec
// rather we are modeling the spec on this one.
package webstepsx

223
internal/engine/experiment/webstepsx/measurer.go
View File

@ -1,223 +0,0 @@
package webstepsx
//
// Measurer
//
// This file contains the client implementation.
//
import (
"context"
"errors"
"net/http"
"net/url"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/measurex"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
const (
testName = "websteps"
testVersion = "0.0.3"
)
// Config contains the experiment config.
type Config struct{}
// TestKeys contains the experiment's test keys.
type TestKeys struct {
*measurex.ArchivalURLMeasurement
}
// Measurer performs the measurement.
type Measurer struct {
Config Config
}
var (
_ model.ExperimentMeasurer = &Measurer{}
_ model.ExperimentMeasurerAsync = &Measurer{}
)
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return &Measurer{Config: config}
}
// ExperimentName implements ExperimentMeasurer.ExperExperimentName.
func (mx *Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion.
func (mx *Measurer) ExperimentVersion() string {
return testVersion
}
var (
// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
ErrNoAvailableTestHelpers = errors.New("no available helpers")
// ErrNoInput indicates that no input was provided.
ErrNoInput = errors.New("no input provided")
// ErrInputIsNotAnURL indicates that the input is not an URL.
ErrInputIsNotAnURL = errors.New("input is not an URL")
// ErrUnsupportedInput indicates that the input URL scheme is unsupported.
ErrUnsupportedInput = errors.New("unsupported input scheme")
)
// RunAsync implements ExperimentMeasurerAsync.RunAsync.
func (mx *Measurer) RunAsync(
ctx context.Context, sess model.ExperimentSession, input string,
callbacks model.ExperimentCallbacks) (<-chan *model.ExperimentAsyncTestKeys, error) {
// 1. Parse and verify URL
URL, err := url.Parse(input)
if err != nil {
return nil, ErrInputIsNotAnURL
}
if URL.Scheme != "http" && URL.Scheme != "https" {
return nil, ErrUnsupportedInput
}
// 2. Find the testhelper
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
var testhelper *model.OOAPIService
for _, th := range testhelpers {
if th.Type == "https" {
testhelper = &th
break
}
}
if testhelper == nil {
return nil, ErrNoAvailableTestHelpers
}
testhelper.Address = "https://1.th.ooni.org/api/v1/websteps" // TODO(bassosimone): remove!
out := make(chan *model.ExperimentAsyncTestKeys)
go mx.runAsync(ctx, sess, input, testhelper, out)
return out, nil
}
var measurerResolvers = []*measurex.ResolverInfo{{
Network: "system",
Address: "",
}, {
Network: "udp",
Address: "8.8.4.4:53",
}}
func (mx *Measurer) runAsync(ctx context.Context, sess model.ExperimentSession,
URL string, th *model.OOAPIService, out chan<- *model.ExperimentAsyncTestKeys) {
defer close(out)
helper := &measurerMeasureURLHelper{
Clnt: sess.DefaultHTTPClient(),
Logger: sess.Logger(),
THURL: th.Address,
UserAgent: sess.UserAgent(),
}
mmx := &measurex.Measurer{
Begin: time.Now(),
HTTPClient: sess.DefaultHTTPClient(),
MeasureURLHelper: helper,
Logger: sess.Logger(),
Resolvers: measurerResolvers,
TLSHandshaker: netxlite.NewTLSHandshakerStdlib(sess.Logger()),
}
cookies := measurex.NewCookieJar()
const parallelism = 3
in := mmx.MeasureURLAndFollowRedirections(
ctx, parallelism, URL, measurex.NewHTTPRequestHeaderForMeasuring(), cookies)
for m := range in {
out <- &model.ExperimentAsyncTestKeys{
Extensions: map[string]int64{
archival.ExtHTTP.Name: archival.ExtHTTP.V,
archival.ExtDNS.Name: archival.ExtDNS.V,
archival.ExtNetevents.Name: archival.ExtNetevents.V,
archival.ExtTCPConnect.Name: archival.ExtTCPConnect.V,
archival.ExtTLSHandshake.Name: archival.ExtTLSHandshake.V,
},
Input: model.MeasurementTarget(m.URL),
MeasurementRuntime: m.TotalRuntime.Seconds(),
TestKeys: &TestKeys{
ArchivalURLMeasurement: measurex.NewArchivalURLMeasurement(m),
},
}
}
}
// measurerMeasureURLHelper injects the TH into the normal
// URL measurement flow implemented by measurex.
type measurerMeasureURLHelper struct {
// Clnt is the MANDATORY client to use
Clnt model.HTTPClient
// Logger is the MANDATORY Logger to use
Logger model.Logger
// THURL is the MANDATORY TH URL.
THURL string
// UserAgent is the OPTIONAL user-agent to use.
UserAgent string
}
func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header,
curEndpoints ...*measurex.HTTPEndpoint) (
[]*measurex.HTTPEndpoint, *measurex.THMeasurement, error) {
cc := &THClientCall{
Endpoints: measurex.HTTPEndpointsToEndpoints(curEndpoints),
HTTPClient: mth.Clnt,
Header: headers,
THURL: mth.THURL,
TargetURL: URL.String(),
UserAgent: mth.UserAgent,
}
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
ol := measurex.NewOperationLogger(
mth.Logger, "THClientCall %s", URL.String())
resp, err := cc.Call(ctx)
ol.Stop(err)
if err != nil {
return nil, resp, err
}
var out []*measurex.HTTPEndpoint
for _, epnt := range resp.Endpoints {
out = append(out, &measurex.HTTPEndpoint{
Domain: URL.Hostname(),
Network: epnt.Network,
Address: epnt.Address,
SNI: URL.Hostname(),
ALPN: measurex.ALPNForHTTPEndpoint(epnt.Network),
URL: URL,
Header: headers,
})
}
return out, resp, nil
}
// Run implements ExperimentMeasurer.Run.
func (mx *Measurer) Run(ctx context.Context, sess model.ExperimentSession,
measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
return errors.New("sync run is not implemented")
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with ooniprobe
// therefore we should be careful when changing it.
type SummaryKeys struct {
Accessible bool `json:"accessible"`
Blocking string `json:"blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (mx *Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{}
return sk, nil
}

372
internal/engine/experiment/webstepsx/th.go
View File

@ -1,372 +0,0 @@
package webstepsx
//
// TH (Test Helper)
//
// This file contains an implementation of the
// (proposed) websteps test helper spec.
//
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/measurex"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version"
)
//
// Messages exchanged by the TH client and server
//
// THClientRequest is the request received by the test helper.
type THClientRequest struct {
// Endpoints is a list of endpoints to measure.
Endpoints []*measurex.Endpoint
// URL is the URL we want to measure.
URL string
// HTTPRequestHeaders contains the request headers.
HTTPRequestHeaders http.Header
}
// THServerResponse is the response from the test helper.
type THServerResponse = measurex.THMeasurement
// thMaxAcceptableBodySize is the maximum acceptable body size by TH code.
const thMaxAcceptableBodySize = 1 << 20
//
// TH client implementation
//
// THClient is the high-level API to invoke the TH. This API
// should be used by command line clients.
type THClient struct {
// DNSServers is the MANDATORY list of DNS-over-UDP
// servers to use to discover endpoints locally.
DNServers []*measurex.ResolverInfo
// HTTPClient is the MANDATORY HTTP client to
// use for contacting the TH.
HTTPClient model.HTTPClient
// ServerURL is the MANDATORY URL of the TH HTTP endpoint.
ServerURL string
}
// Run calls the TH and returns the response or an error.
//
// Arguments:
//
// - ctx is the context with timeout/deadline/cancellation
//
// - URL is the URL the TH server should measure for us
//
// Algorithm:
//
// - use DNSServers to discover extra endpoints for the target URL
//
// - call the TH using the HTTPClient and the ServerURL
//
// - return response or error.
func (c *THClient) Run(ctx context.Context, URL string) (*THServerResponse, error) {
parsed, err := url.Parse(URL)
if err != nil {
return nil, err
}
mx := measurex.NewMeasurerWithDefaultSettings()
var dns []*measurex.DNSMeasurement
const parallelism = 3
for m := range mx.LookupURLHostParallel(ctx, parallelism, parsed, c.DNServers...) {
dns = append(dns, m)
}
endpoints, err := measurex.AllEndpointsForURL(parsed, dns...)
if err != nil {
return nil, err
}
return (&THClientCall{
Endpoints: endpoints,
HTTPClient: c.HTTPClient,
Header: measurex.NewHTTPRequestHeaderForMeasuring(),
THURL: c.ServerURL,
TargetURL: URL,
}).Call(ctx)
}
// THClientCall allows to perform a single TH client call. Make sure
// you fill all the fields marked as MANDATORY before use.
type THClientCall struct {
// Endpoints contains the MANDATORY endpoints we discovered.
Endpoints []*measurex.Endpoint
// HTTPClient is the MANDATORY HTTP client to
// use for contacting the TH.
HTTPClient model.HTTPClient
// Header contains the MANDATORY request headers.
Header http.Header
// THURL is the MANDATORY test helper URL.
THURL string
// TargetURL is the MANDATORY URL to measure.
TargetURL string
// UserAgent is the OPTIONAL user-agent to use.
UserAgent string
}
// Call performs the specified TH call and returns either a response or an error.
func (c *THClientCall) Call(ctx context.Context) (*THServerResponse, error) {
creq := &THClientRequest{
Endpoints: c.Endpoints,
URL: c.TargetURL,
HTTPRequestHeaders: c.Header,
}
reqBody, err := json.Marshal(creq)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(
ctx, "POST", c.THURL, bytes.NewReader(reqBody))
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", c.UserAgent)
return c.httpClientDo(req)
}
// errTHRequestFailed is the error returned if the TH response is not 200 Ok.
var errTHRequestFailed = errors.New("th: request failed")
func (c *THClientCall) httpClientDo(req *http.Request) (*THServerResponse, error) {
resp, err := c.HTTPClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 { // THHandler returns either 400 or 200
return nil, errTHRequestFailed
}
r := io.LimitReader(resp.Body, thMaxAcceptableBodySize)
respBody, err := netxlite.ReadAllContext(req.Context(), r)
if err != nil {
return nil, err
}
var sresp THServerResponse
if err := json.Unmarshal(respBody, &sresp); err != nil {
return nil, err
}
return &sresp, nil
}
//
// TH server implementation
//
// THHandler implements the test helper API.
//
// This handler exposes a unique HTTP endpoint that you need to
// mount to the desired path when creating the server.
//
// The canonical mount point for the HTTP endpoint is /api/v1/websteps.
//
// Accepted methods and request body:
//
// - we only accept POST;
//
// - we expect a THClientRequest as the body.
//
// Status code and response body:
//
// - on success, status is 200 and THServerResponse is the body;
//
// - on failure, status is 400 and there is no body.
//
type THHandler struct{}
// ServerHTTP implements http.Handler.ServeHTTP.
func (h *THHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Add("Server", fmt.Sprintf("oohelperd/%s", version.Version))
if req.Method != "POST" {
w.WriteHeader(400)
return
}
reader := io.LimitReader(req.Body, thMaxAcceptableBodySize)
data, err := netxlite.ReadAllContext(req.Context(), reader)
if err != nil {
w.WriteHeader(400)
return
}
var creq THClientRequest
if err := json.Unmarshal(data, &creq); err != nil {
w.WriteHeader(400)
return
}
cresp, err := h.singleStep(req.Context(), &creq)
if err != nil {
w.WriteHeader(400)
return
}
// We assume that the following call cannot fail because it's a
// clearly serializable data structure.
data, err = json.Marshal(cresp)
runtimex.PanicOnError(err, "json.Marshal failed")
w.Header().Add("Content-Type", "application/json")
w.Write(data)
}
// singleStep performs a singleStep measurement.
//
// The function name derives from the definition (we invented)
// of "web steps". Each redirection is a step. For each step you
// need to figure out the endpoints to use with the DNS. After
// that, you need to check all endpoints. Because here we do not
// perform redirection, this is just a single "step".
//
// The algorithm is the following:
//
// 1. parse the URL and return error if it does not parse or
// the scheme is neither HTTP nor HTTPS;
//
// 2. discover additional endpoints using a suitable DoH
// resolver and the URL's hostname as the domain;
//
// 3. measure each discovered endpoint.
//
// The return value is either a THServerResponse or an error.
func (h *THHandler) singleStep(
ctx context.Context, req *THClientRequest) (*THServerResponse, error) {
mx := measurex.NewMeasurerWithDefaultSettings()
mx.MeasureURLHelper = &thMeasureURLHelper{req.Endpoints}
mx.Resolvers = []*measurex.ResolverInfo{{
Network: measurex.ResolverForeign,
ForeignResolver: thResolver,
}}
jar := measurex.NewCookieJar()
const parallelism = 3
meas, err := mx.MeasureURL(ctx, parallelism, req.URL, req.HTTPRequestHeaders, jar)
if err != nil {
return nil, err
}
return &THServerResponse{
DNS: meas.DNS,
Endpoints: h.simplifyEndpoints(meas.Endpoints),
}, nil
}
func (h *THHandler) simplifyEndpoints(
in []*measurex.HTTPEndpointMeasurement) (out []*measurex.HTTPEndpointMeasurement) {
for _, epnt := range in {
out = append(out, &measurex.HTTPEndpointMeasurement{
URL: epnt.URL,
Network: epnt.Network,
Address: epnt.Address,
Measurement: h.simplifyMeasurement(epnt.Measurement),
})
}
return
}
func (h *THHandler) simplifyMeasurement(in *measurex.Measurement) (out *measurex.Measurement) {
out = &measurex.Measurement{
Connect: in.Connect,
TLSHandshake: h.simplifyHandshake(in.TLSHandshake),
QUICHandshake: h.simplifyHandshake(in.QUICHandshake),
LookupHost: in.LookupHost,
LookupHTTPSSvc: in.LookupHTTPSSvc,
HTTPRoundTrip: h.simplifyHTTPRoundTrip(in.HTTPRoundTrip),
}
return
}
func (h *THHandler) simplifyHandshake(
in []*measurex.QUICTLSHandshakeEvent) (out []*measurex.QUICTLSHandshakeEvent) {
for _, ev := range in {
out = append(out, &measurex.QUICTLSHandshakeEvent{
CipherSuite: ev.CipherSuite,
Failure: ev.Failure,
NegotiatedProto: ev.NegotiatedProto,
TLSVersion: ev.TLSVersion,
PeerCerts: nil,
Finished: 0,
RemoteAddr: ev.RemoteAddr,
SNI: ev.SNI,
ALPN: ev.ALPN,
SkipVerify: ev.SkipVerify,
Oddity: ev.Oddity,
Network: ev.Network,
Started: 0,
})
}
return
}
func (h *THHandler) simplifyHTTPRoundTrip(
in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) {
for _, ev := range in {
out = append(out, &measurex.HTTPRoundTripEvent{
Failure: ev.Failure,
Method: ev.Method,
URL: ev.URL,
RequestHeaders: ev.RequestHeaders,
StatusCode: ev.StatusCode,
ResponseHeaders: ev.ResponseHeaders,
ResponseBody: nil, // we don't transfer the body
ResponseBodyLength: ev.ResponseBodyLength,
ResponseBodyIsTruncated: ev.ResponseBodyIsTruncated,
ResponseBodyIsUTF8: ev.ResponseBodyIsUTF8,
Finished: ev.Finished,
Started: ev.Started,
Oddity: ev.Oddity,
})
}
return
}
type thMeasureURLHelper struct {
epnts []*measurex.Endpoint
}
func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header,
serverEpnts ...*measurex.HTTPEndpoint) (
epnts []*measurex.HTTPEndpoint, thMeaurement *measurex.THMeasurement, err error) {
for _, epnt := range thh.epnts {
epnts = append(epnts, &measurex.HTTPEndpoint{
Domain: URL.Hostname(),
Network: epnt.Network,
Address: epnt.Address,
SNI: URL.Hostname(),
ALPN: measurex.ALPNForHTTPEndpoint(epnt.Network),
URL: URL,
Header: headers, // but overriden later anyway
})
}
return
}
// thResolverURL is the DNS resolver URL used by the TH. We use an
// encrypted resolver to reduce the risk that there is DNS-over-UDP
// censorship in the place where we deploy the TH.
const thResolverURL = "https://dns.google/dns-query"
// thResolver is the DNS resolver used by the TH.
//
// Here we're using github.com/apex/log as the logger, which
// is fine because this is backend only code.
var thResolver = netxlite.WrapResolver(log.Log, netxlite.NewSerialResolver(
netxlite.NewDNSOverHTTPSTransport(http.DefaultClient, thResolverURL),
))

357
internal/netxlite/filtering/tproxy.go
View File

@ -1,357 +0,0 @@
package filtering
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"time"
"github.com/miekg/dns"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// TProxyPolicy is a policy for TPRoxy.
type TProxyPolicy string
const (
// TProxyPolicyTCPDropSYN simulates a SYN segment being dropped.
TProxyPolicyTCPDropSYN = TProxyPolicy("tcp-drop-syn")
// TProxyPolicyTCPRejectSYN simulates a closed TCP port.
TProxyPolicyTCPRejectSYN = TProxyPolicy("tcp-reject-syn")
// TProxyPolicyDropData drops outgoing data of an
// established TCP/UDP connection.
TProxyPolicyDropData = TProxyPolicy("drop-data")
// TProxyPolicyHijackDNS causes the dialer to replace the target
// address with the address of the local censored resolver.
TProxyPolicyHijackDNS = TProxyPolicy("hijack-dns")
// TProxyPolicyHijackTLS causes the dialer to replace the target
// address with the address of the local censored TLS server.
TProxyPolicyHijackTLS = TProxyPolicy("hijack-tls")
// TProxyPolicyHijackHTTP causes the dialer to replace the target
// address with the address of the local censored HTTP server.
TProxyPolicyHijackHTTP = TProxyPolicy("hijack-http")
)
// TProxyConfig contains configuration for TProxy.
type TProxyConfig struct {
// DNSCache is the cached used when the domains policy is "cache". Note
// that the map MUST contain FQDNs. That is, you need to append
// a final dot to the domain name (e.g., `example.com.`). If you
// use the NewTProxyConfig factory, you don't need to worry about this
// issue, because the factory will canonicalize non-canonical
// entries. Otherwise, you can explicitly call the CanonicalizeDNS
// method _before_ using the TProxy.
DNSCache map[string][]string
// Domains contains rules for filtering the lookup of domains. Note
// that the map MUST contain FQDNs. That is, you need to append
// a final dot to the domain name (e.g., `example.com.`). If you
// use the NewTProxyConfig factory, you don't need to worry about this
// issue, because the factory will canonicalize non-canonical
// entries. Otherwise, you can explicitly call the CanonicalizeDNS
// method _before_ using the TProxy.
Domains map[string]DNSAction
// Endpoints contains rules for filtering TCP/UDP endpoints.
Endpoints map[string]TProxyPolicy
// SNIs contains rules for filtering TLS SNIs.
SNIs map[string]TLSAction
// Hosts contains rules for filtering by HTTP host.
Hosts map[string]HTTPAction
}
// NewTProxyConfig reads the TProxyConfig from the given file.
func NewTProxyConfig(file string) (*TProxyConfig, error) {
data, err := os.ReadFile(file)
if err != nil {
return nil, err
}
var config TProxyConfig
if err := json.Unmarshal(data, &config); err != nil {
return nil, err
}
config.CanonicalizeDNS()
return &config, nil
}
// CanonicalizeDNS ensures all DNS names are canonicalized. This method
// modifies the TProxyConfig structure in place.
func (c *TProxyConfig) CanonicalizeDNS() {
domains := make(map[string]DNSAction)
for domain, policy := range c.Domains {
domains[dns.CanonicalName(domain)] = policy
}
c.Domains = domains
cache := make(map[string][]string)
for domain, addrs := range c.DNSCache {
cache[dns.CanonicalName(domain)] = addrs
}
c.DNSCache = cache
}
// TProxy is a model.UnderlyingNetworkLibrary that implements self censorship.
type TProxy struct {
// config contains settings for TProxy.
config *TProxyConfig
// dnsClient is the DNS client we'll internally use.
dnsClient model.Resolver
// dnsListener is the DNS listener.
dnsListener DNSListener
// httpListener is the HTTP listener.
httpListener net.Listener
// listenUDP allows overriding net.ListenUDP calls in tests
listenUDP func(network string, laddr *net.UDPAddr) (model.UDPLikeConn, error)
// logger is the underlying logger to use.
logger model.InfoLogger
// tlsListener is the TLS listener.
tlsListener net.Listener
}
//
// Constructor and destructor
//
// NewTProxy creates a new TProxy instance.
func NewTProxy(config *TProxyConfig, logger model.InfoLogger) (*TProxy, error) {
return newTProxy(config, logger, "127.0.0.1:0", "127.0.0.1:0", "127.0.0.1:0")
}
func newTProxy(config *TProxyConfig, logger model.InfoLogger, dnsListenerAddr,
tlsListenerAddr, httpListenerAddr string) (*TProxy, error) {
p := &TProxy{
config: config,
listenUDP: func(network string, laddr *net.UDPAddr) (model.UDPLikeConn, error) {
return net.ListenUDP(network, laddr)
},
logger: logger,
}
if err := p.newDNSListener(dnsListenerAddr); err != nil {
return nil, err
}
p.newDNSClient(logger)
if err := p.newTLSListener(tlsListenerAddr, logger); err != nil {
p.dnsListener.Close()
return nil, err
}
if err := p.newHTTPListener(httpListenerAddr); err != nil {
p.dnsListener.Close()
p.tlsListener.Close()
return nil, err
}
return p, nil
}
func (p *TProxy) newDNSListener(listenAddr string) error {
var err error
dnsProxy := &DNSProxy{Cache: p.config.DNSCache, OnQuery: p.onQuery}
p.dnsListener, err = dnsProxy.Start(listenAddr)
return err
}
func (p *TProxy) newDNSClient(logger model.DebugLogger) {
dialer := netxlite.NewDialerWithoutResolver(logger)
p.dnsClient = netxlite.NewResolverUDP(logger, dialer, p.dnsListener.LocalAddr().String())
}
func (p *TProxy) newTLSListener(listenAddr string, logger model.DebugLogger) error {
var err error
tlsProxy := &TLSProxy{OnIncomingSNI: p.onIncomingSNI}
p.tlsListener, err = tlsProxy.Start(listenAddr)
return err
}
func (p *TProxy) newHTTPListener(listenAddr string) error {
var err error
httpProxy := &HTTPProxy{OnIncomingHost: p.onIncomingHost}
p.httpListener, err = httpProxy.Start(listenAddr)
return err
}
// Close closes the resources used by a TProxy.
func (p *TProxy) Close() error {
p.dnsClient.CloseIdleConnections()
p.dnsListener.Close()
p.httpListener.Close()
p.tlsListener.Close()
return nil
}
//
// QUIC
//
// ListenUDP implements netxlite.TProxy.ListenUDP.
func (p *TProxy) ListenUDP(network string, laddr *net.UDPAddr) (model.UDPLikeConn, error) {
pconn, err := p.listenUDP(network, laddr)
if err != nil {
return nil, err
}
return &tProxyUDPLikeConn{UDPLikeConn: pconn, proxy: p}, nil
}
// tProxyUDPLikeConn is a TProxy-aware UDPLikeConn.
type tProxyUDPLikeConn struct {
// UDPLikeConn is the underlying conn type.
model.UDPLikeConn
// proxy refers to the TProxy.
proxy *TProxy
}
// WriteTo implements UDPLikeConn.WriteTo. This function will
// apply the proper tproxy policies, if required.
func (c *tProxyUDPLikeConn) WriteTo(pkt []byte, addr net.Addr) (int, error) {
endpoint := fmt.Sprintf("%s/%s", addr.String(), addr.Network())
policy := c.proxy.config.Endpoints[endpoint]
switch policy {
case TProxyPolicyDropData:
c.proxy.logger.Infof("tproxy: WriteTo: %s => %s", endpoint, policy)
return len(pkt), nil
default:
return c.UDPLikeConn.WriteTo(pkt, addr)
}
}
//
// System resolver
//
// LookupHost implements netxlite.TProxy.LookupHost.
func (p *TProxy) LookupHost(ctx context.Context, domain string) ([]string, error) {
return p.dnsClient.LookupHost(ctx, domain)
}
//
// Dialer
//
// NewSimpleDialer implements netxlite.TProxy.NewTProxyDialer.
func (p *TProxy) NewSimpleDialer(timeout time.Duration) model.SimpleDialer {
return &tProxyDialer{
dialer: &net.Dialer{Timeout: timeout},
proxy: p,
}
}
// tProxyDialer is a TProxy-aware Dialer.
type tProxyDialer struct {
// dialer is the underlying network dialer.
dialer *net.Dialer
// proxy refers to the TProxy.
proxy *TProxy
}
// DialContext behaves like net.Dialer.DialContext. This function will
// apply the proper tproxy policies, if required.
func (d *tProxyDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
endpoint := fmt.Sprintf("%s/%s", address, network)
policy := d.proxy.config.Endpoints[endpoint]
switch policy {
case TProxyPolicyTCPDropSYN:
d.proxy.logger.Infof("tproxy: DialContext: %s/%s => %s", address, network, policy)
var cancel context.CancelFunc
const timeout = 70 * time.Second
ctx, cancel = context.WithTimeout(ctx, timeout)
defer cancel()
<-ctx.Done()
return nil, errors.New("i/o timeout")
case TProxyPolicyTCPRejectSYN:
d.proxy.logger.Infof("tproxy: DialContext: %s/%s => %s", address, network, policy)
return nil, netxlite.ECONNREFUSED
case TProxyPolicyHijackDNS:
d.proxy.logger.Infof("tproxy: DialContext: %s/%s => %s", address, network, policy)
address = d.proxy.dnsListener.LocalAddr().String()
case TProxyPolicyHijackTLS:
d.proxy.logger.Infof("tproxy: DialContext: %s/%s => %s", address, network, policy)
address = d.proxy.tlsListener.Addr().String()
case TProxyPolicyHijackHTTP:
d.proxy.logger.Infof("tproxy: DialContext: %s/%s => %s", address, network, policy)
address = d.proxy.httpListener.Addr().String()
default:
// nothing
}
conn, err := d.dialer.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
return &tProxyConn{Conn: conn, proxy: d.proxy}, nil
}
// tProxyConn is a TProxy-aware net.Conn.
type tProxyConn struct {
// Conn is the underlying conn.
net.Conn
// proxy refers to the TProxy.
proxy *TProxy
}
// Write implements Conn.Write. This function will apply
// the proper tproxy policies, if required.
func (c *tProxyConn) Write(b []byte) (int, error) {
addr := c.Conn.RemoteAddr()
endpoint := fmt.Sprintf("%s/%s", addr.String(), addr.Network())
policy := c.proxy.config.Endpoints[endpoint]
switch policy {
case TProxyPolicyDropData:
c.proxy.logger.Infof("tproxy: Write: %s => %s", endpoint, policy)
return len(b), nil
default:
return c.Conn.Write(b)
}
}
//
// Filtering policies implementation
//
// onQuery is called for filtering outgoing DNS queries.
func (p *TProxy) onQuery(domain string) DNSAction {
policy := p.config.Domains[domain]
if policy == "" {
policy = DNSActionPass
} else {
p.logger.Infof("tproxy: DNS: %s => %s", domain, policy)
}
return policy
}
// onIncomingSNI is called for filtering SNI values.
func (p *TProxy) onIncomingSNI(sni string) TLSAction {
policy := p.config.SNIs[sni]
if policy == "" {
policy = TLSActionPass
} else {
p.logger.Infof("tproxy: TLS: %s => %s", sni, policy)
}
return policy
}
// onIncomingHost is called for filtering HTTP hosts.
func (p *TProxy) onIncomingHost(host string) HTTPAction {
policy := p.config.Hosts[host]
if policy == "" {
policy = HTTPActionPass
} else {
p.logger.Infof("tproxy: HTTP: %s => %s", host, policy)
}
return policy
}

578
internal/netxlite/filtering/tproxy_test.go
View File

@ -1,578 +0,0 @@
package filtering
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/model/mocks"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// tProxyDialerAdapter adapts a netxlite.TProxyDialer to be a netxlite.Dialer.
type tProxyDialerAdapter struct {
model.SimpleDialer
}
// CloseIdleConnections implements Dialer.CloseIdleConnections.
func (*tProxyDialerAdapter) CloseIdleConnections() {
// nothing
}
func TestNewTProxyConfig(t *testing.T) {
t.Run("with nonexistent file", func(t *testing.T) {
config, err := NewTProxyConfig(filepath.Join("testdata", "nonexistent"))
if !errors.Is(err, syscall.ENOENT) {
t.Fatal("unexpected err", err)
}
if config != nil {
t.Fatal("expected nil config here")
}
})
t.Run("with file containing invalid JSON", func(t *testing.T) {
config, err := NewTProxyConfig(filepath.Join("testdata", "invalid.json"))
if err == nil || !strings.HasSuffix(err.Error(), "unexpected end of JSON input") {
t.Fatal("unexpected err", err)
}
if config != nil {
t.Fatal("expected nil config here")
}
})
t.Run("with file containing valid JSON", func(t *testing.T) {
config, err := NewTProxyConfig(filepath.Join("testdata", "valid.json"))
if err != nil {
t.Fatal(err)
}
if config == nil {
t.Fatal("expected non-nil config here")
}
if config.Domains["x.org."] != "pass" {
t.Fatal("did not auto-canonicalize config.Domains")
}
if len(config.DNSCache["dns.google."]) != 2 {
t.Fatal("did not auto-canonicalize config.DNSCache")
}
})
}
func TestNewTProxy(t *testing.T) {
t.Run("successful creation and destruction", func(t *testing.T) {
config := &TProxyConfig{}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
if err := proxy.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("cannot create DNS listener", func(t *testing.T) {
config := &TProxyConfig{}
proxy, err := newTProxy(config, log.Log, "127.0.0.1", "", "")
if err == nil || !strings.HasSuffix(err.Error(), "missing port in address") {
t.Fatal("unexpected err", err)
}
if proxy != nil {
t.Fatal("expected nil proxy here")
}
})
t.Run("cannot create TLS listener", func(t *testing.T) {
config := &TProxyConfig{}
proxy, err := newTProxy(config, log.Log, "127.0.0.1:0", "127.0.0.1", "")
if err == nil || !strings.HasSuffix(err.Error(), "missing port in address") {
t.Fatal("unexpected err", err)
}
if proxy != nil {
t.Fatal("expected nil proxy here")
}
})
t.Run("cannot create HTTP listener", func(t *testing.T) {
config := &TProxyConfig{}
proxy, err := newTProxy(config, log.Log, "127.0.0.1:0", "127.0.0.1:0", "127.0.0.1")
if err == nil || !strings.HasSuffix(err.Error(), "missing port in address") {
t.Fatal("unexpected err", err)
}
if proxy != nil {
t.Fatal("expected nil proxy here")
}
})
}
func TestTProxyQUIC(t *testing.T) {
t.Run("ListenUDP", func(t *testing.T) {
t.Run("failure", func(t *testing.T) {
proxy, err := NewTProxy(&TProxyConfig{}, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
pconn, err := proxy.ListenUDP("tcp", &net.UDPAddr{})
if err == nil || !strings.HasSuffix(err.Error(), "unknown network tcp") {
t.Fatal("unexpected err", err)
}
if pconn != nil {
t.Fatal("expected nil pconn here")
}
})
t.Run("success", func(t *testing.T) {
proxy, err := NewTProxy(&TProxyConfig{}, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
pconn, err := proxy.ListenUDP("udp", &net.UDPAddr{})
if err != nil {
t.Fatal(err)
}
uconn := pconn.(*tProxyUDPLikeConn)
if uconn.proxy != proxy {
t.Fatal("proxy not correctly set")
}
if _, okay := uconn.UDPLikeConn.(*net.UDPConn); !okay {
t.Fatal("underlying connection should be an UDPConn")
}
uconn.Close()
})
})
t.Run("WriteTo", func(t *testing.T) {
t.Run("without the drop policy", func(t *testing.T) {
proxy, err := NewTProxy(&TProxyConfig{}, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
var called bool
proxy.listenUDP = func(network string, laddr *net.UDPAddr) (model.UDPLikeConn, error) {
return &mocks.UDPLikeConn{
MockWriteTo: func(p []byte, addr net.Addr) (int, error) {
called = true
return len(p), nil
},
}, nil
}
pconn, err := proxy.ListenUDP("udp", &net.UDPAddr{})
if err != nil {
t.Fatal(err)
}
data := make([]byte, 128)
count, err := pconn.WriteTo(data, &net.UDPAddr{})
if err != nil {
t.Fatal(err)
}
if count != len(data) {
t.Fatal("unexpected number of bytes written")
}
if !called {
t.Fatal("not called")
}
})
t.Run("with the drop policy", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"127.0.0.1:1234/udp": TProxyPolicyDropData,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
var called bool
proxy.listenUDP = func(network string, laddr *net.UDPAddr) (model.UDPLikeConn, error) {
return &mocks.UDPLikeConn{
MockWriteTo: func(p []byte, addr net.Addr) (int, error) {
called = true
return len(p), nil
},
}, nil
}
pconn, err := proxy.ListenUDP("udp", &net.UDPAddr{})
if err != nil {
t.Fatal(err)
}
data := make([]byte, 128)
destAddr := &net.UDPAddr{
IP: net.IPv4(127, 0, 0, 1),
Port: 1234,
Zone: "",
}
count, err := pconn.WriteTo(data, destAddr)
if err != nil {
t.Fatal(err)
}
if count != len(data) {
t.Fatal("unexpected number of bytes written")
}
if called {
t.Fatal("called")
}
})
})
}
func TestTProxyLookupHost(t *testing.T) {
t.Run("without filtering", func(t *testing.T) {
config := &TProxyConfig{}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
ctx := context.Background()
addrs, err := proxy.LookupHost(ctx, "dns.google")
if err != nil {
t.Fatal(err)
}
if len(addrs) < 2 {
t.Fatal("too few addrs")
}
})
t.Run("with filtering", func(t *testing.T) {
config := &TProxyConfig{
Domains: map[string]DNSAction{
"dns.google.": DNSActionNXDOMAIN,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
ctx := context.Background()
addrs, err := proxy.LookupHost(ctx, "dns.google")
if err == nil || err.Error() != "dns_nxdomain_error" {
t.Fatal("unexpected err", err)
}
if len(addrs) != 0 {
t.Fatal("too many addrs")
}
})
}
func TestTProxyOnIncomingSNI(t *testing.T) {
t.Run("without filtering", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"8.8.8.8:443/tcp": TProxyPolicyHijackTLS,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
ctx := context.Background()
dialer := proxy.NewSimpleDialer(10 * time.Second)
conn, err := dialer.DialContext(ctx, "tcp", "8.8.8.8:443")
if err != nil {
t.Fatal(err)
}
tconn := tls.Client(conn, &tls.Config{ServerName: "dns.google"})
err = tconn.HandshakeContext(ctx)
if err != nil {
t.Fatal(err)
}
tconn.Close()
})
t.Run("with filtering", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"8.8.8.8:443/tcp": TProxyPolicyHijackTLS,
},
SNIs: map[string]TLSAction{
"dns.google": TLSActionReset,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
ctx := context.Background()
dialer := proxy.NewSimpleDialer(10 * time.Second)
conn, err := dialer.DialContext(ctx, "tcp", "8.8.8.8:443")
if err != nil {
t.Fatal(err)
}
tlsh := netxlite.NewTLSHandshakerStdlib(log.Log)
tconn, _, err := tlsh.Handshake(ctx, conn, &tls.Config{ServerName: "dns.google"})
if err == nil || err.Error() != netxlite.FailureConnectionReset {
t.Fatal("unexpected err", err)
}
if tconn != nil {
t.Fatal("expected nil tconn")
}
conn.Close()
})
}
func TestTProxyOnIncomingHost(t *testing.T) {
t.Run("without filtering", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"130.192.16.171:80/tcp": TProxyPolicyHijackHTTP,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
dialer := proxy.NewSimpleDialer(10 * time.Second)
req, err := http.NewRequest("GET", "http://130.192.16.171:80", nil)
if err != nil {
t.Fatal(err)
}
req.Host = "nexa.polito.it"
txp := &http.Transport{DialContext: dialer.DialContext}
resp, err := txp.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
})
t.Run("with filtering", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"130.192.16.171:80/tcp": TProxyPolicyHijackHTTP,
},
Hosts: map[string]HTTPAction{
"nexa.polito.it": HTTPActionReset,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
dialer := netxlite.WrapDialer(
log.Log,
netxlite.NewResolverStdlib(log.Log),
&tProxyDialerAdapter{
proxy.NewSimpleDialer(10 * time.Second),
},
)
req, err := http.NewRequest("GET", "http://130.192.16.171:80", nil)
if err != nil {
t.Fatal(err)
}
req.Host = "nexa.polito.it"
txp := &http.Transport{DialContext: dialer.DialContext}
resp, err := txp.RoundTrip(req)
if err == nil || !strings.HasSuffix(err.Error(), netxlite.FailureConnectionReset) {
t.Fatal("unexpected err", err)
}
if resp != nil {
t.Fatal("expected nil resp here")
}
})
}
func TestTProxyDial(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
t.Run("with drop SYN", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"130.192.16.171:80/tcp": TProxyPolicyTCPDropSYN,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
dialer := proxy.NewSimpleDialer(10 * time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", "http://130.192.16.171:80", nil)
if err != nil {
t.Fatal(err)
}
req.Host = "nexa.polito.it"
txp := &http.Transport{DialContext: dialer.DialContext}
resp, err := txp.RoundTrip(req)
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatal("unexpected err", err)
}
if resp != nil {
t.Fatal("expected nil resp here")
}
})
t.Run("with reject SYN", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"130.192.16.171:80/tcp": TProxyPolicyTCPRejectSYN,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
dialer := netxlite.WrapDialer(log.Log,
netxlite.NewResolverStdlib(log.Log),
&tProxyDialerAdapter{
proxy.NewSimpleDialer(10 * time.Second)})
req, err := http.NewRequest("GET", "http://130.192.16.171:80", nil)
if err != nil {
t.Fatal(err)
}
req.Host = "nexa.polito.it"
txp := &http.Transport{DialContext: dialer.DialContext}
resp, err := txp.RoundTrip(req)
if err == nil || !strings.HasSuffix(err.Error(), netxlite.FailureConnectionRefused) {
t.Fatal("unexpected err", err)
}
if resp != nil {
t.Fatal("expected nil resp here")
}
})
t.Run("with drop data", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"130.192.16.171:80/tcp": TProxyPolicyDropData,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
dialer := proxy.NewSimpleDialer(10 * time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(
ctx, "GET", "http://130.192.16.171:80", nil)
if err != nil {
t.Fatal(err)
}
req.Host = "nexa.polito.it"
txp := &http.Transport{DialContext: dialer.DialContext}
resp, err := txp.RoundTrip(req)
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatal("unexpected err", err)
}
if resp != nil {
t.Fatal("expected nil resp here")
}
})
t.Run("with hijack DNS", func(t *testing.T) {
config := &TProxyConfig{
Endpoints: map[string]TProxyPolicy{
"8.8.8.8:53/udp": TProxyPolicyHijackDNS,
},
Domains: map[string]DNSAction{
"example.com.": DNSActionNXDOMAIN,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
dialer := proxy.NewSimpleDialer(10 * time.Second)
resolver := netxlite.NewResolverUDP(
log.Log, &tProxyDialerAdapter{dialer}, "8.8.8.8:53")
addrs, err := resolver.LookupHost(context.Background(), "example.com")
if err == nil || err.Error() != netxlite.FailureDNSNXDOMAINError {
t.Fatal("unexpected err", err)
}
if len(addrs) != 0 {
t.Fatal("expected no addrs here")
}
})
t.Run("with invalid destination address", func(t *testing.T) {
config := &TProxyConfig{}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
defer proxy.Close()
dialer := proxy.NewSimpleDialer(10 * time.Second)
ctx := context.Background()
conn, err := dialer.DialContext(ctx, "tcp", "127.0.0.1")
if err == nil || !strings.HasSuffix(err.Error(), "missing port in address") {
t.Fatal("unexpected err", err)
}
if conn != nil {
t.Fatal("expected nil conn here")
}
})
}
func TestTProxyDNSCache(t *testing.T) {
t.Run("without cache but with the cache rule", func(t *testing.T) {
config := &TProxyConfig{
Domains: map[string]DNSAction{
"dns.google.": DNSActionCache,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
addrs, err := proxy.LookupHost(ctx, "dns.google")
if err == nil || err.Error() != netxlite.FailureDNSNXDOMAINError {
t.Fatal("unexpected err", err)
}
if addrs != nil {
t.Fatal("expected nil addrs")
}
})
t.Run("with cache", func(t *testing.T) {
config := &TProxyConfig{
DNSCache: map[string][]string{
"dns.google.": {"8.8.8.8", "8.8.4.4"},
},
Domains: map[string]DNSAction{
"dns.google.": DNSActionCache,
},
}
proxy, err := NewTProxy(config, log.Log)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
addrs, err := proxy.LookupHost(ctx, "dns.google")
if err != nil {
t.Fatal(err)
}
if len(addrs) != 2 {
t.Fatal("expected two addrs")
}
if addrs[0] != "8.8.8.8" {
t.Fatal("invalid first address")
}
if addrs[1] != "8.8.4.4" {
t.Fatal("invalid second address")
}
})
}

49
internal/ooapi/apimodel/checkin.go
View File

@ -1,49 +0,0 @@
package apimodel
import "github.com/ooni/probe-cli/v3/internal/model"
// CheckInRequestWebConnectivity contains WebConnectivity
// specific parameters to include into CheckInRequest
type CheckInRequestWebConnectivity struct {
CategoryCodes []string `json:"category_codes"`
}
// CheckInRequest is the check-in API request
type CheckInRequest struct {
Charging bool `json:"charging"`
OnWiFi bool `json:"on_wifi"`
Platform string `json:"platform"`
ProbeASN string `json:"probe_asn"`
ProbeCC string `json:"probe_cc"`
RunType model.RunType `json:"run_type"`
SoftwareName string `json:"software_name"`
SoftwareVersion string `json:"software_version"`
WebConnectivity CheckInRequestWebConnectivity `json:"web_connectivity"`
}
// CheckInResponseURLInfo contains information about an URL.
type CheckInResponseURLInfo struct {
CategoryCode string `json:"category_code"`
CountryCode string `json:"country_code"`
URL string `json:"url"`
}
// CheckInResponseWebConnectivity contains WebConnectivity
// specific information of a CheckInResponse
type CheckInResponseWebConnectivity struct {
ReportID string `json:"report_id"`
URLs []CheckInResponseURLInfo `json:"urls"`
}
// CheckInResponse is the check-in API response
type CheckInResponse struct {
ProbeASN string `json:"probe_asn"`
ProbeCC string `json:"probe_cc"`
Tests CheckInResponseTests `json:"tests"`
V int64 `json:"v"`
}
// CheckInResponseTests contains configuration for tests
type CheckInResponseTests struct {
WebConnectivity CheckInResponseWebConnectivity `json:"web_connectivity"`
}

13
internal/ooapi/apimodel/checkreportid.go
View File

@ -1,13 +0,0 @@
package apimodel
// CheckReportIDRequest is the CheckReportID request.
type CheckReportIDRequest struct {
ReportID string `query:"report_id" required:"true"`
}
// CheckReportIDResponse is the CheckReportID response.
type CheckReportIDResponse struct {
Error string `json:"error"`
Found bool `json:"found"`
V int64 `json:"v"`
}

22
internal/ooapi/apimodel/doc.go
View File

@ -1,22 +0,0 @@
// Package apimodel describes the data types used by OONI's API.
//
// If you edit this package to integrate the data model, remember to
// run `go generate ./...`.
//
// We annotate fields with tagging. When a field should be sent
// over as JSON, use the usual `json` tag.
//
// When a field needs to be sent using the query string, use
// the `query` tag instead. We limit what can be sent using the
// query string to int64, string, and bool.
//
// The `path` tag indicates that the URL path contains a
// template. We will replace the value of this field with
// the template. Note that the template should use the
// Go name of the field (e.g. `{{ .ReportID }}`) as opposed
// to the name in the tag, which is only used when we
// generate the API Swagger.
//
// The `required` tag indicates required fields. A required
// field cannot be empty (for the Go definition of empty).
package apimodel

15
internal/ooapi/apimodel/login.go
View File

@ -1,15 +0,0 @@
package apimodel
import "time"
// LoginRequest is the login API request
type LoginRequest struct {
ClientID string `json:"username"`
Password string `json:"password"`
}
// LoginResponse is the login API response
type LoginResponse struct {
Expire time.Time `json:"expire"`
Token string `json:"token"`
}

25
internal/ooapi/apimodel/measurementmeta.go
View File

@ -1,25 +0,0 @@
package apimodel
// MeasurementMetaRequest is the MeasurementMeta Request.
type MeasurementMetaRequest struct {
ReportID string `query:"report_id" required:"true"`
Full bool `query:"full"`
Input string `query:"input"`
}
// MeasurementMetaResponse is the MeasurementMeta Response.
type MeasurementMetaResponse struct {
Anomaly bool `json:"anomaly"`
CategoryCode string `json:"category_code"`
Confirmed bool `json:"confirmed"`
Failure bool `json:"failure"`
Input string `json:"input"`
MeasurementStartTime string `json:"measurement_start_time"`
ProbeASN int64 `json:"probe_asn"`
ProbeCC string `json:"probe_cc"`
RawMeasurement string `json:"raw_measurement"`
ReportID string `json:"report_id"`
Scores string `json:"scores"`
TestName string `json:"test_name"`
TestStartTime string `json:"test_start_time"`
}

21
internal/ooapi/apimodel/openreport.go
View File

@ -1,21 +0,0 @@
package apimodel
// OpenReportRequest is the OpenReport request.
type OpenReportRequest struct {
DataFormatVersion string `json:"data_format_version"`
Format string `json:"format"`
ProbeASN string `json:"probe_asn"`
ProbeCC string `json:"probe_cc"`
SoftwareName string `json:"software_name"`
SoftwareVersion string `json:"software_version"`
TestName string `json:"test_name"`
TestStartTime string `json:"test_start_time"`
TestVersion string `json:"test_version"`
}
// OpenReportResponse is the OpenReport response.
type OpenReportResponse struct {
BackendVersion string `json:"backend_version"`
ReportID string `json:"report_id"`
SupportedFormats []string `json:"supported_formats"`
}

7
internal/ooapi/apimodel/psiphonconfig.go
View File

@ -1,7 +0,0 @@
package apimodel
// PsiphonConfigRequest is the request for the PsiphonConfig API
type PsiphonConfigRequest struct{}
// PsiphonConfigResponse is the response from the PsiphonConfig API
type PsiphonConfigResponse map[string]interface{}

26
internal/ooapi/apimodel/register.go
View File

@ -1,26 +0,0 @@
package apimodel
// RegisterRequest is the request for the Register API.
type RegisterRequest struct {
// just password
Password string `json:"password"`
// metadata
AvailableBandwidth string `json:"available_bandwidth,omitempty"`
DeviceToken string `json:"device_token,omitempty"`
Language string `json:"language,omitempty"`
NetworkType string `json:"network_type,omitempty"`
Platform string `json:"platform"`
ProbeASN string `json:"probe_asn"`
ProbeCC string `json:"probe_cc"`
ProbeFamily string `json:"probe_family,omitempty"`
ProbeTimezone string `json:"probe_timezone,omitempty"`
SoftwareName string `json:"software_name"`
SoftwareVersion string `json:"software_version"`
SupportedTests []string `json:"supported_tests"`
}
// RegisterResponse is the response from the Register API.
type RegisterResponse struct {
ClientID string `json:"client_id"`
}

13
internal/ooapi/apimodel/submitmeasurement.go
View File

@ -1,13 +0,0 @@
package apimodel
// SubmitMeasurementRequest is the SubmitMeasurement request.
type SubmitMeasurementRequest struct {
ReportID string `path:"report_id"`
Format string `json:"format"`
Content interface{} `json:"content"`
}
// SubmitMeasurementResponse is the SubmitMeasurement response.
type SubmitMeasurementResponse struct {
MeasurementUID string `json:"measurement_uid"`
}

15
internal/ooapi/apimodel/testhelpers.go
View File

@ -1,15 +0,0 @@
package apimodel
// TestHelpersRequest is the TestHelpers request.
type TestHelpersRequest struct{}
// TestHelpersResponse is the TestHelpers response.
type TestHelpersResponse map[string][]TestHelpersHelperInfo
// TestHelpersHelperInfo is a single helper within the
// response returned by the TestHelpers API.
type TestHelpersHelperInfo struct {
Address string `json:"address"`
Type string `json:"type"`
Front string `json:"front,omitempty"`
}

16
internal/ooapi/apimodel/tortargets.go
View File

@ -1,16 +0,0 @@
package apimodel
// TorTargetsRequest is a request for the TorTargets API.
type TorTargetsRequest struct{}
// TorTargetsResponse is the response from the TorTargets API.
type TorTargetsResponse map[string]TorTargetsTarget
// TorTargetsTarget is a target for the tor experiment.
type TorTargetsTarget struct {
Address string `json:"address"`
Name string `json:"name"`
Params map[string][]string `json:"params"`
Protocol string `json:"protocol"`
Source string `json:"source"`
}

26
internal/ooapi/apimodel/urls.go
View File

@ -1,26 +0,0 @@
package apimodel
// URLsRequest is the URLs request.
type URLsRequest struct {
CategoryCodes string `query:"category_codes"`
CountryCode string `query:"country_code"`
Limit int64 `query:"limit"`
}
// URLsResponse is the URLs response.
type URLsResponse struct {
Metadata URLsMetadata `json:"metadata"`
Results []URLsResponseURL `json:"results"`
}
// URLsMetadata contains metadata in the URLs response.
type URLsMetadata struct {
Count int64 `json:"count"`
}
// URLsResponseURL is a single URL in the URLs response.
type URLsResponseURL struct {
CategoryCode string `json:"category_code"`
CountryCode string `json:"country_code"`
URL string `json:"url"`
}

618
internal/ooapi/apis.go
View File

@ -1,618 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:46.371368 +0200 CEST m=+0.001320792
package ooapi
//go:generate go run ./internal/generator -file apis.go
import (
"context"
"net/http"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
// simpleCheckReportIDAPI implements the CheckReportID API.
type simpleCheckReportIDAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
UserAgent string // optional
}
func (api *simpleCheckReportIDAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleCheckReportIDAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleCheckReportIDAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleCheckReportIDAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the CheckReportID API.
func (api *simpleCheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleCheckInAPI implements the CheckIn API.
type simpleCheckInAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
UserAgent string // optional
}
func (api *simpleCheckInAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleCheckInAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleCheckInAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleCheckInAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the CheckIn API.
func (api *simpleCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleLoginAPI implements the Login API.
type simpleLoginAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
UserAgent string // optional
}
func (api *simpleLoginAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleLoginAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleLoginAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleLoginAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the Login API.
func (api *simpleLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleMeasurementMetaAPI implements the MeasurementMeta API.
type simpleMeasurementMetaAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
UserAgent string // optional
}
func (api *simpleMeasurementMetaAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleMeasurementMetaAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleMeasurementMetaAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleMeasurementMetaAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the MeasurementMeta API.
func (api *simpleMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleRegisterAPI implements the Register API.
type simpleRegisterAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
UserAgent string // optional
}
func (api *simpleRegisterAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleRegisterAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleRegisterAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleRegisterAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the Register API.
func (api *simpleRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleTestHelpersAPI implements the TestHelpers API.
type simpleTestHelpersAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
UserAgent string // optional
}
func (api *simpleTestHelpersAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleTestHelpersAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleTestHelpersAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleTestHelpersAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the TestHelpers API.
func (api *simpleTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simplePsiphonConfigAPI implements the PsiphonConfig API.
type simplePsiphonConfigAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
Token string // mandatory
RequestMaker RequestMaker // optional
UserAgent string // optional
}
// WithToken returns a copy of the API where the
// value of the Token field is replaced with token.
func (api *simplePsiphonConfigAPI) WithToken(token string) callerForPsiphonConfigAPI {
out := &simplePsiphonConfigAPI{}
out.BaseURL = api.BaseURL
out.HTTPClient = api.HTTPClient
out.JSONCodec = api.JSONCodec
out.RequestMaker = api.RequestMaker
out.UserAgent = api.UserAgent
out.Token = token
return out
}
func (api *simplePsiphonConfigAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simplePsiphonConfigAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simplePsiphonConfigAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simplePsiphonConfigAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the PsiphonConfig API.
func (api *simplePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.Token == "" {
return nil, ErrMissingToken
}
httpReq.Header.Add("Authorization", newAuthorizationHeader(api.Token))
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleTorTargetsAPI implements the TorTargets API.
type simpleTorTargetsAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
Token string // mandatory
RequestMaker RequestMaker // optional
UserAgent string // optional
}
// WithToken returns a copy of the API where the
// value of the Token field is replaced with token.
func (api *simpleTorTargetsAPI) WithToken(token string) callerForTorTargetsAPI {
out := &simpleTorTargetsAPI{}
out.BaseURL = api.BaseURL
out.HTTPClient = api.HTTPClient
out.JSONCodec = api.JSONCodec
out.RequestMaker = api.RequestMaker
out.UserAgent = api.UserAgent
out.Token = token
return out
}
func (api *simpleTorTargetsAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleTorTargetsAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleTorTargetsAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleTorTargetsAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the TorTargets API.
func (api *simpleTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.Token == "" {
return nil, ErrMissingToken
}
httpReq.Header.Add("Authorization", newAuthorizationHeader(api.Token))
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleURLsAPI implements the URLs API.
type simpleURLsAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
UserAgent string // optional
}
func (api *simpleURLsAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleURLsAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleURLsAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleURLsAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the URLs API.
func (api *simpleURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleOpenReportAPI implements the OpenReport API.
type simpleOpenReportAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
UserAgent string // optional
}
func (api *simpleOpenReportAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleOpenReportAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleOpenReportAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleOpenReportAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the OpenReport API.
func (api *simpleOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}
// simpleSubmitMeasurementAPI implements the SubmitMeasurement API.
type simpleSubmitMeasurementAPI struct {
BaseURL string // optional
HTTPClient HTTPClient // optional
JSONCodec JSONCodec // optional
RequestMaker RequestMaker // optional
TemplateExecutor templateExecutor // optional
UserAgent string // optional
}
func (api *simpleSubmitMeasurementAPI) baseURL() string {
if api.BaseURL != "" {
return api.BaseURL
}
return "https://ps1.ooni.io"
}
func (api *simpleSubmitMeasurementAPI) requestMaker() RequestMaker {
if api.RequestMaker != nil {
return api.RequestMaker
}
return &defaultRequestMaker{}
}
func (api *simpleSubmitMeasurementAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *simpleSubmitMeasurementAPI) templateExecutor() templateExecutor {
if api.TemplateExecutor != nil {
return api.TemplateExecutor
}
return &defaultTemplateExecutor{}
}
func (api *simpleSubmitMeasurementAPI) httpClient() HTTPClient {
if api.HTTPClient != nil {
return api.HTTPClient
}
return http.DefaultClient
}
// Call calls the SubmitMeasurement API.
func (api *simpleSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error) {
httpReq, err := api.newRequest(ctx, req)
if err != nil {
return nil, err
}
httpReq.Header.Add("Accept", "application/json")
if api.UserAgent != "" {
httpReq.Header.Add("User-Agent", api.UserAgent)
}
httpResp, err := api.httpClient().Do(httpReq)
return api.newResponse(ctx, httpResp, err)
}

2776
internal/ooapi/apis_test.go
View File

File diff suppressed because it is too large Load Diff

98
internal/ooapi/caching.go
View File

@ -1,98 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:47.186361 +0200 CEST m=+0.000721751
package ooapi
//go:generate go run ./internal/generator -file caching.go
import (
"context"
"reflect"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
// withCacheMeasurementMetaAPI implements caching for simpleMeasurementMetaAPI.
type withCacheMeasurementMetaAPI struct {
API callerForMeasurementMetaAPI // mandatory
GobCodec GobCodec // optional
KVStore KVStore // mandatory
}
type cacheEntryForMeasurementMetaAPI struct {
Req *apimodel.MeasurementMetaRequest
Resp *apimodel.MeasurementMetaResponse
}
// Call calls the API and implements caching.
func (c *withCacheMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
if resp, _ := c.readcache(req); resp != nil {
return resp, nil
}
resp, err := c.API.Call(ctx, req)
if err != nil {
return nil, err
}
if err := c.writecache(req, resp); err != nil {
return nil, err
}
return resp, nil
}
func (c *withCacheMeasurementMetaAPI) gobCodec() GobCodec {
if c.GobCodec != nil {
return c.GobCodec
}
return &defaultGobCodec{}
}
func (c *withCacheMeasurementMetaAPI) getcache() ([]cacheEntryForMeasurementMetaAPI, error) {
data, err := c.KVStore.Get("MeasurementMeta.cache")
if err != nil {
return nil, err
}
var out []cacheEntryForMeasurementMetaAPI
if err := c.gobCodec().Decode(data, &out); err != nil {
return nil, err
}
return out, nil
}
func (c *withCacheMeasurementMetaAPI) setcache(in []cacheEntryForMeasurementMetaAPI) error {
data, err := c.gobCodec().Encode(in)
if err != nil {
return err
}
return c.KVStore.Set("MeasurementMeta.cache", data)
}
func (c *withCacheMeasurementMetaAPI) readcache(req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
cache, err := c.getcache()
if err != nil {
return nil, err
}
for _, cur := range cache {
if reflect.DeepEqual(req, cur.Req) {
return cur.Resp, nil
}
}
return nil, errCacheNotFound
}
func (c *withCacheMeasurementMetaAPI) writecache(req *apimodel.MeasurementMetaRequest, resp *apimodel.MeasurementMetaResponse) error {
cache, _ := c.getcache()
out := []cacheEntryForMeasurementMetaAPI{{Req: req, Resp: resp}}
const toomany = 64
for idx, cur := range cache {
if reflect.DeepEqual(req, cur.Req) {
continue // we already updated the cache
}
if idx > toomany {
break
}
out = append(out, cur)
}
return c.setcache(out)
}
var _ callerForMeasurementMetaAPI = &withCacheMeasurementMetaAPI{}

223
internal/ooapi/caching_test.go
View File

@ -1,223 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:47.604277 +0200 CEST m=+0.000796918
package ooapi
//go:generate go run ./internal/generator -file caching_test.go
import (
"context"
"errors"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
func TestCachesimpleMeasurementMetaAPISuccess(t *testing.T) {
ff := &fakeFill{}
var expect *apimodel.MeasurementMetaResponse
ff.Fill(&expect)
cache := &withCacheMeasurementMetaAPI{
API: &FakeMeasurementMetaAPI{
Response: expect,
},
KVStore: &kvstore.Memory{},
}
var req *apimodel.MeasurementMetaRequest
ff.Fill(&req)
ctx := context.Background()
resp, err := cache.Call(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response")
}
if diff := cmp.Diff(expect, resp); diff != "" {
t.Fatal(diff)
}
}
func TestCachesimpleMeasurementMetaAPIWriteCacheError(t *testing.T) {
errMocked := errors.New("mocked error")
ff := &fakeFill{}
var expect *apimodel.MeasurementMetaResponse
ff.Fill(&expect)
cache := &withCacheMeasurementMetaAPI{
API: &FakeMeasurementMetaAPI{
Response: expect,
},
KVStore: &FakeKVStore{SetError: errMocked},
}
var req *apimodel.MeasurementMetaRequest
ff.Fill(&req)
ctx := context.Background()
resp, err := cache.Call(ctx, req)
if !errors.Is(err, errMocked) {
t.Fatal("not the error we expected", err)
}
if resp != nil {
t.Fatal("expected nil response")
}
}
func TestCachesimpleMeasurementMetaAPIFailureWithNoCache(t *testing.T) {
errMocked := errors.New("mocked error")
ff := &fakeFill{}
cache := &withCacheMeasurementMetaAPI{
API: &FakeMeasurementMetaAPI{
Err: errMocked,
},
KVStore: &kvstore.Memory{},
}
var req *apimodel.MeasurementMetaRequest
ff.Fill(&req)
ctx := context.Background()
resp, err := cache.Call(ctx, req)
if !errors.Is(err, errMocked) {
t.Fatal("not the error we expected", err)
}
if resp != nil {
t.Fatal("expected nil response")
}
}
func TestCachesimpleMeasurementMetaAPIFailureWithPreviousCache(t *testing.T) {
ff := &fakeFill{}
var expect *apimodel.MeasurementMetaResponse
ff.Fill(&expect)
fakeapi := &FakeMeasurementMetaAPI{
Response: expect,
}
cache := &withCacheMeasurementMetaAPI{
API: fakeapi,
KVStore: &kvstore.Memory{},
}
var req *apimodel.MeasurementMetaRequest
ff.Fill(&req)
ctx := context.Background()
// first pass with no error at all
// use a separate scope to be sure we avoid mistakes
{
resp, err := cache.Call(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response")
}
if diff := cmp.Diff(expect, resp); diff != "" {
t.Fatal(diff)
}
}
// second pass with failure
errMocked := errors.New("mocked error")
fakeapi.Err = errMocked
fakeapi.Response = nil
resp2, err := cache.Call(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp2 == nil {
t.Fatal("expected non-nil response")
}
if diff := cmp.Diff(expect, resp2); diff != "" {
t.Fatal(diff)
}
}
func TestCachesimpleMeasurementMetaAPISetcacheWithEncodeError(t *testing.T) {
ff := &fakeFill{}
errMocked := errors.New("mocked error")
var in []cacheEntryForMeasurementMetaAPI
ff.Fill(&in)
cache := &withCacheMeasurementMetaAPI{
GobCodec: &FakeCodec{EncodeErr: errMocked},
}
err := cache.setcache(in)
if !errors.Is(err, errMocked) {
t.Fatal("not the error we expected", err)
}
}
func TestCachesimpleMeasurementMetaAPIReadCacheNotFound(t *testing.T) {
ff := &fakeFill{}
var incache []cacheEntryForMeasurementMetaAPI
ff.Fill(&incache)
cache := &withCacheMeasurementMetaAPI{
KVStore: &kvstore.Memory{},
}
err := cache.setcache(incache)
if err != nil {
t.Fatal(err)
}
var req *apimodel.MeasurementMetaRequest
ff.Fill(&req)
out, err := cache.readcache(req)
if !errors.Is(err, errCacheNotFound) {
t.Fatal("not the error we expected", err)
}
if out != nil {
t.Fatal("expected nil here")
}
}
func TestCachesimpleMeasurementMetaAPIWriteCacheDuplicate(t *testing.T) {
ff := &fakeFill{}
var req *apimodel.MeasurementMetaRequest
ff.Fill(&req)
var resp1 *apimodel.MeasurementMetaResponse
ff.Fill(&resp1)
var resp2 *apimodel.MeasurementMetaResponse
ff.Fill(&resp2)
cache := &withCacheMeasurementMetaAPI{
KVStore: &kvstore.Memory{},
}
err := cache.writecache(req, resp1)
if err != nil {
t.Fatal(err)
}
err = cache.writecache(req, resp2)
if err != nil {
t.Fatal(err)
}
out, err := cache.readcache(req)
if err != nil {
t.Fatal(err)
}
if out == nil {
t.Fatal("expected non-nil here")
}
if diff := cmp.Diff(resp2, out); diff != "" {
t.Fatal(diff)
}
}
func TestCachesimpleMeasurementMetaAPICacheSizeLimited(t *testing.T) {
ff := &fakeFill{}
cache := &withCacheMeasurementMetaAPI{
KVStore: &kvstore.Memory{},
}
var prev int
for {
var req *apimodel.MeasurementMetaRequest
ff.Fill(&req)
var resp *apimodel.MeasurementMetaResponse
ff.Fill(&resp)
err := cache.writecache(req, resp)
if err != nil {
t.Fatal(err)
}
out, err := cache.getcache()
if err != nil {
t.Fatal(err)
}
if len(out) > prev {
prev = len(out)
continue
}
break
}
}

78
internal/ooapi/callers.go
View File

@ -1,78 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:48.00142 +0200 CEST m=+0.000791126
package ooapi
//go:generate go run ./internal/generator -file callers.go
import (
"context"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
// callerForCheckReportIDAPI represents any type exposing a method
// like simpleCheckReportIDAPI.Call.
type callerForCheckReportIDAPI interface {
Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error)
}
// callerForCheckInAPI represents any type exposing a method
// like simpleCheckInAPI.Call.
type callerForCheckInAPI interface {
Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error)
}
// callerForLoginAPI represents any type exposing a method
// like simpleLoginAPI.Call.
type callerForLoginAPI interface {
Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error)
}
// callerForMeasurementMetaAPI represents any type exposing a method
// like simpleMeasurementMetaAPI.Call.
type callerForMeasurementMetaAPI interface {
Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error)
}
// callerForRegisterAPI represents any type exposing a method
// like simpleRegisterAPI.Call.
type callerForRegisterAPI interface {
Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error)
}
// callerForTestHelpersAPI represents any type exposing a method
// like simpleTestHelpersAPI.Call.
type callerForTestHelpersAPI interface {
Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error)
}
// callerForPsiphonConfigAPI represents any type exposing a method
// like simplePsiphonConfigAPI.Call.
type callerForPsiphonConfigAPI interface {
Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error)
}
// callerForTorTargetsAPI represents any type exposing a method
// like simpleTorTargetsAPI.Call.
type callerForTorTargetsAPI interface {
Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error)
}
// callerForURLsAPI represents any type exposing a method
// like simpleURLsAPI.Call.
type callerForURLsAPI interface {
Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error)
}
// callerForOpenReportAPI represents any type exposing a method
// like simpleOpenReportAPI.Call.
type callerForOpenReportAPI interface {
Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error)
}
// callerForSubmitMeasurementAPI represents any type exposing a method
// like simpleSubmitMeasurementAPI.Call.
type callerForSubmitMeasurementAPI interface {
Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error)
}

18
internal/ooapi/client.go
View File

@ -1,18 +0,0 @@
package ooapi
// Client is a client for speaking with the OONI API. Make sure you
// fill in the mandatory fields.
type Client struct {
// KVStore is the MANDATORY key-value store. You can use
// the kvstore.Memory{} struct for an in-memory store.
KVStore KVStore
// The following fields are optional. When they are empty
// we will fallback to sensible defaults.
BaseURL string
GobCodec GobCodec
HTTPClient HTTPClient
JSONCodec JSONCodec
RequestMaker RequestMaker
UserAgent string
}

214
internal/ooapi/clientcall.go
View File

@ -1,214 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:48.430212 +0200 CEST m=+0.000888918
package ooapi
//go:generate go run ./internal/generator -file clientcall.go
import (
"context"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
func (c *Client) newCheckReportIDCaller() callerForCheckReportIDAPI {
return &simpleCheckReportIDAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
}
}
// CheckReportID calls the CheckReportID API.
func (c *Client) CheckReportID(
ctx context.Context, req *apimodel.CheckReportIDRequest,
) (*apimodel.CheckReportIDResponse, error) {
api := c.newCheckReportIDCaller()
return api.Call(ctx, req)
}
func (c *Client) newCheckInCaller() callerForCheckInAPI {
return &simpleCheckInAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
}
}
// CheckIn calls the CheckIn API.
func (c *Client) CheckIn(
ctx context.Context, req *apimodel.CheckInRequest,
) (*apimodel.CheckInResponse, error) {
api := c.newCheckInCaller()
return api.Call(ctx, req)
}
func (c *Client) newMeasurementMetaCaller() callerForMeasurementMetaAPI {
return &withCacheMeasurementMetaAPI{
API: &simpleMeasurementMetaAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
},
GobCodec: c.GobCodec,
KVStore: c.KVStore,
}
}
// MeasurementMeta calls the MeasurementMeta API.
func (c *Client) MeasurementMeta(
ctx context.Context, req *apimodel.MeasurementMetaRequest,
) (*apimodel.MeasurementMetaResponse, error) {
api := c.newMeasurementMetaCaller()
return api.Call(ctx, req)
}
func (c *Client) newTestHelpersCaller() callerForTestHelpersAPI {
return &simpleTestHelpersAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
}
}
// TestHelpers calls the TestHelpers API.
func (c *Client) TestHelpers(
ctx context.Context, req *apimodel.TestHelpersRequest,
) (apimodel.TestHelpersResponse, error) {
api := c.newTestHelpersCaller()
return api.Call(ctx, req)
}
func (c *Client) newPsiphonConfigCaller() callerForPsiphonConfigAPI {
return &withLoginPsiphonConfigAPI{
API: &simplePsiphonConfigAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
},
JSONCodec: c.JSONCodec,
KVStore: c.KVStore,
RegisterAPI: &simpleRegisterAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
},
LoginAPI: &simpleLoginAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
},
}
}
// PsiphonConfig calls the PsiphonConfig API.
func (c *Client) PsiphonConfig(
ctx context.Context, req *apimodel.PsiphonConfigRequest,
) (apimodel.PsiphonConfigResponse, error) {
api := c.newPsiphonConfigCaller()
return api.Call(ctx, req)
}
func (c *Client) newTorTargetsCaller() callerForTorTargetsAPI {
return &withLoginTorTargetsAPI{
API: &simpleTorTargetsAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
},
JSONCodec: c.JSONCodec,
KVStore: c.KVStore,
RegisterAPI: &simpleRegisterAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
},
LoginAPI: &simpleLoginAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
},
}
}
// TorTargets calls the TorTargets API.
func (c *Client) TorTargets(
ctx context.Context, req *apimodel.TorTargetsRequest,
) (apimodel.TorTargetsResponse, error) {
api := c.newTorTargetsCaller()
return api.Call(ctx, req)
}
func (c *Client) newURLsCaller() callerForURLsAPI {
return &simpleURLsAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
}
}
// URLs calls the URLs API.
func (c *Client) URLs(
ctx context.Context, req *apimodel.URLsRequest,
) (*apimodel.URLsResponse, error) {
api := c.newURLsCaller()
return api.Call(ctx, req)
}
func (c *Client) newOpenReportCaller() callerForOpenReportAPI {
return &simpleOpenReportAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
}
}
// OpenReport calls the OpenReport API.
func (c *Client) OpenReport(
ctx context.Context, req *apimodel.OpenReportRequest,
) (*apimodel.OpenReportResponse, error) {
api := c.newOpenReportCaller()
return api.Call(ctx, req)
}
func (c *Client) newSubmitMeasurementCaller() callerForSubmitMeasurementAPI {
return &simpleSubmitMeasurementAPI{
BaseURL: c.BaseURL,
HTTPClient: c.HTTPClient,
JSONCodec: c.JSONCodec,
RequestMaker: c.RequestMaker,
UserAgent: c.UserAgent,
}
}
// SubmitMeasurement calls the SubmitMeasurement API.
func (c *Client) SubmitMeasurement(
ctx context.Context, req *apimodel.SubmitMeasurementRequest,
) (*apimodel.SubmitMeasurementResponse, error) {
api := c.newSubmitMeasurementCaller()
return api.Call(ctx, req)
}

899
internal/ooapi/clientcall_test.go
View File

@ -1,899 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:48.846389 +0200 CEST m=+0.000695543
package ooapi
//go:generate go run ./internal/generator -file clientcall_test.go
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"sync"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
type handleClientCallCheckReportID struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp *apimodel.CheckReportIDResponse
url *url.URL
userAgent string
}
func (h *handleClientCallCheckReportID) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out *apimodel.CheckReportIDResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestCheckReportIDClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallCheckReportID{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.CheckReportIDRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.CheckReportID(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "GET" {
t.Fatal("invalid method")
}
// check the query
api := &simpleCheckReportIDAPI{BaseURL: srvr.URL}
httpReq, err := api.newRequest(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
t.Fatal(diff)
}
}
type handleClientCallCheckIn struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp *apimodel.CheckInResponse
url *url.URL
userAgent string
}
func (h *handleClientCallCheckIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out *apimodel.CheckInResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestCheckInClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallCheckIn{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.CheckInRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.CheckIn(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "POST" {
t.Fatal("invalid method")
}
// check the body
if handler.contentType != "application/json" {
t.Fatal("invalid content-type header")
}
got := &apimodel.CheckInRequest{}
if err := json.Unmarshal(handler.body, &got); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(req, got); diff != "" {
t.Fatal(diff)
}
}
type handleClientCallMeasurementMeta struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp *apimodel.MeasurementMetaResponse
url *url.URL
userAgent string
}
func (h *handleClientCallMeasurementMeta) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out *apimodel.MeasurementMetaResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestMeasurementMetaClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallMeasurementMeta{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.MeasurementMetaRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.MeasurementMeta(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "GET" {
t.Fatal("invalid method")
}
// check the query
api := &simpleMeasurementMetaAPI{BaseURL: srvr.URL}
httpReq, err := api.newRequest(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
t.Fatal(diff)
}
}
type handleClientCallTestHelpers struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp apimodel.TestHelpersResponse
url *url.URL
userAgent string
}
func (h *handleClientCallTestHelpers) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out apimodel.TestHelpersResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestTestHelpersClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallTestHelpers{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.TestHelpersRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.TestHelpers(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "GET" {
t.Fatal("invalid method")
}
// check the query
api := &simpleTestHelpersAPI{BaseURL: srvr.URL}
httpReq, err := api.newRequest(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
t.Fatal(diff)
}
}
type handleClientCallPsiphonConfig struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp apimodel.PsiphonConfigResponse
url *url.URL
userAgent string
}
func (h *handleClientCallPsiphonConfig) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
if r.URL.Path == "/api/v1/register" {
var out apimodel.RegisterResponse
ff.Fill(&out)
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
return
}
if r.URL.Path == "/api/v1/login" {
var out apimodel.LoginResponse
ff.Fill(&out)
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
return
}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out apimodel.PsiphonConfigResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestPsiphonConfigClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallPsiphonConfig{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.PsiphonConfigRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.PsiphonConfig(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "GET" {
t.Fatal("invalid method")
}
// check the query
api := &simplePsiphonConfigAPI{BaseURL: srvr.URL}
httpReq, err := api.newRequest(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
t.Fatal(diff)
}
}
type handleClientCallTorTargets struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp apimodel.TorTargetsResponse
url *url.URL
userAgent string
}
func (h *handleClientCallTorTargets) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
if r.URL.Path == "/api/v1/register" {
var out apimodel.RegisterResponse
ff.Fill(&out)
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
return
}
if r.URL.Path == "/api/v1/login" {
var out apimodel.LoginResponse
ff.Fill(&out)
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
return
}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out apimodel.TorTargetsResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestTorTargetsClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallTorTargets{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.TorTargetsRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.TorTargets(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "GET" {
t.Fatal("invalid method")
}
// check the query
api := &simpleTorTargetsAPI{BaseURL: srvr.URL}
httpReq, err := api.newRequest(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
t.Fatal(diff)
}
}
type handleClientCallURLs struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp *apimodel.URLsResponse
url *url.URL
userAgent string
}
func (h *handleClientCallURLs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out *apimodel.URLsResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestURLsClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallURLs{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.URLsRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.URLs(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "GET" {
t.Fatal("invalid method")
}
// check the query
api := &simpleURLsAPI{BaseURL: srvr.URL}
httpReq, err := api.newRequest(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != "" {
t.Fatal(diff)
}
}
type handleClientCallOpenReport struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp *apimodel.OpenReportResponse
url *url.URL
userAgent string
}
func (h *handleClientCallOpenReport) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out *apimodel.OpenReportResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestOpenReportClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallOpenReport{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.OpenReportRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.OpenReport(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "POST" {
t.Fatal("invalid method")
}
// check the body
if handler.contentType != "application/json" {
t.Fatal("invalid content-type header")
}
got := &apimodel.OpenReportRequest{}
if err := json.Unmarshal(handler.body, &got); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(req, got); diff != "" {
t.Fatal(diff)
}
}
type handleClientCallSubmitMeasurement struct {
accept string
body []byte
contentType string
count int32
method string
mu sync.Mutex
resp *apimodel.SubmitMeasurementResponse
url *url.URL
userAgent string
}
func (h *handleClientCallSubmitMeasurement) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ff := fakeFill{}
defer h.mu.Unlock()
h.mu.Lock()
if h.count > 0 {
w.WriteHeader(400)
return
}
h.count++
if r.Body != nil {
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
h.body = data
}
h.method = r.Method
h.url = r.URL
h.accept = r.Header.Get("Accept")
h.contentType = r.Header.Get("Content-Type")
h.userAgent = r.Header.Get("User-Agent")
var out *apimodel.SubmitMeasurementResponse
ff.Fill(&out)
h.resp = out
data, err := json.Marshal(out)
if err != nil {
w.WriteHeader(400)
return
}
w.Write(data)
}
func TestSubmitMeasurementClientCallRoundTrip(t *testing.T) {
// setup
handler := &handleClientCallSubmitMeasurement{}
srvr := httptest.NewServer(handler)
defer srvr.Close()
req := &apimodel.SubmitMeasurementRequest{}
ff := &fakeFill{}
ff.Fill(&req)
clnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}
ff.Fill(&clnt.UserAgent)
// issue request
ctx := context.Background()
resp, err := clnt.SubmitMeasurement(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non-nil response here")
}
// compare our response and server's one
if diff := cmp.Diff(handler.resp, resp); diff != "" {
t.Fatal(diff)
}
// check whether headers are OK
if handler.accept != "application/json" {
t.Fatal("invalid accept header")
}
if handler.userAgent != clnt.UserAgent {
t.Fatal("invalid user-agent header")
}
// check whether the method is OK
if handler.method != "POST" {
t.Fatal("invalid method")
}
// check the body
if handler.contentType != "application/json" {
t.Fatal("invalid content-type header")
}
got := &apimodel.SubmitMeasurementRequest{}
if err := json.Unmarshal(handler.body, &got); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(req, got); diff != "" {
t.Fatal(diff)
}
}

18
internal/ooapi/cloners.go
View File

@ -1,18 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:49.209515 +0200 CEST m=+0.000689210
package ooapi
//go:generate go run ./internal/generator -file cloners.go
// clonerForPsiphonConfigAPI represents any type exposing a method
// like simplePsiphonConfigAPI.WithToken.
type clonerForPsiphonConfigAPI interface {
WithToken(token string) callerForPsiphonConfigAPI
}
// clonerForTorTargetsAPI represents any type exposing a method
// like simpleTorTargetsAPI.WithToken.
type clonerForTorTargetsAPI interface {
WithToken(token string) callerForTorTargetsAPI
}

57
internal/ooapi/default.go
View File

@ -1,57 +0,0 @@
package ooapi
import (
"bytes"
"context"
"encoding/gob"
"encoding/json"
"io"
"net/http"
"strings"
"text/template"
)
type defaultRequestMaker struct{}
func (*defaultRequestMaker) NewRequest(
ctx context.Context, method, URL string, body io.Reader) (*http.Request, error) {
return http.NewRequestWithContext(ctx, method, URL, body)
}
type defaultJSONCodec struct{}
func (*defaultJSONCodec) Encode(v interface{}) ([]byte, error) {
return json.Marshal(v)
}
func (*defaultJSONCodec) Decode(b []byte, v interface{}) error {
return json.Unmarshal(b, v)
}
type defaultTemplateExecutor struct{}
func (*defaultTemplateExecutor) Execute(tmpl string, v interface{}) (string, error) {
to, err := template.New("t").Parse(tmpl)
if err != nil {
return "", err
}
var sb strings.Builder
if err := to.Execute(&sb, v); err != nil {
return "", err
}
return sb.String(), nil
}
type defaultGobCodec struct{}
func (*defaultGobCodec) Encode(v interface{}) ([]byte, error) {
var bb bytes.Buffer
if err := gob.NewEncoder(&bb).Encode(v); err != nil {
return nil, err
}
return bb.Bytes(), nil
}
func (*defaultGobCodec) Decode(b []byte, v interface{}) error {
return gob.NewDecoder(bytes.NewReader(b)).Decode(v)
}

41
internal/ooapi/default_test.go
View File

@ -1,41 +0,0 @@
package ooapi
import (
"strings"
"testing"
)
func TestDefaultTemplateExecutorParseError(t *testing.T) {
te := &defaultTemplateExecutor{}
out, err := te.Execute("{{ .Foo", nil)
if err == nil || !strings.HasSuffix(err.Error(), "unclosed action") {
t.Fatal("not the error we expected", err)
}
if out != "" {
t.Fatal("expected empty string")
}
}
func TestDefaultTemplateExecutorExecError(t *testing.T) {
te := &defaultTemplateExecutor{}
arg := make(chan interface{})
out, err := te.Execute("{{ .Foo }}", arg)
if err == nil || !strings.Contains(err.Error(), `can't evaluate field Foo`) {
t.Fatal("not the error we expected", err)
}
if out != "" {
t.Fatal("expected empty string")
}
}
func TestDefaultGobCodecEncodeError(t *testing.T) {
codec := &defaultGobCodec{}
arg := make(chan interface{})
data, err := codec.Encode(arg)
if err == nil || !strings.Contains(err.Error(), "can't handle type") {
t.Fatal("not the error we expected", err)
}
if data != nil {
t.Fatal("expected nil data")
}
}

59
internal/ooapi/dependencies.go
View File

@ -1,59 +0,0 @@
package ooapi
import (
"context"
"io"
"net/http"
"github.com/ooni/probe-cli/v3/internal/model"
)
// JSONCodec is a JSON encoder and decoder. Generally, we use a
// default JSONCodec in Client. This is the interface to implement
// if you want to override such a default.
type JSONCodec interface {
// Encode encodes v as a serialized JSON byte slice.
Encode(v interface{}) ([]byte, error)
// Decode decodes the serialized JSON byte slice into v.
Decode(b []byte, v interface{}) error
}
// RequestMaker makes an HTTP request. Generally, we use a
// default RequestMaker in Client. This is the interface to implement
// if you want to override such a default.
type RequestMaker interface {
// NewRequest creates a new HTTP request.
NewRequest(ctx context.Context, method, URL string, body io.Reader) (*http.Request, error)
}
// templateExecutor parses and executes a text template.
type templateExecutor interface {
// Execute takes in input a template string and some piece of data. It
// returns either a string where template parameters have been replaced,
// on success, or an error, on failure.
Execute(tmpl string, v interface{}) (string, error)
}
// HTTPClient is the interface of a generic HTTP client. The
// stdlib's http.Client implements this interface. We use
// http.DefaultClient as the default HTTPClient used by Client.
// Consumers of this package typically provide a custom HTTPClient
// with additional functionality (e.g., DoH, circumvention).
type HTTPClient = model.HTTPClient
// GobCodec is a Gob encoder and decoder. Generally, we use a
// default GobCodec in Client. This is the interface to implement
// if you want to override such a default.
type GobCodec interface {
// Encode encodes v as a serialized gob byte slice.
Encode(v interface{}) ([]byte, error)
// Decode decodes the serialized gob byte slice into v.
Decode(b []byte, v interface{}) error
}
// KVStore is a key-value store. This is the interface the
// client expect for the key-value store used to save persistent
// state (typically on the file system).
type KVStore = model.KeyValueStore

58
internal/ooapi/doc.go
View File

@ -1,58 +0,0 @@
// Package ooapi contains a client for the OONI API. We
// automatically generate the code in this package from the
// apimodel and internal/generator packages.
//
// Note
//
// This package is currrently unused. We plan on replacing
// existing code to speak with the OONI API with it.
//
// Usage
//
// You need to create a Client. Make sure you set all
// the mandatory fields. You will then have a function
// for every supported OONI API. This function will
// take in input a context and a request. You need to
// fill the request, of course. The return value is
// either a response or an error.
//
// If an API requires login, we will automatically
// perform the login. If an API uses caching, we will
// automatically use the cache.
//
// Design
//
// Most of the code in this package is auto-generated from the
// data model in ./apimodel and the definition of APIs provided
// by ./internal/generator/spec.go.
//
// We keep the generated files up-to-date by running
//
// go generate ./...
//
// We have tests that ensure that the definition of the API
// used here is reasonably close to the server's one.
//
// Testing
//
// The following command
//
// go test ./...
//
// will, among other things, ensure that the our API spec
// is consistent with the server's one. Running
//
// go test -short ./...
//
// will exclude most (slow) integration tests.
//
// Architecture
//
// The ./apimodel sub-package contains the definition of request
// and response messages. We rely on tagging to specify how
// we should encode and decode messages.
//
// The ./internal/generator sub-package contains code to generate most
// code in this package. In particular, the spec.go file is
// the specification of the APIs.
package ooapi

14
internal/ooapi/errors.go
View File

@ -1,14 +0,0 @@
package ooapi
import "errors"
// Errors defined by this package.
var (
ErrAPICallFailed = errors.New("ooapi: API call failed")
ErrEmptyField = errors.New("ooapi: empty field")
ErrHTTPFailure = errors.New("ooapi: http request failed")
ErrJSONLiteralNull = errors.New("ooapi: server returned us a literal null")
ErrMissingToken = errors.New("ooapi: missing auth token")
ErrUnauthorized = errors.New("ooapi: not authorized")
errCacheNotFound = errors.New("ooapi: not found in cache")
)

103
internal/ooapi/fake_test.go
View File

@ -1,103 +0,0 @@
package ooapi
import (
"context"
"io"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/fakefill"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// fakeFill forwards the fakefill.Filler type
type fakeFill = fakefill.Filler
type FakeCodec struct {
DecodeErr error
EncodeData []byte
EncodeErr error
}
func (mc *FakeCodec) Encode(v interface{}) ([]byte, error) {
return mc.EncodeData, mc.EncodeErr
}
func (mc *FakeCodec) Decode(b []byte, v interface{}) error {
return mc.DecodeErr
}
type FakeHTTPClient struct {
Err error
Resp *http.Response
}
func (c *FakeHTTPClient) Do(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond)
if req.Body != nil {
_, _ = netxlite.ReadAllContext(req.Context(), req.Body)
req.Body.Close()
}
if c.Err != nil {
return nil, c.Err
}
c.Resp.Request = req // non thread safe but it doesn't matter
return c.Resp, nil
}
func (c *FakeHTTPClient) CloseIdleConnections() {}
type FakeBody struct {
Data []byte
Err error
}
func (fb *FakeBody) Read(p []byte) (int, error) {
time.Sleep(10 * time.Microsecond)
if fb.Err != nil {
return 0, fb.Err
}
if len(fb.Data) <= 0 {
return 0, io.EOF
}
n := copy(p, fb.Data)
fb.Data = fb.Data[n:]
return n, nil
}
func (fb *FakeBody) Close() error {
return nil
}
type FakeRequestMaker struct {
Req *http.Request
Err error
}
func (frm *FakeRequestMaker) NewRequest(
ctx context.Context, method, URL string, body io.Reader) (*http.Request, error) {
return frm.Req, frm.Err
}
type FakeTemplateExecutor struct {
Out string
Err error
}
func (fte *FakeTemplateExecutor) Execute(tmpl string, v interface{}) (string, error) {
return fte.Out, fte.Err
}
type FakeKVStore struct {
SetError error
GetData []byte
GetError error
}
func (fs *FakeKVStore) Get(key string) ([]byte, error) {
return fs.GetData, fs.GetError
}
func (fs *FakeKVStore) Set(key string, value []byte) error {
return fs.SetError
}

212
internal/ooapi/fakeapi_test.go
View File

@ -1,212 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:49.546037 +0200 CEST m=+0.000695501
package ooapi
//go:generate go run ./internal/generator -file fakeapi_test.go
import (
"context"
"github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
type FakeCheckReportIDAPI struct {
Err error
Response *apimodel.CheckReportIDResponse
CountCall *atomicx.Int64
}
func (fapi *FakeCheckReportIDAPI) Call(ctx context.Context, req *apimodel.CheckReportIDRequest) (*apimodel.CheckReportIDResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForCheckReportIDAPI = &FakeCheckReportIDAPI{}
)
type FakeCheckInAPI struct {
Err error
Response *apimodel.CheckInResponse
CountCall *atomicx.Int64
}
func (fapi *FakeCheckInAPI) Call(ctx context.Context, req *apimodel.CheckInRequest) (*apimodel.CheckInResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForCheckInAPI = &FakeCheckInAPI{}
)
type FakeLoginAPI struct {
Err error
Response *apimodel.LoginResponse
CountCall *atomicx.Int64
}
func (fapi *FakeLoginAPI) Call(ctx context.Context, req *apimodel.LoginRequest) (*apimodel.LoginResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForLoginAPI = &FakeLoginAPI{}
)
type FakeMeasurementMetaAPI struct {
Err error
Response *apimodel.MeasurementMetaResponse
CountCall *atomicx.Int64
}
func (fapi *FakeMeasurementMetaAPI) Call(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*apimodel.MeasurementMetaResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForMeasurementMetaAPI = &FakeMeasurementMetaAPI{}
)
type FakeRegisterAPI struct {
Err error
Response *apimodel.RegisterResponse
CountCall *atomicx.Int64
}
func (fapi *FakeRegisterAPI) Call(ctx context.Context, req *apimodel.RegisterRequest) (*apimodel.RegisterResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForRegisterAPI = &FakeRegisterAPI{}
)
type FakeTestHelpersAPI struct {
Err error
Response apimodel.TestHelpersResponse
CountCall *atomicx.Int64
}
func (fapi *FakeTestHelpersAPI) Call(ctx context.Context, req *apimodel.TestHelpersRequest) (apimodel.TestHelpersResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForTestHelpersAPI = &FakeTestHelpersAPI{}
)
type FakePsiphonConfigAPI struct {
WithResult callerForPsiphonConfigAPI
Err error
Response apimodel.PsiphonConfigResponse
CountCall *atomicx.Int64
}
func (fapi *FakePsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
func (fapi *FakePsiphonConfigAPI) WithToken(token string) callerForPsiphonConfigAPI {
return fapi.WithResult
}
var (
_ callerForPsiphonConfigAPI = &FakePsiphonConfigAPI{}
_ clonerForPsiphonConfigAPI = &FakePsiphonConfigAPI{}
)
type FakeTorTargetsAPI struct {
WithResult callerForTorTargetsAPI
Err error
Response apimodel.TorTargetsResponse
CountCall *atomicx.Int64
}
func (fapi *FakeTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
func (fapi *FakeTorTargetsAPI) WithToken(token string) callerForTorTargetsAPI {
return fapi.WithResult
}
var (
_ callerForTorTargetsAPI = &FakeTorTargetsAPI{}
_ clonerForTorTargetsAPI = &FakeTorTargetsAPI{}
)
type FakeURLsAPI struct {
Err error
Response *apimodel.URLsResponse
CountCall *atomicx.Int64
}
func (fapi *FakeURLsAPI) Call(ctx context.Context, req *apimodel.URLsRequest) (*apimodel.URLsResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForURLsAPI = &FakeURLsAPI{}
)
type FakeOpenReportAPI struct {
Err error
Response *apimodel.OpenReportResponse
CountCall *atomicx.Int64
}
func (fapi *FakeOpenReportAPI) Call(ctx context.Context, req *apimodel.OpenReportRequest) (*apimodel.OpenReportResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForOpenReportAPI = &FakeOpenReportAPI{}
)
type FakeSubmitMeasurementAPI struct {
Err error
Response *apimodel.SubmitMeasurementResponse
CountCall *atomicx.Int64
}
func (fapi *FakeSubmitMeasurementAPI) Call(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*apimodel.SubmitMeasurementResponse, error) {
if fapi.CountCall != nil {
fapi.CountCall.Add(1)
}
return fapi.Response, fapi.Err
}
var (
_ callerForSubmitMeasurementAPI = &FakeSubmitMeasurementAPI{}
)

23
internal/ooapi/httpclient_test.go
View File

@ -1,23 +0,0 @@
package ooapi
import (
"net/http"
"testing"
)
type VerboseHTTPClient struct {
T *testing.T
}
func (c *VerboseHTTPClient) Do(req *http.Request) (*http.Response, error) {
c.T.Logf("> %s %s", req.Method, req.URL.String())
resp, err := http.DefaultClient.Do(req)
if err != nil {
c.T.Logf("< %s", err.Error())
return nil, err
}
c.T.Logf("< %d", resp.StatusCode)
return resp, nil
}
func (c *VerboseHTTPClient) CloseIdleConnections() {}

167
internal/ooapi/integration_test.go
View File

@ -1,167 +0,0 @@
package ooapi_test
import (
"context"
"testing"
"github.com/ooni/probe-cli/v3/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/ooapi"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
func TestWithRealServerDoCheckIn(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
req := &apimodel.CheckInRequest{
Charging: true,
OnWiFi: true,
Platform: "android",
ProbeASN: "AS12353",
ProbeCC: "IT",
RunType: model.RunTypeTimed,
SoftwareName: "ooniprobe-android",
SoftwareVersion: "2.7.1",
WebConnectivity: apimodel.CheckInRequestWebConnectivity{
CategoryCodes: []string{"NEWS", "CULTR"},
},
}
httpClnt := &ooapi.VerboseHTTPClient{T: t}
clnt := &ooapi.Client{HTTPClient: httpClnt, KVStore: &kvstore.Memory{}}
ctx := context.Background()
resp, err := clnt.CheckIn(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non nil pointer here")
}
for idx, url := range resp.Tests.WebConnectivity.URLs {
if idx >= 3 {
break
}
t.Logf("- %+v", url)
}
}
func TestWithRealServerDoCheckReportID(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
req := &apimodel.CheckReportIDRequest{
ReportID: "20210223T093606Z_ndt_JO_8376_n1_kDYToqrugDY54Soy",
}
clnt := &ooapi.Client{KVStore: &kvstore.Memory{}}
ctx := context.Background()
resp, err := clnt.CheckReportID(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non nil pointer here")
}
t.Logf("%+v", resp)
}
func TestWithRealServerDoMeasurementMeta(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
req := &apimodel.MeasurementMetaRequest{
ReportID: "20210223T093606Z_ndt_JO_8376_n1_kDYToqrugDY54Soy",
}
clnt := &ooapi.Client{KVStore: &kvstore.Memory{}}
ctx := context.Background()
resp, err := clnt.MeasurementMeta(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non nil pointer here")
}
t.Logf("%+v", resp)
}
func TestWithRealServerDoOpenReport(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
req := &apimodel.OpenReportRequest{
DataFormatVersion: "0.2.0",
Format: "json",
ProbeASN: "AS137",
ProbeCC: "IT",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
TestName: "example",
TestStartTime: "2018-11-01 15:33:20",
TestVersion: "0.1.0",
}
clnt := &ooapi.Client{KVStore: &kvstore.Memory{}}
ctx := context.Background()
resp, err := clnt.OpenReport(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non nil pointer here")
}
t.Logf("%+v", resp)
}
func TestWithRealServerDoPsiphonConfig(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
req := &apimodel.PsiphonConfigRequest{}
httpClnt := &ooapi.VerboseHTTPClient{T: t}
clnt := &ooapi.Client{HTTPClient: httpClnt, KVStore: &kvstore.Memory{}}
ctx := context.Background()
resp, err := clnt.PsiphonConfig(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non nil pointer here")
}
t.Logf("%+v", resp != nil)
}
func TestWithRealServerDoTorTargets(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
req := &apimodel.TorTargetsRequest{}
httpClnt := &ooapi.VerboseHTTPClient{T: t}
clnt := &ooapi.Client{HTTPClient: httpClnt, KVStore: &kvstore.Memory{}}
ctx := context.Background()
resp, err := clnt.TorTargets(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non nil pointer here")
}
t.Logf("%+v", resp != nil)
}
func TestWithRealServerDoURLs(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
req := &apimodel.URLsRequest{
CountryCode: "IT",
Limit: 3,
}
clnt := &ooapi.Client{KVStore: &kvstore.Memory{}}
ctx := context.Background()
resp, err := clnt.URLs(ctx, req)
if err != nil {
t.Fatal(err)
}
if resp == nil {
t.Fatal("expected non nil pointer here")
}
t.Logf("%+v", resp)
}

181
internal/ooapi/internal/generator/apis.go
View File

@ -1,181 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
// apiField contains the fields of an API data structure
type apiField struct {
// name is the field name
name string
// kind is the filed type
kind string
// comment is a brief comment to document the field
comment string
// ifLogin indicates whether this field should only be
// emitted when the API requires login
ifLogin bool
// ifTemplate indicates whether this field should only be
// emitted when the URL path is a template
ifTemplate bool
// noClone is true when this field should not be copied
// from the parent data structure when cloning
noClone bool
}
var apiFields = []apiField{{
name: "BaseURL",
kind: "string",
comment: "optional",
}, {
name: "HTTPClient",
kind: "HTTPClient",
comment: "optional",
}, {
name: "JSONCodec",
kind: "JSONCodec",
comment: "optional",
}, {
name: "Token",
kind: "string",
comment: "mandatory",
ifLogin: true,
noClone: true,
}, {
name: "RequestMaker",
kind: "RequestMaker",
comment: "optional",
}, {
name: "TemplateExecutor",
kind: "templateExecutor",
comment: "optional",
ifTemplate: true,
}, {
name: "UserAgent",
kind: "string",
comment: "optional",
}}
func (d *Descriptor) genNewAPI(sb *strings.Builder) {
fmt.Fprintf(sb, "// %s implements the %s API.\n", d.APIStructName(), d.Name)
fmt.Fprintf(sb, "type %s struct {\n", d.APIStructName())
for _, f := range apiFields {
if !d.RequiresLogin && f.ifLogin {
continue
}
if !d.URLPath.IsTemplate && f.ifTemplate {
continue
}
fmt.Fprintf(sb, "\t%s %s // %s\n", f.name, f.kind, f.comment)
}
fmt.Fprint(sb, "}\n\n")
if d.RequiresLogin {
fmt.Fprintf(sb, "// WithToken returns a copy of the API where the\n")
fmt.Fprintf(sb, "// value of the Token field is replaced with token.\n")
fmt.Fprintf(sb, "func (api *%s) WithToken(token string) %s {\n",
d.APIStructName(), d.CallerInterfaceName())
fmt.Fprintf(sb, "out := &%s{}\n", d.APIStructName())
for _, f := range apiFields {
if !d.URLPath.IsTemplate && f.ifTemplate {
continue
}
if f.noClone == true {
continue
}
fmt.Fprintf(sb, "out.%s = api.%s\n", f.name, f.name)
}
fmt.Fprint(sb, "out.Token = token\n")
fmt.Fprint(sb, "return out\n")
fmt.Fprint(sb, "}\n\n")
}
fmt.Fprintf(sb, "func (api *%s) baseURL() string {\n", d.APIStructName())
fmt.Fprint(sb, "\tif api.BaseURL != \"\" {\n")
fmt.Fprint(sb, "\t\treturn api.BaseURL\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn \"https://ps1.ooni.io\"\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) requestMaker() RequestMaker {\n", d.APIStructName())
fmt.Fprint(sb, "\tif api.RequestMaker != nil {\n")
fmt.Fprint(sb, "\t\treturn api.RequestMaker\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn &defaultRequestMaker{}\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) jsonCodec() JSONCodec {\n", d.APIStructName())
fmt.Fprint(sb, "\tif api.JSONCodec != nil {\n")
fmt.Fprint(sb, "\t\treturn api.JSONCodec\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn &defaultJSONCodec{}\n")
fmt.Fprint(sb, "}\n\n")
if d.URLPath.IsTemplate {
fmt.Fprintf(
sb, "func (api *%s) templateExecutor() templateExecutor {\n",
d.APIStructName())
fmt.Fprint(sb, "\tif api.TemplateExecutor != nil {\n")
fmt.Fprint(sb, "\t\treturn api.TemplateExecutor\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn &defaultTemplateExecutor{}\n")
fmt.Fprint(sb, "}\n\n")
}
fmt.Fprintf(
sb, "func (api *%s) httpClient() HTTPClient {\n",
d.APIStructName())
fmt.Fprint(sb, "\tif api.HTTPClient != nil {\n")
fmt.Fprint(sb, "\t\treturn api.HTTPClient\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn http.DefaultClient\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "// Call calls the %s API.\n", d.Name)
fmt.Fprintf(
sb, "func (api *%s) Call(ctx context.Context, req %s) (%s, error) {\n",
d.APIStructName(), d.RequestTypeName(), d.ResponseTypeName())
fmt.Fprint(sb, "\thttpReq, err := api.newRequest(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\thttpReq.Header.Add(\"Accept\", \"application/json\")\n")
if d.RequiresLogin {
fmt.Fprint(sb, "\tif api.Token == \"\" {\n")
fmt.Fprint(sb, "\t\treturn nil, ErrMissingToken\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\thttpReq.Header.Add(\"Authorization\", newAuthorizationHeader(api.Token))\n")
}
fmt.Fprint(sb, "\tif api.UserAgent != \"\" {\n")
fmt.Fprint(sb, "\t\thttpReq.Header.Add(\"User-Agent\", api.UserAgent)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\thttpResp, err := api.httpClient().Do(httpReq)\n")
fmt.Fprint(sb, "\treturn api.newResponse(ctx, httpResp, err)\n")
fmt.Fprint(sb, "}\n\n")
}
// GenAPIsGo generates apis.go.
func GenAPIsGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
desc.genNewAPI(&sb)
}
writefile(file, &sb)
}

461
internal/ooapi/internal/generator/apistest.go
View File

@ -1,461 +0,0 @@
package main
import (
"fmt"
"reflect"
"strings"
"time"
)
func (d *Descriptor) genTestNewRequest(sb *strings.Builder) {
fmt.Fprintf(sb, "\treq := &%s{}\n", d.RequestTypeNameAsStruct())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\tff.Fill(req)\n")
}
func (d *Descriptor) genTestInvalidURL(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sInvalidURL(t *testing.T) {\n", d.Name)
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tBaseURL: \"\\t\", // invalid\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err == nil || !strings.HasSuffix(err.Error(), \"invalid control character in URL\") {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWithMissingToken(sb *strings.Builder) {
if d.RequiresLogin == false {
return // does not make sense when login isn't required
}
fmt.Fprintf(sb, "func Test%sWithMissingToken(t *testing.T) {\n", d.Name)
fmt.Fprintf(sb, "\tapi := &%s{} // no token\n", d.APIStructName())
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, ErrMissingToken) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWithHTTPErr(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWithHTTPErr(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tclnt := &FakeHTTPClient{Err: errMocked}\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: clnt,\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestMarshalErr(sb *strings.Builder) {
if d.Method != "POST" {
return // does not make sense when we don't send a request body
}
fmt.Fprintf(sb, "func Test%sMarshalErr(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tJSONCodec: &FakeCodec{EncodeErr: errMocked},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWithNewRequestErr(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWithNewRequestErr(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tRequestMaker: &FakeRequestMaker{Err: errMocked},\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWith401(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWith401(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tclnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 401}}\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: clnt,\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, ErrUnauthorized) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWith400(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWith400(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tclnt := &FakeHTTPClient{Resp: &http.Response{StatusCode: 400}}\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: clnt,\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, ErrHTTPFailure) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWithResponseBodyReadErr(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWithResponseBodyReadErr(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tclnt := &FakeHTTPClient{Resp: &http.Response{\n")
fmt.Fprint(sb, "\t\tStatusCode: 200,\n")
fmt.Fprint(sb, "\t\tBody: &FakeBody{Err: errMocked},\n")
fmt.Fprint(sb, "\t}}\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: clnt,\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWithUnmarshalFailure(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWithUnmarshalFailure(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tclnt := &FakeHTTPClient{Resp: &http.Response{\n")
fmt.Fprint(sb, "\t\tStatusCode: 200,\n")
fmt.Fprint(sb, "\t\tBody: &FakeBody{Data: []byte(`{}`)},\n")
fmt.Fprint(sb, "\t}}\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: clnt,\n")
fmt.Fprintf(sb, "\t\tJSONCodec: &FakeCodec{DecodeErr: errMocked},\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprintf(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestRoundTrip(sb *strings.Builder) {
// generate the type of the handler
fmt.Fprintf(sb, "type handle%s struct {\n", d.Name)
fmt.Fprint(sb, "\taccept string\n")
fmt.Fprint(sb, "\tbody []byte\n")
fmt.Fprint(sb, "\tcontentType string\n")
fmt.Fprint(sb, "\tcount int32\n")
fmt.Fprint(sb, "\tmethod string\n")
fmt.Fprint(sb, "\tmu sync.Mutex\n")
fmt.Fprintf(sb, "\tresp %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\turl *url.URL\n")
fmt.Fprint(sb, "\tuserAgent string\n")
fmt.Fprint(sb, "}\n\n")
// generate the handling function
fmt.Fprintf(sb,
"func (h *handle%s) ServeHTTP(w http.ResponseWriter, r *http.Request) {",
d.Name)
fmt.Fprint(sb, "\tdefer h.mu.Unlock()\n")
fmt.Fprint(sb, "\th.mu.Lock()\n")
fmt.Fprint(sb, "\tif h.count > 0 {\n")
fmt.Fprint(sb, "\t\tw.WriteHeader(400)\n")
fmt.Fprint(sb, "\t\treturn\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\th.count++\n")
fmt.Fprint(sb, "\tif r.Body != nil {\n")
fmt.Fprint(sb, "\t\tdata, err := netxlite.ReadAllContext(r.Context(), r.Body)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\t\treturn\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\th.body = data\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\th.method = r.Method\n")
fmt.Fprint(sb, "\th.url = r.URL\n")
fmt.Fprint(sb, "\th.accept = r.Header.Get(\"Accept\")\n")
fmt.Fprint(sb, "\th.contentType = r.Header.Get(\"Content-Type\")\n")
fmt.Fprint(sb, "\th.userAgent = r.Header.Get(\"User-Agent\")\n")
fmt.Fprintf(sb, "\tvar out %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff := fakeFill{}\n")
fmt.Fprint(sb, "\tff.Fill(&out)\n")
fmt.Fprintf(sb, "\th.resp = out\n")
fmt.Fprintf(sb, "\tdata, err := json.Marshal(out)\n")
fmt.Fprintf(sb, "\tif err != nil {\n")
fmt.Fprintf(sb, "\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\treturn\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tw.Write(data)\n")
fmt.Fprintf(sb, "\t}\n\n")
// generate the test itself
fmt.Fprintf(sb, "func Test%sRoundTrip(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\t// setup\n")
fmt.Fprintf(sb, "\thandler := &handle%s{}\n", d.Name)
fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n")
fmt.Fprint(sb, "\tdefer srvr.Close()\n")
fmt.Fprintf(sb, "\treq := &%s{}\n", d.RequestTypeNameAsStruct())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprintf(sb, "\tapi := &%s{BaseURL: srvr.URL}\n", d.APIStructName())
fmt.Fprint(sb, "\tff.Fill(&api.UserAgent)\n")
if d.RequiresLogin {
fmt.Fprint(sb, "\tff.Fill(&api.Token)\n")
}
fmt.Fprint(sb, "\t// issue request\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response here\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// compare our response and server's one\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.resp, resp); diff != \"\" {")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// check whether headers are OK\n")
fmt.Fprint(sb, "\tif handler.accept != \"application/json\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid accept header\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.userAgent != api.UserAgent {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid user-agent header\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// check whether the method is OK\n")
fmt.Fprintf(sb, "\tif handler.method != \"%s\" {\n", d.Method)
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid method\")\n")
fmt.Fprint(sb, "\t}\n")
if d.Method == "POST" {
fmt.Fprint(sb, "\t// check the body\n")
fmt.Fprint(sb, "\tif handler.contentType != \"application/json\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid content-type header\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tgot := &%s{}\n", d.RequestTypeNameAsStruct())
fmt.Fprintf(sb, "\tif err := json.Unmarshal(handler.body, &got); err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(req, got); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
} else {
fmt.Fprint(sb, "\t// check the query\n")
fmt.Fprint(sb, "\thttpReq, err := api.newRequest(context.Background(), req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
}
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestResponseLiteralNull(sb *strings.Builder) {
switch d.ResponseTypeKind() {
case reflect.Map:
// fallthrough
case reflect.Struct:
return // test not applicable
}
fmt.Fprintf(sb, "func Test%sResponseLiteralNull(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tclnt := &FakeHTTPClient{Resp: &http.Response{\n")
fmt.Fprint(sb, "\t\tStatusCode: 200,\n")
fmt.Fprint(sb, "\t\tBody: &FakeBody{Data: []byte(`null`)},\n")
fmt.Fprint(sb, "\t}}\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: clnt,\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, ErrJSONLiteralNull) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestMandatoryFields(sb *strings.Builder) {
fields := d.StructFieldsWithTag(d.Request, tagForRequired)
if len(fields) < 1 {
return // nothing to test
}
fmt.Fprintf(sb, "func Test%sMandatoryFields(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tclnt := &FakeHTTPClient{Resp: &http.Response{\n")
fmt.Fprint(sb, "\t\tStatusCode: 500,\n")
fmt.Fprint(sb, "\t}}\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: clnt,\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprintf(sb, "\treq := &%s{} // deliberately empty\n", d.RequestTypeNameAsStruct())
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, ErrEmptyField) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestTemplateErr(sb *strings.Builder) {
if !d.URLPath.IsTemplate {
return // nothing to test
}
fmt.Fprintf(sb, "func Test%sTemplateErr(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tclnt := &FakeHTTPClient{Resp: &http.Response{\n")
fmt.Fprint(sb, "\t\tStatusCode: 500,\n")
fmt.Fprint(sb, "\t}}\n")
fmt.Fprintf(sb, "\tapi := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: clnt,\n")
if d.RequiresLogin == true {
fmt.Fprint(sb, "\t\tToken: \"fakeToken\",\n")
}
fmt.Fprint(sb, "\t\tTemplateExecutor: &FakeTemplateExecutor{Err: errMocked},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
d.genTestNewRequest(sb)
fmt.Fprint(sb, "\tresp, err := api.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil resp\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
// TODO(bassosimone): we should add a panic for every switch for
// the type of a request or a response for robustness.
func (d *Descriptor) genAPITests(sb *strings.Builder) {
d.genTestInvalidURL(sb)
d.genTestWithMissingToken(sb)
d.genTestWithHTTPErr(sb)
d.genTestMarshalErr(sb)
d.genTestWithNewRequestErr(sb)
d.genTestWith401(sb)
d.genTestWith400(sb)
d.genTestWithResponseBodyReadErr(sb)
d.genTestWithUnmarshalFailure(sb)
d.genTestRoundTrip(sb)
d.genTestResponseLiteralNull(sb)
d.genTestMandatoryFields(sb)
d.genTestTemplateErr(sb)
}
// GenAPIsTestGo generates apis_test.go.
func GenAPIsTestGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"encoding/json\"\n")
fmt.Fprint(&sb, "\t\"errors\"\n")
fmt.Fprint(&sb, "\t\"net/http/httptest\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\t\"net/url\"\n")
fmt.Fprint(&sb, "\t\"strings\"\n")
fmt.Fprint(&sb, "\t\"testing\"\n")
fmt.Fprint(&sb, "\t\"sync\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/netxlite\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
desc.genAPITests(&sb)
}
writefile(file, &sb)
}

130
internal/ooapi/internal/generator/caching.go
View File

@ -1,130 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) genNewCache(sb *strings.Builder) {
fmt.Fprintf(sb, "// %s implements caching for %s.\n",
d.WithCacheAPIStructName(), d.APIStructName())
fmt.Fprintf(sb, "type %s struct {\n", d.WithCacheAPIStructName())
fmt.Fprintf(sb, "\tAPI %s // mandatory\n", d.CallerInterfaceName())
fmt.Fprint(sb, "\tGobCodec GobCodec // optional\n")
fmt.Fprint(sb, "\tKVStore KVStore // mandatory\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "type %s struct {\n", d.CacheEntryName())
fmt.Fprintf(sb, "\tReq %s\n", d.RequestTypeName())
fmt.Fprintf(sb, "\tResp %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "// Call calls the API and implements caching.\n")
fmt.Fprintf(sb, "func (c *%s) Call(ctx context.Context, req %s) (%s, error) {\n",
d.WithCacheAPIStructName(), d.RequestTypeName(), d.ResponseTypeName())
if d.CachePolicy == CacheAlways {
fmt.Fprint(sb, "\tif resp, _ := c.readcache(req); resp != nil {\n")
fmt.Fprint(sb, "\t\treturn resp, nil\n")
fmt.Fprint(sb, "\t}\n")
}
fmt.Fprint(sb, "\tresp, err := c.API.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
if d.CachePolicy == CacheFallback {
fmt.Fprint(sb, "\t\tif resp, _ := c.readcache(req); resp != nil {\n")
fmt.Fprint(sb, "\t\t\treturn resp, nil\n")
fmt.Fprint(sb, "\t\t}\n")
}
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif err := c.writecache(req, resp); err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn resp, nil\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (c *%s) gobCodec() GobCodec {\n", d.WithCacheAPIStructName())
fmt.Fprint(sb, "\tif c.GobCodec != nil {\n")
fmt.Fprint(sb, "\t\treturn c.GobCodec\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn &defaultGobCodec{}\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (c *%s) getcache() ([]%s, error) {\n",
d.WithCacheAPIStructName(), d.CacheEntryName())
fmt.Fprintf(sb, "\tdata, err := c.KVStore.Get(\"%s\")\n", d.CacheKey())
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar out []%s\n", d.CacheEntryName())
fmt.Fprint(sb, "\tif err := c.gobCodec().Decode(data, &out); err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn out, nil\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (c *%s) setcache(in []%s) error {\n",
d.WithCacheAPIStructName(), d.CacheEntryName())
fmt.Fprint(sb, "\tdata, err := c.gobCodec().Encode(in)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\treturn c.KVStore.Set(\"%s\", data)\n", d.CacheKey())
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (c *%s) readcache(req %s) (%s, error) {\n",
d.WithCacheAPIStructName(), d.RequestTypeName(), d.ResponseTypeName())
fmt.Fprint(sb, "\tcache, err := c.getcache()\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tfor _, cur := range cache {\n")
fmt.Fprint(sb, "\t\tif reflect.DeepEqual(req, cur.Req) {\n")
fmt.Fprint(sb, "\t\t\treturn cur.Resp, nil\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn nil, errCacheNotFound\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (c *%s) writecache(req %s, resp %s) error {\n",
d.WithCacheAPIStructName(), d.RequestTypeName(), d.ResponseTypeName())
fmt.Fprint(sb, "\tcache, _ := c.getcache()\n")
fmt.Fprintf(sb, "\tout := []%s{{Req: req, Resp: resp}}\n", d.CacheEntryName())
fmt.Fprint(sb, "\tconst toomany = 64\n")
fmt.Fprint(sb, "\tfor idx, cur := range cache {\n")
fmt.Fprint(sb, "\t\tif reflect.DeepEqual(req, cur.Req) {\n")
fmt.Fprint(sb, "\t\t\tcontinue // we already updated the cache\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif idx > toomany {\n")
fmt.Fprint(sb, "\t\t\tbreak\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tout = append(out, cur)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn c.setcache(out)\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "var _ %s = &%s{}\n\n", d.CallerInterfaceName(),
d.WithCacheAPIStructName())
}
// GenCachingGo generates caching.go.
func GenCachingGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"reflect\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
if desc.CachePolicy == CacheNone {
continue
}
desc.genNewCache(&sb)
}
writefile(file, &sb)
}

275
internal/ooapi/internal/generator/cachingtest.go
View File

@ -1,275 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) genTestCacheSuccess(sb *strings.Builder) {
fmt.Fprintf(sb, "func TestCache%sSuccess(t *testing.T) {\n", d.APIStructName())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := cache.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(expect, resp); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWriteCacheError(sb *strings.Builder) {
fmt.Fprintf(sb, "func TestCache%sWriteCacheError(t *testing.T) {\n", d.APIStructName())
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tKVStore: &FakeKVStore{SetError: errMocked},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := cache.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestFailureWithNoCache(sb *strings.Builder) {
fmt.Fprintf(sb, "func TestCache%sFailureWithNoCache(t *testing.T) {\n", d.APIStructName())
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\tErr: errMocked,\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := cache.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestFailureWithPreviousCache(sb *strings.Builder) {
// This works for both caching policies.
fmt.Fprintf(sb, "func TestCache%sFailureWithPreviousCache(t *testing.T) {\n", d.APIStructName())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprintf(sb, "\tfakeapi := &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName())
fmt.Fprint(sb, "\t\tAPI: fakeapi,\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\t// first pass with no error at all\n")
fmt.Fprint(sb, "\t// use a separate scope to be sure we avoid mistakes\n")
fmt.Fprint(sb, "\t{\n")
fmt.Fprint(sb, "\t\tresp, err := cache.Call(ctx, req)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif resp == nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif diff := cmp.Diff(expect, resp); diff != \"\" {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// second pass with failure\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tfakeapi.Err = errMocked\n")
fmt.Fprint(sb, "\tfakeapi.Response = nil\n")
fmt.Fprint(sb, "\tresp2, err := cache.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp2 == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(expect, resp2); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestSetcacheWithEncodeError(sb *strings.Builder) {
fmt.Fprintf(sb, "func TestCache%sSetcacheWithEncodeError(t *testing.T) {\n", d.APIStructName())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprintf(sb, "\tvar in []%s\n", d.CacheEntryName())
fmt.Fprint(sb, "\tff.Fill(&in)\n")
fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName())
fmt.Fprint(sb, "\t\tGobCodec: &FakeCodec{EncodeErr: errMocked},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\terr := cache.setcache(in)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestReadCacheNotFound(sb *strings.Builder) {
if fields := d.StructFields(d.Request); len(fields) <= 0 {
// this test cannot work when there are no fields in the
// request because we will always find a match.
// TODO(bassosimone): how to avoid having uncovered code?
return
}
fmt.Fprintf(sb, "func TestCache%sReadCacheNotFound(t *testing.T) {\n", d.APIStructName())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar incache []%s\n", d.CacheEntryName())
fmt.Fprint(sb, "\tff.Fill(&incache)\n")
fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName())
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\terr := cache.setcache(incache)\n")
fmt.Fprintf(sb, "\tif err != nil {\n")
fmt.Fprintf(sb, "\t\tt.Fatal(err)\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprintf(sb, "\tout, err := cache.readcache(req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errCacheNotFound) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif out != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil here\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWriteCacheDuplicate(sb *strings.Builder) {
fmt.Fprintf(sb, "func TestCache%sWriteCacheDuplicate(t *testing.T) {\n", d.APIStructName())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprintf(sb, "\tvar resp1 %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&resp1)\n")
fmt.Fprintf(sb, "\tvar resp2 %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&resp2)\n")
fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName())
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\terr := cache.writecache(req, resp1)\n")
fmt.Fprintf(sb, "\tif err != nil {\n")
fmt.Fprintf(sb, "\t\tt.Fatal(err)\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\terr = cache.writecache(req, resp2)\n")
fmt.Fprintf(sb, "\tif err != nil {\n")
fmt.Fprintf(sb, "\t\tt.Fatal(err)\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tout, err := cache.readcache(req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif out == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil here\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(resp2, out); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestCachSizeLimited(sb *strings.Builder) {
if fields := d.StructFields(d.Request); len(fields) <= 0 {
// this test cannot work when there are no fields in the
// request because we will always find a match.
// TODO(bassosimone): how to avoid having uncovered code?
return
}
fmt.Fprintf(sb, "func TestCache%sCacheSizeLimited(t *testing.T) {\n", d.APIStructName())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tcache := &%s{\n", d.WithCacheAPIStructName())
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar prev int\n")
fmt.Fprintf(sb, "\tfor {\n")
fmt.Fprintf(sb, "\t\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\t\tff.Fill(&req)\n")
fmt.Fprintf(sb, "\t\tvar resp %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\t\tff.Fill(&resp)\n")
fmt.Fprintf(sb, "\t\terr := cache.writecache(req, resp)\n")
fmt.Fprintf(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprintf(sb, "\t\t}\n")
fmt.Fprintf(sb, "\t\tout, err := cache.getcache()\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif len(out) > prev {\n")
fmt.Fprint(sb, "\t\t\tprev = len(out)\n")
fmt.Fprint(sb, "\t\t\tcontinue\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tbreak\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
// GenCachingTestGo generates caching_test.go.
func GenCachingTestGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"errors\"\n")
fmt.Fprint(&sb, "\t\"testing\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/kvstore\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
if desc.CachePolicy == CacheNone {
continue
}
desc.genTestCacheSuccess(&sb)
desc.genTestWriteCacheError(&sb)
desc.genTestFailureWithNoCache(&sb)
desc.genTestFailureWithPreviousCache(&sb)
desc.genTestSetcacheWithEncodeError(&sb)
desc.genTestReadCacheNotFound(&sb)
desc.genTestWriteCacheDuplicate(&sb)
desc.genTestCachSizeLimited(&sb)
}
writefile(file, &sb)
}

35
internal/ooapi/internal/generator/callers.go
View File

@ -1,35 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) genNewCaller(sb *strings.Builder) {
fmt.Fprintf(sb, "// %s represents any type exposing a method\n",
d.CallerInterfaceName())
fmt.Fprintf(sb, "// like %s.Call.\n", d.APIStructName())
fmt.Fprintf(sb, "type %s interface {\n", d.CallerInterfaceName())
fmt.Fprintf(sb, "\tCall(ctx context.Context, req %s) (%s, error)\n",
d.RequestTypeName(), d.ResponseTypeName())
fmt.Fprint(sb, "}\n\n")
}
// GenCallersGo generates callers.go.
func GenCallersGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
desc.genNewCaller(&sb)
}
writefile(file, &sb)
}

104
internal/ooapi/internal/generator/clientcall.go
View File

@ -1,104 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) clientMakeAPIBase(sb *strings.Builder) {
fmt.Fprintf(sb, "&%s{\n", d.APIStructName())
for _, field := range apiFields {
if field.ifLogin || field.ifTemplate {
continue
}
fmt.Fprintf(sb, "\t%s: c.%s,\n", field.name, field.name)
}
fmt.Fprint(sb, "}")
}
func (d *Descriptor) clientMakeAPI(sb *strings.Builder) {
if d.RequiresLogin && d.CachePolicy != CacheNone {
panic("we don't support requiresLogin with caching")
}
if d.RequiresLogin {
fmt.Fprintf(sb, "&%s{\n", d.WithLoginAPIStructName())
fmt.Fprint(sb, "\tAPI:")
d.clientMakeAPIBase(sb)
fmt.Fprint(sb, ",\n")
fmt.Fprint(sb, "\tJSONCodec: c.JSONCodec,\n")
fmt.Fprint(sb, "\tKVStore: c.KVStore,\n")
fmt.Fprint(sb, "\tRegisterAPI: &simpleRegisterAPI{\n")
for _, field := range apiFields {
if field.ifLogin || field.ifTemplate {
continue
}
fmt.Fprintf(sb, "\t%s: c.%s,\n", field.name, field.name)
}
fmt.Fprint(sb, "\t},\n")
fmt.Fprint(sb, "\tLoginAPI: &simpleLoginAPI{\n")
for _, field := range apiFields {
if field.ifLogin || field.ifTemplate {
continue
}
fmt.Fprintf(sb, "\t%s: c.%s,\n", field.name, field.name)
}
fmt.Fprint(sb, "\t},\n")
fmt.Fprint(sb, "}\n")
return
}
if d.CachePolicy != CacheNone {
fmt.Fprintf(sb, "&%s{\n", d.WithCacheAPIStructName())
fmt.Fprint(sb, "\tAPI:")
d.clientMakeAPIBase(sb)
fmt.Fprint(sb, ",\n")
fmt.Fprint(sb, "\tGobCodec: c.GobCodec,\n")
fmt.Fprint(sb, "\tKVStore: c.KVStore,\n")
fmt.Fprint(sb, "}\n")
return
}
d.clientMakeAPIBase(sb)
fmt.Fprint(sb, "\n")
}
func (d *Descriptor) genClientNewCaller(sb *strings.Builder) {
fmt.Fprintf(sb, "func (c *Client) new%sCaller() ", d.Name)
fmt.Fprintf(sb, "%s {\n", d.CallerInterfaceName())
fmt.Fprint(sb, "\treturn ")
d.clientMakeAPI(sb)
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genClientCall(sb *strings.Builder) {
fmt.Fprintf(sb, "// %s calls the %s API.\n", d.Name, d.Name)
fmt.Fprintf(sb, "func (c *Client) %s(\n", d.Name)
fmt.Fprintf(sb, "ctx context.Context, req %s,\n) ", d.RequestTypeName())
fmt.Fprintf(sb, "(%s, error) {\n", d.ResponseTypeName())
fmt.Fprintf(sb, "\tapi := c.new%sCaller()\n", d.Name)
fmt.Fprint(sb, "\treturn api.Call(ctx, req)\n")
fmt.Fprint(sb, "}\n\n")
}
// GenClientCallGo generates clientcall.go.
func GenClientCallGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
switch desc.Name {
case "Register", "Login":
// We don't want to generate these APIs as toplevel.
continue
}
desc.genClientNewCaller(&sb)
desc.genClientCall(&sb)
}
writefile(file, &sb)
}

182
internal/ooapi/internal/generator/clientcalltest.go
View File

@ -1,182 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) genTestClientCallRoundTrip(sb *strings.Builder) {
// generate the type of the handler
fmt.Fprintf(sb, "type handleClientCall%s struct {\n", d.Name)
fmt.Fprint(sb, "\taccept string\n")
fmt.Fprint(sb, "\tbody []byte\n")
fmt.Fprint(sb, "\tcontentType string\n")
fmt.Fprint(sb, "\tcount int32\n")
fmt.Fprint(sb, "\tmethod string\n")
fmt.Fprint(sb, "\tmu sync.Mutex\n")
fmt.Fprintf(sb, "\tresp %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\turl *url.URL\n")
fmt.Fprint(sb, "\tuserAgent string\n")
fmt.Fprint(sb, "}\n\n")
// generate the handling function
fmt.Fprintf(sb,
"func (h *handleClientCall%s) ServeHTTP(w http.ResponseWriter, r *http.Request) {",
d.Name)
fmt.Fprint(sb, "\tff := fakeFill{}\n")
if d.RequiresLogin {
fmt.Fprintf(sb, "\tif r.URL.Path == \"/api/v1/register\" {\n")
fmt.Fprintf(sb, "\t\tvar out apimodel.RegisterResponse\n")
fmt.Fprintf(sb, "\t\tff.Fill(&out)\n")
fmt.Fprintf(sb, "\t\tdata, err := json.Marshal(out)\n")
fmt.Fprintf(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\t\treturn\n")
fmt.Fprintf(sb, "\t\t}\n")
fmt.Fprintf(sb, "\t\tw.Write(data)\n")
fmt.Fprintf(sb, "\t\treturn\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tif r.URL.Path == \"/api/v1/login\" {\n")
fmt.Fprintf(sb, "\t\tvar out apimodel.LoginResponse\n")
fmt.Fprintf(sb, "\t\tff.Fill(&out)\n")
fmt.Fprintf(sb, "\t\tdata, err := json.Marshal(out)\n")
fmt.Fprintf(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\t\treturn\n")
fmt.Fprintf(sb, "\t\t}\n")
fmt.Fprintf(sb, "\t\tw.Write(data)\n")
fmt.Fprintf(sb, "\t\treturn\n")
fmt.Fprintf(sb, "\t}\n")
}
fmt.Fprint(sb, "\tdefer h.mu.Unlock()\n")
fmt.Fprint(sb, "\th.mu.Lock()\n")
fmt.Fprint(sb, "\tif h.count > 0 {\n")
fmt.Fprint(sb, "\t\tw.WriteHeader(400)\n")
fmt.Fprint(sb, "\t\treturn\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\th.count++\n")
fmt.Fprint(sb, "\tif r.Body != nil {\n")
fmt.Fprint(sb, "\t\tdata, err := netxlite.ReadAllContext(r.Context(), r.Body)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprintf(sb, "\t\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\t\treturn\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\th.body = data\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\th.method = r.Method\n")
fmt.Fprint(sb, "\th.url = r.URL\n")
fmt.Fprint(sb, "\th.accept = r.Header.Get(\"Accept\")\n")
fmt.Fprint(sb, "\th.contentType = r.Header.Get(\"Content-Type\")\n")
fmt.Fprint(sb, "\th.userAgent = r.Header.Get(\"User-Agent\")\n")
fmt.Fprintf(sb, "\tvar out %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&out)\n")
fmt.Fprintf(sb, "\th.resp = out\n")
fmt.Fprintf(sb, "\tdata, err := json.Marshal(out)\n")
fmt.Fprintf(sb, "\tif err != nil {\n")
fmt.Fprintf(sb, "\t\tw.WriteHeader(400)\n")
fmt.Fprintf(sb, "\t\treturn\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tw.Write(data)\n")
fmt.Fprintf(sb, "\t}\n\n")
// generate the test itself
fmt.Fprintf(sb, "func Test%sClientCallRoundTrip(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\t// setup\n")
fmt.Fprintf(sb, "\thandler := &handleClientCall%s{}\n", d.Name)
fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n")
fmt.Fprint(sb, "\tdefer srvr.Close()\n")
fmt.Fprintf(sb, "\treq := &%s{}\n", d.RequestTypeNameAsStruct())
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tclnt := &Client{KVStore: &kvstore.Memory{}, BaseURL: srvr.URL}\n")
fmt.Fprint(sb, "\tff.Fill(&clnt.UserAgent)\n")
fmt.Fprint(sb, "\t// issue request\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprintf(sb, "\tresp, err := clnt.%s(ctx, req)\n", d.Name)
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response here\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// compare our response and server's one\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.resp, resp); diff != \"\" {")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// check whether headers are OK\n")
fmt.Fprint(sb, "\tif handler.accept != \"application/json\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid accept header\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.userAgent != clnt.UserAgent {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid user-agent header\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// check whether the method is OK\n")
fmt.Fprintf(sb, "\tif handler.method != \"%s\" {\n", d.Method)
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid method\")\n")
fmt.Fprint(sb, "\t}\n")
if d.Method == "POST" {
fmt.Fprint(sb, "\t// check the body\n")
fmt.Fprint(sb, "\tif handler.contentType != \"application/json\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid content-type header\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tgot := &%s{}\n", d.RequestTypeNameAsStruct())
fmt.Fprintf(sb, "\tif err := json.Unmarshal(handler.body, &got); err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(req, got); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
} else {
fmt.Fprint(sb, "\t// check the query\n")
fmt.Fprintf(sb, "\tapi := &%s{BaseURL: srvr.URL}\n", d.APIStructName())
fmt.Fprint(sb, "\thttpReq, err := api.newRequest(context.Background(), req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.url.Path, httpReq.URL.Path); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(handler.url.RawQuery, httpReq.URL.RawQuery); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
}
fmt.Fprint(sb, "}\n\n")
}
// GenClientCallTestGo generates clientcall_test.go.
func GenClientCallTestGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"encoding/json\"\n")
fmt.Fprint(&sb, "\t\"net/http/httptest\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\t\"net/url\"\n")
fmt.Fprint(&sb, "\t\"testing\"\n")
fmt.Fprint(&sb, "\t\"sync\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/netxlite\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/kvstore\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
if desc.Name == "Login" || desc.Name == "Register" {
continue // they cannot be called directly
}
desc.genTestClientCallRoundTrip(&sb)
}
writefile(file, &sb)
}

32
internal/ooapi/internal/generator/cloners.go
View File

@ -1,32 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) genNewCloner(sb *strings.Builder) {
fmt.Fprintf(sb, "// %s represents any type exposing a method\n",
d.ClonerInterfaceName())
fmt.Fprintf(sb, "// like %s.WithToken.\n", d.APIStructName())
fmt.Fprintf(sb, "type %s interface {\n", d.ClonerInterfaceName())
fmt.Fprintf(sb, "\tWithToken(token string) %s\n", d.CallerInterfaceName())
fmt.Fprint(sb, "}\n\n")
}
// GenClonersGo generates cloners.go.
func GenClonersGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
for _, desc := range Descriptors {
if !desc.RequiresLogin {
continue
}
desc.genNewCloner(&sb)
}
writefile(file, &sb)
}

61
internal/ooapi/internal/generator/fakeapitest.go
View File

@ -1,61 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) genNewFakeAPI(sb *strings.Builder) {
fmt.Fprintf(sb, "type %s struct {\n", d.FakeAPIStructName())
if d.RequiresLogin {
fmt.Fprintf(sb, "\tWithResult %s\n", d.CallerInterfaceName())
}
fmt.Fprint(sb, "\tErr error\n")
fmt.Fprintf(sb, "\tResponse %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tCountCall *atomicx.Int64\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (fapi *%s) Call(ctx context.Context, req %s) (%s, error) {\n",
d.FakeAPIStructName(), d.RequestTypeName(), d.ResponseTypeName())
fmt.Fprint(sb, "\tif fapi.CountCall != nil {\n")
fmt.Fprint(sb, "\t\tfapi.CountCall.Add(1)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn fapi.Response, fapi.Err\n")
fmt.Fprint(sb, "}\n\n")
if d.RequiresLogin {
fmt.Fprintf(sb, "func (fapi *%s) WithToken(token string) %s {\n",
d.FakeAPIStructName(), d.CallerInterfaceName())
fmt.Fprint(sb, "\treturn fapi.WithResult\n")
fmt.Fprint(sb, "}\n\n")
}
fmt.Fprint(sb, "var (\n")
fmt.Fprintf(sb, "\t_ %s = &%s{}\n", d.CallerInterfaceName(),
d.FakeAPIStructName())
if d.RequiresLogin {
fmt.Fprintf(sb, "\t_ %s = &%s{}\n", d.ClonerInterfaceName(),
d.FakeAPIStructName())
}
fmt.Fprint(sb, ")\n\n")
}
// GenFakeAPITestGo generates fakeapi_test.go.
func GenFakeAPITestGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/atomicx\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
desc.genNewFakeAPI(&sb)
}
writefile(file, &sb)
}

57
internal/ooapi/internal/generator/generator.go
View File

@ -1,57 +0,0 @@
// Command generator generates code in the ooapi package.
//
// To this end, it uses the content of the apimodel package as
// well as the content of the spec.go file.
//
// The apimodel package defines the model, i.e., the structure
// of requests and responses and how messages should be sent
// and received.
//
// The spec.go file describes all the implemented APIs.
//
// If you change apimodel or spec.go, remember to run the
// `go generate ./...` command to regenerate all files.
package main
import (
"flag"
"fmt"
)
var flagFile = flag.String("file", "", "Indicate which file to regenerate")
func main() {
flag.Parse()
switch file := *flagFile; file {
case "apis.go":
GenAPIsGo(file)
case "responses.go":
GenResponsesGo(file)
case "requests.go":
GenRequestsGo(file)
case "swagger_test.go":
GenSwaggerTestGo(file)
case "apis_test.go":
GenAPIsTestGo(file)
case "callers.go":
GenCallersGo(file)
case "caching.go":
GenCachingGo(file)
case "login.go":
GenLoginGo(file)
case "cloners.go":
GenClonersGo(file)
case "fakeapi_test.go":
GenFakeAPITestGo(file)
case "caching_test.go":
GenCachingTestGo(file)
case "login_test.go":
GenLoginTestGo(file)
case "clientcall.go":
GenClientCallGo(file)
case "clientcall_test.go":
GenClientCallTestGo(file)
default:
panic(fmt.Sprintf("don't know how to create this file: %s", file))
}
}

182
internal/ooapi/internal/generator/login.go
View File

@ -1,182 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) genNewLogin(sb *strings.Builder) {
fmt.Fprintf(sb, "// %s implements login for %s.\n",
d.WithLoginAPIStructName(), d.APIStructName())
fmt.Fprintf(sb, "type %s struct {\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\tAPI %s // mandatory\n", d.ClonerInterfaceName())
fmt.Fprint(sb, "\tJSONCodec JSONCodec // optional\n")
fmt.Fprint(sb, "\tKVStore KVStore // mandatory\n")
fmt.Fprint(sb, "\tRegisterAPI callerForRegisterAPI // mandatory\n")
fmt.Fprint(sb, "\tLoginAPI callerForLoginAPI // mandatory\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "// Call logins, if needed, then calls the API.\n")
fmt.Fprintf(sb, "func (api *%s) Call(ctx context.Context, req %s) (%s, error) {\n",
d.WithLoginAPIStructName(), d.RequestTypeName(), d.ResponseTypeName())
fmt.Fprint(sb, "\ttoken, err := api.maybeLogin(ctx)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tresp, err := api.API.WithToken(token).Call(ctx, req)\n")
fmt.Fprint(sb, "\tif errors.Is(err, ErrUnauthorized) {\n")
fmt.Fprint(sb, "\t\t// Maybe the clock is just off? Let's try to obtain\n")
fmt.Fprint(sb, "\t\t// a token again and see if this fixes it.\n")
fmt.Fprint(sb, "\t\tif token, err = api.forceLogin(ctx); err == nil {\n")
fmt.Fprint(sb, "\t\t\tswitch resp, err = api.API.WithToken(token).Call(ctx, req); err {\n")
fmt.Fprint(sb, "\t\t\tcase nil:\n")
fmt.Fprint(sb, "\t\t\t\treturn resp, nil\n")
fmt.Fprint(sb, "\t\t\tcase ErrUnauthorized:\n")
fmt.Fprint(sb, "\t\t\t\t// fallthrough\n")
fmt.Fprint(sb, "\t\t\tdefault:\n")
fmt.Fprint(sb, "\t\t\t\treturn nil, err\n")
fmt.Fprint(sb, "\t\t\t}\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\t// Okay, this seems a broader problem. How about we try\n")
fmt.Fprint(sb, "\t\t// and re-register ourselves again instead?\n")
fmt.Fprint(sb, "\t\ttoken, err = api.forceRegister(ctx)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\treturn nil, err\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tresp, err = api.API.WithToken(token).Call(ctx, req)\n")
fmt.Fprint(sb, "\t\t// fallthrough\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn resp, nil\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) jsonCodec() JSONCodec {\n",
d.WithLoginAPIStructName())
fmt.Fprint(sb, "\tif api.JSONCodec != nil {\n")
fmt.Fprint(sb, "\t\treturn api.JSONCodec\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn &defaultJSONCodec{}\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) readstate() (*loginState, error) {\n",
d.WithLoginAPIStructName())
fmt.Fprint(sb, "\tdata, err := api.KVStore.Get(loginKey)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tvar ls loginState\n")
fmt.Fprint(sb, "\tif err := api.jsonCodec().Decode(data, &ls); err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn &ls, nil\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) writestate(ls *loginState) error {\n",
d.WithLoginAPIStructName())
fmt.Fprint(sb, "\tdata, err := api.jsonCodec().Encode(*ls)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn api.KVStore.Set(loginKey, data)\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) doRegister(ctx context.Context, password string) (string, error) {\n",
d.WithLoginAPIStructName())
fmt.Fprint(sb, "\treq := newRegisterRequest(password)\n")
fmt.Fprint(sb, "\tls := &loginState{}\n")
fmt.Fprint(sb, "\tresp, err := api.RegisterAPI.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn \"\", err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tls.ClientID = resp.ClientID\n")
fmt.Fprint(sb, "\tls.Password = req.Password\n")
fmt.Fprint(sb, "\treturn api.doLogin(ctx, ls)\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) forceRegister(ctx context.Context) (string, error) {\n",
d.WithLoginAPIStructName())
fmt.Fprint(sb, "\tvar password string\n")
fmt.Fprint(sb, "\t// If we already have a previous password, let us keep\n")
fmt.Fprint(sb, "\t// using it. This will allow a new version of the API to\n")
fmt.Fprint(sb, "\t// be able to continue to identify this probe. (This\n")
fmt.Fprint(sb, "\t// assumes that we have a stateless API that generates\n")
fmt.Fprint(sb, "\t// the user ID as a signature of the password plus a\n")
fmt.Fprint(sb, "\t// timestamp and that the key to generate the signature\n")
fmt.Fprint(sb, "\t// is not lost. If all these conditions are met, we\n")
fmt.Fprint(sb, "\t// can then serve better test targets to more long running\n")
fmt.Fprint(sb, "\t// (and therefore trusted) probes.)\n")
fmt.Fprint(sb, "\tif ls, err := api.readstate(); err == nil {\n")
fmt.Fprint(sb, "\t\tpassword = ls.Password\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif password == \"\" {\n")
fmt.Fprint(sb, "\t\tpassword = newRandomPassword()\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn api.doRegister(ctx, password)\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) forceLogin(ctx context.Context) (string, error) {\n",
d.WithLoginAPIStructName())
fmt.Fprint(sb, "\tls, err := api.readstate()\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn \"\", err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn api.doLogin(ctx, ls)\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) maybeLogin(ctx context.Context) (string, error) {\n",
d.WithLoginAPIStructName())
fmt.Fprint(sb, "\tls, _ := api.readstate()\n")
fmt.Fprint(sb, "\tif ls == nil || !ls.credentialsValid() {\n")
fmt.Fprint(sb, "\t\treturn api.forceRegister(ctx)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif !ls.tokenValid() {\n")
fmt.Fprint(sb, "\t\treturn api.doLogin(ctx, ls)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn ls.Token, nil\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "func (api *%s) doLogin(ctx context.Context, ls *loginState) (string, error) {\n",
d.WithLoginAPIStructName())
fmt.Fprint(sb, "\treq := &apimodel.LoginRequest{\n")
fmt.Fprint(sb, "\t\tClientID: ls.ClientID,\n")
fmt.Fprint(sb, "\t\tPassword: ls.Password,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tresp, err := api.LoginAPI.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn \"\", err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tls.Token = resp.Token\n")
fmt.Fprint(sb, "\tls.Expire = resp.Expire\n")
fmt.Fprint(sb, "\tif err := api.writestate(ls); err != nil {\n")
fmt.Fprint(sb, "\t\treturn \"\", err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\treturn ls.Token, nil\n")
fmt.Fprint(sb, "}\n\n")
fmt.Fprintf(sb, "var _ %s = &%s{}\n\n", d.CallerInterfaceName(),
d.WithLoginAPIStructName())
}
// GenLoginGo generates login.go.
func GenLoginGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"errors\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
if !desc.RequiresLogin {
continue
}
desc.genNewLogin(&sb)
}
writefile(file, &sb)
}

899
internal/ooapi/internal/generator/logintest.go
View File

@ -1,899 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
)
func (d *Descriptor) genTestRegisterAndLoginSuccess(sb *strings.Builder) {
fmt.Fprintf(sb, "func TestRegisterAndLogin%sSuccess(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprint(sb, "\tregisterAPI := &FakeRegisterAPI{\n")
fmt.Fprint(sb, "\t\tResponse: &apimodel.RegisterResponse{\n")
fmt.Fprint(sb, "\t\t\tClientID: \"antani-antani\",\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &FakeLoginAPI{\n")
fmt.Fprint(sb, "\t\t\tResponse: &apimodel.LoginResponse{\n")
fmt.Fprint(sb, "\t\t\t\tExpire: time.Now().Add(3600*time.Second),\n")
fmt.Fprint(sb, "\t\t\t\tToken: \"antani-antani-token\",\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(expect, resp); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif loginAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid loginAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif registerAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid registerAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestContinueUsingToken(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sContinueUsingToken(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprint(sb, "\tregisterAPI := &FakeRegisterAPI{\n")
fmt.Fprint(sb, "\t\tResponse: &apimodel.RegisterResponse{\n")
fmt.Fprint(sb, "\t\t\tClientID: \"antani-antani\",\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &FakeLoginAPI{\n")
fmt.Fprint(sb, "\t\t\tResponse: &apimodel.LoginResponse{\n")
fmt.Fprint(sb, "\t\t\t\tExpire: time.Now().Add(3600*time.Second),\n")
fmt.Fprint(sb, "\t\t\t\tToken: \"antani-antani-token\",\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\t// step 1: we register and login and use the token\n")
fmt.Fprint(sb, "\t// inside a scope just to avoid mistakes\n")
fmt.Fprint(sb, "\t{\n")
fmt.Fprint(sb, "\t\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif resp == nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif diff := cmp.Diff(expect, resp); diff != \"\" {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif loginAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid loginAPI.CountCall\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif registerAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid registerAPI.CountCall\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// step 2: we disable register and login but we\n")
fmt.Fprint(sb, "\t// should be okay because of the token\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tregisterAPI.Err = errMocked\n")
fmt.Fprint(sb, "\tregisterAPI.Response = nil\n")
fmt.Fprint(sb, "\tloginAPI.Err = errMocked\n")
fmt.Fprint(sb, "\tloginAPI.Response = nil\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(expect, resp); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif loginAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid loginAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif registerAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid registerAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWithValidButExpiredToken(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWithValidButExpiredToken(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tregisterAPI := &FakeRegisterAPI{\n")
fmt.Fprint(sb, "\t\tErr: errMocked,\n")
fmt.Fprint(sb, "\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &FakeLoginAPI{\n")
fmt.Fprint(sb, "\t\t\tResponse: &apimodel.LoginResponse{\n")
fmt.Fprint(sb, "\t\t\t\tExpire: time.Now().Add(3600*time.Second),\n")
fmt.Fprint(sb, "\t\t\t\tToken: \"antani-antani-token\",\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tls := &loginState{\n")
fmt.Fprintf(sb, "\t\tClientID: \"antani-antani\",\n")
fmt.Fprintf(sb, "\t\tExpire: time.Now().Add(-5 * time.Second),\n")
fmt.Fprintf(sb, "\t\tToken: \"antani-antani-token\",\n")
fmt.Fprintf(sb, "\t\tPassword: \"antani-antani-password\",\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tif err := login.writestate(ls); err != nil {\n")
fmt.Fprintf(sb, "\t\tt.Fatal(err)\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif diff := cmp.Diff(expect, resp); diff != \"\" {\n")
fmt.Fprint(sb, "\t\tt.Fatal(diff)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif loginAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid loginAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif registerAPI.CountCall.Load() != 0 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid registerAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWithRegisterAPIError(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWithRegisterAPIError(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\tregisterAPI := &FakeRegisterAPI{\n")
fmt.Fprint(sb, "\t\tErr: errMocked,\n")
fmt.Fprint(sb, "\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif registerAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid registerAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestWithLoginFailure(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sWithLoginFailure(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprint(sb, "\tregisterAPI := &FakeRegisterAPI{\n")
fmt.Fprint(sb, "\t\tResponse: &apimodel.RegisterResponse{\n")
fmt.Fprint(sb, "\t\t\tClientID: \"antani-antani\",\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprint(sb, "\t\tloginAPI := &FakeLoginAPI{\n")
fmt.Fprint(sb, "\t\t\tErr: errMocked,\n")
fmt.Fprint(sb, "\t\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif loginAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid loginAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif registerAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid registerAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestRegisterAndLoginThenFail(sb *strings.Builder) {
fmt.Fprintf(sb, "func TestRegisterAndLogin%sThenFail(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprint(sb, "\tregisterAPI := &FakeRegisterAPI{\n")
fmt.Fprint(sb, "\t\tResponse: &apimodel.RegisterResponse{\n")
fmt.Fprint(sb, "\t\t\tClientID: \"antani-antani\",\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &FakeLoginAPI{\n")
fmt.Fprint(sb, "\t\t\tResponse: &apimodel.LoginResponse{\n")
fmt.Fprint(sb, "\t\t\t\tExpire: time.Now().Add(3600*time.Second),\n")
fmt.Fprint(sb, "\t\t\t\tToken: \"antani-antani-token\",\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\t\tErr: errMocked,\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif loginAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid loginAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif registerAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid registerAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestTheDatabaseIsReplaced(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sTheDatabaseIsReplaced(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\thandler := &LoginHandler{\n")
fmt.Fprint(sb, "\t\tlogins: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tregisters: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tt: t,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n")
fmt.Fprint(sb, "\tdefer srvr.Close()\n")
fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\tAPI : baseAPI,\n")
fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\t// step 1: we register and login and use the token\n")
fmt.Fprint(sb, "\t// inside a scope just to avoid mistakes\n")
fmt.Fprint(sb, "\t{\n")
fmt.Fprint(sb, "\t\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif resp == nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.logins.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.registers.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// step 2: we forget accounts and try again.\n")
fmt.Fprint(sb, "\thandler.forgetLogins()\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.logins.Load() != 3 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.registers.Load() != 2 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestTheDatabaseIsReplacedThenFailure(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sTheDatabaseIsReplacedThenFailure(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\thandler := &LoginHandler{\n")
fmt.Fprint(sb, "\t\tlogins: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tregisters: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tt: t,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n")
fmt.Fprint(sb, "\tdefer srvr.Close()\n")
fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\tAPI : baseAPI,\n")
fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\t// step 1: we register and login and use the token\n")
fmt.Fprint(sb, "\t// inside a scope just to avoid mistakes\n")
fmt.Fprint(sb, "\t{\n")
fmt.Fprint(sb, "\t\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif resp == nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.logins.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.registers.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// step 2: we forget accounts and try again.\n")
fmt.Fprint(sb, "\t// but registrations are also failing.\n")
fmt.Fprint(sb, "\thandler.forgetLogins()\n")
fmt.Fprint(sb, "\thandler.noRegister = true\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, ErrHTTPFailure) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.logins.Load() != 2 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.registers.Load() != 2 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestRegisterAndLoginCannotWriteState(sb *strings.Builder) {
fmt.Fprintf(sb, "func TestRegisterAndLogin%sCannotWriteState(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprint(sb, "\tregisterAPI := &FakeRegisterAPI{\n")
fmt.Fprint(sb, "\t\tResponse: &apimodel.RegisterResponse{\n")
fmt.Fprint(sb, "\t\t\tClientID: \"antani-antani\",\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &FakeLoginAPI{\n")
fmt.Fprint(sb, "\t\t\tResponse: &apimodel.LoginResponse{\n")
fmt.Fprint(sb, "\t\t\t\tExpire: time.Now().Add(3600*time.Second),\n")
fmt.Fprint(sb, "\t\t\t\tToken: \"antani-antani-token\",\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t\tCountCall: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\t\tAPI: &%s{\n", d.FakeAPIStructName())
fmt.Fprintf(sb, "\t\t\tWithResult: &%s{\n", d.FakeAPIStructName())
fmt.Fprint(sb, "\t\t\t\tResponse: expect,\n")
fmt.Fprint(sb, "\t\t\t},\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\t\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t\tJSONCodec: &FakeCodec{\n")
fmt.Fprint(sb, "\t\t\tEncodeErr: errMocked,\n")
fmt.Fprint(sb, "\t\t},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, errMocked) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif loginAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid loginAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif registerAPI.CountCall.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid registerAPI.CountCall\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestReadStateDecodeFailure(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sReadStateDecodeFailure(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprintf(sb, "\tvar expect %s\n", d.ResponseTypeName())
fmt.Fprint(sb, "\tff.Fill(&expect)\n")
fmt.Fprint(sb, "\terrMocked := errors.New(\"mocked error\")\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprint(sb, "\t\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t\tJSONCodec: &FakeCodec{DecodeErr: errMocked},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tls := &loginState{\n")
fmt.Fprintf(sb, "\t\tClientID: \"antani-antani\",\n")
fmt.Fprintf(sb, "\t\tExpire: time.Now().Add(-5 * time.Second),\n")
fmt.Fprintf(sb, "\t\tToken: \"antani-antani-token\",\n")
fmt.Fprintf(sb, "\t\tPassword: \"antani-antani-password\",\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tif err := login.writestate(ls); err != nil {\n")
fmt.Fprintf(sb, "\t\tt.Fatal(err)\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "\tout, err := login.forceLogin(context.Background())\n")
fmt.Fprintf(sb, "if !errors.Is(err, errMocked) {\n")
fmt.Fprintf(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprintf(sb, "if out != \"\" {\n")
fmt.Fprintf(sb, "\t\tt.Fatal(\"expected empty string here\")\n")
fmt.Fprintf(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestClockIsOffThenSuccess(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sClockIsOffThenSuccess(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\thandler := &LoginHandler{\n")
fmt.Fprint(sb, "\t\tlogins: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tregisters: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tt: t,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n")
fmt.Fprint(sb, "\tdefer srvr.Close()\n")
fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\tAPI : baseAPI,\n")
fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\t// step 1: we register and login and use the token\n")
fmt.Fprint(sb, "\t// inside a scope just to avoid mistakes\n")
fmt.Fprint(sb, "\t{\n")
fmt.Fprint(sb, "\t\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif resp == nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.logins.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.registers.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// step 2: we forget tokens and try again.\n")
fmt.Fprint(sb, "\t// this should simulate the client clock\n")
fmt.Fprint(sb, "\t// being off and considering a token still valid\n")
fmt.Fprint(sb, "\thandler.forgetTokens()\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.logins.Load() != 2 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.registers.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestClockIsOffThen401(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sClockIsOffThen401(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\thandler := &LoginHandler{\n")
fmt.Fprint(sb, "\t\tlogins: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tregisters: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tt: t,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n")
fmt.Fprint(sb, "\tdefer srvr.Close()\n")
fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\tAPI : baseAPI,\n")
fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\t// step 1: we register and login and use the token\n")
fmt.Fprint(sb, "\t// inside a scope just to avoid mistakes\n")
fmt.Fprint(sb, "\t{\n")
fmt.Fprint(sb, "\t\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif resp == nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.logins.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.registers.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// step 2: we forget tokens and try again.\n")
fmt.Fprint(sb, "\t// this should simulate the client clock\n")
fmt.Fprint(sb, "\t// being off and considering a token still valid\n")
fmt.Fprint(sb, "\thandler.forgetTokens()\n")
fmt.Fprint(sb, "\thandler.failCallWith = []int{401, 401}\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp == nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.logins.Load() != 3 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.registers.Load() != 2 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
func (d *Descriptor) genTestClockIsOffThen500(sb *strings.Builder) {
fmt.Fprintf(sb, "func Test%sClockIsOffThen500(t *testing.T) {\n", d.Name)
fmt.Fprint(sb, "\tff := &fakeFill{}\n")
fmt.Fprint(sb, "\thandler := &LoginHandler{\n")
fmt.Fprint(sb, "\t\tlogins: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tregisters: &atomicx.Int64{},\n")
fmt.Fprint(sb, "\t\tt: t,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tsrvr := httptest.NewServer(handler)\n")
fmt.Fprint(sb, "\tdefer srvr.Close()\n")
fmt.Fprint(sb, "\tregisterAPI := &simpleRegisterAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t\tloginAPI := &simpleLoginAPI{\n")
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprintf(sb, "\tbaseAPI := &%s{\n", d.APIStructName())
fmt.Fprint(sb, "\t\tHTTPClient: &VerboseHTTPClient{T: t},\n")
fmt.Fprint(sb, "\t\tBaseURL: srvr.URL,\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tlogin := &%s{\n", d.WithLoginAPIStructName())
fmt.Fprintf(sb, "\tAPI : baseAPI,\n")
fmt.Fprint(sb, "\tRegisterAPI: registerAPI,\n")
fmt.Fprint(sb, "\tLoginAPI: loginAPI,\n")
fmt.Fprint(sb, "\tKVStore: &kvstore.Memory{},\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tvar req %s\n", d.RequestTypeName())
fmt.Fprint(sb, "\tff.Fill(&req)\n")
fmt.Fprint(sb, "\tctx := context.Background()\n")
fmt.Fprint(sb, "\t// step 1: we register and login and use the token\n")
fmt.Fprint(sb, "\t// inside a scope just to avoid mistakes\n")
fmt.Fprint(sb, "\t{\n")
fmt.Fprint(sb, "\t\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\t\tif err != nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(err)\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif resp == nil {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"expected non-nil response\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.logins.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t\tif handler.registers.Load() != 1 {\n")
fmt.Fprint(sb, "\t\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t\t}\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\t// step 2: we forget tokens and try again.\n")
fmt.Fprint(sb, "\t// this should simulate the client clock\n")
fmt.Fprint(sb, "\t// being off and considering a token still valid\n")
fmt.Fprint(sb, "\thandler.forgetTokens()\n")
fmt.Fprint(sb, "\thandler.failCallWith = []int{401, 500}\n")
fmt.Fprint(sb, "\tresp, err := login.Call(ctx, req)\n")
fmt.Fprint(sb, "\tif !errors.Is(err, ErrHTTPFailure) {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"not the error we expected\", err)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp != nil {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"expected nil response\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.logins.Load() != 2 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.logins\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif handler.registers.Load() != 1 {\n")
fmt.Fprint(sb, "\t\tt.Fatal(\"invalid handler.registers\")\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "}\n\n")
}
// GenLoginTestGo generates login_test.go.
func GenLoginTestGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"errors\"\n")
fmt.Fprint(&sb, "\t\"net/http/httptest\"\n")
fmt.Fprint(&sb, "\t\"testing\"\n")
fmt.Fprint(&sb, "\t\"time\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/google/go-cmp/cmp\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/atomicx\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/kvstore\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n")
for _, desc := range Descriptors {
if !desc.RequiresLogin {
continue
}
desc.genTestRegisterAndLoginSuccess(&sb)
desc.genTestContinueUsingToken(&sb)
desc.genTestWithValidButExpiredToken(&sb)
desc.genTestWithRegisterAPIError(&sb)
desc.genTestWithLoginFailure(&sb)
desc.genTestRegisterAndLoginThenFail(&sb)
desc.genTestTheDatabaseIsReplaced(&sb)
desc.genTestRegisterAndLoginCannotWriteState(&sb)
desc.genTestReadStateDecodeFailure(&sb)
desc.genTestTheDatabaseIsReplacedThenFailure(&sb)
desc.genTestClockIsOffThenSuccess(&sb)
desc.genTestClockIsOffThen401(&sb)
desc.genTestClockIsOffThen500(&sb)
}
writefile(file, &sb)
}

153
internal/ooapi/internal/generator/reflect.go
View File

@ -1,153 +0,0 @@
package main
import (
"fmt"
"reflect"
)
// TypeName returns v's package-qualified type name.
func (d *Descriptor) TypeName(v interface{}) string {
return reflect.TypeOf(v).String()
}
// RequestTypeName calls d.TypeName(d.Request).
func (d *Descriptor) RequestTypeName() string {
return d.TypeName(d.Request)
}
// ResponseTypeName calls d.TypeName(d.Response).
func (d *Descriptor) ResponseTypeName() string {
return d.TypeName(d.Response)
}
// APIStructName returns the correct struct type name
// for the API we're currently processing.
func (d *Descriptor) APIStructName() string {
return fmt.Sprintf("simple%sAPI", d.Name)
}
// FakeAPIStructName returns the correct struct type name
// for the fake for the API we're currently processing.
func (d *Descriptor) FakeAPIStructName() string {
return fmt.Sprintf("Fake%sAPI", d.Name)
}
// WithLoginAPIStructName returns the correct struct type name
// for the WithLoginAPI we're currently processing.
func (d *Descriptor) WithLoginAPIStructName() string {
return fmt.Sprintf("withLogin%sAPI", d.Name)
}
// CallerInterfaceName returns the correct caller interface name
// for the API we're currently processing.
func (d *Descriptor) CallerInterfaceName() string {
return fmt.Sprintf("callerFor%sAPI", d.Name)
}
// ClonerInterfaceName returns the correct cloner interface name
// for the API we're currently processing.
func (d *Descriptor) ClonerInterfaceName() string {
return fmt.Sprintf("clonerFor%sAPI", d.Name)
}
// WithCacheAPIStructName returns the correct struct type name for
// the cache for the API we're currently processing.
func (d *Descriptor) WithCacheAPIStructName() string {
return fmt.Sprintf("withCache%sAPI", d.Name)
}
// CacheEntryName returns the correct struct type name for the
// cache entry for the API we're currently processing.
func (d *Descriptor) CacheEntryName() string {
return fmt.Sprintf("cacheEntryFor%sAPI", d.Name)
}
// CacheKey returns the correct cache key for the API
// we're currently processing.
func (d *Descriptor) CacheKey() string {
return fmt.Sprintf("%s.cache", d.Name)
}
// StructFields returns all the struct fields of in. This function
// assumes that in is a pointer to struct, and will otherwise panic.
func (d *Descriptor) StructFields(in interface{}) []*reflect.StructField {
t := reflect.TypeOf(in)
if t.Kind() != reflect.Ptr {
panic("not a pointer")
}
t = t.Elem()
if t.Kind() != reflect.Struct {
panic("not a struct")
}
var out []*reflect.StructField
for idx := 0; idx < t.NumField(); idx++ {
f := t.Field(idx)
out = append(out, &f)
}
return out
}
// StructFieldsWithTag returns all the struct fields of
// in that have the specified tag.
func (d *Descriptor) StructFieldsWithTag(in interface{}, tag string) []*reflect.StructField {
var out []*reflect.StructField
for _, f := range d.StructFields(in) {
if f.Tag.Get(tag) != "" {
out = append(out, f)
}
}
return out
}
// RequestOrResponseTypeKind returns the type kind of in, which should
// be a request or a response. This function assumes that in is either a
// pointer to struct or a map and will panic otherwise.
func (d *Descriptor) RequestOrResponseTypeKind(in interface{}) reflect.Kind {
t := reflect.TypeOf(in)
if t.Kind() == reflect.Ptr {
t = t.Elem()
if t.Kind() != reflect.Struct {
panic("not a struct")
}
return reflect.Struct
}
if t.Kind() != reflect.Map {
panic("not a map")
}
return reflect.Map
}
// RequestTypeKind calls d.RequestOrResponseTypeKind(d.Request).
func (d *Descriptor) RequestTypeKind() reflect.Kind {
return d.RequestOrResponseTypeKind(d.Request)
}
// ResponseTypeKind calls d.RequestOrResponseTypeKind(d.Response).
func (d *Descriptor) ResponseTypeKind() reflect.Kind {
return d.RequestOrResponseTypeKind(d.Response)
}
// TypeNameAsStruct assumes that in is a pointer to struct and
// returns the type of the corresponding struct. The returned
// type is package qualified.
func (d *Descriptor) TypeNameAsStruct(in interface{}) string {
t := reflect.TypeOf(in)
if t.Kind() != reflect.Ptr {
panic("not a pointer")
}
t = t.Elem()
if t.Kind() != reflect.Struct {
panic("not a struct")
}
return t.String()
}
// RequestTypeNameAsStruct calls d.TypeNameAsStruct(d.Request)
func (d *Descriptor) RequestTypeNameAsStruct() string {
return d.TypeNameAsStruct(d.Request)
}
// ResponseTypeNameAsStruct calls d.TypeNameAsStruct(d.Response)
func (d *Descriptor) ResponseTypeNameAsStruct() string {
return d.TypeNameAsStruct(d.Response)
}

141
internal/ooapi/internal/generator/requests.go
View File

@ -1,141 +0,0 @@
package main
import (
"fmt"
"reflect"
"strings"
"time"
)
const (
tagForQuery = "query"
tagForRequired = "required"
)
func (d *Descriptor) genNewRequestQueryElemString(sb *strings.Builder, f *reflect.StructField) {
name := f.Name
query := f.Tag.Get(tagForQuery)
if f.Tag.Get(tagForRequired) == "true" {
fmt.Fprintf(sb, "\tif req.%s == \"\" {\n", name)
fmt.Fprintf(sb, "\t\treturn nil, newErrEmptyField(\"%s\")\n", name)
fmt.Fprint(sb, "\t}\n")
fmt.Fprintf(sb, "\tq.Add(\"%s\", req.%s)\n", query, name)
return
}
fmt.Fprintf(sb, "\tif req.%s != \"\" {\n", name)
fmt.Fprintf(sb, "\t\tq.Add(\"%s\", req.%s)\n", query, name)
fmt.Fprint(sb, "\t}\n")
}
func (d *Descriptor) genNewRequestQueryElemBool(sb *strings.Builder, f *reflect.StructField) {
// required does not make much sense for a boolean field
name := f.Name
query := f.Tag.Get(tagForQuery)
fmt.Fprintf(sb, "\tif req.%s {\n", name)
fmt.Fprintf(sb, "\t\tq.Add(\"%s\", \"true\")\n", query)
fmt.Fprint(sb, "\t}\n")
}
func (d *Descriptor) genNewRequestQueryElemInt64(sb *strings.Builder, f *reflect.StructField) {
// required does not make much sense for an integer field
name := f.Name
query := f.Tag.Get(tagForQuery)
fmt.Fprintf(sb, "\tif req.%s != 0 {\n", name)
fmt.Fprintf(sb, "\t\tq.Add(\"%s\", newQueryFieldInt64(req.%s))\n", query, name)
fmt.Fprint(sb, "\t}\n")
}
func (d *Descriptor) genNewRequestQuery(sb *strings.Builder) {
if d.Method != "GET" {
return // we only generate query for GET
}
fields := d.StructFieldsWithTag(d.Request, tagForQuery)
if len(fields) <= 0 {
return
}
fmt.Fprint(sb, "\tq := url.Values{}\n")
for idx, f := range fields {
switch f.Type.Kind() {
case reflect.String:
d.genNewRequestQueryElemString(sb, f)
case reflect.Bool:
d.genNewRequestQueryElemBool(sb, f)
case reflect.Int64:
d.genNewRequestQueryElemInt64(sb, f)
default:
panic(fmt.Sprintf("unexpected query type at index %d", idx))
}
}
fmt.Fprint(sb, "\tURL.RawQuery = q.Encode()\n")
}
func (d *Descriptor) genNewRequestCallNewRequest(sb *strings.Builder) {
if d.Method == "POST" {
fmt.Fprint(sb, "\tbody, err := api.jsonCodec().Encode(req)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tout, err := api.requestMaker().NewRequest(")
fmt.Fprintf(sb, "ctx, \"%s\", URL.String(), ", d.Method)
fmt.Fprint(sb, "bytes.NewReader(body))\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tout.Header.Set(\"Content-Type\", \"application/json\")\n")
fmt.Fprint(sb, "\treturn out, nil\n")
return
}
fmt.Fprint(sb, "\treturn api.requestMaker().NewRequest(")
fmt.Fprintf(sb, "ctx, \"%s\", URL.String(), ", d.Method)
fmt.Fprint(sb, "nil)\n")
}
func (d *Descriptor) genNewRequest(sb *strings.Builder) {
fmt.Fprintf(
sb, "func (api *%s) newRequest(ctx context.Context, req %s) %s {\n",
d.APIStructName(), d.RequestTypeName(), "(*http.Request, error)")
fmt.Fprint(sb, "\tURL, err := url.Parse(api.baseURL())\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
switch d.URLPath.IsTemplate {
case false:
fmt.Fprintf(sb, "\tURL.Path = \"%s\"\n", d.URLPath.Value)
case true:
fmt.Fprintf(
sb, "\tup, err := api.templateExecutor().Execute(\"%s\", req)\n",
d.URLPath.Value)
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tURL.Path = up\n")
}
d.genNewRequestQuery(sb)
d.genNewRequestCallNewRequest(sb)
fmt.Fprintf(sb, "}\n\n")
}
// GenRequestsGo generates requests.go.
func GenRequestsGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"bytes\"\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\t\"net/url\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n\n")
for _, desc := range Descriptors {
desc.genNewRequest(&sb)
}
writefile(file, &sb)
}

81
internal/ooapi/internal/generator/responses.go
View File

@ -1,81 +0,0 @@
package main
import (
"fmt"
"reflect"
"strings"
"time"
)
func (d *Descriptor) genNewResponse(sb *strings.Builder) {
fmt.Fprintf(sb,
"func (api *%s) newResponse(ctx context.Context, resp *http.Response, err error) (%s, error) {\n",
d.APIStructName(), d.ResponseTypeName())
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp.StatusCode == 401 {\n")
fmt.Fprint(sb, "\t\treturn nil, ErrUnauthorized\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tif resp.StatusCode != 200 {\n")
fmt.Fprint(sb, "\t\treturn nil, newHTTPFailure(resp.StatusCode)\n")
fmt.Fprint(sb, "\t}\n")
fmt.Fprint(sb, "\tdefer resp.Body.Close()\n")
fmt.Fprint(sb, "\treader := io.LimitReader(resp.Body, 4<<20)\n")
fmt.Fprint(sb, "\tdata, err := netxlite.ReadAllContext(ctx, reader)\n")
fmt.Fprint(sb, "\tif err != nil {\n")
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
switch d.ResponseTypeKind() {
case reflect.Map:
fmt.Fprintf(sb, "\tout := %s{}\n", d.ResponseTypeName())
case reflect.Struct:
fmt.Fprintf(sb, "\tout := &%s{}\n", d.ResponseTypeNameAsStruct())
}
switch d.ResponseTypeKind() {
case reflect.Map:
fmt.Fprint(sb, "\tif err := api.jsonCodec().Decode(data, &out); err != nil {\n")
case reflect.Struct:
fmt.Fprint(sb, "\tif err := api.jsonCodec().Decode(data, out); err != nil {\n")
}
fmt.Fprint(sb, "\t\treturn nil, err\n")
fmt.Fprint(sb, "\t}\n")
switch d.ResponseTypeKind() {
case reflect.Map:
// For rationale, see https://play.golang.org/p/m9-MsTaQ5wt and
// https://play.golang.org/p/6h-v-PShMk9.
fmt.Fprint(sb, "\tif out == nil {\n")
fmt.Fprint(sb, "\t\treturn nil, ErrJSONLiteralNull\n")
fmt.Fprint(sb, "\t}\n")
case reflect.Struct:
// nothing
}
fmt.Fprintf(sb, "\treturn out, nil\n")
fmt.Fprintf(sb, "}\n\n")
}
// GenResponsesGo generates responses.go.
func GenResponsesGo(file string) {
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprint(&sb, "import (\n")
fmt.Fprint(&sb, "\t\"context\"\n")
fmt.Fprint(&sb, "\t\"io\"\n")
fmt.Fprint(&sb, "\t\"net/http\"\n")
fmt.Fprint(&sb, "\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/netxlite\"\n")
fmt.Fprint(&sb, "\t\"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel\"\n")
fmt.Fprint(&sb, ")\n\n")
for _, desc := range Descriptors {
desc.genNewResponse(&sb)
}
writefile(file, &sb)
}

136
internal/ooapi/internal/generator/spec.go
View File

@ -1,136 +0,0 @@
package main
import "github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
// URLPath describes a URLPath.
type URLPath struct {
// IsTemplate indicates whether Value contains a template. A future
// version of this implementation will automatically deduce that.
IsTemplate bool
// Value is the value of the URL path.
Value string
// InSwagger indicates the corresponding name to be used in
// the Swagger specification.
InSwagger string
}
// Descriptor is an API descriptor. It tells the generator
// what code it should emit for a given API.
type Descriptor struct {
// Name is the name of the API.
Name string
// CachePolicy indicates the caching policy to use.
CachePolicy int
// RequiresLogin indicates whether the API requires login.
RequiresLogin bool
// Method is the method to use ("GET" or "POST").
Method string
// URLPath is the URL path.
URLPath URLPath
// Request is an instance of the request type.
Request interface{}
// Response is an instance of the response type.
Response interface{}
}
// These are the caching policies.
const (
// CacheNone indicates we don't use a cache.
CacheNone = iota
// CacheFallback indicates we fallback to the cache
// when there is a failure.
CacheFallback
// CacheAlways indicates that we always check the
// cache before sending a request.
CacheAlways
)
// Descriptors describes all the APIs.
//
// Note that it matters whether the requests and responses
// are pointers. Generally speaking, if the message is a
// struct, use a pointer. If it's a map, don't.
var Descriptors = []Descriptor{{
Name: "CheckReportID",
Method: "GET",
URLPath: URLPath{Value: "/api/_/check_report_id"},
Request: &apimodel.CheckReportIDRequest{},
Response: &apimodel.CheckReportIDResponse{},
}, {
Name: "CheckIn",
Method: "POST",
URLPath: URLPath{Value: "/api/v1/check-in"},
Request: &apimodel.CheckInRequest{},
Response: &apimodel.CheckInResponse{},
}, {
Name: "Login",
Method: "POST",
URLPath: URLPath{Value: "/api/v1/login"},
Request: &apimodel.LoginRequest{},
Response: &apimodel.LoginResponse{},
}, {
Name: "MeasurementMeta",
Method: "GET",
URLPath: URLPath{Value: "/api/v1/measurement_meta"},
Request: &apimodel.MeasurementMetaRequest{},
Response: &apimodel.MeasurementMetaResponse{},
CachePolicy: CacheAlways,
}, {
Name: "Register",
Method: "POST",
URLPath: URLPath{Value: "/api/v1/register"},
Request: &apimodel.RegisterRequest{},
Response: &apimodel.RegisterResponse{},
}, {
Name: "TestHelpers",
Method: "GET",
URLPath: URLPath{Value: "/api/v1/test-helpers"},
Request: &apimodel.TestHelpersRequest{},
Response: apimodel.TestHelpersResponse{},
}, {
Name: "PsiphonConfig",
RequiresLogin: true,
Method: "GET",
URLPath: URLPath{Value: "/api/v1/test-list/psiphon-config"},
Request: &apimodel.PsiphonConfigRequest{},
Response: apimodel.PsiphonConfigResponse{},
}, {
Name: "TorTargets",
RequiresLogin: true,
Method: "GET",
URLPath: URLPath{Value: "/api/v1/test-list/tor-targets"},
Request: &apimodel.TorTargetsRequest{},
Response: apimodel.TorTargetsResponse{},
}, {
Name: "URLs",
Method: "GET",
URLPath: URLPath{Value: "/api/v1/test-list/urls"},
Request: &apimodel.URLsRequest{},
Response: &apimodel.URLsResponse{},
}, {
Name: "OpenReport",
Method: "POST",
URLPath: URLPath{Value: "/report"},
Request: &apimodel.OpenReportRequest{},
Response: &apimodel.OpenReportResponse{},
}, {
Name: "SubmitMeasurement",
Method: "POST",
URLPath: URLPath{
InSwagger: "/report/{report_id}",
IsTemplate: true,
Value: "/report/{{ .ReportID }}",
},
Request: &apimodel.SubmitMeasurementRequest{},
Response: &apimodel.SubmitMeasurementResponse{},
}}

194
internal/ooapi/internal/generator/swaggertest.go
View File

@ -1,194 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"log"
"reflect"
"strings"
"sync"
"time"
"github.com/ooni/probe-cli/v3/internal/ooapi/internal/openapi"
)
const (
tagForJSON = "json"
tagForPath = "path"
)
func (d *Descriptor) genSwaggerURLPath() string {
up := d.URLPath
if up.InSwagger != "" {
return up.InSwagger
}
if up.IsTemplate {
panic("we should always use InSwapper and IsTemplate together")
}
return up.Value
}
func (d *Descriptor) genSwaggerSchema(cur reflect.Type) *openapi.Schema {
switch cur.Kind() {
case reflect.String:
return &openapi.Schema{Type: "string"}
case reflect.Bool:
return &openapi.Schema{Type: "boolean"}
case reflect.Int64:
return &openapi.Schema{Type: "integer"}
case reflect.Slice:
return &openapi.Schema{Type: "array", Items: d.genSwaggerSchema(cur.Elem())}
case reflect.Map:
return &openapi.Schema{Type: "object"}
case reflect.Ptr:
return d.genSwaggerSchema(cur.Elem())
case reflect.Struct:
if cur.String() == "time.Time" {
// Implementation note: we don't want to dive into time.Time but
// rather we want to pretend it's a string. The JSON parser for
// time.Time can indeed reconstruct a time.Time from a string, and
// it's much easier for us to let it do the parsing.
return &openapi.Schema{Type: "string"}
}
sinfo := &openapi.Schema{Type: "object"}
var once sync.Once
initmap := func() {
sinfo.Properties = make(map[string]*openapi.Schema)
}
for idx := 0; idx < cur.NumField(); idx++ {
field := cur.Field(idx)
if field.Tag.Get(tagForPath) != "" {
continue // skipping because this is a path param
}
if field.Tag.Get(tagForQuery) != "" {
continue // skipping because this is a query param
}
v := field.Name
if j := field.Tag.Get(tagForJSON); j != "" {
j = strings.Replace(j, ",omitempty", "", 1) // remove options
if j == "-" {
continue // not exported via JSON
}
v = j
}
once.Do(initmap)
sinfo.Properties[v] = d.genSwaggerSchema(field.Type)
}
return sinfo
case reflect.Interface:
return &openapi.Schema{Type: "object"}
default:
panic("unsupported type")
}
}
func (d *Descriptor) swaggerParamForType(t reflect.Type) string {
switch t.Kind() {
case reflect.String:
return "string"
case reflect.Bool:
return "boolean"
case reflect.Int64:
return "integer"
default:
panic("unsupported type")
}
}
func (d *Descriptor) genSwaggerParams(cur reflect.Type) []*openapi.Parameter {
// when we have params the input must be a pointer to struct
if cur.Kind() != reflect.Ptr {
panic("not a pointer")
}
cur = cur.Elem()
if cur.Kind() != reflect.Struct {
panic("not a pointer to struct")
}
// now that we're sure of the type, inspect the fields
var out []*openapi.Parameter
for idx := 0; idx < cur.NumField(); idx++ {
f := cur.Field(idx)
if q := f.Tag.Get(tagForQuery); q != "" {
out = append(
out, &openapi.Parameter{
Name: q,
In: "query",
Required: f.Tag.Get(tagForRequired) == "true",
Type: d.swaggerParamForType(f.Type),
})
continue
}
if p := f.Tag.Get(tagForPath); p != "" {
out = append(out, &openapi.Parameter{
Name: p,
In: "path",
Required: true,
Type: d.swaggerParamForType(f.Type),
})
continue
}
}
return out
}
func (d *Descriptor) genSwaggerPath() (string, *openapi.Path) {
pathStr, pathInfo := d.genSwaggerURLPath(), &openapi.Path{}
rtinfo := &openapi.RoundTrip{Produces: []string{"application/json"}}
switch d.Method {
case "GET":
pathInfo.Get = rtinfo
case "POST":
rtinfo.Consumes = append(rtinfo.Consumes, "application/json")
pathInfo.Post = rtinfo
default:
panic("unsupported method")
}
rtinfo.Parameters = d.genSwaggerParams(reflect.TypeOf(d.Request))
if d.Method != "GET" {
rtinfo.Parameters = append(rtinfo.Parameters, &openapi.Parameter{
Name: "body",
In: "body",
Required: true,
Schema: d.genSwaggerSchema(reflect.TypeOf(d.Request)),
})
}
rtinfo.Responses = &openapi.Responses{Successful: openapi.Body{
Description: "all good",
Schema: d.genSwaggerSchema(reflect.TypeOf(d.Response)),
}}
return pathStr, pathInfo
}
func genSwaggerVersion() string {
return time.Now().UTC().Format("0.20060102.1150405")
}
// GenSwaggerTestGo generates swagger_test.go
func GenSwaggerTestGo(file string) {
swagger := openapi.Swagger{
Swagger: "2.0",
Info: openapi.API{
Title: "OONI API specification",
Version: genSwaggerVersion(),
},
Host: "api.ooni.io",
BasePath: "/",
Schemes: []string{"https"},
Paths: make(map[string]*openapi.Path),
}
for _, desc := range Descriptors {
pathStr, pathInfo := desc.genSwaggerPath()
swagger.Paths[pathStr] = pathInfo
}
data, err := json.MarshalIndent(swagger, "", " ")
if err != nil {
log.Fatal(err)
}
var sb strings.Builder
fmt.Fprint(&sb, "// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&sb, "// %s\n\n", time.Now())
fmt.Fprint(&sb, "package ooapi\n\n")
fmt.Fprintf(&sb, "//go:generate go run ./internal/generator -file %s\n\n", file)
fmt.Fprintf(&sb, "const swagger = `%s`\n", string(data))
writefile(file, &sb)
}

27
internal/ooapi/internal/generator/writefile.go
View File

@ -1,27 +0,0 @@
package main
import (
"fmt"
"log"
"os"
"strings"
"golang.org/x/sys/execabs"
)
func writefile(name string, sb *strings.Builder) {
filep, err := os.Create(name)
if err != nil {
log.Fatal(err)
}
if _, err := fmt.Fprint(filep, sb.String()); err != nil {
log.Fatal(err)
}
if err := filep.Close(); err != nil {
log.Fatal(err)
}
cmd := execabs.Command("go", "fmt", name)
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
}

64
internal/ooapi/internal/openapi/openapi.go
View File

@ -1,64 +0,0 @@
// Package openapi contains data structures for Swagger v2.0.
//
// We use these data structures to compare the API specification we
// have here with the one of the server.
package openapi
// Schema is the schema of a specific parameter or
// or the schema used by the response body
type Schema struct {
Properties map[string]*Schema `json:"properties,omitempty"`
Items *Schema `json:"items,omitempty"`
Type string `json:"type"`
}
// Parameter describes an input parameter, which could be in the
// URL path, in the query string, or in the request body
type Parameter struct {
In string `json:"in"`
Name string `json:"name"`
Required bool `json:"required,omitempty"`
Schema *Schema `json:"schema,omitempty"`
Type string `json:"type,omitempty"`
}
// Body describes a response body
type Body struct {
Description interface{} `json:"description,omitempty"`
Schema *Schema `json:"schema"`
}
// Responses describes the possible responses
type Responses struct {
Successful Body `json:"200"`
}
// RoundTrip describes an HTTP round trip with a given method and path
type RoundTrip struct {
Consumes []string `json:"consumes,omitempty"`
Produces []string `json:"produces,omitempty"`
Parameters []*Parameter `json:"parameters,omitempty"`
Responses *Responses `json:"responses,omitempty"`
}
// Path describes a path served by the API
type Path struct {
Get *RoundTrip `json:"get,omitempty"`
Post *RoundTrip `json:"post,omitempty"`
}
// API contains info about the API
type API struct {
Title string `json:"title"`
Version string `json:"version"`
}
// Swagger is the toplevel structure
type Swagger struct {
Swagger string `json:"swagger"`
Info API `json:"info"`
Host string `json:"host"`
BasePath string `json:"basePath"`
Schemes []string `json:"schemes"`
Paths map[string]*Path `json:"paths"`
}

295
internal/ooapi/login.go
View File

@ -1,295 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:49.900013 +0200 CEST m=+0.000719959
package ooapi
//go:generate go run ./internal/generator -file login.go
import (
"context"
"errors"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
// withLoginPsiphonConfigAPI implements login for simplePsiphonConfigAPI.
type withLoginPsiphonConfigAPI struct {
API clonerForPsiphonConfigAPI // mandatory
JSONCodec JSONCodec // optional
KVStore KVStore // mandatory
RegisterAPI callerForRegisterAPI // mandatory
LoginAPI callerForLoginAPI // mandatory
}
// Call logins, if needed, then calls the API.
func (api *withLoginPsiphonConfigAPI) Call(ctx context.Context, req *apimodel.PsiphonConfigRequest) (apimodel.PsiphonConfigResponse, error) {
token, err := api.maybeLogin(ctx)
if err != nil {
return nil, err
}
resp, err := api.API.WithToken(token).Call(ctx, req)
if errors.Is(err, ErrUnauthorized) {
// Maybe the clock is just off? Let's try to obtain
// a token again and see if this fixes it.
if token, err = api.forceLogin(ctx); err == nil {
switch resp, err = api.API.WithToken(token).Call(ctx, req); err {
case nil:
return resp, nil
case ErrUnauthorized:
// fallthrough
default:
return nil, err
}
}
// Okay, this seems a broader problem. How about we try
// and re-register ourselves again instead?
token, err = api.forceRegister(ctx)
if err != nil {
return nil, err
}
resp, err = api.API.WithToken(token).Call(ctx, req)
// fallthrough
}
if err != nil {
return nil, err
}
return resp, nil
}
func (api *withLoginPsiphonConfigAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *withLoginPsiphonConfigAPI) readstate() (*loginState, error) {
data, err := api.KVStore.Get(loginKey)
if err != nil {
return nil, err
}
var ls loginState
if err := api.jsonCodec().Decode(data, &ls); err != nil {
return nil, err
}
return &ls, nil
}
func (api *withLoginPsiphonConfigAPI) writestate(ls *loginState) error {
data, err := api.jsonCodec().Encode(*ls)
if err != nil {
return err
}
return api.KVStore.Set(loginKey, data)
}
func (api *withLoginPsiphonConfigAPI) doRegister(ctx context.Context, password string) (string, error) {
req := newRegisterRequest(password)
ls := &loginState{}
resp, err := api.RegisterAPI.Call(ctx, req)
if err != nil {
return "", err
}
ls.ClientID = resp.ClientID
ls.Password = req.Password
return api.doLogin(ctx, ls)
}
func (api *withLoginPsiphonConfigAPI) forceRegister(ctx context.Context) (string, error) {
var password string
// If we already have a previous password, let us keep
// using it. This will allow a new version of the API to
// be able to continue to identify this probe. (This
// assumes that we have a stateless API that generates
// the user ID as a signature of the password plus a
// timestamp and that the key to generate the signature
// is not lost. If all these conditions are met, we
// can then serve better test targets to more long running
// (and therefore trusted) probes.)
if ls, err := api.readstate(); err == nil {
password = ls.Password
}
if password == "" {
password = newRandomPassword()
}
return api.doRegister(ctx, password)
}
func (api *withLoginPsiphonConfigAPI) forceLogin(ctx context.Context) (string, error) {
ls, err := api.readstate()
if err != nil {
return "", err
}
return api.doLogin(ctx, ls)
}
func (api *withLoginPsiphonConfigAPI) maybeLogin(ctx context.Context) (string, error) {
ls, _ := api.readstate()
if ls == nil || !ls.credentialsValid() {
return api.forceRegister(ctx)
}
if !ls.tokenValid() {
return api.doLogin(ctx, ls)
}
return ls.Token, nil
}
func (api *withLoginPsiphonConfigAPI) doLogin(ctx context.Context, ls *loginState) (string, error) {
req := &apimodel.LoginRequest{
ClientID: ls.ClientID,
Password: ls.Password,
}
resp, err := api.LoginAPI.Call(ctx, req)
if err != nil {
return "", err
}
ls.Token = resp.Token
ls.Expire = resp.Expire
if err := api.writestate(ls); err != nil {
return "", err
}
return ls.Token, nil
}
var _ callerForPsiphonConfigAPI = &withLoginPsiphonConfigAPI{}
// withLoginTorTargetsAPI implements login for simpleTorTargetsAPI.
type withLoginTorTargetsAPI struct {
API clonerForTorTargetsAPI // mandatory
JSONCodec JSONCodec // optional
KVStore KVStore // mandatory
RegisterAPI callerForRegisterAPI // mandatory
LoginAPI callerForLoginAPI // mandatory
}
// Call logins, if needed, then calls the API.
func (api *withLoginTorTargetsAPI) Call(ctx context.Context, req *apimodel.TorTargetsRequest) (apimodel.TorTargetsResponse, error) {
token, err := api.maybeLogin(ctx)
if err != nil {
return nil, err
}
resp, err := api.API.WithToken(token).Call(ctx, req)
if errors.Is(err, ErrUnauthorized) {
// Maybe the clock is just off? Let's try to obtain
// a token again and see if this fixes it.
if token, err = api.forceLogin(ctx); err == nil {
switch resp, err = api.API.WithToken(token).Call(ctx, req); err {
case nil:
return resp, nil
case ErrUnauthorized:
// fallthrough
default:
return nil, err
}
}
// Okay, this seems a broader problem. How about we try
// and re-register ourselves again instead?
token, err = api.forceRegister(ctx)
if err != nil {
return nil, err
}
resp, err = api.API.WithToken(token).Call(ctx, req)
// fallthrough
}
if err != nil {
return nil, err
}
return resp, nil
}
func (api *withLoginTorTargetsAPI) jsonCodec() JSONCodec {
if api.JSONCodec != nil {
return api.JSONCodec
}
return &defaultJSONCodec{}
}
func (api *withLoginTorTargetsAPI) readstate() (*loginState, error) {
data, err := api.KVStore.Get(loginKey)
if err != nil {
return nil, err
}
var ls loginState
if err := api.jsonCodec().Decode(data, &ls); err != nil {
return nil, err
}
return &ls, nil
}
func (api *withLoginTorTargetsAPI) writestate(ls *loginState) error {
data, err := api.jsonCodec().Encode(*ls)
if err != nil {
return err
}
return api.KVStore.Set(loginKey, data)
}
func (api *withLoginTorTargetsAPI) doRegister(ctx context.Context, password string) (string, error) {
req := newRegisterRequest(password)
ls := &loginState{}
resp, err := api.RegisterAPI.Call(ctx, req)
if err != nil {
return "", err
}
ls.ClientID = resp.ClientID
ls.Password = req.Password
return api.doLogin(ctx, ls)
}
func (api *withLoginTorTargetsAPI) forceRegister(ctx context.Context) (string, error) {
var password string
// If we already have a previous password, let us keep
// using it. This will allow a new version of the API to
// be able to continue to identify this probe. (This
// assumes that we have a stateless API that generates
// the user ID as a signature of the password plus a
// timestamp and that the key to generate the signature
// is not lost. If all these conditions are met, we
// can then serve better test targets to more long running
// (and therefore trusted) probes.)
if ls, err := api.readstate(); err == nil {
password = ls.Password
}
if password == "" {
password = newRandomPassword()
}
return api.doRegister(ctx, password)
}
func (api *withLoginTorTargetsAPI) forceLogin(ctx context.Context) (string, error) {
ls, err := api.readstate()
if err != nil {
return "", err
}
return api.doLogin(ctx, ls)
}
func (api *withLoginTorTargetsAPI) maybeLogin(ctx context.Context) (string, error) {
ls, _ := api.readstate()
if ls == nil || !ls.credentialsValid() {
return api.forceRegister(ctx)
}
if !ls.tokenValid() {
return api.doLogin(ctx, ls)
}
return ls.Token, nil
}
func (api *withLoginTorTargetsAPI) doLogin(ctx context.Context, ls *loginState) (string, error) {
req := &apimodel.LoginRequest{
ClientID: ls.ClientID,
Password: ls.Password,
}
resp, err := api.LoginAPI.Call(ctx, req)
if err != nil {
return "", err
}
ls.Token = resp.Token
ls.Expire = resp.Expire
if err := api.writestate(ls); err != nil {
return "", err
}
return ls.Token, nil
}
var _ callerForTorTargetsAPI = &withLoginTorTargetsAPI{}

1433
internal/ooapi/login_test.go
View File

File diff suppressed because it is too large Load Diff

208
internal/ooapi/loginhandler_test.go
View File

@ -1,208 +0,0 @@
package ooapi
import (
"encoding/json"
"net/http"
"strings"
"sync"
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
// LoginHandler is an http.Handler to test login
type LoginHandler struct {
failCallWith []int // ignored by login and register
mu sync.Mutex
noRegister bool
state []*loginState
t *testing.T
logins *atomicx.Int64
registers *atomicx.Int64
}
func (lh *LoginHandler) forgetLogins() {
defer lh.mu.Unlock()
lh.mu.Lock()
lh.state = nil
}
func (lh *LoginHandler) forgetTokens() {
defer lh.mu.Unlock()
lh.mu.Lock()
for _, entry := range lh.state {
// This should be enough to cause all tokens to
// be expired and force clients to relogin.
//
// (It does not matter much whether the client
// clock is off, or the server clock is off,
// thanks Galileo for explaining this to us <3.)
entry.Expire = time.Now().Add(-3600 * time.Second)
}
}
func (lh *LoginHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Implementation note: we don't check for the method
// for simplicity since it's already tested.
switch r.URL.Path {
case "/api/v1/register":
if lh.registers != nil {
lh.registers.Add(1)
}
lh.register(w, r)
case "/api/v1/login":
if lh.logins != nil {
lh.logins.Add(1)
}
lh.login(w, r)
case "/api/v1/test-list/psiphon-config":
lh.psiphon(w, r)
case "/api/v1/test-list/tor-targets":
lh.tor(w, r)
default:
w.WriteHeader(500)
}
}
func (lh *LoginHandler) register(w http.ResponseWriter, r *http.Request) {
if r.Body == nil {
w.WriteHeader(400)
return
}
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
var req apimodel.RegisterRequest
if err := json.Unmarshal(data, &req); err != nil {
w.WriteHeader(400)
return
}
if req.Password == "" {
w.WriteHeader(400)
return
}
defer lh.mu.Unlock()
lh.mu.Lock()
if lh.noRegister {
// We have been asked to stop registering clients so
// we're going to make a boo boo.
w.WriteHeader(500)
return
}
var resp apimodel.RegisterResponse
ff := &fakeFill{}
ff.Fill(&resp)
lh.state = append(lh.state, &loginState{
ClientID: resp.ClientID, Password: req.Password})
data, err = json.Marshal(&resp)
if err != nil {
w.WriteHeader(500)
return
}
lh.t.Logf("register: %+v", string(data))
w.Write(data)
}
func (lh *LoginHandler) login(w http.ResponseWriter, r *http.Request) {
if r.Body == nil {
w.WriteHeader(400)
return
}
data, err := netxlite.ReadAllContext(r.Context(), r.Body)
if err != nil {
w.WriteHeader(400)
return
}
var req apimodel.LoginRequest
if err := json.Unmarshal(data, &req); err != nil {
w.WriteHeader(400)
return
}
defer lh.mu.Unlock()
lh.mu.Lock()
for _, s := range lh.state {
if req.ClientID == s.ClientID && req.Password == s.Password {
var resp apimodel.LoginResponse
ff := &fakeFill{}
ff.Fill(&resp)
// We want the token to be many seconds in the future while
// ff.fill only sets the tokent to now plus a small delta.
resp.Expire = time.Now().Add(3600 * time.Second)
s.Expire = resp.Expire
s.Token = resp.Token
data, err = json.Marshal(&resp)
if err != nil {
w.WriteHeader(500)
return
}
lh.t.Logf("login: %+v", string(data))
w.Write(data)
return
}
}
lh.t.Log("login: 401")
w.WriteHeader(401)
}
func (lh *LoginHandler) psiphon(w http.ResponseWriter, r *http.Request) {
defer lh.mu.Unlock()
lh.mu.Lock()
if len(lh.failCallWith) > 0 {
code := lh.failCallWith[0]
lh.failCallWith = lh.failCallWith[1:]
w.WriteHeader(code)
return
}
token := strings.Replace(r.Header.Get("Authorization"), "Bearer ", "", 1)
for _, s := range lh.state {
if token == s.Token && time.Now().Before(s.Expire) {
var resp apimodel.PsiphonConfigResponse
ff := &fakeFill{}
ff.Fill(&resp)
data, err := json.Marshal(&resp)
if err != nil {
w.WriteHeader(500)
return
}
lh.t.Logf("psiphon: %+v", string(data))
w.Write(data)
return
}
}
lh.t.Log("psiphon: 401")
w.WriteHeader(401)
}
func (lh *LoginHandler) tor(w http.ResponseWriter, r *http.Request) {
defer lh.mu.Unlock()
lh.mu.Lock()
if len(lh.failCallWith) > 0 {
code := lh.failCallWith[0]
lh.failCallWith = lh.failCallWith[1:]
w.WriteHeader(code)
return
}
token := strings.Replace(r.Header.Get("Authorization"), "Bearer ", "", 1)
for _, s := range lh.state {
if token == s.Token && time.Now().Before(s.Expire) {
var resp apimodel.TorTargetsResponse
ff := &fakeFill{}
ff.Fill(&resp)
data, err := json.Marshal(&resp)
if err != nil {
w.WriteHeader(500)
return
}
lh.t.Logf("tor: %+v", string(data))
w.Write(data)
return
}
}
lh.t.Log("tor: 401")
w.WriteHeader(401)
}

59
internal/ooapi/loginmodel.go
View File

@ -1,59 +0,0 @@
package ooapi
import (
"crypto/rand"
"encoding/base64"
"time"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
// loginState is the struct saved in the kvstore
// to keep track of the login state.
type loginState struct {
ClientID string
Expire time.Time
Password string
Token string
}
func (ls *loginState) credentialsValid() bool {
return ls.ClientID != "" && ls.Password != ""
}
func (ls *loginState) tokenValid() bool {
return ls.Token != "" && time.Now().Add(60*time.Second).Before(ls.Expire)
}
// loginKey is the key with which loginState is saved
// into the key-value store used by Client.
const loginKey = "orchestra.state"
// newRandomPassword generates a new random password.
func newRandomPassword() string {
b := make([]byte, 48)
_, err := rand.Read(b)
runtimex.PanicOnError(err, "rand.Read failed")
return base64.StdEncoding.EncodeToString(b)
}
// newRegisterRequest creates a new RegisterRequest.
func newRegisterRequest(password string) *apimodel.RegisterRequest {
return &apimodel.RegisterRequest{
// The original implementation has as its only use case that we
// were registering and logging in for sending an update regarding
// the probe whereabouts. Yet here in probe-engine, the orchestra
// is currently only used to fetch inputs. For this purpose, we don't
// need to communicate any specific information. The code that will
// perform an update used to be responsible of doing that. Now, we
// are not using orchestra for this purpose anymore.
Platform: "miniooni",
ProbeASN: "AS0",
ProbeCC: "ZZ",
SoftwareName: "miniooni",
SoftwareVersion: "0.1.0-dev",
SupportedTests: []string{"web_connectivity"},
Password: password,
}
}

192
internal/ooapi/requests.go
View File

@ -1,192 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:50.891059 +0200 CEST m=+0.000799292
package ooapi
//go:generate go run ./internal/generator -file requests.go
import (
"bytes"
"context"
"net/http"
"net/url"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
func (api *simpleCheckReportIDAPI) newRequest(ctx context.Context, req *apimodel.CheckReportIDRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/_/check_report_id"
q := url.Values{}
if req.ReportID == "" {
return nil, newErrEmptyField("ReportID")
}
q.Add("report_id", req.ReportID)
URL.RawQuery = q.Encode()
return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil)
}
func (api *simpleCheckInAPI) newRequest(ctx context.Context, req *apimodel.CheckInRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/v1/check-in"
body, err := api.jsonCodec().Encode(req)
if err != nil {
return nil, err
}
out, err := api.requestMaker().NewRequest(ctx, "POST", URL.String(), bytes.NewReader(body))
if err != nil {
return nil, err
}
out.Header.Set("Content-Type", "application/json")
return out, nil
}
func (api *simpleLoginAPI) newRequest(ctx context.Context, req *apimodel.LoginRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/v1/login"
body, err := api.jsonCodec().Encode(req)
if err != nil {
return nil, err
}
out, err := api.requestMaker().NewRequest(ctx, "POST", URL.String(), bytes.NewReader(body))
if err != nil {
return nil, err
}
out.Header.Set("Content-Type", "application/json")
return out, nil
}
func (api *simpleMeasurementMetaAPI) newRequest(ctx context.Context, req *apimodel.MeasurementMetaRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/v1/measurement_meta"
q := url.Values{}
if req.ReportID == "" {
return nil, newErrEmptyField("ReportID")
}
q.Add("report_id", req.ReportID)
if req.Full {
q.Add("full", "true")
}
if req.Input != "" {
q.Add("input", req.Input)
}
URL.RawQuery = q.Encode()
return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil)
}
func (api *simpleRegisterAPI) newRequest(ctx context.Context, req *apimodel.RegisterRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/v1/register"
body, err := api.jsonCodec().Encode(req)
if err != nil {
return nil, err
}
out, err := api.requestMaker().NewRequest(ctx, "POST", URL.String(), bytes.NewReader(body))
if err != nil {
return nil, err
}
out.Header.Set("Content-Type", "application/json")
return out, nil
}
func (api *simpleTestHelpersAPI) newRequest(ctx context.Context, req *apimodel.TestHelpersRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/v1/test-helpers"
return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil)
}
func (api *simplePsiphonConfigAPI) newRequest(ctx context.Context, req *apimodel.PsiphonConfigRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/v1/test-list/psiphon-config"
return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil)
}
func (api *simpleTorTargetsAPI) newRequest(ctx context.Context, req *apimodel.TorTargetsRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/v1/test-list/tor-targets"
return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil)
}
func (api *simpleURLsAPI) newRequest(ctx context.Context, req *apimodel.URLsRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/api/v1/test-list/urls"
q := url.Values{}
if req.CategoryCodes != "" {
q.Add("category_codes", req.CategoryCodes)
}
if req.CountryCode != "" {
q.Add("country_code", req.CountryCode)
}
if req.Limit != 0 {
q.Add("limit", newQueryFieldInt64(req.Limit))
}
URL.RawQuery = q.Encode()
return api.requestMaker().NewRequest(ctx, "GET", URL.String(), nil)
}
func (api *simpleOpenReportAPI) newRequest(ctx context.Context, req *apimodel.OpenReportRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
URL.Path = "/report"
body, err := api.jsonCodec().Encode(req)
if err != nil {
return nil, err
}
out, err := api.requestMaker().NewRequest(ctx, "POST", URL.String(), bytes.NewReader(body))
if err != nil {
return nil, err
}
out.Header.Set("Content-Type", "application/json")
return out, nil
}
func (api *simpleSubmitMeasurementAPI) newRequest(ctx context.Context, req *apimodel.SubmitMeasurementRequest) (*http.Request, error) {
URL, err := url.Parse(api.baseURL())
if err != nil {
return nil, err
}
up, err := api.templateExecutor().Execute("/report/{{ .ReportID }}", req)
if err != nil {
return nil, err
}
URL.Path = up
body, err := api.jsonCodec().Encode(req)
if err != nil {
return nil, err
}
out, err := api.requestMaker().NewRequest(ctx, "POST", URL.String(), bytes.NewReader(body))
if err != nil {
return nil, err
}
out.Header.Set("Content-Type", "application/json")
return out, nil
}

277
internal/ooapi/responses.go
View File

@ -1,277 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:51.316851 +0200 CEST m=+0.000816793
package ooapi
//go:generate go run ./internal/generator -file responses.go
import (
"context"
"io"
"net/http"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
)
func (api *simpleCheckReportIDAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.CheckReportIDResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := &apimodel.CheckReportIDResponse{}
if err := api.jsonCodec().Decode(data, out); err != nil {
return nil, err
}
return out, nil
}
func (api *simpleCheckInAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.CheckInResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := &apimodel.CheckInResponse{}
if err := api.jsonCodec().Decode(data, out); err != nil {
return nil, err
}
return out, nil
}
func (api *simpleLoginAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.LoginResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := &apimodel.LoginResponse{}
if err := api.jsonCodec().Decode(data, out); err != nil {
return nil, err
}
return out, nil
}
func (api *simpleMeasurementMetaAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.MeasurementMetaResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := &apimodel.MeasurementMetaResponse{}
if err := api.jsonCodec().Decode(data, out); err != nil {
return nil, err
}
return out, nil
}
func (api *simpleRegisterAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.RegisterResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := &apimodel.RegisterResponse{}
if err := api.jsonCodec().Decode(data, out); err != nil {
return nil, err
}
return out, nil
}
func (api *simpleTestHelpersAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.TestHelpersResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := apimodel.TestHelpersResponse{}
if err := api.jsonCodec().Decode(data, &out); err != nil {
return nil, err
}
if out == nil {
return nil, ErrJSONLiteralNull
}
return out, nil
}
func (api *simplePsiphonConfigAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.PsiphonConfigResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := apimodel.PsiphonConfigResponse{}
if err := api.jsonCodec().Decode(data, &out); err != nil {
return nil, err
}
if out == nil {
return nil, ErrJSONLiteralNull
}
return out, nil
}
func (api *simpleTorTargetsAPI) newResponse(ctx context.Context, resp *http.Response, err error) (apimodel.TorTargetsResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := apimodel.TorTargetsResponse{}
if err := api.jsonCodec().Decode(data, &out); err != nil {
return nil, err
}
if out == nil {
return nil, ErrJSONLiteralNull
}
return out, nil
}
func (api *simpleURLsAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.URLsResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := &apimodel.URLsResponse{}
if err := api.jsonCodec().Decode(data, out); err != nil {
return nil, err
}
return out, nil
}
func (api *simpleOpenReportAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.OpenReportResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := &apimodel.OpenReportResponse{}
if err := api.jsonCodec().Decode(data, out); err != nil {
return nil, err
}
return out, nil
}
func (api *simpleSubmitMeasurementAPI) newResponse(ctx context.Context, resp *http.Response, err error) (*apimodel.SubmitMeasurementResponse, error) {
if err != nil {
return nil, err
}
if resp.StatusCode == 401 {
return nil, ErrUnauthorized
}
if resp.StatusCode != 200 {
return nil, newHTTPFailure(resp.StatusCode)
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 4<<20)
data, err := netxlite.ReadAllContext(ctx, reader)
if err != nil {
return nil, err
}
out := &apimodel.SubmitMeasurementResponse{}
if err := api.jsonCodec().Decode(data, out); err != nil {
return nil, err
}
return out, nil
}

578
internal/ooapi/swagger_test.go
View File

@ -1,578 +0,0 @@
// Code generated by go generate; DO NOT EDIT.
// 2022-05-19 20:30:51.768205 +0200 CEST m=+0.001003793
package ooapi
//go:generate go run ./internal/generator -file swagger_test.go
const swagger = `{
"swagger": "2.0",
"info": {
"title": "OONI API specification",
"version": "0.20220519.5183051"
},
"host": "api.ooni.io",
"basePath": "/",
"schemes": [
"https"
],
"paths": {
"/api/_/check_report_id": {
"get": {
"produces": [
"application/json"
],
"parameters": [
{
"in": "query",
"name": "report_id",
"required": true,
"type": "string"
}
],
"responses": {
"200": {
"description": "all good",
"schema": {
"properties": {
"error": {
"type": "string"
},
"found": {
"type": "boolean"
},
"v": {
"type": "integer"
}
},
"type": "object"
}
}
}
}
},
"/api/v1/check-in": {
"post": {
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"parameters": [
{
"in": "body",
"name": "body",
"required": true,
"schema": {
"properties": {
"charging": {
"type": "boolean"
},
"on_wifi": {
"type": "boolean"
},
"platform": {
"type": "string"
},
"probe_asn": {
"type": "string"
},
"probe_cc": {
"type": "string"
},
"run_type": {
"type": "string"
},
"software_name": {
"type": "string"
},
"software_version": {
"type": "string"
},
"web_connectivity": {
"properties": {
"category_codes": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
}
},
"type": "object"
}
}
],
"responses": {
"200": {
"description": "all good",
"schema": {
"properties": {
"probe_asn": {
"type": "string"
},
"probe_cc": {
"type": "string"
},
"tests": {
"properties": {
"web_connectivity": {
"properties": {
"report_id": {
"type": "string"
},
"urls": {
"items": {
"properties": {
"category_code": {
"type": "string"
},
"country_code": {
"type": "string"
},
"url": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
}
},
"type": "object"
}
},
"type": "object"
},
"v": {
"type": "integer"
}
},
"type": "object"
}
}
}
}
},
"/api/v1/login": {
"post": {
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"parameters": [
{
"in": "body",
"name": "body",
"required": true,
"schema": {
"properties": {
"password": {
"type": "string"
},
"username": {
"type": "string"
}
},
"type": "object"
}
}
],
"responses": {
"200": {
"description": "all good",
"schema": {
"properties": {
"expire": {
"type": "string"
},
"token": {
"type": "string"
}
},
"type": "object"
}
}
}
}
},
"/api/v1/measurement_meta": {
"get": {
"produces": [
"application/json"
],
"parameters": [
{
"in": "query",
"name": "report_id",
"required": true,
"type": "string"
},
{
"in": "query",
"name": "full",
"type": "boolean"
},
{
"in": "query",
"name": "input",
"type": "string"
}
],
"responses": {
"200": {
"description": "all good",
"schema": {
"properties": {
"anomaly": {
"type": "boolean"
},
"category_code": {
"type": "string"
},
"confirmed": {
"type": "boolean"
},
"failure": {
"type": "boolean"
},
"input": {
"type": "string"
},
"measurement_start_time": {
"type": "string"
},
"probe_asn": {
"type": "integer"
},
"probe_cc": {
"type": "string"
},
"raw_measurement": {
"type": "string"
},
"report_id": {
"type": "string"
},
"scores": {
"type": "string"
},
"test_name": {
"type": "string"
},
"test_start_time": {
"type": "string"
}
},
"type": "object"
}
}
}
}
},
"/api/v1/register": {
"post": {
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"parameters": [
{
"in": "body",
"name": "body",
"required": true,
"schema": {
"properties": {
"available_bandwidth": {
"type": "string"
},
"device_token": {
"type": "string"
},
"language": {
"type": "string"
},
"network_type": {
"type": "string"
},
"password": {
"type": "string"
},
"platform": {
"type": "string"
},
"probe_asn": {
"type": "string"
},
"probe_cc": {
"type": "string"
},
"probe_family": {
"type": "string"
},
"probe_timezone": {
"type": "string"
},
"software_name": {
"type": "string"
},
"software_version": {
"type": "string"
},
"supported_tests": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
}
}
],
"responses": {
"200": {
"description": "all good",
"schema": {
"properties": {
"client_id": {
"type": "string"
}
},
"type": "object"
}
}
}
}
},
"/api/v1/test-helpers": {
"get": {
"produces": [
"application/json"
],
"responses": {
"200": {
"description": "all good",
"schema": {
"type": "object"
}
}
}
}
},
"/api/v1/test-list/psiphon-config": {
"get": {
"produces": [
"application/json"
],
"responses": {
"200": {
"description": "all good",
"schema": {
"type": "object"
}
}
}
}
},
"/api/v1/test-list/tor-targets": {
"get": {
"produces": [
"application/json"
],
"responses": {
"200": {
"description": "all good",
"schema": {
"type": "object"
}
}
}
}
},
"/api/v1/test-list/urls": {
"get": {
"produces": [
"application/json"
],
"parameters": [
{
"in": "query",
"name": "category_codes",
"type": "string"
},
{
"in": "query",
"name": "country_code",
"type": "string"
},
{
"in": "query",
"name": "limit",
"type": "integer"
}
],
"responses": {
"200": {
"description": "all good",
"schema": {
"properties": {
"metadata": {
"properties": {
"count": {
"type": "integer"
}
},
"type": "object"
},
"results": {
"items": {
"properties": {
"category_code": {
"type": "string"
},
"country_code": {
"type": "string"
},
"url": {
"type": "string"
}
},
"type": "object"
},
"type": "array"
}
},
"type": "object"
}
}
}
}
},
"/report": {
"post": {
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"parameters": [
{
"in": "body",
"name": "body",
"required": true,
"schema": {
"properties": {
"data_format_version": {
"type": "string"
},
"format": {
"type": "string"
},
"probe_asn": {
"type": "string"
},
"probe_cc": {
"type": "string"
},
"software_name": {
"type": "string"
},
"software_version": {
"type": "string"
},
"test_name": {
"type": "string"
},
"test_start_time": {
"type": "string"
},
"test_version": {
"type": "string"
}
},
"type": "object"
}
}
],
"responses": {
"200": {
"description": "all good",
"schema": {
"properties": {
"backend_version": {
"type": "string"
},
"report_id": {
"type": "string"
},
"supported_formats": {
"items": {
"type": "string"
},
"type": "array"
}
},
"type": "object"
}
}
}
}
},
"/report/{report_id}": {
"post": {
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"parameters": [
{
"in": "path",
"name": "report_id",
"required": true,
"type": "string"
},
{
"in": "body",
"name": "body",
"required": true,
"schema": {
"properties": {
"content": {
"type": "object"
},
"format": {
"type": "string"
}
},
"type": "object"
}
}
],
"responses": {
"200": {
"description": "all good",
"schema": {
"properties": {
"measurement_uid": {
"type": "string"
}
},
"type": "object"
}
}
}
}
}
}
}`

161
internal/ooapi/swaggerdiff_test.go
View File

@ -1,161 +0,0 @@
package ooapi
import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"sort"
"strings"
"testing"
"github.com/hexops/gotextdiff"
"github.com/hexops/gotextdiff/myers"
"github.com/hexops/gotextdiff/span"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/ooapi/internal/openapi"
)
const (
productionURL = "https://api.ooni.io/apispec_1.json"
testingURL = "https://ams-pg-test.ooni.org/apispec_1.json"
)
func makeModel(data []byte) *openapi.Swagger {
var out openapi.Swagger
if err := json.Unmarshal(data, &out); err != nil {
log.Fatal(err)
}
// We reduce irrelevant differences by producing a common header
return &openapi.Swagger{Paths: out.Paths}
}
func getServerModel(serverURL string) *openapi.Swagger {
resp, err := http.Get(serverURL)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
data, err := netxlite.ReadAllContext(context.Background(), resp.Body)
if err != nil {
log.Fatal(err)
}
return makeModel(data)
}
func getClientModel() *openapi.Swagger {
return makeModel([]byte(swagger))
}
func simplifyRoundTrip(rt *openapi.RoundTrip) {
// Normalize the used name when a parameter is in body. This
// should only have a cosmetic impact on the spec.
for _, param := range rt.Parameters {
if param.In == "body" {
param.Name = "body"
}
}
// Sort parameters so the comparison does not depend on order.
sort.SliceStable(rt.Parameters, func(i, j int) bool {
left, right := rt.Parameters[i].Name, rt.Parameters[j].Name
return strings.Compare(left, right) < 0
})
// Normalize description of 200 response
rt.Responses.Successful.Description = "all good"
}
func simplifyInPlace(path *openapi.Path) *openapi.Path {
if path.Get != nil && path.Post != nil {
log.Fatal("unsupported configuration")
}
if path.Get != nil {
simplifyRoundTrip(path.Get)
}
if path.Post != nil {
simplifyRoundTrip(path.Post)
}
return path
}
func jsonify(model interface{}) string {
data, err := json.MarshalIndent(model, "", " ")
if err != nil {
log.Fatal(err)
}
return string(data)
}
type diffable struct {
name string
value string
}
func computediff(server, client *diffable) string {
d := gotextdiff.ToUnified(server.name, client.name, server.value, myers.ComputeEdits(
span.URIFromPath(server.name), server.value, client.value,
))
return fmt.Sprint(d)
}
// maybediff emits the diff between the server and the client and
// returns the length of the diff itself in bytes.
func maybediff(key string, server, client *openapi.Path) int {
diff := computediff(&diffable{
name: fmt.Sprintf("server%s.json", key),
value: jsonify(simplifyInPlace(server)),
}, &diffable{
name: fmt.Sprintf("client%s.json", key),
value: jsonify(simplifyInPlace(client)),
})
if diff != "" {
fmt.Printf("%s", diff)
}
return len(diff)
}
func compare(serverURL string) bool {
good := true
serverModel, clientModel := getServerModel(serverURL), getClientModel()
// Implementation note: the server model is richer than the client
// model, so we ignore everything not defined by the client.
var count int
for key := range serverModel.Paths {
if _, found := clientModel.Paths[key]; !found {
delete(serverModel.Paths, key)
continue
}
count++
if maybediff(key, serverModel.Paths[key], clientModel.Paths[key]) > 0 {
good = false
}
}
if count <= 0 {
panic("no element found")
}
return good
}
func TestWithProductionAPI(t *testing.T) {
t.Skip("skip until we use this part of the codebase")
if testing.Short() {
t.Skip("skip test in short mode")
}
t.Log("using ", productionURL)
if !compare(productionURL) {
t.Fatal("model mismatch (see above)")
}
}
func TestWithTestingAPI(t *testing.T) {
t.Skip("skip until we use this part of the codebase")
if testing.Short() {
t.Skip("skip test in short mode")
}
t.Log("using ", testingURL)
if !compare(testingURL) {
t.Fatal("model mismatch (see above)")
}
}

23
internal/ooapi/utils.go
View File

@ -1,23 +0,0 @@
package ooapi
import "fmt"
func newErrEmptyField(field string) error {
return fmt.Errorf("%w: %s", ErrEmptyField, field)
}
func newHTTPFailure(status int) error {
return fmt.Errorf("%w: %d", ErrHTTPFailure, status)
}
func newQueryFieldInt64(v int64) string {
return fmt.Sprintf("%d", v)
}
func newQueryFieldBool(v bool) string {
return fmt.Sprintf("%v", v)
}
func newAuthorizationHeader(token string) string {
return fmt.Sprintf("Bearer %s", token)
}

12
internal/ooapi/utils_test.go
View File

@ -1,12 +0,0 @@
package ooapi
import "testing"
func TestNewQueryFieldBoolWorks(t *testing.T) {
if s := newQueryFieldBool(true); s != "true" {
t.Fatal("invalid encoding of true")
}
if s := newQueryFieldBool(false); s != "false" {
t.Fatal("invalid encoding of false")
}
}