diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 899b966..5831424 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -8,5 +8,8 @@ jobs: test: runs-on: ubuntu-20.04 steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t ./MOBILE/android/oonimkall.aar + - run: ./mk OONI_PSIPHON_TAGS="" ./MOBILE/android/oonimkall.aar diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 06e4f13..c8f49a7 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -8,5 +8,8 @@ jobs: test: runs-on: macos-10.15 steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t ./MOBILE/ios/oonimkall.framework.zip + - run: ./mk OONI_PSIPHON_TAGS="" XCODE_VERSION=12.4 ./MOBILE/ios/oonimkall.framework.zip diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index adb21dc..97e9231 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -13,11 +13,11 @@ jobs: echo $'{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json sudo service docker restart - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t ./CLI/linux/amd64/ooniprobe + - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/linux/amd64/ooniprobe env: DOCKER_CLI_EXPERIMENTAL: enabled - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe - - run: ./make --disable-embedding-psiphon-config -t debian_amd64 + - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64 - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd - run: | for deb in *.deb; do @@ -35,12 +35,13 @@ jobs: echo $'{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json sudo service docker restart - uses: actions/checkout@v2 - - run: sudo apt-get install qemu-user-static - - run: ./make --disable-embedding-psiphon-config -t ./CLI/linux/arm64/ooniprobe + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static + - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/linux/arm64/ooniprobe env: DOCKER_CLI_EXPERIMENTAL: enabled - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe - - run: ./make --disable-embedding-psiphon-config -t debian_arm64 + - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64 - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd - run: | for deb in *.deb; do diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 85607ce..d32830e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -8,6 +8,9 @@ jobs: build: runs-on: "macos-10.15" steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t ./CLI/darwin/amd64/ooniprobe + - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/darwin/amd64/ooniprobe - run: ./smoketest.sh ./CLI/darwin/amd64/ooniprobe diff --git a/.github/workflows/miniooni.yml b/.github/workflows/miniooni.yml index 5089d22..4aa187c 100644 --- a/.github/workflows/miniooni.yml +++ b/.github/workflows/miniooni.yml @@ -10,8 +10,11 @@ jobs: test: runs-on: ubuntu-20.04 steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t miniooni + - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/miniooni - run: ./CLI/linux/amd64/miniooni --yes -nNi https://example.com web_connectivity diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 11fbb08..51055a4 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -8,9 +8,12 @@ jobs: build: runs-on: "ubuntu-20.04" steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - run: sudo apt install mingw-w64 - - run: ./make --disable-embedding-psiphon-config -t ./CLI/windows/amd64/ooniprobe.exe + - run: ./mk OONI_PSIPHON_TAGS="" MINGW_W64_VERSION="9.3-win32" ./CLI/windows/amd64/ooniprobe.exe - uses: actions/upload-artifact@v2 with: name: ooniprobe.exe diff --git a/CLI/linux/build b/CLI/linux/build index 6fa148e..6235a22 100755 --- a/CLI/linux/build +++ b/CLI/linux/build @@ -1,15 +1,18 @@ #!/bin/sh -# This script is executed by `./make` when building inside +# This script is executed by `./mk` when building inside # an Alpine Linux docker container. Using Alpine Linux, which # uses musl libc, allows us to emit static binaries. set -e if [ "$GOARCH" = "" ]; then - echo 'fatal: $GOARCH is not set' 1>&2 - exit 1 + echo 'fatal: GOARCH is not set' 1>&2 + exit 1 fi set -x apk update apk upgrade apk add --no-progress gcc git linux-headers musl-dev -CGO_ENABLED=1 GOOS=linux GOARCH=$GOARCH go build -o ./CLI/linux/$GOARCH/ \ - -ldflags='-s -w -extldflags "-static"' "$@" ./cmd/ooniprobe +export GOPATH=$GOPATH +export CGO_ENABLED=1 +export GOOS=linux +export GOARCH=$GOARCH +go build -o "./CLI/linux/$GOARCH/" -ldflags='-s -w -extldflags "-static"' "$@" ./cmd/ooniprobe diff --git a/CLI/linux/debian b/CLI/linux/debian index 4b11bec..78e1090 100755 --- a/CLI/linux/debian +++ b/CLI/linux/debian @@ -1,5 +1,5 @@ #!/bin/sh -# This script creates a Debian package. When run by `./make`, it +# This script creates a Debian package. When run by `./mk`, it # is run inside a debian:stable container. It's fine to also # run this script from any debian-like system, as long as the # following ASSUMPTIONS are met: @@ -11,8 +11,8 @@ # architecture of the `ooniprobe` we are packaging. if [ $# -gt 1 ]; then - echo "usage: $0 [run_number]" 1>&2 - exit 1 + echo "usage: $0 [run_number]" 1>&2 + exit 1 fi run_number=$1 @@ -20,32 +20,32 @@ run_number=$1 # by the debian/ooniprobe-cli.install file. rm -rf ./debian/bin mkdir -p ./debian/bin -machine=`uname -m` +machine=$(uname -m) goarch="" case $machine in - x86_64) - cp ./CLI/linux/amd64/ooniprobe ./debian/bin - goarch=amd64 - ;; - aarch64) - cp ./CLI/linux/arm64/ooniprobe ./debian/bin - goarch=arm64 - ;; - *) - # TODO(bassosimone): here we probably want to further extend - # this script to support at least armv7. - echo "FATAL: unsupported machine: $machine" 1>&2 - exit 1 - ;; +x86_64) + cp ./CLI/linux/amd64/ooniprobe ./debian/bin + goarch=amd64 + ;; +aarch64) + cp ./CLI/linux/arm64/ooniprobe ./debian/bin + goarch=arm64 + ;; +*) + # TODO(bassosimone): here we probably want to further extend + # this script to support at least armv7. + echo "FATAL: unsupported machine: $machine" 1>&2 + exit 1 + ;; esac set -ex # figure out the version number from the binary itself (which rests -# on the assumption that `uname -m` can run such a binary) -version=`./debian/bin/ooniprobe version` -if [ ! -z $run_number ]; then - version="${version}~${run_number}" +# on the assumption that we can run such a binary) +version=$(./debian/bin/ooniprobe version) +if [ -n "$run_number" ]; then + version="${version}~${run_number}" fi # The OONI_DEB_DRY_RUN is a semi-undocumented feature allowing @@ -58,7 +58,7 @@ $OONI_DEB_DRY_RUN apt-get build-dep -y --no-install-recommends . # keep the original changelog file safe $OONI_DEB_DRY_RUN cp ./debian/changelog ./debian/changelog.oocopy -$OONI_DEB_DRY_RUN dch -v $version "New version ${version}" +$OONI_DEB_DRY_RUN dch -v "$version" "New version ${version}" $OONI_DEB_DRY_RUN dpkg-buildpackage -us -uc -b # restore the original changelog file @@ -70,4 +70,4 @@ $OONI_DEB_DRY_RUN mv ../*.deb . # install the package on the container as a smoke test to # ensure that it is installable. -DEBIAN_FRONTEND=noninteractive dpkg -i ooniprobe-cli_${version}_${goarch}.deb +DEBIAN_FRONTEND=noninteractive dpkg -i "ooniprobe-cli_${version}_${goarch}.deb" diff --git a/Readme.md b/Readme.md index ea0cd09..757d636 100644 --- a/Readme.md +++ b/Readme.md @@ -34,13 +34,13 @@ This will generate a binary called `ooniprobe` in the current directory. ## Android bindings -Make sure you have Python 3.8+ installed, then run: +Make sure you have GNU make installed, then run: ```bash -./make -t android +./mk android ``` -Builds bindings for Android. (Add `----disable-embedding-psiphon-config` if you +Builds bindings for Android. (Add `OONI_PSIPHON_TAGS=""` if you cannot clone private repositories in the https://github.com/ooni namespace.) The generated bindings are (manually) pushed to the Maven Central package @@ -49,13 +49,13 @@ are published along with the release notes. ## iOS bindings -Make sure you have Python 3.8+ installed, then run: +Make sure you have GNU make installed, then run: ```bash -./make -t ios +./mk ios ``` -Builds bindings for iOS. (Add `----disable-embedding-psiphon-config` if you +Builds bindings for iOS. (Add `OONI_PSIPHON_TAGS=""` if you cannot clone private repositories in the https://github.com/ooni namespace.) The generated bindings are (manually) added to GitHub releases. The instructions @@ -83,4 +83,5 @@ Create an issue according to [the routine release template]( https://github.com/ooni/probe/blob/master/.github/ISSUE_TEMPLATE/routine-sprint-releases.md) and perform any item inside the check-list. -We build releases using `./make`, which requires Python3.8+. +We build releases using `./mk`, which requires GNU make. Try +the `./mk help|less` command for detailed usage. diff --git a/make b/make deleted file mode 100755 index a18c86a..0000000 --- a/make +++ /dev/null @@ -1,1434 +0,0 @@ -#!/usr/bin/env python3 - -""" Build script for ooniprobe. You can get documentation regarding -its usage by running `./make --help`. """ - -from __future__ import annotations -import datetime - -import getopt -import os -import platform -import shlex -import shutil -import subprocess -import sys - -from typing import Any -from typing import Dict -from typing import List -from typing import NoReturn -from typing import Optional -from typing import Protocol -from typing import Tuple - - -def android_cmdlinetools_os() -> str: - """android_cmdlinetools_os maps the name of the current OS to the - name used by the android command-line tools file.""" - system = platform.system() - if system == "Linux": - return "linux" - if system == "Darwin": - return "mac" - raise RuntimeError(system) - - -def android_cmdlinetools_version() -> str: - """android_cmdlinetools_version returns the version of the Android - command-line tools that we'd like to download.""" - return "6858069" - - -def android_cmdlinetools_sha256sum() -> str: - """android_cmdlinetools_sha256sum returns the SHA256 sum of the - Android command-line tools zip file.""" - return { - "linux": "87f6dcf41d4e642e37ba03cb2e387a542aa0bd73cb689a9e7152aad40a6e7a08", - "mac": "58a55d9c5bcacd7c42170d2cf2c9ae2889c6797a6128307aaf69100636f54a13", - }[android_cmdlinetools_os()] - - -def cachedir() -> str: - """cachedir returns the directory where we cache the SDKs.""" - return os.path.join(os.path.expandvars("${HOME}"), ".ooniprobe-build") - - -def goversion() -> str: - """goversion is the Go version we use.""" - return "1.16.3" - - -def gopath() -> str: - """gopath is the GOPATH we use.""" - return os.path.expandvars("${HOME}/go") - - -def gosha256sum() -> str: - """gosha256sum returns the SHA256 sum of the Go tarball.""" - return { - "linux": { - "amd64": "951a3c7c6ce4e56ad883f97d9db74d3d6d80d5fec77455c6ada6c1f7ac4776d2", - "arm64": "566b1d6f17d2bc4ad5f81486f0df44f3088c3ed47a3bec4099d8ed9939e90d5d", - }, - "darwin": { - "amd64": "6bb1cf421f8abc2a9a4e39140b7397cdae6aca3e8d36dcff39a1a77f4f1170ac", - "arm64": "f4e96bbcd5d2d1942f5b55d9e4ab19564da4fad192012f6d7b0b9b055ba4208f", - }, - }[goos()][goarch()] - - -def goos() -> str: - """goos returns the GOOS value for the current system.""" - system = platform.system() - if system == "Linux": - return "linux" - if system == "Darwin": - return "darwin" - raise RuntimeError(system) - - -def goarch() -> str: - """goarch returns the GOARCH value for the current system.""" - machine = platform.machine() - if machine in ("arm64", "arm", "386", "amd64"): - return machine - if machine in ("x86", "i386"): - return "386" - if machine == "x86_64": - return "amd64" - if machine == "aarch64": - return "arm64" - raise RuntimeError(machine) - - -def android_ndk_version() -> str: - """android_ndk_version returns the Android NDK version.""" - return "22.1.7171670" - - -def sdkmanager_install_cmd(binpath: str) -> List[str]: - """sdkmanager_install_cmd returns the command line for installing - all the required dependencies using the sdkmanager.""" - return [ - os.path.join(binpath, "sdkmanager"), - "--install", - "build-tools;29.0.3", - "platforms;android-30", - "ndk;{}".format(android_ndk_version()), - ] - - -def log(msg: str) -> None: - """log prints a message on the standard output.""" - print(msg, flush=True) - - -class Options(Protocol): - """Options contains the configured options.""" - - def debugging(self) -> bool: - """debugging indicates whether to pass -x to `go build...`.""" - - def disable_embedding_psiphon_config(self) -> bool: - """disable_embedding_psiphon_config indicates that the user - does not want us to embed an encrypted psiphon config file into - the binary.""" - - def dry_run(self) -> bool: - """dry_run indicates whether to execute commands.""" - - def target(self) -> str: - """target is the target to build.""" - - def verbose(self) -> bool: - """verbose indicates whether to pass -v to `go build...`.""" - - -class ConfigFromCLI: - """ConfigFromCLI parses options from CLI flags.""" - - @classmethod - def parse(cls, targets: List[str], top_targets: List[Target]) -> ConfigFromCLI: - """parse parses command line options and returns a - suitable configuration object.""" - conf = cls() - conf._parse(targets, top_targets) - return conf - - def __init__(self) -> None: - self._debugging = False - self._disable_embedding_psiphon_config = False - self._dry_run = False - self._target = "" - self._verbose = False - - def debugging(self) -> bool: - return self._debugging - - def disable_embedding_psiphon_config(self) -> bool: - return self._disable_embedding_psiphon_config - - def dry_run(self) -> bool: - return self._dry_run - - def target(self) -> str: - return self._target - - def verbose(self) -> bool: - return self._verbose - - # The main reason why I am using getopt here is such that I am able - # to print a very clear and detailed usage string. (But the same - # could be obtained quite likely w/ argparse.) - - _usage_string = """\ -usage: ./make [--disable-embedding-psiphon-config] [-nvx] -t target - ./make -l - ./make [--help|-h] - -The first form of the command builds the `target` specified using the -`-t` command line flag. If the target has dependencies, this command will -build the dependent targets first. The `-n` flag enables a dry run where -the command only prints the commands it would run. The `-v` and `-x` flags -are passed directly to `go build ...` and `gomobile bind ...`. The -`--disable-embedding-psiphon-config` flag causes this command to disable -embedding a psiphon config file into the generated binary; you should -use this option when you cannot clone the private repository containing -the psiphon configuration file. - -The second form of the command lists all the available targets, showing -top-level targets and their recursive dependencies. - -The third form of the command prints this help screen. -""" - - @classmethod - def _usage(cls, err: str = "", exitcode: int = 0) -> NoReturn: - if err: - sys.stderr.write("error: {}\n".format(err)) - sys.stderr.write(cls._usage_string) - sys.exit(exitcode) - - def _parse(self, targets: List[str], top_targets: List[Target]): - try: - opts, args = getopt.getopt( - sys.argv[1:], "hlnt:vx", ["disable-embedding-psiphon-config", "help"] - ) - except getopt.GetoptError as err: - self._usage(err=err.msg, exitcode=1) - if args: - self._usage(err="unexpected number of positional arguments", exitcode=1) - for key, value in opts: - if key == "--disable-embedding-psiphon-config": - self._disable_embedding_psiphon_config = True - continue - if key in ("-h", "--help"): - self._usage() - if key == "-l": - self._list_targets(top_targets) - if key == "-n": - self._dry_run = True - continue - if key == "-t": - self._target = value - continue - if key == "-v": - self._verbose = True - continue - if key == "-x": - self._debugging = True - continue - raise RuntimeError(key, value) - - if self._target == "": # no arguments is equivalent to --help - self._usage() - - if self._target not in targets: - sys.stderr.write("unknown target: {}\n".format(self._target)) - sys.stderr.write("try `./make -l` to see the available targets.\n") - sys.exit(1) - - def _list_targets(self, top_targets: List[Target]) -> NoReturn: - for tgt in top_targets: - self._print_target(tgt, 0) - sys.exit(0) - - def _print_target(self, target: Target, indent: int) -> None: - sys.stdout.write( - "{}{}{}\n".format( - " " * indent, target.name(), ":" if target.deps() else "" - ) - ) - for dep in target.deps(): - self._print_target(dep, indent + 1) - if indent <= 0: - sys.stdout.write("\n") - - -class Engine(Protocol): - """Engine is an engine for building targets.""" - - def backticks(self, cmdline: List[str]) -> str: - """backticks executes the given command line and returns - the output emitted by the command to the caller.""" - - def cat_sed_redirect( - self, patterns: List[Tuple[str, str]], source: str, dest: str - ) -> None: - """cat_sed_redirect does - `cat $source|sed -e "s/$patterns[0][0]/$patterns[0][1]/g" ... > $dest`.""" - - def echo_to_file(self, content: str, filepath: str) -> None: - """echo_to_file writes the content string to the given file.""" - - def require(self, *executable: str) -> None: - """require fails if executable is not found in path.""" - - def run( - self, - cmdline: List[str], - cwd: Optional[str] = None, - inputbytes: Optional[bytes] = None, - ) -> None: - """run runs the specified command line.""" - - def setenv(self, key: str, value: str) -> Optional[str]: - """setenv sets an environment variable and returns the - previous value of such variable (or None).""" - - def unsetenv(self, key: str) -> None: - """unsetenv clears an environment variable.""" - - -class CommandExecutor: - """CommandExecutor executes commands.""" - - def __init__(self, dry_runner: Engine): - self._dry_runner = dry_runner - - def backticks(self, cmdline: List[str]) -> str: - """backticks implements Engine.backticks""" - # Nothing else to do, because backticks is fully - # implemented by CommandDryRunner. - return self._dry_runner.backticks(cmdline) - - def cat_sed_redirect( - self, patterns: List[Tuple[str, str]], source: str, dest: str - ) -> None: - """cat_sed_redirect implements Engine.cat_sed_redirect.""" - self._dry_runner.cat_sed_redirect(patterns, source, dest) - with open(source, "r") as sourcefp: - data = sourcefp.read() - for p, v in patterns: - data = data.replace(p, v) - with open(dest, "w") as destfp: - destfp.write(data) - - def echo_to_file(self, content: str, filepath: str) -> None: - """echo_to_file implements Engine.echo_to_file""" - self._dry_runner.echo_to_file(content, filepath) - with open(filepath, "w") as filep: - filep.write(content) - filep.write("\n") - - def require(self, *executable: str) -> None: - """require implements Engine.require.""" - for exc in executable: - self._dry_runner.require(exc) - fullpath = shutil.which(exc) - if not fullpath: - log("checking for {}... not found".format(exc)) - sys.exit(1) - log("checking for {}... {}".format(exc, fullpath)) - - def run( - self, - cmdline: List[str], - cwd: Optional[str] = None, - inputbytes: Optional[bytes] = None, - ) -> None: - """run implements Engine.run.""" - self._dry_runner.run(cmdline, cwd, inputbytes) - subprocess.run(cmdline, check=True, cwd=cwd, input=inputbytes) - - def setenv(self, key: str, value: str) -> Optional[str]: - """setenv implements Engine.setenv.""" - # Nothing else to do, because setenv is fully - # implemented by CommandDryRunner. - return self._dry_runner.setenv(key, value) - - def unsetenv(self, key: str) -> None: - """unsetenv implements Engine.unsetenv.""" - # Nothing else to do, because unsetenv is fully - # implemented by CommandDryRunner. - self._dry_runner.unsetenv(key) - - -class CommandDryRunner: - """CommandDryRunner is the dry runner.""" - - # Implementation note: here we try to log valid bash snippets - # such that is really obvious what we are doing. - - def backticks(self, cmdline: List[str]) -> str: - """backticks implements Engine.backticks""" - # The backticks command is used to gather information used by - # other commands. As such, it needs to always run. If it was not - # running, we could not correctly implement the `-n` flag. It's - # also a silent command, because it's not really part of the - # sequence of bash commands that are executed. ¯\_(ツ)_/¯ - popen = subprocess.Popen(cmdline, stdout=subprocess.PIPE) - stdout = popen.communicate()[0] - if popen.returncode != 0: - raise RuntimeError(popen.returncode) - return stdout.decode("utf-8").strip() - - def cat_sed_redirect( - self, patterns: List[Tuple[str, str]], source: str, dest: str - ) -> None: - """cat_sed_redirect implements Engine.cat_sed_redirect.""" - out = "./make: cat {}|sed".format(source) - for p, v in patterns: - out += " -e 's/{}/{}/g'".format(p, v) - out += " > {}".format(dest) - log(out) - - def echo_to_file(self, content: str, filepath: str) -> None: - """echo_to_file implements Engine.echo_to_file""" - log("./make: echo '{}' > {}".format(content, filepath)) - - def require(self, *executable: str) -> None: - """require implements Engine.require.""" - for exc in executable: - log(f"./make: echo -n 'checking for {exc}... '") - log("./make: command -v %s || { echo 'not found'; exit 1 }" % exc) - - def run( - self, - cmdline: List[str], - cwd: Optional[str] = None, - inputbytes: Optional[bytes] = None, - ) -> None: - """run implements Engine.run.""" - cdpart = "" - if cwd: - cdpart = "cd {} && ".format(cwd) - log("./make: {}{}".format(cdpart, shlex.join(cmdline))) - - def setenv(self, key: str, value: str) -> Optional[str]: - """setenv implements Engine.setenv.""" - log("./make: export {}={}".format(key, shlex.join([value]))) - prev = os.environ.get(key) - os.environ[key] = value - return prev - - def unsetenv(self, key: str) -> None: - """unsetenv implements Engine.unsetenv.""" - log("./make: unset {}".format(key)) - del os.environ[key] - - -def new_engine(options: Options) -> Engine: - """new_engine creates a new engine instance""" - out: Engine = CommandDryRunner() - if not options.dry_run(): - out = CommandExecutor(out) - return out - - -class Environ: - """Environ creates a context where specific environment - variables are set. They will be restored to their previous - value when we are leaving the context.""" - - def __init__(self, engine: Engine, key: str, value: str) -> None: - self._engine = engine - self._key = key - self._value = value - self._prev: Optional[str] = None - - def __enter__(self) -> None: - self._prev = self._engine.setenv(self._key, self._value) - - def __exit__(self, type: Any, value: Any, traceback: Any) -> bool: - if self._prev is None: - self._engine.unsetenv(self._key) - return False # propagate exc - self._engine.setenv(self._key, self._prev) - return False # propagate exc - - -class AugmentedPath(Environ): - """AugementedPath is an Environ that prepends the required - directory to the currently existing search path.""" - - def __init__(self, engine: Engine, directory: str): - value = os.pathsep.join([directory, os.environ["PATH"]]) - super().__init__(engine, "PATH", value) - - -class WorkingDir: - """WorkingDir is a context manager that enters into a given working - directory and returns to the previous directory when done.""" - - def __init__(self, dirpath: str) -> None: - self._dirpath = dirpath - self._prev: str = "" - - def __enter__(self) -> None: - self._prev = os.getcwd() - os.chdir(self._dirpath) - - def __exit__(self, type: Any, value: Any, traceback: Any) -> bool: - os.chdir(self._prev) - return False # propagate exc - - -class Target(Protocol): - """Target is a target to build.""" - - def name(self) -> str: - """name returns the target name.""" - - def build(self, engine: Engine, options: Options) -> None: - """build builds the specified target.""" - - def deps(self) -> List[Target]: - """deps returns the target dependencies.""" - - -class BaseTarget: - """BaseTarget is the base class of all targets.""" - - def __init__(self, name: str, deps: List[Target]) -> None: - # prevent child classes from easily using these variables - self.__name = name - self.__deps = deps - - def name(self) -> str: - """name implements Target.name""" - return self.__name - - def build_child_targets(self, engine: Engine, options: Options) -> None: - """build_child_targets builds all the child targets""" - for dep in self.__deps: - dep.build(engine, options) - - def deps(self) -> List[Target]: - """deps implements Target.deps.""" - return self.__deps - - -class SDKGolangGo(BaseTarget): - """SDKGolangGo creates ${cachedir}/SDK/golang.""" - - # We download a golang SDK from upstream to make sure we - # are always using a specific version of golang/go. - - def __init__(self) -> None: - name = os.path.join(cachedir(), "SDK", "golang") - super().__init__(name, []) - - def binpath(self) -> str: - """binpath returns the path where the go binary is installed.""" - return os.path.join(self.name(), "go", "bin") - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isdir(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.require("mkdir", "curl", "shasum", "rm", "tar", "echo") - filename = "go{}.{}-{}.tar.gz".format(goversion(), goos(), goarch()) - url = "https://golang.org/dl/{}".format(filename) - engine.run(["mkdir", "-p", self.name()]) - filepath = os.path.join(self.name(), filename) - engine.run(["curl", "-fsSLo", filepath, url]) - sha256file = os.path.join(cachedir(), "SDK", "SHA256") - engine.echo_to_file("{} {}".format(gosha256sum(), filepath), sha256file) - engine.run(["shasum", "--check", sha256file]) - engine.run(["rm", sha256file]) - engine.run(["tar", "-xf", filename], cwd=self.name()) - engine.run(["rm", filepath]) - - def goroot(self): - """goroot returns the goroot.""" - return os.path.join(self.name(), "go") - - -class SDKOONIGo(BaseTarget): - """SDKOONIGo creates ${cachedir}/SDK/oonigo.""" - - # We use a private fork of golang/go on Android as a - # workaround for https://github.com/ooni/probe/issues/1444 - - def __init__(self) -> None: - name = os.path.join(cachedir(), "SDK", "oonigo") - self._gogo = SDKGolangGo() - super().__init__(name, [self._gogo]) - - def binpath(self) -> str: - """binpath returns the path where the go binary is installed.""" - return os.path.join(self.name(), "bin") - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isdir(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.require("git", "bash") - engine.run( - [ - "git", - "clone", - "-b", - "ooni", - "--single-branch", - "--depth", - "8", - "https://github.com/ooni/go", - self.name(), - ] - ) - with Environ(engine, "GOROOT_BOOTSTRAP", self._gogo.goroot()): - engine.run( - ["./make.bash"], - cwd=os.path.join(self.name(), "src"), - ) - - -class SDKAndroid(BaseTarget): - """SDKAndroid creates ${cachedir}/SDK/android.""" - - def __init__(self) -> None: - name = os.path.join(cachedir(), "SDK", "android") - super().__init__(name, []) - - def home(self) -> str: - """home returns the ANDROID_HOME""" - return self.name() - - def ndk_home(self) -> str: - """ndk_home returns the ANDROID_NDK_HOME""" - return os.path.join(self.home(), "ndk", android_ndk_version()) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isdir(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - log("\n./make: building {}...".format(self.name())) - engine.require("mkdir", "curl", "echo", "shasum", "rm", "unzip", "mv", "java") - filename = "commandlinetools-{}-{}_latest.zip".format( - android_cmdlinetools_os(), android_cmdlinetools_version() - ) - url = "https://dl.google.com/android/repository/{}".format(filename) - engine.run(["mkdir", "-p", self.name()]) - filepath = os.path.join(self.name(), filename) - engine.run(["curl", "-fsSLo", filepath, url]) - sha256file = os.path.join(cachedir(), "SDK", "SHA256") - engine.echo_to_file( - "{} {}".format(android_cmdlinetools_sha256sum(), filepath), sha256file - ) - engine.run(["shasum", "--check", sha256file]) - engine.run(["rm", sha256file]) - engine.run(["unzip", filename], cwd=self.name()) - engine.run(["rm", filepath]) - # See https://stackoverflow.com/a/61176718 to understand why - # we need to reorganize the directories like this: - engine.run( - ["mv", "cmdline-tools", android_cmdlinetools_version()], cwd=self.name() - ) - engine.run(["mkdir", "cmdline-tools"], cwd=self.name()) - engine.run( - ["mv", android_cmdlinetools_version(), "cmdline-tools"], cwd=self.name() - ) - engine.run( - sdkmanager_install_cmd( - os.path.join( - self.name(), - "cmdline-tools", - android_cmdlinetools_version(), - "bin", - ), - ), - inputbytes=b"Y\n", # automatically accept license - ) - - -class OONIProbePrivate(BaseTarget): - """OONIProbePrivate creates ${cachedir}/github.com/ooni/probe-private.""" - - # We use this private repository to copy the psiphon configuration - # file to embed into the ooniprobe binaries - - def __init__(self) -> None: - name = os.path.join(cachedir(), "github.com", "ooni", "probe-private") - super().__init__(name, []) - - def copyfiles(self, engine: Engine, options: Options) -> None: - """copyfiles copies psiphon config to the repository.""" - if options.disable_embedding_psiphon_config(): - log("./make: copy psiphon config: disabled by command line flags") - return - engine.run( - [ - "cp", - os.path.join(self.name(), "psiphon-config.json.age"), - os.path.join("internal", "engine"), - ] - ) - engine.run( - [ - "cp", - os.path.join(self.name(), "psiphon-config.key"), - os.path.join("internal", "engine"), - ] - ) - - def build(self, engine: Engine, options: Options) -> None: - if os.path.isdir(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - if options.disable_embedding_psiphon_config(): - log("\n./make: {}: disabled by command line flags".format(self.name())) - return - log("\n./make: building {}...".format(self.name())) - engine.require("git", "cp") - engine.run( - [ - "git", - "clone", - "git@github.com:ooni/probe-private", - self.name(), - ] - ) - - -class OONIMKAllAAR(BaseTarget): - """OONIMKAllAAR creates ./MOBILE/android/oonimkall.aar.""" - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "android", "oonimkall.aar") - self._ooprivate = OONIProbePrivate() - self._oonigo = SDKOONIGo() - self._android = SDKAndroid() - super().__init__(name, [self._ooprivate, self._oonigo, self._android]) - - def aarfile(self) -> str: - """aarfile returns the aar file path""" - return self.name() - - def srcfile(self) -> str: - """srcfile returns the path to the jar file containing sources.""" - return os.path.join(".", "MOBILE", "android", "oonimkall-sources.jar") - - def build(self, engine: Engine, options: Options) -> None: - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - engine.require("sh", "javac") - self._go_get_gomobile(engine, options, self._oonigo) - self._gomobile_init(engine, self._oonigo, self._android) - self._gomobile_bind(engine, options, self._oonigo, self._android) - - # Implementation note: we use proxy scripts for go and gomobile - # that explicitly print what they resolve go and gomobile to using - # `command -v`. This gives us extra confidence that we are really - # using the oonigo fork of golang/go. - - def _go_get_gomobile( - self, engine: Engine, options: Options, oonigo: SDKOONIGo - ) -> None: - # TODO(bassosimone): find a way to run this command without - # adding extra dependencies to go.mod and go.sum. - cmdline: List[str] = [] - cmdline.append("go") - cmdline.append("get") - cmdline.append("-u") - if options.verbose(): - cmdline.append("-v") - if options.debugging(): - cmdline.append("-x") - cmdline.append("golang.org/x/mobile/cmd/gomobile@latest") - with Environ(engine, "GOPATH", gopath()): - with AugmentedPath(engine, oonigo.binpath()): - engine.require("go") - engine.run(cmdline) - - def _gomobile_init( - self, - engine: Engine, - oonigo: SDKOONIGo, - android: SDKAndroid, - ) -> None: - cmdline: List[str] = [] - cmdline.append("gomobile") - cmdline.append("init") - with Environ(engine, "ANDROID_HOME", android.home()): - with Environ(engine, "ANDROID_NDK_HOME", android.ndk_home()): - with AugmentedPath(engine, oonigo.binpath()): - with AugmentedPath(engine, os.path.join(gopath(), "bin")): - engine.require("gomobile", "go") - engine.run(cmdline) - - def _gomobile_bind( - self, - engine: Engine, - options: Options, - oonigo: SDKOONIGo, - android: SDKAndroid, - ) -> None: - cmdline: List[str] = [] - cmdline.append("gomobile") - cmdline.append("bind") - if options.verbose(): - cmdline.append("-v") - if options.debugging(): - cmdline.append("-x") - cmdline.append("-target") - cmdline.append("android") - cmdline.append("-o") - cmdline.append(self.name()) - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags") - cmdline.append("ooni_psiphon_config") - cmdline.append("-ldflags") - cmdline.append("-s -w") - cmdline.append("./pkg/oonimkall") - with Environ(engine, "ANDROID_HOME", android.home()): - with Environ(engine, "ANDROID_NDK_HOME", android.ndk_home()): - with AugmentedPath(engine, oonigo.binpath()): - with AugmentedPath(engine, os.path.join(gopath(), "bin")): - engine.require("gomobile", "go") - engine.run(cmdline) - - -def sign(engine: Engine, filepath: str) -> str: - """sign signs the given filepath using pgp and returns - the filepath of the signature file.""" - engine.require("gpg") - user = "simone@openobservatory.org" - engine.run(["gpg", "-abu", user, filepath]) - return filepath + ".asc" - - -class BundleJAR(BaseTarget): - """BundleJAR creates ./MOBILE/android/bundle.jar.""" - - # We upload the bundle.jar file to maven central to bless - # a new release of the OONI libraries for Android. - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "android", "bundle.jar") - self._oonimkall = OONIMKAllAAR() - super().__init__(name, [self._oonimkall]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.require("cp", "gpg", "jar") - version = datetime.datetime.now().strftime("%Y.%m.%d-%H%M%S") - engine.run( - [ - "cp", - self._oonimkall.aarfile(), - os.path.join("MOBILE", "android", "oonimkall-{}.aar".format(version)), - ] - ) - engine.run( - [ - "cp", - self._oonimkall.srcfile(), - os.path.join( - "MOBILE", "android", "oonimkall-{}-sources.jar".format(version) - ), - ] - ) - engine.cat_sed_redirect( - [ - ("@VERSION@", version), - ], - os.path.join("MOBILE", "template.pom"), - os.path.join("MOBILE", "android", "oonimkall-{}.pom".format(version)), - ) - names = ( - "oonimkall-{}.aar".format(version), - "oonimkall-{}-sources.jar".format(version), - "oonimkall-{}.pom".format(version), - ) - allnames: List[str] = [] - with WorkingDir(os.path.join(".", "MOBILE", "android")): - for name in names: - allnames.append(name) - allnames.append(sign(engine, name)) - engine.run( - [ - "jar", - "-cf", - "bundle.jar", - *allnames, - ], - cwd=os.path.join(".", "MOBILE", "android"), - ) - - -class Phony(BaseTarget): - """Phony is a phony target that executes one or more other targets.""" - - def __init__(self, name: str, depends: List[Target]): - super().__init__(name, depends) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - self.build_child_targets(engine, options) - - -# Android is the top-level "android" target -ANDROID = Phony("android", [BundleJAR()]) - - -class OONIMKAllFramework(BaseTarget): - """OONIMKAllFramework creates ./MOBILE/ios/oonimkall.framework.""" - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "ios", "oonimkall.framework") - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build.""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - self._go_get_gomobile(engine, options, self._gogo) - self._gomobile_init(engine, self._gogo) - self._gomobile_bind(engine, options, self._gogo) - - def _go_get_gomobile( - self, - engine: Engine, - options: Options, - gogo: SDKGolangGo, - ) -> None: - # TODO(bassosimone): find a way to run this command without - # adding extra dependencies to go.mod and go.sum. - cmdline: List[str] = [] - cmdline.append("go") - cmdline.append("get") - cmdline.append("-u") - if options.verbose(): - cmdline.append("-v") - if options.debugging(): - cmdline.append("-x") - cmdline.append("golang.org/x/mobile/cmd/gomobile@latest") - with AugmentedPath(engine, gogo.binpath()): - with Environ(engine, "GOPATH", gopath()): - engine.require("go") - engine.run(cmdline) - - def _gomobile_init( - self, - engine: Engine, - gogo: SDKGolangGo, - ) -> None: - cmdline: List[str] = [] - cmdline.append("gomobile") - cmdline.append("init") - with AugmentedPath(engine, os.path.join(gopath(), "bin")): - with AugmentedPath(engine, gogo.binpath()): - engine.require("gomobile", "go") - engine.run(cmdline) - - def _gomobile_bind( - self, - engine: Engine, - options: Options, - gogo: SDKGolangGo, - ) -> None: - cmdline: List[str] = [] - cmdline.append("gomobile") - cmdline.append("bind") - if options.verbose(): - cmdline.append("-v") - if options.debugging(): - cmdline.append("-x") - cmdline.append("-target") - cmdline.append("ios") - cmdline.append("-o") - cmdline.append(self.name()) - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags") - cmdline.append("ooni_psiphon_config") - cmdline.append("-ldflags") - cmdline.append("-s -w") - cmdline.append("./pkg/oonimkall") - with AugmentedPath(engine, os.path.join(gopath(), "bin")): - with AugmentedPath(engine, gogo.binpath()): - engine.require("gomobile", "go") - engine.run(cmdline) - - -class OONIMKAllFrameworkZip(BaseTarget): - """OONIMKAllFrameworkZip creates ./MOBILE/ios/oonimkall.framework.zip.""" - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "ios", "oonimkall.framework.zip") - ooframework = OONIMKAllFramework() - super().__init__(name, [ooframework]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - engine.require("zip", "rm") - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.run( - [ - "rm", - "-rf", - "oonimkall.framework.zip", - ], - cwd=os.path.join(".", "MOBILE", "ios"), - ) - engine.run( - [ - "zip", - "-yr", - "oonimkall.framework.zip", - "oonimkall.framework", - ], - cwd=os.path.join(".", "MOBILE", "ios"), - ) - - -class OONIMKAllPodspec(BaseTarget): - """OONIMKAllPodspec creates ./MOBILE/ios/oonimkall.podspec.""" - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "ios", "oonimkall.podspec") - super().__init__(name, []) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("./make: {}: already built".format(self.name())) - return - engine.require("cat", "sed") - release = engine.backticks(["git", "describe", "--tags"]) - version = datetime.datetime.now().strftime("%Y.%m.%d-%H%M%S") - engine.cat_sed_redirect( - [("@VERSION@", version), ("@RELEASE@", release)], - os.path.join(".", "MOBILE", "template.podspec"), - self.name(), - ) - - -# IOS is the top-level "ios" target. -IOS = Phony("ios", [OONIMKAllFrameworkZip(), OONIMKAllPodspec()]) - - -class MiniOONIDarwinOrWindows(BaseTarget): - def __init__(self, goos: str, goarch: str): - self._ext = ".exe" if goos == "windows" else "" - self._os = goos - self._arch = goarch - name = os.path.join(".", "CLI", goos, goarch, "miniooni" + self._ext) - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - cmdline = [ - "go", - "build", - "-o", - self.name(), - "-ldflags=-s -w", - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags=ooni_psiphon_config") - cmdline.append("./internal/cmd/miniooni") - with Environ(engine, "GOOS", self._os): - with Environ(engine, "GOARCH", self._arch): - with Environ(engine, "CGO_ENABLED", "0"): - with AugmentedPath(engine, self._gogo.binpath()): - engine.require("go") - engine.run(cmdline) - - -class MiniOONILinux(BaseTarget): - def __init__(self, goarch: str): - self._arch = goarch - name = os.path.join(".", "CLI", "linux", goarch, "miniooni") - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - if self._arch == "arm": - with Environ(engine, "GOARM", "7"): - self._build(engine, options, self._gogo) - else: - self._build(engine, options, self._gogo) - - def _build(self, engine: Engine, options: Options, gogo: SDKGolangGo) -> None: - cmdline = [ - "go", - "build", - "-o", - os.path.join("CLI", "linux", self._arch, "miniooni"), - "-ldflags=-s -w -extldflags -static", - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - tags = "-tags=netgo" - if not options.disable_embedding_psiphon_config(): - tags += ",ooni_psiphon_config" - cmdline.append(tags) - cmdline.append("./internal/cmd/miniooni") - with Environ(engine, "GOOS", "linux"): - with Environ(engine, "GOARCH", self._arch): - with Environ(engine, "CGO_ENABLED", "0"): - with AugmentedPath(engine, gogo.binpath()): - engine.require("go") - engine.run(cmdline) - - -# MINIOONI is the top-level "miniooni" target. -MINIOONI = Phony( - "miniooni", - [ - MiniOONIDarwinOrWindows("darwin", "amd64"), - MiniOONIDarwinOrWindows("darwin", "arm64"), - MiniOONILinux("386"), - MiniOONILinux("amd64"), - MiniOONILinux("arm"), - MiniOONILinux("arm64"), - MiniOONIDarwinOrWindows("windows", "386"), - MiniOONIDarwinOrWindows("windows", "amd64"), - ], -) - - -class OONIProbeLinux(BaseTarget): - """OONIProbeLinux builds ooniprobe for Linux.""" - - # TODO(bassosimone): this works out of the box on macOS and - # requires qemu-user-static on Fedora/Debian. I'm not sure what - # is the right (set of) command(s) I should be checking for. - - def __init__(self, goarch: str): - self._arch = goarch - name = os.path.join(".", "CLI", "linux", goarch, "ooniprobe") - self._ooprivate = OONIProbePrivate() - super().__init__(name, [self._ooprivate]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build.""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - engine.require("docker") - # make sure we have the latest version of the container image - engine.run( - [ - "docker", - "pull", - "--platform", - "linux/{}".format(self._arch), - "golang:{}-alpine".format(goversion()), - ] - ) - # then run the build inside the container - cmdline = [ - "docker", - "run", - "--platform", - "linux/{}".format(self._arch), - "-e", - "GOARCH={}".format(self._arch), - "-v", - "{}:/ooni".format(os.getcwd()), - "-w", - "/ooni", - "golang:{}-alpine".format(goversion()), - os.path.join(".", "CLI", "linux", "build"), - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags=ooni_psiphon_config,netgo") - else: - cmdline.append("-tags=netgo") - engine.run(cmdline) - - -class OONIProbeWindows(BaseTarget): - """OONIProbeWindows builds ooniprobe for Windows.""" - - def __init__(self, goarch: str): - self._arch = goarch - name = os.path.join(".", "CLI", "windows", goarch, "ooniprobe.exe") - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def _gcc(self) -> str: - if self._arch == "amd64": - return "x86_64-w64-mingw32-gcc" - if self._arch == "386": - return "i686-w64-mingw32-gcc" - raise NotImplementedError - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - cmdline = [ - "go", - "build", - "-o", - self.name(), - "-ldflags=-s -w", - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags=ooni_psiphon_config") - cmdline.append("./cmd/ooniprobe") - with Environ(engine, "GOOS", "windows"): - with Environ(engine, "GOARCH", self._arch): - with Environ(engine, "CGO_ENABLED", "1"): - with Environ(engine, "CC", self._gcc()): - with AugmentedPath(engine, self._gogo.binpath()): - engine.require(self._gcc(), "go") - engine.run(cmdline) - - -class OONIProbeDarwin(BaseTarget): - """OONIProbeDarwin builds ooniprobe for macOS.""" - - def __init__(self, goarch: str): - self._arch = goarch - name = os.path.join(".", "CLI", "darwin", goarch, "ooniprobe") - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - cmdline = [ - "go", - "build", - "-o", - self.name(), - "-ldflags=-s -w", - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags=ooni_psiphon_config") - cmdline.append("./cmd/ooniprobe") - with Environ(engine, "GOOS", "darwin"): - with Environ(engine, "GOARCH", self._arch): - with Environ(engine, "CGO_ENABLED", "1"): - with AugmentedPath(engine, self._gogo.binpath()): - engine.require("gcc", "go") - engine.run(cmdline) - - -class Debian(BaseTarget): - """Debian makes a debian package for a given architecture.""" - - def __init__(self, arch: str): - name = "debian_{}".format(arch) - self._target = OONIProbeLinux(arch) - self._arch = arch - super().__init__(name, [self._target]) - - def build(self, engine: Engine, options: Options) -> None: - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.require("docker") - # make sure we have the latest version of the container image - engine.run( - [ - "docker", - "pull", - "--platform", - "linux/{}".format(self._arch), - "debian:stable", - ] - ) - # then run the build inside the container - cmdline: List[str] = [] - cmdline.append("docker") - cmdline.append("run") - cmdline.append("--platform") - cmdline.append("linux/{}".format(self._arch)) - cmdline.append("-v") - cmdline.append("{}:/ooni".format(os.getcwd())) - cmdline.append("-w") - cmdline.append("/ooni") - cmdline.append("debian:stable") - cmdline.append(os.path.join(".", "CLI", "linux", "debian")) - if os.environ.get("GITHUB_ACTIONS", "") == "true": - # When we're running inside a github action, figure out whether - # we are building a tag or a commit. In the latter case, we will - # append the run number to the version number. - github_ref = os.environ.get("GITHUB_REF") - if not github_ref: - raise RuntimeError("missing GITHUB_REF") - github_run_number = os.environ.get("GITHUB_RUN_NUMBER") - if not github_run_number: - raise RuntimeError("missing GITHUB_RUN_NUMBER") - if not github_ref.startswith("/refs/tags/"): - cmdline.append(github_run_number) - engine.run(cmdline) - - -class Sign(BaseTarget): - """Sign signs a specific target artefact.""" - - def __init__(self, target: Target): - self._target = target - name = self._target.name() + ".asc" - super().__init__(name, [self._target]) - - def build(self, engine: Engine, options: Options) -> None: - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - sign(engine, self._target.name()) - - -# OONIPROBE_RELEASE_DARWIN contains the release darwin targets -OONIPROBE_RELEASE_DARWIN = Phony( - "ooniprobe_release_darwin", - [ - Sign(OONIProbeDarwin("amd64")), - Sign(OONIProbeDarwin("arm64")), - ], -) - -# OONIPROBE_RELEASE_LINUX contains the release linux targets -OONIPROBE_RELEASE_LINUX = Phony( - "ooniprobe_release_linux", - [ - Sign(OONIProbeLinux("amd64")), - Sign(OONIProbeLinux("arm64")), - ], -) - -# OONIPROBE_RELEASE_WINDOWS contains the release windows targets -OONIPROBE_RELEASE_WINDOWS = Phony( - "ooniprobe_release_windows", - [ - Sign(OONIProbeWindows("amd64")), - Sign(OONIProbeWindows("386")), - ], -) - -# DEBIAN is the top-level "debian" target. -DEBIAN = Phony( - "debian", - [ - Debian("arm64"), - Debian("amd64"), - ], -) - -# TOP_TARGETS contains the top-level targets -TOP_TARGETS: List[Target] = [ - ANDROID, - IOS, - MINIOONI, - OONIPROBE_RELEASE_DARWIN, - OONIPROBE_RELEASE_LINUX, - OONIPROBE_RELEASE_WINDOWS, - DEBIAN, -] - - -def expand_targets(targets: List[Target]) -> Dict[str, Target]: - """expand_targets creates a dictionary mapping every existing - target name to its implementation.""" - out: Dict[str, Target] = {} - for tgt in targets: - out.update(expand_targets(tgt.deps())) - out[tgt.name()] = tgt - return out - - -def main() -> None: - """main function""" - alltargets = expand_targets(TOP_TARGETS) - options = ConfigFromCLI.parse(list(alltargets.keys()), TOP_TARGETS) - engine = new_engine(options) - # note that we check whether the target is known in parse() - selected = alltargets[options.target()] - selected.build(engine, options) - - -if __name__ == "__main__": - main() diff --git a/mk b/mk new file mode 100755 index 0000000..b8503b6 --- /dev/null +++ b/mk @@ -0,0 +1,602 @@ +#!/usr/bin/make -f + +#quickhelp: Usage: ./mk [VARIABLE=VALUE ...] TARGET ... +.PHONY: usage +usage: + @cat mk | grep '^#quickhelp:' | sed -e 's/^#quickhelp://' -e 's/^\ *//' + +# Most targets are .PHONY because whether to rebuild is controlled +# by golang. We expose to the user all the .PHONY targets. +#quickhelp: +#quickhelp: The `./mk list-targets` command lists all available targets. +.PHONY: list-targets +list-targets: + @cat mk | grep '^\.PHONY:' | sed -e 's/^\.PHONY://' + +#quickhelp: +#quickhelp: The `./mk help` command provides detailed usage instructions. We +#quickhelp: recommend running `./mk help|less` to page its output. +.PHONY: help +help: + @cat mk | grep -E '^#(quick)?help:' | sed -E -e 's/^#(quick)?help://' -e s'/^\ //' + +#help: +#help: The following variables control the build. You can specify them +#help: on the command line as a key-value pairs (see usage above). + +#help: +#help: * ANDROID_CLI_SHA256 : the SHA256 of the Android CLI tools file. We always +#help: download the Linux version, which seems to work +#help: also on macOS (thank you, Java! :pray:). +ANDROID_CLI_SHA256 = 7a00faadc0864f78edd8f4908a629a46d622375cbe2e5814e82934aebecdb622 + +#help: +#help: * ANDROID_CLI_VERSION : the version of the Android CLI tools. +ANDROID_CLI_VERSION = 7302050 + +#help: +#help: * ANDROID_INSTALL_EXTRA : contains the android tools we install in addition +#help: to the NDK in order to build oonimkall.aar. +ANDROID_INSTALL_EXTRA = 'build-tools;29.0.3' 'platforms;android-30' + +#help: +#help: * ANDROID_NDK_VERSION : Android NDK version. +ANDROID_NDK_VERSION = 22.1.7171670 + +#help: +#help: * DEBIAN_TILDE_VERSION : if non-empty, this should be "[0-9]+" and +#help: will be appended to the package version using +#help: a tilde, thus producing, e.g., "1.0~1234". +DEBIAN_TILDE_VERSION = + +#help: +#help: * GIT_CLONE_DIR : directory where to clone repositories, by default +#help: set to `$HOME/.ooniprobe-build/src`. +GIT_CLONE_DIR = $(HOME)/.ooniprobe-build/src + +# $(GIT_CLONE_DIR) is an internal target that creates $(GIT_CLONE_DIR). +$(GIT_CLONE_DIR): + mkdir -p $(GIT_CLONE_DIR) + +#help: +#help: * GOLANG_DOCKER_GOCACHE : where to store golang's build cache to +#help: speed up subsequent Docker builds. +GOLANG_DOCKER_GOCACHE = $(HOME)/.ooniprobe-build/docker/gocache + +#help: +#help: * GOLANG_DOCKER_GOPATH : GOPATH directory used by builds running +#help: inside docker to significantly speed +#help: up subsequent Docker based builds. +GOLANG_DOCKER_GOPATH := $(HOME)/.ooniprobe-build/docker/gopath + +#help: +#help: * GOLANG_EXTRA_FLAGS : extra flags passed to `go build ...`, empty by +#help: default. Useful to pass flags to `go`, e.g.: +#help: +#help: ./mk GOLANG_EXTRA_FLAGS="-x -v" ./CLI/miniooni +GOLANG_EXTRA_FLAGS = + +#help: +#help: * GOLANG_VERSION_NUMBER : the expected version number for golang. +GOLANG_VERSION_NUMBER = 1.16.4 + +#help: +#help: * GPG_USER : allows overriding the default GPG user used +#help: to sign binary releases, e.g.: +#help: +#help: ./mk GPG_USER=john@doe.com ooniprobe/windows +GPG_USER = simone@openobservatory.org + +#help: +#help: * MINGW_W64_VERSION : the expected mingw-w64 version. +MINGW_W64_VERSION = 10.3.1 + +#help: +#help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause +#help: the build to embed a psiphon configuration file +#help: into the generated binaries. This build tag +#help: implies cloning the git@github.com:ooni/probe-private +#help: repository. If you do not have the permission to +#help: clone it, just clear this variable, e.g.: +#help: +#help: ./mk OONI_PSIPHON_TAGS="" ./CLI/miniooni +OONI_PSIPHON_TAGS = ooni_psiphon_config + +#help: +#help: * OONI_ANDROID_HOME : directory where the Android SDK is downloaded +#help: and installed. You can point this to an existing +#help: copy of the SDK as long as (1) you have the +#help: right version of the command line tools, and +#help: (2) it's okay for us to install packages. +OONI_ANDROID_HOME = $(HOME)/.ooniprobe-build/sdk/android + +#help: +#help: * XCODE_VERSION : the version of Xcode we expect. +XCODE_VERSION = 12.5 + +#quickhelp: +#quickhelp: The `./mk show-config` command shows the current value of the +#quickhelp: variables controlling the build. +.PHONY: show-config +show-config: + @echo "ANDROID_CLI_VERSION=$(ANDROID_CLI_VERSION)" + @echo "ANDROID_CLI_SHA256=$(ANDROID_CLI_SHA256)" + @echo "ANDROID_INSTALL_EXTRA=$(ANDROID_INSTALL_EXTRA)" + @echo "ANDROID_NDK_VERSION=$(ANDROID_NDK_VERSION)" + @echo "DEBIAN_TILDE_VERSION=$(DEBIAN_TILDE_VERSION)" + @echo "GIT_CLONE_DIR=$(GIT_CLONE_DIR)" + @echo "GOLANG_DOCKER_GOCACHE=$(GOLANG_DOCKER_GOCACHE)" + @echo "GOLANG_DOCKER_GOPATH=$(GOLANG_DOCKER_GOPATH)" + @echo "GOLANG_EXTRA_FLAGS=$(GOLANG_EXTRA_FLAGS)" + @echo "GOLANG_VERSION_NUMBER=$(GOLANG_VERSION_NUMBER)" + @echo "GPG_USER=$(GPG_USER)" + @echo "MINGW_W64_VERSION=$(MINGW_W64_VERSION)" + @echo "OONI_PSIPHON_TAGS=$(OONI_PSIPHON_TAGS)" + @echo "OONI_ANDROID_HOME=$(OONI_ANDROID_HOME)" + @echo "XCODE_VERSION=$(XCODE_VERSION)" + +# GOLANG_VERSION_STRING is the expected version string. If we +# run a golang binary that does not emit this version string +# when running `go version`, we stop the build. +GOLANG_VERSION_STRING = go$(GOLANG_VERSION_NUMBER) + +# GOLANG_DOCKER_IMAGE is the golang docker image we use for +# building for Linux systems. It is an Alpine based container +# so that we can easily build static binaries. +GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine + +# Cross-compiling miniooni from any system with Go installed is +# very easy, because it does not use any C code. +#help: +#help: The `./mk ./CLI/miniooni` command builds the miniooni experimental +#help: command line client for all the supported GOOS/GOARCH. +#help: +#help: You can also build the following subtargets: +.PHONY: ./CLI/miniooni +./CLI/miniooni: \ + ./CLI/darwin/amd64/miniooni \ + ./CLI/darwin/arm64/miniooni \ + ./CLI/linux/386/miniooni \ + ./CLI/linux/amd64/miniooni \ + ./CLI/linux/arm/miniooni \ + ./CLI/linux/arm64/miniooni \ + ./CLI/windows/386/miniooni.exe \ + ./CLI/windows/amd64/miniooni.exe + +# All the miniooni targets build with CGO_ENABLED=0 such that the build +# succeeds when the GOOS/GOARCH is such that we aren't crosscompiling +# (e.g., targeting darwin/amd64 on darwin/amd64) _and_ there's no C compiler +# installed on the system. We can afford that since miniooni is pure Go. +#help: +#help: * `./mk ./CLI/darwin/amd64/miniooni`: darwin/amd64 +.PHONY: ./CLI/darwin/amd64/miniooni +./CLI/darwin/amd64/miniooni: search/for/go maybe/copypsiphon + GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/darwin/arm64/miniooni`: darwin/arm64 +.PHONY: ./CLI/darwin/arm64/miniooni +./CLI/darwin/arm64/miniooni: search/for/go maybe/copypsiphon + GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +# When building for Linux we use `-tags netgo` and `-extldflags -static` to produce +# a statically linked binary that completely bypasses libc. +#help: +#help: * `./mk ./CLI/linux/386/miniooni`: linux/386 +.PHONY: ./CLI/linux/386/miniooni +./CLI/linux/386/miniooni: search/for/go maybe/copypsiphon + GOOS=linux GOARCH=386 CGO_ENABLED=0 go build -tags="netgo,$(OONI_PSIPHON_TAGS)" -ldflags="-s -w -extldflags -static" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/linux/amd64/miniooni`: linux/amd64 +.PHONY: ./CLI/linux/amd64/miniooni +./CLI/linux/amd64/miniooni: search/for/go maybe/copypsiphon + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags="netgo,$(OONI_PSIPHON_TAGS)" -ldflags="-s -w -extldflags -static" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +# When building for GOARCH=arm, we always force GOARM=7 (i.e., armhf/armv7). +#help: +#help: * `./mk ./CLI/linux/arm/miniooni`: linux/arm +.PHONY: ./CLI/linux/arm/miniooni +./CLI/linux/arm/miniooni: search/for/go maybe/copypsiphon + GOOS=linux GOARCH=arm CGO_ENABLED=0 GOARM=7 go build -tags="netgo,$(OONI_PSIPHON_TAGS)" -ldflags="-s -w -extldflags -static" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/linux/arm64/miniooni`: linux/arm64 +.PHONY: ./CLI/linux/arm64/miniooni +./CLI/linux/arm64/miniooni: search/for/go maybe/copypsiphon + GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -tags="netgo,$(OONI_PSIPHON_TAGS)" -ldflags="-s -w -extldflags -static" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/windows/386/miniooni.exe`: windows/386 +.PHONY: ./CLI/windows/386/miniooni.exe +./CLI/windows/386/miniooni.exe: search/for/go maybe/copypsiphon + GOOS=windows GOARCH=386 CGO_ENABLED=0 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/windows/amd64/miniooni.exe`: windows/amd64 +.PHONY: ./CLI/windows/amd64/miniooni.exe +./CLI/windows/amd64/miniooni.exe: search/for/go maybe/copypsiphon + GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: The `./mk ./CLI/ooniprobe/darwin` command builds the ooniprobe official +#help: command line client for darwin/amd64 and darwin/arm64. This process +#help: entails building ooniprobe and then GPG-signing the binaries. +#help: +#help: You can also build the following subtargets: +.PHONY: ./CLI/ooniprobe/darwin +./CLI/ooniprobe/darwin: \ + ./CLI/darwin/amd64/ooniprobe.asc \ + ./CLI/darwin/arm64/ooniprobe.asc + +# ./CLI/darwin/amd64/ooniprobe.asc is an internal target for signing +.PHONY: ./CLI/darwin/amd64/ooniprobe.asc +./CLI/darwin/amd64/ooniprobe.asc: ./CLI/darwin/amd64/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +# We force CGO_ENABLED=1 because in principle we may be cross compiling. In +# reality it's hard to see a macOS/darwin build not made on macOS. +#help: +#help: * `./mk ./CLI/darwin/amd64/ooniprobe`: darwin/amd64 +.PHONY: ./CLI/darwin/amd64/ooniprobe +./CLI/darwin/amd64/ooniprobe: search/for/go maybe/copypsiphon + GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./cmd/ooniprobe + +# ./CLI/darwin/arm64/ooniprobe.asc is an internal target for signing +.PHONY: ./CLI/darwin/arm64/ooniprobe.asc +./CLI/darwin/arm64/ooniprobe.asc: ./CLI/darwin/arm64/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +#help: +#help: * `./mk ./CLI/darwin/arm64/ooniprobe`: darwin/arm64 +.PHONY: ./CLI/darwin/arm64/ooniprobe +./CLI/darwin/arm64/ooniprobe: search/for/go maybe/copypsiphon + GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./cmd/ooniprobe + +#help: +#help: The `./mk ./debian` command builds the ooniprobe CLI +#help: debian package for amd64 and arm64. +#help: +#help: You can also build the following subtargets: +.PHONY: ./debian +./debian: \ + ./debian/amd64 \ + ./debian/arm64 + +#help: +#help: * `./mk ./debian/amd64`: debian/amd64 +.PHONY: ./debian/amd64 +# This extra .PHONY for linux/amd64 is to help printing targets 🤷. +.PHONY: ./CLI/linux/amd64/ooniprobe +./debian/amd64: search/for/docker ./CLI/linux/amd64/ooniprobe + docker pull --platform linux/amd64 debian:stable + docker run --platform linux/amd64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian "$(DEBIAN_TILDE_VERSION)" + +#help: +#help: * `./mk ./debian/arm64`: debian/arm64 +.PHONY: ./debian/arm64 +# This extra .PHONY for linux/arm64 is to help printing targets 🤷. +.PHONY: ./CLI/linux/arm64/ooniprobe +./debian/arm64: search/for/docker ./CLI/linux/arm64/ooniprobe + docker pull --platform linux/arm64 debian:stable + docker run --platform linux/arm64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian "$(DEBIAN_TILDE_VERSION)" + +#help: +#help: The `./mk ./CLI/ooniprobe/linux` command builds the ooniprobe official command +#help: line client for amd64 and arm64. This entails building and GPG signing. +#help: +#help: You can also build the following subtargets: +.PHONY: ./CLI/ooniprobe/linux +./CLI/ooniprobe/linux: \ + ./CLI/linux/amd64/ooniprobe.asc \ + ./CLI/linux/arm64/ooniprobe.asc + +# ./CLI/linux/amd64/ooniprobe.asc is an internal task for signing. +.PHONY: ./CLI/linux/amd64/ooniprobe.asc +./CLI/linux/amd64/ooniprobe.asc: ./CLI/linux/amd64/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +# Linux builds use Alpine and Docker so we are sure that we are statically +# linking to musl libc, thus making our binaries extremely portable. +#help: +#help: * `./mk ./CLI/linux/amd64/ooniprobe`: linux/amd64 +.PHONY: ./CLI/linux/amd64/ooniprobe +./CLI/linux/amd64/ooniprobe: search/for/docker maybe/copypsiphon + docker pull --platform linux/amd64 $(GOLANG_DOCKER_IMAGE) + docker run --platform linux/amd64 -e GOPATH=/gopath -e GOARCH=amd64 -v $(GOLANG_DOCKER_GOCACHE)/amd64:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS) + +# ./CLI/linux/arm64/ooniprobe.asc is an internal task for signing. +.PHONY: ./CLI/linux/arm64/ooniprobe.asc +./CLI/linux/arm64/ooniprobe.asc: ./CLI/linux/arm64/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +#help: +#help: * `./mk ./CLI/linux/arm64/ooniprobe`: linux/arm64 +.PHONY: ./CLI/linux/arm64/ooniprobe +./CLI/linux/arm64/ooniprobe: search/for/docker maybe/copypsiphon + docker pull --platform linux/arm64 $(GOLANG_DOCKER_IMAGE) + docker run --platform linux/arm64 -e GOPATH=/gopath -e GOARCH=arm64 -v $(GOLANG_DOCKER_GOCACHE)/arm64:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS) + +#help: +#help: The `./mk ./CLI/ooniprobe/windows` command builds the ooniprobe official +#help: command line client for windows/386 and windows/amd64. This entails +#help: building and PGP signing the executables. +#help: +#help: You can also build the following subtargets: +.PHONY: ./CLI/ooniprobe/windows +./CLI/ooniprobe/windows: \ + ./CLI/windows/386/ooniprobe.exe.asc \ + ./CLI/windows/amd64/ooniprobe.exe.asc + +# ./CLI/windows/386/ooniprobe.exe.asc is an internal signing target +.PHONY: ./CLI/windows/386/ooniprobe.exe.asc +./CLI/windows/386/ooniprobe.exe.asc: ./CLI/windows/386/ooniprobe.exe + rm -f $@ && gpg -abu $(GPG_USER) $< + +#help: +#help: * `./mk ./CLI/windows/386/ooniprobe.exe`: windows/386 +.PHONY: ./CLI/windows/386/ooniprobe.exe +./CLI/windows/386/ooniprobe.exe: search/for/go search/for/mingw-w64 maybe/copypsiphon + GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./cmd/ooniprobe + +# ./CLI/windows/amd64/ooniprobe.exe.asc is an internal signing target +.PHONY: ./CLI/windows/amd64/ooniprobe.exe.asc +./CLI/windows/amd64/ooniprobe.exe.asc: ./CLI/windows/amd64/ooniprobe.exe + rm -f $@ && gpg -abu $(GPG_USER) $< + +#help: +#help: * `./mk ./CLI/windows/amd64/ooniprobe.exe`: windows/amd64 +.PHONY: ./CLI/windows/amd64/ooniprobe.exe +./CLI/windows/amd64/ooniprobe.exe: search/for/go search/for/mingw-w64 maybe/copypsiphon + GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./cmd/ooniprobe + +#help: +#help: The `./mk ./MOBILE/android` command builds the oonimkall library for Android. +#help: +#help: You can also build the following subtargets: +.PHONY: ./MOBILE/android +./MOBILE/android: search/for/gpg search/for/jar ./MOBILE/android/oonimkall.aar + cp ./MOBILE/android/oonimkall.aar ./MOBILE/android/oonimkall-$(OONIMKALL_V).aar + cp ./MOBILE/android/oonimkall-sources.jar ./MOBILE/android/oonimkall-$(OONIMKALL_V)-sources.jar + cat ./MOBILE/template.pom | sed -e "s/@VERSION@/$(OONIMKALL_V)/g" > ./MOBILE/android/oonimkall-$(OONIMKALL_V).pom + gpg -abu $(GPG_USER) ./MOBILE/android/oonimkall-$(OONIMKALL_V).aar + gpg -abu $(GPG_USER) ./MOBILE/android/oonimkall-$(OONIMKALL_V)-sources.jar + gpg -abu $(GPG_USER) ./MOBILE/android/oonimkall-$(OONIMKALL_V).pom + cd ./MOBILE/android && jar -cf bundle.jar oonimkall-$(OONIMKALL_V).aar oonimkall-$(OONIMKALL_V).aar.asc oonimkall-$(OONIMKALL_V)-sources.jar oonimkall-$(OONIMKALL_V)-sources.jar.asc oonimkall-$(OONIMKALL_V).pom oonimkall-$(OONIMKALL_V).pom.asc + +#help: +#help: * `./mk ./MOBILE/android/oonimkall.aar`: the AAR +.PHONY: ./MOBILE/android/oonimkall.aar +./MOBILE/android/oonimkall.aar: android/sdk ooni/go maybe/copypsiphon + PATH=$(OONIGODIR)/bin:$$PATH $(MAKE) -f mk __android_build_with_ooni_go + +# GOMOBILE is the full path location to the gomobile binary. We want to +# execute this command every time, because its output may depend on context, +# for this reason WE ARE NOT using `:=`. +GOMOBILE = $(shell go env GOPATH)/bin/gomobile + +# Here we use ooni/go to work around https://github.com/ooni/probe/issues/1444 +__android_build_with_ooni_go: search/for/go + go get -u golang.org/x/mobile/cmd/gomobile + $(GOMOBILE) init + PATH=$(shell go env GOPATH)/bin:$$PATH ANDROID_HOME=$(OONI_ANDROID_HOME) ANDROID_NDK_HOME=$(OONI_ANDROID_HOME)/ndk/$(ANDROID_NDK_VERSION) $(GOMOBILE) bind -target android -o ./MOBILE/android/oonimkall.aar -tags="$(OONI_PSIPHON_TAGS)" -ldflags '-s -w' $(GOLANG_EXTRA_FLAGS) ./pkg/oonimkall + +#help: +#help: The `./mk ./MOBILE/ios` command builds the oonimkall library for iOS. +#help: +#help: You can also build the following subtargets: +.PHONY: ./MOBILE/ios +./MOBILE/ios: \ + ./MOBILE/ios/oonimkall.framework.zip \ + ./MOBILE/ios/oonimkall.podspec + +#help: +#help: * `./mk ./MOBILE/ios/oonimkall.framework.zip`: zip the framework +.PHONY: ./MOBILE/ios/oonimkall.framework.zip +./MOBILE/ios/oonimkall.framework.zip: search/for/zip ./MOBILE/ios/oonimkall.framework + cd ./MOBILE/ios && rm -rf oonimkall.framework.zip + cd ./MOBILE/ios && zip -yr oonimkall.framework.zip oonimkall.framework + +#help: +#help: * `./mk ./MOBILE/ios/framework`: the framework +.PHONY: ./MOBILE/ios/oonimkall.framework +./MOBILE/ios/oonimkall.framework: search/for/go search/for/xcode + go get -u golang.org/x/mobile/cmd/gomobile + $(GOMOBILE) init + PATH=$(shell go env GOPATH)/bin:$$PATH $(GOMOBILE) bind -target ios -o $@ -tags="$(OONI_PSIPHON_TAGS)" -ldflags '-s -w' $(GOLANG_EXTRA_FLAGS) ./pkg/oonimkall + +#help: +#help: * `./mk ./MOBILE/ios/oonimkall.podspec`: the podspec +.PHONY: ./MOBILE/ios/oonimkall.podspec +./MOBILE/ios/oonimkall.podspec: ./MOBILE/template.podspec + cat $< | sed -e "s/@VERSION@/$(OONIMKALL_V)/g" -e "s/@RELEASE@/$(OONIMKALL_R)/g" > $@ + +# important: OONIMKALL_V and OONIMKALL_R MUST be expanded just once so we use `:=` +OONIMKALL_V := $(shell date -u +%Y.%m.%d-%H%M%S) +OONIMKALL_R := $(shell git describe --tags || echo '0.0.0-dev') + +#help: +#help: The following commands check for the availability of dependencies: +# TODO(bassosimone): make checks more robust? + +#help: +#help: * `./mk search/for/bash`: checks for bash +.PHONY: search/for/bash +search/for/bash: + @printf "checking for bash... " + @command -v bash || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/curl`: checks for curl +.PHONY: search/for/curl +search/for/curl: + @printf "checking for curl... " + @command -v curl || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/docker`: checks for docker +.PHONY: search/for/docker +search/for/docker: + @printf "checking for docker... " + @command -v docker || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/git`: checks for git +.PHONY: search/for/git +search/for/git: + @printf "checking for git... " + @command -v git || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/gpg`: checks for gpg +.PHONY: search/for/gpg +search/for/gpg: + @printf "checking for gpg... " + @command -v gpg || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/jar`: checks for jar +.PHONY: search/for/jar +search/for/jar: + @printf "checking for jar... " + @command -v jar || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/java`: checks for java +.PHONY: search/for/java +search/for/java: + @printf "checking for java... " + @command -v java || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/go`: checks for go +.PHONY: search/for/go +search/for/go: + @printf "checking for go... " + @command -v go || { echo "not found"; exit 1; } + @printf "checking for go version... " + @echo $(__GOVERSION_REAL) + @[ "$(GOLANG_VERSION_STRING)" = "$(__GOVERSION_REAL)" ] || { echo "fatal: go version must be $(GOLANG_VERSION_STRING) instead of $(__GOVERSION_REAL)"; exit 1; } + +# __GOVERSION_REAL is the go version reported by the go binary (we +# SHOULD NOT cache this value so we ARE NOT using `:=`) +__GOVERSION_REAL = $(shell go version | awk '{print $$3}') + +#help: +#help: * `./mk search/for/mingw-w64`: checks for mingw-w64 +.PHONY: search/for/mingw-w64 +search/for/mingw-w64: + @printf "checking for x86_64-w64-mingw32-gcc... " + @command -v x86_64-w64-mingw32-gcc || { echo "not found"; exit 1; } + @printf "checking for x86_64-w64-mingw32-gcc version... " + @echo $(__MINGW32_AMD64_VERSION) + @[ "$(MINGW_W64_VERSION)" = "$(__MINGW32_AMD64_VERSION)" ] || { echo "fatal: x86_64-w64-mingw32-gcc version must be $(MINGW_W64_VERSION) instead of $(__MINGW32_AMD64_VERSION)"; exit 1; } + @printf "checking for i686-w64-mingw32-gcc... " + @command -v i686-w64-mingw32-gcc || { echo "not found"; exit 1; } + @printf "checking for i686-w64-mingw32-gcc version... " + @echo $(__MINGW32_386_VERSION) + @[ "$(MINGW_W64_VERSION)" = "$(__MINGW32_386_VERSION)" ] || { echo "fatal: i686-w64-mingw32-gcc version must be $(MINGW_W64_VERSION) instead of $(__MINGW32_386_VERSION)"; exit 1; } + +# __MINGW32_AMD64_VERSION and __MINGW32_386_VERSION are the versions +# reported by the amd64 and 386 mingw binaries. +__MINGW32_AMD64_VERSION = $(shell x86_64-w64-mingw32-gcc --version | sed -n 1p | awk '{print $$3}') +__MINGW32_386_VERSION = $(shell i686-w64-mingw32-gcc --version | sed -n 1p | awk '{print $$3}') + +#help: +#help: * `./mk search/for/shasum`: checks for shasum +.PHONY: search/for/shasum +search/for/shasum: + @printf "checking for shasum... " + @command -v shasum || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/xcode`: checks for Xcode +.PHONY: search/for/xcode +search/for/xcode: + @printf "checking for xcodebuild... " + @command -v xcodebuild || { echo "not found"; exit 1; } + @printf "checking for Xcode version... " + @echo $(__XCODEVERSION_REAL) + @[ "$(XCODE_VERSION)" = "$(__XCODEVERSION_REAL)" ] || { echo "fatal: Xcode version must be $(XCODE_VERSION) instead of $(__XCODEVERSION_REAL)"; exit 1; } + +# __XCODEVERSION_REAL is the version of Xcode obtained using xcodebuild +__XCODEVERSION_REAL = `xcodebuild -version | grep ^Xcode | awk '{print $$2}'` + +#help: +#help: * `./mk search/for/unzip`: checks for unzip +.PHONY: search/for/unzip +search/for/unzip: + @printf "checking for unzip... " + @command -v unzip || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/zip`: checks for zip +.PHONY: search/for/zip +search/for/zip: + @printf "checking for zip... " + @command -v zip || { echo "not found"; exit 1; } + +#help: +#help: The `./mk maybe/copypsiphon` command copies the private psiphon config +#help: file into the current tree unless `$(OONI_PSIPHON_TAGS)` is empty. +.PHONY: maybe/copypsiphon +maybe/copypsiphon: search/for/git + test -z "$(OONI_PSIPHON_TAGS)" || $(MAKE) -f mk $(OONIPRIVATE) + test -z "$(OONI_PSIPHON_TAGS)" || cp $(OONIPRIVATE)/psiphon-config.key ./internal/engine + test -z "$(OONI_PSIPHON_TAGS)" || cp $(OONIPRIVATE)/psiphon-config.json.age ./internal/engine + +# OONIPRIVATE is the directory where we clone the private repository. +OONIPRIVATE = $(GIT_CLONE_DIR)/github.com/ooni/probe-private + +# OONIPRIVATE_REPO is the private repository URL. +OONIPRIVATE_REPO = git@github.com:ooni/probe-private + +# $(OONIPRIVATE) clones the private repository in $(GIT_CLONE_DIR) +$(OONIPRIVATE): search/for/git $(GIT_CLONE_DIR) + test -d $(OONIPRIVATE) || $(MAKE) -f mk __really_clone_private_repo + +__really_clone_private_repo: + git clone $(OONIPRIVATE_REPO) $(OONIPRIVATE) + +#help: +#help: The `./mk ooni/go` command builds the latest version of ooni/go. +.PHONY: ooni/go +ooni/go: search/for/bash search/for/git search/for/go $(OONIGODIR) + test -d $(OONIGODIR) || git clone -b ooni --single-branch --depth 8 $(OONIGO_REPO) $(OONIGODIR) + cd $(OONIGODIR) && git pull --ff-only + cd $(OONIGODIR)/src && ./make.bash + +# OONIGODIR is the directory in which we clone ooni/go +OONIGODIR = $(GIT_CLONE_DIR)/github.com/ooni/go + +# OONIGO_REPO is the repository for ooni/go +OONIGO_REPO = https://github.com/ooni/go + +#help: +#help: The `./mk android/sdk` command ensures we are using the +#help: correct version of the Android sdk. +.PHONY: android/sdk +android/sdk: search/for/java + test -d $(OONI_ANDROID_HOME) || $(MAKE) -f mk android/sdk/download + echo "Yes" | $(__ANDROID_SDKMANAGER) --install $(ANDROID_INSTALL_EXTRA) 'ndk;$(ANDROID_NDK_VERSION)' + +# __ANDROID_SKDMANAGER is the path to android's sdkmanager tool +__ANDROID_SDKMANAGER = $(OONI_ANDROID_HOME)/cmdline-tools/$(ANDROID_CLI_VERSION)/bin/sdkmanager + +# See https://stackoverflow.com/a/61176718 to understand why +# we need to reorganize the directories like this: +#help: +#help: The `./mk android/sdk/download` unconditionally downloads the +#help: Android SDK at `$(OONI_ANDROID_HOME)`. +android/sdk/download: search/for/curl search/for/java search/for/shasum search/for/unzip + curl -fsSLO https://dl.google.com/android/repository/$(__ANDROID_CLITOOLS_FILE) + echo "$(ANDROID_CLI_SHA256) $(__ANDROID_CLITOOLS_FILE)" > __SHA256 + shasum --check __SHA256 + rm -f __SHA256 + unzip $(__ANDROID_CLITOOLS_FILE) + rm $(__ANDROID_CLITOOLS_FILE) + mkdir -p $(OONI_ANDROID_HOME)/cmdline-tools + mv cmdline-tools $(OONI_ANDROID_HOME)/cmdline-tools/$(ANDROID_CLI_VERSION) + +# __ANDROID_CLITOOLS_FILE is the file name of the android cli tools zip +__ANDROID_CLITOOLS_FILE = commandlinetools-linux-$(ANDROID_CLI_VERSION)_latest.zip