From b2209bb6370bba608169206ba424f66bb9dd1b91 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 11 May 2021 16:15:13 +0200 Subject: [PATCH] refactor: replace ./make (python3) with ./mk (makefile) (#343) This pull request fixes https://github.com/ooni/probe/issues/1471. We have replaced the original build script (`./make`) with the `./mk` makefile (executable using `#!/usr/bin/make -f`). We concluded supporting direct builds from Windows is not worth the effort and halving the code we need to maintain is probably a good plus. Both macOS and Linux install GNU make at `/usr/bin/make`, so we should be okay in the common use cases. I significantly simplified the management of Go versioning by requiring the user to manage it and by enforcing that we are using the desired Go version. This speeds up builds and works in sane operating systems that use the last version of a specific package. Otherwise, it's possible to use the `go get golang.org/dl/go${version}` feature. The remaining question mark was related to updating the Android SDK. I have determined that a good course of action is pinning to the latest CLI tools and always forcing the CLI tools to install the latest required packages (e.g., the NDK). --- .github/workflows/android.yml | 5 +- .github/workflows/ios.yml | 5 +- .github/workflows/linux.yml | 11 +- .github/workflows/macos.yml | 5 +- .github/workflows/miniooni.yml | 5 +- .github/workflows/windows.yml | 5 +- CLI/linux/build | 13 +- CLI/linux/debian | 48 +- Readme.md | 15 +- make | 1434 -------------------------------- mk | 602 ++++++++++++++ 11 files changed, 668 insertions(+), 1480 deletions(-) delete mode 100755 make create mode 100755 mk diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 899b966..5831424 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -8,5 +8,8 @@ jobs: test: runs-on: ubuntu-20.04 steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t ./MOBILE/android/oonimkall.aar + - run: ./mk OONI_PSIPHON_TAGS="" ./MOBILE/android/oonimkall.aar diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 06e4f13..c8f49a7 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -8,5 +8,8 @@ jobs: test: runs-on: macos-10.15 steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t ./MOBILE/ios/oonimkall.framework.zip + - run: ./mk OONI_PSIPHON_TAGS="" XCODE_VERSION=12.4 ./MOBILE/ios/oonimkall.framework.zip diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index adb21dc..97e9231 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -13,11 +13,11 @@ jobs: echo $'{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json sudo service docker restart - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t ./CLI/linux/amd64/ooniprobe + - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/linux/amd64/ooniprobe env: DOCKER_CLI_EXPERIMENTAL: enabled - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe - - run: ./make --disable-embedding-psiphon-config -t debian_amd64 + - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64 - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd - run: | for deb in *.deb; do @@ -35,12 +35,13 @@ jobs: echo $'{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json sudo service docker restart - uses: actions/checkout@v2 - - run: sudo apt-get install qemu-user-static - - run: ./make --disable-embedding-psiphon-config -t ./CLI/linux/arm64/ooniprobe + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static + - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/linux/arm64/ooniprobe env: DOCKER_CLI_EXPERIMENTAL: enabled - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe - - run: ./make --disable-embedding-psiphon-config -t debian_arm64 + - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64 - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd - run: | for deb in *.deb; do diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 85607ce..d32830e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -8,6 +8,9 @@ jobs: build: runs-on: "macos-10.15" steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t ./CLI/darwin/amd64/ooniprobe + - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/darwin/amd64/ooniprobe - run: ./smoketest.sh ./CLI/darwin/amd64/ooniprobe diff --git a/.github/workflows/miniooni.yml b/.github/workflows/miniooni.yml index 5089d22..4aa187c 100644 --- a/.github/workflows/miniooni.yml +++ b/.github/workflows/miniooni.yml @@ -10,8 +10,11 @@ jobs: test: runs-on: ubuntu-20.04 steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - - run: ./make --disable-embedding-psiphon-config -t miniooni + - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/miniooni - run: ./CLI/linux/amd64/miniooni --yes -nNi https://example.com web_connectivity diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 11fbb08..51055a4 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -8,9 +8,12 @@ jobs: build: runs-on: "ubuntu-20.04" steps: + - uses: actions/setup-go@v1 + with: + go-version: "1.16.4" - uses: actions/checkout@v2 - run: sudo apt install mingw-w64 - - run: ./make --disable-embedding-psiphon-config -t ./CLI/windows/amd64/ooniprobe.exe + - run: ./mk OONI_PSIPHON_TAGS="" MINGW_W64_VERSION="9.3-win32" ./CLI/windows/amd64/ooniprobe.exe - uses: actions/upload-artifact@v2 with: name: ooniprobe.exe diff --git a/CLI/linux/build b/CLI/linux/build index 6fa148e..6235a22 100755 --- a/CLI/linux/build +++ b/CLI/linux/build @@ -1,15 +1,18 @@ #!/bin/sh -# This script is executed by `./make` when building inside +# This script is executed by `./mk` when building inside # an Alpine Linux docker container. Using Alpine Linux, which # uses musl libc, allows us to emit static binaries. set -e if [ "$GOARCH" = "" ]; then - echo 'fatal: $GOARCH is not set' 1>&2 - exit 1 + echo 'fatal: GOARCH is not set' 1>&2 + exit 1 fi set -x apk update apk upgrade apk add --no-progress gcc git linux-headers musl-dev -CGO_ENABLED=1 GOOS=linux GOARCH=$GOARCH go build -o ./CLI/linux/$GOARCH/ \ - -ldflags='-s -w -extldflags "-static"' "$@" ./cmd/ooniprobe +export GOPATH=$GOPATH +export CGO_ENABLED=1 +export GOOS=linux +export GOARCH=$GOARCH +go build -o "./CLI/linux/$GOARCH/" -ldflags='-s -w -extldflags "-static"' "$@" ./cmd/ooniprobe diff --git a/CLI/linux/debian b/CLI/linux/debian index 4b11bec..78e1090 100755 --- a/CLI/linux/debian +++ b/CLI/linux/debian @@ -1,5 +1,5 @@ #!/bin/sh -# This script creates a Debian package. When run by `./make`, it +# This script creates a Debian package. When run by `./mk`, it # is run inside a debian:stable container. It's fine to also # run this script from any debian-like system, as long as the # following ASSUMPTIONS are met: @@ -11,8 +11,8 @@ # architecture of the `ooniprobe` we are packaging. if [ $# -gt 1 ]; then - echo "usage: $0 [run_number]" 1>&2 - exit 1 + echo "usage: $0 [run_number]" 1>&2 + exit 1 fi run_number=$1 @@ -20,32 +20,32 @@ run_number=$1 # by the debian/ooniprobe-cli.install file. rm -rf ./debian/bin mkdir -p ./debian/bin -machine=`uname -m` +machine=$(uname -m) goarch="" case $machine in - x86_64) - cp ./CLI/linux/amd64/ooniprobe ./debian/bin - goarch=amd64 - ;; - aarch64) - cp ./CLI/linux/arm64/ooniprobe ./debian/bin - goarch=arm64 - ;; - *) - # TODO(bassosimone): here we probably want to further extend - # this script to support at least armv7. - echo "FATAL: unsupported machine: $machine" 1>&2 - exit 1 - ;; +x86_64) + cp ./CLI/linux/amd64/ooniprobe ./debian/bin + goarch=amd64 + ;; +aarch64) + cp ./CLI/linux/arm64/ooniprobe ./debian/bin + goarch=arm64 + ;; +*) + # TODO(bassosimone): here we probably want to further extend + # this script to support at least armv7. + echo "FATAL: unsupported machine: $machine" 1>&2 + exit 1 + ;; esac set -ex # figure out the version number from the binary itself (which rests -# on the assumption that `uname -m` can run such a binary) -version=`./debian/bin/ooniprobe version` -if [ ! -z $run_number ]; then - version="${version}~${run_number}" +# on the assumption that we can run such a binary) +version=$(./debian/bin/ooniprobe version) +if [ -n "$run_number" ]; then + version="${version}~${run_number}" fi # The OONI_DEB_DRY_RUN is a semi-undocumented feature allowing @@ -58,7 +58,7 @@ $OONI_DEB_DRY_RUN apt-get build-dep -y --no-install-recommends . # keep the original changelog file safe $OONI_DEB_DRY_RUN cp ./debian/changelog ./debian/changelog.oocopy -$OONI_DEB_DRY_RUN dch -v $version "New version ${version}" +$OONI_DEB_DRY_RUN dch -v "$version" "New version ${version}" $OONI_DEB_DRY_RUN dpkg-buildpackage -us -uc -b # restore the original changelog file @@ -70,4 +70,4 @@ $OONI_DEB_DRY_RUN mv ../*.deb . # install the package on the container as a smoke test to # ensure that it is installable. -DEBIAN_FRONTEND=noninteractive dpkg -i ooniprobe-cli_${version}_${goarch}.deb +DEBIAN_FRONTEND=noninteractive dpkg -i "ooniprobe-cli_${version}_${goarch}.deb" diff --git a/Readme.md b/Readme.md index ea0cd09..757d636 100644 --- a/Readme.md +++ b/Readme.md @@ -34,13 +34,13 @@ This will generate a binary called `ooniprobe` in the current directory. ## Android bindings -Make sure you have Python 3.8+ installed, then run: +Make sure you have GNU make installed, then run: ```bash -./make -t android +./mk android ``` -Builds bindings for Android. (Add `----disable-embedding-psiphon-config` if you +Builds bindings for Android. (Add `OONI_PSIPHON_TAGS=""` if you cannot clone private repositories in the https://github.com/ooni namespace.) The generated bindings are (manually) pushed to the Maven Central package @@ -49,13 +49,13 @@ are published along with the release notes. ## iOS bindings -Make sure you have Python 3.8+ installed, then run: +Make sure you have GNU make installed, then run: ```bash -./make -t ios +./mk ios ``` -Builds bindings for iOS. (Add `----disable-embedding-psiphon-config` if you +Builds bindings for iOS. (Add `OONI_PSIPHON_TAGS=""` if you cannot clone private repositories in the https://github.com/ooni namespace.) The generated bindings are (manually) added to GitHub releases. The instructions @@ -83,4 +83,5 @@ Create an issue according to [the routine release template]( https://github.com/ooni/probe/blob/master/.github/ISSUE_TEMPLATE/routine-sprint-releases.md) and perform any item inside the check-list. -We build releases using `./make`, which requires Python3.8+. +We build releases using `./mk`, which requires GNU make. Try +the `./mk help|less` command for detailed usage. diff --git a/make b/make deleted file mode 100755 index a18c86a..0000000 --- a/make +++ /dev/null @@ -1,1434 +0,0 @@ -#!/usr/bin/env python3 - -""" Build script for ooniprobe. You can get documentation regarding -its usage by running `./make --help`. """ - -from __future__ import annotations -import datetime - -import getopt -import os -import platform -import shlex -import shutil -import subprocess -import sys - -from typing import Any -from typing import Dict -from typing import List -from typing import NoReturn -from typing import Optional -from typing import Protocol -from typing import Tuple - - -def android_cmdlinetools_os() -> str: - """android_cmdlinetools_os maps the name of the current OS to the - name used by the android command-line tools file.""" - system = platform.system() - if system == "Linux": - return "linux" - if system == "Darwin": - return "mac" - raise RuntimeError(system) - - -def android_cmdlinetools_version() -> str: - """android_cmdlinetools_version returns the version of the Android - command-line tools that we'd like to download.""" - return "6858069" - - -def android_cmdlinetools_sha256sum() -> str: - """android_cmdlinetools_sha256sum returns the SHA256 sum of the - Android command-line tools zip file.""" - return { - "linux": "87f6dcf41d4e642e37ba03cb2e387a542aa0bd73cb689a9e7152aad40a6e7a08", - "mac": "58a55d9c5bcacd7c42170d2cf2c9ae2889c6797a6128307aaf69100636f54a13", - }[android_cmdlinetools_os()] - - -def cachedir() -> str: - """cachedir returns the directory where we cache the SDKs.""" - return os.path.join(os.path.expandvars("${HOME}"), ".ooniprobe-build") - - -def goversion() -> str: - """goversion is the Go version we use.""" - return "1.16.3" - - -def gopath() -> str: - """gopath is the GOPATH we use.""" - return os.path.expandvars("${HOME}/go") - - -def gosha256sum() -> str: - """gosha256sum returns the SHA256 sum of the Go tarball.""" - return { - "linux": { - "amd64": "951a3c7c6ce4e56ad883f97d9db74d3d6d80d5fec77455c6ada6c1f7ac4776d2", - "arm64": "566b1d6f17d2bc4ad5f81486f0df44f3088c3ed47a3bec4099d8ed9939e90d5d", - }, - "darwin": { - "amd64": "6bb1cf421f8abc2a9a4e39140b7397cdae6aca3e8d36dcff39a1a77f4f1170ac", - "arm64": "f4e96bbcd5d2d1942f5b55d9e4ab19564da4fad192012f6d7b0b9b055ba4208f", - }, - }[goos()][goarch()] - - -def goos() -> str: - """goos returns the GOOS value for the current system.""" - system = platform.system() - if system == "Linux": - return "linux" - if system == "Darwin": - return "darwin" - raise RuntimeError(system) - - -def goarch() -> str: - """goarch returns the GOARCH value for the current system.""" - machine = platform.machine() - if machine in ("arm64", "arm", "386", "amd64"): - return machine - if machine in ("x86", "i386"): - return "386" - if machine == "x86_64": - return "amd64" - if machine == "aarch64": - return "arm64" - raise RuntimeError(machine) - - -def android_ndk_version() -> str: - """android_ndk_version returns the Android NDK version.""" - return "22.1.7171670" - - -def sdkmanager_install_cmd(binpath: str) -> List[str]: - """sdkmanager_install_cmd returns the command line for installing - all the required dependencies using the sdkmanager.""" - return [ - os.path.join(binpath, "sdkmanager"), - "--install", - "build-tools;29.0.3", - "platforms;android-30", - "ndk;{}".format(android_ndk_version()), - ] - - -def log(msg: str) -> None: - """log prints a message on the standard output.""" - print(msg, flush=True) - - -class Options(Protocol): - """Options contains the configured options.""" - - def debugging(self) -> bool: - """debugging indicates whether to pass -x to `go build...`.""" - - def disable_embedding_psiphon_config(self) -> bool: - """disable_embedding_psiphon_config indicates that the user - does not want us to embed an encrypted psiphon config file into - the binary.""" - - def dry_run(self) -> bool: - """dry_run indicates whether to execute commands.""" - - def target(self) -> str: - """target is the target to build.""" - - def verbose(self) -> bool: - """verbose indicates whether to pass -v to `go build...`.""" - - -class ConfigFromCLI: - """ConfigFromCLI parses options from CLI flags.""" - - @classmethod - def parse(cls, targets: List[str], top_targets: List[Target]) -> ConfigFromCLI: - """parse parses command line options and returns a - suitable configuration object.""" - conf = cls() - conf._parse(targets, top_targets) - return conf - - def __init__(self) -> None: - self._debugging = False - self._disable_embedding_psiphon_config = False - self._dry_run = False - self._target = "" - self._verbose = False - - def debugging(self) -> bool: - return self._debugging - - def disable_embedding_psiphon_config(self) -> bool: - return self._disable_embedding_psiphon_config - - def dry_run(self) -> bool: - return self._dry_run - - def target(self) -> str: - return self._target - - def verbose(self) -> bool: - return self._verbose - - # The main reason why I am using getopt here is such that I am able - # to print a very clear and detailed usage string. (But the same - # could be obtained quite likely w/ argparse.) - - _usage_string = """\ -usage: ./make [--disable-embedding-psiphon-config] [-nvx] -t target - ./make -l - ./make [--help|-h] - -The first form of the command builds the `target` specified using the -`-t` command line flag. If the target has dependencies, this command will -build the dependent targets first. The `-n` flag enables a dry run where -the command only prints the commands it would run. The `-v` and `-x` flags -are passed directly to `go build ...` and `gomobile bind ...`. The -`--disable-embedding-psiphon-config` flag causes this command to disable -embedding a psiphon config file into the generated binary; you should -use this option when you cannot clone the private repository containing -the psiphon configuration file. - -The second form of the command lists all the available targets, showing -top-level targets and their recursive dependencies. - -The third form of the command prints this help screen. -""" - - @classmethod - def _usage(cls, err: str = "", exitcode: int = 0) -> NoReturn: - if err: - sys.stderr.write("error: {}\n".format(err)) - sys.stderr.write(cls._usage_string) - sys.exit(exitcode) - - def _parse(self, targets: List[str], top_targets: List[Target]): - try: - opts, args = getopt.getopt( - sys.argv[1:], "hlnt:vx", ["disable-embedding-psiphon-config", "help"] - ) - except getopt.GetoptError as err: - self._usage(err=err.msg, exitcode=1) - if args: - self._usage(err="unexpected number of positional arguments", exitcode=1) - for key, value in opts: - if key == "--disable-embedding-psiphon-config": - self._disable_embedding_psiphon_config = True - continue - if key in ("-h", "--help"): - self._usage() - if key == "-l": - self._list_targets(top_targets) - if key == "-n": - self._dry_run = True - continue - if key == "-t": - self._target = value - continue - if key == "-v": - self._verbose = True - continue - if key == "-x": - self._debugging = True - continue - raise RuntimeError(key, value) - - if self._target == "": # no arguments is equivalent to --help - self._usage() - - if self._target not in targets: - sys.stderr.write("unknown target: {}\n".format(self._target)) - sys.stderr.write("try `./make -l` to see the available targets.\n") - sys.exit(1) - - def _list_targets(self, top_targets: List[Target]) -> NoReturn: - for tgt in top_targets: - self._print_target(tgt, 0) - sys.exit(0) - - def _print_target(self, target: Target, indent: int) -> None: - sys.stdout.write( - "{}{}{}\n".format( - " " * indent, target.name(), ":" if target.deps() else "" - ) - ) - for dep in target.deps(): - self._print_target(dep, indent + 1) - if indent <= 0: - sys.stdout.write("\n") - - -class Engine(Protocol): - """Engine is an engine for building targets.""" - - def backticks(self, cmdline: List[str]) -> str: - """backticks executes the given command line and returns - the output emitted by the command to the caller.""" - - def cat_sed_redirect( - self, patterns: List[Tuple[str, str]], source: str, dest: str - ) -> None: - """cat_sed_redirect does - `cat $source|sed -e "s/$patterns[0][0]/$patterns[0][1]/g" ... > $dest`.""" - - def echo_to_file(self, content: str, filepath: str) -> None: - """echo_to_file writes the content string to the given file.""" - - def require(self, *executable: str) -> None: - """require fails if executable is not found in path.""" - - def run( - self, - cmdline: List[str], - cwd: Optional[str] = None, - inputbytes: Optional[bytes] = None, - ) -> None: - """run runs the specified command line.""" - - def setenv(self, key: str, value: str) -> Optional[str]: - """setenv sets an environment variable and returns the - previous value of such variable (or None).""" - - def unsetenv(self, key: str) -> None: - """unsetenv clears an environment variable.""" - - -class CommandExecutor: - """CommandExecutor executes commands.""" - - def __init__(self, dry_runner: Engine): - self._dry_runner = dry_runner - - def backticks(self, cmdline: List[str]) -> str: - """backticks implements Engine.backticks""" - # Nothing else to do, because backticks is fully - # implemented by CommandDryRunner. - return self._dry_runner.backticks(cmdline) - - def cat_sed_redirect( - self, patterns: List[Tuple[str, str]], source: str, dest: str - ) -> None: - """cat_sed_redirect implements Engine.cat_sed_redirect.""" - self._dry_runner.cat_sed_redirect(patterns, source, dest) - with open(source, "r") as sourcefp: - data = sourcefp.read() - for p, v in patterns: - data = data.replace(p, v) - with open(dest, "w") as destfp: - destfp.write(data) - - def echo_to_file(self, content: str, filepath: str) -> None: - """echo_to_file implements Engine.echo_to_file""" - self._dry_runner.echo_to_file(content, filepath) - with open(filepath, "w") as filep: - filep.write(content) - filep.write("\n") - - def require(self, *executable: str) -> None: - """require implements Engine.require.""" - for exc in executable: - self._dry_runner.require(exc) - fullpath = shutil.which(exc) - if not fullpath: - log("checking for {}... not found".format(exc)) - sys.exit(1) - log("checking for {}... {}".format(exc, fullpath)) - - def run( - self, - cmdline: List[str], - cwd: Optional[str] = None, - inputbytes: Optional[bytes] = None, - ) -> None: - """run implements Engine.run.""" - self._dry_runner.run(cmdline, cwd, inputbytes) - subprocess.run(cmdline, check=True, cwd=cwd, input=inputbytes) - - def setenv(self, key: str, value: str) -> Optional[str]: - """setenv implements Engine.setenv.""" - # Nothing else to do, because setenv is fully - # implemented by CommandDryRunner. - return self._dry_runner.setenv(key, value) - - def unsetenv(self, key: str) -> None: - """unsetenv implements Engine.unsetenv.""" - # Nothing else to do, because unsetenv is fully - # implemented by CommandDryRunner. - self._dry_runner.unsetenv(key) - - -class CommandDryRunner: - """CommandDryRunner is the dry runner.""" - - # Implementation note: here we try to log valid bash snippets - # such that is really obvious what we are doing. - - def backticks(self, cmdline: List[str]) -> str: - """backticks implements Engine.backticks""" - # The backticks command is used to gather information used by - # other commands. As such, it needs to always run. If it was not - # running, we could not correctly implement the `-n` flag. It's - # also a silent command, because it's not really part of the - # sequence of bash commands that are executed. ¯\_(ツ)_/¯ - popen = subprocess.Popen(cmdline, stdout=subprocess.PIPE) - stdout = popen.communicate()[0] - if popen.returncode != 0: - raise RuntimeError(popen.returncode) - return stdout.decode("utf-8").strip() - - def cat_sed_redirect( - self, patterns: List[Tuple[str, str]], source: str, dest: str - ) -> None: - """cat_sed_redirect implements Engine.cat_sed_redirect.""" - out = "./make: cat {}|sed".format(source) - for p, v in patterns: - out += " -e 's/{}/{}/g'".format(p, v) - out += " > {}".format(dest) - log(out) - - def echo_to_file(self, content: str, filepath: str) -> None: - """echo_to_file implements Engine.echo_to_file""" - log("./make: echo '{}' > {}".format(content, filepath)) - - def require(self, *executable: str) -> None: - """require implements Engine.require.""" - for exc in executable: - log(f"./make: echo -n 'checking for {exc}... '") - log("./make: command -v %s || { echo 'not found'; exit 1 }" % exc) - - def run( - self, - cmdline: List[str], - cwd: Optional[str] = None, - inputbytes: Optional[bytes] = None, - ) -> None: - """run implements Engine.run.""" - cdpart = "" - if cwd: - cdpart = "cd {} && ".format(cwd) - log("./make: {}{}".format(cdpart, shlex.join(cmdline))) - - def setenv(self, key: str, value: str) -> Optional[str]: - """setenv implements Engine.setenv.""" - log("./make: export {}={}".format(key, shlex.join([value]))) - prev = os.environ.get(key) - os.environ[key] = value - return prev - - def unsetenv(self, key: str) -> None: - """unsetenv implements Engine.unsetenv.""" - log("./make: unset {}".format(key)) - del os.environ[key] - - -def new_engine(options: Options) -> Engine: - """new_engine creates a new engine instance""" - out: Engine = CommandDryRunner() - if not options.dry_run(): - out = CommandExecutor(out) - return out - - -class Environ: - """Environ creates a context where specific environment - variables are set. They will be restored to their previous - value when we are leaving the context.""" - - def __init__(self, engine: Engine, key: str, value: str) -> None: - self._engine = engine - self._key = key - self._value = value - self._prev: Optional[str] = None - - def __enter__(self) -> None: - self._prev = self._engine.setenv(self._key, self._value) - - def __exit__(self, type: Any, value: Any, traceback: Any) -> bool: - if self._prev is None: - self._engine.unsetenv(self._key) - return False # propagate exc - self._engine.setenv(self._key, self._prev) - return False # propagate exc - - -class AugmentedPath(Environ): - """AugementedPath is an Environ that prepends the required - directory to the currently existing search path.""" - - def __init__(self, engine: Engine, directory: str): - value = os.pathsep.join([directory, os.environ["PATH"]]) - super().__init__(engine, "PATH", value) - - -class WorkingDir: - """WorkingDir is a context manager that enters into a given working - directory and returns to the previous directory when done.""" - - def __init__(self, dirpath: str) -> None: - self._dirpath = dirpath - self._prev: str = "" - - def __enter__(self) -> None: - self._prev = os.getcwd() - os.chdir(self._dirpath) - - def __exit__(self, type: Any, value: Any, traceback: Any) -> bool: - os.chdir(self._prev) - return False # propagate exc - - -class Target(Protocol): - """Target is a target to build.""" - - def name(self) -> str: - """name returns the target name.""" - - def build(self, engine: Engine, options: Options) -> None: - """build builds the specified target.""" - - def deps(self) -> List[Target]: - """deps returns the target dependencies.""" - - -class BaseTarget: - """BaseTarget is the base class of all targets.""" - - def __init__(self, name: str, deps: List[Target]) -> None: - # prevent child classes from easily using these variables - self.__name = name - self.__deps = deps - - def name(self) -> str: - """name implements Target.name""" - return self.__name - - def build_child_targets(self, engine: Engine, options: Options) -> None: - """build_child_targets builds all the child targets""" - for dep in self.__deps: - dep.build(engine, options) - - def deps(self) -> List[Target]: - """deps implements Target.deps.""" - return self.__deps - - -class SDKGolangGo(BaseTarget): - """SDKGolangGo creates ${cachedir}/SDK/golang.""" - - # We download a golang SDK from upstream to make sure we - # are always using a specific version of golang/go. - - def __init__(self) -> None: - name = os.path.join(cachedir(), "SDK", "golang") - super().__init__(name, []) - - def binpath(self) -> str: - """binpath returns the path where the go binary is installed.""" - return os.path.join(self.name(), "go", "bin") - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isdir(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.require("mkdir", "curl", "shasum", "rm", "tar", "echo") - filename = "go{}.{}-{}.tar.gz".format(goversion(), goos(), goarch()) - url = "https://golang.org/dl/{}".format(filename) - engine.run(["mkdir", "-p", self.name()]) - filepath = os.path.join(self.name(), filename) - engine.run(["curl", "-fsSLo", filepath, url]) - sha256file = os.path.join(cachedir(), "SDK", "SHA256") - engine.echo_to_file("{} {}".format(gosha256sum(), filepath), sha256file) - engine.run(["shasum", "--check", sha256file]) - engine.run(["rm", sha256file]) - engine.run(["tar", "-xf", filename], cwd=self.name()) - engine.run(["rm", filepath]) - - def goroot(self): - """goroot returns the goroot.""" - return os.path.join(self.name(), "go") - - -class SDKOONIGo(BaseTarget): - """SDKOONIGo creates ${cachedir}/SDK/oonigo.""" - - # We use a private fork of golang/go on Android as a - # workaround for https://github.com/ooni/probe/issues/1444 - - def __init__(self) -> None: - name = os.path.join(cachedir(), "SDK", "oonigo") - self._gogo = SDKGolangGo() - super().__init__(name, [self._gogo]) - - def binpath(self) -> str: - """binpath returns the path where the go binary is installed.""" - return os.path.join(self.name(), "bin") - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isdir(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.require("git", "bash") - engine.run( - [ - "git", - "clone", - "-b", - "ooni", - "--single-branch", - "--depth", - "8", - "https://github.com/ooni/go", - self.name(), - ] - ) - with Environ(engine, "GOROOT_BOOTSTRAP", self._gogo.goroot()): - engine.run( - ["./make.bash"], - cwd=os.path.join(self.name(), "src"), - ) - - -class SDKAndroid(BaseTarget): - """SDKAndroid creates ${cachedir}/SDK/android.""" - - def __init__(self) -> None: - name = os.path.join(cachedir(), "SDK", "android") - super().__init__(name, []) - - def home(self) -> str: - """home returns the ANDROID_HOME""" - return self.name() - - def ndk_home(self) -> str: - """ndk_home returns the ANDROID_NDK_HOME""" - return os.path.join(self.home(), "ndk", android_ndk_version()) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isdir(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - log("\n./make: building {}...".format(self.name())) - engine.require("mkdir", "curl", "echo", "shasum", "rm", "unzip", "mv", "java") - filename = "commandlinetools-{}-{}_latest.zip".format( - android_cmdlinetools_os(), android_cmdlinetools_version() - ) - url = "https://dl.google.com/android/repository/{}".format(filename) - engine.run(["mkdir", "-p", self.name()]) - filepath = os.path.join(self.name(), filename) - engine.run(["curl", "-fsSLo", filepath, url]) - sha256file = os.path.join(cachedir(), "SDK", "SHA256") - engine.echo_to_file( - "{} {}".format(android_cmdlinetools_sha256sum(), filepath), sha256file - ) - engine.run(["shasum", "--check", sha256file]) - engine.run(["rm", sha256file]) - engine.run(["unzip", filename], cwd=self.name()) - engine.run(["rm", filepath]) - # See https://stackoverflow.com/a/61176718 to understand why - # we need to reorganize the directories like this: - engine.run( - ["mv", "cmdline-tools", android_cmdlinetools_version()], cwd=self.name() - ) - engine.run(["mkdir", "cmdline-tools"], cwd=self.name()) - engine.run( - ["mv", android_cmdlinetools_version(), "cmdline-tools"], cwd=self.name() - ) - engine.run( - sdkmanager_install_cmd( - os.path.join( - self.name(), - "cmdline-tools", - android_cmdlinetools_version(), - "bin", - ), - ), - inputbytes=b"Y\n", # automatically accept license - ) - - -class OONIProbePrivate(BaseTarget): - """OONIProbePrivate creates ${cachedir}/github.com/ooni/probe-private.""" - - # We use this private repository to copy the psiphon configuration - # file to embed into the ooniprobe binaries - - def __init__(self) -> None: - name = os.path.join(cachedir(), "github.com", "ooni", "probe-private") - super().__init__(name, []) - - def copyfiles(self, engine: Engine, options: Options) -> None: - """copyfiles copies psiphon config to the repository.""" - if options.disable_embedding_psiphon_config(): - log("./make: copy psiphon config: disabled by command line flags") - return - engine.run( - [ - "cp", - os.path.join(self.name(), "psiphon-config.json.age"), - os.path.join("internal", "engine"), - ] - ) - engine.run( - [ - "cp", - os.path.join(self.name(), "psiphon-config.key"), - os.path.join("internal", "engine"), - ] - ) - - def build(self, engine: Engine, options: Options) -> None: - if os.path.isdir(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - if options.disable_embedding_psiphon_config(): - log("\n./make: {}: disabled by command line flags".format(self.name())) - return - log("\n./make: building {}...".format(self.name())) - engine.require("git", "cp") - engine.run( - [ - "git", - "clone", - "git@github.com:ooni/probe-private", - self.name(), - ] - ) - - -class OONIMKAllAAR(BaseTarget): - """OONIMKAllAAR creates ./MOBILE/android/oonimkall.aar.""" - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "android", "oonimkall.aar") - self._ooprivate = OONIProbePrivate() - self._oonigo = SDKOONIGo() - self._android = SDKAndroid() - super().__init__(name, [self._ooprivate, self._oonigo, self._android]) - - def aarfile(self) -> str: - """aarfile returns the aar file path""" - return self.name() - - def srcfile(self) -> str: - """srcfile returns the path to the jar file containing sources.""" - return os.path.join(".", "MOBILE", "android", "oonimkall-sources.jar") - - def build(self, engine: Engine, options: Options) -> None: - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - engine.require("sh", "javac") - self._go_get_gomobile(engine, options, self._oonigo) - self._gomobile_init(engine, self._oonigo, self._android) - self._gomobile_bind(engine, options, self._oonigo, self._android) - - # Implementation note: we use proxy scripts for go and gomobile - # that explicitly print what they resolve go and gomobile to using - # `command -v`. This gives us extra confidence that we are really - # using the oonigo fork of golang/go. - - def _go_get_gomobile( - self, engine: Engine, options: Options, oonigo: SDKOONIGo - ) -> None: - # TODO(bassosimone): find a way to run this command without - # adding extra dependencies to go.mod and go.sum. - cmdline: List[str] = [] - cmdline.append("go") - cmdline.append("get") - cmdline.append("-u") - if options.verbose(): - cmdline.append("-v") - if options.debugging(): - cmdline.append("-x") - cmdline.append("golang.org/x/mobile/cmd/gomobile@latest") - with Environ(engine, "GOPATH", gopath()): - with AugmentedPath(engine, oonigo.binpath()): - engine.require("go") - engine.run(cmdline) - - def _gomobile_init( - self, - engine: Engine, - oonigo: SDKOONIGo, - android: SDKAndroid, - ) -> None: - cmdline: List[str] = [] - cmdline.append("gomobile") - cmdline.append("init") - with Environ(engine, "ANDROID_HOME", android.home()): - with Environ(engine, "ANDROID_NDK_HOME", android.ndk_home()): - with AugmentedPath(engine, oonigo.binpath()): - with AugmentedPath(engine, os.path.join(gopath(), "bin")): - engine.require("gomobile", "go") - engine.run(cmdline) - - def _gomobile_bind( - self, - engine: Engine, - options: Options, - oonigo: SDKOONIGo, - android: SDKAndroid, - ) -> None: - cmdline: List[str] = [] - cmdline.append("gomobile") - cmdline.append("bind") - if options.verbose(): - cmdline.append("-v") - if options.debugging(): - cmdline.append("-x") - cmdline.append("-target") - cmdline.append("android") - cmdline.append("-o") - cmdline.append(self.name()) - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags") - cmdline.append("ooni_psiphon_config") - cmdline.append("-ldflags") - cmdline.append("-s -w") - cmdline.append("./pkg/oonimkall") - with Environ(engine, "ANDROID_HOME", android.home()): - with Environ(engine, "ANDROID_NDK_HOME", android.ndk_home()): - with AugmentedPath(engine, oonigo.binpath()): - with AugmentedPath(engine, os.path.join(gopath(), "bin")): - engine.require("gomobile", "go") - engine.run(cmdline) - - -def sign(engine: Engine, filepath: str) -> str: - """sign signs the given filepath using pgp and returns - the filepath of the signature file.""" - engine.require("gpg") - user = "simone@openobservatory.org" - engine.run(["gpg", "-abu", user, filepath]) - return filepath + ".asc" - - -class BundleJAR(BaseTarget): - """BundleJAR creates ./MOBILE/android/bundle.jar.""" - - # We upload the bundle.jar file to maven central to bless - # a new release of the OONI libraries for Android. - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "android", "bundle.jar") - self._oonimkall = OONIMKAllAAR() - super().__init__(name, [self._oonimkall]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.require("cp", "gpg", "jar") - version = datetime.datetime.now().strftime("%Y.%m.%d-%H%M%S") - engine.run( - [ - "cp", - self._oonimkall.aarfile(), - os.path.join("MOBILE", "android", "oonimkall-{}.aar".format(version)), - ] - ) - engine.run( - [ - "cp", - self._oonimkall.srcfile(), - os.path.join( - "MOBILE", "android", "oonimkall-{}-sources.jar".format(version) - ), - ] - ) - engine.cat_sed_redirect( - [ - ("@VERSION@", version), - ], - os.path.join("MOBILE", "template.pom"), - os.path.join("MOBILE", "android", "oonimkall-{}.pom".format(version)), - ) - names = ( - "oonimkall-{}.aar".format(version), - "oonimkall-{}-sources.jar".format(version), - "oonimkall-{}.pom".format(version), - ) - allnames: List[str] = [] - with WorkingDir(os.path.join(".", "MOBILE", "android")): - for name in names: - allnames.append(name) - allnames.append(sign(engine, name)) - engine.run( - [ - "jar", - "-cf", - "bundle.jar", - *allnames, - ], - cwd=os.path.join(".", "MOBILE", "android"), - ) - - -class Phony(BaseTarget): - """Phony is a phony target that executes one or more other targets.""" - - def __init__(self, name: str, depends: List[Target]): - super().__init__(name, depends) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - self.build_child_targets(engine, options) - - -# Android is the top-level "android" target -ANDROID = Phony("android", [BundleJAR()]) - - -class OONIMKAllFramework(BaseTarget): - """OONIMKAllFramework creates ./MOBILE/ios/oonimkall.framework.""" - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "ios", "oonimkall.framework") - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build.""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - self._go_get_gomobile(engine, options, self._gogo) - self._gomobile_init(engine, self._gogo) - self._gomobile_bind(engine, options, self._gogo) - - def _go_get_gomobile( - self, - engine: Engine, - options: Options, - gogo: SDKGolangGo, - ) -> None: - # TODO(bassosimone): find a way to run this command without - # adding extra dependencies to go.mod and go.sum. - cmdline: List[str] = [] - cmdline.append("go") - cmdline.append("get") - cmdline.append("-u") - if options.verbose(): - cmdline.append("-v") - if options.debugging(): - cmdline.append("-x") - cmdline.append("golang.org/x/mobile/cmd/gomobile@latest") - with AugmentedPath(engine, gogo.binpath()): - with Environ(engine, "GOPATH", gopath()): - engine.require("go") - engine.run(cmdline) - - def _gomobile_init( - self, - engine: Engine, - gogo: SDKGolangGo, - ) -> None: - cmdline: List[str] = [] - cmdline.append("gomobile") - cmdline.append("init") - with AugmentedPath(engine, os.path.join(gopath(), "bin")): - with AugmentedPath(engine, gogo.binpath()): - engine.require("gomobile", "go") - engine.run(cmdline) - - def _gomobile_bind( - self, - engine: Engine, - options: Options, - gogo: SDKGolangGo, - ) -> None: - cmdline: List[str] = [] - cmdline.append("gomobile") - cmdline.append("bind") - if options.verbose(): - cmdline.append("-v") - if options.debugging(): - cmdline.append("-x") - cmdline.append("-target") - cmdline.append("ios") - cmdline.append("-o") - cmdline.append(self.name()) - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags") - cmdline.append("ooni_psiphon_config") - cmdline.append("-ldflags") - cmdline.append("-s -w") - cmdline.append("./pkg/oonimkall") - with AugmentedPath(engine, os.path.join(gopath(), "bin")): - with AugmentedPath(engine, gogo.binpath()): - engine.require("gomobile", "go") - engine.run(cmdline) - - -class OONIMKAllFrameworkZip(BaseTarget): - """OONIMKAllFrameworkZip creates ./MOBILE/ios/oonimkall.framework.zip.""" - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "ios", "oonimkall.framework.zip") - ooframework = OONIMKAllFramework() - super().__init__(name, [ooframework]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - engine.require("zip", "rm") - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.run( - [ - "rm", - "-rf", - "oonimkall.framework.zip", - ], - cwd=os.path.join(".", "MOBILE", "ios"), - ) - engine.run( - [ - "zip", - "-yr", - "oonimkall.framework.zip", - "oonimkall.framework", - ], - cwd=os.path.join(".", "MOBILE", "ios"), - ) - - -class OONIMKAllPodspec(BaseTarget): - """OONIMKAllPodspec creates ./MOBILE/ios/oonimkall.podspec.""" - - def __init__(self) -> None: - name = os.path.join(".", "MOBILE", "ios", "oonimkall.podspec") - super().__init__(name, []) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("./make: {}: already built".format(self.name())) - return - engine.require("cat", "sed") - release = engine.backticks(["git", "describe", "--tags"]) - version = datetime.datetime.now().strftime("%Y.%m.%d-%H%M%S") - engine.cat_sed_redirect( - [("@VERSION@", version), ("@RELEASE@", release)], - os.path.join(".", "MOBILE", "template.podspec"), - self.name(), - ) - - -# IOS is the top-level "ios" target. -IOS = Phony("ios", [OONIMKAllFrameworkZip(), OONIMKAllPodspec()]) - - -class MiniOONIDarwinOrWindows(BaseTarget): - def __init__(self, goos: str, goarch: str): - self._ext = ".exe" if goos == "windows" else "" - self._os = goos - self._arch = goarch - name = os.path.join(".", "CLI", goos, goarch, "miniooni" + self._ext) - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - cmdline = [ - "go", - "build", - "-o", - self.name(), - "-ldflags=-s -w", - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags=ooni_psiphon_config") - cmdline.append("./internal/cmd/miniooni") - with Environ(engine, "GOOS", self._os): - with Environ(engine, "GOARCH", self._arch): - with Environ(engine, "CGO_ENABLED", "0"): - with AugmentedPath(engine, self._gogo.binpath()): - engine.require("go") - engine.run(cmdline) - - -class MiniOONILinux(BaseTarget): - def __init__(self, goarch: str): - self._arch = goarch - name = os.path.join(".", "CLI", "linux", goarch, "miniooni") - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - if self._arch == "arm": - with Environ(engine, "GOARM", "7"): - self._build(engine, options, self._gogo) - else: - self._build(engine, options, self._gogo) - - def _build(self, engine: Engine, options: Options, gogo: SDKGolangGo) -> None: - cmdline = [ - "go", - "build", - "-o", - os.path.join("CLI", "linux", self._arch, "miniooni"), - "-ldflags=-s -w -extldflags -static", - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - tags = "-tags=netgo" - if not options.disable_embedding_psiphon_config(): - tags += ",ooni_psiphon_config" - cmdline.append(tags) - cmdline.append("./internal/cmd/miniooni") - with Environ(engine, "GOOS", "linux"): - with Environ(engine, "GOARCH", self._arch): - with Environ(engine, "CGO_ENABLED", "0"): - with AugmentedPath(engine, gogo.binpath()): - engine.require("go") - engine.run(cmdline) - - -# MINIOONI is the top-level "miniooni" target. -MINIOONI = Phony( - "miniooni", - [ - MiniOONIDarwinOrWindows("darwin", "amd64"), - MiniOONIDarwinOrWindows("darwin", "arm64"), - MiniOONILinux("386"), - MiniOONILinux("amd64"), - MiniOONILinux("arm"), - MiniOONILinux("arm64"), - MiniOONIDarwinOrWindows("windows", "386"), - MiniOONIDarwinOrWindows("windows", "amd64"), - ], -) - - -class OONIProbeLinux(BaseTarget): - """OONIProbeLinux builds ooniprobe for Linux.""" - - # TODO(bassosimone): this works out of the box on macOS and - # requires qemu-user-static on Fedora/Debian. I'm not sure what - # is the right (set of) command(s) I should be checking for. - - def __init__(self, goarch: str): - self._arch = goarch - name = os.path.join(".", "CLI", "linux", goarch, "ooniprobe") - self._ooprivate = OONIProbePrivate() - super().__init__(name, [self._ooprivate]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build.""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - engine.require("docker") - # make sure we have the latest version of the container image - engine.run( - [ - "docker", - "pull", - "--platform", - "linux/{}".format(self._arch), - "golang:{}-alpine".format(goversion()), - ] - ) - # then run the build inside the container - cmdline = [ - "docker", - "run", - "--platform", - "linux/{}".format(self._arch), - "-e", - "GOARCH={}".format(self._arch), - "-v", - "{}:/ooni".format(os.getcwd()), - "-w", - "/ooni", - "golang:{}-alpine".format(goversion()), - os.path.join(".", "CLI", "linux", "build"), - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags=ooni_psiphon_config,netgo") - else: - cmdline.append("-tags=netgo") - engine.run(cmdline) - - -class OONIProbeWindows(BaseTarget): - """OONIProbeWindows builds ooniprobe for Windows.""" - - def __init__(self, goarch: str): - self._arch = goarch - name = os.path.join(".", "CLI", "windows", goarch, "ooniprobe.exe") - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def _gcc(self) -> str: - if self._arch == "amd64": - return "x86_64-w64-mingw32-gcc" - if self._arch == "386": - return "i686-w64-mingw32-gcc" - raise NotImplementedError - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - cmdline = [ - "go", - "build", - "-o", - self.name(), - "-ldflags=-s -w", - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags=ooni_psiphon_config") - cmdline.append("./cmd/ooniprobe") - with Environ(engine, "GOOS", "windows"): - with Environ(engine, "GOARCH", self._arch): - with Environ(engine, "CGO_ENABLED", "1"): - with Environ(engine, "CC", self._gcc()): - with AugmentedPath(engine, self._gogo.binpath()): - engine.require(self._gcc(), "go") - engine.run(cmdline) - - -class OONIProbeDarwin(BaseTarget): - """OONIProbeDarwin builds ooniprobe for macOS.""" - - def __init__(self, goarch: str): - self._arch = goarch - name = os.path.join(".", "CLI", "darwin", goarch, "ooniprobe") - self._ooprivate = OONIProbePrivate() - self._gogo = SDKGolangGo() - super().__init__(name, [self._ooprivate, self._gogo]) - - def build(self, engine: Engine, options: Options) -> None: - """build implements Target.build""" - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - self._ooprivate.copyfiles(engine, options) - cmdline = [ - "go", - "build", - "-o", - self.name(), - "-ldflags=-s -w", - ] - if options.debugging(): - cmdline.append("-x") - if options.verbose(): - cmdline.append("-v") - if not options.disable_embedding_psiphon_config(): - cmdline.append("-tags=ooni_psiphon_config") - cmdline.append("./cmd/ooniprobe") - with Environ(engine, "GOOS", "darwin"): - with Environ(engine, "GOARCH", self._arch): - with Environ(engine, "CGO_ENABLED", "1"): - with AugmentedPath(engine, self._gogo.binpath()): - engine.require("gcc", "go") - engine.run(cmdline) - - -class Debian(BaseTarget): - """Debian makes a debian package for a given architecture.""" - - def __init__(self, arch: str): - name = "debian_{}".format(arch) - self._target = OONIProbeLinux(arch) - self._arch = arch - super().__init__(name, [self._target]) - - def build(self, engine: Engine, options: Options) -> None: - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - engine.require("docker") - # make sure we have the latest version of the container image - engine.run( - [ - "docker", - "pull", - "--platform", - "linux/{}".format(self._arch), - "debian:stable", - ] - ) - # then run the build inside the container - cmdline: List[str] = [] - cmdline.append("docker") - cmdline.append("run") - cmdline.append("--platform") - cmdline.append("linux/{}".format(self._arch)) - cmdline.append("-v") - cmdline.append("{}:/ooni".format(os.getcwd())) - cmdline.append("-w") - cmdline.append("/ooni") - cmdline.append("debian:stable") - cmdline.append(os.path.join(".", "CLI", "linux", "debian")) - if os.environ.get("GITHUB_ACTIONS", "") == "true": - # When we're running inside a github action, figure out whether - # we are building a tag or a commit. In the latter case, we will - # append the run number to the version number. - github_ref = os.environ.get("GITHUB_REF") - if not github_ref: - raise RuntimeError("missing GITHUB_REF") - github_run_number = os.environ.get("GITHUB_RUN_NUMBER") - if not github_run_number: - raise RuntimeError("missing GITHUB_RUN_NUMBER") - if not github_ref.startswith("/refs/tags/"): - cmdline.append(github_run_number) - engine.run(cmdline) - - -class Sign(BaseTarget): - """Sign signs a specific target artefact.""" - - def __init__(self, target: Target): - self._target = target - name = self._target.name() + ".asc" - super().__init__(name, [self._target]) - - def build(self, engine: Engine, options: Options) -> None: - if os.path.isfile(self.name()) and not options.dry_run(): - log("\n./make: {}: already built".format(self.name())) - return - self.build_child_targets(engine, options) - log("\n./make: building {}...".format(self.name())) - sign(engine, self._target.name()) - - -# OONIPROBE_RELEASE_DARWIN contains the release darwin targets -OONIPROBE_RELEASE_DARWIN = Phony( - "ooniprobe_release_darwin", - [ - Sign(OONIProbeDarwin("amd64")), - Sign(OONIProbeDarwin("arm64")), - ], -) - -# OONIPROBE_RELEASE_LINUX contains the release linux targets -OONIPROBE_RELEASE_LINUX = Phony( - "ooniprobe_release_linux", - [ - Sign(OONIProbeLinux("amd64")), - Sign(OONIProbeLinux("arm64")), - ], -) - -# OONIPROBE_RELEASE_WINDOWS contains the release windows targets -OONIPROBE_RELEASE_WINDOWS = Phony( - "ooniprobe_release_windows", - [ - Sign(OONIProbeWindows("amd64")), - Sign(OONIProbeWindows("386")), - ], -) - -# DEBIAN is the top-level "debian" target. -DEBIAN = Phony( - "debian", - [ - Debian("arm64"), - Debian("amd64"), - ], -) - -# TOP_TARGETS contains the top-level targets -TOP_TARGETS: List[Target] = [ - ANDROID, - IOS, - MINIOONI, - OONIPROBE_RELEASE_DARWIN, - OONIPROBE_RELEASE_LINUX, - OONIPROBE_RELEASE_WINDOWS, - DEBIAN, -] - - -def expand_targets(targets: List[Target]) -> Dict[str, Target]: - """expand_targets creates a dictionary mapping every existing - target name to its implementation.""" - out: Dict[str, Target] = {} - for tgt in targets: - out.update(expand_targets(tgt.deps())) - out[tgt.name()] = tgt - return out - - -def main() -> None: - """main function""" - alltargets = expand_targets(TOP_TARGETS) - options = ConfigFromCLI.parse(list(alltargets.keys()), TOP_TARGETS) - engine = new_engine(options) - # note that we check whether the target is known in parse() - selected = alltargets[options.target()] - selected.build(engine, options) - - -if __name__ == "__main__": - main() diff --git a/mk b/mk new file mode 100755 index 0000000..b8503b6 --- /dev/null +++ b/mk @@ -0,0 +1,602 @@ +#!/usr/bin/make -f + +#quickhelp: Usage: ./mk [VARIABLE=VALUE ...] TARGET ... +.PHONY: usage +usage: + @cat mk | grep '^#quickhelp:' | sed -e 's/^#quickhelp://' -e 's/^\ *//' + +# Most targets are .PHONY because whether to rebuild is controlled +# by golang. We expose to the user all the .PHONY targets. +#quickhelp: +#quickhelp: The `./mk list-targets` command lists all available targets. +.PHONY: list-targets +list-targets: + @cat mk | grep '^\.PHONY:' | sed -e 's/^\.PHONY://' + +#quickhelp: +#quickhelp: The `./mk help` command provides detailed usage instructions. We +#quickhelp: recommend running `./mk help|less` to page its output. +.PHONY: help +help: + @cat mk | grep -E '^#(quick)?help:' | sed -E -e 's/^#(quick)?help://' -e s'/^\ //' + +#help: +#help: The following variables control the build. You can specify them +#help: on the command line as a key-value pairs (see usage above). + +#help: +#help: * ANDROID_CLI_SHA256 : the SHA256 of the Android CLI tools file. We always +#help: download the Linux version, which seems to work +#help: also on macOS (thank you, Java! :pray:). +ANDROID_CLI_SHA256 = 7a00faadc0864f78edd8f4908a629a46d622375cbe2e5814e82934aebecdb622 + +#help: +#help: * ANDROID_CLI_VERSION : the version of the Android CLI tools. +ANDROID_CLI_VERSION = 7302050 + +#help: +#help: * ANDROID_INSTALL_EXTRA : contains the android tools we install in addition +#help: to the NDK in order to build oonimkall.aar. +ANDROID_INSTALL_EXTRA = 'build-tools;29.0.3' 'platforms;android-30' + +#help: +#help: * ANDROID_NDK_VERSION : Android NDK version. +ANDROID_NDK_VERSION = 22.1.7171670 + +#help: +#help: * DEBIAN_TILDE_VERSION : if non-empty, this should be "[0-9]+" and +#help: will be appended to the package version using +#help: a tilde, thus producing, e.g., "1.0~1234". +DEBIAN_TILDE_VERSION = + +#help: +#help: * GIT_CLONE_DIR : directory where to clone repositories, by default +#help: set to `$HOME/.ooniprobe-build/src`. +GIT_CLONE_DIR = $(HOME)/.ooniprobe-build/src + +# $(GIT_CLONE_DIR) is an internal target that creates $(GIT_CLONE_DIR). +$(GIT_CLONE_DIR): + mkdir -p $(GIT_CLONE_DIR) + +#help: +#help: * GOLANG_DOCKER_GOCACHE : where to store golang's build cache to +#help: speed up subsequent Docker builds. +GOLANG_DOCKER_GOCACHE = $(HOME)/.ooniprobe-build/docker/gocache + +#help: +#help: * GOLANG_DOCKER_GOPATH : GOPATH directory used by builds running +#help: inside docker to significantly speed +#help: up subsequent Docker based builds. +GOLANG_DOCKER_GOPATH := $(HOME)/.ooniprobe-build/docker/gopath + +#help: +#help: * GOLANG_EXTRA_FLAGS : extra flags passed to `go build ...`, empty by +#help: default. Useful to pass flags to `go`, e.g.: +#help: +#help: ./mk GOLANG_EXTRA_FLAGS="-x -v" ./CLI/miniooni +GOLANG_EXTRA_FLAGS = + +#help: +#help: * GOLANG_VERSION_NUMBER : the expected version number for golang. +GOLANG_VERSION_NUMBER = 1.16.4 + +#help: +#help: * GPG_USER : allows overriding the default GPG user used +#help: to sign binary releases, e.g.: +#help: +#help: ./mk GPG_USER=john@doe.com ooniprobe/windows +GPG_USER = simone@openobservatory.org + +#help: +#help: * MINGW_W64_VERSION : the expected mingw-w64 version. +MINGW_W64_VERSION = 10.3.1 + +#help: +#help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause +#help: the build to embed a psiphon configuration file +#help: into the generated binaries. This build tag +#help: implies cloning the git@github.com:ooni/probe-private +#help: repository. If you do not have the permission to +#help: clone it, just clear this variable, e.g.: +#help: +#help: ./mk OONI_PSIPHON_TAGS="" ./CLI/miniooni +OONI_PSIPHON_TAGS = ooni_psiphon_config + +#help: +#help: * OONI_ANDROID_HOME : directory where the Android SDK is downloaded +#help: and installed. You can point this to an existing +#help: copy of the SDK as long as (1) you have the +#help: right version of the command line tools, and +#help: (2) it's okay for us to install packages. +OONI_ANDROID_HOME = $(HOME)/.ooniprobe-build/sdk/android + +#help: +#help: * XCODE_VERSION : the version of Xcode we expect. +XCODE_VERSION = 12.5 + +#quickhelp: +#quickhelp: The `./mk show-config` command shows the current value of the +#quickhelp: variables controlling the build. +.PHONY: show-config +show-config: + @echo "ANDROID_CLI_VERSION=$(ANDROID_CLI_VERSION)" + @echo "ANDROID_CLI_SHA256=$(ANDROID_CLI_SHA256)" + @echo "ANDROID_INSTALL_EXTRA=$(ANDROID_INSTALL_EXTRA)" + @echo "ANDROID_NDK_VERSION=$(ANDROID_NDK_VERSION)" + @echo "DEBIAN_TILDE_VERSION=$(DEBIAN_TILDE_VERSION)" + @echo "GIT_CLONE_DIR=$(GIT_CLONE_DIR)" + @echo "GOLANG_DOCKER_GOCACHE=$(GOLANG_DOCKER_GOCACHE)" + @echo "GOLANG_DOCKER_GOPATH=$(GOLANG_DOCKER_GOPATH)" + @echo "GOLANG_EXTRA_FLAGS=$(GOLANG_EXTRA_FLAGS)" + @echo "GOLANG_VERSION_NUMBER=$(GOLANG_VERSION_NUMBER)" + @echo "GPG_USER=$(GPG_USER)" + @echo "MINGW_W64_VERSION=$(MINGW_W64_VERSION)" + @echo "OONI_PSIPHON_TAGS=$(OONI_PSIPHON_TAGS)" + @echo "OONI_ANDROID_HOME=$(OONI_ANDROID_HOME)" + @echo "XCODE_VERSION=$(XCODE_VERSION)" + +# GOLANG_VERSION_STRING is the expected version string. If we +# run a golang binary that does not emit this version string +# when running `go version`, we stop the build. +GOLANG_VERSION_STRING = go$(GOLANG_VERSION_NUMBER) + +# GOLANG_DOCKER_IMAGE is the golang docker image we use for +# building for Linux systems. It is an Alpine based container +# so that we can easily build static binaries. +GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine + +# Cross-compiling miniooni from any system with Go installed is +# very easy, because it does not use any C code. +#help: +#help: The `./mk ./CLI/miniooni` command builds the miniooni experimental +#help: command line client for all the supported GOOS/GOARCH. +#help: +#help: You can also build the following subtargets: +.PHONY: ./CLI/miniooni +./CLI/miniooni: \ + ./CLI/darwin/amd64/miniooni \ + ./CLI/darwin/arm64/miniooni \ + ./CLI/linux/386/miniooni \ + ./CLI/linux/amd64/miniooni \ + ./CLI/linux/arm/miniooni \ + ./CLI/linux/arm64/miniooni \ + ./CLI/windows/386/miniooni.exe \ + ./CLI/windows/amd64/miniooni.exe + +# All the miniooni targets build with CGO_ENABLED=0 such that the build +# succeeds when the GOOS/GOARCH is such that we aren't crosscompiling +# (e.g., targeting darwin/amd64 on darwin/amd64) _and_ there's no C compiler +# installed on the system. We can afford that since miniooni is pure Go. +#help: +#help: * `./mk ./CLI/darwin/amd64/miniooni`: darwin/amd64 +.PHONY: ./CLI/darwin/amd64/miniooni +./CLI/darwin/amd64/miniooni: search/for/go maybe/copypsiphon + GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/darwin/arm64/miniooni`: darwin/arm64 +.PHONY: ./CLI/darwin/arm64/miniooni +./CLI/darwin/arm64/miniooni: search/for/go maybe/copypsiphon + GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +# When building for Linux we use `-tags netgo` and `-extldflags -static` to produce +# a statically linked binary that completely bypasses libc. +#help: +#help: * `./mk ./CLI/linux/386/miniooni`: linux/386 +.PHONY: ./CLI/linux/386/miniooni +./CLI/linux/386/miniooni: search/for/go maybe/copypsiphon + GOOS=linux GOARCH=386 CGO_ENABLED=0 go build -tags="netgo,$(OONI_PSIPHON_TAGS)" -ldflags="-s -w -extldflags -static" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/linux/amd64/miniooni`: linux/amd64 +.PHONY: ./CLI/linux/amd64/miniooni +./CLI/linux/amd64/miniooni: search/for/go maybe/copypsiphon + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags="netgo,$(OONI_PSIPHON_TAGS)" -ldflags="-s -w -extldflags -static" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +# When building for GOARCH=arm, we always force GOARM=7 (i.e., armhf/armv7). +#help: +#help: * `./mk ./CLI/linux/arm/miniooni`: linux/arm +.PHONY: ./CLI/linux/arm/miniooni +./CLI/linux/arm/miniooni: search/for/go maybe/copypsiphon + GOOS=linux GOARCH=arm CGO_ENABLED=0 GOARM=7 go build -tags="netgo,$(OONI_PSIPHON_TAGS)" -ldflags="-s -w -extldflags -static" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/linux/arm64/miniooni`: linux/arm64 +.PHONY: ./CLI/linux/arm64/miniooni +./CLI/linux/arm64/miniooni: search/for/go maybe/copypsiphon + GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -tags="netgo,$(OONI_PSIPHON_TAGS)" -ldflags="-s -w -extldflags -static" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/windows/386/miniooni.exe`: windows/386 +.PHONY: ./CLI/windows/386/miniooni.exe +./CLI/windows/386/miniooni.exe: search/for/go maybe/copypsiphon + GOOS=windows GOARCH=386 CGO_ENABLED=0 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: * `./mk ./CLI/windows/amd64/miniooni.exe`: windows/amd64 +.PHONY: ./CLI/windows/amd64/miniooni.exe +./CLI/windows/amd64/miniooni.exe: search/for/go maybe/copypsiphon + GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./internal/cmd/miniooni + +#help: +#help: The `./mk ./CLI/ooniprobe/darwin` command builds the ooniprobe official +#help: command line client for darwin/amd64 and darwin/arm64. This process +#help: entails building ooniprobe and then GPG-signing the binaries. +#help: +#help: You can also build the following subtargets: +.PHONY: ./CLI/ooniprobe/darwin +./CLI/ooniprobe/darwin: \ + ./CLI/darwin/amd64/ooniprobe.asc \ + ./CLI/darwin/arm64/ooniprobe.asc + +# ./CLI/darwin/amd64/ooniprobe.asc is an internal target for signing +.PHONY: ./CLI/darwin/amd64/ooniprobe.asc +./CLI/darwin/amd64/ooniprobe.asc: ./CLI/darwin/amd64/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +# We force CGO_ENABLED=1 because in principle we may be cross compiling. In +# reality it's hard to see a macOS/darwin build not made on macOS. +#help: +#help: * `./mk ./CLI/darwin/amd64/ooniprobe`: darwin/amd64 +.PHONY: ./CLI/darwin/amd64/ooniprobe +./CLI/darwin/amd64/ooniprobe: search/for/go maybe/copypsiphon + GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./cmd/ooniprobe + +# ./CLI/darwin/arm64/ooniprobe.asc is an internal target for signing +.PHONY: ./CLI/darwin/arm64/ooniprobe.asc +./CLI/darwin/arm64/ooniprobe.asc: ./CLI/darwin/arm64/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +#help: +#help: * `./mk ./CLI/darwin/arm64/ooniprobe`: darwin/arm64 +.PHONY: ./CLI/darwin/arm64/ooniprobe +./CLI/darwin/arm64/ooniprobe: search/for/go maybe/copypsiphon + GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./cmd/ooniprobe + +#help: +#help: The `./mk ./debian` command builds the ooniprobe CLI +#help: debian package for amd64 and arm64. +#help: +#help: You can also build the following subtargets: +.PHONY: ./debian +./debian: \ + ./debian/amd64 \ + ./debian/arm64 + +#help: +#help: * `./mk ./debian/amd64`: debian/amd64 +.PHONY: ./debian/amd64 +# This extra .PHONY for linux/amd64 is to help printing targets 🤷. +.PHONY: ./CLI/linux/amd64/ooniprobe +./debian/amd64: search/for/docker ./CLI/linux/amd64/ooniprobe + docker pull --platform linux/amd64 debian:stable + docker run --platform linux/amd64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian "$(DEBIAN_TILDE_VERSION)" + +#help: +#help: * `./mk ./debian/arm64`: debian/arm64 +.PHONY: ./debian/arm64 +# This extra .PHONY for linux/arm64 is to help printing targets 🤷. +.PHONY: ./CLI/linux/arm64/ooniprobe +./debian/arm64: search/for/docker ./CLI/linux/arm64/ooniprobe + docker pull --platform linux/arm64 debian:stable + docker run --platform linux/arm64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian "$(DEBIAN_TILDE_VERSION)" + +#help: +#help: The `./mk ./CLI/ooniprobe/linux` command builds the ooniprobe official command +#help: line client for amd64 and arm64. This entails building and GPG signing. +#help: +#help: You can also build the following subtargets: +.PHONY: ./CLI/ooniprobe/linux +./CLI/ooniprobe/linux: \ + ./CLI/linux/amd64/ooniprobe.asc \ + ./CLI/linux/arm64/ooniprobe.asc + +# ./CLI/linux/amd64/ooniprobe.asc is an internal task for signing. +.PHONY: ./CLI/linux/amd64/ooniprobe.asc +./CLI/linux/amd64/ooniprobe.asc: ./CLI/linux/amd64/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +# Linux builds use Alpine and Docker so we are sure that we are statically +# linking to musl libc, thus making our binaries extremely portable. +#help: +#help: * `./mk ./CLI/linux/amd64/ooniprobe`: linux/amd64 +.PHONY: ./CLI/linux/amd64/ooniprobe +./CLI/linux/amd64/ooniprobe: search/for/docker maybe/copypsiphon + docker pull --platform linux/amd64 $(GOLANG_DOCKER_IMAGE) + docker run --platform linux/amd64 -e GOPATH=/gopath -e GOARCH=amd64 -v $(GOLANG_DOCKER_GOCACHE)/amd64:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS) + +# ./CLI/linux/arm64/ooniprobe.asc is an internal task for signing. +.PHONY: ./CLI/linux/arm64/ooniprobe.asc +./CLI/linux/arm64/ooniprobe.asc: ./CLI/linux/arm64/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +#help: +#help: * `./mk ./CLI/linux/arm64/ooniprobe`: linux/arm64 +.PHONY: ./CLI/linux/arm64/ooniprobe +./CLI/linux/arm64/ooniprobe: search/for/docker maybe/copypsiphon + docker pull --platform linux/arm64 $(GOLANG_DOCKER_IMAGE) + docker run --platform linux/arm64 -e GOPATH=/gopath -e GOARCH=arm64 -v $(GOLANG_DOCKER_GOCACHE)/arm64:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS) + +#help: +#help: The `./mk ./CLI/ooniprobe/windows` command builds the ooniprobe official +#help: command line client for windows/386 and windows/amd64. This entails +#help: building and PGP signing the executables. +#help: +#help: You can also build the following subtargets: +.PHONY: ./CLI/ooniprobe/windows +./CLI/ooniprobe/windows: \ + ./CLI/windows/386/ooniprobe.exe.asc \ + ./CLI/windows/amd64/ooniprobe.exe.asc + +# ./CLI/windows/386/ooniprobe.exe.asc is an internal signing target +.PHONY: ./CLI/windows/386/ooniprobe.exe.asc +./CLI/windows/386/ooniprobe.exe.asc: ./CLI/windows/386/ooniprobe.exe + rm -f $@ && gpg -abu $(GPG_USER) $< + +#help: +#help: * `./mk ./CLI/windows/386/ooniprobe.exe`: windows/386 +.PHONY: ./CLI/windows/386/ooniprobe.exe +./CLI/windows/386/ooniprobe.exe: search/for/go search/for/mingw-w64 maybe/copypsiphon + GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./cmd/ooniprobe + +# ./CLI/windows/amd64/ooniprobe.exe.asc is an internal signing target +.PHONY: ./CLI/windows/amd64/ooniprobe.exe.asc +./CLI/windows/amd64/ooniprobe.exe.asc: ./CLI/windows/amd64/ooniprobe.exe + rm -f $@ && gpg -abu $(GPG_USER) $< + +#help: +#help: * `./mk ./CLI/windows/amd64/ooniprobe.exe`: windows/amd64 +.PHONY: ./CLI/windows/amd64/ooniprobe.exe +./CLI/windows/amd64/ooniprobe.exe: search/for/go search/for/mingw-w64 maybe/copypsiphon + GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc go build -tags="$(OONI_PSIPHON_TAGS)" -ldflags="-s -w" $(GOLANG_EXTRA_FLAGS) -o $@ ./cmd/ooniprobe + +#help: +#help: The `./mk ./MOBILE/android` command builds the oonimkall library for Android. +#help: +#help: You can also build the following subtargets: +.PHONY: ./MOBILE/android +./MOBILE/android: search/for/gpg search/for/jar ./MOBILE/android/oonimkall.aar + cp ./MOBILE/android/oonimkall.aar ./MOBILE/android/oonimkall-$(OONIMKALL_V).aar + cp ./MOBILE/android/oonimkall-sources.jar ./MOBILE/android/oonimkall-$(OONIMKALL_V)-sources.jar + cat ./MOBILE/template.pom | sed -e "s/@VERSION@/$(OONIMKALL_V)/g" > ./MOBILE/android/oonimkall-$(OONIMKALL_V).pom + gpg -abu $(GPG_USER) ./MOBILE/android/oonimkall-$(OONIMKALL_V).aar + gpg -abu $(GPG_USER) ./MOBILE/android/oonimkall-$(OONIMKALL_V)-sources.jar + gpg -abu $(GPG_USER) ./MOBILE/android/oonimkall-$(OONIMKALL_V).pom + cd ./MOBILE/android && jar -cf bundle.jar oonimkall-$(OONIMKALL_V).aar oonimkall-$(OONIMKALL_V).aar.asc oonimkall-$(OONIMKALL_V)-sources.jar oonimkall-$(OONIMKALL_V)-sources.jar.asc oonimkall-$(OONIMKALL_V).pom oonimkall-$(OONIMKALL_V).pom.asc + +#help: +#help: * `./mk ./MOBILE/android/oonimkall.aar`: the AAR +.PHONY: ./MOBILE/android/oonimkall.aar +./MOBILE/android/oonimkall.aar: android/sdk ooni/go maybe/copypsiphon + PATH=$(OONIGODIR)/bin:$$PATH $(MAKE) -f mk __android_build_with_ooni_go + +# GOMOBILE is the full path location to the gomobile binary. We want to +# execute this command every time, because its output may depend on context, +# for this reason WE ARE NOT using `:=`. +GOMOBILE = $(shell go env GOPATH)/bin/gomobile + +# Here we use ooni/go to work around https://github.com/ooni/probe/issues/1444 +__android_build_with_ooni_go: search/for/go + go get -u golang.org/x/mobile/cmd/gomobile + $(GOMOBILE) init + PATH=$(shell go env GOPATH)/bin:$$PATH ANDROID_HOME=$(OONI_ANDROID_HOME) ANDROID_NDK_HOME=$(OONI_ANDROID_HOME)/ndk/$(ANDROID_NDK_VERSION) $(GOMOBILE) bind -target android -o ./MOBILE/android/oonimkall.aar -tags="$(OONI_PSIPHON_TAGS)" -ldflags '-s -w' $(GOLANG_EXTRA_FLAGS) ./pkg/oonimkall + +#help: +#help: The `./mk ./MOBILE/ios` command builds the oonimkall library for iOS. +#help: +#help: You can also build the following subtargets: +.PHONY: ./MOBILE/ios +./MOBILE/ios: \ + ./MOBILE/ios/oonimkall.framework.zip \ + ./MOBILE/ios/oonimkall.podspec + +#help: +#help: * `./mk ./MOBILE/ios/oonimkall.framework.zip`: zip the framework +.PHONY: ./MOBILE/ios/oonimkall.framework.zip +./MOBILE/ios/oonimkall.framework.zip: search/for/zip ./MOBILE/ios/oonimkall.framework + cd ./MOBILE/ios && rm -rf oonimkall.framework.zip + cd ./MOBILE/ios && zip -yr oonimkall.framework.zip oonimkall.framework + +#help: +#help: * `./mk ./MOBILE/ios/framework`: the framework +.PHONY: ./MOBILE/ios/oonimkall.framework +./MOBILE/ios/oonimkall.framework: search/for/go search/for/xcode + go get -u golang.org/x/mobile/cmd/gomobile + $(GOMOBILE) init + PATH=$(shell go env GOPATH)/bin:$$PATH $(GOMOBILE) bind -target ios -o $@ -tags="$(OONI_PSIPHON_TAGS)" -ldflags '-s -w' $(GOLANG_EXTRA_FLAGS) ./pkg/oonimkall + +#help: +#help: * `./mk ./MOBILE/ios/oonimkall.podspec`: the podspec +.PHONY: ./MOBILE/ios/oonimkall.podspec +./MOBILE/ios/oonimkall.podspec: ./MOBILE/template.podspec + cat $< | sed -e "s/@VERSION@/$(OONIMKALL_V)/g" -e "s/@RELEASE@/$(OONIMKALL_R)/g" > $@ + +# important: OONIMKALL_V and OONIMKALL_R MUST be expanded just once so we use `:=` +OONIMKALL_V := $(shell date -u +%Y.%m.%d-%H%M%S) +OONIMKALL_R := $(shell git describe --tags || echo '0.0.0-dev') + +#help: +#help: The following commands check for the availability of dependencies: +# TODO(bassosimone): make checks more robust? + +#help: +#help: * `./mk search/for/bash`: checks for bash +.PHONY: search/for/bash +search/for/bash: + @printf "checking for bash... " + @command -v bash || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/curl`: checks for curl +.PHONY: search/for/curl +search/for/curl: + @printf "checking for curl... " + @command -v curl || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/docker`: checks for docker +.PHONY: search/for/docker +search/for/docker: + @printf "checking for docker... " + @command -v docker || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/git`: checks for git +.PHONY: search/for/git +search/for/git: + @printf "checking for git... " + @command -v git || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/gpg`: checks for gpg +.PHONY: search/for/gpg +search/for/gpg: + @printf "checking for gpg... " + @command -v gpg || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/jar`: checks for jar +.PHONY: search/for/jar +search/for/jar: + @printf "checking for jar... " + @command -v jar || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/java`: checks for java +.PHONY: search/for/java +search/for/java: + @printf "checking for java... " + @command -v java || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/go`: checks for go +.PHONY: search/for/go +search/for/go: + @printf "checking for go... " + @command -v go || { echo "not found"; exit 1; } + @printf "checking for go version... " + @echo $(__GOVERSION_REAL) + @[ "$(GOLANG_VERSION_STRING)" = "$(__GOVERSION_REAL)" ] || { echo "fatal: go version must be $(GOLANG_VERSION_STRING) instead of $(__GOVERSION_REAL)"; exit 1; } + +# __GOVERSION_REAL is the go version reported by the go binary (we +# SHOULD NOT cache this value so we ARE NOT using `:=`) +__GOVERSION_REAL = $(shell go version | awk '{print $$3}') + +#help: +#help: * `./mk search/for/mingw-w64`: checks for mingw-w64 +.PHONY: search/for/mingw-w64 +search/for/mingw-w64: + @printf "checking for x86_64-w64-mingw32-gcc... " + @command -v x86_64-w64-mingw32-gcc || { echo "not found"; exit 1; } + @printf "checking for x86_64-w64-mingw32-gcc version... " + @echo $(__MINGW32_AMD64_VERSION) + @[ "$(MINGW_W64_VERSION)" = "$(__MINGW32_AMD64_VERSION)" ] || { echo "fatal: x86_64-w64-mingw32-gcc version must be $(MINGW_W64_VERSION) instead of $(__MINGW32_AMD64_VERSION)"; exit 1; } + @printf "checking for i686-w64-mingw32-gcc... " + @command -v i686-w64-mingw32-gcc || { echo "not found"; exit 1; } + @printf "checking for i686-w64-mingw32-gcc version... " + @echo $(__MINGW32_386_VERSION) + @[ "$(MINGW_W64_VERSION)" = "$(__MINGW32_386_VERSION)" ] || { echo "fatal: i686-w64-mingw32-gcc version must be $(MINGW_W64_VERSION) instead of $(__MINGW32_386_VERSION)"; exit 1; } + +# __MINGW32_AMD64_VERSION and __MINGW32_386_VERSION are the versions +# reported by the amd64 and 386 mingw binaries. +__MINGW32_AMD64_VERSION = $(shell x86_64-w64-mingw32-gcc --version | sed -n 1p | awk '{print $$3}') +__MINGW32_386_VERSION = $(shell i686-w64-mingw32-gcc --version | sed -n 1p | awk '{print $$3}') + +#help: +#help: * `./mk search/for/shasum`: checks for shasum +.PHONY: search/for/shasum +search/for/shasum: + @printf "checking for shasum... " + @command -v shasum || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/xcode`: checks for Xcode +.PHONY: search/for/xcode +search/for/xcode: + @printf "checking for xcodebuild... " + @command -v xcodebuild || { echo "not found"; exit 1; } + @printf "checking for Xcode version... " + @echo $(__XCODEVERSION_REAL) + @[ "$(XCODE_VERSION)" = "$(__XCODEVERSION_REAL)" ] || { echo "fatal: Xcode version must be $(XCODE_VERSION) instead of $(__XCODEVERSION_REAL)"; exit 1; } + +# __XCODEVERSION_REAL is the version of Xcode obtained using xcodebuild +__XCODEVERSION_REAL = `xcodebuild -version | grep ^Xcode | awk '{print $$2}'` + +#help: +#help: * `./mk search/for/unzip`: checks for unzip +.PHONY: search/for/unzip +search/for/unzip: + @printf "checking for unzip... " + @command -v unzip || { echo "not found"; exit 1; } + +#help: +#help: * `./mk search/for/zip`: checks for zip +.PHONY: search/for/zip +search/for/zip: + @printf "checking for zip... " + @command -v zip || { echo "not found"; exit 1; } + +#help: +#help: The `./mk maybe/copypsiphon` command copies the private psiphon config +#help: file into the current tree unless `$(OONI_PSIPHON_TAGS)` is empty. +.PHONY: maybe/copypsiphon +maybe/copypsiphon: search/for/git + test -z "$(OONI_PSIPHON_TAGS)" || $(MAKE) -f mk $(OONIPRIVATE) + test -z "$(OONI_PSIPHON_TAGS)" || cp $(OONIPRIVATE)/psiphon-config.key ./internal/engine + test -z "$(OONI_PSIPHON_TAGS)" || cp $(OONIPRIVATE)/psiphon-config.json.age ./internal/engine + +# OONIPRIVATE is the directory where we clone the private repository. +OONIPRIVATE = $(GIT_CLONE_DIR)/github.com/ooni/probe-private + +# OONIPRIVATE_REPO is the private repository URL. +OONIPRIVATE_REPO = git@github.com:ooni/probe-private + +# $(OONIPRIVATE) clones the private repository in $(GIT_CLONE_DIR) +$(OONIPRIVATE): search/for/git $(GIT_CLONE_DIR) + test -d $(OONIPRIVATE) || $(MAKE) -f mk __really_clone_private_repo + +__really_clone_private_repo: + git clone $(OONIPRIVATE_REPO) $(OONIPRIVATE) + +#help: +#help: The `./mk ooni/go` command builds the latest version of ooni/go. +.PHONY: ooni/go +ooni/go: search/for/bash search/for/git search/for/go $(OONIGODIR) + test -d $(OONIGODIR) || git clone -b ooni --single-branch --depth 8 $(OONIGO_REPO) $(OONIGODIR) + cd $(OONIGODIR) && git pull --ff-only + cd $(OONIGODIR)/src && ./make.bash + +# OONIGODIR is the directory in which we clone ooni/go +OONIGODIR = $(GIT_CLONE_DIR)/github.com/ooni/go + +# OONIGO_REPO is the repository for ooni/go +OONIGO_REPO = https://github.com/ooni/go + +#help: +#help: The `./mk android/sdk` command ensures we are using the +#help: correct version of the Android sdk. +.PHONY: android/sdk +android/sdk: search/for/java + test -d $(OONI_ANDROID_HOME) || $(MAKE) -f mk android/sdk/download + echo "Yes" | $(__ANDROID_SDKMANAGER) --install $(ANDROID_INSTALL_EXTRA) 'ndk;$(ANDROID_NDK_VERSION)' + +# __ANDROID_SKDMANAGER is the path to android's sdkmanager tool +__ANDROID_SDKMANAGER = $(OONI_ANDROID_HOME)/cmdline-tools/$(ANDROID_CLI_VERSION)/bin/sdkmanager + +# See https://stackoverflow.com/a/61176718 to understand why +# we need to reorganize the directories like this: +#help: +#help: The `./mk android/sdk/download` unconditionally downloads the +#help: Android SDK at `$(OONI_ANDROID_HOME)`. +android/sdk/download: search/for/curl search/for/java search/for/shasum search/for/unzip + curl -fsSLO https://dl.google.com/android/repository/$(__ANDROID_CLITOOLS_FILE) + echo "$(ANDROID_CLI_SHA256) $(__ANDROID_CLITOOLS_FILE)" > __SHA256 + shasum --check __SHA256 + rm -f __SHA256 + unzip $(__ANDROID_CLITOOLS_FILE) + rm $(__ANDROID_CLITOOLS_FILE) + mkdir -p $(OONI_ANDROID_HOME)/cmdline-tools + mv cmdline-tools $(OONI_ANDROID_HOME)/cmdline-tools/$(ANDROID_CLI_VERSION) + +# __ANDROID_CLITOOLS_FILE is the file name of the android cli tools zip +__ANDROID_CLITOOLS_FILE = commandlinetools-linux-$(ANDROID_CLI_VERSION)_latest.zip