feat(gha): build debian package using ./make (#331)

Part of https://github.com/ooni/probe/issues/1466. We're building both `arm64` and `amd64`. We are still not publishing `arm64` packages, which is what is asked in the original issue, but we're really close to doing that.

.github/workflows/debian.yml

@@ -5,7 +5,6 @@ on:
   push:
     branches:
       - "master"
-      - "deb-s3"
       - "release/**"
     tags:
       - "v*"
@@ -22,6 +21,8 @@ jobs:
           go-version: "1.16"
       - uses: actions/checkout@v2
       - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64
+      - run: mkdir -p debian/bin
+      - run: cp ./CLI/linux/amd64/ooniprobe debian/bin/ooniprobe
       - run: sudo apt-get update -q
       - run: sudo apt-get build-dep -y --no-install-recommends .
       - name: Install deps

.github/workflows/linux.yml

@@ -17,6 +17,7 @@ jobs:
         env:
           DOCKER_CLI_EXPERIMENTAL: enabled
       - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
+      - run: ./make --disable-embedding-psiphon-config -t debian_amd64
 
   build_arm64:
     runs-on: "ubuntu-20.04"
@@ -30,3 +31,4 @@ jobs:
         env:
           DOCKER_CLI_EXPERIMENTAL: enabled
       - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe
+      - run: ./make --disable-embedding-psiphon-config -t debian_arm64

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+/*.deb
 /*.jsonl
 /*.tar.gz
 /*.zip

CLI/linux/debian

@@ -0,0 +1,73 @@
+#!/bin/sh
+# This script creates a Debian package. When run by `./make`, it
+# is run inside a debian:stable container. It's fine to also
+# run this script from any debian-like system, as long as the
+# following ASSUMPTIONS are met:
+#
+# 1. the `ooniprobe` we are packaging is available at
+# this path `./CLI/linux/$GOARCH/ooniprobe`;
+#
+# 2. we are running on a debian system that has the same
+# architecture of the `ooniprobe` we are packaging.
+
+if [ $# -gt 1 ]; then
+    echo "usage: $0 [run_number]" 1>&2
+    exit 1
+fi
+run_number=$1
+
+# Copy the target binary in the correct location expected
+# by the debian/ooniprobe-cli.install file.
+rm -rf ./debian/bin
+mkdir -p ./debian/bin
+machine=`uname -m`
+goarch=""
+case $machine in
+    x86_64)
+        cp ./CLI/linux/amd64/ooniprobe ./debian/bin
+        goarch=amd64
+    ;;
+    aarch64)
+        cp ./CLI/linux/arm64/ooniprobe ./debian/bin
+        goarch=arm64
+    ;;
+    *)
+        # TODO(bassosimone): here we probably want to further extend
+        # this script to support at least armv7.
+        echo "FATAL: unsupported machine: $machine" 1>&2
+        exit 1
+    ;;
+esac
+
+set -ex
+
+# figure out the version number from the binary itself (which rests
+# on the assumption that `uname -m` can run such a binary)
+version=`./debian/bin/ooniprobe version`
+if [ ! -z $run_number ]; then
+    version="${version}~${run_number}"
+fi
+
+# The OONI_DEB_DRY_RUN is a semi-undocumented feature allowing
+# us to see the commands that would be run by this script.
+
+# install the dependencies required by the build process
+$OONI_DEB_DRY_RUN apt-get update -q
+$OONI_DEB_DRY_RUN apt-get build-dep -y --no-install-recommends .
+
+# keep the original changelog file safe
+$OONI_DEB_DRY_RUN cp ./debian/changelog ./debian/changelog.oocopy
+
+$OONI_DEB_DRY_RUN dch -v $version "New version ${version}"
+$OONI_DEB_DRY_RUN dpkg-buildpackage -us -uc -b
+
+# restore the original changelog file
+$OONI_DEB_DRY_RUN mv ./debian/changelog.oocopy ./debian/changelog
+
+# move the package so that we don't loose track
+# of it when using a build container
+$OONI_DEB_DRY_RUN mv ../*.deb .
+
+# install the package on the container as a smoke test to
+# ensure that it is installable.
+DEBIAN_FRONTEND=noninteractive dpkg -i ooniprobe-cli_${version}_${goarch}.deb

debian/ooniprobe-cli.install

@@ -1,2 +1,2 @@
-./CLI/linux/amd64/ooniprobe usr/bin
+debian/bin/ooniprobe /usr/bin
 debian/ooniprobe.conf.disabled /etc/ooniprobe

make

@@ -1249,6 +1249,58 @@ class OONIProbeDarwin:
                         engine.run(cmdline)
 
 
+class Debian:
+    """Debian makes a debian package of a target artifact. It
+    currently only works with ooniprobe targets."""
+
+    def __init__(self, arch: str, target: Target):
+        self._arch = arch
+        self._target = target
+
+    def name(self) -> str:
+        return "debian_{}".format(self._arch)
+
+    def build(self, engine: Engine, options: Options) -> None:
+        self._target.build(engine, options)
+        log("\n./make: building {}...".format(self.name()))
+        engine.require("docker")
+        # make sure we have the latest version of the container image
+        engine.run(
+            [
+                "docker",
+                "pull",
+                "--platform",
+                "linux/{}".format(self._arch),
+                "debian:stable",
+            ]
+        )
+        # then run the build inside the container
+        cmdline: List[str] = []
+        cmdline.append("docker")
+        cmdline.append("run")
+        cmdline.append("--platform")
+        cmdline.append("linux/{}".format(self._arch))
+        cmdline.append("-v")
+        cmdline.append("{}:/ooni".format(os.getcwd()))
+        cmdline.append("-w")
+        cmdline.append("/ooni")
+        cmdline.append("debian:stable")
+        cmdline.append(os.path.join(".", "CLI", "linux", "debian"))
+        if os.environ.get("GITHUB_ACTIONS", "") == "true":
+            # When we're running inside a github action, figure out whether
+            # we are building a tag or a commit. In the latter case, we will
+            # append the run number to the version number.
+            github_ref = os.environ.get("GITHUB_REF")
+            if not github_ref:
+                raise RuntimeError("missing GITHUB_REF")
+            github_run_number = os.environ.get("GITHUB_RUN_NUMBER")
+            if not github_run_number:
+                raise RuntimeError("missing GITHUB_RUN_NUMBER")
+            if not github_ref.startswith("/refs/tags/"):
+                cmdline.append(github_run_number)
+        engine.run(cmdline)
+
+
 class Sign:
     """Sign signs a specific target artefact."""
 
@@ -1320,6 +1372,15 @@ EXTRA_TARGETS: List[Target] = [
     OONIMKAllFrameworkZip(),
 ]
 
+# DEBIAN_TARGETS contains individual debian targets.
+DEBIAN_TARGETS: List[Target] = [
+    Debian("arm64", OONIProbeLinux("arm64")),
+    Debian("amd64", OONIProbeLinux("amd64")),
+]
+
+# DEBIAN is the top-level "debian" target.
+DEBIAN = Phony("debian", DEBIAN_TARGETS)
+
 # VISIBLE_TARGETS contains all the visible-from-CLI targets
 VISIBLE_TARGETS: List[Target] = (
     OONIPROBE_TARGETS
@@ -1330,6 +1391,8 @@ VISIBLE_TARGETS: List[Target] = (
     + [OONIPROBE_RELEASE_DARWIN]
     + [OONIPROBE_RELEASE_LINUX]
     + [OONIPROBE_RELEASE_WINDOWS]
+    + DEBIAN_TARGETS
+    + [DEBIAN]
 )