From 3109d56aef5c1667cecf4edced11f32696e4fd78 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 6 May 2021 22:13:09 +0200 Subject: [PATCH] feat(gha): build debian package using ./make (#331) Part of https://github.com/ooni/probe/issues/1466. We're building both `arm64` and `amd64`. We are still not publishing `arm64` packages, which is what is asked in the original issue, but we're really close to doing that. --- .github/workflows/debian.yml | 3 +- .github/workflows/linux.yml | 2 + .gitignore | 1 + CLI/linux/debian | 73 ++++++++++++++++++++++++++++++++++++ debian/ooniprobe-cli.install | 2 +- make | 63 +++++++++++++++++++++++++++++++ 6 files changed, 142 insertions(+), 2 deletions(-) create mode 100755 CLI/linux/debian diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 189bbb8..42cf5cb 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -5,7 +5,6 @@ on: push: branches: - "master" - - "deb-s3" - "release/**" tags: - "v*" @@ -22,6 +21,8 @@ jobs: go-version: "1.16" - uses: actions/checkout@v2 - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64 + - run: mkdir -p debian/bin + - run: cp ./CLI/linux/amd64/ooniprobe debian/bin/ooniprobe - run: sudo apt-get update -q - run: sudo apt-get build-dep -y --no-install-recommends . - name: Install deps diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3148d04..cb92687 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,6 +17,7 @@ jobs: env: DOCKER_CLI_EXPERIMENTAL: enabled - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe + - run: ./make --disable-embedding-psiphon-config -t debian_amd64 build_arm64: runs-on: "ubuntu-20.04" @@ -30,3 +31,4 @@ jobs: env: DOCKER_CLI_EXPERIMENTAL: enabled - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe + - run: ./make --disable-embedding-psiphon-config -t debian_arm64 diff --git a/.gitignore b/.gitignore index f57e825..fe853c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store +/*.deb /*.jsonl /*.tar.gz /*.zip diff --git a/CLI/linux/debian b/CLI/linux/debian new file mode 100755 index 0000000..4b11bec --- /dev/null +++ b/CLI/linux/debian @@ -0,0 +1,73 @@ +#!/bin/sh +# This script creates a Debian package. When run by `./make`, it +# is run inside a debian:stable container. It's fine to also +# run this script from any debian-like system, as long as the +# following ASSUMPTIONS are met: +# +# 1. the `ooniprobe` we are packaging is available at +# this path `./CLI/linux/$GOARCH/ooniprobe`; +# +# 2. we are running on a debian system that has the same +# architecture of the `ooniprobe` we are packaging. + +if [ $# -gt 1 ]; then + echo "usage: $0 [run_number]" 1>&2 + exit 1 +fi +run_number=$1 + +# Copy the target binary in the correct location expected +# by the debian/ooniprobe-cli.install file. +rm -rf ./debian/bin +mkdir -p ./debian/bin +machine=`uname -m` +goarch="" +case $machine in + x86_64) + cp ./CLI/linux/amd64/ooniprobe ./debian/bin + goarch=amd64 + ;; + aarch64) + cp ./CLI/linux/arm64/ooniprobe ./debian/bin + goarch=arm64 + ;; + *) + # TODO(bassosimone): here we probably want to further extend + # this script to support at least armv7. + echo "FATAL: unsupported machine: $machine" 1>&2 + exit 1 + ;; +esac + +set -ex + +# figure out the version number from the binary itself (which rests +# on the assumption that `uname -m` can run such a binary) +version=`./debian/bin/ooniprobe version` +if [ ! -z $run_number ]; then + version="${version}~${run_number}" +fi + +# The OONI_DEB_DRY_RUN is a semi-undocumented feature allowing +# us to see the commands that would be run by this script. + +# install the dependencies required by the build process +$OONI_DEB_DRY_RUN apt-get update -q +$OONI_DEB_DRY_RUN apt-get build-dep -y --no-install-recommends . + +# keep the original changelog file safe +$OONI_DEB_DRY_RUN cp ./debian/changelog ./debian/changelog.oocopy + +$OONI_DEB_DRY_RUN dch -v $version "New version ${version}" +$OONI_DEB_DRY_RUN dpkg-buildpackage -us -uc -b + +# restore the original changelog file +$OONI_DEB_DRY_RUN mv ./debian/changelog.oocopy ./debian/changelog + +# move the package so that we don't loose track +# of it when using a build container +$OONI_DEB_DRY_RUN mv ../*.deb . + +# install the package on the container as a smoke test to +# ensure that it is installable. +DEBIAN_FRONTEND=noninteractive dpkg -i ooniprobe-cli_${version}_${goarch}.deb diff --git a/debian/ooniprobe-cli.install b/debian/ooniprobe-cli.install index 8fcfc0b..4580a18 100644 --- a/debian/ooniprobe-cli.install +++ b/debian/ooniprobe-cli.install @@ -1,2 +1,2 @@ -./CLI/linux/amd64/ooniprobe usr/bin +debian/bin/ooniprobe /usr/bin debian/ooniprobe.conf.disabled /etc/ooniprobe diff --git a/make b/make index 1d1a7d0..a69e395 100755 --- a/make +++ b/make @@ -1249,6 +1249,58 @@ class OONIProbeDarwin: engine.run(cmdline) +class Debian: + """Debian makes a debian package of a target artifact. It + currently only works with ooniprobe targets.""" + + def __init__(self, arch: str, target: Target): + self._arch = arch + self._target = target + + def name(self) -> str: + return "debian_{}".format(self._arch) + + def build(self, engine: Engine, options: Options) -> None: + self._target.build(engine, options) + log("\n./make: building {}...".format(self.name())) + engine.require("docker") + # make sure we have the latest version of the container image + engine.run( + [ + "docker", + "pull", + "--platform", + "linux/{}".format(self._arch), + "debian:stable", + ] + ) + # then run the build inside the container + cmdline: List[str] = [] + cmdline.append("docker") + cmdline.append("run") + cmdline.append("--platform") + cmdline.append("linux/{}".format(self._arch)) + cmdline.append("-v") + cmdline.append("{}:/ooni".format(os.getcwd())) + cmdline.append("-w") + cmdline.append("/ooni") + cmdline.append("debian:stable") + cmdline.append(os.path.join(".", "CLI", "linux", "debian")) + if os.environ.get("GITHUB_ACTIONS", "") == "true": + # When we're running inside a github action, figure out whether + # we are building a tag or a commit. In the latter case, we will + # append the run number to the version number. + github_ref = os.environ.get("GITHUB_REF") + if not github_ref: + raise RuntimeError("missing GITHUB_REF") + github_run_number = os.environ.get("GITHUB_RUN_NUMBER") + if not github_run_number: + raise RuntimeError("missing GITHUB_RUN_NUMBER") + if not github_ref.startswith("/refs/tags/"): + cmdline.append(github_run_number) + engine.run(cmdline) + + class Sign: """Sign signs a specific target artefact.""" @@ -1320,6 +1372,15 @@ EXTRA_TARGETS: List[Target] = [ OONIMKAllFrameworkZip(), ] +# DEBIAN_TARGETS contains individual debian targets. +DEBIAN_TARGETS: List[Target] = [ + Debian("arm64", OONIProbeLinux("arm64")), + Debian("amd64", OONIProbeLinux("amd64")), +] + +# DEBIAN is the top-level "debian" target. +DEBIAN = Phony("debian", DEBIAN_TARGETS) + # VISIBLE_TARGETS contains all the visible-from-CLI targets VISIBLE_TARGETS: List[Target] = ( OONIPROBE_TARGETS @@ -1330,6 +1391,8 @@ VISIBLE_TARGETS: List[Target] = ( + [OONIPROBE_RELEASE_DARWIN] + [OONIPROBE_RELEASE_LINUX] + [OONIPROBE_RELEASE_WINDOWS] + + DEBIAN_TARGETS + + [DEBIAN] )