feat: build and publish debian/{armhf,i386} (#344)

* feat: build and publish debian/{armhf,i386} Part of https://github.com/ooni/probe/issues/807 * zap temporary build
2021-05-11 20:16:27 +02:00 · 2021-05-11 20:16:27 +02:00 · f77f54bcc9
commit f77f54bcc9
parent b2209bb637
4 changed files with 118 additions and 42 deletions
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@ -6,18 +6,28 @@ on:
      - "release/**"

 jobs:
+  build_386:
+    runs-on: "ubuntu-20.04"
+    steps:
+      - uses: actions/checkout@v2
+      - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/386
+      - run: ./smoketest.sh ./CLI/linux/386/ooniprobe
+      - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
+      - run: |
+          for deb in *.deb; do
+            ./.github/workflows/debops-ci --arch i386 --show-commands upload --bucket-name ooni-deb $deb
+          done
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
+
  build_amd64:
    runs-on: "ubuntu-20.04"
    steps:
-      - run: |
-          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
-          sudo service docker restart
      - uses: actions/checkout@v2
-      - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/linux/amd64/ooniprobe
-        env:
-          DOCKER_CLI_EXPERIMENTAL: enabled
-      - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
      - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64
+      - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
      - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
      - run: |
          for deb in *.deb; do
@ -28,20 +38,33 @@ jobs:
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}

-  build_arm64:
+  build_arm:
    runs-on: "ubuntu-20.04"
    steps:
-      - run: |
-          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
-          sudo service docker restart
      - uses: actions/checkout@v2
      - run: sudo apt-get update -q
      - run: sudo apt-get install -y qemu-user-static
-      - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/linux/arm64/ooniprobe
+      - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm
+      - run: ./smoketest.sh ./CLI/linux/arm/ooniprobe
+      - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
+      - run: |
+          for deb in *.deb; do
+            ./.github/workflows/debops-ci --arch armhf --show-commands upload --bucket-name ooni-deb $deb
+          done
        env:
-          DOCKER_CLI_EXPERIMENTAL: enabled
-      - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
+
+
+  build_arm64:
+    runs-on: "ubuntu-20.04"
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get update -q
+      - run: sudo apt-get install -y qemu-user-static
      - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64
+      - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe
      - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
      - run: |
          for deb in *.deb; do
--- a/CLI/linux/build
+++ b/CLI/linux/build
@ -11,6 +11,9 @@ set -x
 apk update
 apk upgrade
 apk add --no-progress gcc git linux-headers musl-dev
+# some of the following exports are redundant but are however
+# useful because they provide explicit logging
+export GOARM=$GOARM
 export GOPATH=$GOPATH
 export CGO_ENABLED=1
 export GOOS=linux
--- a/CLI/linux/debian
+++ b/CLI/linux/debian
@ -10,36 +10,19 @@
 # 2. we are running on a debian system that has the same
 # architecture of the `ooniprobe` we are packaging.

-if [ $# -gt 1 ]; then
-	echo "usage: $0 [run_number]" 1>&2
+if [ $# -ne 1 ] && [ $# -ne 2 ]; then
+	echo "usage: $0 {arch} [run_number]" 1>&2
 	exit 1
 fi
-run_number=$1
+goarch=$1
+run_number=$2
+set -ex

 # Copy the target binary in the correct location expected
 # by the debian/ooniprobe-cli.install file.
 rm -rf ./debian/bin
 mkdir -p ./debian/bin
-machine=$(uname -m)
-goarch=""
-case $machine in
-x86_64)
-	cp ./CLI/linux/amd64/ooniprobe ./debian/bin
-	goarch=amd64
-	;;
-aarch64)
-	cp ./CLI/linux/arm64/ooniprobe ./debian/bin
-	goarch=arm64
-	;;
-*)
-	# TODO(bassosimone): here we probably want to further extend
-	# this script to support at least armv7.
-	echo "FATAL: unsupported machine: $machine" 1>&2
-	exit 1
-	;;
-esac
-
-set -ex
+cp "./CLI/linux/$goarch/ooniprobe" ./debian/bin

 # figure out the version number from the binary itself (which rests
 # on the assumption that we can run such a binary)
@ -68,6 +51,22 @@ $OONI_DEB_DRY_RUN mv ./debian/changelog.oocopy ./debian/changelog
 # of it when using a build container
 $OONI_DEB_DRY_RUN mv ../*.deb .

+darch=""
+case $goarch in
+386)
+	darch="i386"
+	;;
+amd64)
+	darch="amd64"
+	;;
+arm)
+	darch="armhf"
+	;;
+arm64)
+	darch="arm64"
+	;;
+esac
+
 # install the package on the container as a smoke test to
 # ensure that it is installable.
-DEBIAN_FRONTEND=noninteractive dpkg -i "ooniprobe-cli_${version}_${goarch}.deb"
+DEBIAN_FRONTEND=noninteractive dpkg -i "ooniprobe-cli_${version}_${darch}.deb"
--- a/59
+++ b/59
@ -1,5 +1,8 @@
 #!/usr/bin/make -f

+# Many rules in here break if run in parallel.
+.NOTPARALLEL:
+
 #quickhelp: Usage: ./mk [VARIABLE=VALUE ...] TARGET ...
 .PHONY: usage
 usage:
@ -260,9 +263,20 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine
 #help: You can also build the following subtargets:
 .PHONY: ./debian
 ./debian:           \
+	./debian/386    \
 	./debian/amd64  \
+	./debian/arm    \
 	./debian/arm64

+#help:
+#help: * `./mk ./debian/386`: debian/386
+.PHONY:   ./debian/386
+# This extra .PHONY for linux/386 is to help printing targets 🤷.
+.PHONY:     ./CLI/linux/386/ooniprobe
+./debian/386: search/for/docker ./CLI/linux/386/ooniprobe
+	docker pull --platform linux/386 debian:stable
+	docker run --platform linux/386 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian 386 "$(DEBIAN_TILDE_VERSION)"
+
 #help:
 #help: * `./mk ./debian/amd64`: debian/amd64
 .PHONY:   ./debian/amd64
@ -270,7 +284,17 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine
 .PHONY:     ./CLI/linux/amd64/ooniprobe
 ./debian/amd64: search/for/docker ./CLI/linux/amd64/ooniprobe
 	docker pull --platform linux/amd64 debian:stable
-	docker run --platform linux/amd64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian "$(DEBIAN_TILDE_VERSION)"
+	docker run --platform linux/amd64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian amd64 "$(DEBIAN_TILDE_VERSION)"
+
+# Note that we're building for armv7 here
+#help:
+#help: * `./mk ./debian/arm`: debian/arm
+.PHONY:   ./debian/arm
+# This extra .PHONY for linux/arm is to help printing targets 🤷.
+.PHONY:     ./CLI/linux/arm/ooniprobe
+./debian/arm: search/for/docker ./CLI/linux/arm/ooniprobe
+	docker pull --platform linux/arm/v7 debian:stable
+	docker run --platform linux/arm/v7 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian arm "$(DEBIAN_TILDE_VERSION)"

 #help:
 #help: * `./mk ./debian/arm64`: debian/arm64
@ -279,7 +303,7 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine
 .PHONY:     ./CLI/linux/arm64/ooniprobe
 ./debian/arm64: search/for/docker ./CLI/linux/arm64/ooniprobe
 	docker pull --platform linux/arm64 debian:stable
-	docker run --platform linux/arm64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian "$(DEBIAN_TILDE_VERSION)"
+	docker run --platform linux/arm64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian arm64 "$(DEBIAN_TILDE_VERSION)"

 #help:
 #help: The `./mk ./CLI/ooniprobe/linux` command builds the ooniprobe official command
@ -288,16 +312,30 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine
 #help: You can also build the following subtargets:
 .PHONY: ./CLI/ooniprobe/linux
 ./CLI/ooniprobe/linux:               \
+	./CLI/linux/386/ooniprobe.asc    \
 	./CLI/linux/amd64/ooniprobe.asc  \
+	./CLI/linux/arm/ooniprobe.asc    \
 	./CLI/linux/arm64/ooniprobe.asc

+# ./CLI/linux/386/ooniprobe.asc is an internal task for signing.
+.PHONY:   ./CLI/linux/386/ooniprobe.asc
+./CLI/linux/386/ooniprobe.asc: ./CLI/linux/386/ooniprobe
+	rm -f $@ && gpg -abu $(GPG_USER) $<
+
+# Linux builds use Alpine and Docker so we are sure that we are statically
+# linking to musl libc, thus making our binaries extremely portable.
+#help:
+#help: * `./mk ./CLI/linux/386/ooniprobe`: linux/386
+.PHONY:     ./CLI/linux/386/ooniprobe
+./CLI/linux/386/ooniprobe: search/for/docker maybe/copypsiphon
+	docker pull --platform linux/386 $(GOLANG_DOCKER_IMAGE)
+	docker run --platform linux/386 -e GOPATH=/gopath -e GOARCH=386 -v $(GOLANG_DOCKER_GOCACHE)/386:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS)
+
 # ./CLI/linux/amd64/ooniprobe.asc is an internal task for signing.
 .PHONY:   ./CLI/linux/amd64/ooniprobe.asc
 ./CLI/linux/amd64/ooniprobe.asc: ./CLI/linux/amd64/ooniprobe
 	rm -f $@ && gpg -abu $(GPG_USER) $<

-# Linux builds use Alpine and Docker so we are sure that we are statically
-# linking to musl libc, thus making our binaries extremely portable.
 #help:
 #help: * `./mk ./CLI/linux/amd64/ooniprobe`: linux/amd64
 .PHONY:     ./CLI/linux/amd64/ooniprobe
@ -305,6 +343,19 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine
 	docker pull --platform linux/amd64 $(GOLANG_DOCKER_IMAGE)
 	docker run --platform linux/amd64 -e GOPATH=/gopath -e GOARCH=amd64 -v $(GOLANG_DOCKER_GOCACHE)/amd64:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS)

+# ./CLI/linux/arm/ooniprobe.asc is an internal task for signing.
+.PHONY:   ./CLI/linux/arm/ooniprobe.asc
+./CLI/linux/arm/ooniprobe.asc: ./CLI/linux/arm/ooniprobe
+	rm -f $@ && gpg -abu $(GPG_USER) $<
+
+# Note that we're building for armv7 here
+#help:
+#help: * `./mk ./CLI/linux/arm/ooniprobe`: linux/arm
+.PHONY:     ./CLI/linux/arm/ooniprobe
+./CLI/linux/arm/ooniprobe: search/for/docker maybe/copypsiphon
+	docker pull --platform linux/arm/v7 $(GOLANG_DOCKER_IMAGE)
+	docker run --platform linux/arm/v7 -e GOPATH=/gopath -e GOARCH=arm -e GOARM=7 -v $(GOLANG_DOCKER_GOCACHE)/arm:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS)
+
 # ./CLI/linux/arm64/ooniprobe.asc is an internal task for signing.
 .PHONY:   ./CLI/linux/arm64/ooniprobe.asc
 ./CLI/linux/arm64/ooniprobe.asc: ./CLI/linux/arm64/ooniprobe