From f77f54bcc97724b0931648d4fb00c1d0370c9cd2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 11 May 2021 20:16:27 +0200 Subject: [PATCH] feat: build and publish debian/{armhf,i386} (#344) * feat: build and publish debian/{armhf,i386} Part of https://github.com/ooni/probe/issues/807 * zap temporary build --- .github/workflows/linux.yml | 51 +++++++++++++++++++++++--------- CLI/linux/build | 3 ++ CLI/linux/debian | 47 +++++++++++++++-------------- mk | 59 ++++++++++++++++++++++++++++++++++--- 4 files changed, 118 insertions(+), 42 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 97e9231..7c4a431 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -6,18 +6,28 @@ on: - "release/**" jobs: + build_386: + runs-on: "ubuntu-20.04" + steps: + - uses: actions/checkout@v2 + - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/386 + - run: ./smoketest.sh ./CLI/linux/386/ooniprobe + - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd + - run: | + for deb in *.deb; do + ./.github/workflows/debops-ci --arch i386 --show-commands upload --bucket-name ooni-deb $deb + done + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }} + build_amd64: runs-on: "ubuntu-20.04" steps: - - run: | - echo $'{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json - sudo service docker restart - uses: actions/checkout@v2 - - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/linux/amd64/ooniprobe - env: - DOCKER_CLI_EXPERIMENTAL: enabled - - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64 + - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd - run: | for deb in *.deb; do @@ -28,20 +38,33 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }} - build_arm64: + build_arm: runs-on: "ubuntu-20.04" steps: - - run: | - echo $'{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json - sudo service docker restart - uses: actions/checkout@v2 - run: sudo apt-get update -q - run: sudo apt-get install -y qemu-user-static - - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/linux/arm64/ooniprobe + - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm + - run: ./smoketest.sh ./CLI/linux/arm/ooniprobe + - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd + - run: | + for deb in *.deb; do + ./.github/workflows/debops-ci --arch armhf --show-commands upload --bucket-name ooni-deb $deb + done env: - DOCKER_CLI_EXPERIMENTAL: enabled - - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }} + + + build_arm64: + runs-on: "ubuntu-20.04" + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64 + - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd - run: | for deb in *.deb; do diff --git a/CLI/linux/build b/CLI/linux/build index 6235a22..e1f64bc 100755 --- a/CLI/linux/build +++ b/CLI/linux/build @@ -11,6 +11,9 @@ set -x apk update apk upgrade apk add --no-progress gcc git linux-headers musl-dev +# some of the following exports are redundant but are however +# useful because they provide explicit logging +export GOARM=$GOARM export GOPATH=$GOPATH export CGO_ENABLED=1 export GOOS=linux diff --git a/CLI/linux/debian b/CLI/linux/debian index 78e1090..57377f9 100755 --- a/CLI/linux/debian +++ b/CLI/linux/debian @@ -10,36 +10,19 @@ # 2. we are running on a debian system that has the same # architecture of the `ooniprobe` we are packaging. -if [ $# -gt 1 ]; then - echo "usage: $0 [run_number]" 1>&2 +if [ $# -ne 1 ] && [ $# -ne 2 ]; then + echo "usage: $0 {arch} [run_number]" 1>&2 exit 1 fi -run_number=$1 +goarch=$1 +run_number=$2 +set -ex # Copy the target binary in the correct location expected # by the debian/ooniprobe-cli.install file. rm -rf ./debian/bin mkdir -p ./debian/bin -machine=$(uname -m) -goarch="" -case $machine in -x86_64) - cp ./CLI/linux/amd64/ooniprobe ./debian/bin - goarch=amd64 - ;; -aarch64) - cp ./CLI/linux/arm64/ooniprobe ./debian/bin - goarch=arm64 - ;; -*) - # TODO(bassosimone): here we probably want to further extend - # this script to support at least armv7. - echo "FATAL: unsupported machine: $machine" 1>&2 - exit 1 - ;; -esac - -set -ex +cp "./CLI/linux/$goarch/ooniprobe" ./debian/bin # figure out the version number from the binary itself (which rests # on the assumption that we can run such a binary) @@ -68,6 +51,22 @@ $OONI_DEB_DRY_RUN mv ./debian/changelog.oocopy ./debian/changelog # of it when using a build container $OONI_DEB_DRY_RUN mv ../*.deb . +darch="" +case $goarch in +386) + darch="i386" + ;; +amd64) + darch="amd64" + ;; +arm) + darch="armhf" + ;; +arm64) + darch="arm64" + ;; +esac + # install the package on the container as a smoke test to # ensure that it is installable. -DEBIAN_FRONTEND=noninteractive dpkg -i "ooniprobe-cli_${version}_${goarch}.deb" +DEBIAN_FRONTEND=noninteractive dpkg -i "ooniprobe-cli_${version}_${darch}.deb" diff --git a/mk b/mk index b8503b6..d4a128a 100755 --- a/mk +++ b/mk @@ -1,5 +1,8 @@ #!/usr/bin/make -f +# Many rules in here break if run in parallel. +.NOTPARALLEL: + #quickhelp: Usage: ./mk [VARIABLE=VALUE ...] TARGET ... .PHONY: usage usage: @@ -260,9 +263,20 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine #help: You can also build the following subtargets: .PHONY: ./debian ./debian: \ + ./debian/386 \ ./debian/amd64 \ + ./debian/arm \ ./debian/arm64 +#help: +#help: * `./mk ./debian/386`: debian/386 +.PHONY: ./debian/386 +# This extra .PHONY for linux/386 is to help printing targets 🤷. +.PHONY: ./CLI/linux/386/ooniprobe +./debian/386: search/for/docker ./CLI/linux/386/ooniprobe + docker pull --platform linux/386 debian:stable + docker run --platform linux/386 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian 386 "$(DEBIAN_TILDE_VERSION)" + #help: #help: * `./mk ./debian/amd64`: debian/amd64 .PHONY: ./debian/amd64 @@ -270,7 +284,17 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine .PHONY: ./CLI/linux/amd64/ooniprobe ./debian/amd64: search/for/docker ./CLI/linux/amd64/ooniprobe docker pull --platform linux/amd64 debian:stable - docker run --platform linux/amd64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian "$(DEBIAN_TILDE_VERSION)" + docker run --platform linux/amd64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian amd64 "$(DEBIAN_TILDE_VERSION)" + +# Note that we're building for armv7 here +#help: +#help: * `./mk ./debian/arm`: debian/arm +.PHONY: ./debian/arm +# This extra .PHONY for linux/arm is to help printing targets 🤷. +.PHONY: ./CLI/linux/arm/ooniprobe +./debian/arm: search/for/docker ./CLI/linux/arm/ooniprobe + docker pull --platform linux/arm/v7 debian:stable + docker run --platform linux/arm/v7 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian arm "$(DEBIAN_TILDE_VERSION)" #help: #help: * `./mk ./debian/arm64`: debian/arm64 @@ -279,7 +303,7 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine .PHONY: ./CLI/linux/arm64/ooniprobe ./debian/arm64: search/for/docker ./CLI/linux/arm64/ooniprobe docker pull --platform linux/arm64 debian:stable - docker run --platform linux/arm64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian "$(DEBIAN_TILDE_VERSION)" + docker run --platform linux/arm64 -v $(shell pwd):/ooni -w /ooni debian:stable ./CLI/linux/debian arm64 "$(DEBIAN_TILDE_VERSION)" #help: #help: The `./mk ./CLI/ooniprobe/linux` command builds the ooniprobe official command @@ -288,16 +312,30 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine #help: You can also build the following subtargets: .PHONY: ./CLI/ooniprobe/linux ./CLI/ooniprobe/linux: \ + ./CLI/linux/386/ooniprobe.asc \ ./CLI/linux/amd64/ooniprobe.asc \ + ./CLI/linux/arm/ooniprobe.asc \ ./CLI/linux/arm64/ooniprobe.asc +# ./CLI/linux/386/ooniprobe.asc is an internal task for signing. +.PHONY: ./CLI/linux/386/ooniprobe.asc +./CLI/linux/386/ooniprobe.asc: ./CLI/linux/386/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +# Linux builds use Alpine and Docker so we are sure that we are statically +# linking to musl libc, thus making our binaries extremely portable. +#help: +#help: * `./mk ./CLI/linux/386/ooniprobe`: linux/386 +.PHONY: ./CLI/linux/386/ooniprobe +./CLI/linux/386/ooniprobe: search/for/docker maybe/copypsiphon + docker pull --platform linux/386 $(GOLANG_DOCKER_IMAGE) + docker run --platform linux/386 -e GOPATH=/gopath -e GOARCH=386 -v $(GOLANG_DOCKER_GOCACHE)/386:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS) + # ./CLI/linux/amd64/ooniprobe.asc is an internal task for signing. .PHONY: ./CLI/linux/amd64/ooniprobe.asc ./CLI/linux/amd64/ooniprobe.asc: ./CLI/linux/amd64/ooniprobe rm -f $@ && gpg -abu $(GPG_USER) $< -# Linux builds use Alpine and Docker so we are sure that we are statically -# linking to musl libc, thus making our binaries extremely portable. #help: #help: * `./mk ./CLI/linux/amd64/ooniprobe`: linux/amd64 .PHONY: ./CLI/linux/amd64/ooniprobe @@ -305,6 +343,19 @@ GOLANG_DOCKER_IMAGE = golang:$(GOLANG_VERSION_NUMBER)-alpine docker pull --platform linux/amd64 $(GOLANG_DOCKER_IMAGE) docker run --platform linux/amd64 -e GOPATH=/gopath -e GOARCH=amd64 -v $(GOLANG_DOCKER_GOCACHE)/amd64:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS) +# ./CLI/linux/arm/ooniprobe.asc is an internal task for signing. +.PHONY: ./CLI/linux/arm/ooniprobe.asc +./CLI/linux/arm/ooniprobe.asc: ./CLI/linux/arm/ooniprobe + rm -f $@ && gpg -abu $(GPG_USER) $< + +# Note that we're building for armv7 here +#help: +#help: * `./mk ./CLI/linux/arm/ooniprobe`: linux/arm +.PHONY: ./CLI/linux/arm/ooniprobe +./CLI/linux/arm/ooniprobe: search/for/docker maybe/copypsiphon + docker pull --platform linux/arm/v7 $(GOLANG_DOCKER_IMAGE) + docker run --platform linux/arm/v7 -e GOPATH=/gopath -e GOARCH=arm -e GOARM=7 -v $(GOLANG_DOCKER_GOCACHE)/arm:/root/.cache/go-build -v $(GOLANG_DOCKER_GOPATH):/gopath -v $(shell pwd):/ooni -w /ooni $(GOLANG_DOCKER_IMAGE) ./CLI/linux/build -tags=netgo,$(OONI_PSIPHON_TAGS) $(GOLANG_EXTRA_FLAGS) + # ./CLI/linux/arm64/ooniprobe.asc is an internal task for signing. .PHONY: ./CLI/linux/arm64/ooniprobe.asc ./CLI/linux/arm64/ooniprobe.asc: ./CLI/linux/arm64/ooniprobe