fix: attempt to make linux builds faster (#911)

This work aims to make Linux builds faster to make https://github.com/ooni/probe/issues/2249 more convenient. Since those builds runs inside Docker, the problem to solve here is to save/restore the Go caches notwithstanding Docker. Because Docker runs as root, we need to modify the build a bit to run as a normal user. Otherwise, we will not be able to save the Go cache using actions/cache@v3. (Other approaches such as using `sudo` are possible but running the build as an unprivileged user actually looks cleaner, so I chose to do that.) While there, add a `.editorconfig`.

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = tab
+indent_size = 4
+trim_trailing_whitespace = true
+
+[*.{py}]
+indent_style = space
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2

.github/workflows/linux.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - "release/**"
       - "fullbuild"
+      - "linuxbuild"
     tags:
       - "v*"
 
@@ -17,14 +18,23 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
           echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
+
+      - uses: actions/cache@v3
+        with:
+          path: GOCACHE
+          key: linux-build-cache-386
+
       - run: make CLI/linux-static-386
+
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386
+
       - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -38,14 +48,23 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
           echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
+
+      - uses: actions/cache@v3
+        with:
+          path: GOCACHE
+          key: linux-build-cache-amd64
+
       - run: make CLI/linux-static-amd64
+
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-amd64
+
       - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-amd64 ./CLI/miniooni-linux-amd64
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -58,16 +77,27 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
       - run: sudo apt-get update -q
+
       - run: sudo apt-get install -y qemu-user-static
+
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
           echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
+
+      - uses: actions/cache@v3
+        with:
+          path: GOCACHE
+          key: linux-build-cache-armv6
+
       - run: make CLI/linux-static-armv6
+
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv6
+
       - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv6 ./CLI/miniooni-linux-armv6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -80,16 +110,27 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
       - run: sudo apt-get update -q
+
       - run: sudo apt-get install -y qemu-user-static
+
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
           echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
+
+      - uses: actions/cache@v3
+        with:
+          path: GOCACHE
+          key: linux-build-cache-armv7
+
       - run: make CLI/linux-static-armv7
+
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv7
+
       - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv7 ./CLI/miniooni-linux-armv7
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -102,16 +143,27 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
       - run: sudo apt-get update -q
+
       - run: sudo apt-get install -y qemu-user-static
+
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
           echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
+
+      - uses: actions/cache@v3
+        with:
+          path: GOCACHE
+          key: linux-build-cache-arm64
+
       - run: make CLI/linux-static-arm64
+
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-arm64
+
       - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-arm64 ./CLI/miniooni-linux-arm64
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.vscode/settings.json

@@ -1,3 +1,8 @@
 {
-    "python.formatting.provider": "black"
+    "python.formatting.provider": "black",
-}
+    "gopls": {
+        "build.directoryFilters": [
+            "-GOCACHE"
+        ]
+    }
+}

CLI/.gitignore

@@ -1,2 +1,3 @@
+/Dockerfile
 /miniooni-*
 /ooniprobe-*

CLI/go-build-alpine

@@ -1,15 +1,11 @@
 #!/bin/sh
 set -euxo pipefail
-apk update
-apk upgrade
-apk add --no-progress gcc git linux-headers musl-dev
-# We need to force git to look into this repository owned by the
-# user outside docker rather than by the user running docker
-git config --global --add safe.directory $(pwd)
 # Some of the following exports are redundant but are however
 # useful because they provide explicit logging
 export CGO_ENABLED=1
 export GOARM=$GOARM
+export GOCACHE=$GOCACHE
+export GOMODCACHE=$GOMODCACHE
 export GOOS=$GOOS
 export GOARCH=$GOARCH
 for PACKAGE in $@; do

CLI/go-build-linux-static

@@ -2,11 +2,14 @@
 
 set -euo pipefail
 
-if [[ $# -lt 2 ]]; then
+if [[ $# -lt 3 ]]; then
 	echo "" 1>&2
-	echo "Compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2
+	echo "Docker-based compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2
 	echo "" 1>&2
-	echo "usage: $0 OONIARCH PACKAGE..." 1>&2
+	echo "usage: $0 OOGOCACHEDIR OONIARCH PACKAGE..." 1>&2
+	echo "" 1>&2
+	echo "OOGOCACHEDIR is the directory under which to put GOCACHE and GOMODCACHE for" 1>&2
+	echo "this build, which will be mounted and passed to Docker." 1>&2
 	echo "" 1>&2
 	echo "OONIARCH must be one of: 386, amd64, arm64, armv6, armv7." 1>&2
 	echo "" 1>&2
@@ -25,6 +28,8 @@ fi
 
 GOLANG_DOCKER_IMAGE=golang:$(cat GOVERSION)-alpine
 GOOS=linux
+OOGOCACHEDIR=$1
+shift
 OONIARCH=$1
 shift
 
@@ -49,12 +54,41 @@ else
 	OONI_PSIPHON_TAGS=""
 fi
 
+# Implementation note: we must run docker as the user that invokes
+# it for actions/cache@v3 to be able to cache OOGOCACHEDIR. This
+# constraint forces us to run all privileged operations early
+# using a Dockerfile, so the build proper runs as $(id -u):$(id -g).
+
+GOCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/buildcache
+GOMODCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/modcache
+
+cat > CLI/Dockerfile << EOF
+FROM --platform=linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE
+RUN apk update
+RUN apk upgrade
+RUN apk add --no-progress gcc git linux-headers musl-dev
+RUN adduser -D -h /home/oobuild -G nobody -u $(id -u) oobuild
+ENV HOME=/home/oobuild
+EOF
+
+TAGGED_IMAGE=oobuild-$OONIARCH-$(date +%Y%m%d%H)
+
+DOCKER_USER_OPTS="--user $(id -u):$(id -g)"
+
 set -x
 
+mkdir -p $GOCACHE $GOMODCACHE
+
 docker pull --platform linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE
 
-docker run --platform linux/$DOCKER_ARCH \
+if ! docker inspect --type=image $TAGGED_IMAGE 1>/dev/null 2>/dev/null; then
+	docker build --platform linux/$DOCKER_ARCH -t $TAGGED_IMAGE CLI
+fi
+
+docker run --platform linux/$DOCKER_ARCH $DOCKER_USER_OPTS \
+	-e GOCACHE=/__gocache -e GOMODCACHE=/__gomodcache \
+	-v "$GOCACHE:/__gocache" -v "$GOMODCACHE:/__gomodcache" \
 	-e GOARM=$GOARM -e GOOS=$GOOS -e GOARCH=$GOARCH \
 	-e OONI_PSIPHON_TAGS=$OONI_PSIPHON_TAGS \
 	-e OONIARCH=$OONIARCH -e GOLANG_EXTRA_FLAGS="${GOLANG_EXTRA_FLAGS:-}" \
-	-v $(pwd):/ooni -w /ooni $GOLANG_DOCKER_IMAGE ./CLI/go-build-alpine "$@"
+	-v $(pwd):/ooni -w /ooni $TAGGED_IMAGE ./CLI/go-build-alpine "$@"

GOCACHE/.gitignore

@@ -0,0 +1 @@
+/oonibuild

GOCACHE/README.md

@@ -0,0 +1,8 @@
+# Directory GOCACHE
+
+This directory contains the GOCACHE and GOMODCACHE we use when
+statically compiling Linux binaries using Docker.
+
+If you keep the content of this directory, subsequent builds will be
+faster. You will notice this especially for builds using qemu-user-static
+to build for different architectures.

Makefile

@@ -26,19 +26,26 @@ help:
 #help: on the command line as a key-value pairs (see usage above).
 
 #help:
-#help: * GIT_CLONE_DIR         : directory where to clone repositories, by default
+#help: * GIT_CLONE_DIR           : directory where to clone repositories, by default
-#help:                           set to `$HOME/.ooniprobe-build/src`.
+#help:                             set to `$HOME/.ooniprobe-build/src`.
 GIT_CLONE_DIR = $(HOME)/.ooniprobe-build/src
 
 #help:
-#help: * OONI_PSIPHON_TAGS     : build tags for `go build -tags ...` that cause
+#help: * OONI_GO_DOCKER_GOCACHE  : base directory to put GOMODCACHE and GOCACHE
-#help:                           the build to embed a psiphon configuration file
+#help:                             when building using Docker. By default this
-#help:                           into the generated binaries. This build tag
+#help:                             is set to `$HOME/.ooniprobe-build/cache`
-#help:                           implies cloning the git@github.com:ooni/probe-private
-#help:                           repository. If you do not have the permission to
-#help:                           clone it, just clear this variable, e.g.:
 #help:
-#help:                               make OONI_PSIPHON_TAGS="" CLI/miniooni
+OONI_GO_DOCKER_GOCACHE = $$(pwd)/GOCACHE
+
+#help:
+#help: * OONI_PSIPHON_TAGS       : build tags for `go build -tags ...` that cause
+#help:                             the build to embed a psiphon configuration file
+#help:                             into the generated binaries. This build tag
+#help:                             implies cloning the git@github.com:ooni/probe-private
+#help:                             repository. If you do not have the permission to
+#help:                             clone it, just clear this variable, e.g.:
+#help:
+#help:                                 make OONI_PSIPHON_TAGS="" CLI/miniooni
 OONI_PSIPHON_TAGS = ooni_psiphon_config
 
 #quickhelp:
@@ -92,35 +99,35 @@ CLI/darwin: search/for/go maybe/copypsiphon
 #help: ooniprobe and miniooni binaries for linux/386.
 .PHONY: CLI/linux-static-386
 CLI/linux-static-386: search/for/docker maybe/copypsiphon
-	./CLI/go-build-linux-static 386 ./cmd/ooniprobe ./internal/cmd/miniooni
+	./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) 386 ./cmd/ooniprobe ./internal/cmd/miniooni
 
 #help:
 #help: The `make CLI/linux-static-amd64` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/amd64.
 .PHONY: CLI/linux-static-amd64
 CLI/linux-static-amd64: search/for/docker maybe/copypsiphon
-	./CLI/go-build-linux-static amd64 ./cmd/ooniprobe ./internal/cmd/miniooni
+	./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) amd64 ./cmd/ooniprobe ./internal/cmd/miniooni
 
 #help:
 #help: The `make CLI/linux-static-armv6` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/arm/v6.
 .PHONY: CLI/linux-static-armv6
 CLI/linux-static-armv6: search/for/docker maybe/copypsiphon
-	./CLI/go-build-linux-static armv6 ./cmd/ooniprobe ./internal/cmd/miniooni
+	./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv6 ./cmd/ooniprobe ./internal/cmd/miniooni
 
 #help:
 #help: The `make CLI/linux-static-armv7` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/arm/v7.
 .PHONY: CLI/linux-static-armv7
 CLI/linux-static-armv7: search/for/docker maybe/copypsiphon
-	./CLI/go-build-linux-static armv7 ./cmd/ooniprobe ./internal/cmd/miniooni
+	./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv7 ./cmd/ooniprobe ./internal/cmd/miniooni
 
 #help:
 #help: The `make CLI/linux-static-arm64` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/arm64.
 .PHONY: CLI/linux-static-arm64
 CLI/linux-static-arm64: search/for/docker maybe/copypsiphon
-	./CLI/go-build-linux-static arm64 ./cmd/ooniprobe ./internal/cmd/miniooni
+	./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) arm64 ./cmd/ooniprobe ./internal/cmd/miniooni
 
 #help:
 #help: The `make CLI/windows` command builds the ooniprobe and miniooni