c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: attempt to make linux builds faster (#911)

Browse Source 
This work aims to make Linux builds faster to make https://github.com/ooni/probe/issues/2249 more convenient. Since those builds runs inside Docker, the problem to solve here is to save/restore the Go caches notwithstanding Docker. Because Docker runs as root, we need to modify the build a bit to run as a normal user. Otherwise, we will not be able to save the Go cache using actions/cache@v3. (Other approaches such as using `sudo` are possible but running the build as an unprivileged user actually looks cleaner, so I chose to do that.) While there, add a `.editorconfig`.
This commit is contained in:
Simone Basso 2022-08-30 21:13:33 +02:00 committed by GitHub
parent d10ab88444
commit 196ac55493
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 146 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.editorconfig Normal file
View File

@ -0,0 +1,15 @@
root = true
[*]
end_of_line = lf
insert_final_newline = true
indent_style = tab
indent_size = 4
trim_trailing_whitespace = true
[*.{py}]
indent_style = space
[*.{yml,yaml}]
indent_style = space
indent_size = 2

52
.github/workflows/linux.yml vendored
View File

@ -5,6 +5,7 @@ on:
branches:
- "release/**"
- "fullbuild"
- "linuxbuild"
tags:
- "v*"
@ -17,14 +18,23 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
env:
PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- uses: actions/cache@v3
with:
path: GOCACHE
key: linux-build-cache-386
- run: make CLI/linux-static-386
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -38,14 +48,23 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
env:
PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- uses: actions/cache@v3
with:
path: GOCACHE
key: linux-build-cache-amd64
- run: make CLI/linux-static-amd64
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-amd64
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-amd64 ./CLI/miniooni-linux-amd64
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -58,16 +77,27 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- run: sudo apt-get update -q
- run: sudo apt-get install -y qemu-user-static
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
env:
PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- uses: actions/cache@v3
with:
path: GOCACHE
key: linux-build-cache-armv6
- run: make CLI/linux-static-armv6
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv6
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv6 ./CLI/miniooni-linux-armv6
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -80,16 +110,27 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- run: sudo apt-get update -q
- run: sudo apt-get install -y qemu-user-static
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
env:
PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- uses: actions/cache@v3
with:
path: GOCACHE
key: linux-build-cache-armv7
- run: make CLI/linux-static-armv7
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv7
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv7 ./CLI/miniooni-linux-armv7
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -102,16 +143,27 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- run: sudo apt-get update -q
- run: sudo apt-get install -y qemu-user-static
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age
env:
PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- uses: actions/cache@v3
with:
path: GOCACHE
key: linux-build-cache-arm64
- run: make CLI/linux-static-arm64
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-arm64
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-arm64 ./CLI/miniooni-linux-arm64
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

9
.vscode/settings.json vendored
View File

@ -1,3 +1,8 @@
{
"python.formatting.provider": "black"
}
"python.formatting.provider": "black",
"gopls": {
"build.directoryFilters": [
"-GOCACHE"
]
}
}

1
CLI/.gitignore vendored
View File

@ -1,2 +1,3 @@
/Dockerfile
/miniooni-*
/ooniprobe-*

8
CLI/go-build-alpine
View File

@ -1,15 +1,11 @@
#!/bin/sh
set -euxo pipefail
apk update
apk upgrade
apk add --no-progress gcc git linux-headers musl-dev
# We need to force git to look into this repository owned by the
# user outside docker rather than by the user running docker
git config --global --add safe.directory $(pwd)
# Some of the following exports are redundant but are however
# useful because they provide explicit logging
export CGO_ENABLED=1
export GOARM=$GOARM
export GOCACHE=$GOCACHE
export GOMODCACHE=$GOMODCACHE
export GOOS=$GOOS
export GOARCH=$GOARCH
for PACKAGE in $@; do

44
CLI/go-build-linux-static
View File

@ -2,11 +2,14 @@
set -euo pipefail
if [[ $# -lt 2 ]]; then
if [[ $# -lt 3 ]]; then
echo "" 1>&2
echo "Compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2
echo "Docker-based compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2
echo "" 1>&2
echo "usage: $0 OONIARCH PACKAGE..." 1>&2
echo "usage: $0 OOGOCACHEDIR OONIARCH PACKAGE..." 1>&2
echo "" 1>&2
echo "OOGOCACHEDIR is the directory under which to put GOCACHE and GOMODCACHE for" 1>&2
echo "this build, which will be mounted and passed to Docker." 1>&2
echo "" 1>&2
echo "OONIARCH must be one of: 386, amd64, arm64, armv6, armv7." 1>&2
echo "" 1>&2
@ -25,6 +28,8 @@ fi
GOLANG_DOCKER_IMAGE=golang:$(cat GOVERSION)-alpine
GOOS=linux
OOGOCACHEDIR=$1
shift
OONIARCH=$1
shift
@ -49,12 +54,41 @@ else
OONI_PSIPHON_TAGS=""
fi
# Implementation note: we must run docker as the user that invokes
# it for actions/cache@v3 to be able to cache OOGOCACHEDIR. This
# constraint forces us to run all privileged operations early
# using a Dockerfile, so the build proper runs as $(id -u):$(id -g).
GOCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/buildcache
GOMODCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/modcache
cat > CLI/Dockerfile << EOF
FROM --platform=linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE
RUN apk update
RUN apk upgrade
RUN apk add --no-progress gcc git linux-headers musl-dev
RUN adduser -D -h /home/oobuild -G nobody -u $(id -u) oobuild
ENV HOME=/home/oobuild
EOF
TAGGED_IMAGE=oobuild-$OONIARCH-$(date +%Y%m%d%H)
DOCKER_USER_OPTS="--user $(id -u):$(id -g)"
set -x
mkdir -p $GOCACHE $GOMODCACHE
docker pull --platform linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE
docker run --platform linux/$DOCKER_ARCH \
if ! docker inspect --type=image $TAGGED_IMAGE 1>/dev/null 2>/dev/null; then
docker build --platform linux/$DOCKER_ARCH -t $TAGGED_IMAGE CLI
fi
docker run --platform linux/$DOCKER_ARCH $DOCKER_USER_OPTS \
-e GOCACHE=/__gocache -e GOMODCACHE=/__gomodcache \
-v "$GOCACHE:/__gocache" -v "$GOMODCACHE:/__gomodcache" \
-e GOARM=$GOARM -e GOOS=$GOOS -e GOARCH=$GOARCH \
-e OONI_PSIPHON_TAGS=$OONI_PSIPHON_TAGS \
-e OONIARCH=$OONIARCH -e GOLANG_EXTRA_FLAGS="${GOLANG_EXTRA_FLAGS:-}" \
-v $(pwd):/ooni -w /ooni $GOLANG_DOCKER_IMAGE ./CLI/go-build-alpine "$@"
-v $(pwd):/ooni -w /ooni $TAGGED_IMAGE ./CLI/go-build-alpine "$@"

1
GOCACHE/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/oonibuild

8
GOCACHE/README.md Normal file
View File

@ -0,0 +1,8 @@
# Directory GOCACHE
This directory contains the GOCACHE and GOMODCACHE we use when
statically compiling Linux binaries using Docker.
If you keep the content of this directory, subsequent builds will be
faster. You will notice this especially for builds using qemu-user-static
to build for different architectures.

35
Makefile
View File

@ -26,19 +26,26 @@ help:
#help: on the command line as a key-value pairs (see usage above).
#help:
#help: * GIT_CLONE_DIR : directory where to clone repositories, by default
#help: set to `$HOME/.ooniprobe-build/src`.
#help: * GIT_CLONE_DIR : directory where to clone repositories, by default
#help: set to `$HOME/.ooniprobe-build/src`.
GIT_CLONE_DIR = $(HOME)/.ooniprobe-build/src
#help:
#help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause
#help: the build to embed a psiphon configuration file
#help: into the generated binaries. This build tag
#help: implies cloning the git@github.com:ooni/probe-private
#help: repository. If you do not have the permission to
#help: clone it, just clear this variable, e.g.:
#help: * OONI_GO_DOCKER_GOCACHE : base directory to put GOMODCACHE and GOCACHE
#help: when building using Docker. By default this
#help: is set to `$HOME/.ooniprobe-build/cache`
#help:
#help: make OONI_PSIPHON_TAGS="" CLI/miniooni
OONI_GO_DOCKER_GOCACHE = $$(pwd)/GOCACHE
#help:
#help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause
#help: the build to embed a psiphon configuration file
#help: into the generated binaries. This build tag
#help: implies cloning the git@github.com:ooni/probe-private
#help: repository. If you do not have the permission to
#help: clone it, just clear this variable, e.g.:
#help:
#help: make OONI_PSIPHON_TAGS="" CLI/miniooni
OONI_PSIPHON_TAGS = ooni_psiphon_config
#quickhelp:
@ -92,35 +99,35 @@ CLI/darwin: search/for/go maybe/copypsiphon
#help: ooniprobe and miniooni binaries for linux/386.
.PHONY: CLI/linux-static-386
CLI/linux-static-386: search/for/docker maybe/copypsiphon
./CLI/go-build-linux-static 386 ./cmd/ooniprobe ./internal/cmd/miniooni
./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) 386 ./cmd/ooniprobe ./internal/cmd/miniooni
#help:
#help: The `make CLI/linux-static-amd64` command builds and statically links the
#help: ooniprobe and miniooni binaries for linux/amd64.
.PHONY: CLI/linux-static-amd64
CLI/linux-static-amd64: search/for/docker maybe/copypsiphon
./CLI/go-build-linux-static amd64 ./cmd/ooniprobe ./internal/cmd/miniooni
./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) amd64 ./cmd/ooniprobe ./internal/cmd/miniooni
#help:
#help: The `make CLI/linux-static-armv6` command builds and statically links the
#help: ooniprobe and miniooni binaries for linux/arm/v6.
.PHONY: CLI/linux-static-armv6
CLI/linux-static-armv6: search/for/docker maybe/copypsiphon
./CLI/go-build-linux-static armv6 ./cmd/ooniprobe ./internal/cmd/miniooni
./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv6 ./cmd/ooniprobe ./internal/cmd/miniooni
#help:
#help: The `make CLI/linux-static-armv7` command builds and statically links the
#help: ooniprobe and miniooni binaries for linux/arm/v7.
.PHONY: CLI/linux-static-armv7
CLI/linux-static-armv7: search/for/docker maybe/copypsiphon
./CLI/go-build-linux-static armv7 ./cmd/ooniprobe ./internal/cmd/miniooni
./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv7 ./cmd/ooniprobe ./internal/cmd/miniooni
#help:
#help: The `make CLI/linux-static-arm64` command builds and statically links the
#help: ooniprobe and miniooni binaries for linux/arm64.
.PHONY: CLI/linux-static-arm64
CLI/linux-static-arm64: search/for/docker maybe/copypsiphon
./CLI/go-build-linux-static arm64 ./cmd/ooniprobe ./internal/cmd/miniooni
./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) arm64 ./cmd/ooniprobe ./internal/cmd/miniooni
#help:
#help: The `make CLI/windows` command builds the ooniprobe and miniooni