diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ca2efb8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +indent_style = tab +indent_size = 4 +trim_trailing_whitespace = true + +[*.{py}] +indent_style = space + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ad4e686..a99e8f9 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -5,6 +5,7 @@ on: branches: - "release/**" - "fullbuild" + - "linuxbuild" tags: - "v*" @@ -17,14 +18,23 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: GOCACHE + key: linux-build-cache-386 + - run: make CLI/linux-static-386 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -38,14 +48,23 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: GOCACHE + key: linux-build-cache-amd64 + - run: make CLI/linux-static-amd64 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-amd64 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-amd64 ./CLI/miniooni-linux-amd64 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -58,16 +77,27 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: GOCACHE + key: linux-build-cache-armv6 + - run: make CLI/linux-static-armv6 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv6 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv6 ./CLI/miniooni-linux-armv6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -80,16 +110,27 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: GOCACHE + key: linux-build-cache-armv7 + - run: make CLI/linux-static-armv7 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv7 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv7 ./CLI/miniooni-linux-armv7 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -102,16 +143,27 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: GOCACHE + key: linux-build-cache-arm64 + - run: make CLI/linux-static-arm64 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-arm64 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-arm64 ./CLI/miniooni-linux-arm64 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.vscode/settings.json b/.vscode/settings.json index de288e1..c57221a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,8 @@ { - "python.formatting.provider": "black" -} \ No newline at end of file + "python.formatting.provider": "black", + "gopls": { + "build.directoryFilters": [ + "-GOCACHE" + ] + } +} diff --git a/CLI/.gitignore b/CLI/.gitignore index 34bcffc..93e7f32 100644 --- a/CLI/.gitignore +++ b/CLI/.gitignore @@ -1,2 +1,3 @@ +/Dockerfile /miniooni-* /ooniprobe-* diff --git a/CLI/go-build-alpine b/CLI/go-build-alpine index b1995e7..6ea546c 100755 --- a/CLI/go-build-alpine +++ b/CLI/go-build-alpine @@ -1,15 +1,11 @@ #!/bin/sh set -euxo pipefail -apk update -apk upgrade -apk add --no-progress gcc git linux-headers musl-dev -# We need to force git to look into this repository owned by the -# user outside docker rather than by the user running docker -git config --global --add safe.directory $(pwd) # Some of the following exports are redundant but are however # useful because they provide explicit logging export CGO_ENABLED=1 export GOARM=$GOARM +export GOCACHE=$GOCACHE +export GOMODCACHE=$GOMODCACHE export GOOS=$GOOS export GOARCH=$GOARCH for PACKAGE in $@; do diff --git a/CLI/go-build-linux-static b/CLI/go-build-linux-static index e4f5fc1..3ef177b 100755 --- a/CLI/go-build-linux-static +++ b/CLI/go-build-linux-static @@ -2,11 +2,14 @@ set -euo pipefail -if [[ $# -lt 2 ]]; then +if [[ $# -lt 3 ]]; then echo "" 1>&2 - echo "Compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2 + echo "Docker-based compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2 echo "" 1>&2 - echo "usage: $0 OONIARCH PACKAGE..." 1>&2 + echo "usage: $0 OOGOCACHEDIR OONIARCH PACKAGE..." 1>&2 + echo "" 1>&2 + echo "OOGOCACHEDIR is the directory under which to put GOCACHE and GOMODCACHE for" 1>&2 + echo "this build, which will be mounted and passed to Docker." 1>&2 echo "" 1>&2 echo "OONIARCH must be one of: 386, amd64, arm64, armv6, armv7." 1>&2 echo "" 1>&2 @@ -25,6 +28,8 @@ fi GOLANG_DOCKER_IMAGE=golang:$(cat GOVERSION)-alpine GOOS=linux +OOGOCACHEDIR=$1 +shift OONIARCH=$1 shift @@ -49,12 +54,41 @@ else OONI_PSIPHON_TAGS="" fi +# Implementation note: we must run docker as the user that invokes +# it for actions/cache@v3 to be able to cache OOGOCACHEDIR. This +# constraint forces us to run all privileged operations early +# using a Dockerfile, so the build proper runs as $(id -u):$(id -g). + +GOCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/buildcache +GOMODCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/modcache + +cat > CLI/Dockerfile << EOF +FROM --platform=linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE +RUN apk update +RUN apk upgrade +RUN apk add --no-progress gcc git linux-headers musl-dev +RUN adduser -D -h /home/oobuild -G nobody -u $(id -u) oobuild +ENV HOME=/home/oobuild +EOF + +TAGGED_IMAGE=oobuild-$OONIARCH-$(date +%Y%m%d%H) + +DOCKER_USER_OPTS="--user $(id -u):$(id -g)" + set -x +mkdir -p $GOCACHE $GOMODCACHE + docker pull --platform linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE -docker run --platform linux/$DOCKER_ARCH \ +if ! docker inspect --type=image $TAGGED_IMAGE 1>/dev/null 2>/dev/null; then + docker build --platform linux/$DOCKER_ARCH -t $TAGGED_IMAGE CLI +fi + +docker run --platform linux/$DOCKER_ARCH $DOCKER_USER_OPTS \ + -e GOCACHE=/__gocache -e GOMODCACHE=/__gomodcache \ + -v "$GOCACHE:/__gocache" -v "$GOMODCACHE:/__gomodcache" \ -e GOARM=$GOARM -e GOOS=$GOOS -e GOARCH=$GOARCH \ -e OONI_PSIPHON_TAGS=$OONI_PSIPHON_TAGS \ -e OONIARCH=$OONIARCH -e GOLANG_EXTRA_FLAGS="${GOLANG_EXTRA_FLAGS:-}" \ - -v $(pwd):/ooni -w /ooni $GOLANG_DOCKER_IMAGE ./CLI/go-build-alpine "$@" + -v $(pwd):/ooni -w /ooni $TAGGED_IMAGE ./CLI/go-build-alpine "$@" diff --git a/GOCACHE/.gitignore b/GOCACHE/.gitignore new file mode 100644 index 0000000..61ce03b --- /dev/null +++ b/GOCACHE/.gitignore @@ -0,0 +1 @@ +/oonibuild diff --git a/GOCACHE/README.md b/GOCACHE/README.md new file mode 100644 index 0000000..2316f87 --- /dev/null +++ b/GOCACHE/README.md @@ -0,0 +1,8 @@ +# Directory GOCACHE + +This directory contains the GOCACHE and GOMODCACHE we use when +statically compiling Linux binaries using Docker. + +If you keep the content of this directory, subsequent builds will be +faster. You will notice this especially for builds using qemu-user-static +to build for different architectures. diff --git a/Makefile b/Makefile index a87514b..95784c3 100644 --- a/Makefile +++ b/Makefile @@ -26,19 +26,26 @@ help: #help: on the command line as a key-value pairs (see usage above). #help: -#help: * GIT_CLONE_DIR : directory where to clone repositories, by default -#help: set to `$HOME/.ooniprobe-build/src`. +#help: * GIT_CLONE_DIR : directory where to clone repositories, by default +#help: set to `$HOME/.ooniprobe-build/src`. GIT_CLONE_DIR = $(HOME)/.ooniprobe-build/src #help: -#help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause -#help: the build to embed a psiphon configuration file -#help: into the generated binaries. This build tag -#help: implies cloning the git@github.com:ooni/probe-private -#help: repository. If you do not have the permission to -#help: clone it, just clear this variable, e.g.: +#help: * OONI_GO_DOCKER_GOCACHE : base directory to put GOMODCACHE and GOCACHE +#help: when building using Docker. By default this +#help: is set to `$HOME/.ooniprobe-build/cache` #help: -#help: make OONI_PSIPHON_TAGS="" CLI/miniooni +OONI_GO_DOCKER_GOCACHE = $$(pwd)/GOCACHE + +#help: +#help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause +#help: the build to embed a psiphon configuration file +#help: into the generated binaries. This build tag +#help: implies cloning the git@github.com:ooni/probe-private +#help: repository. If you do not have the permission to +#help: clone it, just clear this variable, e.g.: +#help: +#help: make OONI_PSIPHON_TAGS="" CLI/miniooni OONI_PSIPHON_TAGS = ooni_psiphon_config #quickhelp: @@ -92,35 +99,35 @@ CLI/darwin: search/for/go maybe/copypsiphon #help: ooniprobe and miniooni binaries for linux/386. .PHONY: CLI/linux-static-386 CLI/linux-static-386: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static 386 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) 386 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/linux-static-amd64` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/amd64. .PHONY: CLI/linux-static-amd64 CLI/linux-static-amd64: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static amd64 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) amd64 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/linux-static-armv6` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm/v6. .PHONY: CLI/linux-static-armv6 CLI/linux-static-armv6: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static armv6 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv6 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/linux-static-armv7` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm/v7. .PHONY: CLI/linux-static-armv7 CLI/linux-static-armv7: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static armv7 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv7 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/linux-static-arm64` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm64. .PHONY: CLI/linux-static-arm64 CLI/linux-static-arm64: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static arm64 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) arm64 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/windows` command builds the ooniprobe and miniooni