feat: implement rolling builds (#910)

This diff modifies all the github actions that produce assets to
publish on a release called rolling when we are not building a tag.

If everything goes as planned, we should be able to provide
people with automatically generated fresh binaries for testing.

While there, introduce caching for all builds to make them
as fast as possible. I suspect gomobile based builds will not
see any speed up but other builds most likely will.

See https://github.com/ooni/probe/issues/2249

.github/workflows/alltests.yml

@@ -16,8 +16,9 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-alltests-${{ steps.goversion.outputs.version }}"
 
       - run: go test -race -tags shaping ./...

.github/workflows/android.yml

@@ -25,9 +25,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-android-oonimkall-${{ steps.goversion.outputs.version }}"
 
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@@ -39,12 +40,9 @@ jobs:
       - run: make MOBILE/android
 
       - run: |
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          ./script/ghpublish.bash ./MOBILE/android/oonimkall.aar \
-          gh release create -p $tag --target $GITHUB_SHA || true
+                                  ./MOBILE/android/oonimkall-sources.jar \
-          gh release upload $tag --clobber ./MOBILE/android/oonimkall.aar \
+                                  ./MOBILE/android/oonimkall.pom
-                                           ./MOBILE/android/oonimkall-sources.jar \
-                                           ./MOBILE/android/oonimkall.pom
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -62,9 +60,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-android-cli-386-${{ steps.goversion.outputs.version }}"
 
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@@ -76,11 +75,8 @@ jobs:
       - run: make CLI/android-386
 
       - run: |
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          ./script/ghpublish.bash ./CLI/miniooni-android-386 \
-          gh release create -p $tag --target $GITHUB_SHA || true
+                                  ./CLI/ooniprobe-android-386
-          gh release upload $tag --clobber ./CLI/miniooni-android-386 \
-                                           ./CLI/ooniprobe-android-386
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -98,9 +94,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-android-cli-amd64-${{ steps.goversion.outputs.version }}"
 
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@@ -112,11 +109,8 @@ jobs:
       - run: make CLI/android-amd64
 
       - run: |
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          ./script/ghpublish.bash ./CLI/miniooni-android-amd64 \
-          gh release create -p $tag --target $GITHUB_SHA || true
+                                  ./CLI/ooniprobe-android-amd64
-          gh release upload $tag --clobber ./CLI/miniooni-android-amd64 \
-                                           ./CLI/ooniprobe-android-amd64
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -134,9 +128,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-android-cli-arm-${{ steps.goversion.outputs.version }}"
 
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@@ -148,11 +143,8 @@ jobs:
       - run: make CLI/android-arm
 
       - run: |
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          ./script/ghpublish.bash ./CLI/miniooni-android-arm \
-          gh release create -p $tag --target $GITHUB_SHA || true
+                                  ./CLI/ooniprobe-android-arm
-          gh release upload $tag --clobber ./CLI/miniooni-android-arm \
-                                           ./CLI/ooniprobe-android-arm
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -170,9 +162,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-android-cli-arm64-${{ steps.goversion.outputs.version }}"
 
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@@ -184,10 +177,7 @@ jobs:
       - run: make CLI/android-arm64
 
       - run: |
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          ./script/ghpublish.bash ./CLI/miniooni-android-arm64 \
-          gh release create -p $tag --target $GITHUB_SHA || true
+                                  ./CLI/ooniprobe-android-arm64
-          gh release upload $tag --clobber ./CLI/miniooni-android-arm64 \
-                                           ./CLI/ooniprobe-android-arm64
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/generate.yml

@@ -16,8 +16,9 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-generate-${{ steps.goversion.outputs.version }}"
 
       - run: go generate ./...

.github/workflows/gosec.yml

@@ -20,9 +20,10 @@ jobs:
       id: goversion
       run: echo ::set-output name=version::$(cat GOVERSION)
 
-    - uses: actions/setup-go@v3
+    - uses: magnetikonline/action-golang-cache@v2
       with:
         go-version: "${{ steps.goversion.outputs.version }}"
+        cache-key-suffix: "-gosec-${{ steps.goversion.outputs.version }}"
 
     - name: Run Gosec security scanner
       continue-on-error: true # TODO(https://github.com/ooni/probe/issues/2180)

.github/workflows/ios.yml

@@ -23,9 +23,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-ios-${{ steps.goversion.outputs.version }}"
 
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@@ -37,10 +38,7 @@ jobs:
       - run: make EXPECTED_XCODE_VERSION=12.4 MOBILE/ios
 
       - run: |
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          ./script/ghpublish.bash ./MOBILE/ios/oonimkall.xcframework.zip \
-          gh release create -p $tag --target $GITHUB_SHA || true
+                                  ./MOBILE/ios/oonimkall.podspec
-          gh release upload $tag --clobber ./MOBILE/ios/oonimkall.xcframework.zip \
-                                           ./MOBILE/ios/oonimkall.podspec
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/jafar.yml

@@ -16,9 +16,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-jafar-${{ steps.goversion.outputs.version }}"
 
       - run: go build -v ./internal/cmd/jafar
 

.github/workflows/linux.yml

@@ -25,11 +25,7 @@ jobs:
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
       - run: make CLI/linux-static-386
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386
-      - run: |
+      - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
-          gh release create -p $tag --target $GITHUB_SHA || true
-          gh release upload $tag --clobber ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -50,11 +46,7 @@ jobs:
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
       - run: make CLI/linux-static-amd64
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-amd64
-      - run: |
+      - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-amd64 ./CLI/miniooni-linux-amd64
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
-          gh release create -p $tag --target $GITHUB_SHA || true
-          gh release upload $tag --clobber ./CLI/ooniprobe-linux-amd64 ./CLI/miniooni-linux-amd64
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -76,11 +68,7 @@ jobs:
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
       - run: make CLI/linux-static-armv6
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv6
-      - run: |
+      - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv6 ./CLI/miniooni-linux-armv6
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
-          gh release create -p $tag --target $GITHUB_SHA || true
-          gh release upload $tag --clobber ./CLI/ooniprobe-linux-armv6 ./CLI/miniooni-linux-armv6
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -102,11 +90,7 @@ jobs:
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
       - run: make CLI/linux-static-armv7
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv7
-      - run: |
+      - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv7 ./CLI/miniooni-linux-armv7
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
-          gh release create -p $tag --target $GITHUB_SHA || true
-          gh release upload $tag --clobber ./CLI/ooniprobe-linux-armv7 ./CLI/miniooni-linux-armv7
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -128,10 +112,6 @@ jobs:
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
       - run: make CLI/linux-static-arm64
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-arm64
-      - run: |
+      - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-arm64 ./CLI/miniooni-linux-arm64
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
-          gh release create -p $tag --target $GITHUB_SHA || true
-          gh release upload $tag --clobber ./CLI/ooniprobe-linux-arm64 ./CLI/miniooni-linux-arm64
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/macos.yml

@@ -23,9 +23,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-macos-${{ steps.goversion.outputs.version }}"
 
       - run: |
           echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@@ -39,12 +40,9 @@ jobs:
       - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-darwin-amd64
 
       - run: |
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          ./script/ghpublish.bash ./CLI/ooniprobe-darwin-amd64 \
-          gh release create -p $tag --target $GITHUB_SHA || true
+                                  ./CLI/ooniprobe-darwin-arm64 \
-          gh release upload $tag --clobber ./CLI/ooniprobe-darwin-amd64 \
+                                  ./CLI/miniooni-darwin-amd64 \
-                                           ./CLI/ooniprobe-darwin-arm64 \
+                                  ./CLI/miniooni-darwin-arm64
-                                           ./CLI/miniooni-darwin-amd64 \
-                                           ./CLI/miniooni-darwin-arm64
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/oohelperd.yml

@@ -20,17 +20,14 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-oohelperd-${{ steps.goversion.outputs.version }}"
 
       - name: build oohelperd binary
         run: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ./CLI/oohelperd-linux-amd64 -v -tags netgo -ldflags="-s -w -extldflags -static" ./internal/cmd/oohelperd
 
-      - run: |
+      - run: ./script/ghpublish.bash ./CLI/oohelperd-linux-amd64
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
-          gh release create -p $tag --target $GITHUB_SHA || true
-          gh release upload $tag --clobber ./CLI/oohelperd-linux-amd64
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/qafbmessenger.yml

@@ -11,13 +11,4 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
-
-      - name: Get GOVERSION content
-        id: goversion
-        run: echo ::set-output name=version::$(cat GOVERSION)
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "${{ steps.goversion.outputs.version }}"
-
       - run: ./QA/rundocker.bash "fbmessenger"

.github/workflows/qahhfm.yml

@@ -11,13 +11,4 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
-
-      - name: Get GOVERSION content
-        id: goversion
-        run: echo ::set-output name=version::$(cat GOVERSION)
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "${{ steps.goversion.outputs.version }}"
-
       - run: ./QA/rundocker.bash "hhfm"

.github/workflows/qahirl.yml

@@ -11,13 +11,4 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
-
-      - name: Get GOVERSION content
-        id: goversion
-        run: echo ::set-output name=version::$(cat GOVERSION)
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "${{ steps.goversion.outputs.version }}"
-
       - run: ./QA/rundocker.bash "hirl"

.github/workflows/qatelegram.yml

@@ -11,13 +11,4 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
-
-      - name: Get GOVERSION content
-        id: goversion
-        run: echo ::set-output name=version::$(cat GOVERSION)
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "${{ steps.goversion.outputs.version }}"
-
       - run: ./QA/rundocker.bash "telegram"

.github/workflows/qawebconnectivity.yml

@@ -11,13 +11,4 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
-
-      - name: Get GOVERSION content
-        id: goversion
-        run: echo ::set-output name=version::$(cat GOVERSION)
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "${{ steps.goversion.outputs.version }}"
-
       - run: ./QA/rundocker.bash "webconnectivity"

.github/workflows/qawhatsapp.yml

@@ -11,13 +11,4 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
-
-      - name: Get GOVERSION content
-        id: goversion
-        run: echo ::set-output name=version::$(cat GOVERSION)
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "${{ steps.goversion.outputs.version }}"
-
       - run: ./QA/rundocker.bash "whatsapp"

.github/workflows/tarball.yml

@@ -23,19 +23,15 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-tarball-${{ steps.goversion.outputs.version }}"
 
-      - name: Generate release tarball
+      - name: Generate the release tarball
-        run: |
+        run: ./script/maketarball.bash
-          VERSION=${GITHUB_REF_NAME#v}
-          go mod vendor
-          tar czf ooni-probe-cli-${VERSION}.tar.gz --transform "s,^,ooni-probe-cli-${VERSION}/," *
 
       - name: Upload release tarball
-        run: |
+        run: ./script/ghpublish.bash ooni-probe-cli-*.tar.gz
-          gh release create -p $GITHUB_REF_NAME --target $GITHUB_SHA || true
-          gh release upload $GITHUB_REF_NAME --clobber ooni-probe-cli-*.tar.gz
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/windows.yml

@@ -20,9 +20,10 @@ jobs:
         id: goversion
         run: echo ::set-output name=version::$(cat GOVERSION)
 
-      - uses: actions/setup-go@v3
+      - uses: magnetikonline/action-golang-cache@v2
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
+          cache-key-suffix: "-windows-${{ steps.goversion.outputs.version }}"
 
       - run: sudo apt install mingw-w64
       - run: |
@@ -93,12 +94,9 @@ jobs:
           name: miniooni-windows-386.exe
 
       - run: |
-          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          ./script/ghpublish.bash ooniprobe-windows-386.exe \
-          gh release create -p $tag --target $GITHUB_SHA || true
+                                  ooniprobe-windows-amd64.exe \
-          gh release upload $tag --clobber ooniprobe-windows-386.exe \
+                                  miniooni-windows-386.exe \
-                                           ooniprobe-windows-amd64.exe \
+                                  miniooni-windows-amd64.exe
-                                           miniooni-windows-386.exe
-                                           miniooni-windows-amd64.exe
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

script/ghpublish.bash

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -euo pipefail
+
+# 1. obtain the github ref of this action run
+__ref=${GITHUB_REF:-}
+
+if [[ $__ref == "" ]]; then
+	echo "FATAL: missing github ref" 1>&2
+	exit 1
+fi
+
+# 2. determine whether to publish to a release or to rolling
+if [[ $__ref =~ ^refs/tags/v ]]; then
+	__tag=${__ref#refs/tags/}
+else
+	__tag=rolling
+fi
+
+set -x
+
+# 3. create the release as a pre-release unless it already exists
+gh release create -p $__tag --target $GITHUB_SHA || true
+
+# 4. publish all the assets passed as arguments to the target release
+gh release upload $__tag --clobber "$@"

script/maketarball.bash

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# 1. obtain the github ref of this action run
+__ref=${GITHUB_REF:-}
+
+if [[ $__ref == "" ]]; then
+	echo "FATAL: missing github ref" 1>&2
+	exit 1
+fi
+
+# 2. determine whether to publish to a release or to rolling
+if [[ $__ref =~ ^refs/tags/v ]]; then
+	__version=${__ref#refs/tags/v}
+else
+	__version=rolling
+fi
+
+set -x
+
+# 3. make sure we're using the correct go version
+./CLI/check-go-version
+
+# 4. generate the actual tarball
+go mod vendor
+tar -czf ooni-probe-cli-${__version}.tar.gz --transform "s,^,ooni-probe-cli-${__version}/," *
+