c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: implement rolling builds (#910)

Browse Source 
This diff modifies all the github actions that produce assets to
publish on a release called rolling when we are not building a tag.

If everything goes as planned, we should be able to provide
people with automatically generated fresh binaries for testing.

While there, introduce caching for all builds to make them
as fast as possible. I suspect gomobile based builds will not
see any speed up but other builds most likely will.

See https://github.com/ooni/probe/issues/2249
This commit is contained in:
Simone Basso 2022-08-30 15:29:09 +02:00 committed by GitHub
parent d1b172f3ac
commit d10ab88444
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 111 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/alltests.yml vendored
View File

@ -16,8 +16,9 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-alltests-${{ steps.goversion.outputs.version }}"
- run: go test -race -tags shaping ./...

52
.github/workflows/android.yml vendored
View File

@ -25,9 +25,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-android-oonimkall-${{ steps.goversion.outputs.version }}"
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@ -39,12 +40,9 @@ jobs:
- run: make MOBILE/android
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./MOBILE/android/oonimkall.aar \
./MOBILE/android/oonimkall-sources.jar \
./MOBILE/android/oonimkall.pom
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
./script/ghpublish.bash ./MOBILE/android/oonimkall.aar \
./MOBILE/android/oonimkall-sources.jar \
./MOBILE/android/oonimkall.pom
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -62,9 +60,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-android-cli-386-${{ steps.goversion.outputs.version }}"
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@ -76,11 +75,8 @@ jobs:
- run: make CLI/android-386
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/miniooni-android-386 \
./CLI/ooniprobe-android-386
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
./script/ghpublish.bash ./CLI/miniooni-android-386 \
./CLI/ooniprobe-android-386
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -98,9 +94,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-android-cli-amd64-${{ steps.goversion.outputs.version }}"
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@ -112,11 +109,8 @@ jobs:
- run: make CLI/android-amd64
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/miniooni-android-amd64 \
./CLI/ooniprobe-android-amd64
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
./script/ghpublish.bash ./CLI/miniooni-android-amd64 \
./CLI/ooniprobe-android-amd64
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -134,9 +128,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-android-cli-arm-${{ steps.goversion.outputs.version }}"
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@ -148,11 +143,8 @@ jobs:
- run: make CLI/android-arm
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/miniooni-android-arm \
./CLI/ooniprobe-android-arm
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
./script/ghpublish.bash ./CLI/miniooni-android-arm \
./CLI/ooniprobe-android-arm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -170,9 +162,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-android-cli-arm64-${{ steps.goversion.outputs.version }}"
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@ -184,10 +177,7 @@ jobs:
- run: make CLI/android-arm64
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/miniooni-android-arm64 \
./CLI/ooniprobe-android-arm64
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
./script/ghpublish.bash ./CLI/miniooni-android-arm64 \
./CLI/ooniprobe-android-arm64
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

3
.github/workflows/generate.yml vendored
View File

@ -16,8 +16,9 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-generate-${{ steps.goversion.outputs.version }}"
- run: go generate ./...

3
.github/workflows/gosec.yml vendored
View File

@ -20,9 +20,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-gosec-${{ steps.goversion.outputs.version }}"
- name: Run Gosec security scanner
continue-on-error: true # TODO(https://github.com/ooni/probe/issues/2180)

10
.github/workflows/ios.yml vendored
View File

@ -23,9 +23,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-ios-${{ steps.goversion.outputs.version }}"
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@ -37,10 +38,7 @@ jobs:
- run: make EXPECTED_XCODE_VERSION=12.4 MOBILE/ios
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./MOBILE/ios/oonimkall.xcframework.zip \
./MOBILE/ios/oonimkall.podspec
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
./script/ghpublish.bash ./MOBILE/ios/oonimkall.xcframework.zip \
./MOBILE/ios/oonimkall.podspec
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

3
.github/workflows/jafar.yml vendored
View File

@ -16,9 +16,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-jafar-${{ steps.goversion.outputs.version }}"
- run: go build -v ./internal/cmd/jafar

30
.github/workflows/linux.yml vendored
View File

@ -25,11 +25,7 @@ jobs:
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- run: make CLI/linux-static-386
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -50,11 +46,7 @@ jobs:
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- run: make CLI/linux-static-amd64
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-amd64
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/ooniprobe-linux-amd64 ./CLI/miniooni-linux-amd64
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-amd64 ./CLI/miniooni-linux-amd64
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -76,11 +68,7 @@ jobs:
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- run: make CLI/linux-static-armv6
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv6
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/ooniprobe-linux-armv6 ./CLI/miniooni-linux-armv6
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv6 ./CLI/miniooni-linux-armv6
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -102,11 +90,7 @@ jobs:
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- run: make CLI/linux-static-armv7
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv7
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/ooniprobe-linux-armv7 ./CLI/miniooni-linux-armv7
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv7 ./CLI/miniooni-linux-armv7
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -128,10 +112,6 @@ jobs:
PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
- run: make CLI/linux-static-arm64
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-arm64
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/ooniprobe-linux-arm64 ./CLI/miniooni-linux-arm64
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
- run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-arm64 ./CLI/miniooni-linux-arm64
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

14
.github/workflows/macos.yml vendored
View File

@ -23,9 +23,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-macos-${{ steps.goversion.outputs.version }}"
- run: |
echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key
@ -39,12 +40,9 @@ jobs:
- run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-darwin-amd64
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/ooniprobe-darwin-amd64 \
./CLI/ooniprobe-darwin-arm64 \
./CLI/miniooni-darwin-amd64 \
./CLI/miniooni-darwin-arm64
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
./script/ghpublish.bash ./CLI/ooniprobe-darwin-amd64 \
./CLI/ooniprobe-darwin-arm64 \
./CLI/miniooni-darwin-amd64 \
./CLI/miniooni-darwin-arm64
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

9
.github/workflows/oohelperd.yml vendored
View File

@ -20,17 +20,14 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-oohelperd-${{ steps.goversion.outputs.version }}"
- name: build oohelperd binary
run: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ./CLI/oohelperd-linux-amd64 -v -tags netgo -ldflags="-s -w -extldflags -static" ./internal/cmd/oohelperd
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ./CLI/oohelperd-linux-amd64
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
- run: ./script/ghpublish.bash ./CLI/oohelperd-linux-amd64
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

9
.github/workflows/qafbmessenger.yml vendored
View File

@ -11,13 +11,4 @@ jobs:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Get GOVERSION content
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
with:
go-version: "${{ steps.goversion.outputs.version }}"
- run: ./QA/rundocker.bash "fbmessenger"

9
.github/workflows/qahhfm.yml vendored
View File

@ -11,13 +11,4 @@ jobs:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Get GOVERSION content
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
with:
go-version: "${{ steps.goversion.outputs.version }}"
- run: ./QA/rundocker.bash "hhfm"

9
.github/workflows/qahirl.yml vendored
View File

@ -11,13 +11,4 @@ jobs:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Get GOVERSION content
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
with:
go-version: "${{ steps.goversion.outputs.version }}"
- run: ./QA/rundocker.bash "hirl"

9
.github/workflows/qatelegram.yml vendored
View File

@ -11,13 +11,4 @@ jobs:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Get GOVERSION content
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
with:
go-version: "${{ steps.goversion.outputs.version }}"
- run: ./QA/rundocker.bash "telegram"

9
.github/workflows/qawebconnectivity.yml vendored
View File

@ -11,13 +11,4 @@ jobs:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Get GOVERSION content
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
with:
go-version: "${{ steps.goversion.outputs.version }}"
- run: ./QA/rundocker.bash "webconnectivity"

9
.github/workflows/qawhatsapp.yml vendored
View File

@ -11,13 +11,4 @@ jobs:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Get GOVERSION content
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
with:
go-version: "${{ steps.goversion.outputs.version }}"
- run: ./QA/rundocker.bash "whatsapp"

14
.github/workflows/tarball.yml vendored
View File

@ -23,19 +23,15 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-tarball-${{ steps.goversion.outputs.version }}"
- name: Generate release tarball
run: |
VERSION=${GITHUB_REF_NAME#v}
go mod vendor
tar czf ooni-probe-cli-${VERSION}.tar.gz --transform "s,^,ooni-probe-cli-${VERSION}/," *
- name: Generate the release tarball
run: ./script/maketarball.bash
- name: Upload release tarball
run: |
gh release create -p $GITHUB_REF_NAME --target $GITHUB_SHA || true
gh release upload $GITHUB_REF_NAME --clobber ooni-probe-cli-*.tar.gz
run: ./script/ghpublish.bash ooni-probe-cli-*.tar.gz
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

14
.github/workflows/windows.yml vendored
View File

@ -20,9 +20,10 @@ jobs:
id: goversion
run: echo ::set-output name=version::$(cat GOVERSION)
- uses: actions/setup-go@v3
- uses: magnetikonline/action-golang-cache@v2
with:
go-version: "${{ steps.goversion.outputs.version }}"
cache-key-suffix: "-windows-${{ steps.goversion.outputs.version }}"
- run: sudo apt install mingw-w64
- run: |
@ -93,12 +94,9 @@ jobs:
name: miniooni-windows-386.exe
- run: |
tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
gh release create -p $tag --target $GITHUB_SHA || true
gh release upload $tag --clobber ooniprobe-windows-386.exe \
ooniprobe-windows-amd64.exe \
miniooni-windows-386.exe
miniooni-windows-amd64.exe
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
./script/ghpublish.bash ooniprobe-windows-386.exe \
ooniprobe-windows-amd64.exe \
miniooni-windows-386.exe \
miniooni-windows-amd64.exe
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

25
script/ghpublish.bash Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
set -euo pipefail
# 1. obtain the github ref of this action run
__ref=${GITHUB_REF:-}
if [[ $__ref == "" ]]; then
echo "FATAL: missing github ref" 1>&2
exit 1
fi
# 2. determine whether to publish to a release or to rolling
if [[ $__ref =~ ^refs/tags/v ]]; then
__tag=${__ref#refs/tags/}
else
__tag=rolling
fi
set -x
# 3. create the release as a pre-release unless it already exists
gh release create -p $__tag --target $GITHUB_SHA || true
# 4. publish all the assets passed as arguments to the target release
gh release upload $__tag --clobber "$@"

28
script/maketarball.bash Executable file
View File

@ -0,0 +1,28 @@
#!/bin/bash
set -euo pipefail
# 1. obtain the github ref of this action run
__ref=${GITHUB_REF:-}
if [[ $__ref == "" ]]; then
echo "FATAL: missing github ref" 1>&2
exit 1
fi
# 2. determine whether to publish to a release or to rolling
if [[ $__ref =~ ^refs/tags/v ]]; then
__version=${__ref#refs/tags/v}
else
__version=rolling
fi
set -x
# 3. make sure we're using the correct go version
./CLI/check-go-version
# 4. generate the actual tarball
go mod vendor
tar -czf ooni-probe-cli-${__version}.tar.gz --transform "s,^,ooni-probe-cli-${__version}/," *