From e753e57da30324aa5bf06f0f55428208185da2a3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 7 May 2021 09:22:46 +0200 Subject: [PATCH] feat(gha/linux): upload deb packages for arm64 (#337) We're still working on https://github.com/ooni/probe/issues/1466. The idea here is to teach the GH action for Linux to publish the debian package for arm64. When this is done, we can cleanup legacy build scripts and GH actions, because there is no remaining use case for them: we now build everything using the `./make` tool. --- .github/workflows/cross.yml | 58 --------------------- .github/workflows/debian.yml | 52 ------------------- .github/workflows/debops-ci | 4 +- .github/workflows/linux.yml | 20 +++++++- build.sh | 98 ------------------------------------ 5 files changed, 22 insertions(+), 210 deletions(-) delete mode 100644 .github/workflows/cross.yml delete mode 100644 .github/workflows/debian.yml delete mode 100755 build.sh diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml deleted file mode 100644 index 932e20b..0000000 --- a/.github/workflows/cross.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Shows how to cross compile ooniprobe -name: cross -on: - push: - branches: - - 'release/**' - schedule: - - cron: "14 17 * * 3" -jobs: - - windows_from_linux_build: - runs-on: "ubuntu-latest" - steps: - - uses: actions/setup-go@v1 - with: - go-version: "1.16" - - uses: actions/checkout@v2 - - run: sudo apt update - - run: sudo apt install --yes mingw-w64 - - run: ./build.sh windows - - uses: actions/upload-artifact@v1 - with: - name: ooniprobe-windows-amd64-compiled-from-linux - path: ./CLI/windows/amd64/ooniprobe.exe - - test_build_from_linux: - needs: windows_from_linux_build - runs-on: "windows-latest" - steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 - with: - name: ooniprobe-windows-amd64-compiled-from-linux - - run: bash.exe ./smoketest.sh ./ooniprobe.exe - - windows_from_macos_build: - runs-on: "macos-latest" - steps: - - uses: actions/setup-go@v1 - with: - go-version: "1.16" - - uses: actions/checkout@v2 - - run: brew install mingw-w64 - - run: ./build.sh windows - - uses: actions/upload-artifact@v1 - with: - name: ooniprobe-windows-amd64-compiled-from-macos - path: ./CLI/windows/amd64/ooniprobe.exe - - test_build_from_macos: - needs: windows_from_macos_build - runs-on: "windows-latest" - steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 - with: - name: ooniprobe-windows-amd64-compiled-from-macos - - run: bash.exe ./smoketest.sh ./ooniprobe.exe diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml deleted file mode 100644 index 42cf5cb..0000000 --- a/.github/workflows/debian.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -# Build and publish Debian packages -name: debian -on: - push: - branches: - - "master" - - "release/**" - tags: - - "v*" -jobs: - build: - runs-on: "ubuntu-20.04" - steps: - - name: Docker - run: | - echo $'{\n "experimental": true\n}' | sudo tee /etc/docker/daemon.json - sudo service docker restart - - uses: actions/setup-go@v1 - with: - go-version: "1.16" - - uses: actions/checkout@v2 - - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64 - - run: mkdir -p debian/bin - - run: cp ./CLI/linux/amd64/ooniprobe debian/bin/ooniprobe - - run: sudo apt-get update -q - - run: sudo apt-get build-dep -y --no-install-recommends . - - name: Install deps - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd - - name: Sign and upload - run: | - VER=$(./CLI/linux/amd64/ooniprobe version) - if [[ ! $GITHUB_REF =~ ^refs/tags/* ]]; then - VER="${VER}~${GITHUB_RUN_NUMBER}" - dch -v $VER "New test version" - BT_REPO="${BT_REPO}-test" - else - dch -v $VER "New release" - fi - dpkg-buildpackage -us -uc -b - find ../ -name "*.deb" -type f - DEB="../ooniprobe-cli_${VER}_amd64.deb" - echo no | sudo dpkg -i $DEB - export DEBFNAME="ooniprobe-cli_${VER}_amd64.deb" - cd $GITHUB_WORKSPACE - # AWS user debci-s3 - ./.github/workflows/debops-ci --show-commands upload --bucket-name ooni-deb $DEB - env: - DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - BT_REPO: ooniprobe-debian diff --git a/.github/workflows/debops-ci b/.github/workflows/debops-ci index dcdef7a..86d5b1a 100755 --- a/.github/workflows/debops-ci +++ b/.github/workflows/debops-ci @@ -304,8 +304,10 @@ def check_duplicate_package(pkgblock, packages_text): pname = li[0].split(" ", 1)[1] assert li[1].startswith("Version: "), li pver = li[1].split(" ", 1)[1] + assert li[2].startswith("Architecture: "), li + parch = li[2].split(" ", 1)[1] - m = f"Package: {pname}\nVersion: {pver}" + m = f"Package: {pname}\nVersion: {pver}\nArchitecture: {parch}" if m in packages_text: raise DuplicatePkgError() diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index cb92687..adb21dc 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -4,8 +4,8 @@ on: push: branches: - "release/**" -jobs: +jobs: build_amd64: runs-on: "ubuntu-20.04" steps: @@ -18,6 +18,15 @@ jobs: DOCKER_CLI_EXPERIMENTAL: enabled - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe - run: ./make --disable-embedding-psiphon-config -t debian_amd64 + - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd + - run: | + for deb in *.deb; do + ./.github/workflows/debops-ci --arch amd64 --show-commands upload --bucket-name ooni-deb $deb + done + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }} build_arm64: runs-on: "ubuntu-20.04" @@ -32,3 +41,12 @@ jobs: DOCKER_CLI_EXPERIMENTAL: enabled - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe - run: ./make --disable-embedding-psiphon-config -t debian_arm64 + - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd + - run: | + for deb in *.deb; do + ./.github/workflows/debops-ci --arch arm64 --show-commands upload --bucket-name ooni-deb $deb + done + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }} diff --git a/build.sh b/build.sh deleted file mode 100755 index 456f28e..0000000 --- a/build.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/sh -set -e - -# We don't have a git repository when running in github actions -v=`git describe --tags || echo $GITHUB_SHA` - -case $1 in - windows) - set -x - $0 windows_amd64 - $0 windows_386 - ;; - - windows_amd64) - # Note! This assumes we've installed the mingw-w64 compiler. - GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \ - go build -ldflags='-s -w' ./cmd/ooniprobe - tar -cvzf ooniprobe_${v}_windows_amd64.tar.gz LICENSE.md Readme.md ooniprobe.exe - # We don't have zip inside the github actions runner - zip ooniprobe_${v}_windows_amd64.zip LICENSE.md Readme.md ooniprobe.exe || true - mv ooniprobe.exe ./CLI/windows/amd64/ - ;; - - windows_386) - # Note! This assumes we've installed the mingw-w64 compiler. - GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc \ - go build -ldflags='-s -w' ./cmd/ooniprobe - tar -cvzf ooniprobe_${v}_windows_386.tar.gz LICENSE.md Readme.md ooniprobe.exe - # We don't have zip inside the github actions runner - zip ooniprobe_${v}_windows_386.zip LICENSE.md Readme.md ooniprobe.exe || true - mv ooniprobe.exe ./CLI/windows/386/ - ;; - - linux) - set -x - $0 linux_amd64 - $0 linux_386 - ;; - - linux_amd64) - docker pull --platform linux/amd64 golang:1.16-alpine - docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine - tar -cvzf ooniprobe_${v}_linux_amd64.tar.gz LICENSE.md Readme.md ooniprobe - mv ooniprobe ./CLI/linux/amd64/ - ;; - - linux_386) - docker pull --platform linux/386 golang:1.16-alpine - docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine - tar -cvzf ooniprobe_${v}_linux_386.tar.gz LICENSE.md Readme.md ooniprobe - mv ooniprobe ./CLI/linux/386/ - ;; - - _alpine) - apk update - apk upgrade - apk add --no-progress gcc git linux-headers musl-dev - go build -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/ooniprobe - ;; - - macos|darwin) - set -x - # Note! The following line _assumes_ you have a working C compiler. If you - # have Xcode command line tools installed, you are fine. - go build -ldflags='-s -w' ./cmd/ooniprobe - tar -cvzf ooniprobe_${v}_darwin_amd64.tar.gz LICENSE.md Readme.md ooniprobe - mv ooniprobe ./CLI/darwin/amd64/ - ;; - - release) - $0 linux - $0 windows - $0 darwin - shasum -a 256 ooniprobe_${v}_*_*.* > ooniprobe_checksums.txt - ;; - - *) - - set +x - echo "Usage: $0 darwin|linux|macos|windows|release" - echo "" - echo "You need a C compiler and Go >= 1.16. The C compiler must be a" - echo "UNIX like compiler like GCC, Clang, Mingw-w64." - echo "" - echo "To build a static Linux binary, we use Docker and Alpine. We currently" - echo "build for linux/386 and linux/amd64." - echo "" - echo "You can cross compile for Windows from macOS or Linux. You can" - echo "compile for Linux as long as you have Docker. Cross compiling for" - echo "macOS has never been tested. We have a bunch of cross compiling" - echo "checks inside the .github/workflows/cross.yml file." - echo "" - echo "The macos rule is an alias for the darwin rule. The generated" - echo 'binary file is named ooniprobe_${version}_darwin_${arch}.tar.gz' - echo "because the platform name is darwin." - echo "" - ;; -esac