feat(gha/linux): upload deb packages for arm64 (#337)

We're still working on https://github.com/ooni/probe/issues/1466. The idea here is to teach the GH action for Linux to publish the debian package for arm64. When this is done, we can cleanup legacy build scripts and GH actions, because there is no remaining use case for them: we now build everything using the `./make` tool.

.github/workflows/cross.yml

@@ -1,58 +0,0 @@
-# Shows how to cross compile ooniprobe
-name: cross
-on:
-  push:
-    branches:
-      - 'release/**'
-  schedule:
-    - cron: "14 17 * * 3"
-jobs:
-
-  windows_from_linux_build:
-    runs-on: "ubuntu-latest"
-    steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.16"
-      - uses: actions/checkout@v2
-      - run: sudo apt update
-      - run: sudo apt install --yes mingw-w64
-      - run: ./build.sh windows
-      - uses: actions/upload-artifact@v1
-        with:
-          name: ooniprobe-windows-amd64-compiled-from-linux
-          path: ./CLI/windows/amd64/ooniprobe.exe
-
-  test_build_from_linux:
-    needs: windows_from_linux_build
-    runs-on: "windows-latest"
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        with:
-          name: ooniprobe-windows-amd64-compiled-from-linux
-      - run: bash.exe ./smoketest.sh ./ooniprobe.exe
-
-  windows_from_macos_build:
-    runs-on: "macos-latest"
-    steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.16"
-      - uses: actions/checkout@v2
-      - run: brew install mingw-w64
-      - run: ./build.sh windows
-      - uses: actions/upload-artifact@v1
-        with:
-          name: ooniprobe-windows-amd64-compiled-from-macos
-          path: ./CLI/windows/amd64/ooniprobe.exe
-
-  test_build_from_macos:
-    needs: windows_from_macos_build
-    runs-on: "windows-latest"
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        with:
-          name: ooniprobe-windows-amd64-compiled-from-macos
-      - run: bash.exe ./smoketest.sh ./ooniprobe.exe

.github/workflows/debian.yml

@@ -1,52 +0,0 @@
----
-# Build and publish Debian packages
-name: debian
-on:
-  push:
-    branches:
-      - "master"
-      - "release/**"
-    tags:
-      - "v*"
-jobs:
-  build:
-    runs-on: "ubuntu-20.04"
-    steps:
-      - name: Docker
-        run: |
-          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
-          sudo service docker restart
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.16"
-      - uses: actions/checkout@v2
-      - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64
-      - run: mkdir -p debian/bin
-      - run: cp ./CLI/linux/amd64/ooniprobe debian/bin/ooniprobe
-      - run: sudo apt-get update -q
-      - run: sudo apt-get build-dep -y --no-install-recommends .
-      - name: Install deps
-        run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
-      - name: Sign and upload
-        run: |
-          VER=$(./CLI/linux/amd64/ooniprobe version)
-          if [[ ! $GITHUB_REF =~ ^refs/tags/* ]]; then
-            VER="${VER}~${GITHUB_RUN_NUMBER}"
-            dch -v $VER "New test version"
-            BT_REPO="${BT_REPO}-test"
-          else
-            dch -v $VER "New release"
-          fi
-          dpkg-buildpackage -us -uc -b
-          find ../ -name "*.deb" -type f
-          DEB="../ooniprobe-cli_${VER}_amd64.deb"
-          echo no | sudo dpkg -i $DEB
-          export DEBFNAME="ooniprobe-cli_${VER}_amd64.deb"
-          cd $GITHUB_WORKSPACE
-          # AWS user debci-s3
-          ./.github/workflows/debops-ci --show-commands upload --bucket-name ooni-deb $DEB
-        env:
-          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          BT_REPO: ooniprobe-debian

.github/workflows/debops-ci

@@ -304,8 +304,10 @@ def check_duplicate_package(pkgblock, packages_text):
     pname = li[0].split(" ", 1)[1]
     assert li[1].startswith("Version: "), li
     pver = li[1].split(" ", 1)[1]
+    assert li[2].startswith("Architecture: "), li
+    parch = li[2].split(" ", 1)[1]
 
-    m = f"Package: {pname}\nVersion: {pver}"
+    m = f"Package: {pname}\nVersion: {pver}\nArchitecture: {parch}"
     if m in packages_text:
         raise DuplicatePkgError()
 

.github/workflows/linux.yml

@@ -4,8 +4,8 @@ on:
   push:
     branches:
       - "release/**"
-jobs:
 
+jobs:
   build_amd64:
     runs-on: "ubuntu-20.04"
     steps:
@@ -18,6 +18,15 @@ jobs:
           DOCKER_CLI_EXPERIMENTAL: enabled
       - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
       - run: ./make --disable-embedding-psiphon-config -t debian_amd64
+      - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
+      - run: |
+          for deb in *.deb; do
+            ./.github/workflows/debops-ci --arch amd64 --show-commands upload --bucket-name ooni-deb $deb
+          done
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
 
   build_arm64:
     runs-on: "ubuntu-20.04"
@@ -32,3 +41,12 @@ jobs:
           DOCKER_CLI_EXPERIMENTAL: enabled
       - run: ./smoketest.sh ./CLI/linux/arm64/ooniprobe
       - run: ./make --disable-embedding-psiphon-config -t debian_arm64
+      - run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd
+      - run: |
+          for deb in *.deb; do
+            ./.github/workflows/debops-ci --arch arm64 --show-commands upload --bucket-name ooni-deb $deb
+          done
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}

build.sh

@@ -1,98 +0,0 @@
-#!/bin/sh
-set -e
-
-# We don't have a git repository when running in github actions
-v=`git describe --tags || echo $GITHUB_SHA`
-
-case $1 in
-  windows)
-    set -x
-    $0 windows_amd64
-    $0 windows_386
-    ;;
-
-  windows_amd64)
-    # Note! This assumes we've installed the mingw-w64 compiler.
-    GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \
-      go build -ldflags='-s -w' ./cmd/ooniprobe
-    tar -cvzf ooniprobe_${v}_windows_amd64.tar.gz LICENSE.md Readme.md ooniprobe.exe
-    # We don't have zip inside the github actions runner
-    zip ooniprobe_${v}_windows_amd64.zip LICENSE.md Readme.md ooniprobe.exe || true
-    mv ooniprobe.exe ./CLI/windows/amd64/
-    ;;
-
-  windows_386)
-    # Note! This assumes we've installed the mingw-w64 compiler.
-    GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc \
-      go build -ldflags='-s -w' ./cmd/ooniprobe
-    tar -cvzf ooniprobe_${v}_windows_386.tar.gz LICENSE.md Readme.md ooniprobe.exe
-    # We don't have zip inside the github actions runner
-    zip ooniprobe_${v}_windows_386.zip LICENSE.md Readme.md ooniprobe.exe || true
-    mv ooniprobe.exe ./CLI/windows/386/
-    ;;
-
-  linux)
-    set -x
-    $0 linux_amd64
-    $0 linux_386
-    ;;
-
-  linux_amd64)
-    docker pull --platform linux/amd64 golang:1.16-alpine
-    docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine
-    tar -cvzf ooniprobe_${v}_linux_amd64.tar.gz LICENSE.md Readme.md ooniprobe
-    mv ooniprobe ./CLI/linux/amd64/
-    ;;
-
-  linux_386)
-    docker pull --platform linux/386 golang:1.16-alpine
-    docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine
-    tar -cvzf ooniprobe_${v}_linux_386.tar.gz LICENSE.md Readme.md ooniprobe
-    mv ooniprobe ./CLI/linux/386/
-    ;;
-
-  _alpine)
-    apk update
-    apk upgrade
-    apk add --no-progress gcc git linux-headers musl-dev
-    go build -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/ooniprobe
-    ;;
-
-  macos|darwin)
-    set -x
-    # Note! The following line _assumes_ you have a working C compiler. If you
-    # have Xcode command line tools installed, you are fine.
-    go build -ldflags='-s -w' ./cmd/ooniprobe
-    tar -cvzf ooniprobe_${v}_darwin_amd64.tar.gz LICENSE.md Readme.md ooniprobe
-    mv ooniprobe ./CLI/darwin/amd64/
-    ;;
-
-  release)
-    $0 linux
-    $0 windows
-    $0 darwin
-    shasum -a 256 ooniprobe_${v}_*_*.* > ooniprobe_checksums.txt
-    ;;
-
-  *)
-
-    set +x
-    echo "Usage: $0 darwin|linux|macos|windows|release"
-    echo ""
-    echo "You need a C compiler and Go >= 1.16. The C compiler must be a"
-    echo "UNIX like compiler like GCC, Clang, Mingw-w64."
-    echo ""
-    echo "To build a static Linux binary, we use Docker and Alpine. We currently"
-    echo "build for linux/386 and linux/amd64."
-    echo ""
-    echo "You can cross compile for Windows from macOS or Linux. You can"
-    echo "compile for Linux as long as you have Docker. Cross compiling for"
-    echo "macOS has never been tested. We have a bunch of cross compiling"
-    echo "checks inside the .github/workflows/cross.yml file."
-    echo ""
-    echo "The macos rule is an alias for the darwin rule. The generated"
-    echo 'binary file is named ooniprobe_${version}_darwin_${arch}.tar.gz'
-    echo "because the platform name is darwin."
-    echo ""
-    ;;
-esac