build.sh: build windows/386 and linux/386, update Debian packaging (#193)

* feat(build.sh): build for windows/386

Part of https://github.com/ooni/probe/issues/1315

* fix(build.sh): docker pull so we have latest image

Part of https://github.com/ooni/probe/issues/1315

* fix(build.sh): apk update && apk upgrade to have latest packages

It does not currently seem necessary. But let's be defensive.

Part of https://github.com/ooni/probe/issues/1315.

* fix(windows.yml): for now disable windows/386

It's working locally but not remotely on Windows. We need to
figure out how to install the 386 compiler.

Part of https://github.com/ooni/probe/issues/1315.

* feat(build.sh): build for windows/386

Part of https://github.com/ooni/probe/issues/1315.

* fix(linux.yml): try to enable experimental docker flags

Related to https://github.com/ooni/probe/issues/1315.

See also https://github.com/actions/virtual-environments/issues/368#issuecomment-713588127.

* fix(debian): try to have single workflow with experimental docker

More commits probably needed.

Part of https://github.com/ooni/ooni.org/issues/677.

* remove restriction on the branch name

* maybe we need a pattern also for branches

* less stringent constraints

* consolidated debian workflow into a single workflow

* further fix the debian workflow

* only run debian action for specific branches or tags

* fix(build.sh): package the 386 windows version properly

.github/workflows/debian.yml

@@ -0,0 +1,46 @@
+# Build and publish Debian packages
+name: debian
+on:
+  push:
+    branches:
+      - "master"
+      - "release/**"
+    tags:
+      - "v*"
+jobs:
+  build:
+    runs-on: "ubuntu-20.04"
+    steps:
+      - name: Docker
+        run: |
+          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+          sudo service docker restart
+      - uses: actions/setup-go@v1
+        with:
+          go-version: "1.14"
+      - uses: actions/checkout@v2
+      - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64
+      - run: sudo apt-get update -q
+      - run: sudo apt-get build-dep -y --no-install-recommends .
+      - run: |
+          VER=$(./CLI/linux/amd64/ooniprobe version)
+          if [[ ! $GITHUB_REF =~ ^refs/tags/* ]]; then
+            VER="${VER}~${GITHUB_RUN_NUMBER}"
+            dch -v $VER "New test version"
+            BT_REPO="${BT_REPO}-test"
+          else
+            dch -v $VER "New release"
+          fi
+          dpkg-buildpackage -us -uc -b
+          find ../ -name "*.deb" -type f
+          DEB="../ooniprobe-cli_${VER}_amd64.deb"
+          BT_FNAME="ooniprobe-cli_${VER}_amd64.deb"
+          curl --upload-file "${DEB}" -u "${BT_APIUSER}:${BT_APIKEY}" \
+            "https://api.bintray.com/content/${BT_ORG}/${BT_REPO}/${BT_PKGNAME}/${VER}/${BT_FNAME};deb_distribution=${DEBDIST};deb_component=main;deb_architecture=amd64;publish=1"
+        env:
+          DEBDIST: unstable
+          BT_APIKEY: ${{ secrets.BT_APIKEY }}
+          BT_APIUSER: federicoceratto
+          BT_ORG: ooni
+          BT_PKGNAME: ooniprobe
+          BT_REPO: ooniprobe-debian

.github/workflows/linux-debian-packages-release.yml

@@ -1,37 +0,0 @@
-# Build a Debian package only when a relase tag is applied
-# and publish it on the public/release archive
-name: linux-debian-packages
-on:
-  push:
-    tags:
-      - '*'
-jobs:
-  build:
-    runs-on: "ubuntu-20.04"
-    steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.14"
-      - uses: actions/checkout@v2
-      - run: ./build.sh linux
-      # - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
-      - run: find . -name ooniprobe -type f -executable
-      - run: sudo apt-get update -q
-      - run: sudo apt-get build-dep -y --no-install-recommends .
-      # Use probe version as package version
-      - run: |
-          DVER=$(./CLI/linux/amd64/ooniprobe version)
-          dch -v $DVER "New release"
-          dpkg-buildpackage -us -uc -b
-          find ../ -name "*.deb" -type f
-          DEB="../ooniprobe-cli_${DVER}_amd64.deb"
-          BT_FNAME="ooniprobe-cli_${DVER}_amd64.deb"
-          curl --upload-file "${DEB}" -u "${BT_APIUSER}:${BT_APIKEY}" \
-            "https://api.bintray.com/content/${BT_ORG}/${BT_REPO}/${BT_PKGNAME}/${DVER}/${BT_FNAME};deb_distribution=${DEBDIST};deb_component=main;deb_architecture=amd64;publish=1"
-        env:
-          DEBDIST: unstable
-          BT_APIKEY: ${{ secrets.BT_APIKEY }}
-          BT_APIUSER: federicoceratto
-          BT_ORG: ooni
-          BT_PKGNAME: ooniprobe
-          BT_REPO: ooniprobe-debian

.github/workflows/linux-debian-packages.yml

@@ -1,38 +0,0 @@
-# Build a Debian package and publish on a test/internal archive
-name: linux-debian-packages
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-jobs:
-  build:
-    runs-on: "ubuntu-20.04"
-    steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.14"
-      - uses: actions/checkout@v2
-      - run: ./build.sh linux
-      # - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
-      - run: find . -name ooniprobe -type f -executable
-      - run: sudo apt-get update -q
-      - run: sudo apt-get build-dep -y --no-install-recommends .
-      # Use <probe version>~<github build number> as package version
-      - run: |
-          VER=$(./CLI/linux/amd64/ooniprobe version)
-          DVER="${VER}~${GITHUB_RUN_NUMBER}"
-          dch -v $DVER "test version"
-          dpkg-buildpackage -us -uc -b
-          find ../ -name "*.deb" -type f
-          DEB="../ooniprobe-cli_${DVER}_amd64.deb"
-          BT_FNAME="ooniprobe-cli_${DVER}_amd64.deb"
-          curl --upload-file "${DEB}" -u "${BT_APIUSER}:${BT_APIKEY}" \
-            "https://api.bintray.com/content/${BT_ORG}/${BT_REPO}/${BT_PKGNAME}/${DVER}/${BT_FNAME};deb_distribution=${DEBDIST};deb_component=main;deb_architecture=amd64;publish=1"
-        env:
-          DEBDIST: unstable
-          BT_APIKEY: ${{ secrets.BT_APIKEY }}
-          BT_APIUSER: federicoceratto
-          BT_ORG: ooni
-          BT_PKGNAME: ooniprobe
-          BT_REPO: ooniprobe-debian-test

.github/workflows/linux.yml

@@ -8,11 +8,15 @@ jobs:
   build:
     runs-on: "ubuntu-latest"
     steps:
+      - name: Docker
+        run: |
+          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+          sudo service docker restart
       - uses: actions/setup-go@v1
         with:
           go-version: "1.14"
       - uses: actions/checkout@v2
-      - run: ./build.sh linux
+      - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux
       - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe
       - uses: actions/upload-artifact@v1
         with:

.github/workflows/windows.yml

@@ -12,7 +12,7 @@ jobs:
         with:
           go-version: "1.14"
       - uses: actions/checkout@v2
-      - run: bash.exe ./build.sh windows
+      - run: bash.exe ./build.sh windows_amd64  # TODO(bassosimone): make windows_386 work
       - run: bash.exe ./smoketest.sh ./CLI/windows/amd64/ooniprobe.exe
       - uses: actions/upload-artifact@v1
         with:

CLI/darwin/amd64/.gitignore

@@ -1 +1 @@
-*
+/ooniprobe

CLI/linux/386/.gitignore

@@ -0,0 +1 @@
+/ooniprobe

CLI/linux/amd64/.gitignore

@@ -1 +1 @@
-*
+/ooniprobe

CLI/windows/386/.gitignore

@@ -0,0 +1 @@
+/ooniprobe.exe

CLI/windows/amd64/.gitignore

@@ -1 +1 @@
-*
+/ooniprobe.exe

build.sh

@@ -6,6 +6,11 @@ v=`git describe --tags || echo $GITHUB_SHA`
 
 case $1 in
   windows)
+    $0 windows_amd64
+    $0 windows_386
+    ;;
+
+  windows_amd64)
     # Note! This assumes we've installed the mingw-w64 compiler.
     GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \
       go build -ldflags='-s -w' ./cmd/ooniprobe
@@ -15,13 +20,38 @@ case $1 in
     mv ooniprobe.exe ./CLI/windows/amd64/
     ;;
 
+  windows_386)
+    # Note! This assumes we've installed the mingw-w64 compiler.
+    GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc \
+      go build -ldflags='-s -w' ./cmd/ooniprobe
+    tar -cvzf ooniprobe_${v}_windows_386.tar.gz LICENSE.md Readme.md ooniprobe.exe
+    # We don't have zip inside the github actions runner
+    zip ooniprobe_${v}_windows_386.zip LICENSE.md Readme.md ooniprobe.exe || true
+    mv ooniprobe.exe ./CLI/windows/386/
+    ;;
+
   linux)
-    docker run -v`pwd`:/ooni -w/ooni golang:1.14-alpine ./build.sh _alpine
+    $0 linux_amd64
+    $0 linux_386
+    ;;
+
+  linux_amd64)
+    docker pull --platform linux/amd64 golang:1.14-alpine
+    docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.14-alpine ./build.sh _alpine
     tar -cvzf ooniprobe_${v}_linux_amd64.tar.gz LICENSE.md Readme.md ooniprobe
     mv ooniprobe ./CLI/linux/amd64/
     ;;
 
+  linux_386)
+    docker pull --platform linux/386 golang:1.14-alpine
+    docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.14-alpine ./build.sh _alpine
+    tar -cvzf ooniprobe_${v}_linux_386.tar.gz LICENSE.md Readme.md ooniprobe
+    mv ooniprobe ./CLI/linux/386/
+    ;;
+
   _alpine)
+    apk update
+    apk upgrade
     apk add --no-progress gcc git linux-headers musl-dev
     go build -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/ooniprobe
     ;;
@@ -38,7 +68,7 @@ case $1 in
     $0 linux
     $0 windows
     $0 darwin
-    shasum -a 256 ooniprobe_${v}_*_amd64.* > ooniprobe_checksums.txt
+    shasum -a 256 ooniprobe_${v}_*_*.* > ooniprobe_checksums.txt
     ;;
 
   *)