feat: use go1.16 and resources embedding (#235)

* feat: use go1.16 embedding for resources

We want to embed everything that can be easily embedded. We should, at a
minimum, replace the downloading of resources and bindata.

Ref: https://github.com/ooni/probe/issues/1367.

* fix: get rid of bindata and use go embed instead

* fix: start unbreaking some automatic tests

* fix: fetch resources as part of the mobile build

* fix: convert more stuff to go1.16

I still expect many breakages, but we'll fix them.

* fix: make the windows CI green

* fix: get resources before running QA

* fix: go1.16 uses modules by default

* hopefully fix all other outstanding issues

* fix(QA/telegram.py): add another DC IP address

* Apply suggestions from code review

.github/workflows/alltests.yml

@@ -11,6 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
+      - run: go run ./internal/cmd/getresources
       - run: go test -race -tags shaping ./...

.github/workflows/android.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: brew install --cask android-sdk
       - run: echo y | sdkmanager --install "platforms;android-29"

.github/workflows/bindata.yml

@@ -1,22 +0,0 @@
-# Make sure we can embed bindata
-name: bindata
-on:
-  push:
-  schedule:
-    - cron: "14 17 * * 3"
-jobs:
-  test:
-    runs-on: "${{ matrix.os }}"
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        go: ["1.14"]
-    steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: ${{ matrix.go }}
-      - uses: actions/checkout@v2
-      - run: ./updatebindata.sh
-      - run: go mod tidy  # revert changes caused by installing bindata
-      - run: git diff --exit-code  # if this fails, run ./updatebindata.sh locally and push

.github/workflows/coverage.yml

@@ -9,12 +9,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ "1.14" ]
+        go: [ "1.16" ]
     steps:
       - uses: actions/setup-go@v1
         with:
           go-version: "${{ matrix.go }}"
       - uses: actions/checkout@v2
+      - run: go run ./internal/cmd/getresources
       - run: go test -short -race -tags shaping -coverprofile=probe-cli.cov ./...
       - uses: shogo82148/actions-goveralls@v1
         with:

.github/workflows/cross.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: sudo apt update
       - run: sudo apt install --yes mingw-w64
@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: brew install mingw-w64
       - run: ./build.sh windows

.github/workflows/debian.yml

@@ -17,7 +17,7 @@ jobs:
           sudo service docker restart
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64
       - run: sudo apt-get update -q

.github/workflows/e2eminiooni.yml

@@ -13,6 +13,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./E2E/miniooni.bash

.github/workflows/generate.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: go generate ./...

.github/workflows/ios.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./build-ios.bash
       - run: ./publish-ios.bash

.github/workflows/jafar.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: go build -v ./internal/cmd/jafar
       - run: sudo ./testjafar.bash

.github/workflows/linux.yml

@@ -14,7 +14,7 @@ jobs:
           sudo service docker restart
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux
       - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe

.github/workflows/macos.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./build.sh darwin
       - run: ./smoketest.sh ./CLI/darwin/amd64/ooniprobe

.github/workflows/miniooni.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
 
       - run: ./build-miniooni.sh linux

.github/workflows/qafbmessenger.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "fbmessenger"

.github/workflows/qahhfm.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "hhfm"

.github/workflows/qahirl.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "hirl"

.github/workflows/qatelegram.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "telegram"

.github/workflows/qawebconnectivity.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "webconnectivity"

.github/workflows/qawhatsapp.yml

@@ -11,6 +11,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: ./QA/rundocker.bash "whatsapp"

.github/workflows/shorttests.yml

@@ -9,10 +9,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ "1.14", "1.15" ]
+        go: [ "1.16" ]
     steps:
       - uses: actions/setup-go@v1
         with:
           go-version: "${{ matrix.go }}"
       - uses: actions/checkout@v2
+      - run: go run ./internal/cmd/getresources
       - run: go test -short -race -tags shaping ./...

.github/workflows/windows.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v1
         with:
-          go-version: "1.14"
+          go-version: "1.16"
       - uses: actions/checkout@v2
       - run: bash.exe ./build.sh windows_amd64  # TODO(bassosimone): make windows_386 work
       - run: bash.exe ./smoketest.sh ./CLI/windows/amd64/ooniprobe.exe