feat(make): build miniooni (#322)

This change is useful to move forward with blessing a new
release (https://github.com/ooni/probe/issues/1439).

.github/workflows/android.yml

@@ -1,8 +1,10 @@
+# android verifies we can still build for Android
 name: android
 on:
   push:
     branches:
       - "release/**"
+      - "master"
 jobs:
   test:
     runs-on: ubuntu-20.04

.github/workflows/ios.yml

@@ -1,8 +1,10 @@
+# ios verifies we can still build for iOS
 name: ios
 on:
   push:
     branches:
       - 'release/**'
+      - 'master'
 jobs:
   test:
     runs-on: macos-10.15

.github/workflows/miniooni.yml

@@ -1,47 +1,33 @@
+# miniooni checks whether we can build the research client miniooni
+# and publishes all linux binaries as artefacts. There is no point in
+# publishing windows or darwin binaries b/c they are not signed.
 name: miniooni
 on:
   push:
-    branches:
-      - 'release/**'
-  schedule:
-    - cron: "0 0 * * */1"
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/setup-go@v1
-        with:
-          go-version: "1.16"
       - uses: actions/checkout@v2
-
-      - run: ./build-miniooni.sh linux
+      - run: ./make --disable-embedding-psiphon-config -t miniooni
       - run: ./CLI/linux/amd64/miniooni --yes -nNi https://example.com web_connectivity
+
       - uses: actions/upload-artifact@v1
         with:
           name: miniooni-linux-386
           path: ./CLI/linux/386/miniooni
+
       - uses: actions/upload-artifact@v1
         with:
           name: miniooni-linux-amd64
           path: ./CLI/linux/amd64/miniooni
+
       - uses: actions/upload-artifact@v1
         with:
           name: miniooni-linux-arm
           path: ./CLI/linux/arm/miniooni
+
       - uses: actions/upload-artifact@v1
         with:
           name: miniooni-linux-arm64
           path: ./CLI/linux/arm64/miniooni
-
-      - run: ./build-miniooni.sh darwin
-      - uses: actions/upload-artifact@v1
-        with:
-          name: miniooni-darwin-amd64
-          path: ./CLI/darwin/amd64/miniooni
-
-      - run: sudo apt install --yes mingw-w64
-      - run: ./build-miniooni.sh windows
-      - uses: actions/upload-artifact@v1
-        with:
-          name: miniooni-windows-amd64.exe
-          path: ./CLI/windows/amd64/miniooni.exe

CLI/darwin/arm64/.gitignore

@@ -0,0 +1,2 @@
+/ooniprobe
+/miniooni

Readme.md

@@ -21,7 +21,7 @@ Please, make sure you tag such issues using the `ooni/probe-cli` label.
 
 Every top-level directory contains an explanatory README file.
 
-## Development setup
+## OONIProbe
 
 Be sure you have golang >= 1.16 and a C compiler (when developing for Windows, you
 need Mingw-w64 installed). You can build using:
@@ -34,6 +34,8 @@ This will generate a binary called `ooniprobe` in the current directory.
 
 ## Android bindings
 
+Make sure you have Python 3.8+ installed, then run:
+
 ```bash
 ./make -t android
 ```
@@ -47,6 +49,8 @@ are published along with the release notes.
 
 ## iOS bindings
 
+Make sure you have Python 3.8+ installed, then run:
+
 ```bash
 ./make -t ios
 ```
@@ -57,6 +61,16 @@ cannot clone private repositories in the https://github.com/ooni namespace.)
 The generated bindings are (manually) added to GitHub releases. The instructions
 explaining how to integrate these bindings are published along with the release notes.
 
+## miniooni
+
+Miniooni is the experimental OONI client used for research. Compile using:
+
+```bash
+go build -v ./internal/cmd/miniooni
+```
+
+This will generate a binary called `miniooni` in the current directory.
+
 ## Updating dependencies
 
 ```bash
@@ -68,3 +82,5 @@ go get -u -v ./... && go mod tidy
 Create an issue according to [the routine release template](
 https://github.com/ooni/probe/blob/master/.github/ISSUE_TEMPLATE/routine-sprint-releases.md)
 and perform any item inside the check-list.
+
+We build releases using `./make`, which requires Python3.8+.

build-miniooni.sh

@@ -1,31 +0,0 @@
-#!/bin/sh
-set -e
-case $1 in
-  macos|darwin)
-    export GOOS=darwin GOARCH=amd64
-    go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./internal/cmd/miniooni
-    echo "Binary ready at ./CLI/darwin/amd64/miniooni";;
-  linux)
-    export GOOS=linux GOARCH=386
-    go build -o ./CLI/linux/386 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni
-    echo "Binary ready at ./CLI/linux/386/miniooni"
-    export GOOS=linux GOARCH=amd64
-    go build -o ./CLI/linux/amd64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni
-    echo "Binary ready at ./CLI/linux/amd64/miniooni"
-    export GOOS=linux GOARCH=arm GOARM=7
-    go build -o ./CLI/linux/arm -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni
-    echo "Binary ready at ./CLI/linux/arm/miniooni"
-    export GOOS=linux GOARCH=arm64
-    go build -o ./CLI/linux/arm64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni
-    echo "Binary ready at ./CLI/linux/arm64/miniooni";;
-  windows)
-    export GOOS=windows GOARCH=386
-    go build -o ./CLI/windows/386 -ldflags="-s -w" ./internal/cmd/miniooni
-    echo "Binary ready at ./CLI/windows/386/miniooni.exe"
-    export GOOS=windows GOARCH=amd64
-    go build -o ./CLI/windows/amd64 -ldflags="-s -w" ./internal/cmd/miniooni
-    echo "Binary ready at ./CLI/windows/amd64/miniooni.exe";;
-  *)
-    echo "usage: $0 darwin|linux|windows" 1>&2
-    exit 1
-esac

make

@@ -259,7 +259,7 @@ class Engine(Protocol):
         output_variable: str,
         cmdline: List[str],
         output: List[bytes],
-        )->None:
+    ) -> None:
         """backticks executes output_variable=`*cmdline`."""
 
     def cat_sed_redirect(
@@ -283,6 +283,12 @@ class Engine(Protocol):
     ) -> None:
         """run runs the specified command line."""
 
+    def setenv(self, key: str, value: str) -> None:
+        """setenv sets an environment variable."""
+
+    def unsetenv(self, key: str) -> None:
+        """unsetenv clears an environment variable."""
+
 
 class CommandRealExecutor:
     """CommandRealExecutor executes commands."""
@@ -292,7 +298,7 @@ class CommandRealExecutor:
         output_variable: str,
         cmdline: List[str],
         output: List[bytes],
-        )->None:
+    ) -> None:
         """backticks implements Engine.backticks"""
         # Implemented in CommandDryRunner
 
@@ -331,6 +337,14 @@ class CommandRealExecutor:
                 env[key] = value
         subprocess.run(cmdline, check=True, cwd=cwd, env=env, input=inputbytes)
 
+    def setenv(self, key: str, value: str) -> None:
+        """setenv implements Engine.setenv."""
+        os.environ[key] = value
+
+    def unsetenv(self, key: str) -> None:
+        """unsetenv implements Engine.unsetenv."""
+        del os.environ[key]
+
 
 class CommandDryRunner:
     """CommandDryRunner is the dry runner."""
@@ -346,9 +360,9 @@ class CommandDryRunner:
         output_variable: str,
         cmdline: List[str],
         output: List[bytes],
-        )->None:
+    ) -> None:
         """backticks implements Engine.backticks"""
-        log('./make: {}=`{}`'.format(output_variable, shlex.join(cmdline)))
+        log("./make: {}=`{}`".format(output_variable, shlex.join(cmdline)))
         # implemented here because we want to see the result of backticks
         # command invocations when we're doing a dry run
         popen = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
@@ -402,6 +416,14 @@ class CommandDryRunner:
                 envpart += shlex.join(["{}={}".format(key, value)]) + " "
         log("./make: {}{}{}".format(cdpart, envpart, shlex.join(cmdline)))
 
+    def setenv(self, key: str, value: str) -> None:
+        """setenv implements Engine.setenv."""
+        log("./make: export {}={}".format(key, shlex.join([value])))
+
+    def unsetenv(self, key: str) -> None:
+        """unsetenv implements Engine.unsetenv."""
+        log("./make: unset {}".format(key))
+
 
 class EngineComposer:
     """EngineComposer composes two engines."""
@@ -415,7 +437,7 @@ class EngineComposer:
         output_variable: str,
         cmdline: List[str],
         output: List[bytes],
-        )->None:
+    ) -> None:
         """backticks implements Engine.backticks"""
         self._first.backticks(output_variable, cmdline, output)
         self._second.backticks(output_variable, cmdline, output)
@@ -448,6 +470,16 @@ class EngineComposer:
         self._first.run(cmdline, cwd=cwd, extra_env=extra_env, inputbytes=inputbytes)
         self._second.run(cmdline, cwd=cwd, extra_env=extra_env, inputbytes=inputbytes)
 
+    def setenv(self, key: str, value: str) -> None:
+        """setenv implements Engine.setenv."""
+        self._first.setenv(key, value)
+        self._second.setenv(key, value)
+
+    def unsetenv(self, key: str) -> None:
+        """unsetenv implements Engine.unsetenv."""
+        self._first.unsetenv(key)
+        self._second.unsetenv(key)
+
 
 def new_engine(options: Options) -> Engine:
     """new_engine creates a new engine instance"""
@@ -816,7 +848,9 @@ class BundleJAR:
             ]
         )
         engine.cat_sed_redirect(
-            [("@VERSION@", version),],
+            [
+                ("@VERSION@", version),
+            ],
             os.path.join("MOBILE", "template.pom"),
             os.path.join("MOBILE", "android", "oonimkall-{}.pom".format(version)),
         )
@@ -961,7 +995,7 @@ class OONIMKAllFramework:
                 "PATH": os.pathsep.join(
                     [
                         os.path.join(gopath(), "bin"),  # for gomobile
-                        gogo.binpath(),  # for our go fork
+                        gogo.binpath(),  # for golang/go
                         os.environ["PATH"],  # original environment
                     ]
                 ),
@@ -1040,9 +1074,142 @@ class iOS:
         oopodspec.build(engine, options)
 
 
+class MiniOONIDarwinOrWindows:
+    def __init__(self, goos: str, goarch: str):
+        self.__ext = ".exe" if goos == "windows" else ""
+        self.__name = os.path.join(".", "CLI", goos, goarch, "miniooni" + self.__ext)
+        self.__os = goos
+        self.__arch = goarch
+
+    def name(self) -> str:
+        return self.__name
+
+    def build(self, engine: Engine, options: Options) -> None:
+        if os.path.isfile(self.__name) and not options.dry_run():
+            log("./make: {}: already built".format(self.__name))
+            return
+        ooprivate = OONIProbePrivate()
+        ooprivate.build(engine, options)
+        gogo = SDKGolangGo()
+        gogo.build(engine, options)
+        log("./make: building {}...".format(self.__name))
+        ooprivate.copyfiles(engine, options)
+        engine.setenv("CGO_ENABLED", "0")
+        engine.setenv("GOOS", self.__os)
+        engine.setenv("GOARCH", self.__arch)
+        cmdline = [
+            "go",
+            "build",
+            "-o",
+            self.__name,
+            "-ldflags=-s -w",
+        ]
+        if options.debugging():
+            cmdline.append("-x")
+        if options.verbose():
+            cmdline.append("-v")
+        if not options.disable_embedding_psiphon_config():
+            cmdline.append("-tags=ooni_psiphon_config")
+        cmdline.append("./internal/cmd/miniooni")
+        engine.run(
+            cmdline,
+            extra_env={
+                "PATH": os.pathsep.join(
+                    [
+                        gogo.binpath(),  # for golang/go
+                        os.environ["PATH"],  # original path
+                    ]
+                ),
+            },
+        )
+        engine.unsetenv("CGO_ENABLED")
+        engine.unsetenv("GOARCH")
+        engine.unsetenv("GOOS")
+
+
+class MiniOONILinux:
+    def __init__(self, goarch: str):
+        self.__name = os.path.join(".", "CLI", "linux", goarch, "miniooni")
+        self.__arch = goarch
+
+    def name(self) -> str:
+        return self.__name
+
+    def build(self, engine: Engine, options: Options) -> None:
+        if os.path.isfile(self.__name) and not options.dry_run():
+            log("./make: {}: already built".format(self.__name))
+            return
+        ooprivate = OONIProbePrivate()
+        ooprivate.build(engine, options)
+        gogo = SDKGolangGo()
+        gogo.build(engine, options)
+        log("./make: building {}...".format(self.__name))
+        ooprivate.copyfiles(engine, options)
+        engine.setenv("CGO_ENABLED", "0")
+        engine.setenv("GOOS", "linux")
+        engine.setenv("GOARCH", self.__arch)
+        if self.__arch == "arm":
+            engine.setenv("GOARM", "7")
+        cmdline = [
+            "go",
+            "build",
+            "-o",
+            os.path.join("CLI", "linux", self.__arch, "miniooni"),
+            "-ldflags=-s -w -extldflags -static",
+        ]
+        if options.debugging():
+            cmdline.append("-x")
+        if options.verbose():
+            cmdline.append("-v")
+        tags = "-tags=netgo"
+        if not options.disable_embedding_psiphon_config():
+            tags += ",ooni_psiphon_config"
+        cmdline.append(tags)
+        cmdline.append("./internal/cmd/miniooni")
+        engine.run(
+            cmdline,
+            extra_env={
+                "PATH": os.pathsep.join(
+                    [
+                        gogo.binpath(),  # for golang/go
+                        os.environ["PATH"],  # original path
+                    ]
+                ),
+            },
+        )
+        engine.unsetenv("CGO_ENABLED")
+        engine.unsetenv("GOARCH")
+        engine.unsetenv("GOOS")
+        if self.__arch == "arm":
+            engine.unsetenv("GOARM")
+
+
+class MiniOONI:
+    """MiniOONI is the top-level 'miniooni' target."""
+
+    __name = "miniooni"
+
+    def name(self) -> str:
+        return self.__name
+
+    def build(self, engine: Engine, options: Options) -> None:
+        for builder in (
+            MiniOONIDarwinOrWindows("darwin", "amd64"),
+            MiniOONIDarwinOrWindows("darwin", "arm64"),
+            MiniOONILinux("386"),
+            MiniOONILinux("amd64"),
+            MiniOONILinux("arm"),
+            MiniOONILinux("arm64"),
+            MiniOONIDarwinOrWindows("windows", "386"),
+            MiniOONIDarwinOrWindows("windows", "amd64"),
+        ):
+            builder.build(engine, options)
+
+
 TARGETS: List[Target] = [
     Android(),
     iOS(),
+    MiniOONI(),
     OONIMKAllAAR(),
     OONIMKAllFrameworkZip(),
 ]
@@ -1050,11 +1217,11 @@ TARGETS: List[Target] = [
 
 def main() -> None:
     """main function"""
-    alltargets: Dict[str, Target] = dict((t.name(), t) for t in TARGETS)
-    options = ConfigFromCLI.parse(list(alltargets.keys()))
+    toptargets: Dict[str, Target] = dict((t.name(), t) for t in TARGETS)
+    options = ConfigFromCLI.parse(list(toptargets.keys()))
     engine = new_engine(options)
     # note that we check whether the target is known in parse()
-    selected = alltargets[options.target()]
+    selected = toptargets[options.target()]
     selected.build(engine, options)