diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 3d229e8..9c7b4f1 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,8 +1,10 @@ +# android verifies we can still build for Android name: android on: push: branches: - "release/**" + - "master" jobs: test: runs-on: ubuntu-20.04 diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 6dce9f9..beb2a46 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -1,8 +1,10 @@ +# ios verifies we can still build for iOS name: ios on: push: branches: - 'release/**' + - 'master' jobs: test: runs-on: macos-10.15 diff --git a/.github/workflows/miniooni.yml b/.github/workflows/miniooni.yml index 350d3bd..611fcd1 100644 --- a/.github/workflows/miniooni.yml +++ b/.github/workflows/miniooni.yml @@ -1,47 +1,33 @@ +# miniooni checks whether we can build the research client miniooni +# and publishes all linux binaries as artefacts. There is no point in +# publishing windows or darwin binaries b/c they are not signed. name: miniooni on: push: - branches: - - 'release/**' - schedule: - - cron: "0 0 * * */1" jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - - uses: actions/setup-go@v1 - with: - go-version: "1.16" - uses: actions/checkout@v2 - - - run: ./build-miniooni.sh linux + - run: ./make --disable-embedding-psiphon-config -t miniooni - run: ./CLI/linux/amd64/miniooni --yes -nNi https://example.com web_connectivity + - uses: actions/upload-artifact@v1 with: name: miniooni-linux-386 path: ./CLI/linux/386/miniooni + - uses: actions/upload-artifact@v1 with: name: miniooni-linux-amd64 path: ./CLI/linux/amd64/miniooni + - uses: actions/upload-artifact@v1 with: name: miniooni-linux-arm path: ./CLI/linux/arm/miniooni + - uses: actions/upload-artifact@v1 with: name: miniooni-linux-arm64 path: ./CLI/linux/arm64/miniooni - - - run: ./build-miniooni.sh darwin - - uses: actions/upload-artifact@v1 - with: - name: miniooni-darwin-amd64 - path: ./CLI/darwin/amd64/miniooni - - - run: sudo apt install --yes mingw-w64 - - run: ./build-miniooni.sh windows - - uses: actions/upload-artifact@v1 - with: - name: miniooni-windows-amd64.exe - path: ./CLI/windows/amd64/miniooni.exe diff --git a/CLI/darwin/arm64/.gitignore b/CLI/darwin/arm64/.gitignore new file mode 100644 index 0000000..ab810d8 --- /dev/null +++ b/CLI/darwin/arm64/.gitignore @@ -0,0 +1,2 @@ +/ooniprobe +/miniooni diff --git a/Readme.md b/Readme.md index 717a3be..ea0cd09 100644 --- a/Readme.md +++ b/Readme.md @@ -21,7 +21,7 @@ Please, make sure you tag such issues using the `ooni/probe-cli` label. Every top-level directory contains an explanatory README file. -## Development setup +## OONIProbe Be sure you have golang >= 1.16 and a C compiler (when developing for Windows, you need Mingw-w64 installed). You can build using: @@ -34,6 +34,8 @@ This will generate a binary called `ooniprobe` in the current directory. ## Android bindings +Make sure you have Python 3.8+ installed, then run: + ```bash ./make -t android ``` @@ -47,6 +49,8 @@ are published along with the release notes. ## iOS bindings +Make sure you have Python 3.8+ installed, then run: + ```bash ./make -t ios ``` @@ -57,6 +61,16 @@ cannot clone private repositories in the https://github.com/ooni namespace.) The generated bindings are (manually) added to GitHub releases. The instructions explaining how to integrate these bindings are published along with the release notes. +## miniooni + +Miniooni is the experimental OONI client used for research. Compile using: + +```bash +go build -v ./internal/cmd/miniooni +``` + +This will generate a binary called `miniooni` in the current directory. + ## Updating dependencies ```bash @@ -68,3 +82,5 @@ go get -u -v ./... && go mod tidy Create an issue according to [the routine release template]( https://github.com/ooni/probe/blob/master/.github/ISSUE_TEMPLATE/routine-sprint-releases.md) and perform any item inside the check-list. + +We build releases using `./make`, which requires Python3.8+. diff --git a/build-miniooni.sh b/build-miniooni.sh deleted file mode 100755 index 18b2434..0000000 --- a/build-miniooni.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -set -e -case $1 in - macos|darwin) - export GOOS=darwin GOARCH=amd64 - go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./internal/cmd/miniooni - echo "Binary ready at ./CLI/darwin/amd64/miniooni";; - linux) - export GOOS=linux GOARCH=386 - go build -o ./CLI/linux/386 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni - echo "Binary ready at ./CLI/linux/386/miniooni" - export GOOS=linux GOARCH=amd64 - go build -o ./CLI/linux/amd64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni - echo "Binary ready at ./CLI/linux/amd64/miniooni" - export GOOS=linux GOARCH=arm GOARM=7 - go build -o ./CLI/linux/arm -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni - echo "Binary ready at ./CLI/linux/arm/miniooni" - export GOOS=linux GOARCH=arm64 - go build -o ./CLI/linux/arm64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni - echo "Binary ready at ./CLI/linux/arm64/miniooni";; - windows) - export GOOS=windows GOARCH=386 - go build -o ./CLI/windows/386 -ldflags="-s -w" ./internal/cmd/miniooni - echo "Binary ready at ./CLI/windows/386/miniooni.exe" - export GOOS=windows GOARCH=amd64 - go build -o ./CLI/windows/amd64 -ldflags="-s -w" ./internal/cmd/miniooni - echo "Binary ready at ./CLI/windows/amd64/miniooni.exe";; - *) - echo "usage: $0 darwin|linux|windows" 1>&2 - exit 1 -esac diff --git a/make b/make index 072ad56..f18c062 100755 --- a/make +++ b/make @@ -259,7 +259,7 @@ class Engine(Protocol): output_variable: str, cmdline: List[str], output: List[bytes], - )->None: + ) -> None: """backticks executes output_variable=`*cmdline`.""" def cat_sed_redirect( @@ -283,6 +283,12 @@ class Engine(Protocol): ) -> None: """run runs the specified command line.""" + def setenv(self, key: str, value: str) -> None: + """setenv sets an environment variable.""" + + def unsetenv(self, key: str) -> None: + """unsetenv clears an environment variable.""" + class CommandRealExecutor: """CommandRealExecutor executes commands.""" @@ -292,7 +298,7 @@ class CommandRealExecutor: output_variable: str, cmdline: List[str], output: List[bytes], - )->None: + ) -> None: """backticks implements Engine.backticks""" # Implemented in CommandDryRunner @@ -331,6 +337,14 @@ class CommandRealExecutor: env[key] = value subprocess.run(cmdline, check=True, cwd=cwd, env=env, input=inputbytes) + def setenv(self, key: str, value: str) -> None: + """setenv implements Engine.setenv.""" + os.environ[key] = value + + def unsetenv(self, key: str) -> None: + """unsetenv implements Engine.unsetenv.""" + del os.environ[key] + class CommandDryRunner: """CommandDryRunner is the dry runner.""" @@ -346,9 +360,9 @@ class CommandDryRunner: output_variable: str, cmdline: List[str], output: List[bytes], - )->None: + ) -> None: """backticks implements Engine.backticks""" - log('./make: {}=`{}`'.format(output_variable, shlex.join(cmdline))) + log("./make: {}=`{}`".format(output_variable, shlex.join(cmdline))) # implemented here because we want to see the result of backticks # command invocations when we're doing a dry run popen = subprocess.Popen(cmdline, stdout=subprocess.PIPE) @@ -402,6 +416,14 @@ class CommandDryRunner: envpart += shlex.join(["{}={}".format(key, value)]) + " " log("./make: {}{}{}".format(cdpart, envpart, shlex.join(cmdline))) + def setenv(self, key: str, value: str) -> None: + """setenv implements Engine.setenv.""" + log("./make: export {}={}".format(key, shlex.join([value]))) + + def unsetenv(self, key: str) -> None: + """unsetenv implements Engine.unsetenv.""" + log("./make: unset {}".format(key)) + class EngineComposer: """EngineComposer composes two engines.""" @@ -415,7 +437,7 @@ class EngineComposer: output_variable: str, cmdline: List[str], output: List[bytes], - )->None: + ) -> None: """backticks implements Engine.backticks""" self._first.backticks(output_variable, cmdline, output) self._second.backticks(output_variable, cmdline, output) @@ -448,6 +470,16 @@ class EngineComposer: self._first.run(cmdline, cwd=cwd, extra_env=extra_env, inputbytes=inputbytes) self._second.run(cmdline, cwd=cwd, extra_env=extra_env, inputbytes=inputbytes) + def setenv(self, key: str, value: str) -> None: + """setenv implements Engine.setenv.""" + self._first.setenv(key, value) + self._second.setenv(key, value) + + def unsetenv(self, key: str) -> None: + """unsetenv implements Engine.unsetenv.""" + self._first.unsetenv(key) + self._second.unsetenv(key) + def new_engine(options: Options) -> Engine: """new_engine creates a new engine instance""" @@ -816,7 +848,9 @@ class BundleJAR: ] ) engine.cat_sed_redirect( - [("@VERSION@", version),], + [ + ("@VERSION@", version), + ], os.path.join("MOBILE", "template.pom"), os.path.join("MOBILE", "android", "oonimkall-{}.pom".format(version)), ) @@ -961,7 +995,7 @@ class OONIMKAllFramework: "PATH": os.pathsep.join( [ os.path.join(gopath(), "bin"), # for gomobile - gogo.binpath(), # for our go fork + gogo.binpath(), # for golang/go os.environ["PATH"], # original environment ] ), @@ -1040,9 +1074,142 @@ class iOS: oopodspec.build(engine, options) +class MiniOONIDarwinOrWindows: + def __init__(self, goos: str, goarch: str): + self.__ext = ".exe" if goos == "windows" else "" + self.__name = os.path.join(".", "CLI", goos, goarch, "miniooni" + self.__ext) + self.__os = goos + self.__arch = goarch + + def name(self) -> str: + return self.__name + + def build(self, engine: Engine, options: Options) -> None: + if os.path.isfile(self.__name) and not options.dry_run(): + log("./make: {}: already built".format(self.__name)) + return + ooprivate = OONIProbePrivate() + ooprivate.build(engine, options) + gogo = SDKGolangGo() + gogo.build(engine, options) + log("./make: building {}...".format(self.__name)) + ooprivate.copyfiles(engine, options) + engine.setenv("CGO_ENABLED", "0") + engine.setenv("GOOS", self.__os) + engine.setenv("GOARCH", self.__arch) + cmdline = [ + "go", + "build", + "-o", + self.__name, + "-ldflags=-s -w", + ] + if options.debugging(): + cmdline.append("-x") + if options.verbose(): + cmdline.append("-v") + if not options.disable_embedding_psiphon_config(): + cmdline.append("-tags=ooni_psiphon_config") + cmdline.append("./internal/cmd/miniooni") + engine.run( + cmdline, + extra_env={ + "PATH": os.pathsep.join( + [ + gogo.binpath(), # for golang/go + os.environ["PATH"], # original path + ] + ), + }, + ) + engine.unsetenv("CGO_ENABLED") + engine.unsetenv("GOARCH") + engine.unsetenv("GOOS") + + +class MiniOONILinux: + def __init__(self, goarch: str): + self.__name = os.path.join(".", "CLI", "linux", goarch, "miniooni") + self.__arch = goarch + + def name(self) -> str: + return self.__name + + def build(self, engine: Engine, options: Options) -> None: + if os.path.isfile(self.__name) and not options.dry_run(): + log("./make: {}: already built".format(self.__name)) + return + ooprivate = OONIProbePrivate() + ooprivate.build(engine, options) + gogo = SDKGolangGo() + gogo.build(engine, options) + log("./make: building {}...".format(self.__name)) + ooprivate.copyfiles(engine, options) + engine.setenv("CGO_ENABLED", "0") + engine.setenv("GOOS", "linux") + engine.setenv("GOARCH", self.__arch) + if self.__arch == "arm": + engine.setenv("GOARM", "7") + cmdline = [ + "go", + "build", + "-o", + os.path.join("CLI", "linux", self.__arch, "miniooni"), + "-ldflags=-s -w -extldflags -static", + ] + if options.debugging(): + cmdline.append("-x") + if options.verbose(): + cmdline.append("-v") + tags = "-tags=netgo" + if not options.disable_embedding_psiphon_config(): + tags += ",ooni_psiphon_config" + cmdline.append(tags) + cmdline.append("./internal/cmd/miniooni") + engine.run( + cmdline, + extra_env={ + "PATH": os.pathsep.join( + [ + gogo.binpath(), # for golang/go + os.environ["PATH"], # original path + ] + ), + }, + ) + engine.unsetenv("CGO_ENABLED") + engine.unsetenv("GOARCH") + engine.unsetenv("GOOS") + if self.__arch == "arm": + engine.unsetenv("GOARM") + + +class MiniOONI: + """MiniOONI is the top-level 'miniooni' target.""" + + __name = "miniooni" + + def name(self) -> str: + return self.__name + + def build(self, engine: Engine, options: Options) -> None: + for builder in ( + MiniOONIDarwinOrWindows("darwin", "amd64"), + MiniOONIDarwinOrWindows("darwin", "arm64"), + MiniOONILinux("386"), + MiniOONILinux("amd64"), + MiniOONILinux("arm"), + MiniOONILinux("arm64"), + MiniOONIDarwinOrWindows("windows", "386"), + MiniOONIDarwinOrWindows("windows", "amd64"), + ): + builder.build(engine, options) + + TARGETS: List[Target] = [ Android(), iOS(), + MiniOONI(), OONIMKAllAAR(), OONIMKAllFrameworkZip(), ] @@ -1050,11 +1217,11 @@ TARGETS: List[Target] = [ def main() -> None: """main function""" - alltargets: Dict[str, Target] = dict((t.name(), t) for t in TARGETS) - options = ConfigFromCLI.parse(list(alltargets.keys())) + toptargets: Dict[str, Target] = dict((t.name(), t) for t in TARGETS) + options = ConfigFromCLI.parse(list(toptargets.keys())) engine = new_engine(options) # note that we check whether the target is known in parse() - selected = alltargets[options.target()] + selected = toptargets[options.target()] selected.build(engine, options)