From eb43c7994a2b6040bc09aa082975b725a1853a9b Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 5 May 2021 12:12:34 +0200 Subject: [PATCH] feat(make): build ooniprobe for darwin, windows, and linux (#328) In https://github.com/ooni/probe/issues/1466, a user is asking about arm64 builds for Debian. We already had some code for that in https://github.com/ooni/probe-cli/pull/311. Let us adapt the code to the `./make` script to have arm64 builds. While there, also adapt the code for darwin and windows. --- CLI/linux/arm64/.gitignore | 1 + CLI/linux/build | 12 +++ make | 198 ++++++++++++++++++++++++++++++++++--- 3 files changed, 198 insertions(+), 13 deletions(-) create mode 100755 CLI/linux/build diff --git a/CLI/linux/arm64/.gitignore b/CLI/linux/arm64/.gitignore index a47ad4c..0f1d4ac 100644 --- a/CLI/linux/arm64/.gitignore +++ b/CLI/linux/arm64/.gitignore @@ -1 +1,2 @@ /miniooni +/ooniprobe diff --git a/CLI/linux/build b/CLI/linux/build new file mode 100755 index 0000000..93380ca --- /dev/null +++ b/CLI/linux/build @@ -0,0 +1,12 @@ +#!/bin/sh +set -e +if [ "$GOARCH" = "" ]; then + echo 'fatal: $GOARCH is not set' 1>&2 + exit 1 +fi +set -x +apk update +apk upgrade +apk add --no-progress gcc git linux-headers musl-dev +CGO_ENABLED=1 GOOS=linux GOARCH=$GOARCH go build -o ./CLI/linux/$GOARCH/ \ + -ldflags='-s -w -extldflags "-static"' "$@" ./cmd/ooniprobe diff --git a/make b/make index b9ff4a1..f64ff4c 100755 --- a/make +++ b/make @@ -1094,6 +1094,19 @@ class MiniOONILinux: engine.run(cmdline) +# MINIOONI_TARGETS contains all miniooni targets +MINIOONI_TARGETS: List[Target] = [ + MiniOONIDarwinOrWindows("darwin", "amd64"), + MiniOONIDarwinOrWindows("darwin", "arm64"), + MiniOONILinux("386"), + MiniOONILinux("amd64"), + MiniOONILinux("arm"), + MiniOONILinux("arm64"), + MiniOONIDarwinOrWindows("windows", "386"), + MiniOONIDarwinOrWindows("windows", "amd64"), +] + + class MiniOONI: """MiniOONI is the top-level 'miniooni' target.""" @@ -1103,31 +1116,190 @@ class MiniOONI: return self.__name def build(self, engine: Engine, options: Options) -> None: - for builder in ( - MiniOONIDarwinOrWindows("darwin", "amd64"), - MiniOONIDarwinOrWindows("darwin", "arm64"), - MiniOONILinux("386"), - MiniOONILinux("amd64"), - MiniOONILinux("arm"), - MiniOONILinux("arm64"), - MiniOONIDarwinOrWindows("windows", "386"), - MiniOONIDarwinOrWindows("windows", "amd64"), - ): - builder.build(engine, options) + for target in MINIOONI_TARGETS: + target.build(engine, options) -TARGETS: List[Target] = [ +class OONIProbeLinux: + """OONIProbeLinux builds ooniprobe for Linux.""" + + def __init__(self, goarch: str): + self.__name = os.path.join(".", "CLI", "linux", goarch, "ooniprobe") + self.__arch = goarch + + def name(self) -> str: + return self.__name + + def build(self, engine: Engine, options: Options) -> None: + if os.path.isfile(self.__name) and not options.dry_run(): + log("\n./make: {}: already built".format(self.__name)) + return + ooprivate = OONIProbePrivate() + ooprivate.build(engine, options) + log("\n./make: building {}...".format(self.__name)) + ooprivate.copyfiles(engine, options) + engine.require("docker") + # make sure we have the latest version of the container image + engine.run( + [ + "docker", + "pull", + "--platform", + "linux/{}".format(self.__arch), + "golang:{}-alpine".format(goversion()), + ] + ) + # then run the build inside the container + cmdline = [ + "docker", + "run", + "--platform", + "linux/{}".format(self.__arch), + "-e", + "GOARCH={}".format(self.__arch), + "-it", + "-v", + "{}:/ooni".format(os.getcwd()), + "-w", + "/ooni", + "golang:{}-alpine".format(goversion()), + os.path.join(".", "CLI", "linux", "build"), + ] + if options.debugging(): + cmdline.append("-x") + if options.verbose(): + cmdline.append("-v") + if not options.disable_embedding_psiphon_config(): + cmdline.append("-tags=ooni_psiphon_config,netgo") + else: + cmdline.append("-tags=netgo") + engine.run(cmdline) + + +class OONIProbeWindows: + """OONIProbeWindows builds ooniprobe for Windows.""" + + def __init__(self, goarch: str): + self.__name = os.path.join(".", "CLI", "windows", goarch, "ooniprobe.exe") + self.__arch = goarch + + def name(self) -> str: + return self.__name + + def _gcc(self) -> str: + if self.__arch == "amd64": + return "x86_64-w64-mingw32-gcc" + if self.__arch == "386": + return "i686-w64-mingw32-gcc" + raise NotImplementedError + + def build(self, engine: Engine, options: Options) -> None: + if os.path.isfile(self.__name) and not options.dry_run(): + log("\n./make: {}: already built".format(self.__name)) + return + ooprivate = OONIProbePrivate() + ooprivate.build(engine, options) + gogo = SDKGolangGo() + gogo.build(engine, options) + log("\n./make: building {}...".format(self.__name)) + ooprivate.copyfiles(engine, options) + cmdline = [ + "go", + "build", + "-o", + self.__name, + "-ldflags=-s -w", + ] + if options.debugging(): + cmdline.append("-x") + if options.verbose(): + cmdline.append("-v") + if not options.disable_embedding_psiphon_config(): + cmdline.append("-tags=ooni_psiphon_config") + cmdline.append("./cmd/ooniprobe") + with Environ(engine, "GOOS", "windows"): + with Environ(engine, "GOARCH", self.__arch): + with Environ(engine, "CGO_ENABLED", "1"): + with Environ(engine, "CC", self._gcc()): + with AugmentedPath(engine, gogo.binpath()): + engine.require(self._gcc(), "go") + engine.run(cmdline) + + +class OONIProbeDarwin: + """OONIProbeDarwin builds ooniprobe for macOS.""" + + def __init__(self, goarch: str): + self.__name = os.path.join(".", "CLI", "darwin", goarch, "ooniprobe") + self.__arch = goarch + + def name(self) -> str: + return self.__name + + def build(self, engine: Engine, options: Options) -> None: + if os.path.isfile(self.__name) and not options.dry_run(): + log("\n./make: {}: already built".format(self.__name)) + return + ooprivate = OONIProbePrivate() + ooprivate.build(engine, options) + gogo = SDKGolangGo() + gogo.build(engine, options) + log("\n./make: building {}...".format(self.__name)) + ooprivate.copyfiles(engine, options) + cmdline = [ + "go", + "build", + "-o", + self.__name, + "-ldflags=-s -w", + ] + if options.debugging(): + cmdline.append("-x") + if options.verbose(): + cmdline.append("-v") + if not options.disable_embedding_psiphon_config(): + cmdline.append("-tags=ooni_psiphon_config") + cmdline.append("./cmd/ooniprobe") + with Environ(engine, "GOOS", "darwin"): + with Environ(engine, "GOARCH", self.__arch): + with Environ(engine, "CGO_ENABLED", "1"): + with AugmentedPath(engine, gogo.binpath()): + engine.require("gcc", "go") + engine.run(cmdline) + + +# OONIPROBE_TARGETS contains all the ooniprobe targets +OONIPROBE_TARGETS: List[Target] = [ + OONIProbeDarwin("amd64"), + OONIProbeDarwin("arm64"), + OONIProbeLinux("amd64"), + OONIProbeLinux("arm64"), + OONIProbeWindows("amd64"), + OONIProbeWindows("386"), +] + +# MOBILE_TARGETS contains the top-level mobile targets. +MOBILE_TARGETS: List[Target] = [ Android(), iOS(), +] + +# EXTRA_TARGETS contains extra top-level targets. +EXTRA_TARGETS: List[Target] = [ MiniOONI(), OONIMKAllAAR(), OONIMKAllFrameworkZip(), ] +# VISIBLE_TARGETS contains all the visible-from-CLI targets +VISIBLE_TARGETS: List[Target] = ( + OONIPROBE_TARGETS + MOBILE_TARGETS + EXTRA_TARGETS + MINIOONI_TARGETS +) + def main() -> None: """main function""" - toptargets: Dict[str, Target] = dict((t.name(), t) for t in TARGETS) + toptargets: Dict[str, Target] = dict((t.name(), t) for t in VISIBLE_TARGETS) options = ConfigFromCLI.parse(list(toptargets.keys())) engine = new_engine(options) # note that we check whether the target is known in parse()