From 77973301acd63bd3d70e55e34ba06b75a178197f Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 29 Apr 2021 10:30:39 +0200 Subject: [PATCH] feat(make): implement iOS build (#316) Part of https://github.com/ooni/probe/issues/1440. Basically, let us continue to update our build infrastructure so that we can release v3.10.0-beta. Now, it's the turn of iOS. --- .github/workflows/android.yml | 13 +- .github/workflows/ios.yml | 7 +- .gitignore | 1 + MOBILE/{android => }/go | 0 MOBILE/{android => }/gomobile | 0 make | 228 ++++++++++++++++++++++++++++++++-- 6 files changed, 223 insertions(+), 26 deletions(-) rename MOBILE/{android => }/go (100%) rename MOBILE/{android => }/gomobile (100%) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 0bed789..0244e13 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -4,18 +4,9 @@ on: branches: - mobile-staging - "release/**" - - "**android**" jobs: test: - runs-on: macos-latest + runs-on: ubuntu-20.04 steps: - - uses: actions/setup-go@v1 - with: - go-version: "1.16" - uses: actions/checkout@v2 - - run: brew install --cask android-sdk - - run: echo y | sdkmanager --install "platforms;android-29" - - run: echo y | sdkmanager --install "ndk-bundle" - - run: ./build-android.bash - env: - ANDROID_HOME: /usr/local/Caskroom/android-sdk/4333796 + - run: ./make --disable-embedding-psiphon-config -t ./MOBILE/android/oonimkall.aar diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index a07e105..a5a68e1 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -6,10 +6,7 @@ on: - 'release/**' jobs: test: - runs-on: macos-latest + runs-on: macos-10.15 steps: - - uses: actions/setup-go@v1 - with: - go-version: "1.16" - uses: actions/checkout@v2 - - run: ./build-ios.bash + - run: ./make --disable-embedding-psiphon-config -t ios diff --git a/.gitignore b/.gitignore index f57e825..9eb8bf7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /*.jsonl /*.tar.gz /*.zip +/.vscode /apitool /apitool.exe /coverage.cov diff --git a/MOBILE/android/go b/MOBILE/go similarity index 100% rename from MOBILE/android/go rename to MOBILE/go diff --git a/MOBILE/android/gomobile b/MOBILE/gomobile similarity index 100% rename from MOBILE/android/gomobile rename to MOBILE/gomobile diff --git a/make b/make index acde74e..d746210 100755 --- a/make +++ b/make @@ -129,6 +129,11 @@ class Options(Protocol): def debugging(self) -> bool: """debugging indicates whether to pass -x to `go build...`.""" + def disable_embedding_psiphon_config(self) -> bool: + """disable_embedding_psiphon_config indicates that the user + does not want us to embed an encrypted psiphon config file into + the binary.""" + def dry_run(self) -> bool: """dry_run indicates whether to execute commands.""" @@ -152,6 +157,7 @@ class ConfigFromCLI: def __init__(self) -> None: self._debugging = False + self._disable_embedding_psiphon_config = False self._dry_run = False self._target = "" self._verbose = False @@ -159,6 +165,9 @@ class ConfigFromCLI: def debugging(self) -> bool: return self._debugging + def disable_embedding_psiphon_config(self) -> bool: + return self._disable_embedding_psiphon_config + def dry_run(self) -> bool: return self._dry_run @@ -173,7 +182,7 @@ class ConfigFromCLI: # could be obtained quite likely w/ argparse.) __usage_string = """\ -usage: ./make [-nvx] -t target +usage: ./make [--disable-embedding-psiphon-config] [-nvx] -t target ./make -l ./make [--help|-h] @@ -181,7 +190,11 @@ The first form of the command builds the `target` specified using the `-t` command line flag. If the target has dependencies, this command will build the dependent targets first. The `-n` flag enables a dry run where the command only prints the commands it would run. The `-v` and `-x` flags -are passed directly to `go build ...` and `gomobile bind ...`. +are passed directly to `go build ...` and `gomobile bind ...`. The +`--disable-embedding-psiphon-config` flag causes this command to disable +embedding a psiphon config file into the generated binary; you should +use this option when you cannot clone the private repository containing +the psiphon configuration file. The second form of the command lists all the available targets as a pretty-printed JSON list. @@ -198,12 +211,17 @@ The third form of the command prints this help screen. def _parse(self, targets: List[str]): try: - opts, args = getopt.getopt(sys.argv[1:], "hlnt:vx", ["help"]) + opts, args = getopt.getopt( + sys.argv[1:], "hlnt:vx", ["disable-embedding-psiphon-config", "help"] + ) except getopt.GetoptError as err: self._usage(err=err.msg, exitcode=1) if args: self._usage(err="unexpected number of positional arguments", exitcode=1) for key, value in opts: + if key == "--disable-embedding-psiphon-config": + self._disable_embedding_psiphon_config = True + continue if key in ("-h", "--help"): self._usage() if key == "-l": @@ -406,6 +424,9 @@ class SDKGolangGo: def name(self) -> str: return self.__name + def binpath(self) -> str: + return os.path.join(self.__name, "go", "bin") + def build(self, engine: Engine, options: Options) -> None: if os.path.isdir(self.__name) and not options.dry_run(): log("./make: {}: already built".format(self.__name)) @@ -539,8 +560,11 @@ class OONIProbePrivate: def name(self) -> str: return self.__name - def copyfiles(self, engine: Engine) -> None: + def copyfiles(self, engine: Engine, options: Options) -> None: """copyfiles copies psiphon config to the repository.""" + if options.disable_embedding_psiphon_config(): + log("./make: copy psiphon config: disabled by command line flags") + return engine.run( [ "cp", @@ -560,6 +584,9 @@ class OONIProbePrivate: if os.path.isdir(self.__name) and not options.dry_run(): log("./make: {}: already built".format(self.__name)) return + if options.disable_embedding_psiphon_config(): + log("./make: {}: disabled by command line flags".format(self.__name)) + return log("./make: building {}...".format(self.__name)) engine.require("git", "cp") engine.run( @@ -597,7 +624,7 @@ class OONIMKAllAAR: android = SDKAndroid() android.build(engine, options) log("./make: building {}...".format(self.__name)) - ooprivate.copyfiles(engine) + ooprivate.copyfiles(engine, options) engine.require("sh", "javac") self._go_get_gomobile(engine, options, oonigo) self._gomobile_init(engine, oonigo, android) @@ -614,7 +641,7 @@ class OONIMKAllAAR: # TODO(bassosimone): find a way to run this command without # adding extra dependencies to go.mod and go.sum. cmdline: List[str] = [] - cmdline.append(os.path.join(".", "MOBILE", "android", "go")) + cmdline.append(os.path.join(".", "MOBILE", "go")) cmdline.append("get") cmdline.append("-u") if options.verbose(): @@ -642,7 +669,7 @@ class OONIMKAllAAR: android: SDKAndroid, ) -> None: cmdline: List[str] = [] - cmdline.append(os.path.join(".", "MOBILE", "android", "gomobile")) + cmdline.append(os.path.join(".", "MOBILE", "gomobile")) cmdline.append("init") engine.run( cmdline, @@ -667,7 +694,7 @@ class OONIMKAllAAR: android: SDKAndroid, ) -> None: cmdline: List[str] = [] - cmdline.append(os.path.join(".", "MOBILE", "android", "gomobile")) + cmdline.append(os.path.join(".", "MOBILE", "gomobile")) cmdline.append("bind") if options.verbose(): cmdline.append("-v") @@ -677,8 +704,9 @@ class OONIMKAllAAR: cmdline.append("android") cmdline.append("-o") cmdline.append(self.__name) - cmdline.append("-tags") - cmdline.append("ooni_psiphon_config") + if not options.disable_embedding_psiphon_config(): + cmdline.append("-tags") + cmdline.append("ooni_psiphon_config") cmdline.append("-ldflags") cmdline.append("-s -w") cmdline.append("./pkg/oonimkall") @@ -779,8 +807,188 @@ class Android: bundlejar.build(engine, options) +class OONIMKAllFramework: + """OONIMKAllFramework creates ./MOBILE/ios/oonimkall.framework.""" + + __name = os.path.join(".", "MOBILE", "ios", "oonimkall.framework") + + def name(self) -> str: + return self.__name + + def build(self, engine: Engine, options: Options) -> None: + if os.path.isfile(self.__name) and not options.dry_run(): + log("./make: {}: already built".format(self.__name)) + return + ooprivate = OONIProbePrivate() + ooprivate.build(engine, options) + gogo = SDKGolangGo() + gogo.build(engine, options) + log("./make: building {}...".format(self.__name)) + ooprivate.copyfiles(engine, options) + self._go_get_gomobile(engine, options, gogo) + self._gomobile_init(engine, gogo) + self._gomobile_bind(engine, options, gogo) + + def _go_get_gomobile( + self, + engine: Engine, + options: Options, + gogo: SDKGolangGo, + ) -> None: + # TODO(bassosimone): find a way to run this command without + # adding extra dependencies to go.mod and go.sum. + cmdline: List[str] = [] + cmdline.append(os.path.join(".", "MOBILE", "go")) + cmdline.append("get") + cmdline.append("-u") + if options.verbose(): + cmdline.append("-v") + if options.debugging(): + cmdline.append("-x") + cmdline.append("golang.org/x/mobile/cmd/gomobile@latest") + engine.run( + cmdline, + extra_env={ + "PATH": os.pathsep.join( + [ + gogo.binpath(), # so we use this binary + os.environ["PATH"], # original path + ] + ), + "GOPATH": gopath(), # where to install gomobile + }, + ) + + def _gomobile_init( + self, + engine: Engine, + gogo: SDKGolangGo, + ) -> None: + cmdline: List[str] = [] + cmdline.append(os.path.join(".", "MOBILE", "gomobile")) + cmdline.append("init") + engine.run( + cmdline, + extra_env={ + "PATH": os.pathsep.join( + [ + os.path.join(gopath(), "bin"), # for gomobile + gogo.binpath(), # for our go fork + os.environ["PATH"], # original environment + ] + ), + }, + ) + + def _gomobile_bind( + self, + engine: Engine, + options: Options, + gogo: SDKGolangGo, + ) -> None: + cmdline: List[str] = [] + cmdline.append(os.path.join(".", "MOBILE", "gomobile")) + cmdline.append("bind") + if options.verbose(): + cmdline.append("-v") + if options.debugging(): + cmdline.append("-x") + cmdline.append("-target") + cmdline.append("ios") + cmdline.append("-o") + cmdline.append(self.__name) + if not options.disable_embedding_psiphon_config(): + cmdline.append("-tags") + cmdline.append("ooni_psiphon_config") + cmdline.append("-ldflags") + cmdline.append("-s -w") + cmdline.append("./pkg/oonimkall") + engine.run( + cmdline, + extra_env={ + "PATH": os.pathsep.join( + [ + os.path.join(gopath(), "bin"), # for gomobile + gogo.binpath(), # for our go fork + os.environ["PATH"], # original environment + ] + ), + }, + ) + + +class OONIMKAllFrameworkZip: + """OONIMKAllFrameworkZip creates ./MOBILE/ios/oonimkall.framework.zip.""" + + __name = os.path.join(".", "MOBILE", "ios", "oonimkall.framework.zip") + + def name(self) -> str: + return self.__name + + def build(self, engine: Engine, options: Options) -> None: + if os.path.isfile(self.__name) and not options.dry_run(): + log("./make: {}: already built".format(self.__name)) + return + engine.require("zip", "rm") + ooframework = OONIMKAllFramework() + ooframework.build(engine, options) + log("./make: building {}...".format(self.__name)) + engine.run( + [ + "rm", + "-rf", + "oonimkall.framework.zip", + ], + cwd=os.path.join(".", "MOBILE", "ios"), + ) + engine.run( + [ + "zip", + "-yr", + "oonimkall.framework.zip", + "oonimkall.framework", + ], + cwd=os.path.join(".", "MOBILE", "ios"), + ) + + +class OONIMKAllPodspec: + """OONIMKAllPodspec creates ./MOBILE/ios/oonimkall.podspec.""" + + __name = os.path.join(".", "MOBILE", "ios", "oonimkall.podspec") + + def name(self) -> str: + return self.__name + + def build(self, engine: Engine, options: Options) -> None: + if os.path.isfile(self.__name) and not options.dry_run(): + log("./make: {}: already built".format(self.__name)) + return + version = datetime.datetime.now().strftime("%Y.%m.%d-%H%M%S") + engine.cat_sed_redirect( + "@VERSION@", + version, + os.path.join(".", "MOBILE", "template.podspec"), + self.__name, + ) + + +class iOS: + """iOS is the toplevel ios target.""" + + def name(self) -> str: + return "ios" + + def build(self, engine: Engine, options: Options) -> None: + ooframeworkzip = OONIMKAllFrameworkZip() + ooframeworkzip.build(engine, options) + oopodspec = OONIMKAllPodspec() + oopodspec.build(engine, options) + + TARGETS: List[Target] = [ Android(), + iOS(), BundleJAR(), OONIMKAllAAR(), OONIProbePrivate(),