c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(make): easier unified environment management (#326)

Browse Source 
Further simplifies working with the environment.

Part of https://github.com/ooni/probe/issues/1466.
This commit is contained in:
Simone Basso 2021-05-05 11:03:48 +02:00 committed by GitHub
parent 42dd9cb55e
commit 4bd1533c86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 113 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

273
make
View File

@ -15,6 +15,7 @@ import shutil
import subprocess import subprocess
import sys import sys
from typing import Any
from typing import Dict from typing import Dict
from typing import List from typing import List
from typing import NoReturn from typing import NoReturn
@ -278,13 +279,13 @@ class Engine(Protocol):
self, self,
cmdline: List[str], cmdline: List[str],
cwd: Optional[str] = None, cwd: Optional[str] = None,
extra_env: Optional[Dict[str, str]] = None,
inputbytes: Optional[bytes] = None, inputbytes: Optional[bytes] = None,
) -> None: ) -> None:
"""run runs the specified command line.""" """run runs the specified command line."""
def setenv(self, key: str, value: str) -> None: def setenv(self, key: str, value: str) -> Optional[str]:
"""setenv sets an environment variable.""" """setenv sets an environment variable and returns the
previous value of such variable (or None)."""
def unsetenv(self, key: str) -> None: def unsetenv(self, key: str) -> None:
"""unsetenv clears an environment variable.""" """unsetenv clears an environment variable."""
@ -336,26 +337,23 @@ class CommandExecutor:
self, self,
cmdline: List[str], cmdline: List[str],
cwd: Optional[str] = None, cwd: Optional[str] = None,
extra_env: Optional[Dict[str, str]] = None,
inputbytes: Optional[bytes] = None, inputbytes: Optional[bytes] = None,
) -> None: ) -> None:
"""run implements Engine.run.""" """run implements Engine.run."""
self._dry_runner.run(cmdline, cwd, extra_env, inputbytes) self._dry_runner.run(cmdline, cwd, inputbytes)
env = os.environ.copy() subprocess.run(cmdline, check=True, cwd=cwd, input=inputbytes)
if extra_env:
for key, value in extra_env.items():
env[key] = value
subprocess.run(cmdline, check=True, cwd=cwd, env=env, input=inputbytes)
def setenv(self, key: str, value: str) -> None: def setenv(self, key: str, value: str) -> Optional[str]:
"""setenv implements Engine.setenv.""" """setenv implements Engine.setenv."""
self._dry_runner.setenv(key, value) # Nothing else to do, because executable is fully
os.environ[key] = value # implemented by CommandDryRunner.
return self._dry_runner.setenv(key, value)
def unsetenv(self, key: str) -> None: def unsetenv(self, key: str) -> None:
"""unsetenv implements Engine.unsetenv.""" """unsetenv implements Engine.unsetenv."""
# Nothing else to do, because executable is fully
# implemented by CommandDryRunner.
self._dry_runner.unsetenv(key) self._dry_runner.unsetenv(key)
del os.environ[key]
class CommandDryRunner: class CommandDryRunner:
@ -414,26 +412,25 @@ class CommandDryRunner:
self, self,
cmdline: List[str], cmdline: List[str],
cwd: Optional[str] = None, cwd: Optional[str] = None,
extra_env: Optional[Dict[str, str]] = None,
inputbytes: Optional[bytes] = None, inputbytes: Optional[bytes] = None,
) -> None: ) -> None:
"""run implements Engine.run.""" """run implements Engine.run."""
cdpart = "" cdpart = ""
if cwd: if cwd:
cdpart = "cd {} && ".format(cwd) cdpart = "cd {} && ".format(cwd)
envpart = "" log("./make: {}{}".format(cdpart, shlex.join(cmdline)))
if extra_env:
for key, value in extra_env.items():
envpart += shlex.join(["{}={}".format(key, value)]) + " "
log("./make: {}{}{}".format(cdpart, envpart, shlex.join(cmdline)))
def setenv(self, key: str, value: str) -> None: def setenv(self, key: str, value: str) -> Optional[str]:
"""setenv implements Engine.setenv.""" """setenv implements Engine.setenv."""
log("./make: export {}={}".format(key, shlex.join([value]))) log("./make: export {}={}".format(key, shlex.join([value])))
prev = os.environ.get(key)
os.environ[key] = value
return prev
def unsetenv(self, key: str) -> None: def unsetenv(self, key: str) -> None:
"""unsetenv implements Engine.unsetenv.""" """unsetenv implements Engine.unsetenv."""
log("./make: unset {}".format(key)) log("./make: unset {}".format(key))
del os.environ[key]
def new_engine(options: Options) -> Engine: def new_engine(options: Options) -> Engine:
@ -444,6 +441,37 @@ def new_engine(options: Options) -> Engine:
return out return out
class Environ:
"""Environ creates a context where specific environment
variables are set. They will be restored to their previous
value when we are leaving the context."""
def __init__(self, engine: Engine, key: str, value: str):
self._engine = engine
self._key = key
self._value = value
self._prev: Optional[str] = None
def __enter__(self) -> None:
self._prev = self._engine.setenv(self._key, self._value)
def __exit__(self, type: Any, value: Any, traceback: Any) -> bool:
if self._prev is None:
self._engine.unsetenv(self._key)
return True
self._engine.setenv(self._key, self._prev)
return True
class AugmentedPath(Environ):
"""AugementedPath is an Environ that prepends the required
directory to the currently existing search path."""
def __init__(self, engine: Engine, directory: str):
value = os.pathsep.join([directory, os.environ["PATH"]])
super().__init__(engine, "PATH", value)
class Target(Protocol): class Target(Protocol):
"""Target is a target to build.""" """Target is a target to build."""
@ -470,9 +498,9 @@ class SDKGolangGo:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isdir(self.__name) and not options.dry_run(): if os.path.isdir(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
engine.require("mkdir", "curl", "shasum", "rm", "tar", "echo") engine.require("mkdir", "curl", "shasum", "rm", "tar", "echo")
filename = "go{}.{}-{}.tar.gz".format(goversion(), goos(), goarch()) filename = "go{}.{}-{}.tar.gz".format(goversion(), goos(), goarch())
url = "https://golang.org/dl/{}".format(filename) url = "https://golang.org/dl/{}".format(filename)
@ -507,11 +535,11 @@ class SDKOONIGo:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isdir(self.__name) and not options.dry_run(): if os.path.isdir(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
golang_go = SDKGolangGo() golang_go = SDKGolangGo()
golang_go.build(engine, options) golang_go.build(engine, options)
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
engine.require("git", "bash") engine.require("git", "bash")
engine.run( engine.run(
[ [
@ -526,11 +554,11 @@ class SDKOONIGo:
self.__name, self.__name,
] ]
) )
engine.run( with Environ(engine, "GOROOT_BOOTSTRAP", golang_go.goroot()):
["./make.bash"], engine.run(
cwd=os.path.join(self.__name, "src"), ["./make.bash"],
extra_env={"GOROOT_BOOTSTRAP": golang_go.goroot()}, cwd=os.path.join(self.__name, "src"),
) )
class SDKAndroid: class SDKAndroid:
@ -549,9 +577,9 @@ class SDKAndroid:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isdir(self.__name) and not options.dry_run(): if os.path.isdir(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
engine.require("mkdir", "curl", "echo", "shasum", "rm", "unzip", "mv", "java") engine.require("mkdir", "curl", "echo", "shasum", "rm", "unzip", "mv", "java")
filename = "commandlinetools-{}-{}_latest.zip".format( filename = "commandlinetools-{}-{}_latest.zip".format(
android_cmdlinetools_os(), android_cmdlinetools_version() android_cmdlinetools_os(), android_cmdlinetools_version()
@ -623,12 +651,12 @@ class OONIProbePrivate:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isdir(self.__name) and not options.dry_run(): if os.path.isdir(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
if options.disable_embedding_psiphon_config(): if options.disable_embedding_psiphon_config():
log("./make: {}: disabled by command line flags".format(self.__name)) log("\n./make: {}: disabled by command line flags".format(self.__name))
return return
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
engine.require("git", "cp") engine.require("git", "cp")
engine.run( engine.run(
[ [
@ -656,7 +684,7 @@ class OONIMKAllAAR:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isfile(self.__name) and not options.dry_run(): if os.path.isfile(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
ooprivate = OONIProbePrivate() ooprivate = OONIProbePrivate()
ooprivate.build(engine, options) ooprivate.build(engine, options)
@ -664,7 +692,7 @@ class OONIMKAllAAR:
oonigo.build(engine, options) oonigo.build(engine, options)
android = SDKAndroid() android = SDKAndroid()
android.build(engine, options) android.build(engine, options)
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
ooprivate.copyfiles(engine, options) ooprivate.copyfiles(engine, options)
engine.require("sh", "javac") engine.require("sh", "javac")
self._go_get_gomobile(engine, options, oonigo) self._go_get_gomobile(engine, options, oonigo)
@ -690,18 +718,9 @@ class OONIMKAllAAR:
if options.debugging(): if options.debugging():
cmdline.append("-x") cmdline.append("-x")
cmdline.append("golang.org/x/mobile/cmd/gomobile@latest") cmdline.append("golang.org/x/mobile/cmd/gomobile@latest")
engine.run( with Environ(engine, "GOPATH", gopath()):
cmdline, with AugmentedPath(engine, oonigo.binpath()):
extra_env={ engine.run(cmdline)
"PATH": os.pathsep.join(
[
oonigo.binpath(), # so we use our go fork
os.environ["PATH"], # original path
]
),
"GOPATH": gopath(), # where to install gomobile
},
)
def _gomobile_init( def _gomobile_init(
self, self,
@ -712,20 +731,11 @@ class OONIMKAllAAR:
cmdline: List[str] = [] cmdline: List[str] = []
cmdline.append(os.path.join(".", "MOBILE", "gomobile")) cmdline.append(os.path.join(".", "MOBILE", "gomobile"))
cmdline.append("init") cmdline.append("init")
engine.run( with Environ(engine, "ANDROID_HOME", android.home()):
cmdline, with Environ(engine, "ANDROID_NDK_HOME", android.ndk_home()):
extra_env={ with AugmentedPath(engine, oonigo.binpath()):
"PATH": os.pathsep.join( with AugmentedPath(engine, os.path.join(gopath(), "bin")):
[ engine.run(cmdline)
os.path.join(gopath(), "bin"), # for gomobile
oonigo.binpath(), # for our go fork
os.environ["PATH"], # original environment
]
),
"ANDROID_HOME": android.home(),
"ANDROID_NDK_HOME": android.ndk_home(),
},
)
def _gomobile_bind( def _gomobile_bind(
self, self,
@ -751,20 +761,11 @@ class OONIMKAllAAR:
cmdline.append("-ldflags") cmdline.append("-ldflags")
cmdline.append("-s -w") cmdline.append("-s -w")
cmdline.append("./pkg/oonimkall") cmdline.append("./pkg/oonimkall")
engine.run( with Environ(engine, "ANDROID_HOME", android.home()):
cmdline, with Environ(engine, "ANDROID_NDK_HOME", android.ndk_home()):
extra_env={ with AugmentedPath(engine, oonigo.binpath()):
"PATH": os.pathsep.join( with AugmentedPath(engine, os.path.join(gopath(), "bin")):
[ engine.run(cmdline)
os.path.join(gopath(), "bin"), # for gomobile
oonigo.binpath(), # for our go fork
os.environ["PATH"], # original environment
]
),
"ANDROID_HOME": android.home(),
"ANDROID_NDK_HOME": android.ndk_home(),
},
)
class BundleJAR: class BundleJAR:
@ -780,11 +781,11 @@ class BundleJAR:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isfile(self.__name) and not options.dry_run(): if os.path.isfile(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
oonimkall = OONIMKAllAAR() oonimkall = OONIMKAllAAR()
oonimkall.build(engine, options) oonimkall.build(engine, options)
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
engine.require("cp", "gpg", "jar") engine.require("cp", "gpg", "jar")
version = datetime.datetime.now().strftime("%Y.%m.%d-%H%M%S") version = datetime.datetime.now().strftime("%Y.%m.%d-%H%M%S")
engine.run( engine.run(
@ -859,13 +860,13 @@ class OONIMKAllFramework:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isfile(self.__name) and not options.dry_run(): if os.path.isfile(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
ooprivate = OONIProbePrivate() ooprivate = OONIProbePrivate()
ooprivate.build(engine, options) ooprivate.build(engine, options)
gogo = SDKGolangGo() gogo = SDKGolangGo()
gogo.build(engine, options) gogo.build(engine, options)
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
ooprivate.copyfiles(engine, options) ooprivate.copyfiles(engine, options)
self._go_get_gomobile(engine, options, gogo) self._go_get_gomobile(engine, options, gogo)
self._gomobile_init(engine, gogo) self._gomobile_init(engine, gogo)
@ -888,18 +889,9 @@ class OONIMKAllFramework:
if options.debugging(): if options.debugging():
cmdline.append("-x") cmdline.append("-x")
cmdline.append("golang.org/x/mobile/cmd/gomobile@latest") cmdline.append("golang.org/x/mobile/cmd/gomobile@latest")
engine.run( with AugmentedPath(engine, gogo.binpath()):
cmdline, with Environ(engine, "GOPATH", gopath()):
extra_env={ engine.run(cmdline)
"PATH": os.pathsep.join(
[
gogo.binpath(), # so we use this binary
os.environ["PATH"], # original path
]
),
"GOPATH": gopath(), # where to install gomobile
},
)
def _gomobile_init( def _gomobile_init(
self, self,
@ -909,18 +901,9 @@ class OONIMKAllFramework:
cmdline: List[str] = [] cmdline: List[str] = []
cmdline.append(os.path.join(".", "MOBILE", "gomobile")) cmdline.append(os.path.join(".", "MOBILE", "gomobile"))
cmdline.append("init") cmdline.append("init")
engine.run( with AugmentedPath(engine, os.path.join(gopath(), "bin")):
cmdline, with AugmentedPath(engine, gogo.binpath()):
extra_env={ engine.run(cmdline)
"PATH": os.pathsep.join(
[
os.path.join(gopath(), "bin"), # for gomobile
gogo.binpath(), # for our go fork
os.environ["PATH"], # original environment
]
),
},
)
def _gomobile_bind( def _gomobile_bind(
self, self,
@ -945,18 +928,9 @@ class OONIMKAllFramework:
cmdline.append("-ldflags") cmdline.append("-ldflags")
cmdline.append("-s -w") cmdline.append("-s -w")
cmdline.append("./pkg/oonimkall") cmdline.append("./pkg/oonimkall")
engine.run( with AugmentedPath(engine, os.path.join(gopath(), "bin")):
cmdline, with AugmentedPath(engine, gogo.binpath()):
extra_env={ engine.run(cmdline)
"PATH": os.pathsep.join(
[
os.path.join(gopath(), "bin"), # for gomobile
gogo.binpath(), # for golang/go
os.environ["PATH"], # original environment
]
),
},
)
class OONIMKAllFrameworkZip: class OONIMKAllFrameworkZip:
@ -969,12 +943,12 @@ class OONIMKAllFrameworkZip:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isfile(self.__name) and not options.dry_run(): if os.path.isfile(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
engine.require("zip", "rm") engine.require("zip", "rm")
ooframework = OONIMKAllFramework() ooframework = OONIMKAllFramework()
ooframework.build(engine, options) ooframework.build(engine, options)
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
engine.run( engine.run(
[ [
"rm", "rm",
@ -1041,17 +1015,14 @@ class MiniOONIDarwinOrWindows:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isfile(self.__name) and not options.dry_run(): if os.path.isfile(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
ooprivate = OONIProbePrivate() ooprivate = OONIProbePrivate()
ooprivate.build(engine, options) ooprivate.build(engine, options)
gogo = SDKGolangGo() gogo = SDKGolangGo()
gogo.build(engine, options) gogo.build(engine, options)
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
ooprivate.copyfiles(engine, options) ooprivate.copyfiles(engine, options)
engine.setenv("CGO_ENABLED", "0")
engine.setenv("GOOS", self.__os)
engine.setenv("GOARCH", self.__arch)
cmdline = [ cmdline = [
"go", "go",
"build", "build",
@ -1066,20 +1037,11 @@ class MiniOONIDarwinOrWindows:
if not options.disable_embedding_psiphon_config(): if not options.disable_embedding_psiphon_config():
cmdline.append("-tags=ooni_psiphon_config") cmdline.append("-tags=ooni_psiphon_config")
cmdline.append("./internal/cmd/miniooni") cmdline.append("./internal/cmd/miniooni")
engine.run( with Environ(engine, "GOOS", self.__os):
cmdline, with Environ(engine, "GOARCH", self.__arch):
extra_env={ with Environ(engine, "CGO_ENABLED", "0"):
"PATH": os.pathsep.join( with AugmentedPath(engine, gogo.binpath()):
[ engine.run(cmdline)
gogo.binpath(), # for golang/go
os.environ["PATH"], # original path
]
),
},
)
engine.unsetenv("CGO_ENABLED")
engine.unsetenv("GOARCH")
engine.unsetenv("GOOS")
class MiniOONILinux: class MiniOONILinux:
@ -1092,19 +1054,21 @@ class MiniOONILinux:
def build(self, engine: Engine, options: Options) -> None: def build(self, engine: Engine, options: Options) -> None:
if os.path.isfile(self.__name) and not options.dry_run(): if os.path.isfile(self.__name) and not options.dry_run():
log("./make: {}: already built".format(self.__name)) log("\n./make: {}: already built".format(self.__name))
return return
ooprivate = OONIProbePrivate() ooprivate = OONIProbePrivate()
ooprivate.build(engine, options) ooprivate.build(engine, options)
gogo = SDKGolangGo() gogo = SDKGolangGo()
gogo.build(engine, options) gogo.build(engine, options)
log("./make: building {}...".format(self.__name)) log("\n./make: building {}...".format(self.__name))
ooprivate.copyfiles(engine, options) ooprivate.copyfiles(engine, options)
engine.setenv("CGO_ENABLED", "0")
engine.setenv("GOOS", "linux")
engine.setenv("GOARCH", self.__arch)
if self.__arch == "arm": if self.__arch == "arm":
engine.setenv("GOARM", "7") with Environ(engine, "GOARM", "7"):
self._build(engine, options, gogo)
else:
self._build(engine, options, gogo)
def _build(self, engine: Engine, options: Options, gogo: SDKGolangGo) -> None:
cmdline = [ cmdline = [
"go", "go",
"build", "build",
@ -1121,22 +1085,11 @@ class MiniOONILinux:
tags += ",ooni_psiphon_config" tags += ",ooni_psiphon_config"
cmdline.append(tags) cmdline.append(tags)
cmdline.append("./internal/cmd/miniooni") cmdline.append("./internal/cmd/miniooni")
engine.run( with Environ(engine, "GOOS", "linux"):
cmdline, with Environ(engine, "GOARCH", self.__arch):
extra_env={ with Environ(engine, "CGO_ENABLED", "0"):
"PATH": os.pathsep.join( with AugmentedPath(engine, gogo.binpath()):
[ engine.run(cmdline)
gogo.binpath(), # for golang/go
os.environ["PATH"], # original path
]
),
},
)
engine.unsetenv("CGO_ENABLED")
engine.unsetenv("GOARCH")
engine.unsetenv("GOOS")
if self.__arch == "arm":
engine.unsetenv("GOARM")
class MiniOONI: class MiniOONI: