e753e57da3
We're still working on https://github.com/ooni/probe/issues/1466. The idea here is to teach the GH action for Linux to publish the debian package for arm64. When this is done, we can cleanup legacy build scripts and GH actions, because there is no remaining use case for them: we now build everything using the `./make` tool.
562 lines
18 KiB
Python
Executable File
562 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
"""
|
|
Builds deb packages and uploads them to S3-compatible storage or bintray.
|
|
Works locally and on GitHub Actions and CircleCI
|
|
Detects which package[s] need to be built.
|
|
Support "release" and PR/testing archives.
|
|
|
|
scan - scan the current repository for packages to be built
|
|
build - locate and build packages
|
|
upload <filename> - upload one package to S3 or Bintray
|
|
ci - detect CircleCI PRs, build and upload packages
|
|
delete_from_archive <filename> - delete filename from archive
|
|
|
|
Features:
|
|
- Implement CI/CD workflow using package archives
|
|
- Support adding packages to an existing S3 archive without requiring a
|
|
local mirror
|
|
- Support multiple packages in the same git repository
|
|
- GPG signing
|
|
- Phased updates (rolling deployments)
|
|
- Update changelogs automatically
|
|
- Easy to debug
|
|
"""
|
|
|
|
# TODO: fix S3 credentials passing security
|
|
# TODO: Phased-Update-Percentage
|
|
|
|
# debdeps: git
|
|
from argparse import ArgumentParser
|
|
import os
|
|
from os import getenv
|
|
from pathlib import Path
|
|
from requests.auth import HTTPBasicAuth
|
|
from subprocess import run
|
|
from tempfile import mkdtemp, NamedTemporaryFile
|
|
from textwrap import dedent
|
|
from time import sleep
|
|
from typing import List
|
|
from hashlib import sha256
|
|
import requests
|
|
import sys
|
|
|
|
try:
|
|
import gnupg
|
|
except ImportError:
|
|
gnupg = None
|
|
|
|
# TODO remove these
|
|
BINTRAY_API = "https://bintray.com/api/v1"
|
|
DEFAULT_ORG = "ooni"
|
|
DEFAULT_PR_REPO = "internal-pull-requests"
|
|
DEFAULT_MASTER_REPO = "internal-master"
|
|
DEFAULT_REPO = "internal-pull-requests"
|
|
|
|
EXAMPLE_CONFIG = """
|
|
"""
|
|
|
|
assert sys.version_info >= (3, 7, 0), "Python 3.7.0 or later is required"
|
|
|
|
conf = None
|
|
|
|
|
|
def run2(cmd, **kw):
|
|
if conf.show_commands:
|
|
print(f"Running {cmd}\nKW: {kw}")
|
|
p = run(cmd.split(), capture_output=True, **kw)
|
|
if p.returncode != 0:
|
|
stdout = p.stdout.decode().strip()
|
|
print(f"--stdout--\n{stdout}\n----\n")
|
|
stderr = p.stderr.decode().strip()
|
|
print(f"--stderr--\n{stderr}\n----\n")
|
|
raise Exception(f"'{cmd}' returned: {p.returncode}")
|
|
return p.stdout.decode().strip()
|
|
|
|
|
|
def runi(cmd: str, cwd: Path, sudo=False) -> None:
|
|
if sudo:
|
|
cmd = f"sudo {cmd}"
|
|
run(cmd.split(), cwd=cwd, check=True)
|
|
|
|
|
|
def runc(cmd):
|
|
print("Running:", cmd)
|
|
r = run(cmd.split(), capture_output=True)
|
|
print("Retcode", r.returncode)
|
|
return r.returncode, r.stdout.decode()
|
|
|
|
|
|
def detect_changed_packages() -> List[Path]:
|
|
"""Detects files named debian/changelog
|
|
that have been changed in the current branch
|
|
"""
|
|
DCH = "debian/changelog"
|
|
# TODO: find a cleaner method:
|
|
commit = run2("git merge-base remotes/origin/master HEAD")
|
|
changes = run2(f"git diff --name-only {commit}")
|
|
pkgs = set()
|
|
for c in changes.splitlines():
|
|
c = Path(c)
|
|
if c.as_posix().endswith(DCH):
|
|
pkgs.add(c.parent.parent)
|
|
continue
|
|
while c.name:
|
|
if c.joinpath(DCH).is_file():
|
|
pkgs.add(c)
|
|
c = c.parent
|
|
|
|
return sorted(pkgs)
|
|
|
|
|
|
def trim_compare(url: str) -> str:
|
|
"""Shorten GitHub URLs used to compare changes"""
|
|
if url.startswith("https://github.com/") and "..." in url:
|
|
base, commits = url.rsplit("/", 1)
|
|
if len(commits) == 83:
|
|
beginning = commits[0:8]
|
|
end = commits[43 : 43 + 8]
|
|
return f"{base}/{beginning}...{end}"
|
|
|
|
return url
|
|
|
|
|
|
def _set_pkg_version_from_circleci(p, ver):
|
|
comp = trim_compare(getenv("CIRCLE_COMPARE_URL", "")) # show changes in VCS
|
|
if not comp:
|
|
# https://discuss.circleci.com/t/circle-compare-url-is-empty/24549/8
|
|
comp = getenv("CIRCLE_PULL_REQUEST")
|
|
|
|
if getenv("CIRCLE_PULL_REQUEST"):
|
|
# This is a PR: build ~pr<N>-<N> version. CIRCLE_PR_NUMBER is broken
|
|
pr_num = getenv("CIRCLE_PULL_REQUEST", "").rsplit("/", 1)[-1]
|
|
build_num = getenv("CIRCLE_BUILD_NUM")
|
|
ver = f"{ver}~pr{pr_num}-{build_num}"
|
|
print(f"CircleCI Pull Request detected - using version {ver}")
|
|
run2(f"dch -b -v {ver} {comp}", cwd=p)
|
|
run2(f"dch -r {ver} {comp}", cwd=p)
|
|
ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p)
|
|
assert ver == ver2, ver + " <--> " + ver2
|
|
|
|
elif getenv("CIRCLE_BRANCH") == "master":
|
|
# This is a build outside of a PR and in the mainline branch
|
|
print(f"CircleCI mainline build detected - using version {ver}")
|
|
run2(f"dch -b -v {ver} {comp}", cwd=p)
|
|
run2(f"dch -r {ver} {comp}", cwd=p)
|
|
ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p)
|
|
assert ver == ver2, ver + " <--> " + ver2
|
|
|
|
else:
|
|
# This is a build for a new branch but without a PR: ignore it
|
|
return []
|
|
|
|
|
|
def _set_pkg_version_from_github_actions(p, ver):
|
|
"""When running in GitHub Actions, access env vars to set
|
|
the package version"""
|
|
# GITHUB_REF syntax: refs/heads/<branch-name> or refs/pull/<PR#>/merge
|
|
gh_ref = getenv("GITHUB_REF")
|
|
try:
|
|
pr_num = int(gh_ref.split("/")[2])
|
|
except ValueError:
|
|
pr_num = None
|
|
|
|
gh_run_number = int(getenv("GITHUB_RUN_NUMBER"))
|
|
print(f"GitHub Actions PR #: {pr_num} Run #: {gh_run_number}")
|
|
print("SHA " + getenv("GITHUB_SHA"))
|
|
|
|
comp = ""
|
|
if pr_num is None:
|
|
if gh_ref.endswith("/master"):
|
|
print(f"GitHub release build detected - using version {ver}")
|
|
run2(f"dch -b -v {ver} ''", cwd=p)
|
|
run2(f"dch --release ''", cwd=p)
|
|
ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p)
|
|
assert ver == ver2, ver + " <--> " + ver2
|
|
return True
|
|
|
|
else:
|
|
print("Not a PR or release build. Skipping.") # run by "on: push"
|
|
return False
|
|
|
|
else:
|
|
# This is a PR: build ~pr<N>-<N> version.
|
|
ver = f"{ver}~pr{pr_num}-{gh_run_number}"
|
|
print(f"GitHub Pull Request detected - using version {ver}")
|
|
run2(f"dch -b -v {ver} ''", cwd=p)
|
|
run2(f"dch --release ''", cwd=p)
|
|
ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p)
|
|
assert ver == ver2, ver + " <--> " + ver2
|
|
return True
|
|
|
|
|
|
def buildpkg(p) -> List[Path]:
|
|
"""Build one package, installing required dependencies"""
|
|
print(f"Building package in {p}")
|
|
ver = run2("dpkg-parsechangelog --show-field Version", cwd=p)
|
|
assert ver, f"No version number found in {p}/debian/changelog"
|
|
sudo = True
|
|
should_build = False
|
|
if getenv("CIRCLECI"):
|
|
# Running in CircleCI
|
|
sudo = False
|
|
_set_pkg_version_from_circleci(p, ver)
|
|
elif getenv("GITHUB_EVENT_PATH"):
|
|
sudo = False
|
|
should_build = _set_pkg_version_from_github_actions(p, ver)
|
|
|
|
if not should_build:
|
|
return []
|
|
|
|
runi("apt-get build-dep -qy --no-install-recommends .", p, sudo=sudo)
|
|
runi("fakeroot debian/rules build", p)
|
|
runi("fakeroot debian/rules binary", p)
|
|
with p.joinpath("debian/files").open() as f:
|
|
return [p.parent.joinpath(line.split()[0]) for line in f]
|
|
|
|
|
|
def detect_archive_backend():
|
|
if getenv("BINTRAY_USERNAME") and getenv("BINTRAY_API_KEY"):
|
|
return "bintray"
|
|
|
|
if getenv("AWS_ACCESS_KEY_ID") and getenv("AWS_SECRET_ACCESS_KEY"):
|
|
return "s3"
|
|
|
|
|
|
def setup_gpg_key(keyfp, tmpdir):
|
|
"""Import key from env var or use existing keyring"""
|
|
if gnupg is None:
|
|
print("Please install python3-gnupg")
|
|
sys.exit(1)
|
|
|
|
if keyfp is None and "DEB_GPG_KEY" not in os.environ:
|
|
print(
|
|
"Error: place a GPG key in the DEB_GPG_KEY env var or"
|
|
" fetch it from the local keyring using --gpg-key-fp"
|
|
)
|
|
sys.exit(1)
|
|
|
|
if "DEB_GPG_KEY" in os.environ:
|
|
gpg = gnupg.GPG(gnupghome=tmpdir.as_posix())
|
|
import_result = gpg.import_keys(os.getenv("DEB_GPG_KEY"))
|
|
assert import_result.count == 1
|
|
fp = import_result.fingerprints[0]
|
|
if keyfp:
|
|
assert keyfp == fp
|
|
|
|
else:
|
|
gpg = gnupg.GPG()
|
|
assert gpg.list_keys(keys=keyfp)
|
|
|
|
return gpg, keyfp
|
|
|
|
|
|
def ci(args) -> None:
|
|
# TODO: detect sudo presence
|
|
|
|
backend_name = detect_archive_backend()
|
|
if backend_name == "bintray":
|
|
backend = Bintray()
|
|
elif backend_name == "s3":
|
|
backend = S3()
|
|
else:
|
|
print(
|
|
"Either set BINTRAY_USERNAME / BINTRAY_API_KEY env vars or "
|
|
"AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY"
|
|
)
|
|
sys.exit(1)
|
|
del backend_name
|
|
|
|
run2("apt-get update -q")
|
|
run2("apt-get install -qy --no-install-recommends git")
|
|
pkgdirs = detect_changed_packages()
|
|
if not pkgdirs:
|
|
print("Nothing to build")
|
|
return
|
|
|
|
print(f"Building {pkgdirs}")
|
|
run2("apt-get install -qy --no-install-recommends devscripts")
|
|
|
|
pkgs_lists = [buildpkg(pd) for pd in pkgdirs]
|
|
print(f"Processing {pkgs_lists}")
|
|
for pli in pkgs_lists:
|
|
for p in pli:
|
|
backend.upload(p, args)
|
|
|
|
|
|
def build() -> None:
|
|
"""Run manual build on workstation"""
|
|
pkgdirs = detect_changed_packages()
|
|
pkgs_lists = [buildpkg(pd) for pd in pkgdirs]
|
|
print("Outputs:")
|
|
for pli in pkgs_lists:
|
|
for p in pli:
|
|
print(p)
|
|
|
|
|
|
class DuplicatePkgError(Exception):
|
|
pass
|
|
|
|
|
|
def check_duplicate_package(pkgblock, packages_text):
|
|
li = pkgblock.splitlines()
|
|
assert li[0].startswith("Package: "), li
|
|
pname = li[0].split(" ", 1)[1]
|
|
assert li[1].startswith("Version: "), li
|
|
pver = li[1].split(" ", 1)[1]
|
|
assert li[2].startswith("Architecture: "), li
|
|
parch = li[2].split(" ", 1)[1]
|
|
|
|
m = f"Package: {pname}\nVersion: {pver}\nArchitecture: {parch}"
|
|
if m in packages_text:
|
|
raise DuplicatePkgError()
|
|
|
|
|
|
class Bintray:
|
|
"""Bintray backend"""
|
|
|
|
def __init__(self):
|
|
self._btuser = getenv("BINTRAY_USERNAME")
|
|
assert self._btuser, "Missing BINTRAY_USERNAME"
|
|
|
|
def upload(self, fi, args) -> None:
|
|
"""Upload to Bintray"""
|
|
# FIXME: specify repo
|
|
assert repo, "Please specify a repository"
|
|
assert fi.is_file()
|
|
pname, pver, arch = fi.name.split("_")
|
|
auth = HTTPBasicAuth(self._btuser, getenv("BINTRAY_API_KEY"))
|
|
dist = "unstable"
|
|
url = (
|
|
f"{BINTRAY_API}/content/{args.org}/{repo}/{pname}/{pver}/{fi.name};"
|
|
f"deb_distribution={dist};deb_component=main;deb_architecture=amd64;publish=1"
|
|
)
|
|
with open(fi, "rb") as f:
|
|
resp = requests.put(url, auth=auth, data=f)
|
|
|
|
if not resp.ok:
|
|
print(f"Error {resp.text} when calling {resp.request.url}")
|
|
sys.exit(1)
|
|
|
|
def delete_package(self, args, extra) -> None:
|
|
"""Delete package from Bintray"""
|
|
auth = HTTPBasicAuth(self._btuser, getenv("BINTRAY_API_KEY"))
|
|
filename = extra[0]
|
|
assert filename.endswith(".deb")
|
|
assert args.repo, "Please specify a repository"
|
|
url = f"{BINTRAY_API}/content/{args.org}/{args.repo}/{filename}"
|
|
resp = requests.delete(url, auth=auth)
|
|
if not resp.ok:
|
|
print(f"Error {resp.text} when calling {resp.request.url}")
|
|
sys.exit(1)
|
|
|
|
|
|
class S3:
|
|
"""S3 backend"""
|
|
|
|
def generate_release_file(self, conf, sha, size):
|
|
r = dedent(
|
|
f"""
|
|
Acquire-By-Hash: no
|
|
Architectures: {conf.arch}
|
|
Codename: {conf.distro}
|
|
Components: main
|
|
Date: Thu, 07 Nov 2019 14:23:37 UTC
|
|
Origin: private
|
|
Valid-Until: Thu, 14 Nov 2029 14:23:37 UTC
|
|
SHA256:
|
|
{sha} {size} main/binary-{conf.arch}/Packages
|
|
"""
|
|
)
|
|
return r
|
|
|
|
def init_archive(self, conf):
|
|
"""Initialize the archive"""
|
|
assert conf.bucket_name
|
|
r, o = runc(f"s3cmd mb s3://{conf.bucket_name}")
|
|
if r == 0:
|
|
print("Bucket created")
|
|
runc(f"s3cmd ws-create s3://{conf.bucket_name}")
|
|
|
|
r, out = runc(f"s3cmd ws-info s3://{conf.bucket_name}")
|
|
for li in out.splitlines():
|
|
if li.startswith("Website endpoint"):
|
|
s3url = li.split()[2]
|
|
break
|
|
|
|
# Initialize distro if needed. Check for InRelease
|
|
baseuri = f"s3://{conf.bucket_name}/dists/{conf.distro}"
|
|
r, o = runc(f"s3cmd info --no-progress {baseuri}/InRelease")
|
|
if r == 0:
|
|
return
|
|
|
|
if r != 12:
|
|
print(f"Unexpected return code {r} {o}")
|
|
sys.exit(1)
|
|
|
|
# InRelease file not found: create lock file
|
|
print("Creating initial lock file")
|
|
tf = NamedTemporaryFile()
|
|
# put = "s3cmd --acl-public --guess-mime-type --no-progress put"
|
|
put = "s3cmd --guess-mime-type --no-progress put"
|
|
r2, o = runc(f"{put} --no-progress {tf.name} {baseuri}/.debrepos3.lock")
|
|
assert r2 == 0, repr(o)
|
|
|
|
# Create empty InRelease
|
|
r2, o = runc(f"{put} {tf.name} {baseuri}/InRelease")
|
|
|
|
# Create empty Packages
|
|
r, o = runc(f"{put} {tf.name} {baseuri}/main/binary-{conf.arch}/Packages")
|
|
|
|
# Create index
|
|
html = dedent(
|
|
f"""
|
|
<html><body>
|
|
<p>Create /etc/apt/sources.list.d/{conf.distro}.list containing:</p>
|
|
<pre>deb {s3url} {conf.distro} main</pre>
|
|
</body></html>
|
|
"""
|
|
)
|
|
with open(tf.name, "w") as f:
|
|
f.write(html)
|
|
|
|
r, o = runc(f"{put} {tf.name} {baseuri}/index.html")
|
|
|
|
def lock(self, conf, baseuri):
|
|
"""Rename semaphore file"""
|
|
print(f"Locking {baseuri} ...")
|
|
cmd = f"s3cmd mv --no-progress {baseuri}/.debrepos3.nolock {baseuri}/.debrepos3.lock"
|
|
while True:
|
|
r, o = runc(cmd)
|
|
print(r)
|
|
if r == 0:
|
|
return
|
|
|
|
print("The distro is locked. Waiting...")
|
|
sleep(10)
|
|
|
|
def unlock(self, baseuri):
|
|
"""Rename semaphore file"""
|
|
r, o = runc(
|
|
f"s3cmd mv --no-progress {baseuri}/.debrepos3.lock {baseuri}/.debrepos3.nolock"
|
|
)
|
|
print(r)
|
|
|
|
def scanpackages(self, conf, debfn) -> str:
|
|
r, o = runc(f"dpkg-scanpackages {debfn}")
|
|
assert r == 0, repr(r)
|
|
out = []
|
|
for line in o.splitlines():
|
|
if line.startswith("Filename: "):
|
|
fn = line.split("/")[-1]
|
|
line = f"Filename: dists/{conf.distro}/main/binary-{conf.arch}/{fn}"
|
|
out.append(line)
|
|
|
|
return "\n".join(out) + "\n"
|
|
|
|
def _inner_upload(self, debfn, tmpdir, baseuri, pkgblock: str, gpg, gpgkeyfp):
|
|
# Fetch existing Packages file
|
|
packages = tmpdir / "Packages"
|
|
uri = f"{baseuri}/main/binary-{conf.arch}/Packages {packages}"
|
|
run2(f"s3cmd --no-progress get {uri}")
|
|
|
|
# Check for already uploaded package
|
|
check_duplicate_package(pkgblock, packages.read_text())
|
|
|
|
# Append, then read whole file back
|
|
with packages.open("a") as f:
|
|
f.write(pkgblock)
|
|
|
|
data = packages.read_bytes()
|
|
packagesf_size = len(data)
|
|
packagesf_sha = sha256(data).hexdigest()
|
|
del data
|
|
|
|
# Create, sign, upload InRelease
|
|
release = tmpdir / "Release"
|
|
inrelease = tmpdir / "InRelease"
|
|
rfdata = self.generate_release_file(conf, packagesf_sha, packagesf_size)
|
|
# print(rfdata)
|
|
release.write_text(rfdata)
|
|
# r, o = runc(f"gpg -a -s --clearsign -o {inrelease} {release}")
|
|
# if r != 0:
|
|
# self.unlock(baseuri)
|
|
# print("Error during GPG signature")
|
|
# sys.exit(1)
|
|
sig = gpg.sign(release.read_text(), keyid=gpgkeyfp)
|
|
assert sig.status == "signature created"
|
|
inrelease.write_bytes(sig.data)
|
|
|
|
# Upload InRelease and Packages
|
|
put = "s3cmd --acl-public --guess-mime-type --no-progress put"
|
|
run2(f"{put} {inrelease} {baseuri}/InRelease")
|
|
run2(f"{put} {packages} {baseuri}/main/binary-{conf.arch}/Packages")
|
|
run2(f"{put} {debfn} {baseuri}/main/binary-{conf.arch}/")
|
|
|
|
def upload(self, debfn, conf):
|
|
assert conf.bucket_name
|
|
tmpdir = Path(mkdtemp(prefix="debops-ci"))
|
|
|
|
self.init_archive(conf)
|
|
baseuri = f"s3://{conf.bucket_name}/dists/{conf.distro}"
|
|
|
|
pkgblock = self.scanpackages(conf, debfn)
|
|
|
|
gpg, gpgkeyfp = setup_gpg_key(conf.gpg_key_fp, tmpdir)
|
|
|
|
# Lock distro on S3 to prevent race during appends to Packages
|
|
self.lock(conf, baseuri)
|
|
|
|
try:
|
|
self._inner_upload(debfn, tmpdir, baseuri, pkgblock, gpg, gpgkeyfp)
|
|
except DuplicatePkgError:
|
|
print(f"Error: {debfn} is already in the archive. Not uploading.")
|
|
sys.exit(1) # the unlock in the finally block is still executed
|
|
finally:
|
|
self.unlock(baseuri)
|
|
|
|
# # TODO check
|
|
# dpkg-scanpackages $1 | gzip >> Packages.gz
|
|
# upload Packages.gz
|
|
# rm Packages.gz
|
|
|
|
|
|
def main():
|
|
global conf
|
|
ap = ArgumentParser(usage=__doc__)
|
|
ap.add_argument(
|
|
"action", choices=("upload", "scan", "ci", "build", "delete_from_archive")
|
|
)
|
|
ap.add_argument("-r", "--repo", default=None, help="S3/Bintray repository name")
|
|
ap.add_argument("-o", "--org", default=DEFAULT_ORG, help="S3/Bintray org name")
|
|
ap.add_argument("--bucket-name", help="S3 bucket name")
|
|
ap.add_argument("--distro", default="unstable", help="Debian distribution name")
|
|
ap.add_argument("--arch", default="amd64", help="Debian architecture name")
|
|
ap.add_argument("--gpg-key-fp", help="GPG key fingerprint")
|
|
ap.add_argument("--show-commands", action="store_true", help="Show shell commands")
|
|
args, extra = ap.parse_known_args()
|
|
conf = args
|
|
|
|
if args.action == "ci":
|
|
ci(args)
|
|
elif args.action == "scan":
|
|
for p in sorted(detect_changed_packages()):
|
|
print(p.as_posix())
|
|
elif args.action == "upload":
|
|
# TODO select backend
|
|
# bk = Bintray()
|
|
bk = S3()
|
|
for fn in extra:
|
|
bk.upload(Path(fn), args)
|
|
elif args.action == "delete_from_archive":
|
|
# TODO select backend
|
|
# bk = Bintray()
|
|
bk = S3()
|
|
bk.delete_package(args, extra)
|
|
elif args.action == "build":
|
|
build()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|