From 2539a17ff94ed0d8f453c93b3857039b02a5deec Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 3 May 2021 15:50:27 +0100 Subject: [PATCH] Switch .deb package publishing to S3 (#314) This PR switches the` .deb` package publishing to s3 because Bintray has been shut down. --- .github/workflows/debian.yml | 22 +- .github/workflows/debops-ci | 559 +++++++++++++++++++++++++++++++++++ 2 files changed, 572 insertions(+), 9 deletions(-) create mode 100755 .github/workflows/debops-ci diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index a169495..189bbb8 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -1,9 +1,11 @@ +--- # Build and publish Debian packages name: debian on: push: branches: - "master" + - "deb-s3" - "release/**" tags: - "v*" @@ -22,7 +24,10 @@ jobs: - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64 - run: sudo apt-get update -q - run: sudo apt-get build-dep -y --no-install-recommends . - - run: | + - name: Install deps + run: sudo apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd + - name: Sign and upload + run: | VER=$(./CLI/linux/amd64/ooniprobe version) if [[ ! $GITHUB_REF =~ ^refs/tags/* ]]; then VER="${VER}~${GITHUB_RUN_NUMBER}" @@ -35,13 +40,12 @@ jobs: find ../ -name "*.deb" -type f DEB="../ooniprobe-cli_${VER}_amd64.deb" echo no | sudo dpkg -i $DEB - BT_FNAME="ooniprobe-cli_${VER}_amd64.deb" - curl --upload-file "${DEB}" -u "${BT_APIUSER}:${BT_APIKEY}" \ - "https://api.bintray.com/content/${BT_ORG}/${BT_REPO}/${BT_PKGNAME}/${VER}/${BT_FNAME};deb_distribution=${DEBDIST};deb_component=main;deb_architecture=amd64;publish=1" + export DEBFNAME="ooniprobe-cli_${VER}_amd64.deb" + cd $GITHUB_WORKSPACE + # AWS user debci-s3 + ./.github/workflows/debops-ci --show-commands upload --bucket-name ooni-deb $DEB env: - DEBDIST: unstable - BT_APIKEY: ${{ secrets.BT_APIKEY }} - BT_APIUSER: federicoceratto - BT_ORG: ooni - BT_PKGNAME: ooniprobe + DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} BT_REPO: ooniprobe-debian diff --git a/.github/workflows/debops-ci b/.github/workflows/debops-ci new file mode 100755 index 0000000..dcdef7a --- /dev/null +++ b/.github/workflows/debops-ci @@ -0,0 +1,559 @@ +#!/usr/bin/env python3 + +""" +Builds deb packages and uploads them to S3-compatible storage or bintray. +Works locally and on GitHub Actions and CircleCI +Detects which package[s] need to be built. +Support "release" and PR/testing archives. + +scan - scan the current repository for packages to be built +build - locate and build packages +upload - upload one package to S3 or Bintray +ci - detect CircleCI PRs, build and upload packages +delete_from_archive - delete filename from archive + +Features: + - Implement CI/CD workflow using package archives + - Support adding packages to an existing S3 archive without requiring a + local mirror + - Support multiple packages in the same git repository + - GPG signing + - Phased updates (rolling deployments) + - Update changelogs automatically + - Easy to debug +""" + +# TODO: fix S3 credentials passing security +# TODO: Phased-Update-Percentage + +# debdeps: git +from argparse import ArgumentParser +import os +from os import getenv +from pathlib import Path +from requests.auth import HTTPBasicAuth +from subprocess import run +from tempfile import mkdtemp, NamedTemporaryFile +from textwrap import dedent +from time import sleep +from typing import List +from hashlib import sha256 +import requests +import sys + +try: + import gnupg +except ImportError: + gnupg = None + +# TODO remove these +BINTRAY_API = "https://bintray.com/api/v1" +DEFAULT_ORG = "ooni" +DEFAULT_PR_REPO = "internal-pull-requests" +DEFAULT_MASTER_REPO = "internal-master" +DEFAULT_REPO = "internal-pull-requests" + +EXAMPLE_CONFIG = """ +""" + +assert sys.version_info >= (3, 7, 0), "Python 3.7.0 or later is required" + +conf = None + + +def run2(cmd, **kw): + if conf.show_commands: + print(f"Running {cmd}\nKW: {kw}") + p = run(cmd.split(), capture_output=True, **kw) + if p.returncode != 0: + stdout = p.stdout.decode().strip() + print(f"--stdout--\n{stdout}\n----\n") + stderr = p.stderr.decode().strip() + print(f"--stderr--\n{stderr}\n----\n") + raise Exception(f"'{cmd}' returned: {p.returncode}") + return p.stdout.decode().strip() + + +def runi(cmd: str, cwd: Path, sudo=False) -> None: + if sudo: + cmd = f"sudo {cmd}" + run(cmd.split(), cwd=cwd, check=True) + + +def runc(cmd): + print("Running:", cmd) + r = run(cmd.split(), capture_output=True) + print("Retcode", r.returncode) + return r.returncode, r.stdout.decode() + + +def detect_changed_packages() -> List[Path]: + """Detects files named debian/changelog + that have been changed in the current branch + """ + DCH = "debian/changelog" + # TODO: find a cleaner method: + commit = run2("git merge-base remotes/origin/master HEAD") + changes = run2(f"git diff --name-only {commit}") + pkgs = set() + for c in changes.splitlines(): + c = Path(c) + if c.as_posix().endswith(DCH): + pkgs.add(c.parent.parent) + continue + while c.name: + if c.joinpath(DCH).is_file(): + pkgs.add(c) + c = c.parent + + return sorted(pkgs) + + +def trim_compare(url: str) -> str: + """Shorten GitHub URLs used to compare changes""" + if url.startswith("https://github.com/") and "..." in url: + base, commits = url.rsplit("/", 1) + if len(commits) == 83: + beginning = commits[0:8] + end = commits[43 : 43 + 8] + return f"{base}/{beginning}...{end}" + + return url + + +def _set_pkg_version_from_circleci(p, ver): + comp = trim_compare(getenv("CIRCLE_COMPARE_URL", "")) # show changes in VCS + if not comp: + # https://discuss.circleci.com/t/circle-compare-url-is-empty/24549/8 + comp = getenv("CIRCLE_PULL_REQUEST") + + if getenv("CIRCLE_PULL_REQUEST"): + # This is a PR: build ~pr- version. CIRCLE_PR_NUMBER is broken + pr_num = getenv("CIRCLE_PULL_REQUEST", "").rsplit("/", 1)[-1] + build_num = getenv("CIRCLE_BUILD_NUM") + ver = f"{ver}~pr{pr_num}-{build_num}" + print(f"CircleCI Pull Request detected - using version {ver}") + run2(f"dch -b -v {ver} {comp}", cwd=p) + run2(f"dch -r {ver} {comp}", cwd=p) + ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p) + assert ver == ver2, ver + " <--> " + ver2 + + elif getenv("CIRCLE_BRANCH") == "master": + # This is a build outside of a PR and in the mainline branch + print(f"CircleCI mainline build detected - using version {ver}") + run2(f"dch -b -v {ver} {comp}", cwd=p) + run2(f"dch -r {ver} {comp}", cwd=p) + ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p) + assert ver == ver2, ver + " <--> " + ver2 + + else: + # This is a build for a new branch but without a PR: ignore it + return [] + + +def _set_pkg_version_from_github_actions(p, ver): + """When running in GitHub Actions, access env vars to set + the package version""" + # GITHUB_REF syntax: refs/heads/ or refs/pull//merge + gh_ref = getenv("GITHUB_REF") + try: + pr_num = int(gh_ref.split("/")[2]) + except ValueError: + pr_num = None + + gh_run_number = int(getenv("GITHUB_RUN_NUMBER")) + print(f"GitHub Actions PR #: {pr_num} Run #: {gh_run_number}") + print("SHA " + getenv("GITHUB_SHA")) + + comp = "" + if pr_num is None: + if gh_ref.endswith("/master"): + print(f"GitHub release build detected - using version {ver}") + run2(f"dch -b -v {ver} ''", cwd=p) + run2(f"dch --release ''", cwd=p) + ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p) + assert ver == ver2, ver + " <--> " + ver2 + return True + + else: + print("Not a PR or release build. Skipping.") # run by "on: push" + return False + + else: + # This is a PR: build ~pr- version. + ver = f"{ver}~pr{pr_num}-{gh_run_number}" + print(f"GitHub Pull Request detected - using version {ver}") + run2(f"dch -b -v {ver} ''", cwd=p) + run2(f"dch --release ''", cwd=p) + ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p) + assert ver == ver2, ver + " <--> " + ver2 + return True + + +def buildpkg(p) -> List[Path]: + """Build one package, installing required dependencies""" + print(f"Building package in {p}") + ver = run2("dpkg-parsechangelog --show-field Version", cwd=p) + assert ver, f"No version number found in {p}/debian/changelog" + sudo = True + should_build = False + if getenv("CIRCLECI"): + # Running in CircleCI + sudo = False + _set_pkg_version_from_circleci(p, ver) + elif getenv("GITHUB_EVENT_PATH"): + sudo = False + should_build = _set_pkg_version_from_github_actions(p, ver) + + if not should_build: + return [] + + runi("apt-get build-dep -qy --no-install-recommends .", p, sudo=sudo) + runi("fakeroot debian/rules build", p) + runi("fakeroot debian/rules binary", p) + with p.joinpath("debian/files").open() as f: + return [p.parent.joinpath(line.split()[0]) for line in f] + + +def detect_archive_backend(): + if getenv("BINTRAY_USERNAME") and getenv("BINTRAY_API_KEY"): + return "bintray" + + if getenv("AWS_ACCESS_KEY_ID") and getenv("AWS_SECRET_ACCESS_KEY"): + return "s3" + + +def setup_gpg_key(keyfp, tmpdir): + """Import key from env var or use existing keyring""" + if gnupg is None: + print("Please install python3-gnupg") + sys.exit(1) + + if keyfp is None and "DEB_GPG_KEY" not in os.environ: + print( + "Error: place a GPG key in the DEB_GPG_KEY env var or" + " fetch it from the local keyring using --gpg-key-fp" + ) + sys.exit(1) + + if "DEB_GPG_KEY" in os.environ: + gpg = gnupg.GPG(gnupghome=tmpdir.as_posix()) + import_result = gpg.import_keys(os.getenv("DEB_GPG_KEY")) + assert import_result.count == 1 + fp = import_result.fingerprints[0] + if keyfp: + assert keyfp == fp + + else: + gpg = gnupg.GPG() + assert gpg.list_keys(keys=keyfp) + + return gpg, keyfp + + +def ci(args) -> None: + # TODO: detect sudo presence + + backend_name = detect_archive_backend() + if backend_name == "bintray": + backend = Bintray() + elif backend_name == "s3": + backend = S3() + else: + print( + "Either set BINTRAY_USERNAME / BINTRAY_API_KEY env vars or " + "AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY" + ) + sys.exit(1) + del backend_name + + run2("apt-get update -q") + run2("apt-get install -qy --no-install-recommends git") + pkgdirs = detect_changed_packages() + if not pkgdirs: + print("Nothing to build") + return + + print(f"Building {pkgdirs}") + run2("apt-get install -qy --no-install-recommends devscripts") + + pkgs_lists = [buildpkg(pd) for pd in pkgdirs] + print(f"Processing {pkgs_lists}") + for pli in pkgs_lists: + for p in pli: + backend.upload(p, args) + + +def build() -> None: + """Run manual build on workstation""" + pkgdirs = detect_changed_packages() + pkgs_lists = [buildpkg(pd) for pd in pkgdirs] + print("Outputs:") + for pli in pkgs_lists: + for p in pli: + print(p) + + +class DuplicatePkgError(Exception): + pass + + +def check_duplicate_package(pkgblock, packages_text): + li = pkgblock.splitlines() + assert li[0].startswith("Package: "), li + pname = li[0].split(" ", 1)[1] + assert li[1].startswith("Version: "), li + pver = li[1].split(" ", 1)[1] + + m = f"Package: {pname}\nVersion: {pver}" + if m in packages_text: + raise DuplicatePkgError() + + +class Bintray: + """Bintray backend""" + + def __init__(self): + self._btuser = getenv("BINTRAY_USERNAME") + assert self._btuser, "Missing BINTRAY_USERNAME" + + def upload(self, fi, args) -> None: + """Upload to Bintray""" + # FIXME: specify repo + assert repo, "Please specify a repository" + assert fi.is_file() + pname, pver, arch = fi.name.split("_") + auth = HTTPBasicAuth(self._btuser, getenv("BINTRAY_API_KEY")) + dist = "unstable" + url = ( + f"{BINTRAY_API}/content/{args.org}/{repo}/{pname}/{pver}/{fi.name};" + f"deb_distribution={dist};deb_component=main;deb_architecture=amd64;publish=1" + ) + with open(fi, "rb") as f: + resp = requests.put(url, auth=auth, data=f) + + if not resp.ok: + print(f"Error {resp.text} when calling {resp.request.url}") + sys.exit(1) + + def delete_package(self, args, extra) -> None: + """Delete package from Bintray""" + auth = HTTPBasicAuth(self._btuser, getenv("BINTRAY_API_KEY")) + filename = extra[0] + assert filename.endswith(".deb") + assert args.repo, "Please specify a repository" + url = f"{BINTRAY_API}/content/{args.org}/{args.repo}/{filename}" + resp = requests.delete(url, auth=auth) + if not resp.ok: + print(f"Error {resp.text} when calling {resp.request.url}") + sys.exit(1) + + +class S3: + """S3 backend""" + + def generate_release_file(self, conf, sha, size): + r = dedent( + f""" + Acquire-By-Hash: no + Architectures: {conf.arch} + Codename: {conf.distro} + Components: main + Date: Thu, 07 Nov 2019 14:23:37 UTC + Origin: private + Valid-Until: Thu, 14 Nov 2029 14:23:37 UTC + SHA256: + {sha} {size} main/binary-{conf.arch}/Packages + """ + ) + return r + + def init_archive(self, conf): + """Initialize the archive""" + assert conf.bucket_name + r, o = runc(f"s3cmd mb s3://{conf.bucket_name}") + if r == 0: + print("Bucket created") + runc(f"s3cmd ws-create s3://{conf.bucket_name}") + + r, out = runc(f"s3cmd ws-info s3://{conf.bucket_name}") + for li in out.splitlines(): + if li.startswith("Website endpoint"): + s3url = li.split()[2] + break + + # Initialize distro if needed. Check for InRelease + baseuri = f"s3://{conf.bucket_name}/dists/{conf.distro}" + r, o = runc(f"s3cmd info --no-progress {baseuri}/InRelease") + if r == 0: + return + + if r != 12: + print(f"Unexpected return code {r} {o}") + sys.exit(1) + + # InRelease file not found: create lock file + print("Creating initial lock file") + tf = NamedTemporaryFile() + # put = "s3cmd --acl-public --guess-mime-type --no-progress put" + put = "s3cmd --guess-mime-type --no-progress put" + r2, o = runc(f"{put} --no-progress {tf.name} {baseuri}/.debrepos3.lock") + assert r2 == 0, repr(o) + + # Create empty InRelease + r2, o = runc(f"{put} {tf.name} {baseuri}/InRelease") + + # Create empty Packages + r, o = runc(f"{put} {tf.name} {baseuri}/main/binary-{conf.arch}/Packages") + + # Create index + html = dedent( + f""" + +

Create /etc/apt/sources.list.d/{conf.distro}.list containing:

+ 
deb {s3url} {conf.distro} main
+ + """ + ) + with open(tf.name, "w") as f: + f.write(html) + + r, o = runc(f"{put} {tf.name} {baseuri}/index.html") + + def lock(self, conf, baseuri): + """Rename semaphore file""" + print(f"Locking {baseuri} ...") + cmd = f"s3cmd mv --no-progress {baseuri}/.debrepos3.nolock {baseuri}/.debrepos3.lock" + while True: + r, o = runc(cmd) + print(r) + if r == 0: + return + + print("The distro is locked. Waiting...") + sleep(10) + + def unlock(self, baseuri): + """Rename semaphore file""" + r, o = runc( + f"s3cmd mv --no-progress {baseuri}/.debrepos3.lock {baseuri}/.debrepos3.nolock" + ) + print(r) + + def scanpackages(self, conf, debfn) -> str: + r, o = runc(f"dpkg-scanpackages {debfn}") + assert r == 0, repr(r) + out = [] + for line in o.splitlines(): + if line.startswith("Filename: "): + fn = line.split("/")[-1] + line = f"Filename: dists/{conf.distro}/main/binary-{conf.arch}/{fn}" + out.append(line) + + return "\n".join(out) + "\n" + + def _inner_upload(self, debfn, tmpdir, baseuri, pkgblock: str, gpg, gpgkeyfp): + # Fetch existing Packages file + packages = tmpdir / "Packages" + uri = f"{baseuri}/main/binary-{conf.arch}/Packages {packages}" + run2(f"s3cmd --no-progress get {uri}") + + # Check for already uploaded package + check_duplicate_package(pkgblock, packages.read_text()) + + # Append, then read whole file back + with packages.open("a") as f: + f.write(pkgblock) + + data = packages.read_bytes() + packagesf_size = len(data) + packagesf_sha = sha256(data).hexdigest() + del data + + # Create, sign, upload InRelease + release = tmpdir / "Release" + inrelease = tmpdir / "InRelease" + rfdata = self.generate_release_file(conf, packagesf_sha, packagesf_size) + # print(rfdata) + release.write_text(rfdata) + # r, o = runc(f"gpg -a -s --clearsign -o {inrelease} {release}") + # if r != 0: + # self.unlock(baseuri) + # print("Error during GPG signature") + # sys.exit(1) + sig = gpg.sign(release.read_text(), keyid=gpgkeyfp) + assert sig.status == "signature created" + inrelease.write_bytes(sig.data) + + # Upload InRelease and Packages + put = "s3cmd --acl-public --guess-mime-type --no-progress put" + run2(f"{put} {inrelease} {baseuri}/InRelease") + run2(f"{put} {packages} {baseuri}/main/binary-{conf.arch}/Packages") + run2(f"{put} {debfn} {baseuri}/main/binary-{conf.arch}/") + + def upload(self, debfn, conf): + assert conf.bucket_name + tmpdir = Path(mkdtemp(prefix="debops-ci")) + + self.init_archive(conf) + baseuri = f"s3://{conf.bucket_name}/dists/{conf.distro}" + + pkgblock = self.scanpackages(conf, debfn) + + gpg, gpgkeyfp = setup_gpg_key(conf.gpg_key_fp, tmpdir) + + # Lock distro on S3 to prevent race during appends to Packages + self.lock(conf, baseuri) + + try: + self._inner_upload(debfn, tmpdir, baseuri, pkgblock, gpg, gpgkeyfp) + except DuplicatePkgError: + print(f"Error: {debfn} is already in the archive. Not uploading.") + sys.exit(1) # the unlock in the finally block is still executed + finally: + self.unlock(baseuri) + + # # TODO check + # dpkg-scanpackages $1 | gzip >> Packages.gz + # upload Packages.gz + # rm Packages.gz + + +def main(): + global conf + ap = ArgumentParser(usage=__doc__) + ap.add_argument( + "action", choices=("upload", "scan", "ci", "build", "delete_from_archive") + ) + ap.add_argument("-r", "--repo", default=None, help="S3/Bintray repository name") + ap.add_argument("-o", "--org", default=DEFAULT_ORG, help="S3/Bintray org name") + ap.add_argument("--bucket-name", help="S3 bucket name") + ap.add_argument("--distro", default="unstable", help="Debian distribution name") + ap.add_argument("--arch", default="amd64", help="Debian architecture name") + ap.add_argument("--gpg-key-fp", help="GPG key fingerprint") + ap.add_argument("--show-commands", action="store_true", help="Show shell commands") + args, extra = ap.parse_known_args() + conf = args + + if args.action == "ci": + ci(args) + elif args.action == "scan": + for p in sorted(detect_changed_packages()): + print(p.as_posix()) + elif args.action == "upload": + # TODO select backend + # bk = Bintray() + bk = S3() + for fn in extra: + bk.upload(Path(fn), args) + elif args.action == "delete_from_archive": + # TODO select backend + # bk = Bintray() + bk = S3() + bk.delete_package(args, extra) + elif args.action == "build": + build() + + +if __name__ == "__main__": + main()