#!/usr/bin/env python3

"""
Builds deb packages and uploads them to S3-compatible storage or bintray.
Works locally and on GitHub Actions and CircleCI
Detects which package[s] need to be built.
Support "release" and PR/testing archives.

scan              - scan the current repository for packages to be built
build             - locate and build packages
upload <filename> - upload one package to S3 or Bintray
ci                - detect CircleCI PRs, build and upload packages
delete_from_archive <filename> - delete filename from archive

Features:
 - Implement CI/CD workflow using package archives
 - Support adding packages to an existing S3 archive without requiring a
   local mirror
 - Support multiple packages in the same git repository
 - GPG signing
 - Phased updates (rolling deployments)
 - Update changelogs automatically
 - Easy to debug
"""

# TODO: fix S3 credentials passing security
# TODO: Phased-Update-Percentage

# debdeps: git
from argparse import ArgumentParser
import os
from os import getenv
from pathlib import Path
from requests.auth import HTTPBasicAuth
from subprocess import run
from tempfile import mkdtemp, NamedTemporaryFile
from textwrap import dedent
from time import sleep
from typing import List
from hashlib import sha256
import requests
import sys

try:
    import gnupg
except ImportError:
    gnupg = None

# TODO remove these
BINTRAY_API = "https://bintray.com/api/v1"
DEFAULT_ORG = "ooni"
DEFAULT_PR_REPO = "internal-pull-requests"
DEFAULT_MASTER_REPO = "internal-master"
DEFAULT_REPO = "internal-pull-requests"

EXAMPLE_CONFIG = """
"""

assert sys.version_info >= (3, 7, 0), "Python 3.7.0 or later is required"

conf = None


def run2(cmd, **kw):
    if conf.show_commands:
        print(f"Running {cmd}\nKW: {kw}")
    p = run(cmd.split(), capture_output=True, **kw)
    if p.returncode != 0:
        stdout = p.stdout.decode().strip()
        print(f"--stdout--\n{stdout}\n----\n")
        stderr = p.stderr.decode().strip()
        print(f"--stderr--\n{stderr}\n----\n")
        raise Exception(f"'{cmd}' returned: {p.returncode}")
    return p.stdout.decode().strip()


def runi(cmd: str, cwd: Path, sudo=False) -> None:
    if sudo:
        cmd = f"sudo {cmd}"
    run(cmd.split(), cwd=cwd, check=True)


def runc(cmd):
    print("Running:", cmd)
    r = run(cmd.split(), capture_output=True)
    print("Retcode", r.returncode)
    return r.returncode, r.stdout.decode()


def detect_changed_packages() -> List[Path]:
    """Detects files named debian/changelog
    that have been changed in the current branch
    """
    DCH = "debian/changelog"
    # TODO: find a cleaner method:
    commit = run2("git merge-base remotes/origin/master HEAD")
    changes = run2(f"git diff --name-only {commit}")
    pkgs = set()
    for c in changes.splitlines():
        c = Path(c)
        if c.as_posix().endswith(DCH):
            pkgs.add(c.parent.parent)
            continue
        while c.name:
            if c.joinpath(DCH).is_file():
                pkgs.add(c)
            c = c.parent

    return sorted(pkgs)


def trim_compare(url: str) -> str:
    """Shorten GitHub URLs used to compare changes"""
    if url.startswith("https://github.com/") and "..." in url:
        base, commits = url.rsplit("/", 1)
        if len(commits) == 83:
            beginning = commits[0:8]
            end = commits[43 : 43 + 8]
            return f"{base}/{beginning}...{end}"

    return url


def _set_pkg_version_from_circleci(p, ver):
    comp = trim_compare(getenv("CIRCLE_COMPARE_URL", ""))  # show changes in VCS
    if not comp:
        # https://discuss.circleci.com/t/circle-compare-url-is-empty/24549/8
        comp = getenv("CIRCLE_PULL_REQUEST")

    if getenv("CIRCLE_PULL_REQUEST"):
        # This is a PR: build ~pr<N>-<N> version. CIRCLE_PR_NUMBER is broken
        pr_num = getenv("CIRCLE_PULL_REQUEST", "").rsplit("/", 1)[-1]
        build_num = getenv("CIRCLE_BUILD_NUM")
        ver = f"{ver}~pr{pr_num}-{build_num}"
        print(f"CircleCI Pull Request detected - using version {ver}")
        run2(f"dch -b -v {ver} {comp}", cwd=p)
        run2(f"dch -r {ver} {comp}", cwd=p)
        ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p)
        assert ver == ver2, ver + " <--> " + ver2

    elif getenv("CIRCLE_BRANCH") == "master":
        # This is a build outside of a PR and in the mainline branch
        print(f"CircleCI mainline build detected - using version {ver}")
        run2(f"dch -b -v {ver} {comp}", cwd=p)
        run2(f"dch -r {ver} {comp}", cwd=p)
        ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p)
        assert ver == ver2, ver + " <--> " + ver2

    else:
        # This is a build for a new branch but without a PR: ignore it
        return []


def _set_pkg_version_from_github_actions(p, ver):
    """When running in GitHub Actions, access env vars to set
    the package version"""
    # GITHUB_REF syntax: refs/heads/<branch-name> or refs/pull/<PR#>/merge
    gh_ref = getenv("GITHUB_REF")
    try:
        pr_num = int(gh_ref.split("/")[2])
    except ValueError:
        pr_num = None

    gh_run_number = int(getenv("GITHUB_RUN_NUMBER"))
    print(f"GitHub Actions PR #: {pr_num} Run #: {gh_run_number}")
    print("SHA " + getenv("GITHUB_SHA"))

    comp = ""
    if pr_num is None:
        if gh_ref.endswith("/master"):
            print(f"GitHub release build detected - using version {ver}")
            run2(f"dch -b -v {ver} ''", cwd=p)
            run2(f"dch --release ''", cwd=p)
            ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p)
            assert ver == ver2, ver + " <--> " + ver2
            return True

        else:
            print("Not a PR or release build. Skipping.")  # run by "on: push"
            return False

    else:
        # This is a PR: build ~pr<N>-<N> version.
        ver = f"{ver}~pr{pr_num}-{gh_run_number}"
        print(f"GitHub Pull Request detected - using version {ver}")
        run2(f"dch -b -v {ver} ''", cwd=p)
        run2(f"dch --release ''", cwd=p)
        ver2 = run2("dpkg-parsechangelog --show-field Version", cwd=p)
        assert ver == ver2, ver + " <--> " + ver2
        return True


def buildpkg(p) -> List[Path]:
    """Build one package, installing required dependencies"""
    print(f"Building package in {p}")
    ver = run2("dpkg-parsechangelog --show-field Version", cwd=p)
    assert ver, f"No version number found in {p}/debian/changelog"
    sudo = True
    should_build = False
    if getenv("CIRCLECI"):
        # Running in CircleCI
        sudo = False
        _set_pkg_version_from_circleci(p, ver)
    elif getenv("GITHUB_EVENT_PATH"):
        sudo = False
        should_build = _set_pkg_version_from_github_actions(p, ver)

    if not should_build:
        return []

    runi("apt-get build-dep -qy --no-install-recommends .", p, sudo=sudo)
    runi("fakeroot debian/rules build", p)
    runi("fakeroot debian/rules binary", p)
    with p.joinpath("debian/files").open() as f:
        return [p.parent.joinpath(line.split()[0]) for line in f]


def detect_archive_backend():
    if getenv("BINTRAY_USERNAME") and getenv("BINTRAY_API_KEY"):
        return "bintray"

    if getenv("AWS_ACCESS_KEY_ID") and getenv("AWS_SECRET_ACCESS_KEY"):
        return "s3"


def setup_gpg_key(keyfp, tmpdir):
    """Import key from env var or use existing keyring"""
    if gnupg is None:
        print("Please install python3-gnupg")
        sys.exit(1)

    if keyfp is None and "DEB_GPG_KEY" not in os.environ:
        print(
            "Error: place a GPG key in the DEB_GPG_KEY env var or"
            " fetch it from the local keyring using --gpg-key-fp"
        )
        sys.exit(1)

    if "DEB_GPG_KEY" in os.environ:
        gpg = gnupg.GPG(gnupghome=tmpdir.as_posix())
        import_result = gpg.import_keys(os.getenv("DEB_GPG_KEY"))
        assert import_result.count == 1
        fp = import_result.fingerprints[0]
        if keyfp:
            assert keyfp == fp

    else:
        gpg = gnupg.GPG()
        assert gpg.list_keys(keys=keyfp)

    return gpg, keyfp


def ci(args) -> None:
    # TODO: detect sudo presence

    backend_name = detect_archive_backend()
    if backend_name == "bintray":
        backend = Bintray()
    elif backend_name == "s3":
        backend = S3()
    else:
        print(
            "Either set BINTRAY_USERNAME / BINTRAY_API_KEY env vars or "
            "AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY"
        )
        sys.exit(1)
    del backend_name

    run2("apt-get update -q")
    run2("apt-get install -qy --no-install-recommends git")
    pkgdirs = detect_changed_packages()
    if not pkgdirs:
        print("Nothing to build")
        return

    print(f"Building {pkgdirs}")
    run2("apt-get install -qy --no-install-recommends devscripts")

    pkgs_lists = [buildpkg(pd) for pd in pkgdirs]
    print(f"Processing {pkgs_lists}")
    for pli in pkgs_lists:
        for p in pli:
            backend.upload(p, args)


def build() -> None:
    """Run manual build on workstation"""
    pkgdirs = detect_changed_packages()
    pkgs_lists = [buildpkg(pd) for pd in pkgdirs]
    print("Outputs:")
    for pli in pkgs_lists:
        for p in pli:
            print(p)


class DuplicatePkgError(Exception):
    pass


def check_duplicate_package(pkgblock, packages_text):
    li = pkgblock.splitlines()
    assert li[0].startswith("Package: "), li
    pname = li[0].split(" ", 1)[1]
    assert li[1].startswith("Version: "), li
    pver = li[1].split(" ", 1)[1]
    assert li[2].startswith("Architecture: "), li
    parch = li[2].split(" ", 1)[1]

    m = f"Package: {pname}\nVersion: {pver}\nArchitecture: {parch}"
    if m in packages_text:
        raise DuplicatePkgError()


class Bintray:
    """Bintray backend"""

    def __init__(self):
        self._btuser = getenv("BINTRAY_USERNAME")
        assert self._btuser, "Missing BINTRAY_USERNAME"

    def upload(self, fi, args) -> None:
        """Upload to Bintray"""
        # FIXME: specify repo
        assert repo, "Please specify a repository"
        assert fi.is_file()
        pname, pver, arch = fi.name.split("_")
        auth = HTTPBasicAuth(self._btuser, getenv("BINTRAY_API_KEY"))
        dist = "unstable"
        url = (
            f"{BINTRAY_API}/content/{args.org}/{repo}/{pname}/{pver}/{fi.name};"
            f"deb_distribution={dist};deb_component=main;deb_architecture=amd64;publish=1"
        )
        with open(fi, "rb") as f:
            resp = requests.put(url, auth=auth, data=f)

        if not resp.ok:
            print(f"Error {resp.text} when calling {resp.request.url}")
            sys.exit(1)

    def delete_package(self, args, extra) -> None:
        """Delete package from Bintray"""
        auth = HTTPBasicAuth(self._btuser, getenv("BINTRAY_API_KEY"))
        filename = extra[0]
        assert filename.endswith(".deb")
        assert args.repo, "Please specify a repository"
        url = f"{BINTRAY_API}/content/{args.org}/{args.repo}/{filename}"
        resp = requests.delete(url, auth=auth)
        if not resp.ok:
            print(f"Error {resp.text} when calling {resp.request.url}")
            sys.exit(1)


class S3:
    """S3 backend"""

    def generate_release_file(self, conf, sha, size):
        r = dedent(
            f"""
            Acquire-By-Hash: no
            Architectures: {conf.arch}
            Codename: {conf.distro}
            Components: main
            Date: Thu, 07 Nov 2019 14:23:37 UTC
            Origin: private
            Valid-Until: Thu, 14 Nov 2029 14:23:37 UTC
            SHA256:
             {sha} {size} main/binary-{conf.arch}/Packages
        """
        )
        return r

    def init_archive(self, conf):
        """Initialize the archive"""
        assert conf.bucket_name
        r, o = runc(f"s3cmd mb s3://{conf.bucket_name}")
        if r == 0:
            print("Bucket created")
            runc(f"s3cmd ws-create s3://{conf.bucket_name}")

        r, out = runc(f"s3cmd ws-info s3://{conf.bucket_name}")
        for li in out.splitlines():
            if li.startswith("Website endpoint"):
                s3url = li.split()[2]
                break

        # Initialize distro if needed. Check for InRelease
        baseuri = f"s3://{conf.bucket_name}/dists/{conf.distro}"
        r, o = runc(f"s3cmd info --no-progress {baseuri}/InRelease")
        if r == 0:
            return

        if r != 12:
            print(f"Unexpected return code {r} {o}")
            sys.exit(1)

        # InRelease file not found: create lock file
        print("Creating initial lock file")
        tf = NamedTemporaryFile()
        # put = "s3cmd --acl-public --guess-mime-type --no-progress put"
        put = "s3cmd --guess-mime-type --no-progress put"
        r2, o = runc(f"{put} --no-progress {tf.name} {baseuri}/.debrepos3.lock")
        assert r2 == 0, repr(o)

        # Create empty InRelease
        r2, o = runc(f"{put} {tf.name} {baseuri}/InRelease")

        # Create empty Packages
        r, o = runc(f"{put} {tf.name} {baseuri}/main/binary-{conf.arch}/Packages")

        # Create index
        html = dedent(
            f"""
            <html><body>
            <p>Create /etc/apt/sources.list.d/{conf.distro}.list containing:</p>
            <pre>deb {s3url} {conf.distro} main</pre>
            </body></html>
        """
        )
        with open(tf.name, "w") as f:
            f.write(html)

        r, o = runc(f"{put} {tf.name} {baseuri}/index.html")

    def lock(self, conf, baseuri):
        """Rename semaphore file"""
        print(f"Locking {baseuri} ...")
        cmd = f"s3cmd mv --no-progress {baseuri}/.debrepos3.nolock {baseuri}/.debrepos3.lock"
        while True:
            r, o = runc(cmd)
            print(r)
            if r == 0:
                return

            print("The distro is locked. Waiting...")
            sleep(10)

    def unlock(self, baseuri):
        """Rename semaphore file"""
        r, o = runc(
            f"s3cmd mv --no-progress {baseuri}/.debrepos3.lock {baseuri}/.debrepos3.nolock"
        )
        print(r)

    def scanpackages(self, conf, debfn) -> str:
        r, o = runc(f"dpkg-scanpackages {debfn}")
        assert r == 0, repr(r)
        out = []
        for line in o.splitlines():
            if line.startswith("Filename: "):
                fn = line.split("/")[-1]
                line = f"Filename: dists/{conf.distro}/main/binary-{conf.arch}/{fn}"
            out.append(line)

        return "\n".join(out) + "\n"

    def _inner_upload(self, debfn, tmpdir, baseuri, pkgblock: str, gpg, gpgkeyfp):
        # Fetch existing Packages file
        packages = tmpdir / "Packages"
        uri = f"{baseuri}/main/binary-{conf.arch}/Packages {packages}"
        run2(f"s3cmd --no-progress get {uri}")

        # Check for already uploaded package
        check_duplicate_package(pkgblock, packages.read_text())

        # Append, then read whole file back
        with packages.open("a") as f:
            f.write(pkgblock)

        data = packages.read_bytes()
        packagesf_size = len(data)
        packagesf_sha = sha256(data).hexdigest()
        del data

        # Create, sign, upload InRelease
        release = tmpdir / "Release"
        inrelease = tmpdir / "InRelease"
        rfdata = self.generate_release_file(conf, packagesf_sha, packagesf_size)
        # print(rfdata)
        release.write_text(rfdata)
        # r, o = runc(f"gpg -a -s --clearsign -o {inrelease} {release}")
        # if r != 0:
        #    self.unlock(baseuri)
        #    print("Error during GPG signature")
        #    sys.exit(1)
        sig = gpg.sign(release.read_text(), keyid=gpgkeyfp)
        assert sig.status == "signature created"
        inrelease.write_bytes(sig.data)

        # Upload InRelease and Packages
        put = "s3cmd --acl-public --guess-mime-type --no-progress put"
        run2(f"{put} {inrelease} {baseuri}/InRelease")
        run2(f"{put} {packages} {baseuri}/main/binary-{conf.arch}/Packages")
        run2(f"{put} {debfn} {baseuri}/main/binary-{conf.arch}/")

    def upload(self, debfn, conf):
        assert conf.bucket_name
        tmpdir = Path(mkdtemp(prefix="debops-ci"))

        self.init_archive(conf)
        baseuri = f"s3://{conf.bucket_name}/dists/{conf.distro}"

        pkgblock = self.scanpackages(conf, debfn)

        gpg, gpgkeyfp = setup_gpg_key(conf.gpg_key_fp, tmpdir)

        # Lock distro on S3 to prevent race during appends to Packages
        self.lock(conf, baseuri)

        try:
            self._inner_upload(debfn, tmpdir, baseuri, pkgblock, gpg, gpgkeyfp)
        except DuplicatePkgError:
            print(f"Error: {debfn} is already in the archive. Not uploading.")
            sys.exit(1)  # the unlock in the finally block is still executed
        finally:
            self.unlock(baseuri)

    #  # TODO check
    #  dpkg-scanpackages $1 | gzip >> Packages.gz
    #  upload Packages.gz
    #  rm Packages.gz


def main():
    global conf
    ap = ArgumentParser(usage=__doc__)
    ap.add_argument(
        "action", choices=("upload", "scan", "ci", "build", "delete_from_archive")
    )
    ap.add_argument("-r", "--repo", default=None, help="S3/Bintray repository name")
    ap.add_argument("-o", "--org", default=DEFAULT_ORG, help="S3/Bintray org name")
    ap.add_argument("--bucket-name", help="S3 bucket name")
    ap.add_argument("--distro", default="unstable", help="Debian distribution name")
    ap.add_argument("--arch", default="amd64", help="Debian architecture name")
    ap.add_argument("--gpg-key-fp", help="GPG key fingerprint")
    ap.add_argument("--show-commands", action="store_true", help="Show shell commands")
    args, extra = ap.parse_known_args()
    conf = args

    if args.action == "ci":
        ci(args)
    elif args.action == "scan":
        for p in sorted(detect_changed_packages()):
            print(p.as_posix())
    elif args.action == "upload":
        # TODO select backend
        # bk = Bintray()
        bk = S3()
        for fn in extra:
            bk.upload(Path(fn), args)
    elif args.action == "delete_from_archive":
        # TODO select backend
        # bk = Bintray()
        bk = S3()
        bk.delete_package(args, extra)
    elif args.action == "build":
        build()


if __name__ == "__main__":
    main()