stackpilot/backend/services/git_service.py

"""Deploying a stack from a Git repository.

The repository is the source of truth: a sync makes the stack's files match what
the repo says, and optionally runs ``compose up -d`` when that changed anything.
Two things about *how* are worth stating up front, because both are places this
could quietly destroy data.

**The clone does not live in the stack folder.** It is cached under
``${DATA_DIR}/git/<stack_id>`` and the relevant subtree is copied across. A
stack folder holds more than the repo's files — compose creates bind-mount
directories like ``./config`` right there, full of live application data — so a
``git reset --hard`` or ``git clean`` in that folder would be catastrophic. In
the cache directory both are safe, and the copy step is where the care goes.

**Only files the repo has provided are ever deleted.** Each sync records the
paths it wrote (``GitSource.managed_files``); the next sync removes the ones the
repo no longer has, and nothing else. A file that was never in the repository
cannot be touched, no matter what happened to it.

Credentials never reach a command line. A token is passed to git through
``GIT_ASKPASS`` and an environment variable, an SSH key through a 0600 file
outside the working tree — so neither shows up in ``ps``, in the repo's own
config, or in an error message this module passes on.
"""
from __future__ import annotations

import asyncio
import filecmp
import hmac
import json
import logging
import os
import re
import secrets
import shutil
import stat
from datetime import datetime, timezone
from typing import Optional

from sqlmodel import Session

from config import child_env, settings
from models.git_source import GitSource, SyncResult
from services import compose_service, crypto_service, stack_lock_service

logger = logging.getLogger("stackpilot.git")

GIT_TIMEOUT = 300.0
_SAFE_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,128}$")


class GitError(Exception):
    """A repository that cannot be reached, or a sync that cannot be completed."""


#: What a branch name may look like here. git itself allows more, but not a
#: leading "-": as the refspec of `git fetch origin <branch>` that would be read
#: as an option (`--upload-pack=…` runs a command).
_BRANCH_RE = re.compile(r"^[A-Za-z0-9._/][A-Za-z0-9._/+-]{0,199}$")


def validate_branch(branch: str) -> str:
    branch = (branch or "").strip()
    if not _BRANCH_RE.match(branch) or ".." in branch or branch.endswith((".lock", "/")):
        raise GitError(f"Invalid branch name '{branch}'")
    return branch


def validate_url(url: str) -> str:
    url = (url or "").strip()
    if not url or url.startswith("-") or any(c.isspace() for c in url):
        raise GitError(f"Invalid repository URL '{url}'")
    # git's ext:: and fd:: transports run commands or read file descriptors.
    if url.lower().startswith(("ext::", "fd::")):
        raise GitError("ext:: and fd:: repository URLs are not supported")
    return url


# --------------------------------------------------------------------------- #
# Paths
# --------------------------------------------------------------------------- #


def cache_root() -> str:
    return os.path.join(settings.DATA_DIR, "git")


def repo_dir(stack_id: str) -> str:
    if not _SAFE_ID_RE.match(stack_id or "") or stack_id in (".", ".."):
        raise GitError(f"Invalid stack id '{stack_id}'")
    return os.path.join(cache_root(), stack_id)


def _key_path(stack_id: str) -> str:
    # Beside the clone, never inside it — a working tree gets reset and cleaned.
    return os.path.join(cache_root(), f"{stack_id}.key")


def _askpass_path(stack_id: str) -> str:
    return os.path.join(cache_root(), f"{stack_id}.askpass")


def new_webhook_secret() -> str:
    return secrets.token_urlsafe(24)


# --------------------------------------------------------------------------- #
# Running git
# --------------------------------------------------------------------------- #


def _redact(text: str, *secrets_: Optional[str]) -> str:
    """Strip anything secret out of git's output before it is shown or stored."""
    for value in secrets_:
        if value:
            text = text.replace(value, "••••••")
    # A URL that carries credentials, in case one ever reaches git's output.
    return re.sub(r"(https?://)[^/\s:@]+:[^/\s@]+@", r"\1••••••@", text)


async def _git(args: list[str], env: dict, secret: Optional[str] = None) -> str:
    """Run git, returning stdout. Raises GitError with a redacted message."""
    proc = await asyncio.create_subprocess_exec(
        "git",
        *args,
        env=env,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    try:
        out_b, err_b = await asyncio.wait_for(proc.communicate(), timeout=GIT_TIMEOUT)
    except asyncio.TimeoutError as exc:
        proc.kill()
        raise GitError("git timed out") from exc
    out = out_b.decode("utf-8", "replace")
    if proc.returncode != 0:
        err = _redact(err_b.decode("utf-8", "replace").strip(), secret)
        raise GitError(err or f"git {args[0]} failed (exit {proc.returncode})")
    return out


def _auth_env(source: GitSource) -> tuple[dict, Optional[str]]:
    """Environment for git, plus the plaintext secret so output can be redacted.

    Credentials go in the environment, never in argv: ``ps`` is readable by
    every process on the host, and StackPilot runs in a container people share
    with their whole stack.
    """
    env = {
        **child_env(),
        # No interactive prompting: a private repo without credentials must fail
        # fast rather than hang forever waiting on a terminal that is not there.
        "GIT_TERMINAL_PROMPT": "0",
        "GIT_CONFIG_NOSYSTEM": "1",
        "HOME": cache_root(),
    }
    if source.auth_type == "none" or not source.secret:
        return env, None

    plaintext = crypto_service.decrypt(source.secret)
    os.makedirs(cache_root(), mode=0o700, exist_ok=True)

    if source.auth_type == "ssh":
        key_file = _key_path(source.stack_id)
        fd = os.open(key_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
        with os.fdopen(fd, "w", encoding="utf-8") as fh:
            fh.write(plaintext.rstrip("\n") + "\n")
        env["GIT_SSH_COMMAND"] = (
            f"ssh -i {key_file} -o IdentitiesOnly=yes "
            # accept-new pins the host key on first contact and refuses it if it
            # ever changes, which is the strongest option that does not require
            # the operator to paste a fingerprint by hand.
            "-o StrictHostKeyChecking=accept-new "
            f"-o UserKnownHostsFile={os.path.join(cache_root(), 'known_hosts')}"
        )
        return env, plaintext

    # token: HTTPS basic auth, handed over through an askpass helper.
    askpass = _askpass_path(source.stack_id)
    fd = os.open(askpass, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o700)
    with os.fdopen(fd, "w", encoding="utf-8") as fh:
        fh.write('#!/bin/sh\ncase "$1" in Username*) echo "$GIT_USER";; *) echo "$GIT_TOKEN";; esac\n')
    env["GIT_ASKPASS"] = askpass
    env["GIT_USER"] = source.username or "git"
    env["GIT_TOKEN"] = plaintext
    return env, plaintext


def _cleanup_auth(source: GitSource) -> None:
    for path in (_key_path(source.stack_id), _askpass_path(source.stack_id)):
        try:
            os.remove(path)
        except OSError:
            pass


# --------------------------------------------------------------------------- #
# Fetching
# --------------------------------------------------------------------------- #


async def _fetch(source: GitSource, env: dict, secret: Optional[str]) -> str:
    """Bring the cached clone to the tip of the configured branch. Returns the commit."""
    directory = repo_dir(source.stack_id)
    branch = validate_branch(source.branch or "main")
    validate_url(source.url)
    os.makedirs(cache_root(), mode=0o700, exist_ok=True)

    if os.path.isdir(os.path.join(directory, ".git")):
        try:
            remote = (await _git(["-C", directory, "remote", "get-url", "origin"], env, secret)).strip()
        except GitError:
            remote = ""
        if remote != source.url:
            # Repointed at a different repository: start clean rather than try
            # to reconcile two unrelated histories.
            shutil.rmtree(directory, ignore_errors=True)

    if not os.path.isdir(os.path.join(directory, ".git")):
        # "--" ends the options: a URL that starts with "-" is a URL.
        await _git(
            ["clone", "--depth", "1", "--branch", branch, "--", source.url, directory],
            env, secret,
        )
    else:
        await _git(["-C", directory, "fetch", "--depth", "1", "origin", branch], env, secret)
        await _git(["-C", directory, "checkout", "-B", branch, "FETCH_HEAD"], env, secret)
        await _git(["-C", directory, "reset", "--hard", "FETCH_HEAD"], env, secret)
        # Safe here and only here: this directory holds nothing but the clone.
        await _git(["-C", directory, "clean", "-fdx"], env, secret)

    return (await _git(["-C", directory, "rev-parse", "HEAD"], env, secret)).strip()


# --------------------------------------------------------------------------- #
# Copying the repo's files into the stack
# --------------------------------------------------------------------------- #


def _inside(path: str, parent: str) -> bool:
    return os.path.realpath(path).startswith(os.path.realpath(parent).rstrip("/") + "/")


def _source_tree(source: GitSource) -> str:
    directory = repo_dir(source.stack_id)
    subdir = (source.subdir or "").strip().strip("/")
    if not subdir:
        return directory
    tree = os.path.join(directory, subdir)
    # The subdirectory comes from user input and is about to be walked.
    if not _inside(tree, directory):
        raise GitError(f"Subdirectory '{source.subdir}' leaves the repository")
    if not os.path.isdir(tree):
        raise GitError(f"'{source.subdir}' does not exist in the repository")
    return tree


def _materialise(source: GitSource, previous: list[str]) -> tuple[list[str], list[str], list[str]]:
    """Copy the repo subtree into the stack folder.

    Returns (current, written, removed): everything the repo provides, the
    subset that actually changed on disk, and the files dropped because the repo
    no longer has them.
    """
    tree = _source_tree(source)
    stack_dir = compose_service.stack_dir(source.stack_id)
    os.makedirs(stack_dir, exist_ok=True)

    current: list[str] = []
    written: list[str] = []
    for root, dirs, files in os.walk(tree):
        dirs[:] = [d for d in dirs if d != ".git"]
        for name in files:
            src = os.path.join(root, name)
            rel = os.path.relpath(src, tree)
            dest = os.path.join(stack_dir, rel)
            if not _inside(dest, stack_dir):
                continue  # a symlinked path trying to escape the stack folder
            current.append(rel)
            # shallow=False: compare contents, not just size and mtime, or a
            # revert to a same-sized earlier version would look like no change.
            if os.path.isfile(dest) and filecmp.cmp(src, dest, shallow=False):
                continue
            os.makedirs(os.path.dirname(dest), exist_ok=True)
            shutil.copy2(src, dest)
            written.append(rel)

    removed: list[str] = []
    for rel in previous:
        if rel in current:
            continue
        dest = os.path.join(stack_dir, rel)
        if not _inside(dest, stack_dir) or not os.path.isfile(dest):
            continue
        try:
            os.remove(dest)
            removed.append(rel)
        except OSError as exc:
            logger.warning("Could not remove %s: %s", dest, exc)
    _prune_empty_dirs(stack_dir, removed)
    return sorted(current), sorted(written), sorted(removed)


def _prune_empty_dirs(stack_dir: str, removed: list[str]) -> None:
    """Drop directories left empty by removed files, never the stack folder."""
    for rel in removed:
        directory = os.path.dirname(os.path.join(stack_dir, rel))
        while _inside(directory, stack_dir):
            try:
                os.rmdir(directory)  # fails unless empty, which is what we want
            except OSError:
                break
            directory = os.path.dirname(directory)


# --------------------------------------------------------------------------- #
# Syncing
# --------------------------------------------------------------------------- #


async def sync(session: Session, source: GitSource, actor: str = "system") -> SyncResult:
    """Fetch, copy into the stack, and deploy when something changed."""
    from models.stack import Stack

    if session.get(Stack, source.stack_id) is None:
        # A source left behind by a deleted stack: syncing it would recreate
        # the folder and, with auto deploy, start the stack again.
        raise GitError(f"Stack '{source.stack_id}' no longer exists")
    env, secret = _auth_env(source)
    try:
        commit = await _fetch(source, env, secret)
        previous = _managed(source)
        current, written, removed = _materialise(source, previous)
    except GitError as exc:
        source.last_error = _redact(str(exc), secret)[:1000]
        source.updated_at = datetime.now(timezone.utc)
        session.add(source)
        session.commit()
        raise
    finally:
        _cleanup_auth(source)

    changed = bool(written or removed)
    source.managed_files = json.dumps(current)
    source.last_commit = commit
    source.last_synced_at = datetime.now(timezone.utc)
    source.updated_at = source.last_synced_at
    source.last_error = None
    session.add(source)
    session.commit()

    result = SyncResult(changed=changed, commit=commit, written=written, removed=removed)
    if changed and source.auto_deploy:
        result.deployed, result.detail = await _deploy(session, source.stack_id, actor)
    return result


async def _deploy(session: Session, stack_id: str, actor: str) -> tuple[bool, Optional[str]]:
    """`compose up -d`, under the same lock every other lifecycle action takes."""
    from services import audit_service

    try:
        with stack_lock_service.hold(session, stack_id, "git-deploy", actor):
            outcome = await compose_service.up(stack_id)
    except stack_lock_service.StackBusy as exc:
        # Somebody is already deploying. The files are updated; say so rather
        # than queue a second compose run at the same project.
        return False, f"stack is busy ({exc.action}); files synced but not deployed"
    except Exception as exc:  # noqa: BLE001 - a failed deploy must not lose the sync
        return False, str(exc)[:500]

    ok = outcome.get("returncode") in (0, None)
    audit_service.record(
        session, user=actor, action="stack.git-deploy", target=stack_id,
        detail=f"rc={outcome.get('returncode')}",
    )
    return ok, (outcome.get("stderr") or "").strip()[-1000:] or None


def _managed(source: GitSource) -> list[str]:
    try:
        value = json.loads(source.managed_files or "[]")
    except json.JSONDecodeError:
        return []
    return [str(v) for v in value] if isinstance(value, list) else []


def forget(stack_id: str) -> None:
    """Drop the cached clone and any credential files for a stack."""
    try:
        shutil.rmtree(repo_dir(stack_id), ignore_errors=True)
    except GitError:
        return
    for path in (_key_path(stack_id), _askpass_path(stack_id)):
        try:
            os.remove(path)
        except OSError:
            pass


# --------------------------------------------------------------------------- #
# Webhooks
# --------------------------------------------------------------------------- #


def verify_webhook(source: GitSource, body: bytes, headers) -> bool:
    """Is this webhook really from the forge that holds our secret?

    Supports the two schemes between them covered by GitHub, Gitea, Forgejo and
    GitLab. Both comparisons are constant-time.
    """
    expected = source.webhook_secret or ""
    if not expected:
        return False

    signature = headers.get("X-Hub-Signature-256") or ""
    if signature.startswith("sha256="):
        digest = hmac.new(expected.encode(), body, "sha256").hexdigest()
        return hmac.compare_digest(signature[len("sha256=") :], digest)

    gitlab = headers.get("X-Gitlab-Token") or ""
    if gitlab:
        return hmac.compare_digest(gitlab, expected)

    return False


# --------------------------------------------------------------------------- #
# Polling
# --------------------------------------------------------------------------- #


def due(source: GitSource, now: Optional[datetime] = None) -> bool:
    """Is this source's polling interval up?"""
    if not source.poll_interval_minutes or source.poll_interval_minutes < 1:
        return False
    if source.last_synced_at is None:
        return True
    last = source.last_synced_at
    if last.tzinfo is None:
        last = last.replace(tzinfo=timezone.utc)
    elapsed = (now or datetime.now(timezone.utc)) - last
    return elapsed.total_seconds() >= source.poll_interval_minutes * 60


async def poll_loop(interval: float = 60.0) -> None:
    """Sync every repository whose interval is up. Never dies on one failure."""
    from sqlmodel import select

    from database import engine

    while True:
        await asyncio.sleep(interval)
        try:
            with Session(engine) as session:
                sources = session.exec(select(GitSource)).all()
                for source in sources:
                    if not due(source):
                        continue
                    try:
                        result = await sync(session, source, actor="poll")
                        if result.changed:
                            logger.info(
                                "Git sync updated %s to %s", source.stack_id,
                                (result.commit or "")[:8],
                            )
                    except Exception as exc:  # noqa: BLE001 - one repo, not all
                        logger.warning("Git sync failed for %s: %s", source.stack_id, exc)
        except Exception as exc:  # noqa: BLE001 - the loop outlives everything
            logger.warning("Git polling pass failed: %s", exc)


def ensure_cache_root() -> None:
    """The cache doubles as git's HOME, so it must exist and stay private."""
    try:
        os.makedirs(cache_root(), mode=0o700, exist_ok=True)
        os.chmod(cache_root(), stat.S_IRWXU)
    except OSError as exc:
        logger.warning("Could not create the Git cache directory: %s", exc)