menzeljandClaude Opus 5
052a85067b
docker compose ran with StackPilot's whole environment, and compose
resolves ${VAR} from the calling environment before a stack's own .env.
A stack whose compose file says SECRET_KEY: ${SECRET_KEY} therefore ran
with StackPilot's SECRET_KEY instead of the one in its .env — and with
its own again whenever it was started from a shell. AnsiblePilot hit
exactly this: credentials saved under one key became unreadable under
the other, twice.
config.child_env() is os.environ minus every StackPilot setting
(SECRET_KEY, DATA_DIR, STACKS_DIR, PORT, ...); the docker CLI and git
subprocesses now get that instead.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
448 lines
17 KiB
Python
448 lines
17 KiB
Python
"""Deploying a stack from a Git repository.
|
|
|
|
The repository is the source of truth: a sync makes the stack's files match what
|
|
the repo says, and optionally runs ``compose up -d`` when that changed anything.
|
|
Two things about *how* are worth stating up front, because both are places this
|
|
could quietly destroy data.
|
|
|
|
**The clone does not live in the stack folder.** It is cached under
|
|
``${DATA_DIR}/git/<stack_id>`` and the relevant subtree is copied across. A
|
|
stack folder holds more than the repo's files — compose creates bind-mount
|
|
directories like ``./config`` right there, full of live application data — so a
|
|
``git reset --hard`` or ``git clean`` in that folder would be catastrophic. In
|
|
the cache directory both are safe, and the copy step is where the care goes.
|
|
|
|
**Only files the repo has provided are ever deleted.** Each sync records the
|
|
paths it wrote (``GitSource.managed_files``); the next sync removes the ones the
|
|
repo no longer has, and nothing else. A file that was never in the repository
|
|
cannot be touched, no matter what happened to it.
|
|
|
|
Credentials never reach a command line. A token is passed to git through
|
|
``GIT_ASKPASS`` and an environment variable, an SSH key through a 0600 file
|
|
outside the working tree — so neither shows up in ``ps``, in the repo's own
|
|
config, or in an error message this module passes on.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import filecmp
|
|
import hmac
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import stat
|
|
from datetime import datetime, timezone
|
|
from typing import Optional
|
|
|
|
from sqlmodel import Session
|
|
|
|
from config import child_env, settings
|
|
from models.git_source import GitSource, SyncResult
|
|
from services import compose_service, crypto_service, stack_lock_service
|
|
|
|
logger = logging.getLogger("stackpilot.git")
|
|
|
|
GIT_TIMEOUT = 300.0
|
|
_SAFE_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,128}$")
|
|
|
|
|
|
class GitError(Exception):
|
|
"""A repository that cannot be reached, or a sync that cannot be completed."""
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Paths
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def cache_root() -> str:
|
|
return os.path.join(settings.DATA_DIR, "git")
|
|
|
|
|
|
def repo_dir(stack_id: str) -> str:
|
|
if not _SAFE_ID_RE.match(stack_id or "") or stack_id in (".", ".."):
|
|
raise GitError(f"Invalid stack id '{stack_id}'")
|
|
return os.path.join(cache_root(), stack_id)
|
|
|
|
|
|
def _key_path(stack_id: str) -> str:
|
|
# Beside the clone, never inside it — a working tree gets reset and cleaned.
|
|
return os.path.join(cache_root(), f"{stack_id}.key")
|
|
|
|
|
|
def _askpass_path(stack_id: str) -> str:
|
|
return os.path.join(cache_root(), f"{stack_id}.askpass")
|
|
|
|
|
|
def new_webhook_secret() -> str:
|
|
return secrets.token_urlsafe(24)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Running git
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _redact(text: str, *secrets_: Optional[str]) -> str:
|
|
"""Strip anything secret out of git's output before it is shown or stored."""
|
|
for value in secrets_:
|
|
if value:
|
|
text = text.replace(value, "••••••")
|
|
# A URL that carries credentials, in case one ever reaches git's output.
|
|
return re.sub(r"(https?://)[^/\s:@]+:[^/\s@]+@", r"\1••••••@", text)
|
|
|
|
|
|
async def _git(args: list[str], env: dict, secret: Optional[str] = None) -> str:
|
|
"""Run git, returning stdout. Raises GitError with a redacted message."""
|
|
proc = await asyncio.create_subprocess_exec(
|
|
"git",
|
|
*args,
|
|
env=env,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
try:
|
|
out_b, err_b = await asyncio.wait_for(proc.communicate(), timeout=GIT_TIMEOUT)
|
|
except asyncio.TimeoutError as exc:
|
|
proc.kill()
|
|
raise GitError("git timed out") from exc
|
|
out = out_b.decode("utf-8", "replace")
|
|
if proc.returncode != 0:
|
|
err = _redact(err_b.decode("utf-8", "replace").strip(), secret)
|
|
raise GitError(err or f"git {args[0]} failed (exit {proc.returncode})")
|
|
return out
|
|
|
|
|
|
def _auth_env(source: GitSource) -> tuple[dict, Optional[str]]:
|
|
"""Environment for git, plus the plaintext secret so output can be redacted.
|
|
|
|
Credentials go in the environment, never in argv: ``ps`` is readable by
|
|
every process on the host, and StackPilot runs in a container people share
|
|
with their whole stack.
|
|
"""
|
|
env = {
|
|
**child_env(),
|
|
# No interactive prompting: a private repo without credentials must fail
|
|
# fast rather than hang forever waiting on a terminal that is not there.
|
|
"GIT_TERMINAL_PROMPT": "0",
|
|
"GIT_CONFIG_NOSYSTEM": "1",
|
|
"HOME": cache_root(),
|
|
}
|
|
if source.auth_type == "none" or not source.secret:
|
|
return env, None
|
|
|
|
plaintext = crypto_service.decrypt(source.secret)
|
|
os.makedirs(cache_root(), mode=0o700, exist_ok=True)
|
|
|
|
if source.auth_type == "ssh":
|
|
key_file = _key_path(source.stack_id)
|
|
fd = os.open(key_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
fh.write(plaintext.rstrip("\n") + "\n")
|
|
env["GIT_SSH_COMMAND"] = (
|
|
f"ssh -i {key_file} -o IdentitiesOnly=yes "
|
|
# accept-new pins the host key on first contact and refuses it if it
|
|
# ever changes, which is the strongest option that does not require
|
|
# the operator to paste a fingerprint by hand.
|
|
"-o StrictHostKeyChecking=accept-new "
|
|
f"-o UserKnownHostsFile={os.path.join(cache_root(), 'known_hosts')}"
|
|
)
|
|
return env, plaintext
|
|
|
|
# token: HTTPS basic auth, handed over through an askpass helper.
|
|
askpass = _askpass_path(source.stack_id)
|
|
fd = os.open(askpass, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o700)
|
|
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
fh.write('#!/bin/sh\ncase "$1" in Username*) echo "$GIT_USER";; *) echo "$GIT_TOKEN";; esac\n')
|
|
env["GIT_ASKPASS"] = askpass
|
|
env["GIT_USER"] = source.username or "git"
|
|
env["GIT_TOKEN"] = plaintext
|
|
return env, plaintext
|
|
|
|
|
|
def _cleanup_auth(source: GitSource) -> None:
|
|
for path in (_key_path(source.stack_id), _askpass_path(source.stack_id)):
|
|
try:
|
|
os.remove(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Fetching
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
async def _fetch(source: GitSource, env: dict, secret: Optional[str]) -> str:
|
|
"""Bring the cached clone to the tip of the configured branch. Returns the commit."""
|
|
directory = repo_dir(source.stack_id)
|
|
branch = source.branch or "main"
|
|
os.makedirs(cache_root(), mode=0o700, exist_ok=True)
|
|
|
|
if os.path.isdir(os.path.join(directory, ".git")):
|
|
try:
|
|
remote = (await _git(["-C", directory, "remote", "get-url", "origin"], env, secret)).strip()
|
|
except GitError:
|
|
remote = ""
|
|
if remote != source.url:
|
|
# Repointed at a different repository: start clean rather than try
|
|
# to reconcile two unrelated histories.
|
|
shutil.rmtree(directory, ignore_errors=True)
|
|
|
|
if not os.path.isdir(os.path.join(directory, ".git")):
|
|
await _git(
|
|
["clone", "--depth", "1", "--branch", branch, source.url, directory], env, secret
|
|
)
|
|
else:
|
|
await _git(["-C", directory, "fetch", "--depth", "1", "origin", branch], env, secret)
|
|
await _git(["-C", directory, "checkout", "-B", branch, "FETCH_HEAD"], env, secret)
|
|
await _git(["-C", directory, "reset", "--hard", "FETCH_HEAD"], env, secret)
|
|
# Safe here and only here: this directory holds nothing but the clone.
|
|
await _git(["-C", directory, "clean", "-fdx"], env, secret)
|
|
|
|
return (await _git(["-C", directory, "rev-parse", "HEAD"], env, secret)).strip()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Copying the repo's files into the stack
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _inside(path: str, parent: str) -> bool:
|
|
return os.path.realpath(path).startswith(os.path.realpath(parent).rstrip("/") + "/")
|
|
|
|
|
|
def _source_tree(source: GitSource) -> str:
|
|
directory = repo_dir(source.stack_id)
|
|
subdir = (source.subdir or "").strip().strip("/")
|
|
if not subdir:
|
|
return directory
|
|
tree = os.path.join(directory, subdir)
|
|
# The subdirectory comes from user input and is about to be walked.
|
|
if not _inside(tree, directory):
|
|
raise GitError(f"Subdirectory '{source.subdir}' leaves the repository")
|
|
if not os.path.isdir(tree):
|
|
raise GitError(f"'{source.subdir}' does not exist in the repository")
|
|
return tree
|
|
|
|
|
|
def _materialise(source: GitSource, previous: list[str]) -> tuple[list[str], list[str], list[str]]:
|
|
"""Copy the repo subtree into the stack folder.
|
|
|
|
Returns (current, written, removed): everything the repo provides, the
|
|
subset that actually changed on disk, and the files dropped because the repo
|
|
no longer has them.
|
|
"""
|
|
tree = _source_tree(source)
|
|
stack_dir = compose_service.stack_dir(source.stack_id)
|
|
os.makedirs(stack_dir, exist_ok=True)
|
|
|
|
current: list[str] = []
|
|
written: list[str] = []
|
|
for root, dirs, files in os.walk(tree):
|
|
dirs[:] = [d for d in dirs if d != ".git"]
|
|
for name in files:
|
|
src = os.path.join(root, name)
|
|
rel = os.path.relpath(src, tree)
|
|
dest = os.path.join(stack_dir, rel)
|
|
if not _inside(dest, stack_dir):
|
|
continue # a symlinked path trying to escape the stack folder
|
|
current.append(rel)
|
|
# shallow=False: compare contents, not just size and mtime, or a
|
|
# revert to a same-sized earlier version would look like no change.
|
|
if os.path.isfile(dest) and filecmp.cmp(src, dest, shallow=False):
|
|
continue
|
|
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
shutil.copy2(src, dest)
|
|
written.append(rel)
|
|
|
|
removed: list[str] = []
|
|
for rel in previous:
|
|
if rel in current:
|
|
continue
|
|
dest = os.path.join(stack_dir, rel)
|
|
if not _inside(dest, stack_dir) or not os.path.isfile(dest):
|
|
continue
|
|
try:
|
|
os.remove(dest)
|
|
removed.append(rel)
|
|
except OSError as exc:
|
|
logger.warning("Could not remove %s: %s", dest, exc)
|
|
_prune_empty_dirs(stack_dir, removed)
|
|
return sorted(current), sorted(written), sorted(removed)
|
|
|
|
|
|
def _prune_empty_dirs(stack_dir: str, removed: list[str]) -> None:
|
|
"""Drop directories left empty by removed files, never the stack folder."""
|
|
for rel in removed:
|
|
directory = os.path.dirname(os.path.join(stack_dir, rel))
|
|
while _inside(directory, stack_dir):
|
|
try:
|
|
os.rmdir(directory) # fails unless empty, which is what we want
|
|
except OSError:
|
|
break
|
|
directory = os.path.dirname(directory)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Syncing
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
async def sync(session: Session, source: GitSource, actor: str = "system") -> SyncResult:
|
|
"""Fetch, copy into the stack, and deploy when something changed."""
|
|
env, secret = _auth_env(source)
|
|
try:
|
|
commit = await _fetch(source, env, secret)
|
|
previous = _managed(source)
|
|
current, written, removed = _materialise(source, previous)
|
|
except GitError as exc:
|
|
source.last_error = _redact(str(exc), secret)[:1000]
|
|
source.updated_at = datetime.now(timezone.utc)
|
|
session.add(source)
|
|
session.commit()
|
|
raise
|
|
finally:
|
|
_cleanup_auth(source)
|
|
|
|
changed = bool(written or removed)
|
|
source.managed_files = json.dumps(current)
|
|
source.last_commit = commit
|
|
source.last_synced_at = datetime.now(timezone.utc)
|
|
source.updated_at = source.last_synced_at
|
|
source.last_error = None
|
|
session.add(source)
|
|
session.commit()
|
|
|
|
result = SyncResult(changed=changed, commit=commit, written=written, removed=removed)
|
|
if changed and source.auto_deploy:
|
|
result.deployed, result.detail = await _deploy(session, source.stack_id, actor)
|
|
return result
|
|
|
|
|
|
async def _deploy(session: Session, stack_id: str, actor: str) -> tuple[bool, Optional[str]]:
|
|
"""`compose up -d`, under the same lock every other lifecycle action takes."""
|
|
from services import audit_service
|
|
|
|
try:
|
|
with stack_lock_service.hold(session, stack_id, "git-deploy", actor):
|
|
outcome = await compose_service.up(stack_id)
|
|
except stack_lock_service.StackBusy as exc:
|
|
# Somebody is already deploying. The files are updated; say so rather
|
|
# than queue a second compose run at the same project.
|
|
return False, f"stack is busy ({exc.action}); files synced but not deployed"
|
|
except Exception as exc: # noqa: BLE001 - a failed deploy must not lose the sync
|
|
return False, str(exc)[:500]
|
|
|
|
ok = outcome.get("returncode") in (0, None)
|
|
audit_service.record(
|
|
session, user=actor, action="stack.git-deploy", target=stack_id,
|
|
detail=f"rc={outcome.get('returncode')}",
|
|
)
|
|
return ok, (outcome.get("stderr") or "").strip()[-1000:] or None
|
|
|
|
|
|
def _managed(source: GitSource) -> list[str]:
|
|
try:
|
|
value = json.loads(source.managed_files or "[]")
|
|
except json.JSONDecodeError:
|
|
return []
|
|
return [str(v) for v in value] if isinstance(value, list) else []
|
|
|
|
|
|
def forget(stack_id: str) -> None:
|
|
"""Drop the cached clone and any credential files for a stack."""
|
|
try:
|
|
shutil.rmtree(repo_dir(stack_id), ignore_errors=True)
|
|
except GitError:
|
|
return
|
|
for path in (_key_path(stack_id), _askpass_path(stack_id)):
|
|
try:
|
|
os.remove(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Webhooks
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def verify_webhook(source: GitSource, body: bytes, headers) -> bool:
|
|
"""Is this webhook really from the forge that holds our secret?
|
|
|
|
Supports the two schemes between them covered by GitHub, Gitea, Forgejo and
|
|
GitLab. Both comparisons are constant-time.
|
|
"""
|
|
expected = source.webhook_secret or ""
|
|
if not expected:
|
|
return False
|
|
|
|
signature = headers.get("X-Hub-Signature-256") or ""
|
|
if signature.startswith("sha256="):
|
|
digest = hmac.new(expected.encode(), body, "sha256").hexdigest()
|
|
return hmac.compare_digest(signature[len("sha256=") :], digest)
|
|
|
|
gitlab = headers.get("X-Gitlab-Token") or ""
|
|
if gitlab:
|
|
return hmac.compare_digest(gitlab, expected)
|
|
|
|
return False
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Polling
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def due(source: GitSource, now: Optional[datetime] = None) -> bool:
|
|
"""Is this source's polling interval up?"""
|
|
if not source.poll_interval_minutes or source.poll_interval_minutes < 1:
|
|
return False
|
|
if source.last_synced_at is None:
|
|
return True
|
|
last = source.last_synced_at
|
|
if last.tzinfo is None:
|
|
last = last.replace(tzinfo=timezone.utc)
|
|
elapsed = (now or datetime.now(timezone.utc)) - last
|
|
return elapsed.total_seconds() >= source.poll_interval_minutes * 60
|
|
|
|
|
|
async def poll_loop(interval: float = 60.0) -> None:
|
|
"""Sync every repository whose interval is up. Never dies on one failure."""
|
|
from sqlmodel import select
|
|
|
|
from database import engine
|
|
|
|
while True:
|
|
await asyncio.sleep(interval)
|
|
try:
|
|
with Session(engine) as session:
|
|
sources = session.exec(select(GitSource)).all()
|
|
for source in sources:
|
|
if not due(source):
|
|
continue
|
|
try:
|
|
result = await sync(session, source, actor="poll")
|
|
if result.changed:
|
|
logger.info(
|
|
"Git sync updated %s to %s", source.stack_id,
|
|
(result.commit or "")[:8],
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - one repo, not all
|
|
logger.warning("Git sync failed for %s: %s", source.stack_id, exc)
|
|
except Exception as exc: # noqa: BLE001 - the loop outlives everything
|
|
logger.warning("Git polling pass failed: %s", exc)
|
|
|
|
|
|
def ensure_cache_root() -> None:
|
|
"""The cache doubles as git's HOME, so it must exist and stay private."""
|
|
try:
|
|
os.makedirs(cache_root(), mode=0o700, exist_ok=True)
|
|
os.chmod(cache_root(), stat.S_IRWXU)
|
|
except OSError as exc:
|
|
logger.warning("Could not create the Git cache directory: %s", exc)
|