stackpilot/backend/services/stack_lock_service.py

"""One compose operation per stack at a time.

``docker compose`` does no locking. Two ``update`` calls against the same
project — two open browser tabs, or the auto-update pass landing on a stack
somebody just clicked — both run ``pull`` and then ``up -d``, and race each
other recreating the same containers.

There *was* a busy flag (``compose_service._BUSY``), but it only ever fed the
status column: no lifecycle handler consulted it before acting. This module is
the actual guard, and it lives in the database so it holds across workers and
across a restart.

``compose_service`` keeps its in-process set because it is shared with the
agent, which has no database. The agent is a single process managing one host,
and the central app holds this lock before calling it, so the two do not
conflict.
"""
from __future__ import annotations

import logging
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from typing import Optional

from sqlalchemy.exc import IntegrityError
from sqlmodel import Session, delete, select

from models.runtime_state import StackLock

logger = logging.getLogger("stackpilot.stack_lock")

#: Long enough to outlast the slowest legitimate operation (compose commands
#: time out at 600s, a full pull of a large stack can chain several), short
#: enough that a lock orphaned by a killed worker clears itself within an hour.
DEFAULT_TTL = timedelta(minutes=30)


class StackBusy(Exception):
    """The stack is already running an operation."""

    def __init__(self, stack_id: str, action: str):
        self.stack_id = stack_id
        self.action = action
        super().__init__(f"Stack '{stack_id}' is busy: {action} in progress")


def _now() -> datetime:
    return datetime.now(timezone.utc)


def _aware(value: Optional[datetime]) -> Optional[datetime]:
    """SQLite hands datetimes back naive; compare them as UTC."""
    if value is not None and value.tzinfo is None:
        return value.replace(tzinfo=timezone.utc)
    return value


def acquire(
    session: Session,
    stack_id: str,
    action: str,
    owner: str = "",
    ttl: timedelta = DEFAULT_TTL,
) -> None:
    """Take the lock for ``stack_id`` or raise :class:`StackBusy`.

    An expired lock is taken over — that is the recovery path for a worker that
    died mid-deploy, which would otherwise leave the stack unusable.
    """
    now = _now()
    existing = session.get(StackLock, stack_id)
    if existing is not None:
        if (_aware(existing.expires_at) or now) > now:
            raise StackBusy(stack_id, existing.action)
        logger.warning(
            "Taking over an expired %s lock on '%s' (held by %r since %s)",
            existing.action, stack_id, existing.owner, existing.acquired_at,
        )
        session.delete(existing)
        session.commit()

    session.add(
        StackLock(
            stack_id=stack_id,
            action=action,
            owner=owner,
            acquired_at=now,
            expires_at=now + ttl,
        )
    )
    try:
        session.commit()
    except IntegrityError as exc:
        # Another worker inserted between our check and our commit. The primary
        # key is what actually makes this safe; the read above is only there to
        # give a useful error and to clear stale rows.
        session.rollback()
        raise StackBusy(stack_id, action) from exc


def release(session: Session, stack_id: str) -> None:
    """Drop the lock. Safe to call when it is not held."""
    existing = session.get(StackLock, stack_id)
    if existing is not None:
        session.delete(existing)
        session.commit()


@contextmanager
def hold(session: Session, stack_id: str, action: str, owner: str = ""):
    """Hold the lock for the duration of the block.

    Raises :class:`StackBusy` if somebody else has it. Always releases, so a
    failed deploy does not leave the stack locked.
    """
    acquire(session, stack_id, action, owner)
    try:
        yield
    finally:
        try:
            release(session, stack_id)
        except Exception:  # noqa: BLE001 - never mask the original error
            logger.exception("Failed to release the lock on '%s'", stack_id)


def active(session: Session) -> dict[str, str]:
    """``{stack_id: action}`` for every lock still in force.

    One query for the whole stacks list, rather than a lookup per row.
    """
    now = _now()
    return {
        lock.stack_id: lock.action
        for lock in session.exec(select(StackLock)).all()
        if (_aware(lock.expires_at) or now) > now
    }


def is_busy(session: Session, stack_id: str) -> bool:
    lock = session.get(StackLock, stack_id)
    return lock is not None and (_aware(lock.expires_at) or _now()) > _now()


def prune_expired(session: Session) -> int:
    """Drop locks that have timed out. Called at startup and by the scheduler."""
    result = session.exec(delete(StackLock).where(StackLock.expires_at < _now()))
    session.commit()
    return result.rowcount or 0