Lock stacks during compose runs, cache stats, persist runtime state (0.47.0)

F7 — Nothing stopped two compose operations landing on the same stack. There
was a busy flag, but is_busy() was only ever read to colour the status column;
no lifecycle handler consulted it before acting. Two tabs, or auto-update
picking up a stack somebody had just clicked, both ran pull + up -d against the
same project and raced over recreating containers.

Lifecycle calls, the two deploy WebSockets and the auto-update pass now take a
real lock; a second caller gets 409 (or an error frame and close 4409) and
auto-update skips and retries next cycle. The lock is a row rather than a set
in one worker's memory, so it holds across workers and across a restart, and it
carries an expiry — a worker killed mid-deploy would otherwise strand the stack
with no fix short of editing the database.

F10 — /api/stacks/stats sampled every running container on every call, one
blocking daemon request each, and both the dashboard and the stacks list poll
it every five seconds. Two tabs on a 40-container host meant a sustained ~16
samples a second. Cached for 4s behind a lock so concurrent callers share one
sweep, the same shape dashboard_service already used for its fleet aggregate.

F11 — Three module dicts assumed exactly one uvicorn worker without saying so
and were lost on restart. The busy set is the lock above. The image update
cache is now mirrored to SQLite, so a restart shows the badges immediately
instead of blanking them for up to an hour, and the already-notified marks come
back with them rather than re-announcing the same updates. The login rate
limiter is a table, so it cannot be cleared by getting the process to restart
and no longer multiplies by the worker count.

The constraint that shaped this: compose_service and update_service are shared
with the agent, which has no database. Neither may import one. So the lock is a
separate service the central app enforces at its own entry points, and update
persistence is an opt-in callback the central app registers in its lifespan —
the agent registers nothing and behaves exactly as before. A test asserts
update_service never imports the database, since that is the kind of thing a
later change breaks silently.

Both new nets were checked by reverting the fix: dropping the lock from
_lifecycle fails six tests, removing the stats cache fails the one that names
the behaviour.

Also wires up cache pruning in the same sweep — without it both the dict and
the table grew one entry per image tag ever run, for the life of the install.

31 new tests (729 total).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Dk43rmEeRfYi5wsLDbmfyG

backend/services/auto_update_service.py

@@ -20,7 +20,13 @@ from database import engine
 from models.agent import Agent
 from models.auto_update import AutoUpdate
 from models.setting import EVENT_PULL_FAILED, EVENT_STACK_AUTO_UPDATED
-from services import agent_service, compose_service, notify_service, update_service
+from services import (
+    agent_service,
+    compose_service,
+    notify_service,
+    stack_lock_service,
+    update_service,
+)
 
 logger = logging.getLogger("stackpilot.autoupdate")
 
@@ -104,8 +110,15 @@ async def _run_local(session: Session, policy: AutoUpdate) -> None:
     prev = policy.last_status
     if policy.redeploy:
         try:
-            await compose_service.pull(stack_id)
-            await compose_service.up(stack_id)
+            # Never redeploy underneath somebody: if a user is mid-deploy on
+            # this stack, skip and pick it up next cycle rather than racing
+            # them over the same containers.
+            with stack_lock_service.hold(session, stack_id, "auto-update", "auto-update"):
+                await compose_service.pull(stack_id)
+                await compose_service.up(stack_id)
+        except stack_lock_service.StackBusy as exc:
+            _record(session, policy, "skipped", str(exc))
+            return
         except Exception as exc:  # noqa: BLE001
             _record(session, policy, "error", str(exc))
             await _safe_notify(EVENT_PULL_FAILED, f"Auto-update of '{stack_id}' failed", str(exc), session)

backend/services/image_status_store.py

@@ -0,0 +1,80 @@
+"""Persistence for the image update cache.
+
+``update_service`` holds the registry-digest results in a module dict because
+it is shared with the agent, which has no database. This module is the central
+app's half: it seeds that dict at startup and mirrors every write back into
+SQLite, wired up in ``main.lifespan``.
+
+What it buys: after a restart the update badges are there immediately instead
+of blank until the next background sweep (up to an hour), and the "already
+notified" marks come back with them, so a restart no longer re-announces
+updates the user has already seen.
+"""
+from __future__ import annotations
+
+import logging
+
+from sqlmodel import Session, select
+
+from database import engine
+from models.runtime_state import ImageStatus
+from services import update_service
+
+logger = logging.getLogger("stackpilot.image_status")
+
+
+def _to_status(row: ImageStatus) -> update_service.UpdateStatus:
+    return update_service.UpdateStatus(
+        image=row.image,
+        update_available=row.update_available,
+        current_digest=row.current_digest,
+        remote_digest=row.remote_digest,
+        checked_at=row.checked_at,
+        error=row.error,
+    )
+
+
+def save(status: update_service.UpdateStatus, notified: bool) -> None:
+    """Upsert one image's status. Opens its own session — the caller is the
+    background loop, which has none."""
+    with Session(engine) as session:
+        row = session.get(ImageStatus, status.image)
+        if row is None:
+            row = ImageStatus(image=status.image)
+        row.update_available = status.update_available
+        row.current_digest = status.current_digest
+        row.remote_digest = status.remote_digest
+        row.checked_at = status.checked_at
+        row.error = status.error
+        row.notified = notified
+        session.add(row)
+        session.commit()
+
+
+def install() -> int:
+    """Seed the in-memory cache from the database and start mirroring writes.
+
+    Returns how many entries were restored.
+    """
+    with Session(engine) as session:
+        rows = session.exec(select(ImageStatus)).all()
+        update_service.restore_cache([(_to_status(r), r.notified) for r in rows])
+    update_service.set_persist_callback(save, prune)
+    return len(rows)
+
+
+def prune(keep: set[str]) -> int:
+    """Drop rows for images that are no longer used by any stack.
+
+    Without this the table grows for the life of the install, one row per image
+    tag that was ever running.
+    """
+    removed = 0
+    with Session(engine) as session:
+        for row in session.exec(select(ImageStatus)).all():
+            if row.image not in keep:
+                session.delete(row)
+                removed += 1
+        if removed:
+            session.commit()
+    return removed

backend/services/stack_lock_service.py

@@ -0,0 +1,148 @@
+"""One compose operation per stack at a time.
+
+``docker compose`` does no locking. Two ``update`` calls against the same
+project — two open browser tabs, or the auto-update pass landing on a stack
+somebody just clicked — both run ``pull`` and then ``up -d``, and race each
+other recreating the same containers.
+
+There *was* a busy flag (``compose_service._BUSY``), but it only ever fed the
+status column: no lifecycle handler consulted it before acting. This module is
+the actual guard, and it lives in the database so it holds across workers and
+across a restart.
+
+``compose_service`` keeps its in-process set because it is shared with the
+agent, which has no database. The agent is a single process managing one host,
+and the central app holds this lock before calling it, so the two do not
+conflict.
+"""
+from __future__ import annotations
+
+import logging
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from typing import Optional
+
+from sqlalchemy.exc import IntegrityError
+from sqlmodel import Session, delete, select
+
+from models.runtime_state import StackLock
+
+logger = logging.getLogger("stackpilot.stack_lock")
+
+#: Long enough to outlast the slowest legitimate operation (compose commands
+#: time out at 600s, a full pull of a large stack can chain several), short
+#: enough that a lock orphaned by a killed worker clears itself within an hour.
+DEFAULT_TTL = timedelta(minutes=30)
+
+
+class StackBusy(Exception):
+    """The stack is already running an operation."""
+
+    def __init__(self, stack_id: str, action: str):
+        self.stack_id = stack_id
+        self.action = action
+        super().__init__(f"Stack '{stack_id}' is busy: {action} in progress")
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _aware(value: Optional[datetime]) -> Optional[datetime]:
+    """SQLite hands datetimes back naive; compare them as UTC."""
+    if value is not None and value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value
+
+
+def acquire(
+    session: Session,
+    stack_id: str,
+    action: str,
+    owner: str = "",
+    ttl: timedelta = DEFAULT_TTL,
+) -> None:
+    """Take the lock for ``stack_id`` or raise :class:`StackBusy`.
+
+    An expired lock is taken over — that is the recovery path for a worker that
+    died mid-deploy, which would otherwise leave the stack unusable.
+    """
+    now = _now()
+    existing = session.get(StackLock, stack_id)
+    if existing is not None:
+        if (_aware(existing.expires_at) or now) > now:
+            raise StackBusy(stack_id, existing.action)
+        logger.warning(
+            "Taking over an expired %s lock on '%s' (held by %r since %s)",
+            existing.action, stack_id, existing.owner, existing.acquired_at,
+        )
+        session.delete(existing)
+        session.commit()
+
+    session.add(
+        StackLock(
+            stack_id=stack_id,
+            action=action,
+            owner=owner,
+            acquired_at=now,
+            expires_at=now + ttl,
+        )
+    )
+    try:
+        session.commit()
+    except IntegrityError as exc:
+        # Another worker inserted between our check and our commit. The primary
+        # key is what actually makes this safe; the read above is only there to
+        # give a useful error and to clear stale rows.
+        session.rollback()
+        raise StackBusy(stack_id, action) from exc
+
+
+def release(session: Session, stack_id: str) -> None:
+    """Drop the lock. Safe to call when it is not held."""
+    existing = session.get(StackLock, stack_id)
+    if existing is not None:
+        session.delete(existing)
+        session.commit()
+
+
+@contextmanager
+def hold(session: Session, stack_id: str, action: str, owner: str = ""):
+    """Hold the lock for the duration of the block.
+
+    Raises :class:`StackBusy` if somebody else has it. Always releases, so a
+    failed deploy does not leave the stack locked.
+    """
+    acquire(session, stack_id, action, owner)
+    try:
+        yield
+    finally:
+        try:
+            release(session, stack_id)
+        except Exception:  # noqa: BLE001 - never mask the original error
+            logger.exception("Failed to release the lock on '%s'", stack_id)
+
+
+def active(session: Session) -> dict[str, str]:
+    """``{stack_id: action}`` for every lock still in force.
+
+    One query for the whole stacks list, rather than a lookup per row.
+    """
+    now = _now()
+    return {
+        lock.stack_id: lock.action
+        for lock in session.exec(select(StackLock)).all()
+        if (_aware(lock.expires_at) or now) > now
+    }
+
+
+def is_busy(session: Session, stack_id: str) -> bool:
+    lock = session.get(StackLock, stack_id)
+    return lock is not None and (_aware(lock.expires_at) or _now()) > _now()
+
+
+def prune_expired(session: Session) -> int:
+    """Drop locks that have timed out. Called at startup and by the scheduler."""
+    result = session.exec(delete(StackLock).where(StackLock.expires_at < _now()))
+    session.commit()
+    return result.rowcount or 0

backend/services/stats_service.py

@@ -3,15 +3,33 @@
 Reads a one-shot ``docker stats`` sample per running container (the daemon
 includes ``precpu_stats`` so a single read yields a usable CPU delta) and sums
 them by ``com.docker.compose.project`` label, which equals the stack id.
+
+Sampling is not free: it is one blocking call to the daemon *per running
+container*, and the dashboard and the stacks list both poll this every five
+seconds. Two open tabs on a 40-container host meant a sustained ~16 samples a
+second. Results are therefore cached for :data:`CACHE_TTL`, the same shape
+``dashboard_service`` already uses for its fleet aggregate — one sweep serves
+every reader in the window, and the numbers stay well inside what a
+five-second poll can show.
 """
 from __future__ import annotations
 
+import threading
+import time
 from concurrent.futures import ThreadPoolExecutor
 
 from docker_client import DockerError, get_client, safe_call
 
 COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
 
+#: Slightly under the frontend's 5s poll, so a refresh usually gets fresh
+#: numbers while concurrent readers still share one sweep.
+CACHE_TTL = 4.0
+
+_cache: dict = {"data": None, "ts": 0.0}
+# Held across the sample so N simultaneous callers trigger one sweep, not N.
+_lock = threading.Lock()
+
 
 def _container_stats(container) -> dict | None:
     try:
@@ -60,12 +78,30 @@ def _container_stats(container) -> dict | None:
     }
 
 
-def stack_stats() -> dict:
+def stack_stats(refresh: bool = False) -> dict:
     """Return {stack_id: {cpu_used, cpu_limit, mem_used, mem_limit, containers}}.
 
     Limits are the summed assigned limits across the stack's containers, or null
-    when none of them have that limit set.
+    when none of them have that limit set. Served from a short-lived cache
+    unless ``refresh`` is set.
     """
+    if not refresh and _cache["data"] is not None:
+        if time.monotonic() - _cache["ts"] < CACHE_TTL:
+            return _cache["data"]
+
+    with _lock:
+        # Somebody may have refreshed it while we waited for the lock.
+        if not refresh and _cache["data"] is not None:
+            if time.monotonic() - _cache["ts"] < CACHE_TTL:
+                return _cache["data"]
+        data = _sample()
+        _cache["data"] = data
+        _cache["ts"] = time.monotonic()
+        return data
+
+
+def _sample() -> dict:
+    """One full sweep across every running container."""
     try:
         client = get_client()
         containers = safe_call(client.containers.list)  # running only

backend/services/update_service.py

@@ -10,6 +10,7 @@ import asyncio
 import logging
 import time
 from dataclasses import asdict, dataclass
+from collections.abc import Callable
 from typing import Optional
 
 import httpx
@@ -50,6 +51,49 @@ _CACHE: dict[str, UpdateStatus] = {}
 # background loop doesn't re-notify on every cycle.
 _NOTIFIED: set[str] = set()
 
+#: Optional sink for cache writes.
+#:
+#: This module is shared with the agent, which has no database, so persistence
+#: cannot live here. The central app registers a callback that mirrors each
+#: entry into SQLite (see ``services/image_status_store.py``) and seeds the
+#: cache from it at startup; the agent registers nothing and behaves exactly as
+#: before. Without it a restart blanked every update badge until the next
+#: background sweep — up to an hour — and re-announced updates it had already
+#: notified about.
+_persist_cb: Optional[Callable[[UpdateStatus, bool], None]] = None
+
+
+#: Optional sink for "these images are still in use", same opt-in shape as
+#: _persist_cb. Keeps both the dict and the table from growing one entry per
+#: image tag that was ever running, for the life of the install.
+_prune_cb: Optional[Callable[[set], int]] = None
+
+
+def set_persist_callback(
+    callback: Optional[Callable[[UpdateStatus, bool], None]],
+    prune: Optional[Callable[[set], int]] = None,
+) -> None:
+    global _persist_cb, _prune_cb
+    _persist_cb = callback
+    _prune_cb = prune
+
+
+def restore_cache(entries: list[tuple[UpdateStatus, bool]]) -> None:
+    """Seed the in-memory cache from persisted rows at startup."""
+    for status, notified in entries:
+        _CACHE[status.image] = status
+        if notified:
+            _NOTIFIED.add(status.image)
+
+
+def _store(status: UpdateStatus, notified: bool) -> None:
+    _CACHE[status.image] = status
+    if _persist_cb is not None:
+        try:
+            _persist_cb(status, notified)
+        except Exception as exc:  # noqa: BLE001 - persistence is best-effort
+            logger.debug("Could not persist update status for %s: %s", status.image, exc)
+
 
 # --------------------------------------------------------------------------- #
 # Image reference parsing
@@ -168,9 +212,11 @@ async def check_image(image: str) -> UpdateStatus:
         checked_at=time.time(),
         error=error,
     )
-    _CACHE[image] = status
     if update_available and image not in _NOTIFIED:
+        # Marked before the attempt, not after: a notifier that is down should
+        # not make every cycle re-announce the same update.
         _NOTIFIED.add(image)
+        _store(status, True)
         try:
             await notify_service.notify(
                 EVENT_UPDATE_AVAILABLE,
@@ -179,8 +225,10 @@ async def check_image(image: str) -> UpdateStatus:
             )
         except Exception as exc:  # noqa: BLE001 - notifications are best-effort
             logger.debug("update notify failed for %s: %s", image, exc)
-    elif not update_available:
-        _NOTIFIED.discard(image)
+    else:
+        if not update_available:
+            _NOTIFIED.discard(image)
+        _store(status, image in _NOTIFIED)
     return status
 
 
@@ -291,9 +339,22 @@ async def check_all() -> dict[str, dict]:
     images = _all_running_images()
     for image in images:
         await check_image(image)
+    _forget_unused(set(images))
     return {k: v.to_dict() for k, v in _CACHE.items()}
 
 
+def _forget_unused(keep: set) -> None:
+    """Drop images no running container references any more."""
+    for image in [i for i in _CACHE if i not in keep]:
+        del _CACHE[image]
+        _NOTIFIED.discard(image)
+    if _prune_cb is not None:
+        try:
+            _prune_cb(keep)
+        except Exception as exc:  # noqa: BLE001 - housekeeping is best-effort
+            logger.debug("Could not prune persisted update statuses: %s", exc)
+
+
 def get_cache() -> dict[str, dict]:
     return {k: v.to_dict() for k, v in _CACHE.items()}