Let a stuck backup be stopped instead of stranding the stack (0.67.0)

A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.

Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.

POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".

Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/routers/stacks.py

@@ -1,6 +1,8 @@
 """Stack CRUD + lifecycle endpoints."""
 from __future__ import annotations
 
+import asyncio
+import logging
 import os
 from dataclasses import asdict
 
@@ -34,12 +36,15 @@ from services import (
     icon_service,
     logo_service,
     notify_service,
+    stack_job_service,
     stack_lock_service,
     stats_service,
     update_service,
 )
 from services.convert_service import convert_docker_run
 
+logger = logging.getLogger("stackpilot.stacks")
+
 router = APIRouter(prefix="/api/stacks", tags=["stacks"])
 
 
@@ -77,6 +82,34 @@ def _auto_icon(stack: Stack) -> str | None:
     return f"logo:{slug}" if slug else None
 
 
+def _lock_view(lock: dict | None, stack_id: str) -> dict | None:
+    """The lock plus what can be done about it.
+
+    ``cancellable`` is true only while the operation is actually running in
+    this process and can stop halfway (a backup can, a restore must not).
+    Everything else can still be force-released — that drops the guard without
+    stopping anything, which is the last resort for a lock whose operation is
+    long gone.
+    """
+    if not lock:
+        return None
+    job = stack_job_service.get(stack_id)
+    return {
+        **lock,
+        "cancellable": bool(job and job.cancellable and not job.cancelled),
+        "cancelling": bool(job and job.cancelled),
+        # A backup or restore always registers itself, so a lock of theirs
+        # with no job behind it is a leftover: the run is over and releasing
+        # the row stops nothing.
+        "stale": _is_orphaned(lock["action"], job),
+        "phase": job.phase if job else "",
+    }
+
+
+def _is_orphaned(action: str, job) -> bool:
+    return job is None and action in stack_job_service.TRACKED
+
+
 def _stack_summary(
     stack: Stack, summaries: dict | None = None, busy: dict[str, str] | None = None
 ) -> dict:
@@ -229,7 +262,8 @@ def get_stack(
         containers = []
         status = "unknown"
     # An operation in flight outranks whatever the containers currently say.
-    if stack_lock_service.is_busy(session, stack_id):
+    lock = stack_lock_service.info(session, stack_id)
+    if lock:
         status = "updating"
     return {
         "id": stack.id,
@@ -238,6 +272,9 @@ def get_stack(
         "icon": stack.icon,
         "auto_icon": _auto_icon(stack),
         "status": status,
+        # What is holding the stack, so the page can offer to stop it instead
+        # of only showing a spinner that never ends.
+        "lock": _lock_view(lock, stack_id),
         "yaml": compose_service.read_compose(stack_id),
         # The .env is where credentials live by convention, so it is withheld
         # from the read-only role — same reasoning as the admin-only file
@@ -531,6 +568,113 @@ async def down_stack(stack_id: str, request: Request, session: Session = Depends
     return await _lifecycle(compose_service.down, "down", stack_id, request, session, user)
 
 
+# --------------------------------------------------------------------------- #
+# The lock: what holds a stack, and getting out from under it
+# --------------------------------------------------------------------------- #
+
+#: How long ``unlock`` waits for a cancelled job to unwind. Cancelling removes
+#: the helper container the backup streams through, so the read fails at once;
+#: what takes the time afterwards is compose starting the containers the
+#: backup stopped, which is what we want to have happened before we answer.
+CANCEL_WAIT_SECONDS = 20.0
+_CANCEL_POLL = 0.25
+
+
+@router.get("/{stack_id}/lock")
+def stack_lock(
+    stack_id: str,
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+) -> dict:
+    """What is holding this stack, if anything.
+
+    The stacks list only says "updating"; this says *what*, since when, and
+    whether it can be stopped.
+    """
+    _get_stack_or_404(session, stack_id)
+    return {"locked": stack_lock_service.is_busy(session, stack_id),
+            "lock": _lock_view(stack_lock_service.info(session, stack_id), stack_id)}
+
+
+@router.post("/{stack_id}/unlock")
+async def unlock_stack(
+    stack_id: str,
+    request: Request,
+    force: bool = Query(
+        False,
+        description="Drop the lock even when the operation could not be stopped.",
+    ),
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    """Stop whatever holds the stack, so it can be used again.
+
+    A backup is asked to stop and given a moment to unwind — it throws its
+    half-written archive away, starts the containers it stopped for a
+    consistent snapshot, and drops the lock itself. That is the clean path and
+    the one to try first.
+
+    ``force`` is the other one: it deletes the lock row without stopping
+    anything. For a lock left behind by an operation that is no longer running
+    that is all that is needed; while something *is* still running it means
+    the next deploy and that operation can collide, so it stays an explicit
+    second step.
+    """
+    _get_stack_or_404(session, stack_id)
+    lock = stack_lock_service.info(session, stack_id)
+    job = stack_job_service.get(stack_id)
+    if not lock and not job:
+        raise HTTPException(status_code=409, detail=f"Stack '{stack_id}' is not locked")
+    action = lock["action"] if lock else job.action
+
+    cancelled = False
+    if job is not None and job.cancellable:
+        stack_job_service.cancel(stack_id, user.username)
+        cancelled = True
+        deadline = asyncio.get_running_loop().time() + CANCEL_WAIT_SECONDS
+        while asyncio.get_running_loop().time() < deadline:
+            await asyncio.sleep(_CANCEL_POLL)
+            # The job releases the lock from its own session; ours would
+            # otherwise answer from its identity map.
+            session.expire_all()
+            if not stack_lock_service.is_busy(session, stack_id):
+                break
+
+    session.expire_all()
+    released = not stack_lock_service.is_busy(session, stack_id)
+    forced = False
+    if not released:
+        # A lock left behind by a backup that is no longer running guards
+        # nothing, so it goes without being asked twice.
+        orphaned = _is_orphaned(action, stack_job_service.get(stack_id))
+        if not force and not orphaned:
+            raise HTTPException(
+                status_code=409,
+                detail=(
+                    f"The {action} of '{stack_id}' did not stop in time"
+                    if cancelled
+                    else f"The {action} of '{stack_id}' cannot be stopped from here"
+                )
+                + " — release the lock with force to use the stack anyway.",
+            )
+        stack_lock_service.release(session, stack_id)
+        released, forced = True, not orphaned
+
+    audit_service.record(
+        session, user=user.username, action="stack.unlock", target=stack_id,
+        detail=f"{action}{' cancelled' if cancelled else ''}{' forced' if forced else ''}",
+        ip=_client_ip(request),
+    )
+    logger.info(
+        "Lock on '%s' (%s) released by %s%s%s",
+        stack_id, action, user.username,
+        " after cancelling the job" if cancelled else "",
+        " (forced)" if forced else "",
+    )
+    return {"ok": True, "action": action, "cancelled": cancelled,
+            "released": released, "forced": forced}
+
+
 # --------------------------------------------------------------------------- #
 # logs / export / convert
 # --------------------------------------------------------------------------- #