Let a stuck backup be stopped instead of stranding the stack (0.67.0)

A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.

Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.

POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".

Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/services/backup_service.py

@@ -52,7 +52,12 @@ from sqlmodel import Session
 
 from database import engine
 from docker_client import DockerError, get_long_client, safe_call
-from services import compose_service, stack_lock_service, stack_assets_service as assets
+from services import (
+    compose_service,
+    stack_job_service,
+    stack_lock_service,
+    stack_assets_service as assets,
+)
 
 logger = logging.getLogger("stackpilot.backup")
 
@@ -76,6 +81,15 @@ class StackBusyError(BackupError):
     """The stack is locked by another operation (deploy, update, backup)."""
 
 
+class BackupCancelled(BackupError):
+    """Somebody stopped this backup from the outside.
+
+    Not a failure: the archive is thrown away, the stack is started again and
+    the lock goes, exactly as after an error — but nothing is reported as
+    broken and no "backup failed" notification goes out.
+    """
+
+
 # --------------------------------------------------------------------------- #
 # Backup filename convention
 # --------------------------------------------------------------------------- #
@@ -112,14 +126,23 @@ def backup_name_pattern(stack_id: str, prefix: Optional[str] = None) -> re.Patte
 
 
 @contextmanager
-def _locked(stack_id: str, action: str):
+def _locked(stack_id: str, action: str, owner: str = ""):
+    """Hold the stack lock, and register the job so it can be stopped.
+
+    The lock alone made a stuck backup fatal: nothing could touch the stack
+    until its twelve-hour TTL ran out. The registration is what gives the
+    unlock endpoint something to cancel.
+    """
     with Session(engine) as session:
         try:
-            stack_lock_service.acquire(session, stack_id, action, action, ttl=LOCK_TTL)
+            stack_lock_service.acquire(session, stack_id, action, owner or action, ttl=LOCK_TTL)
         except stack_lock_service.StackBusy as exc:
             raise StackBusyError(str(exc)) from exc
         try:
-            yield
+            with stack_job_service.track(stack_id, action, owner or action):
+                yield
+        except stack_job_service.JobCancelled as exc:
+            raise BackupCancelled(str(exc)) from exc
         finally:
             try:
                 stack_lock_service.release(session, stack_id)
@@ -275,6 +298,10 @@ def _add_tree(
     """
 
     def add(path: str, rel: str, parent_dev: int) -> None:
+        # A stack directory with a data folder in it can take as long as a
+        # volume, so a cancellation lands here too — between two files, with
+        # the half-written archive thrown away by the caller.
+        stack_job_service.check()
         arc = f"{arcroot}/{rel}" if rel else arcroot
         label = rel or "."
         try:
@@ -387,6 +414,7 @@ async def _create(
     binds: Optional[list[str]],
     volumes: Optional[list[str]],
 ) -> tuple[str, dict]:
+    stack_job_service.phase("looking at what to capture")
     selection = await asyncio.to_thread(
         plan, stack_id, include_volumes, include_binds, binds, volumes
     )
@@ -429,6 +457,11 @@ async def _create(
     try:
         parts: list[tuple[str, str]] = []
         for index, bind in enumerate(cap_binds):
+            # Between two items is where a cancelled backup gives up: never in
+            # the middle of one, and never after the stack has been stopped
+            # without the finally below starting it again.
+            stack_job_service.check()
+            stack_job_service.phase(f"bind {bind['source']}")
             part = os.path.join(workdir, f"bind-{index:03d}.tar")
             try:
                 bind["bytes"] = await asyncio.to_thread(
@@ -437,16 +470,22 @@ async def _create(
                 bind["archive"] = f"binds/{index:03d}.tar"
                 parts.append((part, bind["archive"]))
             except Exception as exc:  # noqa: BLE001
+                # Cancelling pulls the helper container out from under the
+                # read, so the failure here *is* the cancellation.
+                stack_job_service.check()
                 logger.warning("Could not archive bind %s: %s", bind["source"], exc)
                 bind["archive"], bind["error"] = None, str(exc)
 
         for index, vol in enumerate(cap_volumes):
+            stack_job_service.check()
+            stack_job_service.phase(f"volume {vol['name']}")
             part = os.path.join(workdir, f"vol-{index:03d}.tar")
             try:
                 vol["bytes"] = await asyncio.to_thread(assets.export_volume, vol["name"], part)
                 vol["archive"] = f"volumes/{vol['name']}.tar"
                 parts.append((part, vol["archive"]))
             except Exception as exc:  # noqa: BLE001
+                stack_job_service.check()
                 logger.warning("Could not archive volume %s: %s", vol["name"], exc)
                 vol["archive"], vol["error"] = None, str(exc)
 
@@ -514,6 +553,8 @@ async def _create(
             ],
         }
 
+        stack_job_service.check()
+        stack_job_service.phase("writing the archive")
         fd, archive_path = tempfile.mkstemp(prefix=f"sp-{stack_id}-", suffix=".tar.gz")
         os.close(fd)
         await asyncio.to_thread(

backend/services/schedule_service.py

@@ -170,6 +170,11 @@ async def _backup_stack(
         return "ok"
     except backup_service.StackBusyError as exc:
         raise _Busy(str(exc)) from exc
+    except backup_service.BackupCancelled as exc:
+        # Somebody stopped this backup from the UI. Not a failure of the
+        # schedule: say so, and don't page anyone with "backup failed".
+        logger.info("Scheduled backup of %s was cancelled: %s", stack_id, exc)
+        return f"cancelled: {exc}"
     except Exception as exc:  # noqa: BLE001
         logger.warning("Scheduled backup of %s failed: %s", stack_id, exc)
         return f"error: {exc}"
@@ -185,7 +190,11 @@ def _summarise(statuses: dict[str, str]) -> str:
     bad = {sid: st for sid, st in statuses.items() if st != "ok"}
     if not bad:
         return "ok"
-    worst = "error" if any(not st.startswith("warning") for st in bad.values()) else "warning"
+    worst = (
+        "error"
+        if any(not st.startswith(("warning", "cancelled")) for st in bad.values())
+        else "warning"
+    )
     if len(statuses) == 1:
         return next(iter(statuses.values()))
     detail = "; ".join(f"{sid}: {st}" for sid, st in sorted(bad.items()))

backend/services/stack_assets_service.py

@@ -32,7 +32,7 @@ from docker_client import (
     run_container,
     safe_call,
 )
-from services import compose_service
+from services import compose_service, stack_job_service
 
 logger = logging.getLogger("stackpilot.assets")
 
@@ -99,16 +99,22 @@ def ensure_helper_image(client) -> None:
 
 
 def _create_helper(client, volumes: dict):
-    return safe_call(
+    container = safe_call(
         client.containers.create,
         settings.BACKUP_HELPER_IMAGE,
         command="true",
         volumes=volumes,
         labels={HELPER_LABEL: "1"},
     )
+    # Streaming a large volume through this container is the part of a backup
+    # that runs for hours; cancelling one works by removing it (see
+    # :mod:`services.stack_job_service`), so the job has to know it exists.
+    stack_job_service.register_helper(container.id)
+    return container
 
 
 def _remove(container) -> None:
+    stack_job_service.drop_helper(getattr(container, "id", ""))
     try:
         container.remove(force=True)
     except Exception:  # noqa: BLE001 - cleanup is best effort
@@ -160,13 +166,15 @@ def _put_archive(container, dest: str, src_file: str, volumes: dict) -> None:
 
 def _create_helper_with(client, volumes: dict, command: list[str]):
     ensure_helper_image(client)
-    return safe_call(
+    container = safe_call(
         client.containers.create,
         settings.BACKUP_HELPER_IMAGE,
         command=command,
         volumes=volumes,
         labels={HELPER_LABEL: "1"},
     )
+    stack_job_service.register_helper(container.id)
+    return container
 
 
 def _unknown() -> dict:

backend/services/stack_job_service.py

@@ -0,0 +1,231 @@
+"""The long-running backup/restore behind a stack lock, and stopping it.
+
+``stack_lock_service`` answers one question — is this stack busy — and
+deliberately knows nothing about who is doing the work. That was enough until
+a backup got stuck: the lock a backup takes lives for twelve hours (a large
+volume legitimately takes that long), and for as long as it is held the stack
+cannot be started, stopped, restarted or deployed. So a backup that hangs — an
+unreachable NFS destination, a helper container reading a volume that never
+ends — or one whose work died without unwinding takes the whole stack with it
+until the TTL runs out.
+
+This module is the other half of the lock: the operation registers itself
+here, and a second request can ask it to stop.
+
+Cancellation is cooperative, and on purpose:
+
+* ``cancel`` sets a flag and force-removes the helper containers the job is
+  streaming through, which is what actually ends the long wait — the stream
+  breaks, the worker thread returns, and the backup unwinds through its own
+  ``finally``. That is what restarts the containers the backup stopped and
+  releases the lock. Killing the task instead would leave both undone.
+* Nothing is cancelled from the outside. A restore is not interruptible at all
+  (it is halfway through writing a stack's data back), so it registers itself
+  to be *seen*, not stopped.
+
+When even that does not help — a job whose process is long gone, leaving only
+the lock row behind — the escape hatch is releasing the lock itself
+(:func:`stack_lock_service.release`), which the unlock endpoint does with
+``force``.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Optional
+
+from docker_client import get_long_client, safe_call
+
+logger = logging.getLogger("stackpilot.stack_job")
+
+#: Actions that can be stopped halfway without leaving a mess behind. A backup
+#: only reads; a restore is mid-way through writing a stack's data back and
+#: must be allowed to finish.
+CANCELLABLE = {"backup"}
+
+#: Actions that always register here while they run. A lock held by one of
+#: these with no job behind it is therefore a leftover — the run is over, only
+#: its row survived — and releasing it stops nothing. Compose operations
+#: (start, update, git-deploy) are not in this set: they run as a subprocess
+#: that outlives the request, so a lock of theirs may well still mean work.
+TRACKED = {"backup", "restore"}
+
+
+class JobCancelled(Exception):
+    """Raised inside a job that has been asked to stop."""
+
+    def __init__(self, stack_id: str, action: str, by: str = ""):
+        self.stack_id = stack_id
+        self.action = action
+        self.by = by
+        who = f" by {by}" if by else ""
+        super().__init__(f"{action} of '{stack_id}' was cancelled{who}")
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+@dataclass
+class Job:
+    """One backup or restore, while it runs."""
+
+    stack_id: str
+    action: str
+    owner: str = ""
+    started_at: datetime = field(default_factory=_now)
+    #: What it is doing right now ("volume media-data", "writing the archive"),
+    #: so a stack that looks stuck can be told from one that is just slow.
+    phase: str = ""
+    cancelled_by: Optional[str] = None
+    _stop: threading.Event = field(default_factory=threading.Event)
+    #: Helper containers this job streams through. Removing them is what ends
+    #: a read that would otherwise run for hours.
+    _helpers: set[str] = field(default_factory=set)
+
+    @property
+    def cancellable(self) -> bool:
+        return self.action in CANCELLABLE
+
+    @property
+    def cancelled(self) -> bool:
+        return self._stop.is_set()
+
+    def to_dict(self) -> dict:
+        return {
+            "stack_id": self.stack_id,
+            "action": self.action,
+            "owner": self.owner,
+            "started_at": self.started_at.isoformat(),
+            "phase": self.phase,
+            "cancellable": self.cancellable,
+            "cancelled": self.cancelled,
+            "cancelled_by": self.cancelled_by,
+        }
+
+
+_jobs: dict[str, Job] = {}
+_guard = threading.Lock()
+#: The job of the current task. ``asyncio.to_thread`` copies the context, so
+#: the worker threads a backup fans out into see their own job too.
+_current: ContextVar[Optional[Job]] = ContextVar("stackpilot_job", default=None)
+
+
+@contextmanager
+def track(stack_id: str, action: str, owner: str = ""):
+    """Register the operation for the duration of the block.
+
+    Always deregisters: a job that is gone from here is one nobody can be
+    waiting on, so a leftover entry would make a free stack look busy.
+    """
+    job = Job(stack_id=stack_id, action=action, owner=owner)
+    with _guard:
+        _jobs[stack_id] = job
+    token = _current.set(job)
+    try:
+        yield job
+    finally:
+        _current.reset(token)
+        with _guard:
+            if _jobs.get(stack_id) is job:
+                del _jobs[stack_id]
+
+
+def current() -> Optional[Job]:
+    return _current.get()
+
+
+def get(stack_id: str) -> Optional[Job]:
+    with _guard:
+        return _jobs.get(stack_id)
+
+
+def active() -> list[dict]:
+    with _guard:
+        return [job.to_dict() for job in _jobs.values()]
+
+
+def phase(text: str) -> None:
+    """Record what the current job is doing. No-op outside a job."""
+    job = _current.get()
+    if job is not None:
+        job.phase = text
+
+
+def cancelled() -> bool:
+    job = _current.get()
+    return job is not None and job.cancelled
+
+
+def check() -> None:
+    """Raise :class:`JobCancelled` if the current job was asked to stop.
+
+    Called at every point a backup can safely give up — between the files and
+    volumes it archives, not in the middle of one.
+    """
+    job = _current.get()
+    if job is not None and job.cancelled:
+        raise JobCancelled(job.stack_id, job.action, job.cancelled_by or "")
+
+
+# --------------------------------------------------------------------------- #
+# Helper containers
+# --------------------------------------------------------------------------- #
+
+
+def register_helper(container_id: str) -> None:
+    """Note a helper container the current job reads or writes through."""
+    job = _current.get()
+    if job is not None:
+        with _guard:
+            job._helpers.add(container_id)
+
+
+def drop_helper(container_id: str) -> None:
+    job = _current.get()
+    if job is not None:
+        with _guard:
+            job._helpers.discard(container_id)
+
+
+def _kill_helpers(job: Job) -> int:
+    with _guard:
+        ids = list(job._helpers)
+    if not ids:
+        return 0
+    killed = 0
+    client = get_long_client()
+    for container_id in ids:
+        try:
+            container = safe_call(client.containers.get, container_id)
+            safe_call(container.remove, force=True)
+            killed += 1
+        except Exception as exc:  # noqa: BLE001 - already gone is a fine outcome
+            logger.debug("Helper %s of '%s' could not be removed: %s", container_id[:12], job.stack_id, exc)
+    return killed
+
+
+def cancel(stack_id: str, by: str = "") -> Optional[Job]:
+    """Ask the job on ``stack_id`` to stop. Returns it, or None if there is none.
+
+    Returns as soon as the job has been told; the job itself still has to
+    unwind — restart what it stopped, drop the lock — which is what the caller
+    waits for afterwards.
+    """
+    job = get(stack_id)
+    if job is None:
+        return None
+    if not job.cancellable:
+        return job
+    job.cancelled_by = by or "unknown"
+    job._stop.set()
+    killed = _kill_helpers(job)
+    logger.info(
+        "Cancelling the %s of '%s' (requested by %s, %d helper container(s) removed)",
+        job.action, stack_id, job.cancelled_by, killed,
+    )
+    return job

backend/services/stack_lock_service.py

@@ -132,6 +132,45 @@ def active(session: Session) -> dict[str, str]:
     }
 
 
+def info(session: Session, stack_id: str) -> Optional[dict]:
+    """The lock in force on ``stack_id``, as plain data — or None.
+
+    What the UI needs to explain a stack that will not budge: which operation
+    holds it, since when, and when it would give up on its own.
+    """
+    lock = session.get(StackLock, stack_id)
+    if lock is None:
+        return None
+    now = _now()
+    expires = _aware(lock.expires_at) or now
+    if expires <= now:
+        return None
+    acquired = _aware(lock.acquired_at) or now
+    return {
+        "action": lock.action,
+        "owner": lock.owner,
+        "acquired_at": acquired.isoformat(),
+        "expires_at": expires.isoformat(),
+        "held_for": int((now - acquired).total_seconds()),
+    }
+
+
+def clear_all(session: Session) -> list[str]:
+    """Drop every lock. Returns the stack ids that were locked.
+
+    Called at startup: a lock only means something while the operation holding
+    it runs, and nothing of ours is in flight yet (StackPilot runs a single
+    uvicorn worker — see the Dockerfile's CMD). Before this, a backup whose
+    process died left its stack unusable for the twelve hours of its TTL.
+    """
+    locks = session.exec(select(StackLock)).all()
+    ids = [lock.stack_id for lock in locks]
+    for lock in locks:
+        session.delete(lock)
+    session.commit()
+    return ids
+
+
 def is_busy(session: Session, stack_id: str) -> bool:
     lock = session.get(StackLock, stack_id)
     return lock is not None and (_aware(lock.expires_at) or _now()) > _now()