Let a stuck backup be stopped instead of stranding the stack (0.67.0)

A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.

Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.

POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".

Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/services/backup_service.py

@@ -52,7 +52,12 @@ from sqlmodel import Session
 
 from database import engine
 from docker_client import DockerError, get_long_client, safe_call
-from services import compose_service, stack_lock_service, stack_assets_service as assets
+from services import (
+    compose_service,
+    stack_job_service,
+    stack_lock_service,
+    stack_assets_service as assets,
+)
 
 logger = logging.getLogger("stackpilot.backup")
 
@@ -76,6 +81,15 @@ class StackBusyError(BackupError):
     """The stack is locked by another operation (deploy, update, backup)."""
 
 
+class BackupCancelled(BackupError):
+    """Somebody stopped this backup from the outside.
+
+    Not a failure: the archive is thrown away, the stack is started again and
+    the lock goes, exactly as after an error — but nothing is reported as
+    broken and no "backup failed" notification goes out.
+    """
+
+
 # --------------------------------------------------------------------------- #
 # Backup filename convention
 # --------------------------------------------------------------------------- #
@@ -112,14 +126,23 @@ def backup_name_pattern(stack_id: str, prefix: Optional[str] = None) -> re.Patte
 
 
 @contextmanager
-def _locked(stack_id: str, action: str):
+def _locked(stack_id: str, action: str, owner: str = ""):
+    """Hold the stack lock, and register the job so it can be stopped.
+
+    The lock alone made a stuck backup fatal: nothing could touch the stack
+    until its twelve-hour TTL ran out. The registration is what gives the
+    unlock endpoint something to cancel.
+    """
     with Session(engine) as session:
         try:
-            stack_lock_service.acquire(session, stack_id, action, action, ttl=LOCK_TTL)
+            stack_lock_service.acquire(session, stack_id, action, owner or action, ttl=LOCK_TTL)
         except stack_lock_service.StackBusy as exc:
             raise StackBusyError(str(exc)) from exc
         try:
-            yield
+            with stack_job_service.track(stack_id, action, owner or action):
+                yield
+        except stack_job_service.JobCancelled as exc:
+            raise BackupCancelled(str(exc)) from exc
         finally:
             try:
                 stack_lock_service.release(session, stack_id)
@@ -275,6 +298,10 @@ def _add_tree(
     """
 
     def add(path: str, rel: str, parent_dev: int) -> None:
+        # A stack directory with a data folder in it can take as long as a
+        # volume, so a cancellation lands here too — between two files, with
+        # the half-written archive thrown away by the caller.
+        stack_job_service.check()
         arc = f"{arcroot}/{rel}" if rel else arcroot
         label = rel or "."
         try:
@@ -387,6 +414,7 @@ async def _create(
     binds: Optional[list[str]],
     volumes: Optional[list[str]],
 ) -> tuple[str, dict]:
+    stack_job_service.phase("looking at what to capture")
     selection = await asyncio.to_thread(
         plan, stack_id, include_volumes, include_binds, binds, volumes
     )
@@ -429,6 +457,11 @@ async def _create(
     try:
         parts: list[tuple[str, str]] = []
         for index, bind in enumerate(cap_binds):
+            # Between two items is where a cancelled backup gives up: never in
+            # the middle of one, and never after the stack has been stopped
+            # without the finally below starting it again.
+            stack_job_service.check()
+            stack_job_service.phase(f"bind {bind['source']}")
             part = os.path.join(workdir, f"bind-{index:03d}.tar")
             try:
                 bind["bytes"] = await asyncio.to_thread(
@@ -437,16 +470,22 @@ async def _create(
                 bind["archive"] = f"binds/{index:03d}.tar"
                 parts.append((part, bind["archive"]))
             except Exception as exc:  # noqa: BLE001
+                # Cancelling pulls the helper container out from under the
+                # read, so the failure here *is* the cancellation.
+                stack_job_service.check()
                 logger.warning("Could not archive bind %s: %s", bind["source"], exc)
                 bind["archive"], bind["error"] = None, str(exc)
 
         for index, vol in enumerate(cap_volumes):
+            stack_job_service.check()
+            stack_job_service.phase(f"volume {vol['name']}")
             part = os.path.join(workdir, f"vol-{index:03d}.tar")
             try:
                 vol["bytes"] = await asyncio.to_thread(assets.export_volume, vol["name"], part)
                 vol["archive"] = f"volumes/{vol['name']}.tar"
                 parts.append((part, vol["archive"]))
             except Exception as exc:  # noqa: BLE001
+                stack_job_service.check()
                 logger.warning("Could not archive volume %s: %s", vol["name"], exc)
                 vol["archive"], vol["error"] = None, str(exc)
 
@@ -514,6 +553,8 @@ async def _create(
             ],
         }
 
+        stack_job_service.check()
+        stack_job_service.phase("writing the archive")
         fd, archive_path = tempfile.mkstemp(prefix=f"sp-{stack_id}-", suffix=".tar.gz")
         os.close(fd)
         await asyncio.to_thread(