Let a stuck backup be stopped instead of stranding the stack (0.67.0)

A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.

Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.

POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".

Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

frontend/package.json

@@ -1,7 +1,7 @@
 {
   "name": "stackpilot-frontend",
   "private": true,
-  "version": "0.66.3",
+  "version": "0.67.0",
   "type": "module",
   "scripts": {
     "dev": "vite",

frontend/src/api/stacks.ts

@@ -1,5 +1,12 @@
 import api from "./client";
-import type { StackDetail, StackStats, StackSummary, StackUpdateInfo } from "@/types";
+import type {
+  StackDetail,
+  StackLock,
+  StackStats,
+  StackSummary,
+  StackUnlockResult,
+  StackUpdateInfo,
+} from "@/types";
 
 export const stacksApi = {
   list: () => api.get<StackSummary[]>("/api/stacks").then((r) => r.data),
@@ -31,6 +38,20 @@ export const stacksApi = {
   clone: (id: string, name: string) =>
     api.post(`/api/stacks/${id}/clone`, { name }).then((r) => r.data),
 
+  /** What holds the stack right now, if anything. */
+  lock: (id: string) =>
+    api
+      .get<{ locked: boolean; lock: StackLock | null }>(`/api/stacks/${id}/lock`)
+      .then((r) => r.data),
+  /** Stop whatever holds the stack. A running backup is asked to stop and
+   *  given a moment to put the stack back the way it found it; `force` drops
+   *  the lock without stopping anything, which is the last resort for one
+   *  left behind by an operation that is no longer running. */
+  unlock: (id: string, force = false) =>
+    api
+      .post<StackUnlockResult>(`/api/stacks/${id}/unlock?force=${force}`)
+      .then((r) => r.data),
+
   start: (id: string) => api.post(`/api/stacks/${id}/start`).then((r) => r.data),
   stop: (id: string) => api.post(`/api/stacks/${id}/stop`).then((r) => r.data),
   restart: (id: string) => api.post(`/api/stacks/${id}/restart`).then((r) => r.data),

frontend/src/components/stacks/StackLockBanner.tsx

@@ -0,0 +1,139 @@
+import { useState } from "react";
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+import { Loader2, Lock, Unlock } from "lucide-react";
+import { toast } from "sonner";
+import { Button } from "@/components/ui";
+import { ConfirmDialog } from "@/components/ui/ConfirmDialog";
+import { stacksApi } from "@/api/stacks";
+import { apiErrorMessage } from "@/api/client";
+import type { StackLock } from "@/types";
+
+/** "14 minutes", "3 hours" — how long this has been going on. */
+function since(seconds: number): string {
+  if (seconds < 90) return `${Math.max(seconds, 0)} seconds`;
+  const minutes = Math.round(seconds / 60);
+  if (minutes < 90) return `${minutes} minutes`;
+  return `${Math.round(minutes / 60)} hours`;
+}
+
+const LABEL: Record<string, string> = {
+  backup: "A backup",
+  restore: "A restore",
+  update: "An update",
+  start: "A deploy",
+  "git-deploy": "A Git deploy",
+  "auto-update": "An automatic update",
+};
+
+function describe(lock: StackLock): string {
+  return LABEL[lock.action] ?? `A ${lock.action}`;
+}
+
+/**
+ * Shown while an operation holds the stack — and, more to the point, offers a
+ * way out of it.
+ *
+ * One operation runs per stack at a time, and a backup may hold that lock for
+ * twelve hours (a large volume legitimately takes that long). So a backup that
+ * dies without unwinding leaves every start, stop and restart answering
+ * "Stack 'x' is busy: backup in progress" with nothing to click. This is the
+ * button that was missing: stop the backup (it throws its half-written archive
+ * away and starts the containers it stopped), or — when there is nothing left
+ * running to stop — drop the lock.
+ */
+export function StackLockBanner({
+  stackId,
+  lock,
+}: {
+  stackId: string;
+  lock: StackLock;
+}) {
+  const qc = useQueryClient();
+  const [confirmForce, setConfirmForce] = useState(false);
+
+  const unlock = useMutation({
+    mutationFn: (force: boolean) => stacksApi.unlock(stackId, force),
+    onSuccess: (result) => {
+      toast.success(
+        result.cancelled
+          ? `${describe(lock)} of ${stackId} was stopped`
+          : `The lock on ${stackId} was released`
+      );
+      setConfirmForce(false);
+      qc.invalidateQueries({ queryKey: ["stack", stackId] });
+      qc.invalidateQueries({ queryKey: ["stacks"] });
+    },
+    onError: (err) => {
+      toast.error(apiErrorMessage(err));
+      // It would not stop: offer the blunt instrument rather than leaving the
+      // user with an error and no next step.
+      setConfirmForce(true);
+    },
+  });
+
+  const running = unlock.isPending;
+  const canStop = lock.cancellable && !lock.cancelling;
+
+  return (
+    <div className="rounded-card border border-amber-300 bg-amber-50 px-3 py-2 text-sm text-amber-900 dark:border-amber-900/60 dark:bg-amber-950/40 dark:text-amber-200">
+      <div className="flex flex-wrap items-center gap-2">
+        {lock.stale ? (
+          <Lock className="h-4 w-4 shrink-0" />
+        ) : (
+          <Loader2 className="h-4 w-4 shrink-0 animate-spin" />
+        )}
+        <span className="flex-1">
+          {lock.stale ? (
+            <>
+              {describe(lock)} left this stack locked, but it is no longer
+              running. Until the lock goes, start, stop and restart are refused.
+            </>
+          ) : (
+            <>
+              {describe(lock)} is running — started {since(lock.held_for)} ago
+              {lock.phase ? ` (${lock.phase})` : ""}. The stack cannot be
+              started, stopped or deployed while it does.
+              {lock.cancelling && " Stopping it…"}
+            </>
+          )}
+        </span>
+        {canStop && (
+          <Button
+            variant="outline"
+            onClick={() => unlock.mutate(false)}
+            loading={running}
+          >
+            Stop {lock.action}
+          </Button>
+        )}
+        <Button
+          variant={canStop ? "ghost" : "outline"}
+          onClick={() =>
+            lock.stale ? unlock.mutate(false) : setConfirmForce(true)
+          }
+          loading={running && !canStop}
+        >
+          <Unlock className="h-4 w-4" /> Release lock
+        </Button>
+      </div>
+
+      {confirmForce && (
+        <ConfirmDialog
+          title={`Release the lock on ${stackId}?`}
+          message={
+            `This only removes the guard — it does not stop ${describe(lock)
+              .charAt(0)
+              .toLowerCase() + describe(lock).slice(1)}. ` +
+            "If it is in fact still running, it and your next deploy can then " +
+            "work on the same stack at the same time."
+          }
+          confirmLabel="Release anyway"
+          danger
+          busy={running}
+          onConfirm={() => unlock.mutate(true)}
+          onCancel={() => setConfirmForce(false)}
+        />
+      )}
+    </div>
+  );
+}

frontend/src/hooks/useStackActions.ts

@@ -23,6 +23,17 @@ export type StackActionStatus = {
  *  or until the same stack is acted on again. */
 const SUCCESS_CLEAR_MS = 5000;
 
+/**
+ * "Stack 'x' is busy: backup in progress" is a dead end on its own — the user
+ * gets it on every button and nothing here says that the operation holding the
+ * stack can be stopped. The stack's own page has that button, so point at it.
+ */
+function withLockHint(message: string): string {
+  return /\bis busy\b/i.test(message)
+    ? `${message}. Open the stack to stop it or release the lock.`
+    : message;
+}
+
 /**
  * Runs stack lifecycle actions and tracks their progress.
  *
@@ -84,7 +95,7 @@ export function useStackActions() {
       toast.success(`${label} ${id} ✓`, { id: t });
       succeed(id, label);
     } catch (err) {
-      const message = apiErrorMessage(err);
+      const message = withLockHint(apiErrorMessage(err));
       toast.error(message, { id: t });
       fail(id, label, message);
     } finally {
@@ -102,7 +113,7 @@ export function useStackActions() {
       toast.success(`${label} ${id} ✓`, { id: t });
       succeed(id, label);
     } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
+      const message = withLockHint(err instanceof Error ? err.message : String(err));
       toast.error(`${label} ${id} failed: ${message}`, { id: t });
       fail(id, label, message);
     } finally {

frontend/src/pages/StackDetail.tsx

@@ -18,6 +18,7 @@ import { ConfirmDialog } from "@/components/ui/ConfirmDialog";
 import { LogViewer } from "@/components/stacks/LogViewer";
 import { ContainerCard } from "@/components/stacks/ContainerCard";
 import { ActionStatusList } from "@/components/stacks/ActionStatusBanner";
+import { StackLockBanner } from "@/components/stacks/StackLockBanner";
 import { AutoUpdatePanel } from "@/components/stacks/AutoUpdatePanel";
 import { SecretsPanel } from "@/components/stacks/SecretsPanel";
 import { GitPanel } from "@/components/stacks/GitPanel";
@@ -43,8 +44,10 @@ export function StackDetail() {
   const { data, isLoading } = useQuery({
     queryKey: ["stack", id],
     queryFn: () => stacksApi.get(id),
-    // Container state arrives over /ws/events; this is the fallback.
-    refetchInterval: 30000,
+    // Container state arrives over /ws/events; this is the fallback. A stack
+    // held by an operation is watched closely instead — the banner's "stop
+    // the backup" has to turn back into the normal page once it worked.
+    refetchInterval: (query) => (query.state.data?.lock ? 3000 : 30000),
   });
 
   if (isLoading || !data) return <Spinner />;
@@ -99,6 +102,8 @@ export function StackDetail() {
         )}
       </div>
 
+      {data.lock && isAdmin && <StackLockBanner stackId={id} lock={data.lock} />}
+
       <ActionStatusList statuses={actions.statuses} onDismiss={actions.dismissStatus} />
 
       {/* Tabs */}

frontend/src/types/index.ts

@@ -48,6 +48,35 @@ export interface ContainerInfo {
   created?: string | null;
 }
 
+/** The operation holding a stack, from `GET /api/stacks/{id}/lock` (and
+ *  inlined in the stack detail). One compose/backup operation runs per stack
+ *  at a time; while the lock is held, start/stop/restart/deploy are refused. */
+export interface StackLock {
+  /** "backup", "restore", "update", "start", "git-deploy", … */
+  action: string;
+  owner: string;
+  acquired_at: string;
+  expires_at: string;
+  held_for: number;
+  /** The operation is running here and can be stopped halfway (a backup can,
+   *  a restore must not). */
+  cancellable: boolean;
+  cancelling: boolean;
+  /** Nothing is running behind this lock any more — it is a leftover and can
+   *  be released without stopping anything. */
+  stale: boolean;
+  /** What the operation is doing right now ("volume media-data"). */
+  phase: string;
+}
+
+export interface StackUnlockResult {
+  ok: boolean;
+  action: string;
+  cancelled: boolean;
+  released: boolean;
+  forced: boolean;
+}
+
 export interface StackDetail {
   id: string;
   name: string;
@@ -55,6 +84,8 @@ export interface StackDetail {
   icon?: string | null;
   auto_icon?: string | null;
   status: StackStatus;
+  /** Set while an operation holds the stack; null otherwise. */
+  lock?: StackLock | null;
   yaml: string;
   env: string;
   containers: ContainerInfo[];