Back up several stacks, or all of them, from one schedule (0.65.0)

A schedule covered exactly one stack, so backing up a host meant one
schedule per stack, each edited separately, and new stacks were only
backed up once someone remembered to add a schedule.

A schedule now takes a list of stacks, or all_stacks, which includes
stacks added later. They are backed up one after another, each with its
own file, upload, retention and status, so one failing stack does not
cost the others their backup:

- stack_status records each stack's result; last_status summarises
  ("error: 2 of 5 stacks - a: ...; c: ...") and one notification per run
  names the stacks with a problem. The dashboard names them too and shows
  an incomplete backup as a warning rather than an error.
- A busy stack (deploy, update) is retried after the others, up to three
  times two minutes apart, replacing the 10-minute postponement of the
  whole schedule.
- Run now starts the run in the background (202) instead of holding the
  request, which a multi-stack run would outlast; the list reports
  running and polls, and the row toasts the outcome. A schedule never
  runs twice at once.
- Settings: "All stacks" or a checklist with select all/none; stacks can
  be changed when editing; deleted stacks stay listed to be unticked.

Schedules from before keep working without a data migration: stack_ids,
all_stacks and stack_status are added with defaults on startup and a row
with only stack_id reads as that one stack. The API still accepts
stack_id and reports it for single-stack schedules.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

frontend/package.json

@@ -1,7 +1,7 @@
 {
   "name": "stackpilot-frontend",
   "private": true,
-  "version": "0.64.1",
+  "version": "0.65.0",
   "type": "module",
   "scripts": {
     "dev": "vite",

frontend/src/api/schedules.ts

@@ -2,7 +2,10 @@ import api from "./client";
 
 export interface BackupSchedule {
   id: number;
-  stack_id: string;
+  /** The chosen stacks; empty when all_stacks is set. */
+  stack_ids: string[];
+  /** Every stack, including ones added later. */
+  all_stacks: boolean;
   destination_id: number;
   destination_name: string | null;
   frequency: "hourly" | "daily" | "weekly";
@@ -15,12 +18,17 @@ export interface BackupSchedule {
   enabled: boolean;
   last_run: string | null;
   last_status: string | null;
+  /** Per stack, from the last run. */
+  stack_status: Record<string, { status: string; at: string }>;
+  /** A run is in progress. */
+  running: boolean;
   next_run: string | null;
   created_at: string;
 }
 
 export interface ScheduleInput {
-  stack_id: string;
+  stack_ids: string[];
+  all_stacks: boolean;
   destination_id: number;
   frequency: string;
   hour: number;
@@ -40,10 +48,7 @@ export const schedulesApi = {
     api.put<BackupSchedule>(`/api/backups/schedules/${id}`, body).then((r) => r.data),
   remove: (id: number) =>
     api.delete(`/api/backups/schedules/${id}`).then((r) => r.data),
+  /** Starts the run in the background; the list then shows `running`. */
   run: (id: number) =>
-    api
-      .post<{ ok: boolean; name?: string; destination?: string; error?: string }>(
-        `/api/backups/schedules/${id}/run`
-      )
-      .then((r) => r.data),
+    api.post<{ started: boolean }>(`/api/backups/schedules/${id}/run`).then((r) => r.data),
 };

frontend/src/pages/Settings.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useState } from "react";
+import { useEffect, useRef, useState } from "react";
 import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 import {
   Bell,
@@ -86,7 +86,15 @@ function describeSchedule(s: BackupSchedule): string {
 
 function SchedulesSection() {
   const qc = useQueryClient();
-  const { data, isLoading } = useQuery({ queryKey: ["schedules"], queryFn: schedulesApi.list });
+  // Poll while a run is in progress, and briefly after "Run now" (the run
+  // registers itself a moment after the request returns).
+  const [pollUntil, setPollUntil] = useState(0);
+  const { data, isLoading } = useQuery({
+    queryKey: ["schedules"],
+    queryFn: schedulesApi.list,
+    refetchInterval: (q) =>
+      q.state.data?.some((s) => s.running) || Date.now() < pollUntil ? 3000 : false,
+  });
   const destinations = useQuery({ queryKey: ["destinations"], queryFn: destinationsApi.list });
   const stacks = useQuery({ queryKey: ["stacks"], queryFn: stacksApi.list });
   const [adding, setAdding] = useState(false);
@@ -100,12 +108,24 @@ function SchedulesSection() {
         {isLoading ? (
           <Spinner />
         ) : (
-          data?.map((s) => <ScheduleRow key={s.id} schedule={s} onChange={invalidate} />)
+          data?.map((s) => (
+            <ScheduleRow
+              key={s.id}
+              schedule={s}
+              stacks={stacks.data ?? []}
+              destinations={destinations.data ?? []}
+              onChange={invalidate}
+              onStarted={() => {
+                setPollUntil(Date.now() + 15000);
+                invalidate();
+              }}
+            />
+          ))
         )}
         {data?.length === 0 && !adding && (
           <Card>
             <p className="text-sm text-slate-500">
-              No scheduled backups. Add one to automatically push a stack to a destination
+              No scheduled backups. Add one to automatically push stacks to a destination
               on a recurring schedule.
             </p>
           </Card>
@@ -130,10 +150,27 @@ function SchedulesSection() {
   );
 }
 
-function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChange: () => void }) {
+function statusTone(status: string | null | undefined): string {
+  if (!status) return "text-slate-500";
+  if (status === "ok") return "text-green-600 dark:text-green-400";
+  if (status.startsWith("warning")) return "text-amber-600 dark:text-amber-400";
+  return "text-red-500";
+}
+
+function ScheduleRow({
+  schedule,
+  stacks,
+  destinations,
+  onChange,
+  onStarted,
+}: {
+  schedule: BackupSchedule;
+  stacks: { id: string; name: string }[];
+  destinations: BackupDestination[];
+  onChange: () => void;
+  onStarted: () => void;
+}) {
   const [editing, setEditing] = useState(false);
-  const destinations = useQuery({ queryKey: ["destinations"], queryFn: destinationsApi.list });
-  const stacks = useQuery({ queryKey: ["stacks"], queryFn: stacksApi.list });
   const toggle = useMutation({
     mutationFn: () => schedulesApi.update(schedule.id, { enabled: !schedule.enabled }),
     onSuccess: onChange,
@@ -141,8 +178,10 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang
   });
   const run = useMutation({
     mutationFn: () => schedulesApi.run(schedule.id),
-    onSuccess: (r) =>
-      r.ok ? toast.success(`Backed up to ${r.destination}`) : toast.error(r.error || "Backup failed"),
+    onSuccess: () => {
+      toast.info("Backup started");
+      onStarted();
+    },
     onError: (e) => toast.error(apiErrorMessage(e)),
   });
   const remove = useMutation({
@@ -151,12 +190,24 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang
     onError: (e) => toast.error(apiErrorMessage(e)),
   });
 
+  // Announce the outcome when a run this page watched finishes.
+  const wasRunning = useRef(schedule.running);
+  useEffect(() => {
+    if (wasRunning.current && !schedule.running) {
+      const status = schedule.last_status ?? "";
+      if (status === "ok") toast.success("Scheduled backup finished");
+      else if (status.startsWith("warning")) toast.warning("Scheduled backup is incomplete", { description: status });
+      else toast.error("Scheduled backup failed", { description: status });
+    }
+    wasRunning.current = schedule.running;
+  }, [schedule.running, schedule.last_status]);
+
   if (editing) {
     return (
       <ScheduleForm
         schedule={schedule}
-        stacks={stacks.data ?? []}
-        destinations={destinations.data ?? []}
+        stacks={stacks}
+        destinations={destinations}
         onDone={() => {
           setEditing(false);
           onChange();
@@ -166,31 +217,43 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang
     );
   }
 
-  const status = schedule.last_status ?? "";
-  const statusClass =
-    status === "ok"
-      ? "text-green-600 dark:text-green-400"
-      : status.startsWith("warning") || status.startsWith("retrying")
-        ? "text-amber-600 dark:text-amber-400"
-        : "text-red-500";
+  const perStack = Object.entries(schedule.stack_status ?? {}).sort(([a], [b]) => a.localeCompare(b));
+  const problems = perStack.filter(([, v]) => v.status !== "ok");
+  const shown = schedule.stack_ids.slice(0, 4);
 
   return (
     <Card className="space-y-2">
       <div className="flex flex-wrap items-center justify-between gap-2">
-        <div className="flex items-center gap-2">
-          <span className="font-mono text-sm font-medium">{schedule.stack_id}</span>
+        <div className="flex min-w-0 flex-wrap items-center gap-2">
+          {schedule.all_stacks ? (
+            <span className="text-sm font-medium">All stacks</span>
+          ) : (
+            <span className="font-mono text-sm font-medium" title={schedule.stack_ids.join(", ")}>
+              {shown.join(", ")}
+              {schedule.stack_ids.length > shown.length && (
+                <span className="text-slate-400"> +{schedule.stack_ids.length - shown.length}</span>
+              )}
+            </span>
+          )}
           <span className="text-slate-400">→</span>
           <Badge>{schedule.destination_name ?? `dest ${schedule.destination_id}`}</Badge>
           {!schedule.enabled && <span className="text-xs text-slate-400">disabled</span>}
+          {schedule.running && (
+            <span className="text-xs text-accent dark:text-accent-dark">running…</span>
+          )}
         </div>
         <div className="flex gap-2">
-          <Button variant="ghost" onClick={() => run.mutate()} loading={run.isPending}>
+          <Button
+            variant="ghost"
+            onClick={() => run.mutate()}
+            loading={run.isPending || schedule.running}
+          >
             <Play className="h-4 w-4" /> Run now
           </Button>
           <Button variant="ghost" onClick={() => toggle.mutate()} loading={toggle.isPending}>
             <Power className="h-4 w-4" /> {schedule.enabled ? "Disable" : "Enable"}
           </Button>
-          <Button variant="ghost" onClick={() => setEditing(true)}>
+          <Button variant="ghost" onClick={() => setEditing(true)} disabled={schedule.running}>
             Edit
           </Button>
           <Button variant="ghost" onClick={() => remove.mutate()} loading={remove.isPending}>
@@ -200,15 +263,28 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang
       </div>
       <div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-slate-500">
         <span>{describeSchedule(schedule)}</span>
-        <span>keep {schedule.keep || "∞"}</span>
+        <span>keep {schedule.keep || "∞"} per stack</span>
         {schedule.include_volumes && <span>+volumes</span>}
         {schedule.next_run && <span>next {relativeTime(schedule.next_run)}</span>}
         {schedule.last_run && (
-          <span className={statusClass}>
-            last {relativeTime(schedule.last_run)} · {schedule.last_status}
+          <span className={statusTone(schedule.last_status)}>
+            last {relativeTime(schedule.last_run)} ·{" "}
+            {problems.length || perStack.length <= 1
+              ? schedule.last_status
+              : `${perStack.length} stacks ok`}
           </span>
         )}
       </div>
+      {problems.length > 0 && perStack.length > 1 && (
+        <ul className="space-y-0.5 text-xs">
+          {problems.map(([sid, v]) => (
+            <li key={sid} className="flex min-w-0 gap-2">
+              <span className="shrink-0 font-mono">{sid}</span>
+              <span className={`min-w-0 break-words ${statusTone(v.status)}`}>{v.status}</span>
+            </li>
+          ))}
+        </ul>
+      )}
     </Card>
   );
 }
@@ -220,7 +296,7 @@ function ScheduleForm({
   onDone,
   onCancel,
 }: {
-  /** Set to edit an existing schedule; its stack cannot change. */
+  /** Set to edit an existing schedule. */
   schedule?: BackupSchedule;
   stacks: { id: string; name: string }[];
   destinations: BackupDestination[];
@@ -229,7 +305,8 @@ function ScheduleForm({
 }) {
   const editing = Boolean(schedule);
   const [form, setForm] = useState({
-    stack_id: schedule?.stack_id ?? stacks[0]?.id ?? "",
+    stack_ids: schedule?.stack_ids ?? ([] as string[]),
+    all_stacks: schedule?.all_stacks ?? false,
     destination_id: schedule?.destination_id ?? destinations[0]?.id ?? 0,
     frequency: schedule?.frequency ?? "daily",
     hour: schedule?.hour ?? 3,
@@ -241,28 +318,28 @@ function ScheduleForm({
     enabled: schedule?.enabled ?? true,
   });
   const set = (k: string, v: unknown) => setForm((f) => ({ ...f, [k]: v }));
+  const toggleStack = (id: string, on: boolean) =>
+    set("stack_ids", on ? [...form.stack_ids, id] : form.stack_ids.filter((s) => s !== id));
 
-  const stackOptions = stacks;
-
-  // Keep stack_id and destination_id valid as the options load or change.
-  useEffect(() => {
-    if (!editing && stackOptions.length && !stackOptions.some((s) => s.id === form.stack_id)) {
-      set("stack_id", stackOptions[0].id);
-    }
-  }, [stackOptions]); // eslint-disable-line react-hooks/exhaustive-deps
+  // Keep destination_id valid as the options load or change.
   useEffect(() => {
     if (destinations.length && !destinations.some((d) => d.id === form.destination_id)) {
       set("destination_id", destinations[0].id);
     }
   }, [destinations]); // eslint-disable-line react-hooks/exhaustive-deps
 
+  // Stacks deleted since the schedule was made stay listed, so they can be
+  // unticked rather than silently dropped.
+  const known = new Set(stacks.map((s) => s.id));
+  const stackOptions = [
+    ...stacks,
+    ...form.stack_ids.filter((id) => !known.has(id)).map((id) => ({ id, name: `${id} (deleted)` })),
+  ];
+  const ready = !!form.destination_id && (form.all_stacks || form.stack_ids.length > 0);
+
   const save = useMutation({
-    mutationFn: () => {
-      if (!schedule) return schedulesApi.create(form);
-      // The stack is fixed once scheduled: its retention matches by stack id.
-      const { stack_id: _stack, ...changes } = form;
-      return schedulesApi.update(schedule.id, changes);
-    },
+    mutationFn: () =>
+      schedule ? schedulesApi.update(schedule.id, form) : schedulesApi.create(form),
     onSuccess: () => {
       toast.success(editing ? "Schedule updated" : "Schedule added");
       onDone();
@@ -272,24 +349,56 @@ function ScheduleForm({
 
   return (
     <Card className="space-y-3">
-      <div className="grid gap-3 sm:grid-cols-3">
-        <label className="space-y-1">
-          <span className="text-xs font-medium text-slate-500">Stack</span>
-          <select
-            className={selectClass}
-            value={form.stack_id}
-            onChange={(e) => set("stack_id", e.target.value)}
-            disabled={editing}
-          >
-            {stackOptions.length === 0 && <option value="">—</option>}
-            {editing && !stackOptions.some((s) => s.id === form.stack_id) && (
-              <option value={form.stack_id}>{form.stack_id}</option>
-            )}
-            {stackOptions.map((s) => (
-              <option key={s.id} value={s.id}>{s.name}</option>
-            ))}
-          </select>
+      <div className="space-y-1">
+        <span className="text-xs font-medium text-slate-500">Stacks</span>
+        <label className="flex items-center gap-2 text-sm">
+          <input
+            type="checkbox"
+            checked={form.all_stacks}
+            onChange={(e) => set("all_stacks", e.target.checked)}
+            className="h-4 w-4"
+          />
+          All stacks — including ones added later
         </label>
+        {!form.all_stacks && (
+          <div className="rounded-lg border border-slate-200 p-2 dark:border-slate-700">
+            <div className="mb-1 flex gap-3 text-xs">
+              <button
+                type="button"
+                className="text-accent hover:underline dark:text-accent-dark"
+                onClick={() => set("stack_ids", stacks.map((s) => s.id))}
+              >
+                Select all
+              </button>
+              <button
+                type="button"
+                className="text-accent hover:underline dark:text-accent-dark"
+                onClick={() => set("stack_ids", [])}
+              >
+                None
+              </button>
+              <span className="text-slate-400">{form.stack_ids.length} selected</span>
+            </div>
+            <div className="grid max-h-48 gap-1 overflow-y-auto sm:grid-cols-2">
+              {stackOptions.length === 0 && (
+                <span className="text-xs text-slate-500">No stacks yet.</span>
+              )}
+              {stackOptions.map((s) => (
+                <label key={s.id} className="flex min-w-0 items-center gap-2 text-sm">
+                  <input
+                    type="checkbox"
+                    checked={form.stack_ids.includes(s.id)}
+                    onChange={(e) => toggleStack(s.id, e.target.checked)}
+                    className="h-4 w-4 shrink-0"
+                  />
+                  <span className="truncate">{s.name}</span>
+                </label>
+              ))}
+            </div>
+          </div>
+        )}
+      </div>
+      <div className="grid gap-3 sm:grid-cols-3">
         <label className="space-y-1">
           <span className="text-xs font-medium text-slate-500">Destination</span>
           <select className={selectClass} value={form.destination_id} onChange={(e) => set("destination_id", Number(e.target.value))}>
@@ -331,7 +440,7 @@ function ScheduleForm({
       </div>
       <div className="grid gap-3 sm:grid-cols-3">
         <label className="space-y-1">
-          <span className="text-xs font-medium text-slate-500">Keep (retention)</span>
+          <span className="text-xs font-medium text-slate-500">Keep per stack (retention)</span>
           <Input type="number" min={0} value={form.keep} onChange={(e) => set("keep", Number(e.target.value))} />
         </label>
         <label className="flex items-end gap-2 pb-2 text-sm">
@@ -343,9 +452,12 @@ function ScheduleForm({
           Stop during backup
         </label>
       </div>
+      <p className="text-[11px] text-slate-500">
+        Stacks are backed up one after another, each to its own file with its own retention.
+      </p>
       <div className="flex justify-end gap-2">
         <Button variant="outline" onClick={onCancel}>Cancel</Button>
-        <Button onClick={() => save.mutate()} loading={save.isPending} disabled={!form.stack_id || !form.destination_id}>
+        <Button onClick={() => save.mutate()} loading={save.isPending} disabled={!ready}>
           {editing ? "Save" : "Add schedule"}
         </Button>
       </div>