Back up several stacks, or all of them, from one schedule (0.65.0)

A schedule covered exactly one stack, so backing up a host meant one
schedule per stack, each edited separately, and new stacks were only
backed up once someone remembered to add a schedule.

A schedule now takes a list of stacks, or all_stacks, which includes
stacks added later. They are backed up one after another, each with its
own file, upload, retention and status, so one failing stack does not
cost the others their backup:

- stack_status records each stack's result; last_status summarises
  ("error: 2 of 5 stacks - a: ...; c: ...") and one notification per run
  names the stacks with a problem. The dashboard names them too and shows
  an incomplete backup as a warning rather than an error.
- A busy stack (deploy, update) is retried after the others, up to three
  times two minutes apart, replacing the 10-minute postponement of the
  whole schedule.
- Run now starts the run in the background (202) instead of holding the
  request, which a multi-stack run would outlast; the list reports
  running and polls, and the row toasts the outcome. A schedule never
  runs twice at once.
- Settings: "All stacks" or a checklist with select all/none; stacks can
  be changed when editing; deleted stacks stay listed to be unticked.

Schedules from before keep working without a data migration: stack_ids,
all_stacks and stack_status are added with defaults on startup and a row
with only stack_id reads as that one stack. The API still accepts
stack_id and reports it for single-stack schedules.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/services/schedule_service.py

@@ -1,14 +1,17 @@
 """Scheduled (recurring) stack backups to a destination.
 
 A background loop wakes every minute, runs any schedules whose ``next_run`` has
-passed, uploads the backup to the destination, prunes old backups per the
-retention setting, and records status + the next run time.
+passed and records status + the next run time. A schedule covers several
+stacks (or all of them); each is backed up on its own — its own file, upload,
+retention and status — one after the other, so one failing stack does not
+cost the others their backup.
 
 Times are stored as naive UTC to match SQLite's datetime handling.
 """
 from __future__ import annotations
 
 import asyncio
+import json
 import logging
 import os
 from datetime import datetime, timedelta, timezone
@@ -17,7 +20,7 @@ from sqlmodel import Session, select
 
 from database import engine
 from models.backup_destination import BackupDestination
-from models.backup_schedule import BackupSchedule
+from models.backup_schedule import BackupSchedule, stacks_of
 from models.setting import EVENT_BACKUP_FAILED
 from models.stack import Stack
 from services import (
@@ -28,9 +31,24 @@ from services import (
 
 logger = logging.getLogger("stackpilot.schedule")
 
-#: A stack that is busy (deploying, updating) when its backup is due is tried
-#: again this much later instead of waiting for the next regular run.
-BUSY_RETRY = timedelta(minutes=10)
+#: A stack that is busy (deploying, updating) when its turn comes is tried
+#: again after the others, up to this many more times, this far apart.
+BUSY_ATTEMPTS = 3
+BUSY_WAIT_SECONDS = 120.0
+
+#: Schedules with a run in progress (scheduler or "Run now"), so the same
+#: schedule never runs twice at once.
+_running: set[int] = set()
+
+
+def is_running(schedule_id: int) -> bool:
+    return schedule_id in _running
+
+
+def resolve_stacks(session: Session, schedule: BackupSchedule) -> list[str]:
+    if schedule.all_stacks:
+        return sorted(s.id for s in session.exec(select(Stack)).all())
+    return stacks_of(schedule)
 
 
 def _now() -> datetime:
@@ -93,70 +111,125 @@ async def _alert(session: Session, title: str, body: str) -> None:
         pass
 
 
-async def run_schedule(session: Session, schedule: BackupSchedule) -> dict:
-    """Execute one schedule now. Updates status + next_run. Returns a summary."""
-    now = _now()
-    dest = session.get(BackupDestination, schedule.destination_id)
-    result: dict = {"ok": False}
-    path = None
-    next_run = None
-    try:
-        if not dest:
-            raise RuntimeError(f"destination {schedule.destination_id} not found")
+class _Busy(Exception):
+    pass
 
-        stack = session.get(Stack, schedule.stack_id)
-        if not stack:
-            raise RuntimeError(f"stack '{schedule.stack_id}' not found")
+
+async def _backup_stack(
+    session: Session, schedule: BackupSchedule, dest: BackupDestination, stack_id: str
+) -> str:
+    """Back up, upload and rotate one stack. Returns its status line; raises
+    _Busy when the stack is locked by another operation."""
+    stack = session.get(Stack, stack_id)
+    if not stack:
+        return "error: stack not found"
+    path = None
+    try:
         path, report = await backup_service.create_backup_ex(
-            schedule.stack_id, stack.name,
+            stack_id, stack.name,
             include_volumes=schedule.include_volumes, stop_first=schedule.stop_first,
         )
-
-        filename = backup_service.backup_filename(schedule.stack_id, schedule.include_volumes)
+        filename = backup_service.backup_filename(stack_id, schedule.include_volumes)
         await asyncio.to_thread(dest_service.upload, dest, path, filename)
         problems = _problems(report)
-        # Retention only after a complete backup: an incomplete one must not
-        # push the last good backup out.
-        pruned = 0 if problems else await asyncio.to_thread(
-            _prune, dest, schedule.stack_id, schedule.keep
-        )
-        result = {"ok": not problems, "name": filename, "destination": dest.name, "pruned": pruned}
         if problems:
-            schedule.last_status = f"warning: {'; '.join(problems)}"[:300]
-            result["error"] = "; ".join(problems)
-            logger.warning("Scheduled backup %s incomplete: %s", schedule.stack_id, problems)
-            await _alert(
-                session,
-                f"Scheduled backup of '{schedule.stack_id}' is incomplete",
-                "\n".join(problems),
-            )
-        else:
-            schedule.last_status = "ok"
-            logger.info(
-                "Scheduled backup %s → %s ok (pruned %d)", schedule.stack_id, dest.name, pruned
-            )
+            # Retention only after a complete backup: an incomplete one must
+            # not push the last good backup out.
+            return f"warning: {'; '.join(problems)}"
+        await asyncio.to_thread(_prune, dest, stack_id, schedule.keep)
+        return "ok"
     except backup_service.StackBusyError as exc:
-        # Deploys and updates are short: try again soon rather than skipping a day.
-        schedule.last_status = f"retrying: {exc}"[:300]
-        result = {"ok": False, "error": str(exc)}
-        next_run = now + BUSY_RETRY
-        logger.info("Scheduled backup %s postponed: %s", schedule.stack_id, exc)
+        raise _Busy(str(exc)) from exc
     except Exception as exc:  # noqa: BLE001
-        schedule.last_status = f"error: {exc}"[:300]
-        result = {"ok": False, "error": str(exc)}
-        logger.warning("Scheduled backup %s failed: %s", schedule.stack_id, exc)
-        await _alert(session, f"Scheduled backup of '{schedule.stack_id}' failed", str(exc))
+        logger.warning("Scheduled backup of %s failed: %s", stack_id, exc)
+        return f"error: {exc}"
     finally:
         if path and os.path.exists(path):
             os.unlink(path)
+
+
+def _summarise(statuses: dict[str, str]) -> str:
+    """One status line for the schedule from the per-stack ones."""
+    if not statuses:
+        return "error: no stacks to back up"
+    bad = {sid: st for sid, st in statuses.items() if st != "ok"}
+    if not bad:
+        return "ok"
+    worst = "error" if any(not st.startswith("warning") for st in bad.values()) else "warning"
+    if len(statuses) == 1:
+        return next(iter(statuses.values()))
+    detail = "; ".join(f"{sid}: {st}" for sid, st in sorted(bad.items()))
+    return f"{worst}: {len(bad)} of {len(statuses)} stacks — {detail}"
+
+
+async def run_schedule(session: Session, schedule: BackupSchedule) -> dict:
+    """Execute one schedule now. Updates status + next_run. Returns a summary."""
+    if schedule.id in _running:
+        return {"ok": False, "error": "this schedule is already running"}
+    _running.add(schedule.id)
+    now = _now()
+    statuses: dict[str, str] = {}
+    dest = session.get(BackupDestination, schedule.destination_id)
+    try:
+        stack_ids = resolve_stacks(session, schedule)
+        if not dest:
+            statuses = {sid: f"error: destination {schedule.destination_id} not found"
+                        for sid in stack_ids}
+        else:
+            pending = list(stack_ids)
+            for attempt in range(BUSY_ATTEMPTS + 1):
+                busy: list[str] = []
+                for stack_id in pending:
+                    try:
+                        statuses[stack_id] = await _backup_stack(session, schedule, dest, stack_id)
+                    except _Busy as exc:
+                        busy.append(stack_id)
+                        statuses[stack_id] = f"error: {exc}"
+                if not busy or attempt == BUSY_ATTEMPTS:
+                    break
+                # Deploys and updates are short: wait, then take the busy
+                # stacks again rather than skipping them until the next run.
+                pending = busy
+                await asyncio.sleep(BUSY_WAIT_SECONDS)
+
+        schedule.last_status = _summarise(statuses)[:300]
+        problems = {sid: st for sid, st in statuses.items() if st != "ok"}
+        if problems or not statuses:
+            await _alert(
+                session,
+                f"Scheduled backup to '{dest.name if dest else schedule.destination_id}' "
+                + ("failed" if schedule.last_status.startswith("error") else "is incomplete"),
+                "\n".join(f"{sid}: {st}" for sid, st in sorted(problems.items()))
+                or schedule.last_status,
+            )
+        logger.info("Scheduled backup %s: %s", schedule.id, schedule.last_status)
+        return {
+            "ok": schedule.last_status == "ok",
+            "destination": dest.name if dest else None,
+            "stacks": statuses,
+            **({} if schedule.last_status == "ok" else {"error": schedule.last_status}),
+        }
+    finally:
+        _running.discard(schedule.id)
+        schedule.stack_status = json.dumps(
+            {sid: {"status": st[:300], "at": now.isoformat()} for sid, st in statuses.items()}
+        )
         schedule.last_run = now
-        schedule.next_run = next_run or compute_next_run(
+        schedule.next_run = compute_next_run(
             schedule.frequency, schedule.hour, schedule.minute, schedule.weekday, now
         )
         session.add(schedule)
         session.commit()
         session.refresh(schedule)
-    return result
+
+
+async def run_in_background(schedule_id: int) -> None:
+    """Run a schedule with its own session, for "Run now": a run over many
+    stacks takes far longer than a request may."""
+    with Session(engine) as session:
+        schedule = session.get(BackupSchedule, schedule_id)
+        if schedule:
+            await run_schedule(session, schedule)
 
 
 async def run_due() -> None:
@@ -174,7 +247,8 @@ async def run_due() -> None:
                 due.append(s)
         session.commit()
         for s in due:
-            await run_schedule(session, s)
+            if not is_running(s.id):
+                await run_schedule(session, s)
 
 
 async def scheduler_loop() -> None: