Back up several stacks, or all of them, from one schedule (0.65.0)

A schedule covered exactly one stack, so backing up a host meant one
schedule per stack, each edited separately, and new stacks were only
backed up once someone remembered to add a schedule.

A schedule now takes a list of stacks, or all_stacks, which includes
stacks added later. They are backed up one after another, each with its
own file, upload, retention and status, so one failing stack does not
cost the others their backup:

- stack_status records each stack's result; last_status summarises
  ("error: 2 of 5 stacks - a: ...; c: ...") and one notification per run
  names the stacks with a problem. The dashboard names them too and shows
  an incomplete backup as a warning rather than an error.
- A busy stack (deploy, update) is retried after the others, up to three
  times two minutes apart, replacing the 10-minute postponement of the
  whole schedule.
- Run now starts the run in the background (202) instead of holding the
  request, which a multi-stack run would outlast; the list reports
  running and polls, and the row toasts the outcome. A schedule never
  runs twice at once.
- Settings: "All stacks" or a checklist with select all/none; stacks can
  be changed when editing; deleted stacks stay listed to be unticked.

Schedules from before keep working without a data migration: stack_ids,
all_stacks and stack_status are added with defaults on startup and a row
with only stack_id reads as that one stack. The API still accepts
stack_id and reports it for single-stack schedules.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/models/backup_schedule.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 from datetime import datetime, timezone
 from typing import Optional
 
@@ -14,10 +15,19 @@ FREQUENCIES = ["hourly", "daily", "weekly"]
 
 
 class BackupSchedule(SQLModel, table=True):
-    """An automatic, recurring backup of a local stack to a destination."""
+    """An automatic, recurring backup of local stacks to a destination.
+
+    Covers the stacks in ``stack_ids``, or every stack when ``all_stacks`` is
+    set (so new stacks are included without editing the schedule). Each stack
+    is backed up on its own, with its own file, retention and status.
+    """
 
     id: Optional[int] = Field(default=None, primary_key=True)
-    stack_id: str
+    #: The single stack of schedules made before 0.65.0. Read through
+    #: stacks_of(); new rows leave it empty and use stack_ids.
+    stack_id: str = ""
+    stack_ids: str = "[]"  # JSON list of stack ids
+    all_stacks: bool = False
     destination_id: int
     frequency: str = "daily"  # one of FREQUENCIES
     hour: int = 3  # UTC, used for daily/weekly
@@ -28,16 +38,51 @@ class BackupSchedule(SQLModel, table=True):
     keep: int = 7  # retention: keep newest N for this stack on the dest (0 = all)
     enabled: bool = True
     last_run: Optional[datetime] = None
-    last_status: Optional[str] = None  # "ok" | "error: ..."
+    last_status: Optional[str] = None  # "ok" | "warning: ..." | "error: ..."
+    #: JSON {stack_id: status} from the last run, for the per-stack view.
+    stack_status: str = "{}"
     next_run: Optional[datetime] = None
     created_at: datetime = Field(default_factory=_now)
 
 
+def stacks_of(schedule: BackupSchedule) -> list[str]:
+    """The explicitly chosen stacks (empty for an all-stacks schedule)."""
+    try:
+        ids = json.loads(schedule.stack_ids or "[]")
+    except ValueError:
+        ids = []
+    if not ids and schedule.stack_id:
+        ids = [schedule.stack_id]  # a pre-0.65.0 row
+    return [str(i) for i in ids]
+
+
+def set_stacks(schedule: BackupSchedule, ids: list[str]) -> None:
+    schedule.stack_ids = json.dumps(sorted(dict.fromkeys(ids)))
+    # Keep the legacy column meaningful for anything still reading it.
+    schedule.stack_id = ids[0] if len(ids) == 1 else ""
+
+
+def status_of(schedule: BackupSchedule) -> dict:
+    try:
+        return json.loads(schedule.stack_status or "{}")
+    except ValueError:
+        return {}
+
+
+def describe_stacks(schedule: BackupSchedule) -> str:
+    if schedule.all_stacks:
+        return "all stacks"
+    return ", ".join(stacks_of(schedule)) or "no stacks"
+
+
 # --- API schemas ---
 
 
 class ScheduleCreate(SQLModel):
-    stack_id: str
+    stack_ids: list[str] = []
+    all_stacks: bool = False
+    #: Accepted for API clients from before multi-stack schedules.
+    stack_id: Optional[str] = None
     destination_id: int
     frequency: str = "daily"
     hour: int = 3
@@ -50,6 +95,8 @@ class ScheduleCreate(SQLModel):
 
 
 class ScheduleUpdate(SQLModel):
+    stack_ids: Optional[list[str]] = None
+    all_stacks: Optional[bool] = None
     destination_id: Optional[int] = None
     frequency: Optional[str] = None
     hour: Optional[int] = None
@@ -63,6 +110,9 @@ class ScheduleUpdate(SQLModel):
 
 class ScheduleRead(SQLModel):
     id: int
+    stack_ids: list[str]
+    all_stacks: bool
+    #: The one stack when there is exactly one, else "" (older API clients).
     stack_id: str
     destination_id: int
     destination_name: Optional[str]
@@ -76,5 +126,9 @@ class ScheduleRead(SQLModel):
     enabled: bool
     last_run: Optional[datetime]
     last_status: Optional[str]
+    #: {stack_id: status} from the last run.
+    stack_status: dict
+    #: A run is in progress right now.
+    running: bool
     next_run: Optional[datetime]
     created_at: datetime