diff --git a/README.md b/README.md index 1a1a139..e1b9008 100644 --- a/README.md +++ b/README.md @@ -643,13 +643,20 @@ it is what your saved destination credentials are encrypted with. ### Phase 7 — Scheduled backups -- **Recurring backups**: schedule a stack to back up to a destination **hourly, +- **Recurring backups**: schedule stacks to back up to a destination **hourly, daily, or weekly** (UTC) under **Settings → Scheduled backups**. A background scheduler runs due jobs every minute and records last/next run + status. +- **Several stacks per schedule** (0.65.0): tick any number of stacks, or + *All stacks* to include stacks added later without editing the schedule. They + are backed up one after another, each to its own file with its own retention + and status, so one failing stack does not cost the others their backup; the + schedule lists which stack had a problem. A stack that is busy (deploying, + updating) is retried after the others. Schedules from before keep working. - **Retention**: keep the newest *N* backups per stack on the destination; older ones are pruned automatically. -- **Run now** for an on-demand run, plus a `backup_failed` notification event - wired into the webhook system. +- **Run now** for an on-demand run (in the background; the row shows it running + and then the result), plus a `backup_failed` notification event wired into + the webhook system. ### Phase 9 — Networks @@ -1090,7 +1097,8 @@ POST /api/stacks/{id}/backup/push POST /api/stacks/restore-from ``` GET /api/backups/schedules POST /api/backups/schedules PUT /api/backups/schedules/{id} DELETE /api/backups/schedules/{id} -POST /api/backups/schedules/{id}/run +POST /api/backups/schedules/{id}/run (202, runs in the background) + body: {"stack_ids": [...] | "all_stacks": true, "destination_id": …, …} ``` ### Phase 9 endpoints diff --git a/backend/models/backup_schedule.py b/backend/models/backup_schedule.py index 5bf9778..13cc195 100644 --- a/backend/models/backup_schedule.py +++ b/backend/models/backup_schedule.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from datetime import datetime, timezone from typing import Optional @@ -14,10 +15,19 @@ FREQUENCIES = ["hourly", "daily", "weekly"] class BackupSchedule(SQLModel, table=True): - """An automatic, recurring backup of a local stack to a destination.""" + """An automatic, recurring backup of local stacks to a destination. + + Covers the stacks in ``stack_ids``, or every stack when ``all_stacks`` is + set (so new stacks are included without editing the schedule). Each stack + is backed up on its own, with its own file, retention and status. + """ id: Optional[int] = Field(default=None, primary_key=True) - stack_id: str + #: The single stack of schedules made before 0.65.0. Read through + #: stacks_of(); new rows leave it empty and use stack_ids. + stack_id: str = "" + stack_ids: str = "[]" # JSON list of stack ids + all_stacks: bool = False destination_id: int frequency: str = "daily" # one of FREQUENCIES hour: int = 3 # UTC, used for daily/weekly @@ -28,16 +38,51 @@ class BackupSchedule(SQLModel, table=True): keep: int = 7 # retention: keep newest N for this stack on the dest (0 = all) enabled: bool = True last_run: Optional[datetime] = None - last_status: Optional[str] = None # "ok" | "error: ..." + last_status: Optional[str] = None # "ok" | "warning: ..." | "error: ..." + #: JSON {stack_id: status} from the last run, for the per-stack view. + stack_status: str = "{}" next_run: Optional[datetime] = None created_at: datetime = Field(default_factory=_now) +def stacks_of(schedule: BackupSchedule) -> list[str]: + """The explicitly chosen stacks (empty for an all-stacks schedule).""" + try: + ids = json.loads(schedule.stack_ids or "[]") + except ValueError: + ids = [] + if not ids and schedule.stack_id: + ids = [schedule.stack_id] # a pre-0.65.0 row + return [str(i) for i in ids] + + +def set_stacks(schedule: BackupSchedule, ids: list[str]) -> None: + schedule.stack_ids = json.dumps(sorted(dict.fromkeys(ids))) + # Keep the legacy column meaningful for anything still reading it. + schedule.stack_id = ids[0] if len(ids) == 1 else "" + + +def status_of(schedule: BackupSchedule) -> dict: + try: + return json.loads(schedule.stack_status or "{}") + except ValueError: + return {} + + +def describe_stacks(schedule: BackupSchedule) -> str: + if schedule.all_stacks: + return "all stacks" + return ", ".join(stacks_of(schedule)) or "no stacks" + + # --- API schemas --- class ScheduleCreate(SQLModel): - stack_id: str + stack_ids: list[str] = [] + all_stacks: bool = False + #: Accepted for API clients from before multi-stack schedules. + stack_id: Optional[str] = None destination_id: int frequency: str = "daily" hour: int = 3 @@ -50,6 +95,8 @@ class ScheduleCreate(SQLModel): class ScheduleUpdate(SQLModel): + stack_ids: Optional[list[str]] = None + all_stacks: Optional[bool] = None destination_id: Optional[int] = None frequency: Optional[str] = None hour: Optional[int] = None @@ -63,6 +110,9 @@ class ScheduleUpdate(SQLModel): class ScheduleRead(SQLModel): id: int + stack_ids: list[str] + all_stacks: bool + #: The one stack when there is exactly one, else "" (older API clients). stack_id: str destination_id: int destination_name: Optional[str] @@ -76,5 +126,9 @@ class ScheduleRead(SQLModel): enabled: bool last_run: Optional[datetime] last_status: Optional[str] + #: {stack_id: status} from the last run. + stack_status: dict + #: A run is in progress right now. + running: bool next_run: Optional[datetime] created_at: datetime diff --git a/backend/routers/schedules.py b/backend/routers/schedules.py index 2c19bad..08f6126 100644 --- a/backend/routers/schedules.py +++ b/backend/routers/schedules.py @@ -1,6 +1,8 @@ """Scheduled backup management.""" from __future__ import annotations +import asyncio + from fastapi import APIRouter, Depends, HTTPException, Request from sqlmodel import Session, select @@ -13,6 +15,10 @@ from models.backup_schedule import ( ScheduleCreate, ScheduleRead, ScheduleUpdate, + describe_stacks, + set_stacks, + stacks_of, + status_of, ) from models.stack import Stack from models.user import User @@ -27,9 +33,12 @@ def _ip(request: Request) -> str: def _to_read(session: Session, s: BackupSchedule) -> ScheduleRead: dest = session.get(BackupDestination, s.destination_id) + stack_ids = stacks_of(s) return ScheduleRead( id=s.id, - stack_id=s.stack_id, + stack_ids=stack_ids, + all_stacks=s.all_stacks, + stack_id=stack_ids[0] if len(stack_ids) == 1 and not s.all_stacks else "", destination_id=s.destination_id, destination_name=dest.name if dest else None, frequency=s.frequency, @@ -42,6 +51,8 @@ def _to_read(session: Session, s: BackupSchedule) -> ScheduleRead: enabled=s.enabled, last_run=s.last_run, last_status=s.last_status, + stack_status=status_of(s), + running=schedule_service.is_running(s.id), next_run=s.next_run, created_at=s.created_at, ) @@ -59,8 +70,14 @@ def _validate(session: Session, schedule: BackupSchedule) -> None: raise HTTPException(status_code=400, detail=f"Unknown frequency '{schedule.frequency}'") if not session.get(BackupDestination, schedule.destination_id): raise HTTPException(status_code=404, detail=f"Destination {schedule.destination_id} not found") - if not session.get(Stack, schedule.stack_id): - raise HTTPException(status_code=404, detail=f"Stack '{schedule.stack_id}' not found") + if schedule.all_stacks: + return + stack_ids = stacks_of(schedule) + if not stack_ids: + raise HTTPException(status_code=400, detail="Choose at least one stack, or all stacks") + missing = [sid for sid in stack_ids if not session.get(Stack, sid)] + if missing: + raise HTTPException(status_code=404, detail=f"Stack(s) not found: {', '.join(missing)}") @router.get("", response_model=list[ScheduleRead]) @@ -79,7 +96,9 @@ def create_schedule( session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> ScheduleRead: - s = BackupSchedule(**body.model_dump()) + data = body.model_dump(exclude={"stack_ids", "stack_id"}) + s = BackupSchedule(**data) + set_stacks(s, body.stack_ids or ([body.stack_id] if body.stack_id else [])) _validate(session, s) s.next_run = schedule_service.compute_next_run( s.frequency, s.hour, s.minute, s.weekday, schedule_service._now() @@ -88,7 +107,7 @@ def create_schedule( session.commit() session.refresh(s) audit_service.record( - session, user=user.username, action="schedule.create", target=s.stack_id, + session, user=user.username, action="schedule.create", target=describe_stacks(s), detail=f"{s.frequency} → dest {s.destination_id}", ip=_ip(request), ) return _to_read(session, s) @@ -104,8 +123,11 @@ def update_schedule( ) -> ScheduleRead: s = _get_or_404(session, schedule_id) data = body.model_dump(exclude_unset=True) + stack_ids = data.pop("stack_ids", None) for k, v in data.items(): setattr(s, k, v) + if stack_ids is not None: + set_stacks(s, stack_ids) _validate(session, s) # Recompute next run when timing fields change. if {"frequency", "hour", "minute", "weekday"} & set(data) or s.next_run is None: @@ -116,7 +138,7 @@ def update_schedule( session.commit() session.refresh(s) audit_service.record( - session, user=user.username, action="schedule.update", target=s.stack_id, + session, user=user.username, action="schedule.update", target=describe_stacks(s), ip=_ip(request), ) return _to_read(session, s) @@ -130,27 +152,39 @@ def delete_schedule( user: User = Depends(require_admin), ) -> dict: s = _get_or_404(session, schedule_id) - stack_id = s.stack_id + target = describe_stacks(s) session.delete(s) session.commit() audit_service.record( - session, user=user.username, action="schedule.delete", target=stack_id, + session, user=user.username, action="schedule.delete", target=target, ip=_ip(request), ) return {"ok": True} -@router.post("/{schedule_id}/run") +#: Keeps "Run now" tasks referenced until they finish (asyncio only holds +#: weak references to tasks). +_tasks: set[asyncio.Task] = set() + + +@router.post("/{schedule_id}/run", status_code=202) async def run_now( schedule_id: int, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + """Start the schedule in the background. Backing up several stacks takes + far longer than a request may; the list shows ``running`` and then the + per-stack results.""" s = _get_or_404(session, schedule_id) - result = await schedule_service.run_schedule(session, s) + if schedule_service.is_running(s.id): + raise HTTPException(status_code=409, detail="This schedule is already running") + task = asyncio.create_task(schedule_service.run_in_background(s.id)) + _tasks.add(task) + task.add_done_callback(_tasks.discard) audit_service.record( - session, user=user.username, action="schedule.run", target=s.stack_id, - detail="ok" if result.get("ok") else result.get("error", "error"), ip=_ip(request), + session, user=user.username, action="schedule.run", target=describe_stacks(s), + ip=_ip(request), ) - return result + return {"started": True} diff --git a/backend/services/dashboard_service.py b/backend/services/dashboard_service.py index 816dede..404f505 100644 --- a/backend/services/dashboard_service.py +++ b/backend/services/dashboard_service.py @@ -15,7 +15,7 @@ from datetime import datetime, timedelta, timezone from sqlmodel import Session, select from docker_client import DockerError, get_client, safe_call -from models.backup_schedule import BackupSchedule +from models.backup_schedule import BackupSchedule, describe_stacks, status_of from services import compose_service, update_service COMPOSE_LABEL = compose_service.COMPOSE_LABEL @@ -191,13 +191,20 @@ def _backup_attention(session: Session) -> list[dict]: for sch in session.exec(select(BackupSchedule).where(BackupSchedule.enabled == True)).all(): # noqa: E712 host = "local" status = (sch.last_status or "").lower() + # Name the stacks that actually had a problem, not the whole schedule. + failing = sorted( + sid for sid, entry in status_of(sch).items() + if str((entry or {}).get("status", "")) != "ok" + ) + what = ", ".join(failing) if failing else describe_stacks(sch) if status and not status.startswith("ok"): - items.append(_attn("error", "backup_failed", host, - f"Backup of {sch.stack_id} failed", sch.last_status or "", - "/settings")) + incomplete = status.startswith("warning") + items.append(_attn("warn" if incomplete else "error", "backup_failed", host, + f"Backup of {what} {'is incomplete' if incomplete else 'failed'}", + sch.last_status or "", "/settings")) elif sch.next_run and _aware(sch.next_run) < now - timedelta(hours=1): items.append(_attn("warn", "backup_overdue", host, - f"Backup of {sch.stack_id} is overdue", + f"Backup of {describe_stacks(sch)} is overdue", "Scheduled run did not happen.", "/settings")) return items diff --git a/backend/services/schedule_service.py b/backend/services/schedule_service.py index 8b31774..03ff1f7 100644 --- a/backend/services/schedule_service.py +++ b/backend/services/schedule_service.py @@ -1,14 +1,17 @@ """Scheduled (recurring) stack backups to a destination. A background loop wakes every minute, runs any schedules whose ``next_run`` has -passed, uploads the backup to the destination, prunes old backups per the -retention setting, and records status + the next run time. +passed and records status + the next run time. A schedule covers several +stacks (or all of them); each is backed up on its own — its own file, upload, +retention and status — one after the other, so one failing stack does not +cost the others their backup. Times are stored as naive UTC to match SQLite's datetime handling. """ from __future__ import annotations import asyncio +import json import logging import os from datetime import datetime, timedelta, timezone @@ -17,7 +20,7 @@ from sqlmodel import Session, select from database import engine from models.backup_destination import BackupDestination -from models.backup_schedule import BackupSchedule +from models.backup_schedule import BackupSchedule, stacks_of from models.setting import EVENT_BACKUP_FAILED from models.stack import Stack from services import ( @@ -28,9 +31,24 @@ from services import ( logger = logging.getLogger("stackpilot.schedule") -#: A stack that is busy (deploying, updating) when its backup is due is tried -#: again this much later instead of waiting for the next regular run. -BUSY_RETRY = timedelta(minutes=10) +#: A stack that is busy (deploying, updating) when its turn comes is tried +#: again after the others, up to this many more times, this far apart. +BUSY_ATTEMPTS = 3 +BUSY_WAIT_SECONDS = 120.0 + +#: Schedules with a run in progress (scheduler or "Run now"), so the same +#: schedule never runs twice at once. +_running: set[int] = set() + + +def is_running(schedule_id: int) -> bool: + return schedule_id in _running + + +def resolve_stacks(session: Session, schedule: BackupSchedule) -> list[str]: + if schedule.all_stacks: + return sorted(s.id for s in session.exec(select(Stack)).all()) + return stacks_of(schedule) def _now() -> datetime: @@ -93,70 +111,125 @@ async def _alert(session: Session, title: str, body: str) -> None: pass -async def run_schedule(session: Session, schedule: BackupSchedule) -> dict: - """Execute one schedule now. Updates status + next_run. Returns a summary.""" - now = _now() - dest = session.get(BackupDestination, schedule.destination_id) - result: dict = {"ok": False} - path = None - next_run = None - try: - if not dest: - raise RuntimeError(f"destination {schedule.destination_id} not found") +class _Busy(Exception): + pass - stack = session.get(Stack, schedule.stack_id) - if not stack: - raise RuntimeError(f"stack '{schedule.stack_id}' not found") + +async def _backup_stack( + session: Session, schedule: BackupSchedule, dest: BackupDestination, stack_id: str +) -> str: + """Back up, upload and rotate one stack. Returns its status line; raises + _Busy when the stack is locked by another operation.""" + stack = session.get(Stack, stack_id) + if not stack: + return "error: stack not found" + path = None + try: path, report = await backup_service.create_backup_ex( - schedule.stack_id, stack.name, + stack_id, stack.name, include_volumes=schedule.include_volumes, stop_first=schedule.stop_first, ) - - filename = backup_service.backup_filename(schedule.stack_id, schedule.include_volumes) + filename = backup_service.backup_filename(stack_id, schedule.include_volumes) await asyncio.to_thread(dest_service.upload, dest, path, filename) problems = _problems(report) - # Retention only after a complete backup: an incomplete one must not - # push the last good backup out. - pruned = 0 if problems else await asyncio.to_thread( - _prune, dest, schedule.stack_id, schedule.keep - ) - result = {"ok": not problems, "name": filename, "destination": dest.name, "pruned": pruned} if problems: - schedule.last_status = f"warning: {'; '.join(problems)}"[:300] - result["error"] = "; ".join(problems) - logger.warning("Scheduled backup %s incomplete: %s", schedule.stack_id, problems) - await _alert( - session, - f"Scheduled backup of '{schedule.stack_id}' is incomplete", - "\n".join(problems), - ) - else: - schedule.last_status = "ok" - logger.info( - "Scheduled backup %s → %s ok (pruned %d)", schedule.stack_id, dest.name, pruned - ) + # Retention only after a complete backup: an incomplete one must + # not push the last good backup out. + return f"warning: {'; '.join(problems)}" + await asyncio.to_thread(_prune, dest, stack_id, schedule.keep) + return "ok" except backup_service.StackBusyError as exc: - # Deploys and updates are short: try again soon rather than skipping a day. - schedule.last_status = f"retrying: {exc}"[:300] - result = {"ok": False, "error": str(exc)} - next_run = now + BUSY_RETRY - logger.info("Scheduled backup %s postponed: %s", schedule.stack_id, exc) + raise _Busy(str(exc)) from exc except Exception as exc: # noqa: BLE001 - schedule.last_status = f"error: {exc}"[:300] - result = {"ok": False, "error": str(exc)} - logger.warning("Scheduled backup %s failed: %s", schedule.stack_id, exc) - await _alert(session, f"Scheduled backup of '{schedule.stack_id}' failed", str(exc)) + logger.warning("Scheduled backup of %s failed: %s", stack_id, exc) + return f"error: {exc}" finally: if path and os.path.exists(path): os.unlink(path) + + +def _summarise(statuses: dict[str, str]) -> str: + """One status line for the schedule from the per-stack ones.""" + if not statuses: + return "error: no stacks to back up" + bad = {sid: st for sid, st in statuses.items() if st != "ok"} + if not bad: + return "ok" + worst = "error" if any(not st.startswith("warning") for st in bad.values()) else "warning" + if len(statuses) == 1: + return next(iter(statuses.values())) + detail = "; ".join(f"{sid}: {st}" for sid, st in sorted(bad.items())) + return f"{worst}: {len(bad)} of {len(statuses)} stacks — {detail}" + + +async def run_schedule(session: Session, schedule: BackupSchedule) -> dict: + """Execute one schedule now. Updates status + next_run. Returns a summary.""" + if schedule.id in _running: + return {"ok": False, "error": "this schedule is already running"} + _running.add(schedule.id) + now = _now() + statuses: dict[str, str] = {} + dest = session.get(BackupDestination, schedule.destination_id) + try: + stack_ids = resolve_stacks(session, schedule) + if not dest: + statuses = {sid: f"error: destination {schedule.destination_id} not found" + for sid in stack_ids} + else: + pending = list(stack_ids) + for attempt in range(BUSY_ATTEMPTS + 1): + busy: list[str] = [] + for stack_id in pending: + try: + statuses[stack_id] = await _backup_stack(session, schedule, dest, stack_id) + except _Busy as exc: + busy.append(stack_id) + statuses[stack_id] = f"error: {exc}" + if not busy or attempt == BUSY_ATTEMPTS: + break + # Deploys and updates are short: wait, then take the busy + # stacks again rather than skipping them until the next run. + pending = busy + await asyncio.sleep(BUSY_WAIT_SECONDS) + + schedule.last_status = _summarise(statuses)[:300] + problems = {sid: st for sid, st in statuses.items() if st != "ok"} + if problems or not statuses: + await _alert( + session, + f"Scheduled backup to '{dest.name if dest else schedule.destination_id}' " + + ("failed" if schedule.last_status.startswith("error") else "is incomplete"), + "\n".join(f"{sid}: {st}" for sid, st in sorted(problems.items())) + or schedule.last_status, + ) + logger.info("Scheduled backup %s: %s", schedule.id, schedule.last_status) + return { + "ok": schedule.last_status == "ok", + "destination": dest.name if dest else None, + "stacks": statuses, + **({} if schedule.last_status == "ok" else {"error": schedule.last_status}), + } + finally: + _running.discard(schedule.id) + schedule.stack_status = json.dumps( + {sid: {"status": st[:300], "at": now.isoformat()} for sid, st in statuses.items()} + ) schedule.last_run = now - schedule.next_run = next_run or compute_next_run( + schedule.next_run = compute_next_run( schedule.frequency, schedule.hour, schedule.minute, schedule.weekday, now ) session.add(schedule) session.commit() session.refresh(schedule) - return result + + +async def run_in_background(schedule_id: int) -> None: + """Run a schedule with its own session, for "Run now": a run over many + stacks takes far longer than a request may.""" + with Session(engine) as session: + schedule = session.get(BackupSchedule, schedule_id) + if schedule: + await run_schedule(session, schedule) async def run_due() -> None: @@ -174,7 +247,8 @@ async def run_due() -> None: due.append(s) session.commit() for s in due: - await run_schedule(session, s) + if not is_running(s.id): + await run_schedule(session, s) async def scheduler_loop() -> None: diff --git a/backend/tests/test_backup_restore.py b/backend/tests/test_backup_restore.py index edb539e..2a95378 100644 --- a/backend/tests/test_backup_restore.py +++ b/backend/tests/test_backup_restore.py @@ -574,7 +574,7 @@ def test_verify_accepts_a_good_archive_and_rejects_garbage(stack, tmp_path): # --------------------------------------------------------------------------- # -# Retention and schedules +# Retention (schedules: test_backup_schedules.py) # --------------------------------------------------------------------------- # @@ -606,77 +606,6 @@ def test_retention_leaves_other_stacks_alone(monkeypatch): assert deleted == ["backup-app-full-20260910-030000.tar.gz"] -def _schedule_env(monkeypatch, db, report=None, error=None): - from sqlmodel import Session - - from models.backup_destination import BackupDestination - from models.backup_schedule import BackupSchedule - from models.stack import Stack - from services import schedule_service - - stack_id = f"s{uuid.uuid4().hex[:8]}" - alerts: list[str] = [] - pruned: list[str] = [] - - async def fake_create(*args, **kwargs): - if error: - raise error - path = os.path.join(compose_service.stacks_root(), f".{stack_id}.tar.gz") - open(path, "wb").close() - return path, report - - async def fake_notify(event, title, body, session): - alerts.append(title) - - monkeypatch.setattr(schedule_service.backup_service, "create_backup_ex", fake_create) - monkeypatch.setattr(schedule_service.dest_service, "upload", lambda d, p, f: f) - monkeypatch.setattr(schedule_service, "_prune", lambda d, s, k: pruned.append(s) or 0) - monkeypatch.setattr(schedule_service.notify_service, "notify", fake_notify) - - session = Session(db) - session.add(Stack(id=stack_id, name=stack_id)) - dest = BackupDestination(name=f"d-{stack_id}", type="sftp", config="{}") - session.add(dest) - session.commit() - schedule = BackupSchedule(stack_id=stack_id, destination_id=dest.id, frequency="daily") - session.add(schedule) - session.commit() - return schedule_service, session, schedule, alerts, pruned - - -def test_incomplete_scheduled_backup_warns_and_keeps_old_backups(monkeypatch, db): - report = {"errors": ["db: helper failed"], "restart_error": None} - svc, session, schedule, alerts, pruned = _schedule_env(monkeypatch, db, report=report) - - result = run(svc.run_schedule(session, schedule)) - - assert result["ok"] is False - assert schedule.last_status.startswith("warning:") - assert pruned == [] # the last good backup must not be rotated out - assert alerts and "incomplete" in alerts[0] - - -def test_complete_scheduled_backup_is_ok_and_rotates(monkeypatch, db): - report = {"errors": [], "restart_error": None} - svc, session, schedule, alerts, pruned = _schedule_env(monkeypatch, db, report=report) - - result = run(svc.run_schedule(session, schedule)) - - assert result["ok"] is True and schedule.last_status == "ok" - assert pruned == [schedule.stack_id] and alerts == [] - - -def test_busy_stack_is_retried_soon(monkeypatch, db): - error = backup_service.StackBusyError("Stack 'x' is busy: update in progress") - svc, session, schedule, alerts, _ = _schedule_env(monkeypatch, db, error=error) - - run(svc.run_schedule(session, schedule)) - - assert schedule.last_status.startswith("retrying:") - assert (schedule.next_run - schedule.last_run) == svc.BUSY_RETRY - assert alerts == [] - - # --------------------------------------------------------------------------- # # Routes # --------------------------------------------------------------------------- # diff --git a/backend/tests/test_backup_schedules.py b/backend/tests/test_backup_schedules.py new file mode 100644 index 0000000..6832333 --- /dev/null +++ b/backend/tests/test_backup_schedules.py @@ -0,0 +1,311 @@ +"""Scheduled backups over several stacks. + +A schedule covers a list of stacks, or all of them (new stacks included +without editing it). Each stack gets its own backup, upload, retention and +status, one after the other: a failing or busy stack must not cost the others +their backup, and the schedule's status must say which stack had a problem. +""" +from __future__ import annotations + +import json +import os +import uuid + +import pytest + +from services import backup_service, compose_service + + +def run(coro): + import asyncio + + return asyncio.run(coro) + + +class Env: + """Stacks with scripted outcomes, a destination and the fakes around them.""" + + def __init__(self, monkeypatch, db, outcomes: dict[str, list]): + from sqlmodel import Session + + from models.backup_destination import BackupDestination + from models.stack import Stack + from services import schedule_service + + self.svc = schedule_service + self.outcomes = outcomes # stack -> list of "ok" | "incomplete" | "busy" | Exception + self.backed_up: list[str] = [] + self.uploaded: list[str] = [] + self.pruned: list[str] = [] + self.alerts: list[tuple[str, str]] = [] + self.sleeps: list[float] = [] + + async def fake_create(stack_id, name, **kwargs): + outcome = self.outcomes[stack_id].pop(0) if len(self.outcomes[stack_id]) > 1 \ + else self.outcomes[stack_id][0] + if outcome == "busy": + raise backup_service.StackBusyError(f"Stack '{stack_id}' is busy: update in progress") + if isinstance(outcome, Exception): + raise outcome + self.backed_up.append(stack_id) + path = os.path.join(compose_service.stacks_root(), f".{stack_id}-{uuid.uuid4().hex}.tgz") + open(path, "wb").close() + errors = ["db: helper failed"] if outcome == "incomplete" else [] + return path, {"errors": errors, "restart_error": None} + + async def fake_notify(event, title, body, session): + self.alerts.append((title, body)) + + async def fake_sleep(seconds): + self.sleeps.append(seconds) + + monkeypatch.setattr(schedule_service.backup_service, "create_backup_ex", fake_create) + monkeypatch.setattr(schedule_service.dest_service, "upload", + lambda d, p, f: self.uploaded.append(f) or f) + monkeypatch.setattr(schedule_service, "_prune", + lambda d, s, k: self.pruned.append(s) or 0) + monkeypatch.setattr(schedule_service.notify_service, "notify", fake_notify) + monkeypatch.setattr(schedule_service.asyncio, "sleep", fake_sleep) + + self.session = Session(db) + prefix = uuid.uuid4().hex[:6] + self.ids = {name: f"{prefix}-{name}" for name in outcomes} + self.outcomes = {self.ids[n]: list(o) for n, o in outcomes.items()} + for sid in self.ids.values(): + self.session.add(Stack(id=sid, name=sid)) + self.dest = BackupDestination(name=f"nas-{prefix}", type="sftp", config="{}") + self.session.add(self.dest) + self.session.commit() + + def schedule(self, names=None, all_stacks=False, legacy=None): + from models.backup_schedule import BackupSchedule, set_stacks + + s = BackupSchedule(destination_id=self.dest.id, frequency="daily", all_stacks=all_stacks) + if legacy: + s.stack_id = self.ids[legacy] # a row from before multi-stack schedules + else: + set_stacks(s, [self.ids[n] for n in names or []]) + self.session.add(s) + self.session.commit() + return s + + def status(self, schedule): + from models.backup_schedule import status_of + + return {sid: v["status"] for sid, v in status_of(schedule).items()} + + +def test_every_chosen_stack_is_backed_up_on_its_own(monkeypatch, db): + env = Env(monkeypatch, db, {"a": ["ok"], "b": ["ok"], "c": ["ok"]}) + schedule = env.schedule(["a", "c"]) + + result = run(env.svc.run_schedule(env.session, schedule)) + + a, c = env.ids["a"], env.ids["c"] + assert result["ok"] is True and schedule.last_status == "ok" + assert sorted(env.backed_up) == sorted([a, c]) + assert sorted(env.pruned) == sorted([a, c]) + assert len(env.uploaded) == 2 and all(f.startswith("backup-") for f in env.uploaded) + assert env.status(schedule) == {a: "ok", c: "ok"} + assert env.alerts == [] + + +def test_all_stacks_includes_stacks_added_later(monkeypatch, db): + from sqlmodel import select + + from models.stack import Stack + + env = Env(monkeypatch, db, {"a": ["ok"]}) + schedule = env.schedule(all_stacks=True) + late = f"{env.ids['a']}-late" + env.session.add(Stack(id=late, name=late)) + env.session.commit() + env.outcomes[late] = ["ok"] + # Other tests' stacks share the database: back them up as fine too. + for stack in env.session.exec(select(Stack)).all(): + env.outcomes.setdefault(stack.id, ["ok"]) + + run(env.svc.run_schedule(env.session, schedule)) + + assert {env.ids["a"], late} <= set(env.backed_up) + + +def test_one_failing_stack_does_not_cost_the_others(monkeypatch, db): + env = Env(monkeypatch, db, {"a": [RuntimeError("disk full")], "b": ["ok"], "c": ["incomplete"]}) + schedule = env.schedule(["a", "b", "c"]) + + result = run(env.svc.run_schedule(env.session, schedule)) + + a, b, c = env.ids["a"], env.ids["b"], env.ids["c"] + assert env.backed_up == [b, c] + assert env.pruned == [b] # incomplete: older backups of c are kept + status = env.status(schedule) + assert status[a] == "error: disk full" + assert status[b] == "ok" + assert status[c].startswith("warning: not captured") + assert schedule.last_status.startswith("error: 2 of 3 stacks") + assert a in schedule.last_status and c in schedule.last_status + assert result["ok"] is False + assert len(env.alerts) == 1 and "failed" in env.alerts[0][0] + assert a in env.alerts[0][1] and c in env.alerts[0][1] + + +def test_only_incomplete_stacks_make_a_warning(monkeypatch, db): + env = Env(monkeypatch, db, {"a": ["ok"], "b": ["incomplete"]}) + schedule = env.schedule(["a", "b"]) + + run(env.svc.run_schedule(env.session, schedule)) + + assert schedule.last_status.startswith("warning: 1 of 2 stacks") + assert "incomplete" in env.alerts[0][0] + + +def test_busy_stack_is_retried_after_the_others(monkeypatch, db): + env = Env(monkeypatch, db, {"a": ["busy", "ok"], "b": ["ok"]}) + schedule = env.schedule(["a", "b"]) + + run(env.svc.run_schedule(env.session, schedule)) + + a, b = env.ids["a"], env.ids["b"] + assert env.backed_up == [b, a] + assert env.sleeps == [env.svc.BUSY_WAIT_SECONDS] + assert schedule.last_status == "ok" and env.alerts == [] + + +def test_stack_busy_every_time_is_reported(monkeypatch, db): + env = Env(monkeypatch, db, {"a": ["busy"]}) + schedule = env.schedule(["a"]) + + run(env.svc.run_schedule(env.session, schedule)) + + assert len(env.sleeps) == env.svc.BUSY_ATTEMPTS + assert "busy" in schedule.last_status and schedule.last_status.startswith("error:") + assert env.alerts + + +def test_schedule_from_before_multi_stack_still_runs(monkeypatch, db): + env = Env(monkeypatch, db, {"a": ["ok"]}) + schedule = env.schedule(legacy="a") + + run(env.svc.run_schedule(env.session, schedule)) + + assert env.backed_up == [env.ids["a"]] + + +def test_deleted_stack_is_an_error_for_that_stack_only(monkeypatch, db): + from models.backup_schedule import set_stacks + + env = Env(monkeypatch, db, {"a": ["ok"]}) + schedule = env.schedule(["a"]) + set_stacks(schedule, [env.ids["a"], "gone-stack"]) + + run(env.svc.run_schedule(env.session, schedule)) + + assert env.status(schedule)["gone-stack"] == "error: stack not found" + assert env.status(schedule)[env.ids["a"]] == "ok" + + +def test_a_running_schedule_is_not_started_twice(monkeypatch, db): + env = Env(monkeypatch, db, {"a": ["ok"]}) + schedule = env.schedule(["a"]) + env.svc._running.add(schedule.id) + try: + result = run(env.svc.run_schedule(env.session, schedule)) + finally: + env.svc._running.discard(schedule.id) + assert result["ok"] is False and env.backed_up == [] + + +# --------------------------------------------------------------------------- # +# API +# --------------------------------------------------------------------------- # + + +@pytest.fixture +def api_env(monkeypatch, db, as_admin): + env = Env(monkeypatch, db, {"a": ["ok"], "b": ["ok"]}) + return env, as_admin + + +def test_create_with_several_stacks(api_env): + env, client = api_env + r = client.post("/api/backups/schedules", json={ + "stack_ids": [env.ids["b"], env.ids["a"]], "destination_id": env.dest.id, + }) + assert r.status_code == 201 + body = r.json() + assert body["stack_ids"] == sorted([env.ids["a"], env.ids["b"]]) + assert body["all_stacks"] is False and body["stack_id"] == "" + assert body["running"] is False and body["stack_status"] == {} + + +def test_create_for_all_stacks_needs_no_list(api_env): + env, client = api_env + r = client.post("/api/backups/schedules", json={ + "all_stacks": True, "destination_id": env.dest.id, + }) + assert r.status_code == 201 and r.json()["all_stacks"] is True + + +def test_old_single_stack_request_still_works(api_env): + env, client = api_env + r = client.post("/api/backups/schedules", json={ + "stack_id": env.ids["a"], "destination_id": env.dest.id, + }) + assert r.status_code == 201 + assert r.json()["stack_ids"] == [env.ids["a"]] and r.json()["stack_id"] == env.ids["a"] + + +def test_no_stacks_or_unknown_stacks_are_refused(api_env): + env, client = api_env + r = client.post("/api/backups/schedules", json={"destination_id": env.dest.id}) + assert r.status_code == 400 + r = client.post("/api/backups/schedules", json={ + "stack_ids": [env.ids["a"], "nope"], "destination_id": env.dest.id, + }) + assert r.status_code == 404 and "nope" in r.json()["detail"] + + +def test_edit_changes_the_stacks(api_env): + env, client = api_env + created = client.post("/api/backups/schedules", json={ + "stack_ids": [env.ids["a"]], "destination_id": env.dest.id, + }).json() + r = client.put(f"/api/backups/schedules/{created['id']}", json={ + "stack_ids": [env.ids["a"], env.ids["b"]], + }) + assert r.status_code == 200 + assert r.json()["stack_ids"] == sorted([env.ids["a"], env.ids["b"]]) + r = client.put(f"/api/backups/schedules/{created['id']}", json={"all_stacks": True}) + assert r.json()["all_stacks"] is True + + +def test_run_now_starts_in_the_background(api_env, monkeypatch): + env, client = api_env + created = client.post("/api/backups/schedules", json={ + "stack_ids": [env.ids["a"]], "destination_id": env.dest.id, + }).json() + started: list[int] = [] + + async def fake_run(schedule_id): + started.append(schedule_id) + + monkeypatch.setattr(env.svc, "run_in_background", fake_run) + r = client.post(f"/api/backups/schedules/{created['id']}/run") + assert r.status_code == 202 and r.json() == {"started": True} + + env.svc._running.add(created["id"]) + try: + assert client.post(f"/api/backups/schedules/{created['id']}/run").status_code == 409 + listed = {s["id"]: s for s in client.get("/api/backups/schedules").json()} + assert listed[created["id"]]["running"] is True + finally: + env.svc._running.discard(created["id"]) + + +def test_stack_status_is_stored_as_json(monkeypatch, db): + env = Env(monkeypatch, db, {"a": ["ok"]}) + schedule = env.schedule(["a"]) + run(env.svc.run_schedule(env.session, schedule)) + stored = json.loads(schedule.stack_status) + assert stored[env.ids["a"]]["status"] == "ok" and "at" in stored[env.ids["a"]] diff --git a/backend/version.py b/backend/version.py index ac52be1..3895630 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1,3 +1,3 @@ """Single source of truth for the StackPilot release version.""" -APP_VERSION = "0.64.1" +APP_VERSION = "0.65.0" diff --git a/frontend/package.json b/frontend/package.json index ee0aec3..6218eab 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "stackpilot-frontend", "private": true, - "version": "0.64.1", + "version": "0.65.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/api/schedules.ts b/frontend/src/api/schedules.ts index 1ed1c36..5235a2e 100644 --- a/frontend/src/api/schedules.ts +++ b/frontend/src/api/schedules.ts @@ -2,7 +2,10 @@ import api from "./client"; export interface BackupSchedule { id: number; - stack_id: string; + /** The chosen stacks; empty when all_stacks is set. */ + stack_ids: string[]; + /** Every stack, including ones added later. */ + all_stacks: boolean; destination_id: number; destination_name: string | null; frequency: "hourly" | "daily" | "weekly"; @@ -15,12 +18,17 @@ export interface BackupSchedule { enabled: boolean; last_run: string | null; last_status: string | null; + /** Per stack, from the last run. */ + stack_status: Record; + /** A run is in progress. */ + running: boolean; next_run: string | null; created_at: string; } export interface ScheduleInput { - stack_id: string; + stack_ids: string[]; + all_stacks: boolean; destination_id: number; frequency: string; hour: number; @@ -40,10 +48,7 @@ export const schedulesApi = { api.put(`/api/backups/schedules/${id}`, body).then((r) => r.data), remove: (id: number) => api.delete(`/api/backups/schedules/${id}`).then((r) => r.data), + /** Starts the run in the background; the list then shows `running`. */ run: (id: number) => - api - .post<{ ok: boolean; name?: string; destination?: string; error?: string }>( - `/api/backups/schedules/${id}/run` - ) - .then((r) => r.data), + api.post<{ started: boolean }>(`/api/backups/schedules/${id}/run`).then((r) => r.data), }; diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index 2b5b791..9f1b033 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { Bell, @@ -86,7 +86,15 @@ function describeSchedule(s: BackupSchedule): string { function SchedulesSection() { const qc = useQueryClient(); - const { data, isLoading } = useQuery({ queryKey: ["schedules"], queryFn: schedulesApi.list }); + // Poll while a run is in progress, and briefly after "Run now" (the run + // registers itself a moment after the request returns). + const [pollUntil, setPollUntil] = useState(0); + const { data, isLoading } = useQuery({ + queryKey: ["schedules"], + queryFn: schedulesApi.list, + refetchInterval: (q) => + q.state.data?.some((s) => s.running) || Date.now() < pollUntil ? 3000 : false, + }); const destinations = useQuery({ queryKey: ["destinations"], queryFn: destinationsApi.list }); const stacks = useQuery({ queryKey: ["stacks"], queryFn: stacksApi.list }); const [adding, setAdding] = useState(false); @@ -100,12 +108,24 @@ function SchedulesSection() { {isLoading ? ( ) : ( - data?.map((s) => ) + data?.map((s) => ( + { + setPollUntil(Date.now() + 15000); + invalidate(); + }} + /> + )) )} {data?.length === 0 && !adding && (

- No scheduled backups. Add one to automatically push a stack to a destination + No scheduled backups. Add one to automatically push stacks to a destination on a recurring schedule.

@@ -130,10 +150,27 @@ function SchedulesSection() { ); } -function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChange: () => void }) { +function statusTone(status: string | null | undefined): string { + if (!status) return "text-slate-500"; + if (status === "ok") return "text-green-600 dark:text-green-400"; + if (status.startsWith("warning")) return "text-amber-600 dark:text-amber-400"; + return "text-red-500"; +} + +function ScheduleRow({ + schedule, + stacks, + destinations, + onChange, + onStarted, +}: { + schedule: BackupSchedule; + stacks: { id: string; name: string }[]; + destinations: BackupDestination[]; + onChange: () => void; + onStarted: () => void; +}) { const [editing, setEditing] = useState(false); - const destinations = useQuery({ queryKey: ["destinations"], queryFn: destinationsApi.list }); - const stacks = useQuery({ queryKey: ["stacks"], queryFn: stacksApi.list }); const toggle = useMutation({ mutationFn: () => schedulesApi.update(schedule.id, { enabled: !schedule.enabled }), onSuccess: onChange, @@ -141,8 +178,10 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang }); const run = useMutation({ mutationFn: () => schedulesApi.run(schedule.id), - onSuccess: (r) => - r.ok ? toast.success(`Backed up to ${r.destination}`) : toast.error(r.error || "Backup failed"), + onSuccess: () => { + toast.info("Backup started"); + onStarted(); + }, onError: (e) => toast.error(apiErrorMessage(e)), }); const remove = useMutation({ @@ -151,12 +190,24 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang onError: (e) => toast.error(apiErrorMessage(e)), }); + // Announce the outcome when a run this page watched finishes. + const wasRunning = useRef(schedule.running); + useEffect(() => { + if (wasRunning.current && !schedule.running) { + const status = schedule.last_status ?? ""; + if (status === "ok") toast.success("Scheduled backup finished"); + else if (status.startsWith("warning")) toast.warning("Scheduled backup is incomplete", { description: status }); + else toast.error("Scheduled backup failed", { description: status }); + } + wasRunning.current = schedule.running; + }, [schedule.running, schedule.last_status]); + if (editing) { return ( { setEditing(false); onChange(); @@ -166,31 +217,43 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang ); } - const status = schedule.last_status ?? ""; - const statusClass = - status === "ok" - ? "text-green-600 dark:text-green-400" - : status.startsWith("warning") || status.startsWith("retrying") - ? "text-amber-600 dark:text-amber-400" - : "text-red-500"; + const perStack = Object.entries(schedule.stack_status ?? {}).sort(([a], [b]) => a.localeCompare(b)); + const problems = perStack.filter(([, v]) => v.status !== "ok"); + const shown = schedule.stack_ids.slice(0, 4); return (
-
- {schedule.stack_id} +
+ {schedule.all_stacks ? ( + All stacks + ) : ( + + {shown.join(", ")} + {schedule.stack_ids.length > shown.length && ( + +{schedule.stack_ids.length - shown.length} + )} + + )} → {schedule.destination_name ?? `dest ${schedule.destination_id}`} {!schedule.enabled && disabled} + {schedule.running && ( + running… + )}
- -
{describeSchedule(schedule)} - keep {schedule.keep || "∞"} + keep {schedule.keep || "∞"} per stack {schedule.include_volumes && +volumes} {schedule.next_run && next {relativeTime(schedule.next_run)}} {schedule.last_run && ( - - last {relativeTime(schedule.last_run)} · {schedule.last_status} + + last {relativeTime(schedule.last_run)} ·{" "} + {problems.length || perStack.length <= 1 + ? schedule.last_status + : `${perStack.length} stacks ok`} )}
+ {problems.length > 0 && perStack.length > 1 && ( +
    + {problems.map(([sid, v]) => ( +
  • + {sid} + {v.status} +
    • + ))} +
+ )} ); } @@ -220,7 +296,7 @@ function ScheduleForm({ onDone, onCancel, }: { - /** Set to edit an existing schedule; its stack cannot change. */ + /** Set to edit an existing schedule. */ schedule?: BackupSchedule; stacks: { id: string; name: string }[]; destinations: BackupDestination[]; @@ -229,7 +305,8 @@ function ScheduleForm({ }) { const editing = Boolean(schedule); const [form, setForm] = useState({ - stack_id: schedule?.stack_id ?? stacks[0]?.id ?? "", + stack_ids: schedule?.stack_ids ?? ([] as string[]), + all_stacks: schedule?.all_stacks ?? false, destination_id: schedule?.destination_id ?? destinations[0]?.id ?? 0, frequency: schedule?.frequency ?? "daily", hour: schedule?.hour ?? 3, @@ -241,28 +318,28 @@ function ScheduleForm({ enabled: schedule?.enabled ?? true, }); const set = (k: string, v: unknown) => setForm((f) => ({ ...f, [k]: v })); + const toggleStack = (id: string, on: boolean) => + set("stack_ids", on ? [...form.stack_ids, id] : form.stack_ids.filter((s) => s !== id)); - const stackOptions = stacks; - - // Keep stack_id and destination_id valid as the options load or change. - useEffect(() => { - if (!editing && stackOptions.length && !stackOptions.some((s) => s.id === form.stack_id)) { - set("stack_id", stackOptions[0].id); - } - }, [stackOptions]); // eslint-disable-line react-hooks/exhaustive-deps + // Keep destination_id valid as the options load or change. useEffect(() => { if (destinations.length && !destinations.some((d) => d.id === form.destination_id)) { set("destination_id", destinations[0].id); } }, [destinations]); // eslint-disable-line react-hooks/exhaustive-deps + // Stacks deleted since the schedule was made stay listed, so they can be + // unticked rather than silently dropped. + const known = new Set(stacks.map((s) => s.id)); + const stackOptions = [ + ...stacks, + ...form.stack_ids.filter((id) => !known.has(id)).map((id) => ({ id, name: `${id} (deleted)` })), + ]; + const ready = !!form.destination_id && (form.all_stacks || form.stack_ids.length > 0); + const save = useMutation({ - mutationFn: () => { - if (!schedule) return schedulesApi.create(form); - // The stack is fixed once scheduled: its retention matches by stack id. - const { stack_id: _stack, ...changes } = form; - return schedulesApi.update(schedule.id, changes); - }, + mutationFn: () => + schedule ? schedulesApi.update(schedule.id, form) : schedulesApi.create(form), onSuccess: () => { toast.success(editing ? "Schedule updated" : "Schedule added"); onDone(); @@ -272,24 +349,56 @@ function ScheduleForm({ return ( -
-