A schedule covered exactly one stack, so backing up a host meant one
schedule per stack, each edited separately, and new stacks were only
backed up once someone remembered to add a schedule.
A schedule now takes a list of stacks, or all_stacks, which includes
stacks added later. They are backed up one after another, each with its
own file, upload, retention and status, so one failing stack does not
cost the others their backup:
- stack_status records each stack's result; last_status summarises
("error: 2 of 5 stacks - a: ...; c: ...") and one notification per run
names the stacks with a problem. The dashboard names them too and shows
an incomplete backup as a warning rather than an error.
- A busy stack (deploy, update) is retried after the others, up to three
times two minutes apart, replacing the 10-minute postponement of the
whole schedule.
- Run now starts the run in the background (202) instead of holding the
request, which a multi-stack run would outlast; the list reports
running and polls, and the row toasts the outcome. A schedule never
runs twice at once.
- Settings: "All stacks" or a checklist with select all/none; stacks can
be changed when editing; deleted stacks stay listed to be unticked.
Schedules from before keep working without a data migration: stack_ids,
all_stacks and stack_status are added with defaults on startup and a row
with only stack_id reads as that one stack. The API still accepts
stack_id and reports it for single-stack schedules.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
191 lines
6.3 KiB
Python
191 lines
6.3 KiB
Python
"""Scheduled backup management."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
from sqlmodel import Session, select
|
|
|
|
from auth import require_admin
|
|
from database import get_session
|
|
from models.backup_destination import BackupDestination
|
|
from models.backup_schedule import (
|
|
FREQUENCIES,
|
|
BackupSchedule,
|
|
ScheduleCreate,
|
|
ScheduleRead,
|
|
ScheduleUpdate,
|
|
describe_stacks,
|
|
set_stacks,
|
|
stacks_of,
|
|
status_of,
|
|
)
|
|
from models.stack import Stack
|
|
from models.user import User
|
|
from services import audit_service, schedule_service
|
|
|
|
router = APIRouter(prefix="/api/backups/schedules", tags=["backups"])
|
|
|
|
|
|
def _ip(request: Request) -> str:
|
|
return request.client.host if request.client else "unknown"
|
|
|
|
|
|
def _to_read(session: Session, s: BackupSchedule) -> ScheduleRead:
|
|
dest = session.get(BackupDestination, s.destination_id)
|
|
stack_ids = stacks_of(s)
|
|
return ScheduleRead(
|
|
id=s.id,
|
|
stack_ids=stack_ids,
|
|
all_stacks=s.all_stacks,
|
|
stack_id=stack_ids[0] if len(stack_ids) == 1 and not s.all_stacks else "",
|
|
destination_id=s.destination_id,
|
|
destination_name=dest.name if dest else None,
|
|
frequency=s.frequency,
|
|
hour=s.hour,
|
|
minute=s.minute,
|
|
weekday=s.weekday,
|
|
include_volumes=s.include_volumes,
|
|
stop_first=s.stop_first,
|
|
keep=s.keep,
|
|
enabled=s.enabled,
|
|
last_run=s.last_run,
|
|
last_status=s.last_status,
|
|
stack_status=status_of(s),
|
|
running=schedule_service.is_running(s.id),
|
|
next_run=s.next_run,
|
|
created_at=s.created_at,
|
|
)
|
|
|
|
|
|
def _get_or_404(session: Session, schedule_id: int) -> BackupSchedule:
|
|
s = session.get(BackupSchedule, schedule_id)
|
|
if not s:
|
|
raise HTTPException(status_code=404, detail=f"Schedule {schedule_id} not found")
|
|
return s
|
|
|
|
|
|
def _validate(session: Session, schedule: BackupSchedule) -> None:
|
|
if schedule.frequency not in FREQUENCIES:
|
|
raise HTTPException(status_code=400, detail=f"Unknown frequency '{schedule.frequency}'")
|
|
if not session.get(BackupDestination, schedule.destination_id):
|
|
raise HTTPException(status_code=404, detail=f"Destination {schedule.destination_id} not found")
|
|
if schedule.all_stacks:
|
|
return
|
|
stack_ids = stacks_of(schedule)
|
|
if not stack_ids:
|
|
raise HTTPException(status_code=400, detail="Choose at least one stack, or all stacks")
|
|
missing = [sid for sid in stack_ids if not session.get(Stack, sid)]
|
|
if missing:
|
|
raise HTTPException(status_code=404, detail=f"Stack(s) not found: {', '.join(missing)}")
|
|
|
|
|
|
@router.get("", response_model=list[ScheduleRead])
|
|
def list_schedules(
|
|
session: Session = Depends(get_session),
|
|
_user: User = Depends(require_admin),
|
|
) -> list[ScheduleRead]:
|
|
rows = session.exec(select(BackupSchedule).order_by(BackupSchedule.id)).all()
|
|
return [_to_read(session, s) for s in rows]
|
|
|
|
|
|
@router.post("", response_model=ScheduleRead, status_code=201)
|
|
def create_schedule(
|
|
body: ScheduleCreate,
|
|
request: Request,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> ScheduleRead:
|
|
data = body.model_dump(exclude={"stack_ids", "stack_id"})
|
|
s = BackupSchedule(**data)
|
|
set_stacks(s, body.stack_ids or ([body.stack_id] if body.stack_id else []))
|
|
_validate(session, s)
|
|
s.next_run = schedule_service.compute_next_run(
|
|
s.frequency, s.hour, s.minute, s.weekday, schedule_service._now()
|
|
)
|
|
session.add(s)
|
|
session.commit()
|
|
session.refresh(s)
|
|
audit_service.record(
|
|
session, user=user.username, action="schedule.create", target=describe_stacks(s),
|
|
detail=f"{s.frequency} → dest {s.destination_id}", ip=_ip(request),
|
|
)
|
|
return _to_read(session, s)
|
|
|
|
|
|
@router.put("/{schedule_id}", response_model=ScheduleRead)
|
|
def update_schedule(
|
|
schedule_id: int,
|
|
body: ScheduleUpdate,
|
|
request: Request,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> ScheduleRead:
|
|
s = _get_or_404(session, schedule_id)
|
|
data = body.model_dump(exclude_unset=True)
|
|
stack_ids = data.pop("stack_ids", None)
|
|
for k, v in data.items():
|
|
setattr(s, k, v)
|
|
if stack_ids is not None:
|
|
set_stacks(s, stack_ids)
|
|
_validate(session, s)
|
|
# Recompute next run when timing fields change.
|
|
if {"frequency", "hour", "minute", "weekday"} & set(data) or s.next_run is None:
|
|
s.next_run = schedule_service.compute_next_run(
|
|
s.frequency, s.hour, s.minute, s.weekday, schedule_service._now()
|
|
)
|
|
session.add(s)
|
|
session.commit()
|
|
session.refresh(s)
|
|
audit_service.record(
|
|
session, user=user.username, action="schedule.update", target=describe_stacks(s),
|
|
ip=_ip(request),
|
|
)
|
|
return _to_read(session, s)
|
|
|
|
|
|
@router.delete("/{schedule_id}")
|
|
def delete_schedule(
|
|
schedule_id: int,
|
|
request: Request,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
s = _get_or_404(session, schedule_id)
|
|
target = describe_stacks(s)
|
|
session.delete(s)
|
|
session.commit()
|
|
audit_service.record(
|
|
session, user=user.username, action="schedule.delete", target=target,
|
|
ip=_ip(request),
|
|
)
|
|
return {"ok": True}
|
|
|
|
|
|
#: Keeps "Run now" tasks referenced until they finish (asyncio only holds
|
|
#: weak references to tasks).
|
|
_tasks: set[asyncio.Task] = set()
|
|
|
|
|
|
@router.post("/{schedule_id}/run", status_code=202)
|
|
async def run_now(
|
|
schedule_id: int,
|
|
request: Request,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
"""Start the schedule in the background. Backing up several stacks takes
|
|
far longer than a request may; the list shows ``running`` and then the
|
|
per-stack results."""
|
|
s = _get_or_404(session, schedule_id)
|
|
if schedule_service.is_running(s.id):
|
|
raise HTTPException(status_code=409, detail="This schedule is already running")
|
|
task = asyncio.create_task(schedule_service.run_in_background(s.id))
|
|
_tasks.add(task)
|
|
task.add_done_callback(_tasks.discard)
|
|
audit_service.record(
|
|
session, user=user.username, action="schedule.run", target=describe_stacks(s),
|
|
ip=_ip(request),
|
|
)
|
|
return {"started": True}
|