"""Scheduled (recurring) stack backups to a destination. A background loop wakes every minute, runs any schedules whose ``next_run`` has passed, uploads the backup to the destination, prunes old backups per the retention setting, and records status + the next run time. Times are stored as naive UTC to match SQLite's datetime handling. """ from __future__ import annotations import asyncio import logging import os from datetime import datetime, timedelta, timezone from sqlmodel import Session, select from database import engine from models.backup_destination import BackupDestination from models.backup_schedule import BackupSchedule from models.setting import EVENT_BACKUP_FAILED from models.stack import Stack from services import ( backup_destination_service as dest_service, backup_service, notify_service, ) logger = logging.getLogger("stackpilot.schedule") #: A stack that is busy (deploying, updating) when its backup is due is tried #: again this much later instead of waiting for the next regular run. BUSY_RETRY = timedelta(minutes=10) def _now() -> datetime: return datetime.now(timezone.utc).replace(tzinfo=None) def compute_next_run( frequency: str, hour: int, minute: int, weekday: int, after: datetime ) -> datetime: """Next run strictly after ``after`` (naive UTC).""" if frequency == "hourly": nxt = after.replace(minute=minute % 60, second=0, microsecond=0) if nxt <= after: nxt += timedelta(hours=1) return nxt base = after.replace(hour=hour % 24, minute=minute % 60, second=0, microsecond=0) if frequency == "weekly": days_ahead = (weekday - base.weekday()) % 7 cand = base + timedelta(days=days_ahead) if cand <= after: cand += timedelta(days=7) return cand # daily (default) if base <= after: base += timedelta(days=1) return base def _prune(dest: BackupDestination, stack_id: str, keep: int) -> int: if keep <= 0: return 0 items = dest_service.list_backups(dest) pattern = backup_service.backup_name_pattern(stack_id) mine = [i for i in items if pattern.match(i["name"])] mine.sort(key=lambda x: x.get("modified") or 0, reverse=True) removed = 0 for old in mine[keep:]: try: dest_service.delete(dest, old["name"]) removed += 1 except dest_service.DestinationError as exc: logger.warning("retention delete failed for %s: %s", old["name"], exc) return removed def _problems(report: dict) -> list[str]: """What makes a backup that was written less than a success.""" problems = [f"not captured: {e}" for e in report.get("errors") or []] if report.get("restart_error"): problems.append(f"stack did not start again: {report['restart_error']}") return problems async def _alert(session: Session, title: str, body: str) -> None: try: await notify_service.notify(EVENT_BACKUP_FAILED, title, body, session) except Exception: # noqa: BLE001 pass async def run_schedule(session: Session, schedule: BackupSchedule) -> dict: """Execute one schedule now. Updates status + next_run. Returns a summary.""" now = _now() dest = session.get(BackupDestination, schedule.destination_id) result: dict = {"ok": False} path = None next_run = None try: if not dest: raise RuntimeError(f"destination {schedule.destination_id} not found") stack = session.get(Stack, schedule.stack_id) if not stack: raise RuntimeError(f"stack '{schedule.stack_id}' not found") path, report = await backup_service.create_backup_ex( schedule.stack_id, stack.name, include_volumes=schedule.include_volumes, stop_first=schedule.stop_first, ) filename = backup_service.backup_filename(schedule.stack_id, schedule.include_volumes) await asyncio.to_thread(dest_service.upload, dest, path, filename) problems = _problems(report) # Retention only after a complete backup: an incomplete one must not # push the last good backup out. pruned = 0 if problems else await asyncio.to_thread( _prune, dest, schedule.stack_id, schedule.keep ) result = {"ok": not problems, "name": filename, "destination": dest.name, "pruned": pruned} if problems: schedule.last_status = f"warning: {'; '.join(problems)}"[:300] result["error"] = "; ".join(problems) logger.warning("Scheduled backup %s incomplete: %s", schedule.stack_id, problems) await _alert( session, f"Scheduled backup of '{schedule.stack_id}' is incomplete", "\n".join(problems), ) else: schedule.last_status = "ok" logger.info( "Scheduled backup %s → %s ok (pruned %d)", schedule.stack_id, dest.name, pruned ) except backup_service.StackBusyError as exc: # Deploys and updates are short: try again soon rather than skipping a day. schedule.last_status = f"retrying: {exc}"[:300] result = {"ok": False, "error": str(exc)} next_run = now + BUSY_RETRY logger.info("Scheduled backup %s postponed: %s", schedule.stack_id, exc) except Exception as exc: # noqa: BLE001 schedule.last_status = f"error: {exc}"[:300] result = {"ok": False, "error": str(exc)} logger.warning("Scheduled backup %s failed: %s", schedule.stack_id, exc) await _alert(session, f"Scheduled backup of '{schedule.stack_id}' failed", str(exc)) finally: if path and os.path.exists(path): os.unlink(path) schedule.last_run = now schedule.next_run = next_run or compute_next_run( schedule.frequency, schedule.hour, schedule.minute, schedule.weekday, now ) session.add(schedule) session.commit() session.refresh(schedule) return result async def run_due() -> None: now = _now() with Session(engine) as session: schedules = session.exec( select(BackupSchedule).where(BackupSchedule.enabled == True) # noqa: E712 ).all() due = [] for s in schedules: if s.next_run is None: s.next_run = compute_next_run(s.frequency, s.hour, s.minute, s.weekday, now) session.add(s) elif s.next_run <= now: due.append(s) session.commit() for s in due: await run_schedule(session, s) async def scheduler_loop() -> None: await asyncio.sleep(20) # let startup settle while True: try: await run_due() except Exception as exc: # noqa: BLE001 logger.warning("scheduler tick failed: %s", exc) await asyncio.sleep(60)