stackpilot/backend/services/schedule_service.py

"""Scheduled (recurring) stack backups to a destination.

A background loop wakes every minute, runs any schedules whose ``next_run`` has
passed and records status + the next run time. A schedule covers several
stacks (or all of them); each is backed up on its own — its own file, upload,
retention and status — one after the other, so one failing stack does not
cost the others their backup.

Times are stored as naive UTC to match SQLite's datetime handling.
"""
from __future__ import annotations

import asyncio
import json
import logging
import os
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError

from sqlmodel import Session, select

from database import engine
from models.backup_destination import BackupDestination
from models.backup_schedule import BackupSchedule, stacks_of
from models.setting import EVENT_BACKUP_FAILED
from models.stack import Stack
from services import (
    backup_destination_service as dest_service,
    backup_service,
    notify_service,
)

logger = logging.getLogger("stackpilot.schedule")

#: A stack that is busy (deploying, updating) when its turn comes is tried
#: again after the others, up to this many more times, this far apart.
BUSY_ATTEMPTS = 3
BUSY_WAIT_SECONDS = 120.0

#: Schedules with a run in progress (scheduler or "Run now"), so the same
#: schedule never runs twice at once.
_running: set[int] = set()


def is_running(schedule_id: int) -> bool:
    return schedule_id in _running


def resolve_stacks(session: Session, schedule: BackupSchedule) -> list[str]:
    if schedule.all_stacks:
        return sorted(s.id for s in session.exec(select(Stack)).all())
    return stacks_of(schedule)


def _now() -> datetime:
    return datetime.now(timezone.utc).replace(tzinfo=None)


def zone(name: str | None) -> ZoneInfo:
    """The schedule's time zone; UTC for an empty or unknown name."""
    try:
        return ZoneInfo(name or "UTC")
    except (ZoneInfoNotFoundError, ValueError):
        return ZoneInfo("UTC")


def compute_next_run(
    frequency: str,
    hour: int,
    minute: int,
    weekday: int,
    after: datetime,
    tz: str | None = "UTC",
) -> datetime:
    """Next run strictly after ``after`` (naive UTC), returned as naive UTC.

    ``hour``/``minute``/``weekday`` are wall-clock time in ``tz``, so a 03:00
    backup stays at 03:00 across daylight saving changes. Each candidate day is
    built from its own date rather than by adding 24 hours, which is what keeps
    it on the wall clock. A time that does not exist on the day clocks go
    forward (02:30) runs an hour later that day.
    """
    zi = zone(tz)
    after_utc = after.replace(tzinfo=timezone.utc)
    local = after_utc.astimezone(zi)

    def utc(dt: datetime) -> datetime:
        return dt.astimezone(timezone.utc).replace(tzinfo=None)

    if frequency == "hourly":
        cand = local.replace(minute=minute % 60, second=0, microsecond=0)
        # Step in UTC: an hour is an hour, whatever the wall clock does.
        nxt = cand.astimezone(timezone.utc)
        while nxt <= after_utc:
            nxt += timedelta(hours=1)
        return nxt.replace(tzinfo=None)

    def at(day) -> datetime:
        return datetime(day.year, day.month, day.day, hour % 24, minute % 60, tzinfo=zi)

    day = local.date()
    if frequency == "weekly":
        day += timedelta(days=(weekday - day.weekday()) % 7)
        step = timedelta(days=7)
    else:  # daily (default)
        step = timedelta(days=1)
    while at(day).astimezone(timezone.utc) <= after_utc:
        day += step
    return utc(at(day))


def _prune(dest: BackupDestination, stack_id: str, keep: int) -> int:
    if keep <= 0:
        return 0
    items = dest_service.list_backups(dest)
    pattern = backup_service.backup_name_pattern(stack_id)
    mine = [i for i in items if pattern.match(i["name"])]
    mine.sort(key=lambda x: x.get("modified") or 0, reverse=True)
    removed = 0
    for old in mine[keep:]:
        try:
            dest_service.delete(dest, old["name"])
            removed += 1
        except dest_service.DestinationError as exc:
            logger.warning("retention delete failed for %s: %s", old["name"], exc)
    return removed


def _problems(report: dict) -> list[str]:
    """What makes a backup that was written less than a success."""
    problems = [f"not captured: {e}" for e in report.get("errors") or []]
    if report.get("restart_error"):
        problems.append(f"stack did not start again: {report['restart_error']}")
    return problems


async def _alert(session: Session, title: str, body: str) -> None:
    try:
        await notify_service.notify(EVENT_BACKUP_FAILED, title, body, session)
    except Exception:  # noqa: BLE001
        pass


class _Busy(Exception):
    pass


async def _backup_stack(
    session: Session, schedule: BackupSchedule, dest: BackupDestination, stack_id: str
) -> str:
    """Back up, upload and rotate one stack. Returns its status line; raises
    _Busy when the stack is locked by another operation."""
    stack = session.get(Stack, stack_id)
    if not stack:
        return "error: stack not found"
    path = None
    try:
        path, report = await backup_service.create_backup_ex(
            stack_id, stack.name,
            include_volumes=schedule.include_volumes, stop_first=schedule.stop_first,
        )
        filename = backup_service.backup_filename(
            stack_id, schedule.include_volumes, tz=schedule.timezone
        )
        await asyncio.to_thread(dest_service.upload, dest, path, filename)
        problems = _problems(report)
        if problems:
            # Retention only after a complete backup: an incomplete one must
            # not push the last good backup out.
            return f"warning: {'; '.join(problems)}"
        await asyncio.to_thread(_prune, dest, stack_id, schedule.keep)
        return "ok"
    except backup_service.StackBusyError as exc:
        raise _Busy(str(exc)) from exc
    except backup_service.BackupCancelled as exc:
        # Somebody stopped this backup from the UI. Not a failure of the
        # schedule: say so, and don't page anyone with "backup failed".
        logger.info("Scheduled backup of %s was cancelled: %s", stack_id, exc)
        return f"cancelled: {exc}"
    except Exception as exc:  # noqa: BLE001
        logger.warning("Scheduled backup of %s failed: %s", stack_id, exc)
        return f"error: {exc}"
    finally:
        if path and os.path.exists(path):
            os.unlink(path)


def _summarise(statuses: dict[str, str]) -> str:
    """One status line for the schedule from the per-stack ones."""
    if not statuses:
        return "error: no stacks to back up"
    bad = {sid: st for sid, st in statuses.items() if st != "ok"}
    if not bad:
        return "ok"
    worst = (
        "error"
        if any(not st.startswith(("warning", "cancelled")) for st in bad.values())
        else "warning"
    )
    if len(statuses) == 1:
        return next(iter(statuses.values()))
    detail = "; ".join(f"{sid}: {st}" for sid, st in sorted(bad.items()))
    return f"{worst}: {len(bad)} of {len(statuses)} stacks — {detail}"


async def run_schedule(session: Session, schedule: BackupSchedule) -> dict:
    """Execute one schedule now. Updates status + next_run. Returns a summary."""
    if schedule.id in _running:
        return {"ok": False, "error": "this schedule is already running"}
    _running.add(schedule.id)
    now = _now()
    statuses: dict[str, str] = {}
    dest = session.get(BackupDestination, schedule.destination_id)
    try:
        stack_ids = resolve_stacks(session, schedule)
        if not dest:
            statuses = {sid: f"error: destination {schedule.destination_id} not found"
                        for sid in stack_ids}
        else:
            pending = list(stack_ids)
            for attempt in range(BUSY_ATTEMPTS + 1):
                busy: list[str] = []
                for stack_id in pending:
                    try:
                        statuses[stack_id] = await _backup_stack(session, schedule, dest, stack_id)
                    except _Busy as exc:
                        busy.append(stack_id)
                        statuses[stack_id] = f"error: {exc}"
                if not busy or attempt == BUSY_ATTEMPTS:
                    break
                # Deploys and updates are short: wait, then take the busy
                # stacks again rather than skipping them until the next run.
                pending = busy
                await asyncio.sleep(BUSY_WAIT_SECONDS)

        schedule.last_status = _summarise(statuses)[:300]
        problems = {sid: st for sid, st in statuses.items() if st != "ok"}
        if problems or not statuses:
            await _alert(
                session,
                f"Scheduled backup to '{dest.name if dest else schedule.destination_id}' "
                + ("failed" if schedule.last_status.startswith("error") else "is incomplete"),
                "\n".join(f"{sid}: {st}" for sid, st in sorted(problems.items()))
                or schedule.last_status,
            )
        logger.info("Scheduled backup %s: %s", schedule.id, schedule.last_status)
        return {
            "ok": schedule.last_status == "ok",
            "destination": dest.name if dest else None,
            "stacks": statuses,
            **({} if schedule.last_status == "ok" else {"error": schedule.last_status}),
        }
    finally:
        _running.discard(schedule.id)
        schedule.stack_status = json.dumps(
            {sid: {"status": st[:300], "at": now.isoformat()} for sid, st in statuses.items()}
        )
        schedule.last_run = now
        schedule.next_run = compute_next_run(
            schedule.frequency, schedule.hour, schedule.minute, schedule.weekday, now,
            schedule.timezone,
        )
        session.add(schedule)
        session.commit()
        session.refresh(schedule)


async def run_in_background(schedule_id: int) -> None:
    """Run a schedule with its own session, for "Run now": a run over many
    stacks takes far longer than a request may."""
    with Session(engine) as session:
        schedule = session.get(BackupSchedule, schedule_id)
        if schedule:
            await run_schedule(session, schedule)


async def run_due() -> None:
    now = _now()
    with Session(engine) as session:
        schedules = session.exec(
            select(BackupSchedule).where(BackupSchedule.enabled == True)  # noqa: E712
        ).all()
        due = []
        for s in schedules:
            if s.next_run is None:
                s.next_run = compute_next_run(
                    s.frequency, s.hour, s.minute, s.weekday, now, s.timezone
                )
                session.add(s)
            elif s.next_run <= now:
                due.append(s)
        session.commit()
        for s in due:
            if not is_running(s.id):
                await run_schedule(session, s)


async def scheduler_loop() -> None:
    await asyncio.sleep(20)  # let startup settle
    while True:
        try:
            await run_due()
        except Exception as exc:  # noqa: BLE001
            logger.warning("scheduler tick failed: %s", exc)
        await asyncio.sleep(60)