menzeljandClaude Opus 5
270ddda51d
A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.
Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.
POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".
Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
304 lines
11 KiB
Python
304 lines
11 KiB
Python
"""Scheduled (recurring) stack backups to a destination.
|
|
|
|
A background loop wakes every minute, runs any schedules whose ``next_run`` has
|
|
passed and records status + the next run time. A schedule covers several
|
|
stacks (or all of them); each is backed up on its own — its own file, upload,
|
|
retention and status — one after the other, so one failing stack does not
|
|
cost the others their backup.
|
|
|
|
Times are stored as naive UTC to match SQLite's datetime handling.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|
|
|
from sqlmodel import Session, select
|
|
|
|
from database import engine
|
|
from models.backup_destination import BackupDestination
|
|
from models.backup_schedule import BackupSchedule, stacks_of
|
|
from models.setting import EVENT_BACKUP_FAILED
|
|
from models.stack import Stack
|
|
from services import (
|
|
backup_destination_service as dest_service,
|
|
backup_service,
|
|
notify_service,
|
|
)
|
|
|
|
logger = logging.getLogger("stackpilot.schedule")
|
|
|
|
#: A stack that is busy (deploying, updating) when its turn comes is tried
|
|
#: again after the others, up to this many more times, this far apart.
|
|
BUSY_ATTEMPTS = 3
|
|
BUSY_WAIT_SECONDS = 120.0
|
|
|
|
#: Schedules with a run in progress (scheduler or "Run now"), so the same
|
|
#: schedule never runs twice at once.
|
|
_running: set[int] = set()
|
|
|
|
|
|
def is_running(schedule_id: int) -> bool:
|
|
return schedule_id in _running
|
|
|
|
|
|
def resolve_stacks(session: Session, schedule: BackupSchedule) -> list[str]:
|
|
if schedule.all_stacks:
|
|
return sorted(s.id for s in session.exec(select(Stack)).all())
|
|
return stacks_of(schedule)
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(timezone.utc).replace(tzinfo=None)
|
|
|
|
|
|
def zone(name: str | None) -> ZoneInfo:
|
|
"""The schedule's time zone; UTC for an empty or unknown name."""
|
|
try:
|
|
return ZoneInfo(name or "UTC")
|
|
except (ZoneInfoNotFoundError, ValueError):
|
|
return ZoneInfo("UTC")
|
|
|
|
|
|
def compute_next_run(
|
|
frequency: str,
|
|
hour: int,
|
|
minute: int,
|
|
weekday: int,
|
|
after: datetime,
|
|
tz: str | None = "UTC",
|
|
) -> datetime:
|
|
"""Next run strictly after ``after`` (naive UTC), returned as naive UTC.
|
|
|
|
``hour``/``minute``/``weekday`` are wall-clock time in ``tz``, so a 03:00
|
|
backup stays at 03:00 across daylight saving changes. Each candidate day is
|
|
built from its own date rather than by adding 24 hours, which is what keeps
|
|
it on the wall clock. A time that does not exist on the day clocks go
|
|
forward (02:30) runs an hour later that day.
|
|
"""
|
|
zi = zone(tz)
|
|
after_utc = after.replace(tzinfo=timezone.utc)
|
|
local = after_utc.astimezone(zi)
|
|
|
|
def utc(dt: datetime) -> datetime:
|
|
return dt.astimezone(timezone.utc).replace(tzinfo=None)
|
|
|
|
if frequency == "hourly":
|
|
cand = local.replace(minute=minute % 60, second=0, microsecond=0)
|
|
# Step in UTC: an hour is an hour, whatever the wall clock does.
|
|
nxt = cand.astimezone(timezone.utc)
|
|
while nxt <= after_utc:
|
|
nxt += timedelta(hours=1)
|
|
return nxt.replace(tzinfo=None)
|
|
|
|
def at(day) -> datetime:
|
|
return datetime(day.year, day.month, day.day, hour % 24, minute % 60, tzinfo=zi)
|
|
|
|
day = local.date()
|
|
if frequency == "weekly":
|
|
day += timedelta(days=(weekday - day.weekday()) % 7)
|
|
step = timedelta(days=7)
|
|
else: # daily (default)
|
|
step = timedelta(days=1)
|
|
while at(day).astimezone(timezone.utc) <= after_utc:
|
|
day += step
|
|
return utc(at(day))
|
|
|
|
|
|
def _prune(dest: BackupDestination, stack_id: str, keep: int) -> int:
|
|
if keep <= 0:
|
|
return 0
|
|
items = dest_service.list_backups(dest)
|
|
pattern = backup_service.backup_name_pattern(stack_id)
|
|
mine = [i for i in items if pattern.match(i["name"])]
|
|
mine.sort(key=lambda x: x.get("modified") or 0, reverse=True)
|
|
removed = 0
|
|
for old in mine[keep:]:
|
|
try:
|
|
dest_service.delete(dest, old["name"])
|
|
removed += 1
|
|
except dest_service.DestinationError as exc:
|
|
logger.warning("retention delete failed for %s: %s", old["name"], exc)
|
|
return removed
|
|
|
|
|
|
def _problems(report: dict) -> list[str]:
|
|
"""What makes a backup that was written less than a success."""
|
|
problems = [f"not captured: {e}" for e in report.get("errors") or []]
|
|
if report.get("restart_error"):
|
|
problems.append(f"stack did not start again: {report['restart_error']}")
|
|
return problems
|
|
|
|
|
|
async def _alert(session: Session, title: str, body: str) -> None:
|
|
try:
|
|
await notify_service.notify(EVENT_BACKUP_FAILED, title, body, session)
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
|
|
class _Busy(Exception):
|
|
pass
|
|
|
|
|
|
async def _backup_stack(
|
|
session: Session, schedule: BackupSchedule, dest: BackupDestination, stack_id: str
|
|
) -> str:
|
|
"""Back up, upload and rotate one stack. Returns its status line; raises
|
|
_Busy when the stack is locked by another operation."""
|
|
stack = session.get(Stack, stack_id)
|
|
if not stack:
|
|
return "error: stack not found"
|
|
path = None
|
|
try:
|
|
path, report = await backup_service.create_backup_ex(
|
|
stack_id, stack.name,
|
|
include_volumes=schedule.include_volumes, stop_first=schedule.stop_first,
|
|
)
|
|
filename = backup_service.backup_filename(stack_id, schedule.include_volumes)
|
|
await asyncio.to_thread(dest_service.upload, dest, path, filename)
|
|
problems = _problems(report)
|
|
if problems:
|
|
# Retention only after a complete backup: an incomplete one must
|
|
# not push the last good backup out.
|
|
return f"warning: {'; '.join(problems)}"
|
|
await asyncio.to_thread(_prune, dest, stack_id, schedule.keep)
|
|
return "ok"
|
|
except backup_service.StackBusyError as exc:
|
|
raise _Busy(str(exc)) from exc
|
|
except backup_service.BackupCancelled as exc:
|
|
# Somebody stopped this backup from the UI. Not a failure of the
|
|
# schedule: say so, and don't page anyone with "backup failed".
|
|
logger.info("Scheduled backup of %s was cancelled: %s", stack_id, exc)
|
|
return f"cancelled: {exc}"
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Scheduled backup of %s failed: %s", stack_id, exc)
|
|
return f"error: {exc}"
|
|
finally:
|
|
if path and os.path.exists(path):
|
|
os.unlink(path)
|
|
|
|
|
|
def _summarise(statuses: dict[str, str]) -> str:
|
|
"""One status line for the schedule from the per-stack ones."""
|
|
if not statuses:
|
|
return "error: no stacks to back up"
|
|
bad = {sid: st for sid, st in statuses.items() if st != "ok"}
|
|
if not bad:
|
|
return "ok"
|
|
worst = (
|
|
"error"
|
|
if any(not st.startswith(("warning", "cancelled")) for st in bad.values())
|
|
else "warning"
|
|
)
|
|
if len(statuses) == 1:
|
|
return next(iter(statuses.values()))
|
|
detail = "; ".join(f"{sid}: {st}" for sid, st in sorted(bad.items()))
|
|
return f"{worst}: {len(bad)} of {len(statuses)} stacks — {detail}"
|
|
|
|
|
|
async def run_schedule(session: Session, schedule: BackupSchedule) -> dict:
|
|
"""Execute one schedule now. Updates status + next_run. Returns a summary."""
|
|
if schedule.id in _running:
|
|
return {"ok": False, "error": "this schedule is already running"}
|
|
_running.add(schedule.id)
|
|
now = _now()
|
|
statuses: dict[str, str] = {}
|
|
dest = session.get(BackupDestination, schedule.destination_id)
|
|
try:
|
|
stack_ids = resolve_stacks(session, schedule)
|
|
if not dest:
|
|
statuses = {sid: f"error: destination {schedule.destination_id} not found"
|
|
for sid in stack_ids}
|
|
else:
|
|
pending = list(stack_ids)
|
|
for attempt in range(BUSY_ATTEMPTS + 1):
|
|
busy: list[str] = []
|
|
for stack_id in pending:
|
|
try:
|
|
statuses[stack_id] = await _backup_stack(session, schedule, dest, stack_id)
|
|
except _Busy as exc:
|
|
busy.append(stack_id)
|
|
statuses[stack_id] = f"error: {exc}"
|
|
if not busy or attempt == BUSY_ATTEMPTS:
|
|
break
|
|
# Deploys and updates are short: wait, then take the busy
|
|
# stacks again rather than skipping them until the next run.
|
|
pending = busy
|
|
await asyncio.sleep(BUSY_WAIT_SECONDS)
|
|
|
|
schedule.last_status = _summarise(statuses)[:300]
|
|
problems = {sid: st for sid, st in statuses.items() if st != "ok"}
|
|
if problems or not statuses:
|
|
await _alert(
|
|
session,
|
|
f"Scheduled backup to '{dest.name if dest else schedule.destination_id}' "
|
|
+ ("failed" if schedule.last_status.startswith("error") else "is incomplete"),
|
|
"\n".join(f"{sid}: {st}" for sid, st in sorted(problems.items()))
|
|
or schedule.last_status,
|
|
)
|
|
logger.info("Scheduled backup %s: %s", schedule.id, schedule.last_status)
|
|
return {
|
|
"ok": schedule.last_status == "ok",
|
|
"destination": dest.name if dest else None,
|
|
"stacks": statuses,
|
|
**({} if schedule.last_status == "ok" else {"error": schedule.last_status}),
|
|
}
|
|
finally:
|
|
_running.discard(schedule.id)
|
|
schedule.stack_status = json.dumps(
|
|
{sid: {"status": st[:300], "at": now.isoformat()} for sid, st in statuses.items()}
|
|
)
|
|
schedule.last_run = now
|
|
schedule.next_run = compute_next_run(
|
|
schedule.frequency, schedule.hour, schedule.minute, schedule.weekday, now,
|
|
schedule.timezone,
|
|
)
|
|
session.add(schedule)
|
|
session.commit()
|
|
session.refresh(schedule)
|
|
|
|
|
|
async def run_in_background(schedule_id: int) -> None:
|
|
"""Run a schedule with its own session, for "Run now": a run over many
|
|
stacks takes far longer than a request may."""
|
|
with Session(engine) as session:
|
|
schedule = session.get(BackupSchedule, schedule_id)
|
|
if schedule:
|
|
await run_schedule(session, schedule)
|
|
|
|
|
|
async def run_due() -> None:
|
|
now = _now()
|
|
with Session(engine) as session:
|
|
schedules = session.exec(
|
|
select(BackupSchedule).where(BackupSchedule.enabled == True) # noqa: E712
|
|
).all()
|
|
due = []
|
|
for s in schedules:
|
|
if s.next_run is None:
|
|
s.next_run = compute_next_run(
|
|
s.frequency, s.hour, s.minute, s.weekday, now, s.timezone
|
|
)
|
|
session.add(s)
|
|
elif s.next_run <= now:
|
|
due.append(s)
|
|
session.commit()
|
|
for s in due:
|
|
if not is_running(s.id):
|
|
await run_schedule(session, s)
|
|
|
|
|
|
async def scheduler_loop() -> None:
|
|
await asyncio.sleep(20) # let startup settle
|
|
while True:
|
|
try:
|
|
await run_due()
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("scheduler tick failed: %s", exc)
|
|
await asyncio.sleep(60)
|