menzeljandClaude Opus 5
8a0d673798
A schedule covered exactly one stack, so backing up a host meant one
schedule per stack, each edited separately, and new stacks were only
backed up once someone remembered to add a schedule.
A schedule now takes a list of stacks, or all_stacks, which includes
stacks added later. They are backed up one after another, each with its
own file, upload, retention and status, so one failing stack does not
cost the others their backup:
- stack_status records each stack's result; last_status summarises
("error: 2 of 5 stacks - a: ...; c: ...") and one notification per run
names the stacks with a problem. The dashboard names them too and shows
an incomplete backup as a warning rather than an error.
- A busy stack (deploy, update) is retried after the others, up to three
times two minutes apart, replacing the 10-minute postponement of the
whole schedule.
- Run now starts the run in the background (202) instead of holding the
request, which a multi-stack run would outlast; the list reports
running and polls, and the row toasts the outcome. A schedule never
runs twice at once.
- Settings: "All stacks" or a checklist with select all/none; stacks can
be changed when editing; deleted stacks stay listed to be unticked.
Schedules from before keep working without a data migration: stack_ids,
all_stacks and stack_status are added with defaults on startup and a row
with only stack_id reads as that one stack. The API still accepts
stack_id and reports it for single-stack schedules.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
312 lines
11 KiB
Python
312 lines
11 KiB
Python
"""Scheduled backups over several stacks.
|
|
|
|
A schedule covers a list of stacks, or all of them (new stacks included
|
|
without editing it). Each stack gets its own backup, upload, retention and
|
|
status, one after the other: a failing or busy stack must not cost the others
|
|
their backup, and the schedule's status must say which stack had a problem.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
from services import backup_service, compose_service
|
|
|
|
|
|
def run(coro):
|
|
import asyncio
|
|
|
|
return asyncio.run(coro)
|
|
|
|
|
|
class Env:
|
|
"""Stacks with scripted outcomes, a destination and the fakes around them."""
|
|
|
|
def __init__(self, monkeypatch, db, outcomes: dict[str, list]):
|
|
from sqlmodel import Session
|
|
|
|
from models.backup_destination import BackupDestination
|
|
from models.stack import Stack
|
|
from services import schedule_service
|
|
|
|
self.svc = schedule_service
|
|
self.outcomes = outcomes # stack -> list of "ok" | "incomplete" | "busy" | Exception
|
|
self.backed_up: list[str] = []
|
|
self.uploaded: list[str] = []
|
|
self.pruned: list[str] = []
|
|
self.alerts: list[tuple[str, str]] = []
|
|
self.sleeps: list[float] = []
|
|
|
|
async def fake_create(stack_id, name, **kwargs):
|
|
outcome = self.outcomes[stack_id].pop(0) if len(self.outcomes[stack_id]) > 1 \
|
|
else self.outcomes[stack_id][0]
|
|
if outcome == "busy":
|
|
raise backup_service.StackBusyError(f"Stack '{stack_id}' is busy: update in progress")
|
|
if isinstance(outcome, Exception):
|
|
raise outcome
|
|
self.backed_up.append(stack_id)
|
|
path = os.path.join(compose_service.stacks_root(), f".{stack_id}-{uuid.uuid4().hex}.tgz")
|
|
open(path, "wb").close()
|
|
errors = ["db: helper failed"] if outcome == "incomplete" else []
|
|
return path, {"errors": errors, "restart_error": None}
|
|
|
|
async def fake_notify(event, title, body, session):
|
|
self.alerts.append((title, body))
|
|
|
|
async def fake_sleep(seconds):
|
|
self.sleeps.append(seconds)
|
|
|
|
monkeypatch.setattr(schedule_service.backup_service, "create_backup_ex", fake_create)
|
|
monkeypatch.setattr(schedule_service.dest_service, "upload",
|
|
lambda d, p, f: self.uploaded.append(f) or f)
|
|
monkeypatch.setattr(schedule_service, "_prune",
|
|
lambda d, s, k: self.pruned.append(s) or 0)
|
|
monkeypatch.setattr(schedule_service.notify_service, "notify", fake_notify)
|
|
monkeypatch.setattr(schedule_service.asyncio, "sleep", fake_sleep)
|
|
|
|
self.session = Session(db)
|
|
prefix = uuid.uuid4().hex[:6]
|
|
self.ids = {name: f"{prefix}-{name}" for name in outcomes}
|
|
self.outcomes = {self.ids[n]: list(o) for n, o in outcomes.items()}
|
|
for sid in self.ids.values():
|
|
self.session.add(Stack(id=sid, name=sid))
|
|
self.dest = BackupDestination(name=f"nas-{prefix}", type="sftp", config="{}")
|
|
self.session.add(self.dest)
|
|
self.session.commit()
|
|
|
|
def schedule(self, names=None, all_stacks=False, legacy=None):
|
|
from models.backup_schedule import BackupSchedule, set_stacks
|
|
|
|
s = BackupSchedule(destination_id=self.dest.id, frequency="daily", all_stacks=all_stacks)
|
|
if legacy:
|
|
s.stack_id = self.ids[legacy] # a row from before multi-stack schedules
|
|
else:
|
|
set_stacks(s, [self.ids[n] for n in names or []])
|
|
self.session.add(s)
|
|
self.session.commit()
|
|
return s
|
|
|
|
def status(self, schedule):
|
|
from models.backup_schedule import status_of
|
|
|
|
return {sid: v["status"] for sid, v in status_of(schedule).items()}
|
|
|
|
|
|
def test_every_chosen_stack_is_backed_up_on_its_own(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"], "b": ["ok"], "c": ["ok"]})
|
|
schedule = env.schedule(["a", "c"])
|
|
|
|
result = run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
a, c = env.ids["a"], env.ids["c"]
|
|
assert result["ok"] is True and schedule.last_status == "ok"
|
|
assert sorted(env.backed_up) == sorted([a, c])
|
|
assert sorted(env.pruned) == sorted([a, c])
|
|
assert len(env.uploaded) == 2 and all(f.startswith("backup-") for f in env.uploaded)
|
|
assert env.status(schedule) == {a: "ok", c: "ok"}
|
|
assert env.alerts == []
|
|
|
|
|
|
def test_all_stacks_includes_stacks_added_later(monkeypatch, db):
|
|
from sqlmodel import select
|
|
|
|
from models.stack import Stack
|
|
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(all_stacks=True)
|
|
late = f"{env.ids['a']}-late"
|
|
env.session.add(Stack(id=late, name=late))
|
|
env.session.commit()
|
|
env.outcomes[late] = ["ok"]
|
|
# Other tests' stacks share the database: back them up as fine too.
|
|
for stack in env.session.exec(select(Stack)).all():
|
|
env.outcomes.setdefault(stack.id, ["ok"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert {env.ids["a"], late} <= set(env.backed_up)
|
|
|
|
|
|
def test_one_failing_stack_does_not_cost_the_others(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": [RuntimeError("disk full")], "b": ["ok"], "c": ["incomplete"]})
|
|
schedule = env.schedule(["a", "b", "c"])
|
|
|
|
result = run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
a, b, c = env.ids["a"], env.ids["b"], env.ids["c"]
|
|
assert env.backed_up == [b, c]
|
|
assert env.pruned == [b] # incomplete: older backups of c are kept
|
|
status = env.status(schedule)
|
|
assert status[a] == "error: disk full"
|
|
assert status[b] == "ok"
|
|
assert status[c].startswith("warning: not captured")
|
|
assert schedule.last_status.startswith("error: 2 of 3 stacks")
|
|
assert a in schedule.last_status and c in schedule.last_status
|
|
assert result["ok"] is False
|
|
assert len(env.alerts) == 1 and "failed" in env.alerts[0][0]
|
|
assert a in env.alerts[0][1] and c in env.alerts[0][1]
|
|
|
|
|
|
def test_only_incomplete_stacks_make_a_warning(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"], "b": ["incomplete"]})
|
|
schedule = env.schedule(["a", "b"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert schedule.last_status.startswith("warning: 1 of 2 stacks")
|
|
assert "incomplete" in env.alerts[0][0]
|
|
|
|
|
|
def test_busy_stack_is_retried_after_the_others(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["busy", "ok"], "b": ["ok"]})
|
|
schedule = env.schedule(["a", "b"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
a, b = env.ids["a"], env.ids["b"]
|
|
assert env.backed_up == [b, a]
|
|
assert env.sleeps == [env.svc.BUSY_WAIT_SECONDS]
|
|
assert schedule.last_status == "ok" and env.alerts == []
|
|
|
|
|
|
def test_stack_busy_every_time_is_reported(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["busy"]})
|
|
schedule = env.schedule(["a"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert len(env.sleeps) == env.svc.BUSY_ATTEMPTS
|
|
assert "busy" in schedule.last_status and schedule.last_status.startswith("error:")
|
|
assert env.alerts
|
|
|
|
|
|
def test_schedule_from_before_multi_stack_still_runs(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(legacy="a")
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert env.backed_up == [env.ids["a"]]
|
|
|
|
|
|
def test_deleted_stack_is_an_error_for_that_stack_only(monkeypatch, db):
|
|
from models.backup_schedule import set_stacks
|
|
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(["a"])
|
|
set_stacks(schedule, [env.ids["a"], "gone-stack"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert env.status(schedule)["gone-stack"] == "error: stack not found"
|
|
assert env.status(schedule)[env.ids["a"]] == "ok"
|
|
|
|
|
|
def test_a_running_schedule_is_not_started_twice(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(["a"])
|
|
env.svc._running.add(schedule.id)
|
|
try:
|
|
result = run(env.svc.run_schedule(env.session, schedule))
|
|
finally:
|
|
env.svc._running.discard(schedule.id)
|
|
assert result["ok"] is False and env.backed_up == []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# API
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.fixture
|
|
def api_env(monkeypatch, db, as_admin):
|
|
env = Env(monkeypatch, db, {"a": ["ok"], "b": ["ok"]})
|
|
return env, as_admin
|
|
|
|
|
|
def test_create_with_several_stacks(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["b"], env.ids["a"]], "destination_id": env.dest.id,
|
|
})
|
|
assert r.status_code == 201
|
|
body = r.json()
|
|
assert body["stack_ids"] == sorted([env.ids["a"], env.ids["b"]])
|
|
assert body["all_stacks"] is False and body["stack_id"] == ""
|
|
assert body["running"] is False and body["stack_status"] == {}
|
|
|
|
|
|
def test_create_for_all_stacks_needs_no_list(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={
|
|
"all_stacks": True, "destination_id": env.dest.id,
|
|
})
|
|
assert r.status_code == 201 and r.json()["all_stacks"] is True
|
|
|
|
|
|
def test_old_single_stack_request_still_works(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={
|
|
"stack_id": env.ids["a"], "destination_id": env.dest.id,
|
|
})
|
|
assert r.status_code == 201
|
|
assert r.json()["stack_ids"] == [env.ids["a"]] and r.json()["stack_id"] == env.ids["a"]
|
|
|
|
|
|
def test_no_stacks_or_unknown_stacks_are_refused(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={"destination_id": env.dest.id})
|
|
assert r.status_code == 400
|
|
r = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["a"], "nope"], "destination_id": env.dest.id,
|
|
})
|
|
assert r.status_code == 404 and "nope" in r.json()["detail"]
|
|
|
|
|
|
def test_edit_changes_the_stacks(api_env):
|
|
env, client = api_env
|
|
created = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["a"]], "destination_id": env.dest.id,
|
|
}).json()
|
|
r = client.put(f"/api/backups/schedules/{created['id']}", json={
|
|
"stack_ids": [env.ids["a"], env.ids["b"]],
|
|
})
|
|
assert r.status_code == 200
|
|
assert r.json()["stack_ids"] == sorted([env.ids["a"], env.ids["b"]])
|
|
r = client.put(f"/api/backups/schedules/{created['id']}", json={"all_stacks": True})
|
|
assert r.json()["all_stacks"] is True
|
|
|
|
|
|
def test_run_now_starts_in_the_background(api_env, monkeypatch):
|
|
env, client = api_env
|
|
created = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["a"]], "destination_id": env.dest.id,
|
|
}).json()
|
|
started: list[int] = []
|
|
|
|
async def fake_run(schedule_id):
|
|
started.append(schedule_id)
|
|
|
|
monkeypatch.setattr(env.svc, "run_in_background", fake_run)
|
|
r = client.post(f"/api/backups/schedules/{created['id']}/run")
|
|
assert r.status_code == 202 and r.json() == {"started": True}
|
|
|
|
env.svc._running.add(created["id"])
|
|
try:
|
|
assert client.post(f"/api/backups/schedules/{created['id']}/run").status_code == 409
|
|
listed = {s["id"]: s for s in client.get("/api/backups/schedules").json()}
|
|
assert listed[created["id"]]["running"] is True
|
|
finally:
|
|
env.svc._running.discard(created["id"])
|
|
|
|
|
|
def test_stack_status_is_stored_as_json(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(["a"])
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
stored = json.loads(schedule.stack_status)
|
|
assert stored[env.ids["a"]]["status"] == "ok" and "at" in stored[env.ids["a"]]
|