"""Scheduled backups over several stacks. A schedule covers a list of stacks, or all of them (new stacks included without editing it). Each stack gets its own backup, upload, retention and status, one after the other: a failing or busy stack must not cost the others their backup, and the schedule's status must say which stack had a problem. """ from __future__ import annotations import json import os import uuid import pytest from services import backup_service, compose_service def run(coro): import asyncio return asyncio.run(coro) class Env: """Stacks with scripted outcomes, a destination and the fakes around them.""" def __init__(self, monkeypatch, db, outcomes: dict[str, list]): from sqlmodel import Session from models.backup_destination import BackupDestination from models.stack import Stack from services import schedule_service self.svc = schedule_service self.outcomes = outcomes # stack -> list of "ok" | "incomplete" | "busy" | Exception self.backed_up: list[str] = [] self.uploaded: list[str] = [] self.pruned: list[str] = [] self.alerts: list[tuple[str, str]] = [] self.sleeps: list[float] = [] async def fake_create(stack_id, name, **kwargs): outcome = self.outcomes[stack_id].pop(0) if len(self.outcomes[stack_id]) > 1 \ else self.outcomes[stack_id][0] if outcome == "busy": raise backup_service.StackBusyError(f"Stack '{stack_id}' is busy: update in progress") if isinstance(outcome, Exception): raise outcome self.backed_up.append(stack_id) path = os.path.join(compose_service.stacks_root(), f".{stack_id}-{uuid.uuid4().hex}.tgz") open(path, "wb").close() errors = ["db: helper failed"] if outcome == "incomplete" else [] return path, {"errors": errors, "restart_error": None} async def fake_notify(event, title, body, session): self.alerts.append((title, body)) async def fake_sleep(seconds): self.sleeps.append(seconds) monkeypatch.setattr(schedule_service.backup_service, "create_backup_ex", fake_create) monkeypatch.setattr(schedule_service.dest_service, "upload", lambda d, p, f: self.uploaded.append(f) or f) monkeypatch.setattr(schedule_service, "_prune", lambda d, s, k: self.pruned.append(s) or 0) monkeypatch.setattr(schedule_service.notify_service, "notify", fake_notify) monkeypatch.setattr(schedule_service.asyncio, "sleep", fake_sleep) self.session = Session(db) prefix = uuid.uuid4().hex[:6] self.ids = {name: f"{prefix}-{name}" for name in outcomes} self.outcomes = {self.ids[n]: list(o) for n, o in outcomes.items()} for sid in self.ids.values(): self.session.add(Stack(id=sid, name=sid)) self.dest = BackupDestination(name=f"nas-{prefix}", type="sftp", config="{}") self.session.add(self.dest) self.session.commit() def schedule(self, names=None, all_stacks=False, legacy=None): from models.backup_schedule import BackupSchedule, set_stacks s = BackupSchedule(destination_id=self.dest.id, frequency="daily", all_stacks=all_stacks) if legacy: s.stack_id = self.ids[legacy] # a row from before multi-stack schedules else: set_stacks(s, [self.ids[n] for n in names or []]) self.session.add(s) self.session.commit() return s def status(self, schedule): from models.backup_schedule import status_of return {sid: v["status"] for sid, v in status_of(schedule).items()} def test_every_chosen_stack_is_backed_up_on_its_own(monkeypatch, db): env = Env(monkeypatch, db, {"a": ["ok"], "b": ["ok"], "c": ["ok"]}) schedule = env.schedule(["a", "c"]) result = run(env.svc.run_schedule(env.session, schedule)) a, c = env.ids["a"], env.ids["c"] assert result["ok"] is True and schedule.last_status == "ok" assert sorted(env.backed_up) == sorted([a, c]) assert sorted(env.pruned) == sorted([a, c]) assert len(env.uploaded) == 2 and all(f.startswith("backup-") for f in env.uploaded) assert env.status(schedule) == {a: "ok", c: "ok"} assert env.alerts == [] def test_all_stacks_includes_stacks_added_later(monkeypatch, db): from sqlmodel import select from models.stack import Stack env = Env(monkeypatch, db, {"a": ["ok"]}) schedule = env.schedule(all_stacks=True) late = f"{env.ids['a']}-late" env.session.add(Stack(id=late, name=late)) env.session.commit() env.outcomes[late] = ["ok"] # Other tests' stacks share the database: back them up as fine too. for stack in env.session.exec(select(Stack)).all(): env.outcomes.setdefault(stack.id, ["ok"]) run(env.svc.run_schedule(env.session, schedule)) assert {env.ids["a"], late} <= set(env.backed_up) def test_one_failing_stack_does_not_cost_the_others(monkeypatch, db): env = Env(monkeypatch, db, {"a": [RuntimeError("disk full")], "b": ["ok"], "c": ["incomplete"]}) schedule = env.schedule(["a", "b", "c"]) result = run(env.svc.run_schedule(env.session, schedule)) a, b, c = env.ids["a"], env.ids["b"], env.ids["c"] assert env.backed_up == [b, c] assert env.pruned == [b] # incomplete: older backups of c are kept status = env.status(schedule) assert status[a] == "error: disk full" assert status[b] == "ok" assert status[c].startswith("warning: not captured") assert schedule.last_status.startswith("error: 2 of 3 stacks") assert a in schedule.last_status and c in schedule.last_status assert result["ok"] is False assert len(env.alerts) == 1 and "failed" in env.alerts[0][0] assert a in env.alerts[0][1] and c in env.alerts[0][1] def test_only_incomplete_stacks_make_a_warning(monkeypatch, db): env = Env(monkeypatch, db, {"a": ["ok"], "b": ["incomplete"]}) schedule = env.schedule(["a", "b"]) run(env.svc.run_schedule(env.session, schedule)) assert schedule.last_status.startswith("warning: 1 of 2 stacks") assert "incomplete" in env.alerts[0][0] def test_busy_stack_is_retried_after_the_others(monkeypatch, db): env = Env(monkeypatch, db, {"a": ["busy", "ok"], "b": ["ok"]}) schedule = env.schedule(["a", "b"]) run(env.svc.run_schedule(env.session, schedule)) a, b = env.ids["a"], env.ids["b"] assert env.backed_up == [b, a] assert env.sleeps == [env.svc.BUSY_WAIT_SECONDS] assert schedule.last_status == "ok" and env.alerts == [] def test_stack_busy_every_time_is_reported(monkeypatch, db): env = Env(monkeypatch, db, {"a": ["busy"]}) schedule = env.schedule(["a"]) run(env.svc.run_schedule(env.session, schedule)) assert len(env.sleeps) == env.svc.BUSY_ATTEMPTS assert "busy" in schedule.last_status and schedule.last_status.startswith("error:") assert env.alerts def test_schedule_from_before_multi_stack_still_runs(monkeypatch, db): env = Env(monkeypatch, db, {"a": ["ok"]}) schedule = env.schedule(legacy="a") run(env.svc.run_schedule(env.session, schedule)) assert env.backed_up == [env.ids["a"]] def test_deleted_stack_is_an_error_for_that_stack_only(monkeypatch, db): from models.backup_schedule import set_stacks env = Env(monkeypatch, db, {"a": ["ok"]}) schedule = env.schedule(["a"]) set_stacks(schedule, [env.ids["a"], "gone-stack"]) run(env.svc.run_schedule(env.session, schedule)) assert env.status(schedule)["gone-stack"] == "error: stack not found" assert env.status(schedule)[env.ids["a"]] == "ok" def test_a_running_schedule_is_not_started_twice(monkeypatch, db): env = Env(monkeypatch, db, {"a": ["ok"]}) schedule = env.schedule(["a"]) env.svc._running.add(schedule.id) try: result = run(env.svc.run_schedule(env.session, schedule)) finally: env.svc._running.discard(schedule.id) assert result["ok"] is False and env.backed_up == [] # --------------------------------------------------------------------------- # # API # --------------------------------------------------------------------------- # @pytest.fixture def api_env(monkeypatch, db, as_admin): env = Env(monkeypatch, db, {"a": ["ok"], "b": ["ok"]}) return env, as_admin def test_create_with_several_stacks(api_env): env, client = api_env r = client.post("/api/backups/schedules", json={ "stack_ids": [env.ids["b"], env.ids["a"]], "destination_id": env.dest.id, }) assert r.status_code == 201 body = r.json() assert body["stack_ids"] == sorted([env.ids["a"], env.ids["b"]]) assert body["all_stacks"] is False and body["stack_id"] == "" assert body["running"] is False and body["stack_status"] == {} def test_create_for_all_stacks_needs_no_list(api_env): env, client = api_env r = client.post("/api/backups/schedules", json={ "all_stacks": True, "destination_id": env.dest.id, }) assert r.status_code == 201 and r.json()["all_stacks"] is True def test_old_single_stack_request_still_works(api_env): env, client = api_env r = client.post("/api/backups/schedules", json={ "stack_id": env.ids["a"], "destination_id": env.dest.id, }) assert r.status_code == 201 assert r.json()["stack_ids"] == [env.ids["a"]] and r.json()["stack_id"] == env.ids["a"] def test_no_stacks_or_unknown_stacks_are_refused(api_env): env, client = api_env r = client.post("/api/backups/schedules", json={"destination_id": env.dest.id}) assert r.status_code == 400 r = client.post("/api/backups/schedules", json={ "stack_ids": [env.ids["a"], "nope"], "destination_id": env.dest.id, }) assert r.status_code == 404 and "nope" in r.json()["detail"] def test_edit_changes_the_stacks(api_env): env, client = api_env created = client.post("/api/backups/schedules", json={ "stack_ids": [env.ids["a"]], "destination_id": env.dest.id, }).json() r = client.put(f"/api/backups/schedules/{created['id']}", json={ "stack_ids": [env.ids["a"], env.ids["b"]], }) assert r.status_code == 200 assert r.json()["stack_ids"] == sorted([env.ids["a"], env.ids["b"]]) r = client.put(f"/api/backups/schedules/{created['id']}", json={"all_stacks": True}) assert r.json()["all_stacks"] is True def test_run_now_starts_in_the_background(api_env, monkeypatch): env, client = api_env created = client.post("/api/backups/schedules", json={ "stack_ids": [env.ids["a"]], "destination_id": env.dest.id, }).json() started: list[int] = [] async def fake_run(schedule_id): started.append(schedule_id) monkeypatch.setattr(env.svc, "run_in_background", fake_run) r = client.post(f"/api/backups/schedules/{created['id']}/run") assert r.status_code == 202 and r.json() == {"started": True} env.svc._running.add(created["id"]) try: assert client.post(f"/api/backups/schedules/{created['id']}/run").status_code == 409 listed = {s["id"]: s for s in client.get("/api/backups/schedules").json()} assert listed[created["id"]]["running"] is True finally: env.svc._running.discard(created["id"]) def test_stack_status_is_stored_as_json(monkeypatch, db): env = Env(monkeypatch, db, {"a": ["ok"]}) schedule = env.schedule(["a"]) run(env.svc.run_schedule(env.session, schedule)) stored = json.loads(schedule.stack_status) assert stored[env.ids["a"]]["status"] == "ok" and "at" in stored[env.ids["a"]] # --------------------------------------------------------------------------- # # Local time # --------------------------------------------------------------------------- # def _next(freq, hour, minute, weekday, after, tz): from datetime import datetime from services import schedule_service return schedule_service.compute_next_run( freq, hour, minute, weekday, datetime.fromisoformat(after), tz ).isoformat() def test_daily_time_is_wall_clock_time_in_the_schedule_zone(): # 03:00 in Berlin is 02:00 UTC in winter... assert _next("daily", 3, 0, 0, "2026-01-10T12:00", "Europe/Berlin") == "2026-01-11T02:00:00" # ...and 01:00 UTC in summer. assert _next("daily", 3, 0, 0, "2026-07-10T12:00", "Europe/Berlin") == "2026-07-11T01:00:00" def test_daily_time_stays_put_across_daylight_saving_changes(): # Clocks go forward on 29 March 2026 and back on 25 October 2026. assert _next("daily", 3, 0, 0, "2026-03-28T02:00", "Europe/Berlin") == "2026-03-29T01:00:00" assert _next("daily", 3, 0, 0, "2026-03-29T01:00", "Europe/Berlin") == "2026-03-30T01:00:00" assert _next("daily", 3, 0, 0, "2026-10-24T01:00", "Europe/Berlin") == "2026-10-25T02:00:00" def test_a_time_skipped_by_the_clock_change_runs_an_hour_later(): # 02:30 does not exist on 29 March in Berlin; it runs at 03:30 local. assert _next("daily", 2, 30, 0, "2026-03-28T12:00", "Europe/Berlin") == "2026-03-29T01:30:00" def test_weekly_is_on_the_local_weekday(): # Sunday 00:30 in Berlin is still Saturday in UTC. assert _next("weekly", 0, 30, 6, "2026-01-05T12:00", "Europe/Berlin") == "2026-01-10T23:30:00" def test_hourly_minute_is_local_minute(): # India is UTC+5:30, so :00 local is :30 UTC. assert _next("hourly", 0, 0, 0, "2026-01-10T12:00", "Asia/Kolkata") == "2026-01-10T12:30:00" assert _next("hourly", 0, 15, 0, "2026-01-10T12:00", "UTC") == "2026-01-10T12:15:00" def test_utc_is_the_default_and_unknown_zones_fall_back_to_it(): assert _next("daily", 3, 0, 0, "2026-01-10T12:00", None) == "2026-01-11T03:00:00" assert _next("daily", 3, 0, 0, "2026-01-10T12:00", "Mars/Olympus") == "2026-01-11T03:00:00" def test_api_keeps_the_zone_and_refuses_bad_values(api_env): env, client = api_env r = client.post("/api/backups/schedules", json={ "stack_ids": [env.ids["a"]], "destination_id": env.dest.id, "hour": 3, "minute": 0, "timezone": "Europe/Berlin", }) assert r.status_code == 201 body = r.json() assert body["timezone"] == "Europe/Berlin" from datetime import datetime from zoneinfo import ZoneInfo local = datetime.fromisoformat(body["next_run"]).replace(tzinfo=ZoneInfo("UTC")).astimezone( ZoneInfo("Europe/Berlin") ) assert (local.hour, local.minute) == (3, 0) r = client.put(f"/api/backups/schedules/{body['id']}", json={"timezone": "Mars/Olympus"}) assert r.status_code == 400 r = client.put(f"/api/backups/schedules/{body['id']}", json={"hour": 24}) assert r.status_code == 400