menzeljandClaude Opus 5
9a357d2b92
Schedule times were UTC, so a nightly 03:00 backup had to be entered as 01:00 or 02:00 depending on the season, and moved by an hour twice a year. Each schedule now has an IANA time zone, preset in the form to the browser's (e.g. Europe/Berlin), and its hour/minute/weekday are wall-clock time there. compute_next_run builds each candidate day from its date in that zone rather than adding 24 hours, so the time stays put across daylight saving changes; a time skipped by the spring change runs an hour later that day. next_run is still stored as naive UTC. - Existing schedules default to UTC and keep their run times until edited. - The API validates the zone and hour/minute/weekday ranges (400) instead of silently wrapping them. - tzdata is a dependency: the slim base image has no system tz database. Tests cover Berlin winter and summer time, both 2026 change days, the skipped hour, a weekly run across the UTC date line, hourly at a +5:30 offset, the UTC fallback and the API checks. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
385 lines
14 KiB
Python
385 lines
14 KiB
Python
"""Scheduled backups over several stacks.
|
|
|
|
A schedule covers a list of stacks, or all of them (new stacks included
|
|
without editing it). Each stack gets its own backup, upload, retention and
|
|
status, one after the other: a failing or busy stack must not cost the others
|
|
their backup, and the schedule's status must say which stack had a problem.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
from services import backup_service, compose_service
|
|
|
|
|
|
def run(coro):
|
|
import asyncio
|
|
|
|
return asyncio.run(coro)
|
|
|
|
|
|
class Env:
|
|
"""Stacks with scripted outcomes, a destination and the fakes around them."""
|
|
|
|
def __init__(self, monkeypatch, db, outcomes: dict[str, list]):
|
|
from sqlmodel import Session
|
|
|
|
from models.backup_destination import BackupDestination
|
|
from models.stack import Stack
|
|
from services import schedule_service
|
|
|
|
self.svc = schedule_service
|
|
self.outcomes = outcomes # stack -> list of "ok" | "incomplete" | "busy" | Exception
|
|
self.backed_up: list[str] = []
|
|
self.uploaded: list[str] = []
|
|
self.pruned: list[str] = []
|
|
self.alerts: list[tuple[str, str]] = []
|
|
self.sleeps: list[float] = []
|
|
|
|
async def fake_create(stack_id, name, **kwargs):
|
|
outcome = self.outcomes[stack_id].pop(0) if len(self.outcomes[stack_id]) > 1 \
|
|
else self.outcomes[stack_id][0]
|
|
if outcome == "busy":
|
|
raise backup_service.StackBusyError(f"Stack '{stack_id}' is busy: update in progress")
|
|
if isinstance(outcome, Exception):
|
|
raise outcome
|
|
self.backed_up.append(stack_id)
|
|
path = os.path.join(compose_service.stacks_root(), f".{stack_id}-{uuid.uuid4().hex}.tgz")
|
|
open(path, "wb").close()
|
|
errors = ["db: helper failed"] if outcome == "incomplete" else []
|
|
return path, {"errors": errors, "restart_error": None}
|
|
|
|
async def fake_notify(event, title, body, session):
|
|
self.alerts.append((title, body))
|
|
|
|
async def fake_sleep(seconds):
|
|
self.sleeps.append(seconds)
|
|
|
|
monkeypatch.setattr(schedule_service.backup_service, "create_backup_ex", fake_create)
|
|
monkeypatch.setattr(schedule_service.dest_service, "upload",
|
|
lambda d, p, f: self.uploaded.append(f) or f)
|
|
monkeypatch.setattr(schedule_service, "_prune",
|
|
lambda d, s, k: self.pruned.append(s) or 0)
|
|
monkeypatch.setattr(schedule_service.notify_service, "notify", fake_notify)
|
|
monkeypatch.setattr(schedule_service.asyncio, "sleep", fake_sleep)
|
|
|
|
self.session = Session(db)
|
|
prefix = uuid.uuid4().hex[:6]
|
|
self.ids = {name: f"{prefix}-{name}" for name in outcomes}
|
|
self.outcomes = {self.ids[n]: list(o) for n, o in outcomes.items()}
|
|
for sid in self.ids.values():
|
|
self.session.add(Stack(id=sid, name=sid))
|
|
self.dest = BackupDestination(name=f"nas-{prefix}", type="sftp", config="{}")
|
|
self.session.add(self.dest)
|
|
self.session.commit()
|
|
|
|
def schedule(self, names=None, all_stacks=False, legacy=None):
|
|
from models.backup_schedule import BackupSchedule, set_stacks
|
|
|
|
s = BackupSchedule(destination_id=self.dest.id, frequency="daily", all_stacks=all_stacks)
|
|
if legacy:
|
|
s.stack_id = self.ids[legacy] # a row from before multi-stack schedules
|
|
else:
|
|
set_stacks(s, [self.ids[n] for n in names or []])
|
|
self.session.add(s)
|
|
self.session.commit()
|
|
return s
|
|
|
|
def status(self, schedule):
|
|
from models.backup_schedule import status_of
|
|
|
|
return {sid: v["status"] for sid, v in status_of(schedule).items()}
|
|
|
|
|
|
def test_every_chosen_stack_is_backed_up_on_its_own(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"], "b": ["ok"], "c": ["ok"]})
|
|
schedule = env.schedule(["a", "c"])
|
|
|
|
result = run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
a, c = env.ids["a"], env.ids["c"]
|
|
assert result["ok"] is True and schedule.last_status == "ok"
|
|
assert sorted(env.backed_up) == sorted([a, c])
|
|
assert sorted(env.pruned) == sorted([a, c])
|
|
assert len(env.uploaded) == 2 and all(f.startswith("backup-") for f in env.uploaded)
|
|
assert env.status(schedule) == {a: "ok", c: "ok"}
|
|
assert env.alerts == []
|
|
|
|
|
|
def test_all_stacks_includes_stacks_added_later(monkeypatch, db):
|
|
from sqlmodel import select
|
|
|
|
from models.stack import Stack
|
|
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(all_stacks=True)
|
|
late = f"{env.ids['a']}-late"
|
|
env.session.add(Stack(id=late, name=late))
|
|
env.session.commit()
|
|
env.outcomes[late] = ["ok"]
|
|
# Other tests' stacks share the database: back them up as fine too.
|
|
for stack in env.session.exec(select(Stack)).all():
|
|
env.outcomes.setdefault(stack.id, ["ok"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert {env.ids["a"], late} <= set(env.backed_up)
|
|
|
|
|
|
def test_one_failing_stack_does_not_cost_the_others(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": [RuntimeError("disk full")], "b": ["ok"], "c": ["incomplete"]})
|
|
schedule = env.schedule(["a", "b", "c"])
|
|
|
|
result = run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
a, b, c = env.ids["a"], env.ids["b"], env.ids["c"]
|
|
assert env.backed_up == [b, c]
|
|
assert env.pruned == [b] # incomplete: older backups of c are kept
|
|
status = env.status(schedule)
|
|
assert status[a] == "error: disk full"
|
|
assert status[b] == "ok"
|
|
assert status[c].startswith("warning: not captured")
|
|
assert schedule.last_status.startswith("error: 2 of 3 stacks")
|
|
assert a in schedule.last_status and c in schedule.last_status
|
|
assert result["ok"] is False
|
|
assert len(env.alerts) == 1 and "failed" in env.alerts[0][0]
|
|
assert a in env.alerts[0][1] and c in env.alerts[0][1]
|
|
|
|
|
|
def test_only_incomplete_stacks_make_a_warning(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"], "b": ["incomplete"]})
|
|
schedule = env.schedule(["a", "b"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert schedule.last_status.startswith("warning: 1 of 2 stacks")
|
|
assert "incomplete" in env.alerts[0][0]
|
|
|
|
|
|
def test_busy_stack_is_retried_after_the_others(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["busy", "ok"], "b": ["ok"]})
|
|
schedule = env.schedule(["a", "b"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
a, b = env.ids["a"], env.ids["b"]
|
|
assert env.backed_up == [b, a]
|
|
assert env.sleeps == [env.svc.BUSY_WAIT_SECONDS]
|
|
assert schedule.last_status == "ok" and env.alerts == []
|
|
|
|
|
|
def test_stack_busy_every_time_is_reported(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["busy"]})
|
|
schedule = env.schedule(["a"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert len(env.sleeps) == env.svc.BUSY_ATTEMPTS
|
|
assert "busy" in schedule.last_status and schedule.last_status.startswith("error:")
|
|
assert env.alerts
|
|
|
|
|
|
def test_schedule_from_before_multi_stack_still_runs(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(legacy="a")
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert env.backed_up == [env.ids["a"]]
|
|
|
|
|
|
def test_deleted_stack_is_an_error_for_that_stack_only(monkeypatch, db):
|
|
from models.backup_schedule import set_stacks
|
|
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(["a"])
|
|
set_stacks(schedule, [env.ids["a"], "gone-stack"])
|
|
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
|
|
assert env.status(schedule)["gone-stack"] == "error: stack not found"
|
|
assert env.status(schedule)[env.ids["a"]] == "ok"
|
|
|
|
|
|
def test_a_running_schedule_is_not_started_twice(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(["a"])
|
|
env.svc._running.add(schedule.id)
|
|
try:
|
|
result = run(env.svc.run_schedule(env.session, schedule))
|
|
finally:
|
|
env.svc._running.discard(schedule.id)
|
|
assert result["ok"] is False and env.backed_up == []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# API
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@pytest.fixture
|
|
def api_env(monkeypatch, db, as_admin):
|
|
env = Env(monkeypatch, db, {"a": ["ok"], "b": ["ok"]})
|
|
return env, as_admin
|
|
|
|
|
|
def test_create_with_several_stacks(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["b"], env.ids["a"]], "destination_id": env.dest.id,
|
|
})
|
|
assert r.status_code == 201
|
|
body = r.json()
|
|
assert body["stack_ids"] == sorted([env.ids["a"], env.ids["b"]])
|
|
assert body["all_stacks"] is False and body["stack_id"] == ""
|
|
assert body["running"] is False and body["stack_status"] == {}
|
|
|
|
|
|
def test_create_for_all_stacks_needs_no_list(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={
|
|
"all_stacks": True, "destination_id": env.dest.id,
|
|
})
|
|
assert r.status_code == 201 and r.json()["all_stacks"] is True
|
|
|
|
|
|
def test_old_single_stack_request_still_works(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={
|
|
"stack_id": env.ids["a"], "destination_id": env.dest.id,
|
|
})
|
|
assert r.status_code == 201
|
|
assert r.json()["stack_ids"] == [env.ids["a"]] and r.json()["stack_id"] == env.ids["a"]
|
|
|
|
|
|
def test_no_stacks_or_unknown_stacks_are_refused(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={"destination_id": env.dest.id})
|
|
assert r.status_code == 400
|
|
r = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["a"], "nope"], "destination_id": env.dest.id,
|
|
})
|
|
assert r.status_code == 404 and "nope" in r.json()["detail"]
|
|
|
|
|
|
def test_edit_changes_the_stacks(api_env):
|
|
env, client = api_env
|
|
created = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["a"]], "destination_id": env.dest.id,
|
|
}).json()
|
|
r = client.put(f"/api/backups/schedules/{created['id']}", json={
|
|
"stack_ids": [env.ids["a"], env.ids["b"]],
|
|
})
|
|
assert r.status_code == 200
|
|
assert r.json()["stack_ids"] == sorted([env.ids["a"], env.ids["b"]])
|
|
r = client.put(f"/api/backups/schedules/{created['id']}", json={"all_stacks": True})
|
|
assert r.json()["all_stacks"] is True
|
|
|
|
|
|
def test_run_now_starts_in_the_background(api_env, monkeypatch):
|
|
env, client = api_env
|
|
created = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["a"]], "destination_id": env.dest.id,
|
|
}).json()
|
|
started: list[int] = []
|
|
|
|
async def fake_run(schedule_id):
|
|
started.append(schedule_id)
|
|
|
|
monkeypatch.setattr(env.svc, "run_in_background", fake_run)
|
|
r = client.post(f"/api/backups/schedules/{created['id']}/run")
|
|
assert r.status_code == 202 and r.json() == {"started": True}
|
|
|
|
env.svc._running.add(created["id"])
|
|
try:
|
|
assert client.post(f"/api/backups/schedules/{created['id']}/run").status_code == 409
|
|
listed = {s["id"]: s for s in client.get("/api/backups/schedules").json()}
|
|
assert listed[created["id"]]["running"] is True
|
|
finally:
|
|
env.svc._running.discard(created["id"])
|
|
|
|
|
|
def test_stack_status_is_stored_as_json(monkeypatch, db):
|
|
env = Env(monkeypatch, db, {"a": ["ok"]})
|
|
schedule = env.schedule(["a"])
|
|
run(env.svc.run_schedule(env.session, schedule))
|
|
stored = json.loads(schedule.stack_status)
|
|
assert stored[env.ids["a"]]["status"] == "ok" and "at" in stored[env.ids["a"]]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Local time
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _next(freq, hour, minute, weekday, after, tz):
|
|
from datetime import datetime
|
|
|
|
from services import schedule_service
|
|
|
|
return schedule_service.compute_next_run(
|
|
freq, hour, minute, weekday, datetime.fromisoformat(after), tz
|
|
).isoformat()
|
|
|
|
|
|
def test_daily_time_is_wall_clock_time_in_the_schedule_zone():
|
|
# 03:00 in Berlin is 02:00 UTC in winter...
|
|
assert _next("daily", 3, 0, 0, "2026-01-10T12:00", "Europe/Berlin") == "2026-01-11T02:00:00"
|
|
# ...and 01:00 UTC in summer.
|
|
assert _next("daily", 3, 0, 0, "2026-07-10T12:00", "Europe/Berlin") == "2026-07-11T01:00:00"
|
|
|
|
|
|
def test_daily_time_stays_put_across_daylight_saving_changes():
|
|
# Clocks go forward on 29 March 2026 and back on 25 October 2026.
|
|
assert _next("daily", 3, 0, 0, "2026-03-28T02:00", "Europe/Berlin") == "2026-03-29T01:00:00"
|
|
assert _next("daily", 3, 0, 0, "2026-03-29T01:00", "Europe/Berlin") == "2026-03-30T01:00:00"
|
|
assert _next("daily", 3, 0, 0, "2026-10-24T01:00", "Europe/Berlin") == "2026-10-25T02:00:00"
|
|
|
|
|
|
def test_a_time_skipped_by_the_clock_change_runs_an_hour_later():
|
|
# 02:30 does not exist on 29 March in Berlin; it runs at 03:30 local.
|
|
assert _next("daily", 2, 30, 0, "2026-03-28T12:00", "Europe/Berlin") == "2026-03-29T01:30:00"
|
|
|
|
|
|
def test_weekly_is_on_the_local_weekday():
|
|
# Sunday 00:30 in Berlin is still Saturday in UTC.
|
|
assert _next("weekly", 0, 30, 6, "2026-01-05T12:00", "Europe/Berlin") == "2026-01-10T23:30:00"
|
|
|
|
|
|
def test_hourly_minute_is_local_minute():
|
|
# India is UTC+5:30, so :00 local is :30 UTC.
|
|
assert _next("hourly", 0, 0, 0, "2026-01-10T12:00", "Asia/Kolkata") == "2026-01-10T12:30:00"
|
|
assert _next("hourly", 0, 15, 0, "2026-01-10T12:00", "UTC") == "2026-01-10T12:15:00"
|
|
|
|
|
|
def test_utc_is_the_default_and_unknown_zones_fall_back_to_it():
|
|
assert _next("daily", 3, 0, 0, "2026-01-10T12:00", None) == "2026-01-11T03:00:00"
|
|
assert _next("daily", 3, 0, 0, "2026-01-10T12:00", "Mars/Olympus") == "2026-01-11T03:00:00"
|
|
|
|
|
|
def test_api_keeps_the_zone_and_refuses_bad_values(api_env):
|
|
env, client = api_env
|
|
r = client.post("/api/backups/schedules", json={
|
|
"stack_ids": [env.ids["a"]], "destination_id": env.dest.id,
|
|
"hour": 3, "minute": 0, "timezone": "Europe/Berlin",
|
|
})
|
|
assert r.status_code == 201
|
|
body = r.json()
|
|
assert body["timezone"] == "Europe/Berlin"
|
|
from datetime import datetime
|
|
from zoneinfo import ZoneInfo
|
|
|
|
local = datetime.fromisoformat(body["next_run"]).replace(tzinfo=ZoneInfo("UTC")).astimezone(
|
|
ZoneInfo("Europe/Berlin")
|
|
)
|
|
assert (local.hour, local.minute) == (3, 0)
|
|
|
|
r = client.put(f"/api/backups/schedules/{body['id']}", json={"timezone": "Mars/Olympus"})
|
|
assert r.status_code == 400
|
|
r = client.put(f"/api/backups/schedules/{body['id']}", json={"hour": 24})
|
|
assert r.status_code == 400
|