Phase 4: backups w/ volumes, notifications, settings & users, audit page (0.4.0)
- Backup/restore: per-stack tar.gz incl. named-volume snapshots (helper container), upload restore with rename/overwrite/conflict detection. - Notifications: ntfy/Discord/Slack/Gotify/generic webhooks, per-event subscriptions; wired into the update checker and stack lifecycle. - Settings page: update-check interval, webhook CRUD + test, user management (with last-admin safeguards). - Audit log page (searchable, paginated). - Mobile-responsive sidebar/layout. Multi-host agents and remote backup destinations (SFTP/S3) deferred. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
menzelj
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
22d9864436
commit
8d19b09abd
@@ -0,0 +1,268 @@
|
||||
"""Stack backup & restore, including named-volume contents.
|
||||
|
||||
A backup is a single ``.tar.gz`` with this layout::
|
||||
|
||||
manifest.json metadata + volume/bind inventory
|
||||
compose/... the full stack directory (compose file, .env, ...)
|
||||
volumes/<full>.tar raw contents of each compose-managed named volume
|
||||
|
||||
Named-volume contents are read/written through a throwaway helper container
|
||||
(``BACKUP_HELPER_IMAGE``) with the volume bind-mounted — this is the portable
|
||||
way to snapshot a volume regardless of its driver/mountpoint.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
from typing import Optional
|
||||
|
||||
from config import settings
|
||||
from docker_client import DockerError, get_client, safe_call
|
||||
from services import compose_service
|
||||
|
||||
logger = logging.getLogger("stackpilot.backup")
|
||||
|
||||
COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
|
||||
COMPOSE_VOLUME_LABEL = "com.docker.compose.volume"
|
||||
MANIFEST_NAME = "manifest.json"
|
||||
BACKUP_FORMAT_VERSION = 1
|
||||
|
||||
|
||||
class BackupError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Helper container for volume I/O
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _ensure_helper_image(client) -> None:
|
||||
image = settings.BACKUP_HELPER_IMAGE
|
||||
try:
|
||||
safe_call(client.images.get, image)
|
||||
except DockerError:
|
||||
logger.info("Pulling backup helper image %s", image)
|
||||
safe_call(client.images.pull, image)
|
||||
|
||||
|
||||
def _export_volume(full_name: str) -> bytes:
|
||||
client = get_client()
|
||||
_ensure_helper_image(client)
|
||||
container = safe_call(
|
||||
client.containers.create,
|
||||
settings.BACKUP_HELPER_IMAGE,
|
||||
command="true",
|
||||
volumes={full_name: {"bind": "/v", "mode": "ro"}},
|
||||
)
|
||||
try:
|
||||
# "/v/." copies the *contents* of the volume (no leading "v/" prefix),
|
||||
# so restore can extract straight back into the volume root.
|
||||
bits, _ = container.get_archive("/v/.")
|
||||
buf = io.BytesIO()
|
||||
for chunk in bits:
|
||||
buf.write(chunk)
|
||||
return buf.getvalue()
|
||||
finally:
|
||||
try:
|
||||
container.remove(force=True)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
def _restore_volume(full_name: str, labels: dict, tar_bytes: bytes) -> None:
|
||||
client = get_client()
|
||||
_ensure_helper_image(client)
|
||||
try:
|
||||
safe_call(client.volumes.get, full_name)
|
||||
except DockerError:
|
||||
safe_call(client.volumes.create, name=full_name, labels=labels or {})
|
||||
container = safe_call(
|
||||
client.containers.create,
|
||||
settings.BACKUP_HELPER_IMAGE,
|
||||
command="true",
|
||||
volumes={full_name: {"bind": "/v", "mode": "rw"}},
|
||||
)
|
||||
try:
|
||||
container.put_archive("/v", tar_bytes)
|
||||
finally:
|
||||
try:
|
||||
container.remove(force=True)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
def _compose_volumes(stack_id: str) -> list[dict]:
|
||||
"""Return [{full, short, labels}] for compose-managed named volumes."""
|
||||
try:
|
||||
client = get_client()
|
||||
vols = safe_call(
|
||||
client.volumes.list,
|
||||
filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"},
|
||||
)
|
||||
except DockerError:
|
||||
return []
|
||||
out = []
|
||||
for v in vols:
|
||||
labels = v.attrs.get("Labels") or {}
|
||||
out.append(
|
||||
{
|
||||
"full": v.name,
|
||||
"short": labels.get(COMPOSE_VOLUME_LABEL, v.name),
|
||||
"labels": labels,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Backup
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
async def create_backup(
|
||||
stack_id: str,
|
||||
name: str,
|
||||
include_volumes: bool = True,
|
||||
stop_first: bool = True,
|
||||
) -> str:
|
||||
"""Create a backup tar.gz and return its path on disk."""
|
||||
directory = compose_service.stack_dir(stack_id)
|
||||
if not os.path.isdir(directory):
|
||||
raise BackupError("Stack directory missing")
|
||||
|
||||
volumes = _compose_volumes(stack_id) if include_volumes else []
|
||||
|
||||
# For a consistent volume snapshot, stop the stack first.
|
||||
stopped = False
|
||||
if include_volumes and stop_first and volumes:
|
||||
try:
|
||||
await compose_service.stop(stack_id)
|
||||
stopped = True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Could not stop %s before backup: %s", stack_id, exc)
|
||||
|
||||
try:
|
||||
manifest = {
|
||||
"format_version": BACKUP_FORMAT_VERSION,
|
||||
"stack_id": stack_id,
|
||||
"name": name,
|
||||
"created_at": compose_service.now().isoformat(),
|
||||
"include_volumes": include_volumes,
|
||||
"volumes": [{"full": v["full"], "short": v["short"], "labels": v["labels"]} for v in volumes],
|
||||
}
|
||||
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
||||
tmp.close()
|
||||
with tarfile.open(tmp.name, "w:gz") as tar:
|
||||
# manifest
|
||||
data = json.dumps(manifest, indent=2).encode("utf-8")
|
||||
info = tarfile.TarInfo(MANIFEST_NAME)
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
# stack directory
|
||||
tar.add(directory, arcname="compose")
|
||||
# volume contents
|
||||
for v in volumes:
|
||||
vbytes = await asyncio.to_thread(_export_volume, v["full"])
|
||||
info = tarfile.TarInfo(f"volumes/{v['full']}.tar")
|
||||
info.size = len(vbytes)
|
||||
tar.addfile(info, io.BytesIO(vbytes))
|
||||
return tmp.name
|
||||
finally:
|
||||
if stopped:
|
||||
try:
|
||||
await compose_service.up(stack_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Could not restart %s after backup: %s", stack_id, exc)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Restore
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def read_manifest(tar_path: str) -> dict:
|
||||
with tarfile.open(tar_path, "r:gz") as tar:
|
||||
member = tar.getmember(MANIFEST_NAME)
|
||||
fh = tar.extractfile(member)
|
||||
if fh is None:
|
||||
raise BackupError("Backup is missing its manifest")
|
||||
return json.loads(fh.read().decode("utf-8"))
|
||||
|
||||
|
||||
def _safe_extract_compose(tar: tarfile.TarFile, dest_dir: str) -> None:
|
||||
"""Extract the ``compose/`` subtree into dest_dir, guarding path traversal."""
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
for member in tar.getmembers():
|
||||
if not member.name.startswith("compose/"):
|
||||
continue
|
||||
rel = member.name[len("compose/") :]
|
||||
if not rel:
|
||||
continue
|
||||
target = os.path.normpath(os.path.join(dest_dir, rel))
|
||||
if not target.startswith(os.path.abspath(dest_dir) + os.sep) and target != os.path.abspath(dest_dir):
|
||||
raise BackupError(f"Refusing unsafe path in backup: {member.name}")
|
||||
if member.isdir():
|
||||
os.makedirs(target, exist_ok=True)
|
||||
elif member.isreg():
|
||||
os.makedirs(os.path.dirname(target), exist_ok=True)
|
||||
src = tar.extractfile(member)
|
||||
if src is not None:
|
||||
with open(target, "wb") as out:
|
||||
shutil.copyfileobj(src, out)
|
||||
|
||||
|
||||
def restore_backup(
|
||||
tar_path: str,
|
||||
target_id: Optional[str] = None,
|
||||
overwrite: bool = False,
|
||||
restore_volumes: bool = True,
|
||||
) -> dict:
|
||||
"""Restore a backup. Returns {stack_id, name, volumes_restored}."""
|
||||
manifest = read_manifest(tar_path)
|
||||
stack_id = target_id or manifest.get("stack_id")
|
||||
if not stack_id:
|
||||
raise BackupError("Backup manifest has no stack id")
|
||||
|
||||
directory = compose_service.stack_dir(stack_id)
|
||||
exists = os.path.isdir(directory)
|
||||
if exists and not overwrite:
|
||||
raise BackupError(f"Stack '{stack_id}' already exists")
|
||||
|
||||
with tarfile.open(tar_path, "r:gz") as tar:
|
||||
if exists:
|
||||
shutil.rmtree(directory)
|
||||
_safe_extract_compose(tar, directory)
|
||||
|
||||
volumes_restored = 0
|
||||
if restore_volumes:
|
||||
for v in manifest.get("volumes", []):
|
||||
member_name = f"volumes/{v['full']}.tar"
|
||||
try:
|
||||
member = tar.getmember(member_name)
|
||||
except KeyError:
|
||||
continue
|
||||
fh = tar.extractfile(member)
|
||||
if fh is None:
|
||||
continue
|
||||
# Re-target volume labels to the (possibly new) stack id.
|
||||
labels = dict(v.get("labels") or {})
|
||||
labels[COMPOSE_PROJECT_LABEL] = stack_id
|
||||
full = v["full"]
|
||||
if target_id and manifest.get("stack_id") and full.startswith(manifest["stack_id"] + "_"):
|
||||
full = stack_id + full[len(manifest["stack_id"]):]
|
||||
_restore_volume(full, labels, fh.read())
|
||||
volumes_restored += 1
|
||||
|
||||
return {
|
||||
"stack_id": stack_id,
|
||||
"name": manifest.get("name", stack_id),
|
||||
"volumes_restored": volumes_restored,
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Outbound notification webhooks.
|
||||
|
||||
Webhooks are configured two ways:
|
||||
* DB-managed (the ``Webhook`` table) — per-webhook type + event subscriptions,
|
||||
editable from the Settings page.
|
||||
* Env ``NOTIFY_WEBHOOKS`` — a comma-separated list of generic JSON endpoints
|
||||
that receive every event (kept for backward compatibility / GitOps setups).
|
||||
|
||||
Supported types: ntfy, discord, slack, gotify, generic (JSON POST).
|
||||
All delivery is best-effort: failures are logged, never raised to the caller.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from config import settings as env_settings
|
||||
from database import engine
|
||||
from models.setting import Webhook
|
||||
|
||||
logger = logging.getLogger("stackpilot.notify")
|
||||
|
||||
_TIMEOUT = 10.0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Payload formatting per webhook type
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _build_request(wtype: str, url: str, event: str, title: str, message: str):
|
||||
"""Return (method-kwargs) for httpx.post for the given webhook type."""
|
||||
if wtype == "ntfy":
|
||||
return {
|
||||
"url": url,
|
||||
"content": message.encode("utf-8"),
|
||||
"headers": {"Title": title, "Tags": _ntfy_tag(event)},
|
||||
}
|
||||
if wtype == "discord":
|
||||
return {"url": url, "json": {"content": f"**{title}**\n{message}"}}
|
||||
if wtype == "slack":
|
||||
return {"url": url, "json": {"text": f"*{title}*\n{message}"}}
|
||||
if wtype == "gotify":
|
||||
priority = 8 if event in ("stack_error", "pull_failed") else 5
|
||||
return {
|
||||
"url": url,
|
||||
"json": {"title": title, "message": message, "priority": priority},
|
||||
}
|
||||
# generic
|
||||
return {
|
||||
"url": url,
|
||||
"json": {"event": event, "title": title, "message": message},
|
||||
}
|
||||
|
||||
|
||||
def _ntfy_tag(event: str) -> str:
|
||||
return {
|
||||
"update_available": "arrow_up",
|
||||
"stack_start": "white_check_mark",
|
||||
"stack_stop": "stop_button",
|
||||
"stack_error": "rotating_light",
|
||||
"pull_failed": "warning",
|
||||
}.get(event, "bell")
|
||||
|
||||
|
||||
async def _deliver(client: httpx.AsyncClient, wtype: str, url: str, event: str, title: str, message: str) -> bool:
|
||||
kwargs = _build_request(wtype, url, event, title, message)
|
||||
target = kwargs.pop("url")
|
||||
try:
|
||||
resp = await client.post(target, timeout=_TIMEOUT, **kwargs)
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
except httpx.HTTPError as exc:
|
||||
logger.warning("Webhook delivery failed (%s): %s", wtype, exc)
|
||||
return False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Public API
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _targets_for_event(session: Session, event: str) -> list[tuple[str, str]]:
|
||||
"""Return [(type, url)] of all destinations subscribed to ``event``."""
|
||||
targets: list[tuple[str, str]] = []
|
||||
for wh in session.exec(select(Webhook)).all():
|
||||
if not wh.enabled:
|
||||
continue
|
||||
subscribed = [e.strip() for e in (wh.events or "").split(",") if e.strip()]
|
||||
if event in subscribed:
|
||||
targets.append((wh.type, wh.url))
|
||||
# Env-configured generic endpoints receive everything.
|
||||
for url in env_settings.NOTIFY_WEBHOOKS:
|
||||
targets.append(("generic", url))
|
||||
return targets
|
||||
|
||||
|
||||
async def notify(
|
||||
event: str,
|
||||
title: str,
|
||||
message: str,
|
||||
session: Optional[Session] = None,
|
||||
) -> int:
|
||||
"""Fan out ``event`` to all subscribed webhooks. Returns delivered count."""
|
||||
if session is None:
|
||||
with Session(engine) as own:
|
||||
return await notify(event, title, message, own)
|
||||
|
||||
targets = _targets_for_event(session, event)
|
||||
if not targets:
|
||||
return 0
|
||||
delivered = 0
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
for wtype, url in targets:
|
||||
if await _deliver(client, wtype, url, event, title, message):
|
||||
delivered += 1
|
||||
return delivered
|
||||
|
||||
|
||||
async def test_webhook(wtype: str, url: str) -> bool:
|
||||
"""Send a one-off test notification to a single destination."""
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
return await _deliver(
|
||||
client,
|
||||
wtype,
|
||||
url,
|
||||
"update_available",
|
||||
"StackPilot test notification",
|
||||
"If you can read this, your webhook is configured correctly. 🚀",
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Runtime settings stored in the DB (key/value), with env fallbacks."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from config import settings as env_settings
|
||||
from database import engine
|
||||
from models.setting import Setting
|
||||
|
||||
KEY_UPDATE_INTERVAL = "update_check_interval_minutes"
|
||||
|
||||
|
||||
def get(session: Session, key: str, default: Any = None) -> Any:
|
||||
row = session.get(Setting, key)
|
||||
if row is None:
|
||||
return default
|
||||
try:
|
||||
return json.loads(row.value)
|
||||
except json.JSONDecodeError:
|
||||
return default
|
||||
|
||||
|
||||
def set_value(session: Session, key: str, value: Any) -> None:
|
||||
row = session.get(Setting, key)
|
||||
encoded = json.dumps(value)
|
||||
if row is None:
|
||||
session.add(Setting(key=key, value=encoded))
|
||||
else:
|
||||
row.value = encoded
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
|
||||
def get_update_interval(session: Optional[Session] = None) -> int:
|
||||
"""Effective update-check interval in minutes (DB override or env default)."""
|
||||
if session is None:
|
||||
with Session(engine) as own:
|
||||
return get_update_interval(own)
|
||||
val = get(session, KEY_UPDATE_INTERVAL)
|
||||
if isinstance(val, int) and val > 0:
|
||||
return val
|
||||
return env_settings.UPDATE_CHECK_INTERVAL_MINUTES
|
||||
@@ -16,6 +16,8 @@ import httpx
|
||||
|
||||
from config import settings
|
||||
from docker_client import DockerError, get_client, safe_call
|
||||
from models.setting import EVENT_UPDATE_AVAILABLE
|
||||
from services import notify_service, settings_service
|
||||
|
||||
logger = logging.getLogger("stackpilot.update")
|
||||
|
||||
@@ -45,6 +47,10 @@ class UpdateStatus:
|
||||
# image ref -> UpdateStatus
|
||||
_CACHE: dict[str, UpdateStatus] = {}
|
||||
|
||||
# images we've already sent an "update available" notification for, so the
|
||||
# background loop doesn't re-notify on every cycle.
|
||||
_NOTIFIED: set[str] = set()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Image reference parsing
|
||||
@@ -164,6 +170,18 @@ async def check_image(image: str) -> UpdateStatus:
|
||||
error=error,
|
||||
)
|
||||
_CACHE[image] = status
|
||||
if update_available and image not in _NOTIFIED:
|
||||
_NOTIFIED.add(image)
|
||||
try:
|
||||
await notify_service.notify(
|
||||
EVENT_UPDATE_AVAILABLE,
|
||||
"Image update available",
|
||||
f"A newer image is available for {image}.",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - notifications are best-effort
|
||||
logger.debug("update notify failed for %s: %s", image, exc)
|
||||
elif not update_available:
|
||||
_NOTIFIED.discard(image)
|
||||
return status
|
||||
|
||||
|
||||
@@ -192,7 +210,6 @@ def get_cache() -> dict[str, dict]:
|
||||
|
||||
|
||||
async def background_loop():
|
||||
interval = max(settings.UPDATE_CHECK_INTERVAL_MINUTES, 5) * 60
|
||||
# initial delay so startup isn't blocked
|
||||
await asyncio.sleep(30)
|
||||
while True:
|
||||
@@ -201,4 +218,6 @@ async def background_loop():
|
||||
logger.info("Image update check complete (%d images)", len(_CACHE))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Image update check failed: %s", exc)
|
||||
# Re-read the interval each cycle so Settings changes take effect.
|
||||
interval = max(settings_service.get_update_interval(), 5) * 60
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
Reference in New Issue
Block a user