diff --git a/README.md b/README.md index 9f01763..1a1a139 100644 --- a/README.md +++ b/README.md @@ -590,6 +590,35 @@ it is what your saved destination credentials are encrypted with. folders back at their host paths and preserves permissions, ownership and symlinks (PUID/PGID-based images such as the *arr suite need this), with optional rename and overwrite/conflict detection. + + **How backup and restore stay reliable** (0.64.0): + - The stack's running containers are stopped whenever any data is captured — + also when the data is only folders inside the stack directory — and exactly + those are started again (`compose start`, not `up`, so services you stopped + stay stopped and undeployed compose edits are not applied). A stack that + cannot be stopped is not backed up. The stack is locked meanwhile, so a + deploy, update or scheduled backup cannot interleave (a scheduled backup + that finds the stack busy retries after 10 minutes). + - Remote storage is kept out by what it *is*, not only by how it is declared: + NFS/CIFS volumes, bind folders on a mounted share (`/mnt/nas/…` via fstab or + autofs), bind-type volumes pointing at one, and shares mounted inside the + stack folder. Folders too big to measure in 20 s are not selected either. + - Volumes the stack uses without owning them (`external: true`) are backed up + too; anonymous volumes are listed but not selected. + - Restore reads the whole archive first and refuses a damaged one before + touching anything, stops the stack while replacing its data, unpacks the + stack folder next to the old one and then swaps it in. With *Overwrite*, + folders the backup left out (too large, deselected, remote) and other + filesystems mounted inside the stack folder are **kept**, not deleted. + - Restoring under a new name never writes into the original stack: folders + outside the stack directory and volumes not named after the stack are + skipped, and a volume owned by another stack or mounted by a running + container is never overwritten. Missing volumes are recreated with their + original driver and options. + - A backup that could not capture everything, or left the stack down, is + reported as incomplete — in the dialog and for scheduled backups, which + then notify and do not rotate out older backups. Retention only matches + the stack's own files (it used to take `app-db`'s backups for `app`). - **Notification webhooks**: ntfy, Discord, Slack, Gotify, or generic JSON, each subscribed to chosen events (image update available, stack start/stop/error, pull failed). Managed in **Settings → Notifications**; env `NOTIFY_WEBHOOKS` diff --git a/backend/routers/backups.py b/backend/routers/backups.py index 3cf441c..24cb908 100644 --- a/backend/routers/backups.py +++ b/backend/routers/backups.py @@ -8,6 +8,7 @@ import tempfile from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import FileResponse +from starlette.background import BackgroundTask from pydantic import BaseModel from sqlmodel import Session @@ -41,15 +42,32 @@ def _compact(report: dict) -> dict: "volumes": report.get("volumes", []), "skipped": report.get("skipped", []), "path_mismatch": report.get("path_mismatch"), + "stopped": report.get("stopped", []), + "errors": report.get("errors", []), + # Capped: a busy log folder can produce a warning per file, and this + # travels in a response header for downloads. + "warnings": report.get("warnings", [])[:20], + "restart_error": report.get("restart_error"), } def _summary(report: dict) -> str: - return ( + text = ( f"binds={len(report.get('binds', []))} " f"volumes={len(report.get('volumes', []))} " f"skipped={len(report.get('skipped', []))}" ) + if report.get("errors"): + text += f" errors={len(report['errors'])}" + if report.get("restart_error"): + text += " restart_failed" + return text + + +def _backup_http_error(exc: backup_service.BackupError) -> HTTPException: + # 409 for a stack that is busy or already exists, 400 for the rest. + conflict = isinstance(exc, backup_service.StackBusyError) or "already exists" in str(exc) + return HTTPException(status_code=409 if conflict else 400, detail=str(exc)) @router.get("/{stack_id}/backup/inventory") @@ -88,7 +106,7 @@ async def backup_stack( binds=binds, volumes=volumes, ) except backup_service.BackupError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc + raise _backup_http_error(exc) from exc audit_service.record( session, user=user.username, action="stack.backup", target=stack_id, detail=_summary(report), ip=_ip(request), @@ -100,6 +118,9 @@ async def backup_stack( # The browser downloads a blob, so the summary of what actually made it # into the archive rides along in a header. headers={"X-Stackpilot-Backup": json.dumps(_compact(report))}, + # The archive is a temp file; without this every download stayed in + # the container's /tmp until it filled the disk. + background=BackgroundTask(os.unlink, path), ) @@ -122,7 +143,7 @@ async def restore_stack( target = compose_service.slugify(target_id) if target_id else None try: - result = backup_service.restore_backup( + result = await backup_service.restore( tmp.name, target_id=target, overwrite=overwrite, @@ -130,9 +151,7 @@ async def restore_stack( restore_binds=restore_binds, ) except backup_service.BackupError as exc: - # 409 for the "already exists" conflict, 400 for malformed backups. - code = 409 if "already exists" in str(exc) else 400 - raise HTTPException(status_code=code, detail=str(exc)) from exc + raise _backup_http_error(exc) from exc stack_id = result["stack_id"] stack = session.get(Stack, stack_id) @@ -199,7 +218,7 @@ async def push_backup( include_binds=body.include_binds, binds=body.binds, volumes=body.volumes, ) except backup_service.BackupError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc + raise _backup_http_error(exc) from exc filename = _backup_filename(stack_id, body.include_volumes) try: @@ -238,14 +257,13 @@ async def restore_from_destination( target = compose_service.slugify(body.target_id) if body.target_id else None try: - result = backup_service.restore_backup( + result = await backup_service.restore( tmp.name, target_id=target, overwrite=body.overwrite, restore_volumes=body.restore_volumes, restore_binds=body.restore_binds, ) except backup_service.BackupError as exc: - code = 409 if "already exists" in str(exc) else 400 - raise HTTPException(status_code=code, detail=str(exc)) from exc + raise _backup_http_error(exc) from exc stack_id = result["stack_id"] if not session.get(Stack, stack_id): diff --git a/backend/services/backup_destination_service.py b/backend/services/backup_destination_service.py index 9b23c8c..a964d6c 100644 --- a/backend/services/backup_destination_service.py +++ b/backend/services/backup_destination_service.py @@ -219,11 +219,14 @@ def _s3_list(cfg: dict) -> list[dict]: if prefix: kwargs["Prefix"] = prefix + "/" try: - resp = client.list_objects_v2(**kwargs) + # Paginated: one call returns at most 1000 keys, and retention sorts + # by age, so the oldest backups were the ones it never saw. + pages = client.get_paginator("list_objects_v2").paginate(**kwargs) + objects = [obj for page in pages for obj in page.get("Contents", [])] except Exception as exc: # noqa: BLE001 raise DestinationError(f"S3 list failed: {exc}") from exc out = [] - for obj in resp.get("Contents", []): + for obj in objects: name = obj["Key"].split("/")[-1] if not name.endswith(".tar.gz"): continue diff --git a/backend/services/backup_service.py b/backend/services/backup_service.py index 2007741..b17670f 100644 --- a/backend/services/backup_service.py +++ b/backend/services/backup_service.py @@ -11,6 +11,25 @@ Bind sources and volumes are read/written through a throwaway helper container (see :mod:`services.stack_assets_service`) so that host paths this container cannot see are still captured — without that, a stack whose data directories live outside StackPilot's own mount would back up as "just the compose file". + +What makes a backup and its restore trustworthy, and what this module keeps to: + +* **Consistent.** Whenever data is captured — including data folders that ride + along inside the stack directory — the stack's running containers are + stopped first and exactly those are started again afterwards. A stack that + cannot be stopped is not backed up half-written. The stack is locked for the + duration, so a deploy, update or second backup cannot interleave. +* **Nothing is lost that the backup does not contain.** A restore replaces what + the backup holds and leaves alone what it does not: folders left out (too + large, deselected, remote storage) and anything mounted into the stack + directory from another filesystem survive an overwrite. +* **Verified before destructive.** The whole archive is read once before a + restore touches anything, and the stack directory is unpacked next to the + old one first. +* **A restore under a new name never writes into the original's data**: + folders outside the stack directory and volumes not named after the stack + are skipped rather than overwritten. +* **Remote storage** (NFS/CIFS/...) is neither archived nor wiped. """ from __future__ import annotations @@ -19,25 +38,44 @@ import io import json import logging import os +import re import shutil +import stat import tarfile import tempfile +import zlib +from contextlib import contextmanager +from datetime import timedelta from typing import Optional -from services import compose_service, stack_assets_service as assets +from sqlmodel import Session + +from database import engine +from docker_client import DockerError, get_long_client, safe_call +from services import compose_service, stack_lock_service, stack_assets_service as assets logger = logging.getLogger("stackpilot.backup") COMPOSE_PROJECT_LABEL = assets.COMPOSE_PROJECT_LABEL COMPOSE_VOLUME_LABEL = assets.COMPOSE_VOLUME_LABEL MANIFEST_NAME = "manifest.json" -BACKUP_FORMAT_VERSION = 2 +#: 3 added compose_binds/excluded (what the compose/ tree does and does not +#: carry) and each volume's driver/options/external. Older ones restore too. +BACKUP_FORMAT_VERSION = 3 + +#: How long a backup or restore may hold the stack lock before another +#: operation may take it over. Generous: a large volume can take hours. +LOCK_TTL = timedelta(hours=12) class BackupError(Exception): pass +class StackBusyError(BackupError): + """The stack is locked by another operation (deploy, update, backup).""" + + # --------------------------------------------------------------------------- # # Backup filename convention # --------------------------------------------------------------------------- # @@ -58,6 +96,100 @@ def backup_filename(stack_id: str, include_volumes: bool, prefix: Optional[str] return f"{backup_basename(stack_id, prefix)}-{suffix}-{date}.tar.gz" +def backup_name_pattern(stack_id: str, prefix: Optional[str] = None) -> re.Pattern: + """Matches exactly this stack's backup files. + + A plain prefix match on ``backup-app-`` also takes ``backup-app-db-…``, so + retention for stack "app" deleted the backups of stack "app-db". + """ + base = re.escape(backup_basename(stack_id, prefix)) + return re.compile(rf"^{base}-(?:full|config)-\d{{8}}-\d{{6}}\.tar\.gz$") + + +# --------------------------------------------------------------------------- # +# Locking and stopping +# --------------------------------------------------------------------------- # + + +@contextmanager +def _locked(stack_id: str, action: str): + with Session(engine) as session: + try: + stack_lock_service.acquire(session, stack_id, action, action, ttl=LOCK_TTL) + except stack_lock_service.StackBusy as exc: + raise StackBusyError(str(exc)) from exc + try: + yield + finally: + try: + stack_lock_service.release(session, stack_id) + except Exception: # noqa: BLE001 - never mask the original error + logger.exception("Failed to release the lock on '%s'", stack_id) + + +async def _running(stack_id: str) -> list: + try: + containers = await asyncio.to_thread(compose_service.containers_for_stack, stack_id) + except DockerError as exc: + raise BackupError(f"Docker is not reachable: {exc.detail or exc.error}") from exc + return [c for c in containers if c.state == "running"] + + +def _tail(result: dict) -> str: + text = (result.get("stderr") or result.get("stdout") or "").strip() + return text.splitlines()[-1] if text else f"exit {result.get('returncode')}" + + +def _docker_each(ids: list[str], action: str) -> None: + client = get_long_client() + for container_id in ids: + container = safe_call(client.containers.get, container_id) + safe_call(getattr(container, action)) + + +async def _compose(stack_id: str, args: list[str]) -> Optional[str]: + """Run a compose command; None on success, else what went wrong.""" + try: + result = await compose_service.run_compose(stack_id, args) + except Exception as exc: # noqa: BLE001 - timeouts, missing binary + return str(exc) + return None if result["returncode"] == 0 else _tail(result) + + +async def _stop(stack_id: str, running: list) -> None: + """Stop exactly these containers, or raise and leave them running.""" + services = sorted({c.service for c in running}) + error = await _compose(stack_id, ["stop", *services]) + if error is None: + return + # Compose fails on a compose file that was edited (or broken) since the + # deploy; the containers themselves can still be stopped directly. + logger.warning("compose stop failed for %s (%s); stopping containers directly", stack_id, error) + try: + await asyncio.to_thread(_docker_each, [c.id for c in running], "stop") + except Exception as exc: # noqa: BLE001 + await _start(stack_id, running) # put back whatever did stop + raise BackupError( + f"Could not stop '{stack_id}' for a consistent snapshot: {error}" + ) from exc + + +async def _start(stack_id: str, running: list) -> Optional[str]: + """Start exactly these containers again; None on success.""" + if not running: + return None + services = sorted({c.service for c in running}) + error = await _compose(stack_id, ["start", *services]) + if error is None: + return None + try: + await asyncio.to_thread(_docker_each, [c.id for c in running], "start") + return None + except Exception as exc: # noqa: BLE001 + logger.warning("Could not restart %s: %s / %s", stack_id, error, exc) + return f"{error}; {exc}" + + # --------------------------------------------------------------------------- # # Selection # --------------------------------------------------------------------------- # @@ -70,7 +202,7 @@ def _decide( ``chosen`` is an explicit list from the caller; without one the inventory's own defaults apply (everything except system paths, oversized directories - and remote-backed volumes). + and remote-backed storage). """ for item in items: if not enabled: @@ -103,22 +235,122 @@ def plan( return inv +# --------------------------------------------------------------------------- # +# Writing the archive +# --------------------------------------------------------------------------- # + + +class _ExactReader: + """Hands tarfile exactly ``size`` bytes of a file that may change while it + is read. A file that shrinks (a log rotated mid-backup) is padded with + zeros instead of aborting inside the member, which would leave the whole + archive unreadable; ``short`` records that it happened.""" + + def __init__(self, fh, size: int): + self._fh = fh + self._remaining = size + self.short = False + + def read(self, n: int = -1) -> bytes: + if n < 0 or n > self._remaining: + n = self._remaining + data = self._fh.read(n) + if len(data) < n: + self.short = True + data += b"\0" * (n - len(data)) + self._remaining -= n + return data + + +def _add_tree( + tar: tarfile.TarFile, root: str, arcroot: str, skip: set[str], warnings: list[str] +) -> None: + """Add the stack directory, tolerating a directory that is in use. + + ``skip`` holds paths (relative to ``root``) whose *contents* stay out: the + directory entry itself is kept so the mount point exists after a restore. + Files that vanish mid-walk are left out and unreadable ones reported, + rather than one of them failing the whole backup. A remote filesystem + mounted somewhere inside is not descended into. + """ + + def add(path: str, rel: str, parent_dev: int) -> None: + arc = f"{arcroot}/{rel}" if rel else arcroot + label = rel or "." + try: + st = os.lstat(path) + if stat.S_ISSOCK(st.st_mode): + return # a runtime artefact, meaningless in an archive + if stat.S_ISREG(st.st_mode): + if rel in skip: + return + with open(path, "rb") as fh: + info = tar.gettarinfo(arcname=arc, fileobj=fh) + if info.isreg(): + reader = _ExactReader(fh, info.size) + tar.addfile(info, reader) + if reader.short: + warnings.append(f"{label}: changed while it was read") + else: # a hard link to a file already in the archive + tar.addfile(info) + return + info = tar.gettarinfo(path, arcname=arc) + except FileNotFoundError: + return + except OSError as exc: + warnings.append(f"{label}: {exc.strerror or exc}") + return + if info is None: + return + tar.addfile(info) + if not stat.S_ISDIR(st.st_mode) or rel in skip: + return + if rel and st.st_dev != parent_dev: + remote = assets.remote_mount_type(path) + if remote: + warnings.append(f"{label}: remote storage ({remote}) mounted here, not included") + return + try: + names = sorted(os.listdir(path)) + except OSError as exc: + warnings.append(f"{label}: {exc.strerror or exc}") + return + for name in names: + add(os.path.join(path, name), f"{rel}/{name}" if rel else name, st.st_dev) + + add(root, "", os.lstat(root).st_dev) + + +def _write_archive( + dest: str, + manifest: dict, + directory: str, + skip: set[str], + parts: list[tuple[str, str]], + warnings: list[str], +) -> None: + with tarfile.open(dest, "w:gz") as tar: + data = json.dumps(manifest, indent=2).encode("utf-8") + info = tarfile.TarInfo(MANIFEST_NAME) + info.size = len(data) + info.mtime = int(compose_service.now().timestamp()) + tar.addfile(info, io.BytesIO(data)) + _add_tree(tar, directory, "compose", skip, warnings) + for part, arcname in parts: + tar.add(part, arcname=arcname) + os.unlink(part) # keep the peak disk use to one copy of the data + + # --------------------------------------------------------------------------- # # Backup # --------------------------------------------------------------------------- # -def _compose_filter(excluded_prefixes: list[str]): - """Drop deselected bind directories from the compose/ tree (keep the mount - point itself, so the stack still starts after a restore).""" - - def _filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: - for prefix in excluded_prefixes: - if info.name.startswith(prefix + "/"): - return None - return info - - return _filter +def _rel_inside(path: str, directory: str) -> Optional[str]: + rel = os.path.relpath(path, directory) + if rel == "." or rel.startswith(".." + os.sep) or rel == "..": + return None + return rel async def create_backup_ex( @@ -130,69 +362,97 @@ async def create_backup_ex( binds: Optional[list[str]] = None, volumes: Optional[list[str]] = None, ) -> tuple[str, dict]: - """Create a backup tar.gz. Returns (path, report).""" + """Create a backup tar.gz. Returns (path, report). + + The report's ``errors`` lists data that should have been captured and was + not, and ``restart_error`` a stack that could not be started again — both + mean the backup must not be reported as a plain success. + """ directory = compose_service.stack_dir(stack_id) if not os.path.isdir(directory): raise BackupError("Stack directory missing") + with _locked(stack_id, "backup"): + return await _create( + stack_id, name, directory, include_volumes, stop_first, include_binds, binds, volumes + ) + +async def _create( + stack_id: str, + name: str, + directory: str, + include_volumes: bool, + stop_first: bool, + include_binds: bool, + binds: Optional[list[str]], + volumes: Optional[list[str]], +) -> tuple[str, dict]: selection = await asyncio.to_thread( plan, stack_id, include_volumes, include_binds, binds, volumes ) + selected_binds = [b for b in selection["binds"] if b["selected"]] # Selected bind sources this process cannot reach through the filesystem # need their own archive; the rest already ride along in compose/. - cap_binds = [b for b in selection["binds"] if b["selected"] and b["via"] == "archive"] + cap_binds = [b for b in selected_binds if b["via"] == "archive"] + tree_binds = [b for b in selected_binds if b["via"] != "archive"] cap_volumes = [v for v in selection["volumes"] if v["selected"]] skipped_binds = [b for b in selection["binds"] if not b["selected"]] skipped_volumes = [v for v in selection["volumes"] if not v["selected"]] - # Consistent snapshot: stop the stack first — but only if it is actually - # running, so backing up a stopped stack doesn't start it. - stopped = False - if stop_first and (cap_volumes or cap_binds): - if compose_service.compute_status(stack_id) not in ("stopped", "unknown"): - try: - await compose_service.stop(stack_id) - stopped = True - except Exception as exc: # noqa: BLE001 - logger.warning("Could not stop %s before backup: %s", stack_id, exc) + # Deselected data that sits inside the stack folder must not sneak into + # the archive through compose/ (that is how a 200 GB downloads folder + # ends up in a "config only" backup). A restore leaves these alone. + excluded = sorted( + { + rel + for b in skipped_binds + if b.get("inside_stack_dir") and (rel := _rel_inside(b["source"], directory)) + } + ) + compose_binds = sorted( + {rel for b in tree_binds if (rel := _rel_inside(b["source"], directory))} + ) + + # Consistent snapshot: stop what runs — data folders inside the stack + # directory count as data too — and start exactly that again afterwards. + stopped: list = [] + if stop_first and (cap_volumes or selected_binds): + running = await _running(stack_id) + if running: + await _stop(stack_id, running) + stopped = running workdir = tempfile.mkdtemp(prefix="sp-backup-") + archive_path: Optional[str] = None + warnings: list[str] = [] + restart_error: Optional[str] = None try: - # Archive bind sources and volumes into the work directory first, so a - # failure on one asset is reported instead of corrupting the tar. + parts: list[tuple[str, str]] = [] for index, bind in enumerate(cap_binds): - bind["archive"] = f"binds/{index:03d}.tar" part = os.path.join(workdir, f"bind-{index:03d}.tar") try: bind["bytes"] = await asyncio.to_thread( assets.export_path, bind["source"], bind["kind"], part ) - bind["_part"] = part + bind["archive"] = f"binds/{index:03d}.tar" + parts.append((part, bind["archive"])) except Exception as exc: # noqa: BLE001 logger.warning("Could not archive bind %s: %s", bind["source"], exc) - bind["archive"] = None - bind["error"] = str(exc) + bind["archive"], bind["error"] = None, str(exc) for index, vol in enumerate(cap_volumes): - vol["archive"] = f"volumes/{vol['name']}.tar" part = os.path.join(workdir, f"vol-{index:03d}.tar") try: vol["bytes"] = await asyncio.to_thread(assets.export_volume, vol["name"], part) - vol["_part"] = part + vol["archive"] = f"volumes/{vol['name']}.tar" + parts.append((part, vol["archive"])) except Exception as exc: # noqa: BLE001 logger.warning("Could not archive volume %s: %s", vol["name"], exc) - vol["archive"] = None - vol["error"] = str(exc) + vol["archive"], vol["error"] = None, str(exc) - # Deselected data that sits inside the stack folder must not sneak into - # the archive through compose/ (that is how a 200 GB downloads folder - # ends up in a "config only" backup). - excluded = [ - "compose/" + os.path.relpath(b["source"], directory) - for b in skipped_binds - if b.get("inside_stack_dir") + failed = [b for b in cap_binds if b.get("error")] + [ + v for v in cap_volumes if v.get("error") ] - manifest = { "format_version": BACKUP_FORMAT_VERSION, "stack_id": stack_id, @@ -202,6 +462,11 @@ async def create_backup_ex( "include_binds": include_binds, "stack_dir": directory, "path_mismatch": selection.get("path_mismatch"), + "stopped": sorted({c.service for c in stopped}), + # Data folders carried inside compose/, and folders whose contents + # were left out of it (kept as they are by a restore). + "compose_binds": compose_binds, + "excluded": excluded, # Volumes keep the v1 shape (full/short/labels) so older StackPilots # can still read the manifest they care about. "volumes": [ @@ -209,10 +474,12 @@ async def create_backup_ex( "full": v["name"], "short": v["short"], "labels": v["labels"], - "archive": v.get("archive"), + "driver": v.get("driver"), + "options": v.get("options") or {}, + "external": v.get("external", False), + "archive": v["archive"], "remote": v.get("remote", False), "bytes": v.get("bytes"), - "error": v.get("error"), } for v in cap_volumes if v.get("archive") @@ -223,9 +490,8 @@ async def create_backup_ex( "kind": b["kind"], "mounts": b["mounts"], "inside_stack_dir": b["inside_stack_dir"], - "archive": b.get("archive"), + "archive": b["archive"], "bytes": b.get("bytes"), - "error": b.get("error"), } for b in cap_binds if b.get("archive") @@ -237,46 +503,51 @@ async def create_backup_ex( + [ {"kind": "volume", "name": v["name"], "reason": v.get("reason")} for v in skipped_volumes + ] + + [ + { + "kind": "volume" if "name" in item else "bind", + **({"name": item["name"]} if "name" in item else {"source": item["source"]}), + "reason": f"backup failed: {item['error']}", + } + for item in failed ], } - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz") - tmp.close() - with tarfile.open(tmp.name, "w:gz") as tar: - data = json.dumps(manifest, indent=2).encode("utf-8") - info = tarfile.TarInfo(MANIFEST_NAME) - info.size = len(data) - tar.addfile(info, io.BytesIO(data)) - tar.add(directory, arcname="compose", filter=_compose_filter(excluded)) - for bind in cap_binds: - if bind.get("_part"): - tar.add(bind["_part"], arcname=bind["archive"]) - for vol in cap_volumes: - if vol.get("_part"): - tar.add(vol["_part"], arcname=vol["archive"]) - - report = { - "file": tmp.name, - "binds": [ - {"source": b["source"], "bytes": b.get("bytes"), "error": b.get("error")} - for b in cap_binds - ], - "volumes": [ - {"name": v["name"], "bytes": v.get("bytes"), "error": v.get("error")} - for v in cap_volumes - ], - "skipped": manifest["skipped"], - "path_mismatch": selection.get("path_mismatch"), - "size": os.path.getsize(tmp.name), - } - return tmp.name, report + fd, archive_path = tempfile.mkstemp(prefix=f"sp-{stack_id}-", suffix=".tar.gz") + os.close(fd) + await asyncio.to_thread( + _write_archive, archive_path, manifest, directory, set(excluded), parts, warnings + ) + except BaseException: + if archive_path and os.path.exists(archive_path): + os.unlink(archive_path) + raise finally: shutil.rmtree(workdir, ignore_errors=True) if stopped: - try: - await compose_service.up(stack_id) - except Exception as exc: # noqa: BLE001 - logger.warning("Could not restart %s after backup: %s", stack_id, exc) + restart_error = await _start(stack_id, stopped) + + errors = [f"{item.get('source') or item.get('name')}: {item['error']}" for item in failed] + report = { + "file": archive_path, + "binds": [ + {"source": b["source"], "bytes": b.get("bytes") or b.get("size"), "error": b.get("error")} + for b in tree_binds + cap_binds + ], + "volumes": [ + {"name": v["name"], "bytes": v.get("bytes"), "error": v.get("error")} + for v in cap_volumes + ], + "skipped": manifest["skipped"], + "path_mismatch": selection.get("path_mismatch"), + "stopped": manifest["stopped"], + "errors": errors, + "warnings": warnings, + "restart_error": restart_error, + "size": os.path.getsize(archive_path), + } + return archive_path, report async def create_backup( @@ -295,20 +566,45 @@ async def create_backup( # --------------------------------------------------------------------------- # -# Restore +# Reading and verifying # --------------------------------------------------------------------------- # def read_manifest(tar_path: str) -> dict: - with tarfile.open(tar_path, "r:gz") as tar: - try: - member = tar.getmember(MANIFEST_NAME) - except KeyError as exc: - raise BackupError("Backup is missing its manifest") from exc - fh = tar.extractfile(member) - if fh is None: - raise BackupError("Backup is missing its manifest") - return json.loads(fh.read().decode("utf-8")) + """The manifest, read without decompressing the rest (it comes first).""" + try: + with tarfile.open(tar_path, "r:gz") as tar: + for member in tar: + if member.name == MANIFEST_NAME: + fh = tar.extractfile(member) + if fh is None: + break + return json.loads(fh.read().decode("utf-8")) + except (tarfile.TarError, OSError, EOFError, zlib.error, ValueError) as exc: + raise BackupError(f"Backup archive is damaged or incomplete: {exc}") from exc + raise BackupError("Backup is missing its manifest") + + +def verify_archive(tar_path: str) -> None: + """Read the whole archive once, so a truncated or corrupt file is refused + before a restore deletes anything. Draining the gzip stream to its end is + what checks its CRC.""" + try: + with tarfile.open(tar_path, "r:gz") as tar: + for member in tar: + if member.isreg(): + fh = tar.extractfile(member) + while fh is not None and fh.read(1024 * 1024): + pass + while tar.fileobj.read(1024 * 1024): + pass + except (tarfile.TarError, OSError, EOFError, zlib.error) as exc: + raise BackupError(f"Backup archive is damaged or incomplete: {exc}") from exc + + +# --------------------------------------------------------------------------- # +# Restore — stack directory +# --------------------------------------------------------------------------- # def _safe_target(dest_dir: str, rel: str) -> str: @@ -329,46 +625,67 @@ def _safe_target(dest_dir: str, rel: str) -> str: def _apply_meta(path: str, member: tarfile.TarInfo) -> None: """Restore mode/ownership/mtime — *arr-style images run as PUID/PGID and break when their config comes back root-owned with default permissions.""" - try: - os.chmod(path, member.mode) - except OSError: - pass try: os.chown(path, member.uid, member.gid) except (OSError, AttributeError): pass + try: + # After chown, which clears setuid/setgid bits. + os.chmod(path, member.mode) + except OSError: + pass try: os.utime(path, (member.mtime, member.mtime)) except OSError: pass -def _extract_tree(tar: tarfile.TarFile, prefix: str, dest_dir: str) -> None: - """Extract one subtree of the archive, preserving metadata and symlinks.""" +def _under(rel: str, prefixes: set[str]) -> bool: + return any(rel.startswith(p + "/") for p in prefixes) + + +def _extract_tree( + tar: tarfile.TarFile, prefix: str, dest_dir: str, skip: frozenset | set = frozenset() +) -> None: + """Extract one subtree of the archive, preserving metadata and symlinks. + + Contents under a ``skip`` path (relative) are not extracted; the path's own + directory entry still is. + """ os.makedirs(dest_dir, exist_ok=True) + root_name = prefix.rstrip("/") dirs: list[tuple[str, tarfile.TarInfo]] = [] for member in tar.getmembers(): + if member.name == root_name: + if member.isdir(): + dirs.append((dest_dir, member)) + continue if not member.name.startswith(prefix): continue rel = member.name[len(prefix) :].lstrip("/") - if not rel: + if not rel or _under(rel, skip): continue target = _safe_target(dest_dir, rel) + existing_dir = os.path.isdir(target) and not os.path.islink(target) if member.isdir(): + if os.path.lexists(target) and not existing_dir: + os.unlink(target) os.makedirs(target, exist_ok=True) dirs.append((target, member)) continue os.makedirs(os.path.dirname(target), exist_ok=True) + # Never write through (or over) something an earlier member put here: + # open() would follow a symlink to wherever it points. + if existing_dir: + shutil.rmtree(target) + elif os.path.lexists(target): + os.unlink(target) if member.issym(): - if os.path.lexists(target): - os.unlink(target) os.symlink(member.linkname, target) continue # chmod/utime would follow the link if member.islnk(): source = _safe_target(dest_dir, member.linkname[len(prefix) :].lstrip("/")) if os.path.exists(source): - if os.path.lexists(target): - os.unlink(target) os.link(source, target) continue if not member.isreg(): @@ -384,105 +701,273 @@ def _extract_tree(tar: tarfile.TarFile, prefix: str, dest_dir: str) -> None: _apply_meta(target, member) +def _remove_tree(path: str, dev: int, rel: str, warnings: list[str]) -> None: + """rmtree that never crosses into another filesystem (a mounted disk or + share inside the stack folder is left as it is).""" + with os.scandir(path) as it: + entries = list(it) + for entry in entries: + entry_rel = f"{rel}/{entry.name}" + if entry.is_dir(follow_symlinks=False): + if entry.stat(follow_symlinks=False).st_dev != dev: + warnings.append(f"{entry_rel}: another filesystem is mounted here, left untouched") + continue + _remove_tree(entry.path, dev, entry_rel, warnings) + else: + os.unlink(entry.path) + try: + os.rmdir(path) + except OSError: + pass # still holds a mount point + + +def _clear(directory: str, preserve: set[str], warnings: list[str]) -> None: + """Empty the stack directory for the restored tree, except what the backup + does not contain: ``preserve`` paths and other filesystems mounted inside.""" + root_dev = os.lstat(directory).st_dev + + def walk(path: str, rel: str) -> None: + with os.scandir(path) as it: + entries = list(it) + for entry in entries: + entry_rel = f"{rel}/{entry.name}" if rel else entry.name + if entry_rel in preserve: + continue + if entry.is_dir(follow_symlinks=False): + if entry.stat(follow_symlinks=False).st_dev != root_dev: + warnings.append( + f"{entry_rel}: another filesystem is mounted here, left untouched" + ) + continue + if any(p.startswith(entry_rel + "/") for p in preserve): + walk(entry.path, entry_rel) + else: + _remove_tree(entry.path, root_dev, entry_rel, warnings) + else: + os.unlink(entry.path) + + walk(directory, "") + + +def _copy_dir_meta(src: str, dst: str) -> None: + st = os.lstat(src) + try: + os.chown(dst, st.st_uid, st.st_gid) + os.chmod(dst, stat.S_IMODE(st.st_mode)) + os.utime(dst, (st.st_atime, st.st_mtime)) + except OSError: + pass + + +def _merge(src: str, dst: str, rel: str, warnings: list[str]) -> None: + """Move the staged tree into the (cleared) stack directory.""" + for name in sorted(os.listdir(src)): + s, d = os.path.join(src, name), os.path.join(dst, name) + entry_rel = f"{rel}/{name}" if rel else name + s_dir = os.path.isdir(s) and not os.path.islink(s) + if os.path.lexists(d): + d_dir = os.path.isdir(d) and not os.path.islink(d) + if s_dir and d_dir: + _merge(s, d, entry_rel, warnings) + _copy_dir_meta(s, d) + continue + if d_dir: + # Only kept (preserved or mounted) directories survive _clear. + warnings.append(f"{entry_rel}: kept the existing folder, the backup has a file here") + continue + os.unlink(d) + shutil.move(s, d) + + +def _restore_tree( + tar: tarfile.TarFile, + directory: str, + preserve: set[str], + skip_extract: set[str], + warnings: list[str], +) -> None: + """Replace the stack directory with the backup's, staged next to it first + so a failing extraction leaves the old one untouched.""" + root = os.path.dirname(directory) + os.makedirs(root, exist_ok=True) + staging = tempfile.mkdtemp(prefix=f".restore-{os.path.basename(directory)}-", dir=root) + try: + _extract_tree(tar, "compose/", staging, skip=skip_extract) + if not os.path.isdir(directory): + shutil.move(staging, directory) + return + _clear(directory, preserve, warnings) + _merge(staging, directory, "", warnings) + _copy_dir_meta(staging, directory) + finally: + shutil.rmtree(staging, ignore_errors=True) + + +def _tree_paths(manifest: dict) -> tuple[set[str], set[str]]: + """(excluded, compose_binds) relative to the stack directory. + + Format 3 records them. For older backups the excluded folders are worked + out from the skipped binds; which data folders compose/ carries is unknown. + """ + if "excluded" in manifest: + return set(manifest.get("excluded") or []), set(manifest.get("compose_binds") or []) + old_dir = manifest.get("stack_dir") or "" + excluded = set() + for item in manifest.get("skipped", []): + if item.get("kind") == "bind" and old_dir and item.get("source"): + rel = _rel_inside(item["source"], old_dir) + if rel: + excluded.add(rel) + return excluded, set() + + +# --------------------------------------------------------------------------- # +# Restore +# --------------------------------------------------------------------------- # + + +def _target_id(manifest: dict, target_id: Optional[str]) -> str: + raw = target_id or manifest.get("stack_id") or "" + if not raw.strip(): + raise BackupError("Backup manifest has no stack id") + # Slugify whichever id we end up using — the manifest comes from an + # uploaded file, so its stack_id must never be able to escape STACKS_DIR. + return compose_service.slugify(raw) + + +def _extract_member(tar: tarfile.TarFile, name: str) -> Optional[str]: + """Copy one archive member to a temp file; None if it is missing.""" + try: + member = tar.getmember(name) + except KeyError: + return None + fh = tar.extractfile(member) + if fh is None: + return None + with tempfile.NamedTemporaryFile(delete=False, suffix=".tar") as tmp: + shutil.copyfileobj(fh, tmp) + return tmp.name + + def restore_backup( tar_path: str, target_id: Optional[str] = None, overwrite: bool = False, restore_volumes: bool = True, restore_binds: bool = True, + verified: bool = False, ) -> dict: - """Restore a backup. Returns a report of what was written.""" + """Restore a backup's files, folders and volumes. Returns a report. + + This is the part that writes; :func:`restore` wraps it with the lock and + with stopping and starting the stack. + """ manifest = read_manifest(tar_path) - raw_id = target_id or manifest.get("stack_id") or "" - if not raw_id.strip(): - raise BackupError("Backup manifest has no stack id") - # Slugify whichever id we end up using — the manifest comes from an - # uploaded file, so its stack_id must never be able to escape STACKS_DIR. - stack_id = compose_service.slugify(raw_id) - old_id = manifest.get("stack_id") or stack_id + stack_id = _target_id(manifest, target_id) + old_id = compose_service.slugify(manifest.get("stack_id") or stack_id) + renamed = stack_id != old_id old_dir = manifest.get("stack_dir") or "" directory = compose_service.stack_dir(stack_id) - exists = os.path.isdir(directory) - if exists and not overwrite: + if os.path.isdir(directory) and not overwrite: raise BackupError(f"Stack '{stack_id}' already exists") + if not verified: + verify_archive(tar_path) + excluded, compose_binds = _tree_paths(manifest) + # Not restoring data folders means keeping the ones there are, not + # emptying them: compose/ carries those inside the stack directory. + held_back = compose_binds if not restore_binds else set() volumes_restored = 0 binds_restored = 0 skipped: list[dict] = [] + warnings: list[str] = [] with tarfile.open(tar_path, "r:gz") as tar: - if exists: - shutil.rmtree(directory) - _extract_tree(tar, "compose/", directory) + _restore_tree(tar, directory, excluded | held_back, held_back, warnings) + skipped += [{"kind": "bind", "source": rel, "reason": "not requested"} for rel in held_back] - if restore_binds: - for bind in manifest.get("binds", []): - archive = bind.get("archive") - if not archive: - continue - try: - member = tar.getmember(archive) - except KeyError: - continue - source = bind["source"] + for bind in manifest.get("binds", []): + archive = bind.get("archive") + if not archive: + continue + source = bind["source"] + if not restore_binds: + skipped.append({"kind": "bind", "source": source, "reason": "not requested"}) + continue + if bind.get("inside_stack_dir") and old_dir: # A renamed stack must not write into the old stack's folder. - if old_dir and bind.get("inside_stack_dir"): - rel = os.path.relpath(source, old_dir) - source = os.path.normpath(os.path.join(directory, rel)) - if assets.is_system_path(source): - skipped.append({"kind": "bind", "source": source, "reason": "system path"}) - continue - fh = tar.extractfile(member) - if fh is None: - continue - with tempfile.NamedTemporaryFile(delete=False, suffix=".tar") as tmp: - shutil.copyfileobj(fh, tmp) - part = tmp.name - try: - assets.import_path(source, bind.get("kind", "dir"), part) - binds_restored += 1 - except Exception as exc: # noqa: BLE001 - logger.warning("Could not restore bind %s: %s", source, exc) - skipped.append({"kind": "bind", "source": source, "reason": str(exc)}) - finally: - os.unlink(part) - elif manifest.get("binds"): - skipped += [ - {"kind": "bind", "source": b["source"], "reason": "not requested"} - for b in manifest["binds"] - ] + source = os.path.normpath(os.path.join(directory, os.path.relpath(source, old_dir))) + elif renamed: + skipped.append({ + "kind": "bind", + "source": source, + "reason": f"belongs to the original stack '{old_id}'; " + "restore under that name to write it back", + }) + continue + if assets.is_system_path(source): + skipped.append({"kind": "bind", "source": source, "reason": "system path"}) + continue + part = _extract_member(tar, archive) + if part is None: + skipped.append({"kind": "bind", "source": source, "reason": "missing from archive"}) + continue + try: + assets.import_path(source, bind.get("kind", "dir"), part) + binds_restored += 1 + except Exception as exc: # noqa: BLE001 + logger.warning("Could not restore bind %s: %s", source, exc) + skipped.append({"kind": "bind", "source": source, "reason": str(exc)}) + finally: + os.unlink(part) - if restore_volumes: - for v in manifest.get("volumes", []): - member_name = v.get("archive") or f"volumes/{v['full']}.tar" - try: - member = tar.getmember(member_name) - except KeyError: - continue - fh = tar.extractfile(member) - if fh is None: - continue - # Re-target volume labels to the (possibly new) stack id. - labels = dict(v.get("labels") or {}) + for v in manifest.get("volumes", []): + full = v["full"] + if not restore_volumes: + skipped.append({"kind": "volume", "name": full, "reason": "not requested"}) + continue + retargeted = False + if renamed and full.startswith(old_id + "_"): + full = stack_id + full[len(old_id) :] + retargeted = True + labels = dict(v.get("labels") or {}) + if not v.get("external"): + # Re-target compose's labels to the (possibly new) stack id. labels[COMPOSE_PROJECT_LABEL] = stack_id - full = v["full"] - if target_id and old_id and full.startswith(old_id + "_"): - full = stack_id + full[len(old_id) :] - with tempfile.NamedTemporaryFile(delete=False, suffix=".tar") as tmp: - shutil.copyfileobj(fh, tmp) - part = tmp.name - try: - # Remote-backed volumes (NFS/CIFS) are never wiped: that - # would delete the share the volume points at. - assets.import_volume(full, labels, part, wipe=not v.get("remote", False)) - volumes_restored += 1 - except Exception as exc: # noqa: BLE001 - logger.warning("Could not restore volume %s: %s", full, exc) - skipped.append({"kind": "volume", "name": full, "reason": str(exc)}) - finally: - os.unlink(part) + reason = _volume_conflict(full, stack_id, old_id, renamed and not retargeted) + if reason: + skipped.append({"kind": "volume", "name": full, "reason": reason}) + continue + part = _extract_member(tar, v.get("archive") or f"volumes/{v['full']}.tar") + if part is None: + skipped.append({"kind": "volume", "name": full, "reason": "missing from archive"}) + continue + try: + # Remote-backed volumes (NFS/CIFS) are never wiped: that + # would delete the share the volume points at. + assets.import_volume( + full, + labels, + part, + wipe=not v.get("remote", False), + driver=v.get("driver"), + driver_opts=v.get("options") or None, + ) + volumes_restored += 1 + except Exception as exc: # noqa: BLE001 + logger.warning("Could not restore volume %s: %s", full, exc) + skipped.append({"kind": "volume", "name": full, "reason": str(exc)}) + finally: + os.unlink(part) + # What went wrong now, as opposed to what the backup itself left out + # (listed in skipped as well, for the full picture). + problems = [ + f"{item.get('name') or item.get('source')}: {item['reason']}" + for item in skipped + if item["reason"] != "not requested" + ] + warnings skipped += manifest.get("skipped", []) return { "stack_id": stack_id, @@ -490,4 +975,69 @@ def restore_backup( "volumes_restored": volumes_restored, "binds_restored": binds_restored, "skipped": skipped, + "warnings": warnings, + "problems": problems, } + + +def _volume_conflict(full: str, stack_id: str, old_id: str, shared_with_original: bool) -> Optional[str]: + """Why a volume must not be overwritten, or None if it may be.""" + attrs = assets.volume_attrs(full) + if attrs is None: + return None # created fresh + owner = (attrs.get("Labels") or {}).get(COMPOSE_PROJECT_LABEL) + if owner and owner != stack_id: + return f"belongs to stack '{owner}'; not overwritten" + if shared_with_original: + return ( + f"is not named after the stack, so it is still the original " + f"'{old_id}' stack's volume; not overwritten" + ) + users = assets.running_users(full) + if users: + return f"in use by running container(s) {', '.join(users)}; not overwritten" + return None + + +async def restore( + tar_path: str, + target_id: Optional[str] = None, + overwrite: bool = False, + restore_volumes: bool = True, + restore_binds: bool = True, +) -> dict: + """Restore with the stack locked, stopped while its data is replaced, and + brought back up afterwards if it was running.""" + manifest = await asyncio.to_thread(read_manifest, tar_path) + stack_id = _target_id(manifest, target_id) + directory = compose_service.stack_dir(stack_id) + if os.path.isdir(directory) and not overwrite: + raise BackupError(f"Stack '{stack_id}' already exists") + + with _locked(stack_id, "restore"): + await asyncio.to_thread(verify_archive, tar_path) + try: + running = await _running(stack_id) + except BackupError: + running = [] # no Docker: the volume part will say so + if running: + # Replacing a database's files under it corrupts it. + await _stop(stack_id, running) + try: + result = await asyncio.to_thread( + restore_backup, tar_path, target_id, overwrite, + restore_volumes, restore_binds, True, + ) + except BaseException: + if running: + await _start(stack_id, running) + raise + result["restarted"] = False + if running: + # up, not start: the restored compose file may differ from what ran. + error = await _compose(stack_id, ["up", "-d", "--remove-orphans"]) + result["restarted"] = error is None + if error: + result["restart_error"] = error + result["problems"].append(f"stack did not start again: {error}") + return result diff --git a/backend/services/compose_service.py b/backend/services/compose_service.py index dcb2933..237ac18 100644 --- a/backend/services/compose_service.py +++ b/backend/services/compose_service.py @@ -127,6 +127,10 @@ def discover_stacks(override: Optional[str] = None) -> list[str]: return [] found = [] for entry in sorted(os.listdir(root)): + # Hidden directories are StackPilot's own scratch space (a restore + # stages there before swapping it in), never a stack. + if entry.startswith("."): + continue directory = os.path.join(root, entry) if os.path.isdir(directory) and find_compose_file(directory): found.append(entry) diff --git a/backend/services/schedule_service.py b/backend/services/schedule_service.py index fe1f029..8b31774 100644 --- a/backend/services/schedule_service.py +++ b/backend/services/schedule_service.py @@ -28,6 +28,10 @@ from services import ( logger = logging.getLogger("stackpilot.schedule") +#: A stack that is busy (deploying, updating) when its backup is due is tried +#: again this much later instead of waiting for the next regular run. +BUSY_RETRY = timedelta(minutes=10) + def _now() -> datetime: return datetime.now(timezone.utc).replace(tzinfo=None) @@ -57,11 +61,12 @@ def compute_next_run( return base -def _prune(dest: BackupDestination, basename: str, keep: int) -> int: +def _prune(dest: BackupDestination, stack_id: str, keep: int) -> int: if keep <= 0: return 0 items = dest_service.list_backups(dest) - mine = [i for i in items if i["name"].startswith(f"{basename}-")] + pattern = backup_service.backup_name_pattern(stack_id) + mine = [i for i in items if pattern.match(i["name"])] mine.sort(key=lambda x: x.get("modified") or 0, reverse=True) removed = 0 for old in mine[keep:]: @@ -73,12 +78,28 @@ def _prune(dest: BackupDestination, basename: str, keep: int) -> int: return removed +def _problems(report: dict) -> list[str]: + """What makes a backup that was written less than a success.""" + problems = [f"not captured: {e}" for e in report.get("errors") or []] + if report.get("restart_error"): + problems.append(f"stack did not start again: {report['restart_error']}") + return problems + + +async def _alert(session: Session, title: str, body: str) -> None: + try: + await notify_service.notify(EVENT_BACKUP_FAILED, title, body, session) + except Exception: # noqa: BLE001 + pass + + async def run_schedule(session: Session, schedule: BackupSchedule) -> dict: """Execute one schedule now. Updates status + next_run. Returns a summary.""" now = _now() dest = session.get(BackupDestination, schedule.destination_id) result: dict = {"ok": False} path = None + next_run = None try: if not dest: raise RuntimeError(f"destination {schedule.destination_id} not found") @@ -86,36 +107,50 @@ async def run_schedule(session: Session, schedule: BackupSchedule) -> dict: stack = session.get(Stack, schedule.stack_id) if not stack: raise RuntimeError(f"stack '{schedule.stack_id}' not found") - path = await backup_service.create_backup( + path, report = await backup_service.create_backup_ex( schedule.stack_id, stack.name, include_volumes=schedule.include_volumes, stop_first=schedule.stop_first, ) filename = backup_service.backup_filename(schedule.stack_id, schedule.include_volumes) - basename = backup_service.backup_basename(schedule.stack_id) await asyncio.to_thread(dest_service.upload, dest, path, filename) - pruned = await asyncio.to_thread(_prune, dest, basename, schedule.keep) - schedule.last_status = "ok" - result = {"ok": True, "name": filename, "destination": dest.name, "pruned": pruned} - logger.info("Scheduled backup %s → %s ok (pruned %d)", schedule.stack_id, dest.name, pruned) + problems = _problems(report) + # Retention only after a complete backup: an incomplete one must not + # push the last good backup out. + pruned = 0 if problems else await asyncio.to_thread( + _prune, dest, schedule.stack_id, schedule.keep + ) + result = {"ok": not problems, "name": filename, "destination": dest.name, "pruned": pruned} + if problems: + schedule.last_status = f"warning: {'; '.join(problems)}"[:300] + result["error"] = "; ".join(problems) + logger.warning("Scheduled backup %s incomplete: %s", schedule.stack_id, problems) + await _alert( + session, + f"Scheduled backup of '{schedule.stack_id}' is incomplete", + "\n".join(problems), + ) + else: + schedule.last_status = "ok" + logger.info( + "Scheduled backup %s → %s ok (pruned %d)", schedule.stack_id, dest.name, pruned + ) + except backup_service.StackBusyError as exc: + # Deploys and updates are short: try again soon rather than skipping a day. + schedule.last_status = f"retrying: {exc}"[:300] + result = {"ok": False, "error": str(exc)} + next_run = now + BUSY_RETRY + logger.info("Scheduled backup %s postponed: %s", schedule.stack_id, exc) except Exception as exc: # noqa: BLE001 schedule.last_status = f"error: {exc}"[:300] result = {"ok": False, "error": str(exc)} logger.warning("Scheduled backup %s failed: %s", schedule.stack_id, exc) - try: - await notify_service.notify( - EVENT_BACKUP_FAILED, - f"Scheduled backup of '{schedule.stack_id}' failed", - str(exc), - session, - ) - except Exception: # noqa: BLE001 - pass + await _alert(session, f"Scheduled backup of '{schedule.stack_id}' failed", str(exc)) finally: if path and os.path.exists(path): os.unlink(path) schedule.last_run = now - schedule.next_run = compute_next_run( + schedule.next_run = next_run or compute_next_run( schedule.frequency, schedule.hour, schedule.minute, schedule.weekday, now ) session.add(schedule) diff --git a/backend/services/stack_assets_service.py b/backend/services/stack_assets_service.py index ced1216..c24860a 100644 --- a/backend/services/stack_assets_service.py +++ b/backend/services/stack_assets_service.py @@ -24,7 +24,7 @@ from typing import Optional import yaml from config import settings -from docker_client import DockerError, get_client, safe_call +from docker_client import DockerError, get_client, get_long_client, safe_call from services import compose_service logger = logging.getLogger("stackpilot.assets") @@ -49,6 +49,24 @@ SYSTEM_PREFIXES = ("/dev", "/proc", "/sys", "/run", "/var/run", "/var/lib/docker # user wants, and *restoring* one would overwrite the share. REMOTE_VOLUME_TYPES = {"nfs", "nfs4", "cifs", "smb", "smb3", "smbfs", "sshfs", "glusterfs"} +# The same for bind mounts and bind-type volumes, which only say *where* they +# are, not what: a host directory can be an NFS mount (fstab/autofs) just as +# well. Recognised by the filesystem magic `stat -f` reports for the path. +REMOTE_FS_MAGIC = { + 0x6969: "nfs", + 0x517B: "smb", + 0xFF534D42: "cifs", + 0xFE534D42: "smb2", + 0x00C36400: "ceph", +} +# Mount types in /proc/self/mountinfo that are remote storage. +REMOTE_MOUNT_TYPES = {"nfs", "nfs4", "cifs", "smb3", "smbfs", "ceph", "glusterfs", + "fuse.sshfs", "fuse.rclone", "fuse.glusterfs"} + +# Volumes Docker names itself (64 hex chars): recreated with every container, so +# a restore could never hand them back to the service. +_ANONYMOUS_RE = re.compile(r"^[0-9a-f]{64}$") + # Bind directories larger than this are listed but not selected by default. DEFAULT_MAX_BIND_BYTES = 2 * 1024**3 @@ -116,7 +134,7 @@ def _put_archive(container, dest: str, src_file: str, volumes: dict) -> None: if not _is_chown_error(exc): raise logger.warning("%s rejects ownership changes; restoring without it", dest) - client = get_client() + client = get_long_client() staging = _create_helper_with( client, volumes, ["sh", "-c", f"cp -R /tmp/. '{dest}/'"] ) @@ -142,26 +160,35 @@ def _create_helper_with(client, volumes: dict, command: list[str]): ) -def inspect_paths(paths: list[str]) -> dict[str, dict]: - """Classify host paths ({path: {"kind", "size"}}) via one helper container. +def _unknown() -> dict: + return {"kind": "unknown", "size": None, "fs": None, "remote": None, "measured": False} - ``kind`` is dir / file / special; ``size`` is bytes (best effort, the walk is - capped so a huge media share can't stall the request). + +def inspect_paths(paths: list[str]) -> dict[str, dict]: + """Classify host paths via one helper container. + + ``{path: {"kind", "size", "fs", "remote", "measured"}}``: ``kind`` is dir / + file / special, ``size`` bytes. ``remote`` names the network filesystem the + path sits on (nfs, cifs, ...), whose contents are not walked at all. + Directory sizes are capped at 20s of ``du``; ``measured`` is False when that + ran out, which on its own says the directory is big. """ unique = [p for p in dict.fromkeys(paths) if p] if not unique: return {} - client = get_client() + client = get_long_client() ensure_helper_image(client) mounts = {p: {"bind": f"/m/{i}", "mode": "ro"} for i, p in enumerate(unique)} + remote_hex = "|".join(f"{m:x}" for m in REMOTE_FS_MAGIC) script_parts = [] for i in range(len(unique)): script_parts.append( - f'd=/m/{i}; ' - f'if [ -d "$d" ]; then s=$(timeout 20 du -sk "$d" 2>/dev/null | cut -f1); ' - f'echo "{i} dir ${{s:-}}"; ' - f'elif [ -f "$d" ]; then echo "{i} file $(stat -c %s "$d" 2>/dev/null)"; ' - f'else echo "{i} special"; fi' + f'd=/m/{i}; t=$(stat -f -c %t "$d" 2>/dev/null); t=${{t:--}}; ' + f'if [ -d "$d" ]; then ' + f'case "$t" in {remote_hex}) echo "{i} dir - $t";; ' + f'*) s=$(timeout 20 du -sk "$d" 2>/dev/null | cut -f1); echo "{i} dir ${{s:--}} $t";; esac; ' + f'elif [ -f "$d" ]; then echo "{i} file $(stat -c %s "$d" 2>/dev/null) $t"; ' + f'else echo "{i} special - $t"; fi' ) script = "; ".join(script_parts) try: @@ -176,9 +203,9 @@ def inspect_paths(paths: list[str]) -> dict[str, dict]: ) except DockerError as exc: logger.warning("Path inspection failed: %s", exc) - return {p: {"kind": "unknown", "size": None} for p in unique} + return {p: _unknown() for p in unique} - result: dict[str, dict] = {p: {"kind": "unknown", "size": None} for p in unique} + result: dict[str, dict] = {p: _unknown() for p in unique} for line in (out or b"").decode("utf-8", "replace").splitlines(): parts = line.strip().split() if len(parts) < 2 or not parts[0].isdigit(): @@ -190,13 +217,24 @@ def inspect_paths(paths: list[str]) -> dict[str, dict]: size: Optional[int] = None if len(parts) > 2 and parts[2].isdigit(): size = int(parts[2]) * 1024 if kind == "dir" else int(parts[2]) - result[unique[idx]] = {"kind": kind, "size": size} + fs = parts[3] if len(parts) > 3 and parts[3] != "-" else None + try: + remote = REMOTE_FS_MAGIC.get(int(fs, 16)) if fs else None + except ValueError: + remote = None + result[unique[idx]] = { + "kind": kind, + "size": size, + "fs": fs, + "remote": remote, + "measured": kind != "dir" or remote is not None or size is not None, + } return result def export_path(source: str, kind: str, dest_file: str) -> int: """Tar a host path (dir contents, or a single file) into ``dest_file``.""" - client = get_client() + client = get_long_client() ensure_helper_image(client) if kind == "file": parent, base = _split(source) @@ -223,7 +261,7 @@ def export_path(source: str, kind: str, dest_file: str) -> int: def import_path(source: str, kind: str, src_file: str) -> None: """Unpack an archive produced by :func:`export_path` back to its host path.""" - client = get_client() + client = get_long_client() ensure_helper_image(client) if kind == "file": parent, _base = _split(source) @@ -239,7 +277,7 @@ def import_path(source: str, kind: str, src_file: str) -> None: def export_volume(full_name: str, dest_file: str) -> int: """Stream a named volume's contents into ``dest_file`` (never into RAM).""" - client = get_client() + client = get_long_client() ensure_helper_image(client) container = _create_helper(client, {full_name: {"bind": "/v", "mode": "ro"}}) written = 0 @@ -254,16 +292,53 @@ def export_volume(full_name: str, dest_file: str) -> int: return written -def import_volume(full_name: str, labels: dict, src_file: str, wipe: bool = True) -> None: - """Restore a volume from an archive, optionally clearing it first.""" - client = get_client() +def volume_attrs(full_name: str) -> Optional[dict]: + """A volume's inspect data, or None if it does not exist.""" + try: + return safe_call(get_client().volumes.get, full_name).attrs or {} + except DockerError: + return None + + +def running_users(full_name: str) -> list[str]: + """Names of running containers that have the volume mounted.""" + try: + containers = safe_call( + get_client().containers.list, filters={"volume": full_name, "status": "running"} + ) + except DockerError: + return [] + return sorted(c.name for c in containers) + + +def import_volume( + full_name: str, + labels: dict, + src_file: str, + wipe: bool = True, + driver: Optional[str] = None, + driver_opts: Optional[dict] = None, +) -> None: + """Restore a volume from an archive, optionally clearing it first. + + A missing volume is recreated with the driver and options it had, not as a + plain local volume: compose refuses to use an existing volume whose + configuration differs from the compose file. + """ + client = get_long_client() ensure_helper_image(client) existed = True try: safe_call(client.volumes.get, full_name) except DockerError: existed = False - safe_call(client.volumes.create, name=full_name, labels=labels or {}) + safe_call( + client.volumes.create, + name=full_name, + driver=driver or "local", + driver_opts=driver_opts or None, + labels=labels or {}, + ) if existed and wipe: # Restore means "back to the snapshot": drop files created since. safe_call( @@ -429,8 +504,77 @@ def _inside(path: str, parent: str) -> bool: return path == parent or path.startswith(parent + "/") +def _volume_entry(name: str, attrs: dict, external: bool) -> dict: + labels = attrs.get("Labels") or {} + options = attrs.get("Options") or {} + driver = attrs.get("Driver", "local") + vtype = str(options.get("type") or "").lower() + device = str(options.get("device") or "") + remote = ( + vtype in REMOTE_VOLUME_TYPES + or driver != "local" + or device.startswith("//") + or device.startswith(":") + ) + # A local volume that binds a host directory (type none, o=bind) is only as + # local as that directory; inventory() checks what it sits on. + bind_device = ( + device + if not remote and device.startswith("/") and "bind" in str(options.get("o") or "") + else None + ) + return { + "name": name, + "short": labels.get(COMPOSE_VOLUME_LABEL, name), + "labels": labels, + "driver": driver, + "options": options, + "remote": remote, + "remote_type": vtype or (driver if driver != "local" else None), + "external": external, + "bind_device": bind_device, + } + + +def _external_volume_names(stack_id: str) -> set[str]: + """Named volumes the stack uses but compose did not create for it: + ``external: true`` in the compose file, or mounted by its containers.""" + names: set[str] = set() + try: + client = get_client() + containers = safe_call( + client.containers.list, + all=True, + filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"}, + ) + for c in containers: + for mount in c.attrs.get("Mounts") or []: + if mount.get("Type") == "volume" and mount.get("Name"): + names.add(mount["Name"]) + except DockerError: + pass + compose_file = compose_service.find_compose_file(compose_service.stack_dir(stack_id)) + if compose_file: + try: + with open(compose_file, "r", encoding="utf-8", errors="replace") as fh: + data = yaml.safe_load(fh) or {} + except (OSError, yaml.YAMLError): + data = {} + env = _env_for_stack(stack_id) + for key, spec in (data.get("volumes") or {}).items(): + if isinstance(spec, dict) and spec.get("external"): + names.add(_interpolate(str(spec.get("name") or key), env)) + return names + + def compose_volumes(stack_id: str) -> list[dict]: - """Compose-managed named volumes, with remote-storage detection.""" + """The stack's named volumes, with remote-storage detection. + + Compose-created ones carry the project label. Volumes the stack only uses + (``external: true``, or created by hand) do not, and were silently missing + from backups before; they are listed with ``external`` set. Anonymous + volumes are listed too, so it is visible that data lives in one. + """ try: client = get_client() vols = safe_call( @@ -439,34 +583,43 @@ def compose_volumes(stack_id: str) -> list[dict]: ) except DockerError: return [] - out = [] - for v in vols: - attrs = v.attrs or {} - labels = attrs.get("Labels") or {} - options = attrs.get("Options") or {} - driver = attrs.get("Driver", "local") - vtype = str(options.get("type") or "").lower() - device = str(options.get("device") or "") - remote = ( - vtype in REMOTE_VOLUME_TYPES - or driver != "local" - or device.startswith("//") - or device.startswith(":") - ) - out.append( - { - "name": v.name, - "short": labels.get(COMPOSE_VOLUME_LABEL, v.name), - "labels": labels, - "driver": driver, - "options": options, - "remote": remote, - "remote_type": vtype or (driver if driver != "local" else None), - } - ) + out = [_volume_entry(v.name, v.attrs or {}, external=False) for v in vols] + known = {v["name"] for v in out} + for name in sorted(_external_volume_names(stack_id) - known): + attrs = volume_attrs(name) + if attrs is None: + continue # declared external but not created yet + out.append(_volume_entry(name, attrs, external=True)) return out +def _mounts() -> list[tuple[str, str]]: + """(mount point, filesystem type) of every mount this process sees.""" + out = [] + try: + with open("/proc/self/mountinfo", "r", encoding="utf-8") as fh: + for line in fh: + left, _, right = line.partition(" - ") + fields, rfields = left.split(), right.split() + if len(fields) >= 5 and rfields: + # Spaces etc. are octal-escaped in mountinfo. + point = fields[4].encode().decode("unicode_escape") + out.append((point, rfields[0])) + except OSError: + pass + return out + + +def remote_mount_type(path: str) -> Optional[str]: + """The remote filesystem type ``path`` lives on as this process sees it, + e.g. an NFS share mounted inside the stacks directory. None if local.""" + best, best_type = "", None + for point, fstype in _mounts(): + if _inside(path, point) and len(point) >= len(best): + best, best_type = point, fstype + return best_type if best_type in REMOTE_MOUNT_TYPES else None + + def inventory(stack_id: str, max_bind_bytes: int = DEFAULT_MAX_BIND_BYTES) -> dict: """What a backup of this stack would (and would not) capture. @@ -488,14 +641,18 @@ def inventory(stack_id: str, max_bind_bytes: int = DEFAULT_MAX_BIND_BYTES) -> di if mount not in entry["mounts"]: entry["mounts"].append(mount) + volumes_raw = compose_volumes(stack_id) real_paths = [p for p in grouped if not is_system_path(p)] - stats = inspect_paths(real_paths) + stats = inspect_paths( + real_paths + [v["bind_device"] for v in volumes_raw if v["bind_device"]] + ) binds = [] for path, entry in sorted(grouped.items()): system = is_system_path(path) - info = stats.get(path, {"kind": "unknown", "size": None}) + info = stats.get(path) or _unknown() kind, size = info["kind"], info["size"] + remote = info.get("remote") or (remote_mount_type(path) if not system else None) inside = _inside(path, directory) # A path inside the stack directory that this process can actually read # is already covered by the compose/ tree in the archive. @@ -504,12 +661,17 @@ def inventory(stack_id: str, max_bind_bytes: int = DEFAULT_MAX_BIND_BYTES) -> di reason = None if system: include, reason = False, "system path" + elif remote: + include, reason = False, f"remote storage ({remote})" elif kind == "special": include, reason = False, "not a regular file or directory" elif kind == "unknown": include, reason = False, "could not inspect path" elif size is not None and size > max_bind_bytes: include, reason = False, f"larger than {max_bind_bytes // 1024**3} GiB" + elif not info.get("measured", True): + # du ran out of time: too big to have been walked in 20s. + include, reason = False, "too large to measure" binds.append( { "source": path, @@ -522,21 +684,24 @@ def inventory(stack_id: str, max_bind_bytes: int = DEFAULT_MAX_BIND_BYTES) -> di "covered_by_compose": visible, "via": "compose" if visible else "archive", "system": system, + "remote": remote, "include_default": include, "reason": reason, } ) volumes = [] - for vol in compose_volumes(stack_id): - include = not vol["remote"] - volumes.append( - { - **vol, - "include_default": include, - "reason": None if include else f"remote storage ({vol['remote_type']})", - } - ) + for vol in volumes_raw: + if vol["bind_device"]: + device_remote = (stats.get(vol["bind_device"]) or {}).get("remote") + if device_remote: + vol = {**vol, "remote": True, "remote_type": device_remote} + include, reason = True, None + if vol["remote"]: + include, reason = False, f"remote storage ({vol['remote_type']})" + elif _ANONYMOUS_RE.match(vol["name"]): + include, reason = False, "anonymous volume (Docker recreates it with the container)" + volumes.append({**vol, "include_default": include, "reason": reason}) return { "stack_id": stack_id, diff --git a/backend/tests/test_backup_restore.py b/backend/tests/test_backup_restore.py new file mode 100644 index 0000000..edb539e --- /dev/null +++ b/backend/tests/test_backup_restore.py @@ -0,0 +1,817 @@ +"""Backup and restore, end to end on real files and real archives. + +Docker is faked (containers, volumes, the helper container); everything that +touches the stack directory and the archive runs for real. The cases are the +ways a backup used to fail to be one: + +* data folders inside the stack directory were archived while the stack kept + writing to them (it was only stopped when named volumes were selected); +* the stack came back with ``up -d`` — starting services that had been + stopped on purpose and applying undeployed compose edits; +* a restore with "overwrite" deleted the whole stack directory, including the + folders the backup had left out (too large, deselected, NFS); +* a truncated archive was only noticed after the old stack was gone; +* a restore under a new name overwrote the original stack's volumes; +* retention for stack "app" deleted the backups of stack "app-db"; +* a backup with a failed volume was reported as "ok". +""" +from __future__ import annotations + +import asyncio +import gzip +import io +import json +import os +import tarfile +import uuid + +import pytest + +from services import backup_service, compose_service, stack_assets_service as assets +from services.compose_service import ContainerInfo + + +# --------------------------------------------------------------------------- # +# Fakes +# --------------------------------------------------------------------------- # + + +class FakeDocker: + """Records what the backup asked Docker/compose to do.""" + + def __init__(self): + self.containers: list[ContainerInfo] = [] + self.compose_calls: list[list[str]] = [] + self.compose_fail: set[str] = set() + self.volumes: dict[str, dict] = {} # name -> {"files": {...}, "labels", ...} + self.running_users: dict[str, list[str]] = {} + self.imported: list[dict] = [] + self.export_fail: set[str] = set() + + def container(self, service: str, state: str = "running") -> ContainerInfo: + c = ContainerInfo( + id=f"id-{service}", name=f"x-{service}-1", service=service, + image="img", state=state, status=state, + ) + self.containers.append(c) + return c + + # compose_service + def containers_for_stack(self, stack_id): + return list(self.containers) + + async def run_compose(self, stack_id, args, override=None, timeout=600.0): + self.compose_calls.append(list(args)) + failed = args[0] in self.compose_fail + if not failed and args[0] == "stop": + for c in self.containers: + if c.service in args[1:]: + c.state = "exited" + if not failed and args[0] in ("start", "up"): + for c in self.containers: + if args[0] == "up" or c.service in args[1:]: + c.state = "running" + return { + "returncode": 1 if failed else 0, + "stdout": "", + "stderr": f"{args[0]} failed" if failed else "", + "command": " ".join(args), + } + + # stack_assets_service + def export_volume(self, name, dest_file): + if name in self.export_fail: + raise RuntimeError("helper container could not mount the volume") + with tarfile.open(dest_file, "w") as tar: + for rel, content in self.volumes[name]["files"].items(): + info = tarfile.TarInfo(rel) + info.size = len(content) + tar.addfile(info, io.BytesIO(content)) + return os.path.getsize(dest_file) + + def import_volume(self, name, labels, src_file, wipe=True, driver=None, driver_opts=None): + with tarfile.open(src_file) as tar: + files = {m.name: tar.extractfile(m).read() for m in tar.getmembers() if m.isreg()} + self.imported.append( + {"name": name, "labels": labels, "wipe": wipe, "driver": driver, + "driver_opts": driver_opts, "files": files} + ) + self.volumes[name] = {"files": files, "labels": labels} + + def volume_attrs(self, name): + vol = self.volumes.get(name) + return None if vol is None else {"Labels": vol.get("labels") or {}} + + +@pytest.fixture +def docker(monkeypatch, db): + fake = FakeDocker() + monkeypatch.setattr(compose_service, "containers_for_stack", fake.containers_for_stack) + monkeypatch.setattr(compose_service, "run_compose", fake.run_compose) + monkeypatch.setattr(assets, "export_volume", fake.export_volume) + monkeypatch.setattr(assets, "import_volume", fake.import_volume) + monkeypatch.setattr(assets, "volume_attrs", fake.volume_attrs) + monkeypatch.setattr(assets, "running_users", lambda name: fake.running_users.get(name, [])) + + def no_docker(ids, action): + raise RuntimeError("docker unavailable") + + monkeypatch.setattr(backup_service, "_docker_each", no_docker) + return fake + + +def _bind(source, directory, size=1024, include=True, reason=None, via="compose"): + inside = source == directory or source.startswith(directory + "/") + return { + "source": source, "mounts": [{"service": "app", "target": "/data"}], + "kind": "dir", "size": size, "inside_stack_dir": inside, + "covered_by_compose": via == "compose", "via": via, "system": False, + "remote": None, "include_default": include, "reason": reason, + } + + +def _volume(name, stack_id, include=True, reason=None, external=False, remote=False): + labels = {} if external else { + assets.COMPOSE_PROJECT_LABEL: stack_id, + assets.COMPOSE_VOLUME_LABEL: name.split("_", 1)[-1], + } + return { + "name": name, "short": name.split("_", 1)[-1], "labels": labels, + "driver": "local", "options": {"type": "none", "o": "bind", "device": "/srv/x"} if remote else {}, + "remote": remote, "remote_type": "nfs" if remote else None, + "external": external, "bind_device": None, + "include_default": include, "reason": reason, + } + + +@pytest.fixture +def stack(monkeypatch, docker): + """A stack laid out the way the user keeps them: compose, .env and data + folders side by side in /opt/stacks/.""" + stack_id = f"s{uuid.uuid4().hex[:8]}" + directory = compose_service.stack_dir(stack_id) + os.makedirs(os.path.join(directory, "config")) + os.makedirs(os.path.join(directory, "media", "movies")) + _write(directory, "compose.yaml", "services:\n app:\n image: img\n") + _write(directory, ".env", "PASSWORD=hunter2\n") + _write(directory, "config/app.db", "state-at-backup") + _write(directory, "media/movies/big.mkv", "x" * 100) + + inv = { + "stack_id": stack_id, "stack_dir": directory, "stack_dir_visible": True, + "path_mismatch": None, + "binds": [ + _bind(os.path.join(directory, "config"), directory), + # Too big for a backup: listed, not selected, left out of compose/. + _bind(os.path.join(directory, "media"), directory, size=500 * 1024**3, + include=False, reason="larger than 2 GiB"), + ], + "volumes": [], + } + monkeypatch.setattr(assets, "inventory", lambda sid, max_bind_bytes=0: json.loads(json.dumps(inv))) + return stack_id, directory, inv + + +def _write(directory, rel, content): + path = os.path.join(directory, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as fh: + fh.write(content) + + +def _read(directory, rel): + with open(os.path.join(directory, rel)) as fh: + return fh.read() + + +def _names(archive): + with tarfile.open(archive, "r:gz") as tar: + return set(tar.getnames()) + + +def run(coro): + return asyncio.run(coro) + + +# --------------------------------------------------------------------------- # +# Backup +# --------------------------------------------------------------------------- # + + +def test_data_folder_in_stack_dir_stops_the_stack_and_starts_exactly_what_ran(stack, docker): + stack_id, directory, _ = stack + docker.container("app", "running") + docker.container("worker", "exited") # stopped on purpose + + path, report = run(backup_service.create_backup_ex(stack_id, "S")) + try: + assert docker.compose_calls == [["stop", "app"], ["start", "app"]] + assert report["stopped"] == ["app"] + assert report["restart_error"] is None and report["errors"] == [] + assert [c.state for c in docker.containers] == ["running", "exited"] + finally: + os.unlink(path) + + +def test_stopped_stack_is_not_started_by_a_backup(stack, docker): + stack_id, _, _ = stack + docker.container("app", "exited") + + path, _ = run(backup_service.create_backup_ex(stack_id, "S")) + os.unlink(path) + assert docker.compose_calls == [] + + +def test_large_folder_stays_out_but_its_mount_point_stays_in(stack): + stack_id, directory, _ = stack + + path, report = run(backup_service.create_backup_ex(stack_id, "S")) + try: + names = _names(path) + manifest = backup_service.read_manifest(path) + finally: + os.unlink(path) + assert {"compose/compose.yaml", "compose/.env", "compose/config/app.db"} <= names + assert "compose/media" in names + assert not any(n.startswith("compose/media/") for n in names) + assert manifest["format_version"] == 3 + assert manifest["excluded"] == ["media"] + assert manifest["compose_binds"] == ["config"] + assert [b["source"] for b in report["binds"]] == [os.path.join(directory, "config")] + + +def test_stack_that_cannot_be_stopped_is_not_backed_up(stack, docker): + stack_id, _, _ = stack + docker.container("app", "running") + docker.compose_fail.add("stop") + + with pytest.raises(backup_service.BackupError, match="consistent snapshot"): + run(backup_service.create_backup_ex(stack_id, "S")) + # Whatever did stop is started again. + assert docker.compose_calls[-1] == ["start", "app"] + + +def test_restart_failure_is_reported_not_swallowed(stack, docker): + stack_id, _, _ = stack + docker.container("app", "running") + docker.compose_fail.add("start") + + path, report = run(backup_service.create_backup_ex(stack_id, "S")) + os.unlink(path) + assert report["restart_error"] + + +def test_failed_volume_is_an_error_and_listed_as_skipped(stack, docker): + stack_id, _, inv = stack + inv["volumes"].append(_volume(f"{stack_id}_db", stack_id)) + docker.export_fail.add(f"{stack_id}_db") + + path, report = run(backup_service.create_backup_ex(stack_id, "S")) + try: + manifest = backup_service.read_manifest(path) + finally: + os.unlink(path) + assert report["errors"] and f"{stack_id}_db" in report["errors"][0] + assert manifest["volumes"] == [] + assert any(s.get("name") == f"{stack_id}_db" and "backup failed" in s["reason"] + for s in manifest["skipped"]) + + +def test_file_that_shrinks_while_read_keeps_the_archive_valid(tmp_path, monkeypatch): + root = tmp_path / "stack" + root.mkdir() + log = root / "log.txt" + log.write_bytes(b"a" * 5000) + + class RotatedMidRead: + """The file is truncated after its size was taken, before it is read.""" + + def __init__(self, fh): + self._fh = fh + self.name = fh.name + + def fileno(self): + return self._fh.fileno() + + def read(self, n=-1): + if log.stat().st_size > 100: + os.truncate(log, 100) + return self._fh.read(n) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self._fh.close() + + real_open = open + + def rotating_open(path, mode="r", *args, **kwargs): + fh = real_open(path, mode, *args, **kwargs) + return RotatedMidRead(fh) if str(path) == str(log) else fh + + monkeypatch.setattr("builtins.open", rotating_open) + archive = tmp_path / "a.tar.gz" + warnings: list[str] = [] + with tarfile.open(archive, "w:gz") as tar: + backup_service._add_tree(tar, str(root), "compose", set(), warnings) + monkeypatch.undo() + + backup_service.verify_archive(str(archive)) + assert warnings == ["log.txt: changed while it was read"] + with tarfile.open(archive, "r:gz") as tar: + data = tar.extractfile("compose/log.txt").read() + assert data == b"a" * 100 + b"\0" * 4900 + + +def test_backup_of_a_busy_stack_is_refused(stack): + from sqlmodel import Session + + from database import engine + from services import stack_lock_service + + stack_id, _, _ = stack + with Session(engine) as session: + stack_lock_service.acquire(session, stack_id, "update", "someone") + try: + with pytest.raises(backup_service.StackBusyError): + run(backup_service.create_backup_ex(stack_id, "S")) + finally: + stack_lock_service.release(session, stack_id) + + +# --------------------------------------------------------------------------- # +# Restore +# --------------------------------------------------------------------------- # + + +def _backup(stack_id): + path, _ = run(backup_service.create_backup_ex(stack_id, "S")) + return path + + +def test_overwrite_restores_the_snapshot_and_keeps_what_was_left_out(stack, docker): + stack_id, directory, _ = stack + archive = _backup(stack_id) + try: + # Life goes on after the backup... + _write(directory, "config/app.db", "state-after-backup") + _write(directory, "config/wal-0002", "newer than the snapshot") + _write(directory, "media/movies/new.mkv", "y") + _write(directory, "compose.yaml", "services: {}\n") + + result = run(backup_service.restore(archive, overwrite=True)) + finally: + os.unlink(archive) + + assert _read(directory, "config/app.db") == "state-at-backup" + assert not os.path.exists(os.path.join(directory, "config/wal-0002")) + assert _read(directory, "compose.yaml").startswith("services:\n app:") + assert _read(directory, ".env") == "PASSWORD=hunter2\n" + # Not in the backup, so not the restore's to delete. + assert _read(directory, "media/movies/big.mkv") == "x" * 100 + assert _read(directory, "media/movies/new.mkv") == "y" + assert result["stack_id"] == stack_id + # "media" was left out when the backup was made: listed, but not a problem. + assert result["problems"] == [] + # Nothing staged is left behind or picked up as a stack. + root = os.path.dirname(directory) + assert not [n for n in os.listdir(root) if n.startswith(".restore-")] + + +def test_restore_stops_a_running_stack_and_brings_it_back_up(stack, docker): + stack_id, _, _ = stack + archive = _backup(stack_id) + docker.container("app", "running") + docker.compose_calls.clear() + try: + result = run(backup_service.restore(archive, overwrite=True)) + finally: + os.unlink(archive) + assert docker.compose_calls == [["stop", "app"], ["up", "-d", "--remove-orphans"]] + assert result["restarted"] is True + + +def test_without_restore_binds_the_data_folders_are_kept(stack): + stack_id, directory, _ = stack + archive = _backup(stack_id) + try: + _write(directory, "config/app.db", "state-after-backup") + _write(directory, "compose.yaml", "services: {}\n") + run(backup_service.restore(archive, overwrite=True, restore_binds=False)) + finally: + os.unlink(archive) + assert _read(directory, "config/app.db") == "state-after-backup" + assert _read(directory, "compose.yaml").startswith("services:\n app:") + + +def test_restore_to_a_new_name_creates_the_stack(stack): + stack_id, directory, _ = stack + archive = _backup(stack_id) + new_id = stack_id + "-copy" + try: + result = run(backup_service.restore(archive, target_id=new_id)) + finally: + os.unlink(archive) + new_dir = compose_service.stack_dir(new_id) + assert result["stack_id"] == new_id + assert _read(new_dir, "config/app.db") == "state-at-backup" + assert os.path.isdir(os.path.join(new_dir, "media")) # the mount point + assert os.listdir(os.path.join(new_dir, "media")) == [] + + +def test_existing_stack_without_overwrite_is_refused(stack): + stack_id, _, _ = stack + archive = _backup(stack_id) + try: + with pytest.raises(backup_service.BackupError, match="already exists"): + run(backup_service.restore(archive)) + finally: + os.unlink(archive) + + +def test_truncated_archive_is_refused_before_anything_is_touched(stack): + stack_id, directory, _ = stack + archive = _backup(stack_id) + with open(archive, "rb") as fh: + data = fh.read() + with open(archive, "wb") as fh: + fh.write(data[: len(data) // 2]) + _write(directory, "config/app.db", "current") + try: + with pytest.raises(backup_service.BackupError, match="damaged or incomplete"): + run(backup_service.restore(archive, overwrite=True)) + finally: + os.unlink(archive) + assert _read(directory, "config/app.db") == "current" + + +def test_archive_cannot_write_through_its_own_symlink(tmp_path): + outside = tmp_path / "outside.txt" + outside.write_text("untouched") + archive = tmp_path / "evil.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + link = tarfile.TarInfo("compose/x") + link.type = tarfile.SYMTYPE + link.linkname = str(outside) + tar.addfile(link) + payload = b"pwned" + info = tarfile.TarInfo("compose/x") + info.size = len(payload) + tar.addfile(info, io.BytesIO(payload)) + dest = tmp_path / "dest" + with tarfile.open(archive, "r:gz") as tar: + backup_service._extract_tree(tar, "compose/", str(dest)) + assert outside.read_text() == "untouched" + assert (dest / "x").read_bytes() == b"pwned" + + +def test_volumes_restore_with_their_driver_and_options(stack, docker): + stack_id, _, inv = stack + vol = _volume(f"{stack_id}_db", stack_id) + vol["options"] = {"type": "none", "o": "bind", "device": "/srv/db"} + inv["volumes"].append(vol) + docker.volumes[f"{stack_id}_db"] = {"files": {"PG_VERSION": b"16"}, "labels": vol["labels"]} + archive = _backup(stack_id) + del docker.volumes[f"{stack_id}_db"] + try: + result = run(backup_service.restore(archive, overwrite=True)) + finally: + os.unlink(archive) + assert result["volumes_restored"] == 1 + imported = docker.imported[0] + assert imported["driver_opts"] == {"type": "none", "o": "bind", "device": "/srv/db"} + assert imported["files"] == {"PG_VERSION": b"16"} + + +def test_renamed_restore_never_touches_the_originals_volumes(stack, docker): + stack_id, _, inv = stack + inv["volumes"] += [ + _volume(f"{stack_id}_db", stack_id), + # explicit `name:` in compose, so not prefixed with the stack id + {**_volume("shared_media", stack_id), "short": "media"}, + ] + docker.volumes[f"{stack_id}_db"] = {"files": {"a": b"1"}, "labels": inv["volumes"][0]["labels"]} + docker.volumes["shared_media"] = {"files": {"b": b"2"}, "labels": inv["volumes"][1]["labels"]} + archive = _backup(stack_id) + new_id = stack_id + "-test" + try: + result = run(backup_service.restore(archive, target_id=new_id)) + finally: + os.unlink(archive) + assert [i["name"] for i in docker.imported] == [f"{new_id}_db"] + assert docker.imported[0]["labels"][assets.COMPOSE_PROJECT_LABEL] == new_id + reasons = {s.get("name"): s["reason"] for s in result["skipped"] if s["kind"] == "volume"} + assert "belongs to stack" in reasons["shared_media"] + assert any(p.startswith("shared_media: belongs to stack") for p in result["problems"]) + assert docker.volumes["shared_media"]["files"] == {"b": b"2"} + + +def test_volume_in_use_by_another_stack_is_not_overwritten(stack, docker): + stack_id, _, inv = stack + inv["volumes"].append(_volume("common", stack_id, external=True)) + docker.volumes["common"] = {"files": {"a": b"1"}, "labels": {}} + archive = _backup(stack_id) + docker.running_users["common"] = ["other-app-1"] + try: + result = run(backup_service.restore(archive, overwrite=True)) + finally: + os.unlink(archive) + assert docker.imported == [] + assert any("other-app-1" in s["reason"] for s in result["skipped"] if s.get("name") == "common") + + +def test_external_volume_keeps_its_labels(stack, docker): + stack_id, _, inv = stack + inv["volumes"].append(_volume("common", stack_id, external=True)) + docker.volumes["common"] = {"files": {"a": b"1"}, "labels": {}} + archive = _backup(stack_id) + del docker.volumes["common"] + try: + run(backup_service.restore(archive, overwrite=True)) + finally: + os.unlink(archive) + assert docker.imported[0]["labels"] == {} + + +def test_version_2_backup_keeps_its_skipped_folders(stack, tmp_path): + """Backups made before this change have no "excluded" list.""" + stack_id, directory, _ = stack + archive = tmp_path / "v2.tar.gz" + manifest = { + "format_version": 2, "stack_id": stack_id, "name": "S", "stack_dir": directory, + "volumes": [], "binds": [], + "skipped": [{"kind": "bind", "source": os.path.join(directory, "media"), + "reason": "larger than 2 GiB"}], + } + with tarfile.open(archive, "w:gz") as tar: + data = json.dumps(manifest).encode() + info = tarfile.TarInfo("manifest.json") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + body = b"services: {}\n" + info = tarfile.TarInfo("compose/compose.yaml") + info.size = len(body) + tar.addfile(info, io.BytesIO(body)) + + run(backup_service.restore(str(archive), overwrite=True)) + assert _read(directory, "compose.yaml") == "services: {}\n" + assert _read(directory, "media/movies/big.mkv") == "x" * 100 + assert not os.path.exists(os.path.join(directory, "config")) + + +def test_verify_accepts_a_good_archive_and_rejects_garbage(stack, tmp_path): + stack_id, _, _ = stack + archive = _backup(stack_id) + try: + backup_service.verify_archive(archive) + finally: + os.unlink(archive) + junk = tmp_path / "junk.tar.gz" + junk.write_bytes(gzip.compress(b"not a tar")) + with pytest.raises(backup_service.BackupError): + backup_service.verify_archive(str(junk)) + + +# --------------------------------------------------------------------------- # +# Retention and schedules +# --------------------------------------------------------------------------- # + + +def test_retention_pattern_is_exact(): + pattern = backup_service.backup_name_pattern("app") + assert pattern.match("backup-app-full-20260918-030000.tar.gz") + assert pattern.match("backup-app-config-20260918-030000.tar.gz") + assert not pattern.match("backup-app-db-full-20260918-030000.tar.gz") + assert not pattern.match("backup-app-full-20260918-030000.tar.gz.part") + + +def test_retention_leaves_other_stacks_alone(monkeypatch): + from services import schedule_service + + names = [ + "backup-app-full-20260910-030000.tar.gz", + "backup-app-full-20260911-030000.tar.gz", + "backup-app-full-20260912-030000.tar.gz", + "backup-app-db-full-20260901-030000.tar.gz", + ] + deleted: list[str] = [] + monkeypatch.setattr( + schedule_service.dest_service, "list_backups", + lambda dest: [{"name": n, "modified": i} for i, n in enumerate(names)], + ) + monkeypatch.setattr(schedule_service.dest_service, "delete", lambda dest, n: deleted.append(n)) + + assert schedule_service._prune(object(), "app", keep=2) == 1 + assert deleted == ["backup-app-full-20260910-030000.tar.gz"] + + +def _schedule_env(monkeypatch, db, report=None, error=None): + from sqlmodel import Session + + from models.backup_destination import BackupDestination + from models.backup_schedule import BackupSchedule + from models.stack import Stack + from services import schedule_service + + stack_id = f"s{uuid.uuid4().hex[:8]}" + alerts: list[str] = [] + pruned: list[str] = [] + + async def fake_create(*args, **kwargs): + if error: + raise error + path = os.path.join(compose_service.stacks_root(), f".{stack_id}.tar.gz") + open(path, "wb").close() + return path, report + + async def fake_notify(event, title, body, session): + alerts.append(title) + + monkeypatch.setattr(schedule_service.backup_service, "create_backup_ex", fake_create) + monkeypatch.setattr(schedule_service.dest_service, "upload", lambda d, p, f: f) + monkeypatch.setattr(schedule_service, "_prune", lambda d, s, k: pruned.append(s) or 0) + monkeypatch.setattr(schedule_service.notify_service, "notify", fake_notify) + + session = Session(db) + session.add(Stack(id=stack_id, name=stack_id)) + dest = BackupDestination(name=f"d-{stack_id}", type="sftp", config="{}") + session.add(dest) + session.commit() + schedule = BackupSchedule(stack_id=stack_id, destination_id=dest.id, frequency="daily") + session.add(schedule) + session.commit() + return schedule_service, session, schedule, alerts, pruned + + +def test_incomplete_scheduled_backup_warns_and_keeps_old_backups(monkeypatch, db): + report = {"errors": ["db: helper failed"], "restart_error": None} + svc, session, schedule, alerts, pruned = _schedule_env(monkeypatch, db, report=report) + + result = run(svc.run_schedule(session, schedule)) + + assert result["ok"] is False + assert schedule.last_status.startswith("warning:") + assert pruned == [] # the last good backup must not be rotated out + assert alerts and "incomplete" in alerts[0] + + +def test_complete_scheduled_backup_is_ok_and_rotates(monkeypatch, db): + report = {"errors": [], "restart_error": None} + svc, session, schedule, alerts, pruned = _schedule_env(monkeypatch, db, report=report) + + result = run(svc.run_schedule(session, schedule)) + + assert result["ok"] is True and schedule.last_status == "ok" + assert pruned == [schedule.stack_id] and alerts == [] + + +def test_busy_stack_is_retried_soon(monkeypatch, db): + error = backup_service.StackBusyError("Stack 'x' is busy: update in progress") + svc, session, schedule, alerts, _ = _schedule_env(monkeypatch, db, error=error) + + run(svc.run_schedule(session, schedule)) + + assert schedule.last_status.startswith("retrying:") + assert (schedule.next_run - schedule.last_run) == svc.BUSY_RETRY + assert alerts == [] + + +# --------------------------------------------------------------------------- # +# Routes +# --------------------------------------------------------------------------- # + + +def test_download_removes_its_temp_file(stack, as_admin, db): + from sqlmodel import Session + + from models.stack import Stack + + stack_id, _, _ = stack + with Session(db) as session: + session.add(Stack(id=stack_id, name=stack_id)) + session.commit() + created: list[str] = [] + real = backup_service.create_backup_ex + + async def spy(*args, **kwargs): + path, report = await real(*args, **kwargs) + created.append(path) + return path, report + + import routers.backups as routes + + original = routes.backup_service.create_backup_ex + routes.backup_service.create_backup_ex = spy + try: + r = as_admin.get(f"/api/stacks/{stack_id}/backup") + finally: + routes.backup_service.create_backup_ex = original + assert r.status_code == 200 + header = json.loads(r.headers["X-Stackpilot-Backup"]) + assert header["errors"] == [] and header["restart_error"] is None + assert created and not os.path.exists(created[0]) + + +# --------------------------------------------------------------------------- # +# Inventory +# --------------------------------------------------------------------------- # + + +class _HelperClient: + def __init__(self, output: str): + self.output = output + self.images = self + self.containers = self + + def get(self, image): + return object() + + def run(self, image, command, **kwargs): + return self.output.encode() + + +def test_inspect_paths_flags_remote_and_unmeasured(monkeypatch): + out = "\n".join([ + "0 dir 1024 ef53", # local ext4, 1 MiB + "1 dir - 6969", # NFS: not walked + "2 dir - ef53", # du ran out of time + "3 file 12 ff534d42", # a file on CIFS + ]) + monkeypatch.setattr(assets, "get_long_client", lambda: _HelperClient(out)) + res = assets.inspect_paths(["/a", "/b", "/c", "/d"]) + assert res["/a"] == {"kind": "dir", "size": 1024 * 1024, "fs": "ef53", "remote": None, "measured": True} + assert res["/b"]["remote"] == "nfs" and res["/b"]["measured"] is True + assert res["/c"]["measured"] is False + assert res["/d"]["remote"] == "cifs" + + +def test_inventory_keeps_remote_and_unmeasured_folders_out(monkeypatch): + stack_id = "inv" + directory = compose_service.stack_dir(stack_id) + specs = [ + {"source": "/mnt/nas/media", "service": "app", "target": "/media"}, + {"source": f"{directory}/cache", "service": "app", "target": "/cache"}, + {"source": f"{directory}/config", "service": "app", "target": "/config"}, + ] + monkeypatch.setattr(assets, "_bind_specs_from_containers", lambda sid: specs) + monkeypatch.setattr(assets, "_bind_specs_from_compose", lambda sid: []) + monkeypatch.setattr(assets, "compose_volumes", lambda sid: [ + {**_volume(f"{stack_id}_db", stack_id), "bind_device": "/mnt/nas/db"}, + _volume("a" * 64, stack_id), + ]) + monkeypatch.setattr(assets, "remote_mount_type", lambda path: None) + monkeypatch.setattr(assets, "inspect_paths", lambda paths: { + "/mnt/nas/media": {"kind": "dir", "size": None, "fs": "6969", "remote": "nfs", "measured": True}, + f"{directory}/cache": {"kind": "dir", "size": None, "fs": "ef53", "remote": None, "measured": False}, + f"{directory}/config": {"kind": "dir", "size": 4096, "fs": "ef53", "remote": None, "measured": True}, + "/mnt/nas/db": {"kind": "dir", "size": None, "fs": "6969", "remote": "nfs", "measured": True}, + }) + + inv = assets.inventory(stack_id) + + binds = {b["source"]: b for b in inv["binds"]} + assert binds["/mnt/nas/media"]["include_default"] is False + assert binds["/mnt/nas/media"]["reason"] == "remote storage (nfs)" + assert binds[f"{directory}/cache"]["reason"] == "too large to measure" + assert binds[f"{directory}/config"]["include_default"] is True + vols = {v["name"]: v for v in inv["volumes"]} + assert vols[f"{stack_id}_db"]["include_default"] is False + assert vols[f"{stack_id}_db"]["reason"] == "remote storage (nfs)" + assert "anonymous" in vols["a" * 64]["reason"] + + +def test_external_volumes_are_found(monkeypatch): + stack_id = f"s{uuid.uuid4().hex[:8]}" + directory = compose_service.stack_dir(stack_id) + os.makedirs(directory) + _write(directory, "compose.yaml", + "services:\n app:\n image: img\n volumes: [shared:/data]\n" + "volumes:\n shared:\n external: true\n name: ${SHARED:-common}\n") + + class Vol: + def __init__(self, name, attrs): + self.name, self.attrs = name, attrs + + class Client: + def __init__(self): + self.volumes = self + self.containers = self + + def list(self, **kwargs): + if "all" in kwargs: # containers + return [] + return [Vol(f"{stack_id}_db", {"Labels": {assets.COMPOSE_PROJECT_LABEL: stack_id}})] + + def get(self, name): + if name != "common": + raise assets.DockerError("not_found") + return Vol("common", {"Labels": {}, "Driver": "local"}) + + client = Client() + monkeypatch.setattr(assets, "get_client", lambda: client) + + vols = {v["name"]: v for v in assets.compose_volumes(stack_id)} + assert set(vols) == {f"{stack_id}_db", "common"} + assert vols["common"]["external"] is True + assert vols[f"{stack_id}_db"]["external"] is False diff --git a/backend/version.py b/backend/version.py index 65e9033..c308f14 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1,3 +1,3 @@ """Single source of truth for the StackPilot release version.""" -APP_VERSION = "0.63.0" +APP_VERSION = "0.64.0" diff --git a/frontend/package.json b/frontend/package.json index 39a9cda..72d6932 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "stackpilot-frontend", "private": true, - "version": "0.63.0", + "version": "0.64.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/api/backups.ts b/frontend/src/api/backups.ts index c12b635..bc6997d 100644 --- a/frontend/src/api/backups.ts +++ b/frontend/src/api/backups.ts @@ -18,6 +18,8 @@ export interface BackupBind { covered_by_compose: boolean; via: "compose" | "archive"; system: boolean; + /** Network filesystem the folder sits on (nfs, cifs, …); never archived. */ + remote?: string | null; include_default: boolean; selected: boolean; reason: string | null; @@ -29,6 +31,8 @@ export interface BackupVolume { driver: string; remote: boolean; remote_type: string | null; + /** Used by the stack but not created by compose for it (external: true). */ + external?: boolean; include_default: boolean; selected: boolean; reason: string | null; @@ -49,6 +53,14 @@ export interface BackupReport { volumes: { name: string; bytes: number | null; error: string | null }[]; skipped: { kind: string; source?: string; name?: string; reason: string | null }[]; path_mismatch: { host: string; container: string } | null; + /** Services stopped for a consistent snapshot (and started again). */ + stopped?: string[]; + /** Data that should have been captured and was not. */ + errors?: string[]; + /** Files that changed or could not be read while archiving. */ + warnings?: string[]; + /** Set when the stack could not be started again after the backup. */ + restart_error?: string | null; } export interface RestoreResult { @@ -57,6 +69,9 @@ export interface RestoreResult { volumes_restored: number; binds_restored?: number; skipped?: { kind: string; source?: string; name?: string; reason: string | null }[]; + /** What went wrong during this restore (not what the backup left out). */ + problems?: string[]; + restarted?: boolean; } export interface BackupOptions { diff --git a/frontend/src/components/stacks/BackupRestore.tsx b/frontend/src/components/stacks/BackupRestore.tsx index 6c3d000..9579e35 100644 --- a/frontend/src/components/stacks/BackupRestore.tsx +++ b/frontend/src/components/stacks/BackupRestore.tsx @@ -150,6 +150,25 @@ export function BackupButton({ stackId }: { stackId: string }) { const skipped = report.skipped.filter((s) => s.reason !== "not requested").length; return `Backup: ${parts.join(" · ")}${skipped ? ` — ${skipped} skipped` : ""}`; }; + // A backup that is missing data, or left the stack down, is not a success. + const announce = (tid: string | number, text: string, report?: BackupReport | null) => { + const problems = [ + ...(report?.errors ?? []).map((e) => `Not captured: ${e}`), + ...(report?.restart_error ? [`Stack did not start again: ${report.restart_error}`] : []), + ]; + if (problems.length) { + toast.error(`${text} — incomplete`, { id: tid, description: problems.join("\n") }); + } else if (report?.warnings?.length) { + toast.warning(text, { + id: tid, + description: `${report.warnings.length} file(s) changed while being read: ${report.warnings + .slice(0, 3) + .join("; ")}`, + }); + } else { + toast.success(text, { id: tid }); + } + }; const run = async () => { setBusy(true); @@ -166,7 +185,7 @@ export function BackupButton({ stackId }: { stackId: string }) { }; if (target === "download") { const report = await backupsApi.download(stackId, opts); - toast.success(describe(report), { id: tid }); + announce(tid, describe(report), report); } else { const body = { destination_id: Number(target), @@ -177,7 +196,7 @@ export function BackupButton({ stackId }: { stackId: string }) { volumes: opts.volumes, }; const res = await backupsApi.push(stackId, body); - toast.success(`Pushed to ${res.destination}`, { id: tid }); + announce(tid, `Pushed to ${res.destination} — ${describe(res.report)}`, res.report); } setOpen(false); } catch (e) { @@ -246,6 +265,7 @@ export function BackupButton({ stackId }: { stackId: string }) { detail={`named volume (${v.driver})`} badges={[ ...(v.remote ? [{ text: v.remote_type || "remote", tone: "warn" as const }] : []), + ...(v.external ? [{ text: "external" }] : []), ...(v.reason && !v.remote ? [{ text: v.reason, tone: "warn" as const }] : []), ]} /> @@ -257,8 +277,9 @@ export function BackupButton({ stackId }: { stackId: string }) { )}

- Remote storage (NFS/CIFS) is unchecked by default — it lives on your NAS and would be - overwritten on restore. + Remote storage (NFS/CIFS volumes, and folders on a mounted share) is never selected by + default — it lives on your NAS. The stack is stopped while its data is read and + started again afterwards.

@@ -354,7 +375,12 @@ export function RestoreButton() { } const bits = [`${res.volumes_restored} volume(s)`]; if (res.binds_restored) bits.push(`${res.binds_restored} folder(s)`); - toast.success(`Restored '${res.stack_id}' — ${bits.join(", ")}`, { id: tid }); + const text = `Restored '${res.stack_id}' — ${bits.join(", ")}`; + if (res.problems?.length) { + toast.warning(text, { id: tid, description: res.problems.slice(0, 5).join("\n") }); + } else { + toast.success(text, { id: tid }); + } qc.invalidateQueries({ queryKey: ["stacks"] }); setOpen(false); setFile(null); @@ -464,7 +490,7 @@ export function RestoreButton() { checked={overwrite} onChange={setOverwrite} label="Overwrite if a stack with this id already exists" - hint="Replaces the existing stack files and volume contents." + hint="Stops the stack, puts back the files, folders and volumes in the backup and starts it again. Folders the backup left out are kept." />