menzeljandClaude Opus 5.5
06234c1067
StackPilot called `docker compose -p <folder> -f compose.yaml`, which is
not what `docker compose` typed in the stack's folder does. A
compose.override.yaml or a COMPOSE_FILE in .env applied in a terminal
and was dropped by the next Start in StackPilot, which recreated the
containers without it. A compose file's `name:` or a COMPOSE_PROJECT_NAME
in .env made the terminal's containers invisible: the stack showed as
stopped while running, and Start set up a second copy next to it.
Compose now runs in the stack's folder without -f, so it loads the same
files, and -p is the name compose itself picks there: COMPOSE_PROJECT_NAME
from .env, else the last `name:` among the compose files (${VAR} from
.env), else the folder name. A stack StackPilot already runs under its
folder name keeps that name while it has containers or volumes under it
and none under the declared one: switching would have brought it up
again on empty volumes. Backups record the project, and a restore takes
it from the files it has just put back.
Changes made outside StackPilot now reach every view:
- docker_watch_service reads Docker's events once for the backend. The
dashboard rollup (cached 25s) and the stats cache are stale as soon as
the event generation moves; the tiles followed a terminal `compose
stop` after 15-30s, now within a second.
- Image events re-read the local digests, so a `docker compose pull` in
a terminal clears the update badge instead of leaving it until the
next registry check (an hour by default).
- The browser catches up when its event socket reconnects; what changed
while it was down waited for the 60s poll.
- The stack detail carries a revision of the compose file and .env;
saving after they changed on disk is refused with 409 and the editor
offers to overwrite or to load the version on disk. A save used to
replace a terminal edit without a word.
- A stack whose folder is gone is flagged "folder missing" instead of
passing for an ordinary stopped stack.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Fx7gcuy8d6nPd95g43rGHu
1111 lines
44 KiB
Python
1111 lines
44 KiB
Python
"""Stack backup & restore — compose files, bind-mount data and named volumes.
|
|
|
|
A backup is a single ``.tar.gz`` with this layout::
|
|
|
|
manifest.json metadata + full inventory of what was captured
|
|
compose/... the stack directory as StackPilot can see it
|
|
binds/<n>.tar contents of each captured bind-mount source
|
|
volumes/<full>.tar contents of each captured named volume
|
|
|
|
Bind sources and volumes are read/written through a throwaway helper container
|
|
(see :mod:`services.stack_assets_service`) so that host paths this container
|
|
cannot see are still captured — without that, a stack whose data directories
|
|
live outside StackPilot's own mount would back up as "just the compose file".
|
|
|
|
What makes a backup and its restore trustworthy, and what this module keeps to:
|
|
|
|
* **Consistent.** Whenever data is captured — including data folders that ride
|
|
along inside the stack directory — the stack's running containers are
|
|
stopped first and exactly those are started again afterwards. A stack that
|
|
cannot be stopped is not backed up half-written. The stack is locked for the
|
|
duration, so a deploy, update or second backup cannot interleave.
|
|
* **Nothing is lost that the backup does not contain.** A restore replaces what
|
|
the backup holds and leaves alone what it does not: folders left out (too
|
|
large, deselected, remote storage) and anything mounted into the stack
|
|
directory from another filesystem survive an overwrite.
|
|
* **Verified before destructive.** The whole archive is read once before a
|
|
restore touches anything, and the stack directory is unpacked next to the
|
|
old one first.
|
|
* **A restore under a new name never writes into the original's data**:
|
|
folders outside the stack directory and volumes not named after the stack
|
|
are skipped rather than overwritten.
|
|
* **Remote storage** (NFS/CIFS/...) is neither archived nor wiped.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import tarfile
|
|
import tempfile
|
|
import zlib
|
|
from contextlib import contextmanager
|
|
from datetime import timedelta
|
|
from typing import Optional
|
|
|
|
from sqlmodel import Session
|
|
|
|
from database import engine
|
|
from docker_client import DockerError, get_long_client, safe_call
|
|
from services import (
|
|
compose_service,
|
|
stack_job_service,
|
|
stack_lock_service,
|
|
stack_assets_service as assets,
|
|
)
|
|
|
|
logger = logging.getLogger("stackpilot.backup")
|
|
|
|
COMPOSE_PROJECT_LABEL = assets.COMPOSE_PROJECT_LABEL
|
|
COMPOSE_VOLUME_LABEL = assets.COMPOSE_VOLUME_LABEL
|
|
MANIFEST_NAME = "manifest.json"
|
|
#: 3 added compose_binds/excluded (what the compose/ tree does and does not
|
|
#: carry) and each volume's driver/options/external. Older ones restore too.
|
|
BACKUP_FORMAT_VERSION = 3
|
|
|
|
#: How long a backup or restore may hold the stack lock before another
|
|
#: operation may take it over. Generous: a large volume can take hours.
|
|
LOCK_TTL = timedelta(hours=12)
|
|
|
|
|
|
class BackupError(Exception):
|
|
pass
|
|
|
|
|
|
class StackBusyError(BackupError):
|
|
"""The stack is locked by another operation (deploy, update, backup)."""
|
|
|
|
|
|
class BackupCancelled(BackupError):
|
|
"""Somebody stopped this backup from the outside.
|
|
|
|
Not a failure: the archive is thrown away, the stack is started again and
|
|
the lock goes, exactly as after an error — but nothing is reported as
|
|
broken and no "backup failed" notification goes out.
|
|
"""
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Backup filename convention
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def backup_basename(stack_id: str, prefix: Optional[str] = None) -> str:
|
|
"""Filename stem used to group a stack's backups (and match for retention).
|
|
|
|
``prefix`` (e.g. a remote host slug) keeps backups of same-named stacks on
|
|
different hosts from colliding / pruning each other on a shared destination.
|
|
"""
|
|
return f"backup-{prefix}-{stack_id}" if prefix else f"backup-{stack_id}"
|
|
|
|
|
|
def backup_filename(
|
|
stack_id: str,
|
|
include_volumes: bool,
|
|
prefix: Optional[str] = None,
|
|
tz: Optional[str] = None,
|
|
) -> str:
|
|
"""``backup-<stack>-<full|config>-YYYYmmdd-HHMMSS.tar.gz``, the time on the
|
|
wall clock of ``tz`` — the schedule's zone, or the browser's for a backup
|
|
made by hand. It used to be UTC, so a backup taken at 12:21 in Berlin was
|
|
called …-102154 and seemed to be from two hours earlier. The format is
|
|
unchanged, so retention still matches older files (it orders by mtime)."""
|
|
from services.schedule_service import zone
|
|
|
|
date = compose_service.now().astimezone(zone(tz)).strftime("%Y%m%d-%H%M%S")
|
|
suffix = "full" if include_volumes else "config"
|
|
return f"{backup_basename(stack_id, prefix)}-{suffix}-{date}.tar.gz"
|
|
|
|
|
|
def backup_name_pattern(stack_id: str, prefix: Optional[str] = None) -> re.Pattern:
|
|
"""Matches exactly this stack's backup files.
|
|
|
|
A plain prefix match on ``backup-app-`` also takes ``backup-app-db-…``, so
|
|
retention for stack "app" deleted the backups of stack "app-db".
|
|
"""
|
|
base = re.escape(backup_basename(stack_id, prefix))
|
|
return re.compile(rf"^{base}-(?:full|config)-\d{{8}}-\d{{6}}\.tar\.gz$")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Locking and stopping
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@contextmanager
|
|
def _locked(stack_id: str, action: str, owner: str = ""):
|
|
"""Hold the stack lock, and register the job so it can be stopped.
|
|
|
|
The lock alone made a stuck backup fatal: nothing could touch the stack
|
|
until its twelve-hour TTL ran out. The registration is what gives the
|
|
unlock endpoint something to cancel.
|
|
"""
|
|
with Session(engine) as session:
|
|
try:
|
|
stack_lock_service.acquire(session, stack_id, action, owner or action, ttl=LOCK_TTL)
|
|
except stack_lock_service.StackBusy as exc:
|
|
raise StackBusyError(str(exc)) from exc
|
|
try:
|
|
with stack_job_service.track(stack_id, action, owner or action):
|
|
yield
|
|
except stack_job_service.JobCancelled as exc:
|
|
raise BackupCancelled(str(exc)) from exc
|
|
finally:
|
|
try:
|
|
stack_lock_service.release(session, stack_id)
|
|
except Exception: # noqa: BLE001 - never mask the original error
|
|
logger.exception("Failed to release the lock on '%s'", stack_id)
|
|
|
|
|
|
async def _running(stack_id: str) -> list:
|
|
try:
|
|
containers = await asyncio.to_thread(compose_service.containers_for_stack, stack_id)
|
|
except DockerError as exc:
|
|
raise BackupError(f"Docker is not reachable: {exc.detail or exc.error}") from exc
|
|
return [c for c in containers if c.state == "running"]
|
|
|
|
|
|
def _tail(result: dict) -> str:
|
|
text = (result.get("stderr") or result.get("stdout") or "").strip()
|
|
return text.splitlines()[-1] if text else f"exit {result.get('returncode')}"
|
|
|
|
|
|
def _docker_each(ids: list[str], action: str) -> None:
|
|
client = get_long_client()
|
|
for container_id in ids:
|
|
container = safe_call(client.containers.get, container_id)
|
|
safe_call(getattr(container, action))
|
|
|
|
|
|
async def _compose(stack_id: str, args: list[str]) -> Optional[str]:
|
|
"""Run a compose command; None on success, else what went wrong."""
|
|
try:
|
|
result = await compose_service.run_compose(stack_id, args)
|
|
except Exception as exc: # noqa: BLE001 - timeouts, missing binary
|
|
return str(exc)
|
|
return None if result["returncode"] == 0 else _tail(result)
|
|
|
|
|
|
async def _stop(stack_id: str, running: list) -> None:
|
|
"""Stop exactly these containers, or raise and leave them running."""
|
|
services = sorted({c.service for c in running})
|
|
error = await _compose(stack_id, ["stop", *services])
|
|
if error is None:
|
|
return
|
|
# Compose fails on a compose file that was edited (or broken) since the
|
|
# deploy; the containers themselves can still be stopped directly.
|
|
logger.warning("compose stop failed for %s (%s); stopping containers directly", stack_id, error)
|
|
try:
|
|
await asyncio.to_thread(_docker_each, [c.id for c in running], "stop")
|
|
except Exception as exc: # noqa: BLE001
|
|
await _start(stack_id, running) # put back whatever did stop
|
|
raise BackupError(
|
|
f"Could not stop '{stack_id}' for a consistent snapshot: {error}"
|
|
) from exc
|
|
|
|
|
|
async def _start(stack_id: str, running: list) -> Optional[str]:
|
|
"""Start exactly these containers again; None on success."""
|
|
if not running:
|
|
return None
|
|
services = sorted({c.service for c in running})
|
|
error = await _compose(stack_id, ["start", *services])
|
|
if error is None:
|
|
return None
|
|
try:
|
|
await asyncio.to_thread(_docker_each, [c.id for c in running], "start")
|
|
return None
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Could not restart %s: %s / %s", stack_id, error, exc)
|
|
return f"{error}; {exc}"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Selection
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _decide(
|
|
items: list[dict], key: str, chosen: Optional[list[str]], enabled: bool
|
|
) -> list[dict]:
|
|
"""Mark each inventory item ``selected`` (with a reason when it is not).
|
|
|
|
``chosen`` is an explicit list from the caller; without one the inventory's
|
|
own defaults apply (everything except system paths, oversized directories
|
|
and remote-backed storage).
|
|
"""
|
|
for item in items:
|
|
if not enabled:
|
|
item["selected"], item["reason"] = False, "not requested"
|
|
elif chosen is not None:
|
|
selected = item[key] in chosen
|
|
item["selected"] = selected
|
|
item["reason"] = None if selected else "not selected"
|
|
else:
|
|
item["selected"] = bool(item.get("include_default"))
|
|
item["reason"] = None if item["selected"] else (item.get("reason") or "not selected")
|
|
# Never archive plumbing, whatever the caller asked for.
|
|
if item["selected"] and (item.get("system") or item.get("kind") in ("special", "unknown")):
|
|
item["selected"] = False
|
|
item["reason"] = item.get("reason") or "not a regular file or directory"
|
|
return items
|
|
|
|
|
|
def plan(
|
|
stack_id: str,
|
|
include_volumes: bool = True,
|
|
include_binds: bool = True,
|
|
binds: Optional[list[str]] = None,
|
|
volumes: Optional[list[str]] = None,
|
|
) -> dict:
|
|
"""Decide what a backup captures. Returned as-is by the inventory endpoint."""
|
|
inv = assets.inventory(stack_id)
|
|
_decide(inv["binds"], "source", binds, include_binds)
|
|
_decide(inv["volumes"], "name", volumes, include_volumes)
|
|
return inv
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Writing the archive
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
class _ExactReader:
|
|
"""Hands tarfile exactly ``size`` bytes of a file that may change while it
|
|
is read. A file that shrinks (a log rotated mid-backup) is padded with
|
|
zeros instead of aborting inside the member, which would leave the whole
|
|
archive unreadable; ``short`` records that it happened."""
|
|
|
|
def __init__(self, fh, size: int):
|
|
self._fh = fh
|
|
self._remaining = size
|
|
self.short = False
|
|
|
|
def read(self, n: int = -1) -> bytes:
|
|
if n < 0 or n > self._remaining:
|
|
n = self._remaining
|
|
data = self._fh.read(n)
|
|
if len(data) < n:
|
|
self.short = True
|
|
data += b"\0" * (n - len(data))
|
|
self._remaining -= n
|
|
return data
|
|
|
|
|
|
def _add_tree(
|
|
tar: tarfile.TarFile, root: str, arcroot: str, skip: set[str], warnings: list[str]
|
|
) -> None:
|
|
"""Add the stack directory, tolerating a directory that is in use.
|
|
|
|
``skip`` holds paths (relative to ``root``) whose *contents* stay out: the
|
|
directory entry itself is kept so the mount point exists after a restore.
|
|
Files that vanish mid-walk are left out and unreadable ones reported,
|
|
rather than one of them failing the whole backup. A remote filesystem
|
|
mounted somewhere inside is not descended into.
|
|
"""
|
|
|
|
def add(path: str, rel: str, parent_dev: int) -> None:
|
|
# A stack directory with a data folder in it can take as long as a
|
|
# volume, so a cancellation lands here too — between two files, with
|
|
# the half-written archive thrown away by the caller.
|
|
stack_job_service.check()
|
|
arc = f"{arcroot}/{rel}" if rel else arcroot
|
|
label = rel or "."
|
|
try:
|
|
st = os.lstat(path)
|
|
if stat.S_ISSOCK(st.st_mode):
|
|
return # a runtime artefact, meaningless in an archive
|
|
if stat.S_ISREG(st.st_mode):
|
|
if rel in skip:
|
|
return
|
|
with open(path, "rb") as fh:
|
|
info = tar.gettarinfo(arcname=arc, fileobj=fh)
|
|
if info.isreg():
|
|
reader = _ExactReader(fh, info.size)
|
|
tar.addfile(info, reader)
|
|
if reader.short:
|
|
warnings.append(f"{label}: changed while it was read")
|
|
else: # a hard link to a file already in the archive
|
|
tar.addfile(info)
|
|
return
|
|
info = tar.gettarinfo(path, arcname=arc)
|
|
except FileNotFoundError:
|
|
return
|
|
except OSError as exc:
|
|
warnings.append(f"{label}: {exc.strerror or exc}")
|
|
return
|
|
if info is None:
|
|
return
|
|
tar.addfile(info)
|
|
if not stat.S_ISDIR(st.st_mode) or rel in skip:
|
|
return
|
|
if rel and st.st_dev != parent_dev:
|
|
remote = assets.remote_mount_type(path)
|
|
if remote:
|
|
warnings.append(f"{label}: remote storage ({remote}) mounted here, not included")
|
|
return
|
|
try:
|
|
names = sorted(os.listdir(path))
|
|
except OSError as exc:
|
|
warnings.append(f"{label}: {exc.strerror or exc}")
|
|
return
|
|
for name in names:
|
|
add(os.path.join(path, name), f"{rel}/{name}" if rel else name, st.st_dev)
|
|
|
|
add(root, "", os.lstat(root).st_dev)
|
|
|
|
|
|
def _write_archive(
|
|
dest: str,
|
|
manifest: dict,
|
|
directory: str,
|
|
skip: set[str],
|
|
parts: list[tuple[str, str]],
|
|
warnings: list[str],
|
|
) -> None:
|
|
with tarfile.open(dest, "w:gz") as tar:
|
|
data = json.dumps(manifest, indent=2).encode("utf-8")
|
|
info = tarfile.TarInfo(MANIFEST_NAME)
|
|
info.size = len(data)
|
|
info.mtime = int(compose_service.now().timestamp())
|
|
tar.addfile(info, io.BytesIO(data))
|
|
_add_tree(tar, directory, "compose", skip, warnings)
|
|
for part, arcname in parts:
|
|
tar.add(part, arcname=arcname)
|
|
os.unlink(part) # keep the peak disk use to one copy of the data
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Backup
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _rel_inside(path: str, directory: str) -> Optional[str]:
|
|
rel = os.path.relpath(path, directory)
|
|
if rel == "." or rel.startswith(".." + os.sep) or rel == "..":
|
|
return None
|
|
return rel
|
|
|
|
|
|
async def create_backup_ex(
|
|
stack_id: str,
|
|
name: str,
|
|
include_volumes: bool = True,
|
|
stop_first: bool = True,
|
|
include_binds: bool = True,
|
|
binds: Optional[list[str]] = None,
|
|
volumes: Optional[list[str]] = None,
|
|
) -> tuple[str, dict]:
|
|
"""Create a backup tar.gz. Returns (path, report).
|
|
|
|
The report's ``errors`` lists data that should have been captured and was
|
|
not, and ``restart_error`` a stack that could not be started again — both
|
|
mean the backup must not be reported as a plain success.
|
|
"""
|
|
directory = compose_service.stack_dir(stack_id)
|
|
if not os.path.isdir(directory):
|
|
raise BackupError("Stack directory missing")
|
|
with _locked(stack_id, "backup"):
|
|
return await _create(
|
|
stack_id, name, directory, include_volumes, stop_first, include_binds, binds, volumes
|
|
)
|
|
|
|
|
|
async def _create(
|
|
stack_id: str,
|
|
name: str,
|
|
directory: str,
|
|
include_volumes: bool,
|
|
stop_first: bool,
|
|
include_binds: bool,
|
|
binds: Optional[list[str]],
|
|
volumes: Optional[list[str]],
|
|
) -> tuple[str, dict]:
|
|
stack_job_service.phase("looking at what to capture")
|
|
selection = await asyncio.to_thread(
|
|
plan, stack_id, include_volumes, include_binds, binds, volumes
|
|
)
|
|
selected_binds = [b for b in selection["binds"] if b["selected"]]
|
|
# Selected bind sources this process cannot reach through the filesystem
|
|
# need their own archive; the rest already ride along in compose/.
|
|
cap_binds = [b for b in selected_binds if b["via"] == "archive"]
|
|
tree_binds = [b for b in selected_binds if b["via"] != "archive"]
|
|
cap_volumes = [v for v in selection["volumes"] if v["selected"]]
|
|
skipped_binds = [b for b in selection["binds"] if not b["selected"]]
|
|
skipped_volumes = [v for v in selection["volumes"] if not v["selected"]]
|
|
|
|
# Deselected data that sits inside the stack folder must not sneak into
|
|
# the archive through compose/ (that is how a 200 GB downloads folder
|
|
# ends up in a "config only" backup). A restore leaves these alone.
|
|
excluded = sorted(
|
|
{
|
|
rel
|
|
for b in skipped_binds
|
|
if b.get("inside_stack_dir") and (rel := _rel_inside(b["source"], directory))
|
|
}
|
|
)
|
|
compose_binds = sorted(
|
|
{rel for b in tree_binds if (rel := _rel_inside(b["source"], directory))}
|
|
)
|
|
|
|
# Consistent snapshot: stop what runs — data folders inside the stack
|
|
# directory count as data too — and start exactly that again afterwards.
|
|
stopped: list = []
|
|
if stop_first and (cap_volumes or selected_binds):
|
|
running = await _running(stack_id)
|
|
if running:
|
|
await _stop(stack_id, running)
|
|
stopped = running
|
|
|
|
workdir = tempfile.mkdtemp(prefix="sp-backup-")
|
|
archive_path: Optional[str] = None
|
|
warnings: list[str] = []
|
|
restart_error: Optional[str] = None
|
|
try:
|
|
parts: list[tuple[str, str]] = []
|
|
for index, bind in enumerate(cap_binds):
|
|
# Between two items is where a cancelled backup gives up: never in
|
|
# the middle of one, and never after the stack has been stopped
|
|
# without the finally below starting it again.
|
|
stack_job_service.check()
|
|
stack_job_service.phase(f"bind {bind['source']}")
|
|
part = os.path.join(workdir, f"bind-{index:03d}.tar")
|
|
try:
|
|
bind["bytes"] = await asyncio.to_thread(
|
|
assets.export_path, bind["source"], bind["kind"], part
|
|
)
|
|
bind["archive"] = f"binds/{index:03d}.tar"
|
|
parts.append((part, bind["archive"]))
|
|
except Exception as exc: # noqa: BLE001
|
|
# Cancelling pulls the helper container out from under the
|
|
# read, so the failure here *is* the cancellation.
|
|
stack_job_service.check()
|
|
logger.warning("Could not archive bind %s: %s", bind["source"], exc)
|
|
bind["archive"], bind["error"] = None, str(exc)
|
|
|
|
for index, vol in enumerate(cap_volumes):
|
|
stack_job_service.check()
|
|
stack_job_service.phase(f"volume {vol['name']}")
|
|
part = os.path.join(workdir, f"vol-{index:03d}.tar")
|
|
try:
|
|
vol["bytes"] = await asyncio.to_thread(assets.export_volume, vol["name"], part)
|
|
vol["archive"] = f"volumes/{vol['name']}.tar"
|
|
parts.append((part, vol["archive"]))
|
|
except Exception as exc: # noqa: BLE001
|
|
stack_job_service.check()
|
|
logger.warning("Could not archive volume %s: %s", vol["name"], exc)
|
|
vol["archive"], vol["error"] = None, str(exc)
|
|
|
|
failed = [b for b in cap_binds if b.get("error")] + [
|
|
v for v in cap_volumes if v.get("error")
|
|
]
|
|
manifest = {
|
|
"format_version": BACKUP_FORMAT_VERSION,
|
|
"stack_id": stack_id,
|
|
# Volumes are prefixed with the project, which a compose file's
|
|
# name: or COMPOSE_PROJECT_NAME can make anything at all.
|
|
"project": compose_service.project_name(stack_id),
|
|
"name": name,
|
|
"created_at": compose_service.now().isoformat(),
|
|
"include_volumes": include_volumes,
|
|
"include_binds": include_binds,
|
|
"stack_dir": directory,
|
|
"path_mismatch": selection.get("path_mismatch"),
|
|
"stopped": sorted({c.service for c in stopped}),
|
|
# Data folders carried inside compose/, and folders whose contents
|
|
# were left out of it (kept as they are by a restore).
|
|
"compose_binds": compose_binds,
|
|
"excluded": excluded,
|
|
# Volumes keep the v1 shape (full/short/labels) so older StackPilots
|
|
# can still read the manifest they care about.
|
|
"volumes": [
|
|
{
|
|
"full": v["name"],
|
|
"short": v["short"],
|
|
"labels": v["labels"],
|
|
"driver": v.get("driver"),
|
|
"options": v.get("options") or {},
|
|
"external": v.get("external", False),
|
|
"archive": v["archive"],
|
|
"remote": v.get("remote", False),
|
|
"bytes": v.get("bytes"),
|
|
}
|
|
for v in cap_volumes
|
|
if v.get("archive")
|
|
],
|
|
"binds": [
|
|
{
|
|
"source": b["source"],
|
|
"kind": b["kind"],
|
|
"mounts": b["mounts"],
|
|
"inside_stack_dir": b["inside_stack_dir"],
|
|
"archive": b["archive"],
|
|
"bytes": b.get("bytes"),
|
|
}
|
|
for b in cap_binds
|
|
if b.get("archive")
|
|
],
|
|
"skipped": [
|
|
{"kind": "bind", "source": b["source"], "reason": b.get("reason")}
|
|
for b in skipped_binds
|
|
]
|
|
+ [
|
|
{"kind": "volume", "name": v["name"], "reason": v.get("reason")}
|
|
for v in skipped_volumes
|
|
]
|
|
+ [
|
|
{
|
|
"kind": "volume" if "name" in item else "bind",
|
|
**({"name": item["name"]} if "name" in item else {"source": item["source"]}),
|
|
"reason": f"backup failed: {item['error']}",
|
|
}
|
|
for item in failed
|
|
],
|
|
}
|
|
|
|
stack_job_service.check()
|
|
stack_job_service.phase("writing the archive")
|
|
fd, archive_path = tempfile.mkstemp(prefix=f"sp-{stack_id}-", suffix=".tar.gz")
|
|
os.close(fd)
|
|
await asyncio.to_thread(
|
|
_write_archive, archive_path, manifest, directory, set(excluded), parts, warnings
|
|
)
|
|
except BaseException:
|
|
if archive_path and os.path.exists(archive_path):
|
|
os.unlink(archive_path)
|
|
raise
|
|
finally:
|
|
shutil.rmtree(workdir, ignore_errors=True)
|
|
if stopped:
|
|
restart_error = await _start(stack_id, stopped)
|
|
|
|
errors = [f"{item.get('source') or item.get('name')}: {item['error']}" for item in failed]
|
|
report = {
|
|
"file": archive_path,
|
|
"binds": [
|
|
{"source": b["source"], "bytes": b.get("bytes") or b.get("size"), "error": b.get("error")}
|
|
for b in tree_binds + cap_binds
|
|
],
|
|
"volumes": [
|
|
{"name": v["name"], "bytes": v.get("bytes"), "error": v.get("error")}
|
|
for v in cap_volumes
|
|
],
|
|
"skipped": manifest["skipped"],
|
|
"path_mismatch": selection.get("path_mismatch"),
|
|
"stopped": manifest["stopped"],
|
|
"errors": errors,
|
|
"warnings": warnings,
|
|
"restart_error": restart_error,
|
|
"size": os.path.getsize(archive_path),
|
|
}
|
|
return archive_path, report
|
|
|
|
|
|
async def create_backup(
|
|
stack_id: str,
|
|
name: str,
|
|
include_volumes: bool = True,
|
|
stop_first: bool = True,
|
|
include_binds: bool = True,
|
|
binds: Optional[list[str]] = None,
|
|
volumes: Optional[list[str]] = None,
|
|
) -> str:
|
|
path, _report = await create_backup_ex(
|
|
stack_id, name, include_volumes, stop_first, include_binds, binds, volumes
|
|
)
|
|
return path
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Reading and verifying
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def read_manifest(tar_path: str) -> dict:
|
|
"""The manifest, read without decompressing the rest (it comes first)."""
|
|
try:
|
|
with tarfile.open(tar_path, "r:gz") as tar:
|
|
for member in tar:
|
|
if member.name == MANIFEST_NAME:
|
|
fh = tar.extractfile(member)
|
|
if fh is None:
|
|
break
|
|
return json.loads(fh.read().decode("utf-8"))
|
|
except (tarfile.TarError, OSError, EOFError, zlib.error, ValueError) as exc:
|
|
raise BackupError(f"Backup archive is damaged or incomplete: {exc}") from exc
|
|
raise BackupError("Backup is missing its manifest")
|
|
|
|
|
|
def verify_archive(tar_path: str) -> None:
|
|
"""Read the whole archive once, so a truncated or corrupt file is refused
|
|
before a restore deletes anything. Draining the gzip stream to its end is
|
|
what checks its CRC."""
|
|
try:
|
|
with tarfile.open(tar_path, "r:gz") as tar:
|
|
for member in tar:
|
|
if member.isreg():
|
|
fh = tar.extractfile(member)
|
|
while fh is not None and fh.read(1024 * 1024):
|
|
pass
|
|
while tar.fileobj.read(1024 * 1024):
|
|
pass
|
|
except (tarfile.TarError, OSError, EOFError, zlib.error) as exc:
|
|
raise BackupError(f"Backup archive is damaged or incomplete: {exc}") from exc
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Restore — stack directory
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _safe_target(dest_dir: str, rel: str) -> str:
|
|
"""Resolve a member path inside dest_dir, refusing traversal *and* writes
|
|
through a symlink planted earlier in the same archive."""
|
|
root = os.path.abspath(dest_dir)
|
|
target = os.path.normpath(os.path.join(root, rel))
|
|
if target != root and not target.startswith(root + os.sep):
|
|
raise BackupError(f"Refusing unsafe path in backup: {rel}")
|
|
parent = os.path.dirname(target)
|
|
if os.path.exists(parent):
|
|
real_parent = os.path.realpath(parent)
|
|
if real_parent != root and not real_parent.startswith(root + os.sep):
|
|
raise BackupError(f"Refusing unsafe path in backup: {rel}")
|
|
return target
|
|
|
|
|
|
def _apply_meta(path: str, member: tarfile.TarInfo) -> None:
|
|
"""Restore mode/ownership/mtime — *arr-style images run as PUID/PGID and
|
|
break when their config comes back root-owned with default permissions."""
|
|
try:
|
|
os.chown(path, member.uid, member.gid)
|
|
except (OSError, AttributeError):
|
|
pass
|
|
try:
|
|
# After chown, which clears setuid/setgid bits.
|
|
os.chmod(path, member.mode)
|
|
except OSError:
|
|
pass
|
|
try:
|
|
os.utime(path, (member.mtime, member.mtime))
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _under(rel: str, prefixes: set[str]) -> bool:
|
|
return any(rel.startswith(p + "/") for p in prefixes)
|
|
|
|
|
|
def _extract_tree(
|
|
tar: tarfile.TarFile, prefix: str, dest_dir: str, skip: frozenset | set = frozenset()
|
|
) -> None:
|
|
"""Extract one subtree of the archive, preserving metadata and symlinks.
|
|
|
|
Contents under a ``skip`` path (relative) are not extracted; the path's own
|
|
directory entry still is.
|
|
"""
|
|
os.makedirs(dest_dir, exist_ok=True)
|
|
root_name = prefix.rstrip("/")
|
|
dirs: list[tuple[str, tarfile.TarInfo]] = []
|
|
for member in tar.getmembers():
|
|
if member.name == root_name:
|
|
if member.isdir():
|
|
dirs.append((dest_dir, member))
|
|
continue
|
|
if not member.name.startswith(prefix):
|
|
continue
|
|
rel = member.name[len(prefix) :].lstrip("/")
|
|
if not rel or _under(rel, skip):
|
|
continue
|
|
target = _safe_target(dest_dir, rel)
|
|
existing_dir = os.path.isdir(target) and not os.path.islink(target)
|
|
if member.isdir():
|
|
if os.path.lexists(target) and not existing_dir:
|
|
os.unlink(target)
|
|
os.makedirs(target, exist_ok=True)
|
|
dirs.append((target, member))
|
|
continue
|
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
# Never write through (or over) something an earlier member put here:
|
|
# open() would follow a symlink to wherever it points.
|
|
if existing_dir:
|
|
shutil.rmtree(target)
|
|
elif os.path.lexists(target):
|
|
os.unlink(target)
|
|
if member.issym():
|
|
os.symlink(member.linkname, target)
|
|
continue # chmod/utime would follow the link
|
|
if member.islnk():
|
|
source = _safe_target(dest_dir, member.linkname[len(prefix) :].lstrip("/"))
|
|
if os.path.exists(source):
|
|
os.link(source, target)
|
|
continue
|
|
if not member.isreg():
|
|
continue # devices/fifos/sockets are runtime artefacts
|
|
src = tar.extractfile(member)
|
|
if src is None:
|
|
continue
|
|
with open(target, "wb") as out:
|
|
shutil.copyfileobj(src, out)
|
|
_apply_meta(target, member)
|
|
# Directory metadata last: a 0500 directory would block writing its files.
|
|
for target, member in sorted(dirs, key=lambda d: len(d[0]), reverse=True):
|
|
_apply_meta(target, member)
|
|
|
|
|
|
def _remove_tree(path: str, dev: int, rel: str, warnings: list[str]) -> None:
|
|
"""rmtree that never crosses into another filesystem (a mounted disk or
|
|
share inside the stack folder is left as it is)."""
|
|
with os.scandir(path) as it:
|
|
entries = list(it)
|
|
for entry in entries:
|
|
entry_rel = f"{rel}/{entry.name}"
|
|
if entry.is_dir(follow_symlinks=False):
|
|
if entry.stat(follow_symlinks=False).st_dev != dev:
|
|
warnings.append(f"{entry_rel}: another filesystem is mounted here, left untouched")
|
|
continue
|
|
_remove_tree(entry.path, dev, entry_rel, warnings)
|
|
else:
|
|
os.unlink(entry.path)
|
|
try:
|
|
os.rmdir(path)
|
|
except OSError:
|
|
pass # still holds a mount point
|
|
|
|
|
|
def _clear(directory: str, preserve: set[str], warnings: list[str]) -> None:
|
|
"""Empty the stack directory for the restored tree, except what the backup
|
|
does not contain: ``preserve`` paths and other filesystems mounted inside."""
|
|
root_dev = os.lstat(directory).st_dev
|
|
|
|
def walk(path: str, rel: str) -> None:
|
|
with os.scandir(path) as it:
|
|
entries = list(it)
|
|
for entry in entries:
|
|
entry_rel = f"{rel}/{entry.name}" if rel else entry.name
|
|
if entry_rel in preserve:
|
|
continue
|
|
if entry.is_dir(follow_symlinks=False):
|
|
if entry.stat(follow_symlinks=False).st_dev != root_dev:
|
|
warnings.append(
|
|
f"{entry_rel}: another filesystem is mounted here, left untouched"
|
|
)
|
|
continue
|
|
if any(p.startswith(entry_rel + "/") for p in preserve):
|
|
walk(entry.path, entry_rel)
|
|
else:
|
|
_remove_tree(entry.path, root_dev, entry_rel, warnings)
|
|
else:
|
|
os.unlink(entry.path)
|
|
|
|
walk(directory, "")
|
|
|
|
|
|
def _copy_dir_meta(src: str, dst: str) -> None:
|
|
st = os.lstat(src)
|
|
try:
|
|
os.chown(dst, st.st_uid, st.st_gid)
|
|
os.chmod(dst, stat.S_IMODE(st.st_mode))
|
|
os.utime(dst, (st.st_atime, st.st_mtime))
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _merge(src: str, dst: str, rel: str, warnings: list[str]) -> None:
|
|
"""Move the staged tree into the (cleared) stack directory."""
|
|
for name in sorted(os.listdir(src)):
|
|
s, d = os.path.join(src, name), os.path.join(dst, name)
|
|
entry_rel = f"{rel}/{name}" if rel else name
|
|
s_dir = os.path.isdir(s) and not os.path.islink(s)
|
|
if os.path.lexists(d):
|
|
d_dir = os.path.isdir(d) and not os.path.islink(d)
|
|
if s_dir and d_dir:
|
|
_merge(s, d, entry_rel, warnings)
|
|
_copy_dir_meta(s, d)
|
|
continue
|
|
if d_dir:
|
|
# Only kept (preserved or mounted) directories survive _clear.
|
|
warnings.append(f"{entry_rel}: kept the existing folder, the backup has a file here")
|
|
continue
|
|
os.unlink(d)
|
|
shutil.move(s, d)
|
|
|
|
|
|
def _restore_tree(
|
|
tar: tarfile.TarFile,
|
|
directory: str,
|
|
preserve: set[str],
|
|
skip_extract: set[str],
|
|
warnings: list[str],
|
|
) -> None:
|
|
"""Replace the stack directory with the backup's, staged next to it first
|
|
so a failing extraction leaves the old one untouched."""
|
|
root = os.path.dirname(directory)
|
|
os.makedirs(root, exist_ok=True)
|
|
staging = tempfile.mkdtemp(prefix=f".restore-{os.path.basename(directory)}-", dir=root)
|
|
try:
|
|
_extract_tree(tar, "compose/", staging, skip=skip_extract)
|
|
if not os.path.isdir(directory):
|
|
shutil.move(staging, directory)
|
|
return
|
|
_clear(directory, preserve, warnings)
|
|
_merge(staging, directory, "", warnings)
|
|
_copy_dir_meta(staging, directory)
|
|
finally:
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
|
|
|
|
def _tree_paths(manifest: dict) -> tuple[set[str], set[str]]:
|
|
"""(excluded, compose_binds) relative to the stack directory.
|
|
|
|
Format 3 records them. For older backups the excluded folders are worked
|
|
out from the skipped binds; which data folders compose/ carries is unknown.
|
|
"""
|
|
if "excluded" in manifest:
|
|
return set(manifest.get("excluded") or []), set(manifest.get("compose_binds") or [])
|
|
old_dir = manifest.get("stack_dir") or ""
|
|
excluded = set()
|
|
for item in manifest.get("skipped", []):
|
|
if item.get("kind") == "bind" and old_dir and item.get("source"):
|
|
rel = _rel_inside(item["source"], old_dir)
|
|
if rel:
|
|
excluded.add(rel)
|
|
return excluded, set()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Restore
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _target_id(manifest: dict, target_id: Optional[str]) -> str:
|
|
raw = target_id or manifest.get("stack_id") or ""
|
|
if not raw.strip():
|
|
raise BackupError("Backup manifest has no stack id")
|
|
# Slugify whichever id we end up using — the manifest comes from an
|
|
# uploaded file, so its stack_id must never be able to escape STACKS_DIR.
|
|
return compose_service.slugify(raw)
|
|
|
|
|
|
def _extract_member(tar: tarfile.TarFile, name: str) -> Optional[str]:
|
|
"""Copy one archive member to a temp file; None if it is missing."""
|
|
try:
|
|
member = tar.getmember(name)
|
|
except KeyError:
|
|
return None
|
|
fh = tar.extractfile(member)
|
|
if fh is None:
|
|
return None
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".tar") as tmp:
|
|
shutil.copyfileobj(fh, tmp)
|
|
return tmp.name
|
|
|
|
|
|
def restore_backup(
|
|
tar_path: str,
|
|
target_id: Optional[str] = None,
|
|
overwrite: bool = False,
|
|
restore_volumes: bool = True,
|
|
restore_binds: bool = True,
|
|
verified: bool = False,
|
|
) -> dict:
|
|
"""Restore a backup's files, folders and volumes. Returns a report.
|
|
|
|
This is the part that writes; :func:`restore` wraps it with the lock and
|
|
with stopping and starting the stack.
|
|
"""
|
|
manifest = read_manifest(tar_path)
|
|
stack_id = _target_id(manifest, target_id)
|
|
old_id = compose_service.slugify(manifest.get("stack_id") or stack_id)
|
|
renamed = stack_id != old_id
|
|
# Volumes are named after the compose *project*, which differs from the id
|
|
# for a folder compose had to normalise ("my.app" -> "myapp_data"), and
|
|
# for one whose files name the project. Backups before 0.67.2 do not say.
|
|
old_project = manifest.get("project") or compose_service.project_name(
|
|
manifest.get("stack_id") or stack_id
|
|
)
|
|
old_dir = manifest.get("stack_dir") or ""
|
|
|
|
directory = compose_service.stack_dir(stack_id)
|
|
if os.path.isdir(directory) and not overwrite:
|
|
raise BackupError(f"Stack '{stack_id}' already exists")
|
|
if not verified:
|
|
verify_archive(tar_path)
|
|
|
|
excluded, compose_binds = _tree_paths(manifest)
|
|
# Not restoring data folders means keeping the ones there are, not
|
|
# emptying them: compose/ carries those inside the stack directory.
|
|
held_back = compose_binds if not restore_binds else set()
|
|
volumes_restored = 0
|
|
binds_restored = 0
|
|
skipped: list[dict] = []
|
|
warnings: list[str] = []
|
|
|
|
with tarfile.open(tar_path, "r:gz") as tar:
|
|
_restore_tree(tar, directory, excluded | held_back, held_back, warnings)
|
|
# Read from the files just put back: their name: / .env decide it.
|
|
new_project = compose_service.project_name(stack_id)
|
|
skipped += [{"kind": "bind", "source": rel, "reason": "not requested"} for rel in held_back]
|
|
|
|
for bind in manifest.get("binds", []):
|
|
archive = bind.get("archive")
|
|
if not archive:
|
|
continue
|
|
source = bind["source"]
|
|
if not restore_binds:
|
|
skipped.append({"kind": "bind", "source": source, "reason": "not requested"})
|
|
continue
|
|
if bind.get("inside_stack_dir") and old_dir:
|
|
# A renamed stack must not write into the old stack's folder.
|
|
source = os.path.normpath(os.path.join(directory, os.path.relpath(source, old_dir)))
|
|
elif renamed:
|
|
skipped.append({
|
|
"kind": "bind",
|
|
"source": source,
|
|
"reason": f"belongs to the original stack '{old_id}'; "
|
|
"restore under that name to write it back",
|
|
})
|
|
continue
|
|
if assets.is_system_path(source):
|
|
skipped.append({"kind": "bind", "source": source, "reason": "system path"})
|
|
continue
|
|
part = _extract_member(tar, archive)
|
|
if part is None:
|
|
skipped.append({"kind": "bind", "source": source, "reason": "missing from archive"})
|
|
continue
|
|
try:
|
|
assets.import_path(source, bind.get("kind", "dir"), part)
|
|
binds_restored += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Could not restore bind %s: %s", source, exc)
|
|
skipped.append({"kind": "bind", "source": source, "reason": str(exc)})
|
|
finally:
|
|
os.unlink(part)
|
|
|
|
for v in manifest.get("volumes", []):
|
|
full = v["full"]
|
|
if not restore_volumes:
|
|
skipped.append({"kind": "volume", "name": full, "reason": "not requested"})
|
|
continue
|
|
retargeted = False
|
|
if old_project != new_project and full.startswith(old_project + "_"):
|
|
full = new_project + full[len(old_project) :]
|
|
retargeted = True
|
|
labels = dict(v.get("labels") or {})
|
|
if not v.get("external"):
|
|
# Re-target compose's labels to the (possibly new) project.
|
|
labels[COMPOSE_PROJECT_LABEL] = new_project
|
|
reason = _volume_conflict(full, new_project, old_id, renamed and not retargeted)
|
|
if reason:
|
|
skipped.append({"kind": "volume", "name": full, "reason": reason})
|
|
continue
|
|
part = _extract_member(tar, v.get("archive") or f"volumes/{v['full']}.tar")
|
|
if part is None:
|
|
skipped.append({"kind": "volume", "name": full, "reason": "missing from archive"})
|
|
continue
|
|
try:
|
|
# Remote-backed volumes (NFS/CIFS) are never wiped: that
|
|
# would delete the share the volume points at.
|
|
assets.import_volume(
|
|
full,
|
|
labels,
|
|
part,
|
|
wipe=not v.get("remote", False),
|
|
driver=v.get("driver"),
|
|
driver_opts=v.get("options") or None,
|
|
)
|
|
volumes_restored += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Could not restore volume %s: %s", full, exc)
|
|
skipped.append({"kind": "volume", "name": full, "reason": str(exc)})
|
|
finally:
|
|
os.unlink(part)
|
|
|
|
# What went wrong now, as opposed to what the backup itself left out
|
|
# (listed in skipped as well, for the full picture).
|
|
problems = [
|
|
f"{item.get('name') or item.get('source')}: {item['reason']}"
|
|
for item in skipped
|
|
if item["reason"] != "not requested"
|
|
] + warnings
|
|
skipped += manifest.get("skipped", [])
|
|
return {
|
|
"stack_id": stack_id,
|
|
# Restored under a new id, the copy must not carry the original's
|
|
# display name — two stacks called the same in the list is how the
|
|
# wrong one gets stopped.
|
|
"name": stack_id if renamed else manifest.get("name", stack_id),
|
|
"volumes_restored": volumes_restored,
|
|
"binds_restored": binds_restored,
|
|
"skipped": skipped,
|
|
"warnings": warnings,
|
|
"problems": problems,
|
|
}
|
|
|
|
|
|
def _volume_conflict(full: str, project: str, old_id: str, shared_with_original: bool) -> Optional[str]:
|
|
"""Why a volume must not be overwritten, or None if it may be."""
|
|
attrs = assets.volume_attrs(full)
|
|
if attrs is None:
|
|
return None # created fresh
|
|
owner = (attrs.get("Labels") or {}).get(COMPOSE_PROJECT_LABEL)
|
|
if owner and owner != project:
|
|
return f"belongs to stack '{owner}'; not overwritten"
|
|
if shared_with_original:
|
|
return (
|
|
f"is not named after the stack, so it is still the original "
|
|
f"'{old_id}' stack's volume; not overwritten"
|
|
)
|
|
users = assets.running_users(full)
|
|
if users:
|
|
return f"in use by running container(s) {', '.join(users)}; not overwritten"
|
|
return None
|
|
|
|
|
|
async def restore(
|
|
tar_path: str,
|
|
target_id: Optional[str] = None,
|
|
overwrite: bool = False,
|
|
restore_volumes: bool = True,
|
|
restore_binds: bool = True,
|
|
) -> dict:
|
|
"""Restore with the stack locked, stopped while its data is replaced, and
|
|
brought back up afterwards if it was running."""
|
|
manifest = await asyncio.to_thread(read_manifest, tar_path)
|
|
stack_id = _target_id(manifest, target_id)
|
|
directory = compose_service.stack_dir(stack_id)
|
|
if os.path.isdir(directory) and not overwrite:
|
|
raise BackupError(f"Stack '{stack_id}' already exists")
|
|
|
|
with _locked(stack_id, "restore"):
|
|
await asyncio.to_thread(verify_archive, tar_path)
|
|
try:
|
|
running = await _running(stack_id)
|
|
except BackupError:
|
|
running = [] # no Docker: the volume part will say so
|
|
if running:
|
|
# Replacing a database's files under it corrupts it.
|
|
await _stop(stack_id, running)
|
|
try:
|
|
result = await asyncio.to_thread(
|
|
restore_backup, tar_path, target_id, overwrite,
|
|
restore_volumes, restore_binds, True,
|
|
)
|
|
except BaseException:
|
|
if running:
|
|
await _start(stack_id, running)
|
|
raise
|
|
result["restarted"] = False
|
|
if running:
|
|
# up, not start: the restored compose file may differ from what ran.
|
|
error = await _compose(stack_id, ["up", "-d", "--remove-orphans"])
|
|
result["restarted"] = error is None
|
|
if error:
|
|
result["restart_error"] = error
|
|
result["problems"].append(f"stack did not start again: {error}")
|
|
return result
|