menzeljonas/stackpilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Back up bind-mount data, not just the compose file (0.40.0)

Browse Source 
A stack's real state lives in its bind-mounted config directories, and those
were never captured: the backup only tarred the stack folder as this container
sees it. When STACKS_HOST_DIR differs from the container's STACKS_DIR, compose
resolves ./config against the container path and the daemon creates it at that
path on the *host* — invisible here, so the archive held little more than
compose.yaml and .env.

New services/stack_assets_service.py inventories a stack's data (bind sources
merged from container mounts + the compose file, named volumes) and does all
data I/O through a throwaway helper container, i.e. by host path, so unseen
directories are captured anyway. It also detects the host/container stacks-path
mismatch and reports it.

- manifest v2: full inventory, per-asset capture result, skip reasons (v1 still
  restores)
- NFS/CIFS-backed volumes are skipped by default and never wiped on restore
- deselected data inside the stack folder no longer sneaks in via compose/
- volume/bind archives stream through temp files instead of RAM
- restore preserves mode, ownership, mtime and symlinks, and writes bind folders
  back to their host paths (rewritten when the stack is renamed)
- backup dialog shows the inventory with sizes and per-item checkboxes; restore
  gained a "restore bind folders" toggle
- new GET /api/stacks/{id}/backup/inventory (+ agent + proxy), backup endpoints
  take include_binds/binds/volumes, restore takes restore_binds

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
menzelj
2026-08-16 18:20:19 +00:00
co-authored by Claude Opus 5
parent ecf780c5e6
commit 5347a36eaf
11 changed files with 1301 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/backup_service.py
+328 -140
View File
@@ -1,14 +1,16 @@
"""Stack backup & restore, including named-volume contents.
"""Stack backup & restore — compose files, bind-mount data and named volumes.
A backup is a single ``.tar.gz`` with this layout::
manifest.json metadata + volume/bind inventory
compose/... the full stack directory (compose file, .env, ...)
volumes/<full>.tar raw contents of each compose-managed named volume
manifest.json metadata + full inventory of what was captured
compose/... the stack directory as StackPilot can see it
binds/<n>.tar contents of each captured bind-mount source
volumes/<full>.tar contents of each captured named volume
Named-volume contents are read/written through a throwaway helper container
(``BACKUP_HELPER_IMAGE``) with the volume bind-mounted — this is the portable
way to snapshot a volume regardless of its driver/mountpoint.
Bind sources and volumes are read/written through a throwaway helper container
(see :mod:`services.stack_assets_service`) so that host paths this container
cannot see are still captured — without that, a stack whose data directories
live outside StackPilot's own mount would back up as "just the compose file".
"""
from __future__ import annotations
@@ -22,16 +24,14 @@ import tarfile
import tempfile
from typing import Optional
from config import settings
from docker_client import DockerError, get_client, safe_call
from services import compose_service
from services import compose_service, stack_assets_service as assets
logger = logging.getLogger("stackpilot.backup")
COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
COMPOSE_VOLUME_LABEL = "com.docker.compose.volume"
COMPOSE_PROJECT_LABEL = assets.COMPOSE_PROJECT_LABEL
COMPOSE_VOLUME_LABEL = assets.COMPOSE_VOLUME_LABEL
MANIFEST_NAME = "manifest.json"
BACKUP_FORMAT_VERSION = 1
BACKUP_FORMAT_VERSION = 2
class BackupError(Exception):
@@ -59,98 +59,48 @@ def backup_filename(stack_id: str, include_volumes: bool, prefix: Optional[str]
# --------------------------------------------------------------------------- #
# Helper container for volume I/O
# Selection
# --------------------------------------------------------------------------- #
def _ensure_helper_image(client) -> None:
image = settings.BACKUP_HELPER_IMAGE
try:
safe_call(client.images.get, image)
except DockerError:
logger.info("Pulling backup helper image %s", image)
safe_call(client.images.pull, image)
def _decide(
items: list[dict], key: str, chosen: Optional[list[str]], enabled: bool
) -> list[dict]:
"""Mark each inventory item ``selected`` (with a reason when it is not).
``chosen`` is an explicit list from the caller; without one the inventory's
own defaults apply (everything except system paths, oversized directories
and remote-backed volumes).
"""
for item in items:
if not enabled:
item["selected"], item["reason"] = False, "not requested"
elif chosen is not None:
selected = item[key] in chosen
item["selected"] = selected
item["reason"] = None if selected else "not selected"
else:
item["selected"] = bool(item.get("include_default"))
item["reason"] = None if item["selected"] else (item.get("reason") or "not selected")
# Never archive plumbing, whatever the caller asked for.
if item["selected"] and (item.get("system") or item.get("kind") in ("special", "unknown")):
item["selected"] = False
item["reason"] = item.get("reason") or "not a regular file or directory"
return items
def _export_volume(full_name: str) -> bytes:
client = get_client()
_ensure_helper_image(client)
container = safe_call(
client.containers.create,
settings.BACKUP_HELPER_IMAGE,
command="true",
volumes={full_name: {"bind": "/v", "mode": "ro"}},
)
try:
# "/v/." copies the *contents* of the volume (no leading "v/" prefix),
# so restore can extract straight back into the volume root.
bits, _ = container.get_archive("/v/.")
buf = io.BytesIO()
for chunk in bits:
buf.write(chunk)
return buf.getvalue()
finally:
try:
container.remove(force=True)
except Exception: # noqa: BLE001
pass
def _restore_volume(full_name: str, labels: dict, tar_bytes: bytes) -> None:
client = get_client()
_ensure_helper_image(client)
existed = True
try:
safe_call(client.volumes.get, full_name)
except DockerError:
existed = False
safe_call(client.volumes.create, name=full_name, labels=labels or {})
if existed:
# Restore means "back to the snapshot": clear the current contents so
# files created/kept since the backup don't survive underneath it.
safe_call(
client.containers.run,
settings.BACKUP_HELPER_IMAGE,
["sh", "-c", "find /v -mindepth 1 -delete"],
volumes={full_name: {"bind": "/v", "mode": "rw"}},
remove=True,
)
container = safe_call(
client.containers.create,
settings.BACKUP_HELPER_IMAGE,
command="true",
volumes={full_name: {"bind": "/v", "mode": "rw"}},
)
try:
container.put_archive("/v", tar_bytes)
finally:
try:
container.remove(force=True)
except Exception: # noqa: BLE001
pass
def _compose_volumes(stack_id: str) -> list[dict]:
"""Return [{full, short, labels}] for compose-managed named volumes."""
try:
client = get_client()
vols = safe_call(
client.volumes.list,
filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"},
)
except DockerError:
return []
out = []
for v in vols:
labels = v.attrs.get("Labels") or {}
out.append(
{
"full": v.name,
"short": labels.get(COMPOSE_VOLUME_LABEL, v.name),
"labels": labels,
}
)
return out
def plan(
stack_id: str,
include_volumes: bool = True,
include_binds: bool = True,
binds: Optional[list[str]] = None,
volumes: Optional[list[str]] = None,
) -> dict:
"""Decide what a backup captures. Returned as-is by the inventory endpoint."""
inv = assets.inventory(stack_id)
_decide(inv["binds"], "source", binds, include_binds)
_decide(inv["volumes"], "name", volumes, include_volumes)
return inv
# --------------------------------------------------------------------------- #
@@ -158,23 +108,47 @@ def _compose_volumes(stack_id: str) -> list[dict]:
# --------------------------------------------------------------------------- #
async def create_backup(
def _compose_filter(excluded_prefixes: list[str]):
"""Drop deselected bind directories from the compose/ tree (keep the mount
point itself, so the stack still starts after a restore)."""
def _filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
for prefix in excluded_prefixes:
if info.name.startswith(prefix + "/"):
return None
return info
return _filter
async def create_backup_ex(
stack_id: str,
name: str,
include_volumes: bool = True,
stop_first: bool = True,
) -> str:
"""Create a backup tar.gz and return its path on disk."""
include_binds: bool = True,
binds: Optional[list[str]] = None,
volumes: Optional[list[str]] = None,
) -> tuple[str, dict]:
"""Create a backup tar.gz. Returns (path, report)."""
directory = compose_service.stack_dir(stack_id)
if not os.path.isdir(directory):
raise BackupError("Stack directory missing")
volumes = _compose_volumes(stack_id) if include_volumes else []
selection = await asyncio.to_thread(
plan, stack_id, include_volumes, include_binds, binds, volumes
)
# Selected bind sources this process cannot reach through the filesystem
# need their own archive; the rest already ride along in compose/.
cap_binds = [b for b in selection["binds"] if b["selected"] and b["via"] == "archive"]
cap_volumes = [v for v in selection["volumes"] if v["selected"]]
skipped_binds = [b for b in selection["binds"] if not b["selected"]]
skipped_volumes = [v for v in selection["volumes"] if not v["selected"]]
# For a consistent volume snapshot, stop the stack first — but only if it
# is actually running, so backing up a stopped stack doesn't start it.
# Consistent snapshot: stop the stack first — but only if it is actually
# running, so backing up a stopped stack doesn't start it.
stopped = False
if include_volumes and stop_first and volumes:
if stop_first and (cap_volumes or cap_binds):
if compose_service.compute_status(stack_id) not in ("stopped", "unknown"):
try:
await compose_service.stop(stack_id)
@@ -182,34 +156,122 @@ async def create_backup(
except Exception as exc: # noqa: BLE001
logger.warning("Could not stop %s before backup: %s", stack_id, exc)
workdir = tempfile.mkdtemp(prefix="sp-backup-")
try:
# Archive bind sources and volumes into the work directory first, so a
# failure on one asset is reported instead of corrupting the tar.
for index, bind in enumerate(cap_binds):
bind["archive"] = f"binds/{index:03d}.tar"
part = os.path.join(workdir, f"bind-{index:03d}.tar")
try:
bind["bytes"] = await asyncio.to_thread(
assets.export_path, bind["source"], bind["kind"], part
)
bind["_part"] = part
except Exception as exc: # noqa: BLE001
logger.warning("Could not archive bind %s: %s", bind["source"], exc)
bind["archive"] = None
bind["error"] = str(exc)
for index, vol in enumerate(cap_volumes):
vol["archive"] = f"volumes/{vol['name']}.tar"
part = os.path.join(workdir, f"vol-{index:03d}.tar")
try:
vol["bytes"] = await asyncio.to_thread(assets.export_volume, vol["name"], part)
vol["_part"] = part
except Exception as exc: # noqa: BLE001
logger.warning("Could not archive volume %s: %s", vol["name"], exc)
vol["archive"] = None
vol["error"] = str(exc)
# Deselected data that sits inside the stack folder must not sneak into
# the archive through compose/ (that is how a 200 GB downloads folder
# ends up in a "config only" backup).
excluded = [
"compose/" + os.path.relpath(b["source"], directory)
for b in skipped_binds
if b.get("inside_stack_dir")
]
manifest = {
"format_version": BACKUP_FORMAT_VERSION,
"stack_id": stack_id,
"name": name,
"created_at": compose_service.now().isoformat(),
"include_volumes": include_volumes,
"volumes": [{"full": v["full"], "short": v["short"], "labels": v["labels"]} for v in volumes],
"include_binds": include_binds,
"stack_dir": directory,
"path_mismatch": selection.get("path_mismatch"),
# Volumes keep the v1 shape (full/short/labels) so older StackPilots
# can still read the manifest they care about.
"volumes": [
{
"full": v["name"],
"short": v["short"],
"labels": v["labels"],
"archive": v.get("archive"),
"remote": v.get("remote", False),
"bytes": v.get("bytes"),
"error": v.get("error"),
}
for v in cap_volumes
if v.get("archive")
],
"binds": [
{
"source": b["source"],
"kind": b["kind"],
"mounts": b["mounts"],
"inside_stack_dir": b["inside_stack_dir"],
"archive": b.get("archive"),
"bytes": b.get("bytes"),
"error": b.get("error"),
}
for b in cap_binds
if b.get("archive")
],
"skipped": [
{"kind": "bind", "source": b["source"], "reason": b.get("reason")}
for b in skipped_binds
]
+ [
{"kind": "volume", "name": v["name"], "reason": v.get("reason")}
for v in skipped_volumes
],
}
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
tmp.close()
with tarfile.open(tmp.name, "w:gz") as tar:
# manifest
data = json.dumps(manifest, indent=2).encode("utf-8")
info = tarfile.TarInfo(MANIFEST_NAME)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
# stack directory
tar.add(directory, arcname="compose")
# volume contents
for v in volumes:
vbytes = await asyncio.to_thread(_export_volume, v["full"])
info = tarfile.TarInfo(f"volumes/{v['full']}.tar")
info.size = len(vbytes)
tar.addfile(info, io.BytesIO(vbytes))
return tmp.name
tar.add(directory, arcname="compose", filter=_compose_filter(excluded))
for bind in cap_binds:
if bind.get("_part"):
tar.add(bind["_part"], arcname=bind["archive"])
for vol in cap_volumes:
if vol.get("_part"):
tar.add(vol["_part"], arcname=vol["archive"])
report = {
"file": tmp.name,
"binds": [
{"source": b["source"], "bytes": b.get("bytes"), "error": b.get("error")}
for b in cap_binds
],
"volumes": [
{"name": v["name"], "bytes": v.get("bytes"), "error": v.get("error")}
for v in cap_volumes
],
"skipped": manifest["skipped"],
"path_mismatch": selection.get("path_mismatch"),
"size": os.path.getsize(tmp.name),
}
return tmp.name, report
finally:
shutil.rmtree(workdir, ignore_errors=True)
if stopped:
try:
await compose_service.up(stack_id)
@@ -217,6 +279,21 @@ async def create_backup(
logger.warning("Could not restart %s after backup: %s", stack_id, exc)
async def create_backup(
stack_id: str,
name: str,
include_volumes: bool = True,
stop_first: bool = True,
include_binds: bool = True,
binds: Optional[list[str]] = None,
volumes: Optional[list[str]] = None,
) -> str:
path, _report = await create_backup_ex(
stack_id, name, include_volumes, stop_first, include_binds, binds, volumes
)
return path
# --------------------------------------------------------------------------- #
# Restore
# --------------------------------------------------------------------------- #
@@ -224,33 +301,87 @@ async def create_backup(
def read_manifest(tar_path: str) -> dict:
with tarfile.open(tar_path, "r:gz") as tar:
member = tar.getmember(MANIFEST_NAME)
try:
member = tar.getmember(MANIFEST_NAME)
except KeyError as exc:
raise BackupError("Backup is missing its manifest") from exc
fh = tar.extractfile(member)
if fh is None:
raise BackupError("Backup is missing its manifest")
return json.loads(fh.read().decode("utf-8"))
def _safe_extract_compose(tar: tarfile.TarFile, dest_dir: str) -> None:
"""Extract the ``compose/`` subtree into dest_dir, guarding path traversal."""
def _safe_target(dest_dir: str, rel: str) -> str:
"""Resolve a member path inside dest_dir, refusing traversal *and* writes
through a symlink planted earlier in the same archive."""
root = os.path.abspath(dest_dir)
target = os.path.normpath(os.path.join(root, rel))
if target != root and not target.startswith(root + os.sep):
raise BackupError(f"Refusing unsafe path in backup: {rel}")
parent = os.path.dirname(target)
if os.path.exists(parent):
real_parent = os.path.realpath(parent)
if real_parent != root and not real_parent.startswith(root + os.sep):
raise BackupError(f"Refusing unsafe path in backup: {rel}")
return target
def _apply_meta(path: str, member: tarfile.TarInfo) -> None:
"""Restore mode/ownership/mtime — *arr-style images run as PUID/PGID and
break when their config comes back root-owned with default permissions."""
try:
os.chmod(path, member.mode)
except OSError:
pass
try:
os.chown(path, member.uid, member.gid)
except (OSError, AttributeError):
pass
try:
os.utime(path, (member.mtime, member.mtime))
except OSError:
pass
def _extract_tree(tar: tarfile.TarFile, prefix: str, dest_dir: str) -> None:
"""Extract one subtree of the archive, preserving metadata and symlinks."""
os.makedirs(dest_dir, exist_ok=True)
dirs: list[tuple[str, tarfile.TarInfo]] = []
for member in tar.getmembers():
if not member.name.startswith("compose/"):
if not member.name.startswith(prefix):
continue
rel = member.name[len("compose/") :]
rel = member.name[len(prefix) :].lstrip("/")
if not rel:
continue
target = os.path.normpath(os.path.join(dest_dir, rel))
if not target.startswith(os.path.abspath(dest_dir) + os.sep) and target != os.path.abspath(dest_dir):
raise BackupError(f"Refusing unsafe path in backup: {member.name}")
target = _safe_target(dest_dir, rel)
if member.isdir():
os.makedirs(target, exist_ok=True)
elif member.isreg():
os.makedirs(os.path.dirname(target), exist_ok=True)
src = tar.extractfile(member)
if src is not None:
with open(target, "wb") as out:
shutil.copyfileobj(src, out)
dirs.append((target, member))
continue
os.makedirs(os.path.dirname(target), exist_ok=True)
if member.issym():
if os.path.lexists(target):
os.unlink(target)
os.symlink(member.linkname, target)
continue # chmod/utime would follow the link
if member.islnk():
source = _safe_target(dest_dir, member.linkname[len(prefix) :].lstrip("/"))
if os.path.exists(source):
if os.path.lexists(target):
os.unlink(target)
os.link(source, target)
continue
if not member.isreg():
continue # devices/fifos/sockets are runtime artefacts
src = tar.extractfile(member)
if src is None:
continue
with open(target, "wb") as out:
shutil.copyfileobj(src, out)
_apply_meta(target, member)
# Directory metadata last: a 0500 directory would block writing its files.
for target, member in sorted(dirs, key=lambda d: len(d[0]), reverse=True):
_apply_meta(target, member)
def restore_backup(
@@ -258,8 +389,9 @@ def restore_backup(
target_id: Optional[str] = None,
overwrite: bool = False,
restore_volumes: bool = True,
restore_binds: bool = True,
) -> dict:
"""Restore a backup. Returns {stack_id, name, volumes_restored}."""
"""Restore a backup. Returns a report of what was written."""
manifest = read_manifest(tar_path)
raw_id = target_id or manifest.get("stack_id") or ""
if not raw_id.strip():
@@ -267,21 +399,63 @@ def restore_backup(
# Slugify whichever id we end up using — the manifest comes from an
# uploaded file, so its stack_id must never be able to escape STACKS_DIR.
stack_id = compose_service.slugify(raw_id)
old_id = manifest.get("stack_id") or stack_id
old_dir = manifest.get("stack_dir") or ""
directory = compose_service.stack_dir(stack_id)
exists = os.path.isdir(directory)
if exists and not overwrite:
raise BackupError(f"Stack '{stack_id}' already exists")
volumes_restored = 0
binds_restored = 0
skipped: list[dict] = []
with tarfile.open(tar_path, "r:gz") as tar:
if exists:
shutil.rmtree(directory)
_safe_extract_compose(tar, directory)
_extract_tree(tar, "compose/", directory)
if restore_binds:
for bind in manifest.get("binds", []):
archive = bind.get("archive")
if not archive:
continue
try:
member = tar.getmember(archive)
except KeyError:
continue
source = bind["source"]
# A renamed stack must not write into the old stack's folder.
if old_dir and bind.get("inside_stack_dir"):
rel = os.path.relpath(source, old_dir)
source = os.path.normpath(os.path.join(directory, rel))
if assets.is_system_path(source):
skipped.append({"kind": "bind", "source": source, "reason": "system path"})
continue
fh = tar.extractfile(member)
if fh is None:
continue
with tempfile.NamedTemporaryFile(delete=False, suffix=".tar") as tmp:
shutil.copyfileobj(fh, tmp)
part = tmp.name
try:
assets.import_path(source, bind.get("kind", "dir"), part)
binds_restored += 1
except Exception as exc: # noqa: BLE001
logger.warning("Could not restore bind %s: %s", source, exc)
skipped.append({"kind": "bind", "source": source, "reason": str(exc)})
finally:
os.unlink(part)
elif manifest.get("binds"):
skipped += [
{"kind": "bind", "source": b["source"], "reason": "not requested"}
for b in manifest["binds"]
]
volumes_restored = 0
if restore_volumes:
for v in manifest.get("volumes", []):
member_name = f"volumes/{v['full']}.tar"
member_name = v.get("archive") or f"volumes/{v['full']}.tar"
try:
member = tar.getmember(member_name)
except KeyError:
@@ -293,13 +467,27 @@ def restore_backup(
labels = dict(v.get("labels") or {})
labels[COMPOSE_PROJECT_LABEL] = stack_id
full = v["full"]
if target_id and manifest.get("stack_id") and full.startswith(manifest["stack_id"] + "_"):
full = stack_id + full[len(manifest["stack_id"]):]
_restore_volume(full, labels, fh.read())
volumes_restored += 1
if target_id and old_id and full.startswith(old_id + "_"):
full = stack_id + full[len(old_id) :]
with tempfile.NamedTemporaryFile(delete=False, suffix=".tar") as tmp:
shutil.copyfileobj(fh, tmp)
part = tmp.name
try:
# Remote-backed volumes (NFS/CIFS) are never wiped: that
# would delete the share the volume points at.
assets.import_volume(full, labels, part, wipe=not v.get("remote", False))
volumes_restored += 1
except Exception as exc: # noqa: BLE001
logger.warning("Could not restore volume %s: %s", full, exc)
skipped.append({"kind": "volume", "name": full, "reason": str(exc)})
finally:
os.unlink(part)
skipped += manifest.get("skipped", [])
return {
"stack_id": stack_id,
"name": manifest.get("name", stack_id),
"volumes_restored": volumes_restored,
"binds_restored": binds_restored,
"skipped": skipped,
}