menzeljonas/stackpilot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Back up bind-mount data, not just the compose file (0.40.0)

Browse Source 
A stack's real state lives in its bind-mounted config directories, and those
were never captured: the backup only tarred the stack folder as this container
sees it. When STACKS_HOST_DIR differs from the container's STACKS_DIR, compose
resolves ./config against the container path and the daemon creates it at that
path on the *host* — invisible here, so the archive held little more than
compose.yaml and .env.

New services/stack_assets_service.py inventories a stack's data (bind sources
merged from container mounts + the compose file, named volumes) and does all
data I/O through a throwaway helper container, i.e. by host path, so unseen
directories are captured anyway. It also detects the host/container stacks-path
mismatch and reports it.

- manifest v2: full inventory, per-asset capture result, skip reasons (v1 still
  restores)
- NFS/CIFS-backed volumes are skipped by default and never wiped on restore
- deselected data inside the stack folder no longer sneaks in via compose/
- volume/bind archives stream through temp files instead of RAM
- restore preserves mode, ownership, mtime and symlinks, and writes bind folders
  back to their host paths (rewritten when the stack is renamed)
- backup dialog shows the inventory with sizes and per-item checkboxes; restore
  gained a "restore bind folders" toggle
- new GET /api/stacks/{id}/backup/inventory (+ agent + proxy), backup endpoints
  take include_binds/binds/volumes, restore takes restore_binds

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
menzelj
2026-08-16 18:20:19 +00:00
co-authored by Claude Opus 5
parent ecf780c5e6
commit 5347a36eaf
11 changed files with 1301 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+13 -3
View File
@@ -86,9 +86,19 @@ as intuitive as Dockge, as capable as Portainer for Compose workflows.
from its container labels) — the helper survives the backend being recreated;
the UI polls `/api/health` and reloads when the new version answers. Installs
not managed by compose get a clear "update manually" error instead.
- **Backup & restore**: per-stack `.tar.gz` backups including named-volume contents
(snapshotted via a throwaway helper container); restore via upload with optional
rename, volume restore, and overwrite/conflict detection.
- **Backup & restore**: per-stack `.tar.gz` backups covering the whole stack —
the stack folder, every **bind-mounted data directory** (`./config`, `/mnt/appdata/…`)
and every named volume. Bind sources and volumes are read through a throwaway
helper container, i.e. by **host path**, so data that StackPilot itself cannot
see is captured too (that is the case whenever `STACKS_HOST_DIR` differs from
the container's `STACKS_DIR` — compose then creates the data directories at the
container path *on the host*, and a naive backup would only find the compose
file). The Backup dialog shows the full inventory with sizes and lets you pick
what goes in; **NFS/CIFS-backed volumes are unchecked by default** because they
live on a NAS and restoring one would overwrite the share. Restore puts bind
folders back at their host paths and preserves permissions, ownership and
symlinks (PUID/PGID-based images such as the *arr suite need this), with
optional rename and overwrite/conflict detection.
- **Notification webhooks**: ntfy, Discord, Slack, Gotify, or generic JSON, each
subscribed to chosen events (image update available, stack start/stop/error,
pull failed). Managed in **Settings → Notifications**; env `NOTIFY_WEBHOOKS`
backend/agent_app.py
+23 -2
View File
@@ -10,6 +10,7 @@ All compose/Docker logic is reused from the backend's ``compose_service`` and
"""
from __future__ import annotations
import asyncio
import logging
import os
import shutil
@@ -464,17 +465,28 @@ def agent_detach_secret(stack_id: str, body: SecretDetachBody) -> dict:
return {"ok": True, "yaml": new_yaml}
@app.get("/agent/stacks/{stack_id}/backup/inventory", dependencies=[Depends(verify_token)])
async def backup_inventory(stack_id: str) -> dict:
"""What a backup of this stack would capture (see routers/backups.py)."""
_ensure_stack(stack_id)
return await asyncio.to_thread(backup_service.plan, stack_id)
@app.get("/agent/stacks/{stack_id}/backup", dependencies=[Depends(verify_token)])
async def backup_stack(
stack_id: str,
include_volumes: bool = Query(True),
include_binds: bool = Query(True),
stop_first: bool = Query(True),
binds: list[str] | None = Query(None),
volumes: list[str] | None = Query(None),
):
if not os.path.isdir(compose_service.stack_dir(stack_id)):
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
try:
path = await backup_service.create_backup(
path, report = await backup_service.create_backup_ex(
stack_id, stack_id, include_volumes=include_volumes, stop_first=stop_first,
include_binds=include_binds, binds=binds, volumes=volumes,
)
except backup_service.BackupError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@@ -482,6 +494,13 @@ async def backup_stack(
path,
media_type="application/gzip",
filename=backup_service.backup_filename(stack_id, include_volumes),
headers={"X-Stackpilot-Backup": json.dumps({
"size": report.get("size"),
"binds": report.get("binds", []),
"volumes": report.get("volumes", []),
"skipped": report.get("skipped", []),
"path_mismatch": report.get("path_mismatch"),
})},
)
@@ -491,6 +510,7 @@ async def restore_stack(
target_id: str | None = Form(None),
overwrite: bool = Form(False),
restore_volumes: bool = Form(True),
restore_binds: bool = Form(True),
) -> dict:
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
try:
@@ -500,7 +520,8 @@ async def restore_stack(
target = compose_service.slugify(target_id) if target_id else None
try:
return backup_service.restore_backup(
tmp.name, target_id=target, overwrite=overwrite, restore_volumes=restore_volumes,
tmp.name, target_id=target, overwrite=overwrite,
restore_volumes=restore_volumes, restore_binds=restore_binds,
)
except backup_service.BackupError as exc:
code = 409 if "already exists" in str(exc) else 400
backend/routers/agents.py
+35 -2
View File
@@ -39,7 +39,10 @@ _ACTIONS = {"start", "stop", "restart", "pull", "update", "down"}
class AgentPushBody(BaseModel):
destination_id: int
include_volumes: bool = True
include_binds: bool = True
stop_first: bool = True
binds: list[str] | None = None
volumes: list[str] | None = None
class AgentRestoreFromBody(BaseModel):
@@ -47,6 +50,7 @@ class AgentRestoreFromBody(BaseModel):
name: str
target_id: str | None = None
overwrite: bool = False
restore_binds: bool = True
restore_volumes: bool = True
@@ -351,13 +355,27 @@ async def agent_lifecycle(
# --------------------------------------------------------------------------- #
@router.get("/{agent_id}/stacks/{stack_id}/backup/inventory")
async def agent_backup_inventory(
agent_id: int,
stack_id: str,
session: Session = Depends(get_session),
user: User = Depends(require_admin),
) -> dict:
agent = _get_or_404(session, agent_id)
return await _proxy(session, agent, "GET", f"/agent/stacks/{stack_id}/backup/inventory")
@router.get("/{agent_id}/stacks/{stack_id}/backup")
async def agent_backup_download(
agent_id: int,
stack_id: str,
request: Request,
include_volumes: bool = Query(True),
include_binds: bool = Query(True),
stop_first: bool = Query(True),
binds: list[str] | None = Query(None),
volumes: list[str] | None = Query(None),
session: Session = Depends(get_session),
user: User = Depends(require_admin),
):
@@ -367,7 +385,13 @@ async def agent_backup_download(
try:
await agent_service.download_to_file(
session, agent, f"/agent/stacks/{stack_id}/backup", tmp.name,
params={"include_volumes": include_volumes, "stop_first": stop_first},
params={
"include_volumes": include_volumes,
"include_binds": include_binds,
"stop_first": stop_first,
**({"binds": binds} if binds else {}),
**({"volumes": volumes} if volumes else {}),
},
)
except AgentError as exc:
if os.path.exists(tmp.name):
@@ -403,7 +427,13 @@ async def agent_backup_push(
try:
await agent_service.download_to_file(
session, agent, f"/agent/stacks/{stack_id}/backup", tmp.name,
params={"include_volumes": body.include_volumes, "stop_first": body.stop_first},
params={
"include_volumes": body.include_volumes,
"include_binds": body.include_binds,
"stop_first": body.stop_first,
**({"binds": body.binds} if body.binds else {}),
**({"volumes": body.volumes} if body.volumes else {}),
},
)
except AgentError as exc:
_raise(exc)
@@ -432,6 +462,7 @@ async def agent_restore_upload(
target_id: str | None = Form(None),
overwrite: bool = Form(False),
restore_volumes: bool = Form(True),
restore_binds: bool = Form(True),
session: Session = Depends(get_session),
user: User = Depends(require_admin),
) -> dict:
@@ -448,6 +479,7 @@ async def agent_restore_upload(
"target_id": target_id or "",
"overwrite": str(overwrite).lower(),
"restore_volumes": str(restore_volumes).lower(),
"restore_binds": str(restore_binds).lower(),
},
)
except AgentError as exc:
@@ -485,6 +517,7 @@ async def agent_restore_from(
"target_id": body.target_id or "",
"overwrite": str(body.overwrite).lower(),
"restore_volumes": str(body.restore_volumes).lower(),
"restore_binds": str(body.restore_binds).lower(),
},
)
except AgentError as exc:
backend/routers/backups.py
+61 -7
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import asyncio
import json
import os
import tempfile
@@ -32,12 +33,48 @@ def _ip(request: Request) -> str:
_backup_filename = backup_service.backup_filename
def _compact(report: dict) -> dict:
"""The parts of a backup report worth showing the user."""
return {
"size": report.get("size"),
"binds": report.get("binds", []),
"volumes": report.get("volumes", []),
"skipped": report.get("skipped", []),
"path_mismatch": report.get("path_mismatch"),
}
def _summary(report: dict) -> str:
return (
f"binds={len(report.get('binds', []))} "
f"volumes={len(report.get('volumes', []))} "
f"skipped={len(report.get('skipped', []))}"
)
@router.get("/{stack_id}/backup/inventory")
async def backup_inventory(
stack_id: str,
session: Session = Depends(get_session),
user: User = Depends(require_admin),
) -> dict:
"""What a backup of this stack would capture: bind-mount sources (with size
and whether they are reachable at all), named volumes, and anything that is
skipped by default with the reason why."""
if not session.get(Stack, stack_id):
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
return await asyncio.to_thread(backup_service.plan, stack_id)
@router.get("/{stack_id}/backup")
async def backup_stack(
stack_id: str,
request: Request,
include_volumes: bool = Query(True),
include_binds: bool = Query(True),
stop_first: bool = Query(True),
binds: list[str] | None = Query(None),
volumes: list[str] | None = Query(None),
session: Session = Depends(get_session),
user: User = Depends(require_admin),
):
@@ -45,19 +82,24 @@ async def backup_stack(
if not stack:
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
try:
path = await backup_service.create_backup(
stack_id, stack.name, include_volumes=include_volumes, stop_first=stop_first,
path, report = await backup_service.create_backup_ex(
stack_id, stack.name, include_volumes=include_volumes,
stop_first=stop_first, include_binds=include_binds,
binds=binds, volumes=volumes,
)
except backup_service.BackupError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
audit_service.record(
session, user=user.username, action="stack.backup", target=stack_id,
detail=f"volumes={include_volumes}", ip=_ip(request),
detail=_summary(report), ip=_ip(request),
)
return FileResponse(
path,
media_type="application/gzip",
filename=_backup_filename(stack_id, include_volumes),
# The browser downloads a blob, so the summary of what actually made it
# into the archive rides along in a header.
headers={"X-Stackpilot-Backup": json.dumps(_compact(report))},
)
@@ -68,6 +110,7 @@ async def restore_stack(
target_id: str | None = Form(None),
overwrite: bool = Form(False),
restore_volumes: bool = Form(True),
restore_binds: bool = Form(True),
session: Session = Depends(get_session),
user: User = Depends(require_admin),
) -> dict:
@@ -84,6 +127,7 @@ async def restore_stack(
target_id=target,
overwrite=overwrite,
restore_volumes=restore_volumes,
restore_binds=restore_binds,
)
except backup_service.BackupError as exc:
# 409 for the "already exists" conflict, 400 for malformed backups.
@@ -97,7 +141,8 @@ async def restore_stack(
session.commit()
audit_service.record(
session, user=user.username, action="stack.restore", target=stack_id,
detail=f"volumes={result['volumes_restored']}", ip=_ip(request),
detail=f"volumes={result['volumes_restored']} binds={result['binds_restored']}",
ip=_ip(request),
)
return result
finally:
@@ -113,7 +158,10 @@ async def restore_stack(
class PushBody(BaseModel):
destination_id: int
include_volumes: bool = True
include_binds: bool = True
stop_first: bool = True
binds: list[str] | None = None
volumes: list[str] | None = None
class RestoreFromBody(BaseModel):
@@ -122,6 +170,7 @@ class RestoreFromBody(BaseModel):
target_id: str | None = None
overwrite: bool = False
restore_volumes: bool = True
restore_binds: bool = True
def _get_dest(session: Session, dest_id: int) -> BackupDestination:
@@ -144,9 +193,10 @@ async def push_backup(
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
dest = _get_dest(session, body.destination_id)
try:
path = await backup_service.create_backup(
path, report = await backup_service.create_backup_ex(
stack_id, stack.name,
include_volumes=body.include_volumes, stop_first=body.stop_first,
include_binds=body.include_binds, binds=body.binds, volumes=body.volumes,
)
except backup_service.BackupError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@@ -162,9 +212,12 @@ async def push_backup(
audit_service.record(
session, user=user.username, action="stack.backup.push",
target=stack_id, detail=f"{dest.name}:{filename}", ip=_ip(request),
target=stack_id, detail=f"{dest.name}:{filename} {_summary(report)}", ip=_ip(request),
)
return {"ok": True, "destination": dest.name, "name": filename, "remote": remote}
return {
"ok": True, "destination": dest.name, "name": filename,
"remote": remote, "report": _compact(report),
}
@router.post("/restore-from")
@@ -188,6 +241,7 @@ async def restore_from_destination(
result = backup_service.restore_backup(
tmp.name, target_id=target,
overwrite=body.overwrite, restore_volumes=body.restore_volumes,
restore_binds=body.restore_binds,
)
except backup_service.BackupError as exc:
code = 409 if "already exists" in str(exc) else 400
backend/services/backup_service.py
+328 -140
View File
@@ -1,14 +1,16 @@
"""Stack backup & restore, including named-volume contents.
"""Stack backup & restore — compose files, bind-mount data and named volumes.
A backup is a single ``.tar.gz`` with this layout::
manifest.json metadata + volume/bind inventory
compose/... the full stack directory (compose file, .env, ...)
volumes/<full>.tar raw contents of each compose-managed named volume
manifest.json metadata + full inventory of what was captured
compose/... the stack directory as StackPilot can see it
binds/<n>.tar contents of each captured bind-mount source
volumes/<full>.tar contents of each captured named volume
Named-volume contents are read/written through a throwaway helper container
(``BACKUP_HELPER_IMAGE``) with the volume bind-mounted — this is the portable
way to snapshot a volume regardless of its driver/mountpoint.
Bind sources and volumes are read/written through a throwaway helper container
(see :mod:`services.stack_assets_service`) so that host paths this container
cannot see are still captured — without that, a stack whose data directories
live outside StackPilot's own mount would back up as "just the compose file".
"""
from __future__ import annotations
@@ -22,16 +24,14 @@ import tarfile
import tempfile
from typing import Optional
from config import settings
from docker_client import DockerError, get_client, safe_call
from services import compose_service
from services import compose_service, stack_assets_service as assets
logger = logging.getLogger("stackpilot.backup")
COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
COMPOSE_VOLUME_LABEL = "com.docker.compose.volume"
COMPOSE_PROJECT_LABEL = assets.COMPOSE_PROJECT_LABEL
COMPOSE_VOLUME_LABEL = assets.COMPOSE_VOLUME_LABEL
MANIFEST_NAME = "manifest.json"
BACKUP_FORMAT_VERSION = 1
BACKUP_FORMAT_VERSION = 2
class BackupError(Exception):
@@ -59,98 +59,48 @@ def backup_filename(stack_id: str, include_volumes: bool, prefix: Optional[str]
# --------------------------------------------------------------------------- #
# Helper container for volume I/O
# Selection
# --------------------------------------------------------------------------- #
def _ensure_helper_image(client) -> None:
image = settings.BACKUP_HELPER_IMAGE
try:
safe_call(client.images.get, image)
except DockerError:
logger.info("Pulling backup helper image %s", image)
safe_call(client.images.pull, image)
def _decide(
items: list[dict], key: str, chosen: Optional[list[str]], enabled: bool
) -> list[dict]:
"""Mark each inventory item ``selected`` (with a reason when it is not).
``chosen`` is an explicit list from the caller; without one the inventory's
own defaults apply (everything except system paths, oversized directories
and remote-backed volumes).
"""
for item in items:
if not enabled:
item["selected"], item["reason"] = False, "not requested"
elif chosen is not None:
selected = item[key] in chosen
item["selected"] = selected
item["reason"] = None if selected else "not selected"
else:
item["selected"] = bool(item.get("include_default"))
item["reason"] = None if item["selected"] else (item.get("reason") or "not selected")
# Never archive plumbing, whatever the caller asked for.
if item["selected"] and (item.get("system") or item.get("kind") in ("special", "unknown")):
item["selected"] = False
item["reason"] = item.get("reason") or "not a regular file or directory"
return items
def _export_volume(full_name: str) -> bytes:
client = get_client()
_ensure_helper_image(client)
container = safe_call(
client.containers.create,
settings.BACKUP_HELPER_IMAGE,
command="true",
volumes={full_name: {"bind": "/v", "mode": "ro"}},
)
try:
# "/v/." copies the *contents* of the volume (no leading "v/" prefix),
# so restore can extract straight back into the volume root.
bits, _ = container.get_archive("/v/.")
buf = io.BytesIO()
for chunk in bits:
buf.write(chunk)
return buf.getvalue()
finally:
try:
container.remove(force=True)
except Exception: # noqa: BLE001
pass
def _restore_volume(full_name: str, labels: dict, tar_bytes: bytes) -> None:
client = get_client()
_ensure_helper_image(client)
existed = True
try:
safe_call(client.volumes.get, full_name)
except DockerError:
existed = False
safe_call(client.volumes.create, name=full_name, labels=labels or {})
if existed:
# Restore means "back to the snapshot": clear the current contents so
# files created/kept since the backup don't survive underneath it.
safe_call(
client.containers.run,
settings.BACKUP_HELPER_IMAGE,
["sh", "-c", "find /v -mindepth 1 -delete"],
volumes={full_name: {"bind": "/v", "mode": "rw"}},
remove=True,
)
container = safe_call(
client.containers.create,
settings.BACKUP_HELPER_IMAGE,
command="true",
volumes={full_name: {"bind": "/v", "mode": "rw"}},
)
try:
container.put_archive("/v", tar_bytes)
finally:
try:
container.remove(force=True)
except Exception: # noqa: BLE001
pass
def _compose_volumes(stack_id: str) -> list[dict]:
"""Return [{full, short, labels}] for compose-managed named volumes."""
try:
client = get_client()
vols = safe_call(
client.volumes.list,
filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"},
)
except DockerError:
return []
out = []
for v in vols:
labels = v.attrs.get("Labels") or {}
out.append(
{
"full": v.name,
"short": labels.get(COMPOSE_VOLUME_LABEL, v.name),
"labels": labels,
}
)
return out
def plan(
stack_id: str,
include_volumes: bool = True,
include_binds: bool = True,
binds: Optional[list[str]] = None,
volumes: Optional[list[str]] = None,
) -> dict:
"""Decide what a backup captures. Returned as-is by the inventory endpoint."""
inv = assets.inventory(stack_id)
_decide(inv["binds"], "source", binds, include_binds)
_decide(inv["volumes"], "name", volumes, include_volumes)
return inv
# --------------------------------------------------------------------------- #
@@ -158,23 +108,47 @@ def _compose_volumes(stack_id: str) -> list[dict]:
# --------------------------------------------------------------------------- #
async def create_backup(
def _compose_filter(excluded_prefixes: list[str]):
"""Drop deselected bind directories from the compose/ tree (keep the mount
point itself, so the stack still starts after a restore)."""
def _filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
for prefix in excluded_prefixes:
if info.name.startswith(prefix + "/"):
return None
return info
return _filter
async def create_backup_ex(
stack_id: str,
name: str,
include_volumes: bool = True,
stop_first: bool = True,
) -> str:
"""Create a backup tar.gz and return its path on disk."""
include_binds: bool = True,
binds: Optional[list[str]] = None,
volumes: Optional[list[str]] = None,
) -> tuple[str, dict]:
"""Create a backup tar.gz. Returns (path, report)."""
directory = compose_service.stack_dir(stack_id)
if not os.path.isdir(directory):
raise BackupError("Stack directory missing")
volumes = _compose_volumes(stack_id) if include_volumes else []
selection = await asyncio.to_thread(
plan, stack_id, include_volumes, include_binds, binds, volumes
)
# Selected bind sources this process cannot reach through the filesystem
# need their own archive; the rest already ride along in compose/.
cap_binds = [b for b in selection["binds"] if b["selected"] and b["via"] == "archive"]
cap_volumes = [v for v in selection["volumes"] if v["selected"]]
skipped_binds = [b for b in selection["binds"] if not b["selected"]]
skipped_volumes = [v for v in selection["volumes"] if not v["selected"]]
# For a consistent volume snapshot, stop the stack first — but only if it
# is actually running, so backing up a stopped stack doesn't start it.
# Consistent snapshot: stop the stack first — but only if it is actually
# running, so backing up a stopped stack doesn't start it.
stopped = False
if include_volumes and stop_first and volumes:
if stop_first and (cap_volumes or cap_binds):
if compose_service.compute_status(stack_id) not in ("stopped", "unknown"):
try:
await compose_service.stop(stack_id)
@@ -182,34 +156,122 @@ async def create_backup(
except Exception as exc: # noqa: BLE001
logger.warning("Could not stop %s before backup: %s", stack_id, exc)
workdir = tempfile.mkdtemp(prefix="sp-backup-")
try:
# Archive bind sources and volumes into the work directory first, so a
# failure on one asset is reported instead of corrupting the tar.
for index, bind in enumerate(cap_binds):
bind["archive"] = f"binds/{index:03d}.tar"
part = os.path.join(workdir, f"bind-{index:03d}.tar")
try:
bind["bytes"] = await asyncio.to_thread(
assets.export_path, bind["source"], bind["kind"], part
)
bind["_part"] = part
except Exception as exc: # noqa: BLE001
logger.warning("Could not archive bind %s: %s", bind["source"], exc)
bind["archive"] = None
bind["error"] = str(exc)
for index, vol in enumerate(cap_volumes):
vol["archive"] = f"volumes/{vol['name']}.tar"
part = os.path.join(workdir, f"vol-{index:03d}.tar")
try:
vol["bytes"] = await asyncio.to_thread(assets.export_volume, vol["name"], part)
vol["_part"] = part
except Exception as exc: # noqa: BLE001
logger.warning("Could not archive volume %s: %s", vol["name"], exc)
vol["archive"] = None
vol["error"] = str(exc)
# Deselected data that sits inside the stack folder must not sneak into
# the archive through compose/ (that is how a 200 GB downloads folder
# ends up in a "config only" backup).
excluded = [
"compose/" + os.path.relpath(b["source"], directory)
for b in skipped_binds
if b.get("inside_stack_dir")
]
manifest = {
"format_version": BACKUP_FORMAT_VERSION,
"stack_id": stack_id,
"name": name,
"created_at": compose_service.now().isoformat(),
"include_volumes": include_volumes,
"volumes": [{"full": v["full"], "short": v["short"], "labels": v["labels"]} for v in volumes],
"include_binds": include_binds,
"stack_dir": directory,
"path_mismatch": selection.get("path_mismatch"),
# Volumes keep the v1 shape (full/short/labels) so older StackPilots
# can still read the manifest they care about.
"volumes": [
{
"full": v["name"],
"short": v["short"],
"labels": v["labels"],
"archive": v.get("archive"),
"remote": v.get("remote", False),
"bytes": v.get("bytes"),
"error": v.get("error"),
}
for v in cap_volumes
if v.get("archive")
],
"binds": [
{
"source": b["source"],
"kind": b["kind"],
"mounts": b["mounts"],
"inside_stack_dir": b["inside_stack_dir"],
"archive": b.get("archive"),
"bytes": b.get("bytes"),
"error": b.get("error"),
}
for b in cap_binds
if b.get("archive")
],
"skipped": [
{"kind": "bind", "source": b["source"], "reason": b.get("reason")}
for b in skipped_binds
]
+ [
{"kind": "volume", "name": v["name"], "reason": v.get("reason")}
for v in skipped_volumes
],
}
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
tmp.close()
with tarfile.open(tmp.name, "w:gz") as tar:
# manifest
data = json.dumps(manifest, indent=2).encode("utf-8")
info = tarfile.TarInfo(MANIFEST_NAME)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
# stack directory
tar.add(directory, arcname="compose")
# volume contents
for v in volumes:
vbytes = await asyncio.to_thread(_export_volume, v["full"])
info = tarfile.TarInfo(f"volumes/{v['full']}.tar")
info.size = len(vbytes)
tar.addfile(info, io.BytesIO(vbytes))
return tmp.name
tar.add(directory, arcname="compose", filter=_compose_filter(excluded))
for bind in cap_binds:
if bind.get("_part"):
tar.add(bind["_part"], arcname=bind["archive"])
for vol in cap_volumes:
if vol.get("_part"):
tar.add(vol["_part"], arcname=vol["archive"])
report = {
"file": tmp.name,
"binds": [
{"source": b["source"], "bytes": b.get("bytes"), "error": b.get("error")}
for b in cap_binds
],
"volumes": [
{"name": v["name"], "bytes": v.get("bytes"), "error": v.get("error")}
for v in cap_volumes
],
"skipped": manifest["skipped"],
"path_mismatch": selection.get("path_mismatch"),
"size": os.path.getsize(tmp.name),
}
return tmp.name, report
finally:
shutil.rmtree(workdir, ignore_errors=True)
if stopped:
try:
await compose_service.up(stack_id)
@@ -217,6 +279,21 @@ async def create_backup(
logger.warning("Could not restart %s after backup: %s", stack_id, exc)
async def create_backup(
stack_id: str,
name: str,
include_volumes: bool = True,
stop_first: bool = True,
include_binds: bool = True,
binds: Optional[list[str]] = None,
volumes: Optional[list[str]] = None,
) -> str:
path, _report = await create_backup_ex(
stack_id, name, include_volumes, stop_first, include_binds, binds, volumes
)
return path
# --------------------------------------------------------------------------- #
# Restore
# --------------------------------------------------------------------------- #
@@ -224,33 +301,87 @@ async def create_backup(
def read_manifest(tar_path: str) -> dict:
with tarfile.open(tar_path, "r:gz") as tar:
member = tar.getmember(MANIFEST_NAME)
try:
member = tar.getmember(MANIFEST_NAME)
except KeyError as exc:
raise BackupError("Backup is missing its manifest") from exc
fh = tar.extractfile(member)
if fh is None:
raise BackupError("Backup is missing its manifest")
return json.loads(fh.read().decode("utf-8"))
def _safe_extract_compose(tar: tarfile.TarFile, dest_dir: str) -> None:
"""Extract the ``compose/`` subtree into dest_dir, guarding path traversal."""
def _safe_target(dest_dir: str, rel: str) -> str:
"""Resolve a member path inside dest_dir, refusing traversal *and* writes
through a symlink planted earlier in the same archive."""
root = os.path.abspath(dest_dir)
target = os.path.normpath(os.path.join(root, rel))
if target != root and not target.startswith(root + os.sep):
raise BackupError(f"Refusing unsafe path in backup: {rel}")
parent = os.path.dirname(target)
if os.path.exists(parent):
real_parent = os.path.realpath(parent)
if real_parent != root and not real_parent.startswith(root + os.sep):
raise BackupError(f"Refusing unsafe path in backup: {rel}")
return target
def _apply_meta(path: str, member: tarfile.TarInfo) -> None:
"""Restore mode/ownership/mtime — *arr-style images run as PUID/PGID and
break when their config comes back root-owned with default permissions."""
try:
os.chmod(path, member.mode)
except OSError:
pass
try:
os.chown(path, member.uid, member.gid)
except (OSError, AttributeError):
pass
try:
os.utime(path, (member.mtime, member.mtime))
except OSError:
pass
def _extract_tree(tar: tarfile.TarFile, prefix: str, dest_dir: str) -> None:
"""Extract one subtree of the archive, preserving metadata and symlinks."""
os.makedirs(dest_dir, exist_ok=True)
dirs: list[tuple[str, tarfile.TarInfo]] = []
for member in tar.getmembers():
if not member.name.startswith("compose/"):
if not member.name.startswith(prefix):
continue
rel = member.name[len("compose/") :]
rel = member.name[len(prefix) :].lstrip("/")
if not rel:
continue
target = os.path.normpath(os.path.join(dest_dir, rel))
if not target.startswith(os.path.abspath(dest_dir) + os.sep) and target != os.path.abspath(dest_dir):
raise BackupError(f"Refusing unsafe path in backup: {member.name}")
target = _safe_target(dest_dir, rel)
if member.isdir():
os.makedirs(target, exist_ok=True)
elif member.isreg():
os.makedirs(os.path.dirname(target), exist_ok=True)
src = tar.extractfile(member)
if src is not None:
with open(target, "wb") as out:
shutil.copyfileobj(src, out)
dirs.append((target, member))
continue
os.makedirs(os.path.dirname(target), exist_ok=True)
if member.issym():
if os.path.lexists(target):
os.unlink(target)
os.symlink(member.linkname, target)
continue # chmod/utime would follow the link
if member.islnk():
source = _safe_target(dest_dir, member.linkname[len(prefix) :].lstrip("/"))
if os.path.exists(source):
if os.path.lexists(target):
os.unlink(target)
os.link(source, target)
continue
if not member.isreg():
continue # devices/fifos/sockets are runtime artefacts
src = tar.extractfile(member)
if src is None:
continue
with open(target, "wb") as out:
shutil.copyfileobj(src, out)
_apply_meta(target, member)
# Directory metadata last: a 0500 directory would block writing its files.
for target, member in sorted(dirs, key=lambda d: len(d[0]), reverse=True):
_apply_meta(target, member)
def restore_backup(
@@ -258,8 +389,9 @@ def restore_backup(
target_id: Optional[str] = None,
overwrite: bool = False,
restore_volumes: bool = True,
restore_binds: bool = True,
) -> dict:
"""Restore a backup. Returns {stack_id, name, volumes_restored}."""
"""Restore a backup. Returns a report of what was written."""
manifest = read_manifest(tar_path)
raw_id = target_id or manifest.get("stack_id") or ""
if not raw_id.strip():
@@ -267,21 +399,63 @@ def restore_backup(
# Slugify whichever id we end up using — the manifest comes from an
# uploaded file, so its stack_id must never be able to escape STACKS_DIR.
stack_id = compose_service.slugify(raw_id)
old_id = manifest.get("stack_id") or stack_id
old_dir = manifest.get("stack_dir") or ""
directory = compose_service.stack_dir(stack_id)
exists = os.path.isdir(directory)
if exists and not overwrite:
raise BackupError(f"Stack '{stack_id}' already exists")
volumes_restored = 0
binds_restored = 0
skipped: list[dict] = []
with tarfile.open(tar_path, "r:gz") as tar:
if exists:
shutil.rmtree(directory)
_safe_extract_compose(tar, directory)
_extract_tree(tar, "compose/", directory)
if restore_binds:
for bind in manifest.get("binds", []):
archive = bind.get("archive")
if not archive:
continue
try:
member = tar.getmember(archive)
except KeyError:
continue
source = bind["source"]
# A renamed stack must not write into the old stack's folder.
if old_dir and bind.get("inside_stack_dir"):
rel = os.path.relpath(source, old_dir)
source = os.path.normpath(os.path.join(directory, rel))
if assets.is_system_path(source):
skipped.append({"kind": "bind", "source": source, "reason": "system path"})
continue
fh = tar.extractfile(member)
if fh is None:
continue
with tempfile.NamedTemporaryFile(delete=False, suffix=".tar") as tmp:
shutil.copyfileobj(fh, tmp)
part = tmp.name
try:
assets.import_path(source, bind.get("kind", "dir"), part)
binds_restored += 1
except Exception as exc: # noqa: BLE001
logger.warning("Could not restore bind %s: %s", source, exc)
skipped.append({"kind": "bind", "source": source, "reason": str(exc)})
finally:
os.unlink(part)
elif manifest.get("binds"):
skipped += [
{"kind": "bind", "source": b["source"], "reason": "not requested"}
for b in manifest["binds"]
]
volumes_restored = 0
if restore_volumes:
for v in manifest.get("volumes", []):
member_name = f"volumes/{v['full']}.tar"
member_name = v.get("archive") or f"volumes/{v['full']}.tar"
try:
member = tar.getmember(member_name)
except KeyError:
@@ -293,13 +467,27 @@ def restore_backup(
labels = dict(v.get("labels") or {})
labels[COMPOSE_PROJECT_LABEL] = stack_id
full = v["full"]
if target_id and manifest.get("stack_id") and full.startswith(manifest["stack_id"] + "_"):
full = stack_id + full[len(manifest["stack_id"]):]
_restore_volume(full, labels, fh.read())
volumes_restored += 1
if target_id and old_id and full.startswith(old_id + "_"):
full = stack_id + full[len(old_id) :]
with tempfile.NamedTemporaryFile(delete=False, suffix=".tar") as tmp:
shutil.copyfileobj(fh, tmp)
part = tmp.name
try:
# Remote-backed volumes (NFS/CIFS) are never wiped: that
# would delete the share the volume points at.
assets.import_volume(full, labels, part, wipe=not v.get("remote", False))
volumes_restored += 1
except Exception as exc: # noqa: BLE001
logger.warning("Could not restore volume %s: %s", full, exc)
skipped.append({"kind": "volume", "name": full, "reason": str(exc)})
finally:
os.unlink(part)
skipped += manifest.get("skipped", [])
return {
"stack_id": stack_id,
"name": manifest.get("name", stack_id),
"volumes_restored": volumes_restored,
"binds_restored": binds_restored,
"skipped": skipped,
}
backend/services/stack_assets_service.py
+500
View File
@@ -0,0 +1,500 @@
"""Inventory of everything a stack's data actually lives in.
A stack is more than its compose file: bind-mounted directories (``./config``)
and named volumes hold the real state. StackPilot itself runs in a container, so
it can only *see* what is mounted into it — a bind source like
``/opt/stacks/arr-stack/gluetun`` may exist on the host and still be invisible
here (that happens whenever the stacks directory is mounted under a different
host path than ``STACKS_DIR``, because compose resolves ``./gluetun`` against the
path *inside* this container and the daemon then creates it at that same path on
the **host**).
Everything in this module therefore reads and writes host paths through a
throwaway helper container: the daemon does the mounting, so the data is
reachable regardless of what StackPilot has mounted. That is what makes backups
complete instead of "just the compose file".
"""
from __future__ import annotations
import logging
import os
import re
from typing import Optional
import yaml
from config import settings
from docker_client import DockerError, get_client, safe_call
from services import compose_service
logger = logging.getLogger("stackpilot.assets")
COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
COMPOSE_SERVICE_LABEL = "com.docker.compose.service"
COMPOSE_VOLUME_LABEL = "com.docker.compose.volume"
# Paths that are plumbing, never stack data.
SYSTEM_PATHS = {
"/var/run/docker.sock",
"/run/docker.sock",
"/etc/localtime",
"/etc/timezone",
"/etc/hosts",
"/etc/resolv.conf",
}
SYSTEM_PREFIXES = ("/dev", "/proc", "/sys", "/run", "/var/run", "/var/lib/docker")
# Volume driver_opts types that point at storage which lives somewhere else
# entirely (a NAS). Pulling a media library through a tar.gz is never what the
# user wants, and *restoring* one would overwrite the share.
REMOTE_VOLUME_TYPES = {"nfs", "nfs4", "cifs", "smb", "smb3", "smbfs", "sshfs", "glusterfs"}
# Bind directories larger than this are listed but not selected by default.
DEFAULT_MAX_BIND_BYTES = 2 * 1024**3
_ENV_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::?-([^}]*))?\}|\$([A-Za-z_][A-Za-z0-9_]*)")
class AssetError(Exception):
pass
# --------------------------------------------------------------------------- #
# Helper container primitives (host-path I/O)
# --------------------------------------------------------------------------- #
def _ensure_helper_image(client) -> None:
image = settings.BACKUP_HELPER_IMAGE
try:
safe_call(client.images.get, image)
except DockerError:
logger.info("Pulling helper image %s", image)
safe_call(client.images.pull, image)
def _create_helper(client, volumes: dict):
return safe_call(
client.containers.create,
settings.BACKUP_HELPER_IMAGE,
command="true",
volumes=volumes,
)
def _remove(container) -> None:
try:
container.remove(force=True)
except Exception: # noqa: BLE001 - cleanup is best effort
pass
def _split(path: str) -> tuple[str, str]:
clean = path.rstrip("/") or "/"
return os.path.dirname(clean) or "/", os.path.basename(clean)
def inspect_paths(paths: list[str]) -> dict[str, dict]:
"""Classify host paths ({path: {"kind", "size"}}) via one helper container.
``kind`` is dir / file / special; ``size`` is bytes (best effort, the walk is
capped so a huge media share can't stall the request).
"""
unique = [p for p in dict.fromkeys(paths) if p]
if not unique:
return {}
client = get_client()
_ensure_helper_image(client)
mounts = {p: {"bind": f"/m/{i}", "mode": "ro"} for i, p in enumerate(unique)}
script_parts = []
for i in range(len(unique)):
script_parts.append(
f'd=/m/{i}; '
f'if [ -d "$d" ]; then s=$(timeout 20 du -sk "$d" 2>/dev/null | cut -f1); '
f'echo "{i} dir ${{s:-}}"; '
f'elif [ -f "$d" ]; then echo "{i} file $(stat -c %s "$d" 2>/dev/null)"; '
f'else echo "{i} special"; fi'
)
script = "; ".join(script_parts)
try:
out = safe_call(
client.containers.run,
settings.BACKUP_HELPER_IMAGE,
["sh", "-c", script],
volumes=mounts,
remove=True,
stdout=True,
stderr=False,
)
except DockerError as exc:
logger.warning("Path inspection failed: %s", exc)
return {p: {"kind": "unknown", "size": None} for p in unique}
result: dict[str, dict] = {p: {"kind": "unknown", "size": None} for p in unique}
for line in (out or b"").decode("utf-8", "replace").splitlines():
parts = line.strip().split()
if len(parts) < 2 or not parts[0].isdigit():
continue
idx = int(parts[0])
if idx >= len(unique):
continue
kind = parts[1]
size: Optional[int] = None
if len(parts) > 2 and parts[2].isdigit():
size = int(parts[2]) * 1024 if kind == "dir" else int(parts[2])
result[unique[idx]] = {"kind": kind, "size": size}
return result
def export_path(source: str, kind: str, dest_file: str) -> int:
"""Tar a host path (dir contents, or a single file) into ``dest_file``."""
client = get_client()
_ensure_helper_image(client)
if kind == "file":
parent, base = _split(source)
if not base:
raise AssetError(f"Cannot archive {source}")
container = _create_helper(client, {parent: {"bind": "/src", "mode": "ro"}})
member = f"/src/{base}"
else:
container = _create_helper(client, {source: {"bind": "/src", "mode": "ro"}})
# "/src/." archives the *contents*, so restore can unpack straight back
# into the directory without a stray prefix.
member = "/src/."
written = 0
try:
bits, _ = container.get_archive(member)
with open(dest_file, "wb") as fh:
for chunk in bits:
fh.write(chunk)
written += len(chunk)
finally:
_remove(container)
return written
def import_path(source: str, kind: str, src_file: str) -> None:
"""Unpack an archive produced by :func:`export_path` back to its host path."""
client = get_client()
_ensure_helper_image(client)
if kind == "file":
parent, _base = _split(source)
container = _create_helper(client, {parent: {"bind": "/dst", "mode": "rw"}})
else:
container = _create_helper(client, {source: {"bind": "/dst", "mode": "rw"}})
try:
with open(src_file, "rb") as fh:
container.put_archive("/dst", fh)
finally:
_remove(container)
def export_volume(full_name: str, dest_file: str) -> int:
"""Stream a named volume's contents into ``dest_file`` (never into RAM)."""
client = get_client()
_ensure_helper_image(client)
container = _create_helper(client, {full_name: {"bind": "/v", "mode": "ro"}})
written = 0
try:
bits, _ = container.get_archive("/v/.")
with open(dest_file, "wb") as fh:
for chunk in bits:
fh.write(chunk)
written += len(chunk)
finally:
_remove(container)
return written
def import_volume(full_name: str, labels: dict, src_file: str, wipe: bool = True) -> None:
"""Restore a volume from an archive, optionally clearing it first."""
client = get_client()
_ensure_helper_image(client)
existed = True
try:
safe_call(client.volumes.get, full_name)
except DockerError:
existed = False
safe_call(client.volumes.create, name=full_name, labels=labels or {})
if existed and wipe:
# Restore means "back to the snapshot": drop files created since.
safe_call(
client.containers.run,
settings.BACKUP_HELPER_IMAGE,
["sh", "-c", "find /v -mindepth 1 -delete"],
volumes={full_name: {"bind": "/v", "mode": "rw"}},
remove=True,
)
container = _create_helper(client, {full_name: {"bind": "/v", "mode": "rw"}})
try:
with open(src_file, "rb") as fh:
container.put_archive("/v", fh)
finally:
_remove(container)
# --------------------------------------------------------------------------- #
# Where does STACKS_DIR really live on the host?
# --------------------------------------------------------------------------- #
def host_stacks_dir() -> Optional[str]:
"""Host path backing ``STACKS_DIR`` inside this container, if detectable.
Read from /proc/self/mountinfo (field 4 is the source subtree on the host
filesystem). Returns None when not running in a container / not bind-mounted.
"""
target = settings.STACKS_DIR.rstrip("/") or "/"
try:
with open("/proc/self/mountinfo", "r", encoding="utf-8") as fh:
for line in fh:
parts = line.split()
if len(parts) < 5:
continue
if parts[4].rstrip("/") == target:
return parts[3]
except OSError:
return None
return None
def stacks_path_mismatch() -> Optional[dict]:
"""Report a host/container path mismatch for the stacks directory.
When they differ, compose resolves a stack's relative bind mounts against
the *container* path, so the daemon creates the data directories at that
path on the host — invisible to StackPilot. Backups then only find the
compose file unless bind sources are captured through a helper container.
"""
host = host_stacks_dir()
container = settings.STACKS_DIR.rstrip("/")
if not host or host.rstrip("/") == container:
return None
return {"host": host, "container": container}
# --------------------------------------------------------------------------- #
# Compose / container mount discovery
# --------------------------------------------------------------------------- #
def _env_for_stack(stack_id: str) -> dict:
env: dict[str, str] = {}
path = os.path.join(compose_service.stack_dir(stack_id), ".env")
try:
with open(path, "r", encoding="utf-8", errors="replace") as fh:
for raw in fh:
line = raw.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
env[key.strip()] = value.strip().strip('"').strip("'")
except OSError:
pass
return env
def _interpolate(text: str, env: dict) -> str:
def repl(m: re.Match) -> str:
name = m.group(1) or m.group(3)
default = m.group(2) or ""
return env.get(name, default)
return _ENV_RE.sub(repl, text)
def _bind_specs_from_compose(stack_id: str) -> list[dict]:
"""Bind sources declared in the compose file (used when no containers exist)."""
directory = compose_service.stack_dir(stack_id)
compose_file = compose_service.find_compose_file(directory)
if not compose_file:
return []
try:
with open(compose_file, "r", encoding="utf-8", errors="replace") as fh:
data = yaml.safe_load(fh) or {}
except (OSError, yaml.YAMLError):
return []
env = _env_for_stack(stack_id)
out: list[dict] = []
for service, spec in (data.get("services") or {}).items():
if not isinstance(spec, dict):
continue
for entry in spec.get("volumes") or []:
source = target = None
if isinstance(entry, str):
parts = _interpolate(entry, env).split(":")
if len(parts) >= 2:
source, target = parts[0], parts[1]
elif isinstance(entry, dict):
if entry.get("type") not in (None, "bind"):
continue
source = _interpolate(str(entry.get("source") or ""), env)
target = _interpolate(str(entry.get("target") or ""), env)
if not source or not target:
continue
if not (source.startswith("/") or source.startswith(".") or source.startswith("~")):
continue # named volume
if source.startswith("~"):
continue # home-relative: resolved by the daemon's user, skip
resolved = source if source.startswith("/") else os.path.normpath(
os.path.join(directory, source)
)
out.append({"source": resolved, "service": str(service), "target": target})
return out
def _bind_specs_from_containers(stack_id: str) -> list[dict]:
"""Bind sources as the daemon actually mounted them (authoritative)."""
try:
client = get_client()
containers = safe_call(
client.containers.list,
all=True,
filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"},
)
except DockerError:
return []
out: list[dict] = []
for c in containers:
service = (c.labels or {}).get(COMPOSE_SERVICE_LABEL, c.name)
for mount in c.attrs.get("Mounts") or []:
if mount.get("Type") != "bind" or not mount.get("Source"):
continue
out.append(
{
"source": mount["Source"],
"service": service,
"target": mount.get("Destination") or "",
}
)
return out
def is_system_path(path: str) -> bool:
if path in SYSTEM_PATHS:
return True
return any(path == p or path.startswith(p + "/") for p in SYSTEM_PREFIXES)
def _inside(path: str, parent: str) -> bool:
parent = parent.rstrip("/")
return path == parent or path.startswith(parent + "/")
def compose_volumes(stack_id: str) -> list[dict]:
"""Compose-managed named volumes, with remote-storage detection."""
try:
client = get_client()
vols = safe_call(
client.volumes.list,
filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"},
)
except DockerError:
return []
out = []
for v in vols:
attrs = v.attrs or {}
labels = attrs.get("Labels") or {}
options = attrs.get("Options") or {}
driver = attrs.get("Driver", "local")
vtype = str(options.get("type") or "").lower()
device = str(options.get("device") or "")
remote = (
vtype in REMOTE_VOLUME_TYPES
or driver != "local"
or device.startswith("//")
or device.startswith(":")
)
out.append(
{
"name": v.name,
"short": labels.get(COMPOSE_VOLUME_LABEL, v.name),
"labels": labels,
"driver": driver,
"options": options,
"remote": remote,
"remote_type": vtype or (driver if driver != "local" else None),
}
)
return out
def inventory(stack_id: str, max_bind_bytes: int = DEFAULT_MAX_BIND_BYTES) -> dict:
"""What a backup of this stack would (and would not) capture.
Bind sources are merged from the running containers (authoritative) and the
compose file (covers stacks that were never started), classified through a
helper container so host-only paths are seen too.
"""
directory = compose_service.stack_dir(stack_id)
specs = _bind_specs_from_containers(stack_id) or []
seen = {(s["source"], s["service"], s["target"]) for s in specs}
for spec in _bind_specs_from_compose(stack_id):
if (spec["source"], spec["service"], spec["target"]) not in seen:
specs.append(spec)
grouped: dict[str, dict] = {}
for spec in specs:
entry = grouped.setdefault(spec["source"], {"source": spec["source"], "mounts": []})
mount = {"service": spec["service"], "target": spec["target"]}
if mount not in entry["mounts"]:
entry["mounts"].append(mount)
real_paths = [p for p in grouped if not is_system_path(p)]
stats = inspect_paths(real_paths)
binds = []
for path, entry in sorted(grouped.items()):
system = is_system_path(path)
info = stats.get(path, {"kind": "unknown", "size": None})
kind, size = info["kind"], info["size"]
inside = _inside(path, directory)
# A path inside the stack directory that this process can actually read
# is already covered by the compose/ tree in the archive.
visible = inside and os.path.exists(path)
include = True
reason = None
if system:
include, reason = False, "system path"
elif kind == "special":
include, reason = False, "not a regular file or directory"
elif kind == "unknown":
include, reason = False, "could not inspect path"
elif size is not None and size > max_bind_bytes:
include, reason = False, f"larger than {max_bind_bytes // 1024**3} GiB"
binds.append(
{
"source": path,
"mounts": entry["mounts"],
"kind": kind,
"size": size,
"inside_stack_dir": inside,
# Readable from here and inside the stack folder → the compose/
# tree already carries it, no separate archive needed.
"covered_by_compose": visible,
"via": "compose" if visible else "archive",
"system": system,
"include_default": include,
"reason": reason,
}
)
volumes = []
for vol in compose_volumes(stack_id):
include = not vol["remote"]
volumes.append(
{
**vol,
"include_default": include,
"reason": None if include else f"remote storage ({vol['remote_type']})",
}
)
return {
"stack_id": stack_id,
"stack_dir": directory,
"stack_dir_visible": os.path.isdir(directory),
"path_mismatch": stacks_path_mismatch(),
"binds": binds,
"volumes": volumes,
}
backend/version.py
+1 -1
View File
@@ -1,3 +1,3 @@
"""Single source of truth for the StackPilot release version."""
APP_VERSION = "0.39.0"
APP_VERSION = "0.40.0"
frontend/package.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "stackpilot-frontend",
"private": true,
"version": "0.39.0",
"version": "0.40.0",
"type": "module",
"scripts": {
"dev": "vite",
frontend/src/api/agents.ts
+27 -16
View File
@@ -1,4 +1,6 @@
import api from "./client";
import { backupParams, readReport } from "./backups";
import type { BackupInventory, BackupOptions, RestoreResult } from "./backups";
import type { Agent, StackDetail, StackStats, StackSummary, StackUpdateInfo } from "@/types";
export type RemoteStackSummary = StackSummary & { agent_id: number; agent_name: string };
@@ -54,13 +56,13 @@ export const agentsApi = {
action: (id: number, stackId: string, action: string) =>
api.post(`/api/agents/${id}/stacks/${stackId}/${action}`).then((r) => r.data),
backupDownload: async (
id: number,
stackId: string,
opts: { includeVolumes: boolean; stopFirst: boolean }
) => {
backupInventory: (id: number, stackId: string) =>
api
.get<BackupInventory>(`/api/agents/${id}/stacks/${stackId}/backup/inventory`)
.then((r) => r.data),
backupDownload: async (id: number, stackId: string, opts: BackupOptions) => {
const res = await api.get(`/api/agents/${id}/stacks/${stackId}/backup`, {
params: { include_volumes: opts.includeVolumes, stop_first: opts.stopFirst },
params: backupParams(opts),
responseType: "blob",
});
const cd = res.headers["content-disposition"] as string | undefined;
@@ -73,11 +75,19 @@ export const agentsApi = {
a.click();
a.remove();
URL.revokeObjectURL(url);
return readReport(res.headers);
},
backupPush: (
id: number,
stackId: string,
body: { destination_id: number; include_volumes: boolean; stop_first: boolean }
body: {
destination_id: number;
include_volumes: boolean;
include_binds?: boolean;
stop_first: boolean;
binds?: string[];
volumes?: string[];
}
) =>
api
.post<{ ok: boolean; destination: string; name: string }>(
@@ -88,18 +98,21 @@ export const agentsApi = {
restoreUpload: (
id: number,
file: File,
opts: { targetId?: string; overwrite: boolean; restoreVolumes: boolean }
opts: {
targetId?: string;
overwrite: boolean;
restoreVolumes: boolean;
restoreBinds?: boolean;
}
) => {
const form = new FormData();
form.append("file", file);
if (opts.targetId) form.append("target_id", opts.targetId);
form.append("overwrite", String(opts.overwrite));
form.append("restore_volumes", String(opts.restoreVolumes));
form.append("restore_binds", String(opts.restoreBinds ?? true));
return api
.post<{ stack_id: string; name: string; volumes_restored: number }>(
`/api/agents/${id}/stacks/restore`,
form
)
.post<RestoreResult>(`/api/agents/${id}/stacks/restore`, form)
.then((r) => r.data);
},
restoreFrom: (
@@ -110,12 +123,10 @@ export const agentsApi = {
target_id?: string;
overwrite: boolean;
restore_volumes: boolean;
restore_binds?: boolean;
}
) =>
api
.post<{ stack_id: string; name: string; volumes_restored: number }>(
`/api/agents/${id}/stacks/restore-from`,
body
)
.post<RestoreResult>(`/api/agents/${id}/stacks/restore-from`, body)
.then((r) => r.data),
};
frontend/src/api/backups.ts
+113 -23
View File
@@ -8,6 +8,87 @@ export interface BackupDestination {
created_at: string;
}
/** One bind-mount source or named volume a backup would capture. */
export interface BackupBind {
source: string;
mounts: { service: string; target: string }[];
kind: string;
size: number | null;
inside_stack_dir: boolean;
covered_by_compose: boolean;
via: "compose" | "archive";
system: boolean;
include_default: boolean;
selected: boolean;
reason: string | null;
}
export interface BackupVolume {
name: string;
short: string;
driver: string;
remote: boolean;
remote_type: string | null;
include_default: boolean;
selected: boolean;
reason: string | null;
}
export interface BackupInventory {
stack_id: string;
stack_dir: string;
stack_dir_visible: boolean;
path_mismatch: { host: string; container: string } | null;
binds: BackupBind[];
volumes: BackupVolume[];
}
export interface BackupReport {
size: number | null;
binds: { source: string; bytes: number | null; error: string | null }[];
volumes: { name: string; bytes: number | null; error: string | null }[];
skipped: { kind: string; source?: string; name?: string; reason: string | null }[];
path_mismatch: { host: string; container: string } | null;
}
export interface RestoreResult {
stack_id: string;
name: string;
volumes_restored: number;
binds_restored?: number;
skipped?: { kind: string; source?: string; name?: string; reason: string | null }[];
}
export interface BackupOptions {
includeVolumes: boolean;
stopFirst: boolean;
includeBinds?: boolean;
binds?: string[];
volumes?: string[];
}
/** Query params shared by the local and the agent-proxied backup endpoints. */
export function backupParams(opts: BackupOptions) {
return {
include_volumes: opts.includeVolumes,
include_binds: opts.includeBinds ?? true,
stop_first: opts.stopFirst,
...(opts.binds ? { binds: opts.binds } : {}),
...(opts.volumes ? { volumes: opts.volumes } : {}),
};
}
/** The backup summary the server attaches to the download response. */
export function readReport(headers: unknown): BackupReport | null {
const raw = (headers as Record<string, string> | undefined)?.["x-stackpilot-backup"];
if (!raw) return null;
try {
return JSON.parse(raw) as BackupReport;
} catch {
return null;
}
}
export interface RemoteBackup {
name: string;
size: number;
@@ -26,46 +107,60 @@ function triggerDownload(blob: Blob, filename: string) {
}
export const backupsApi = {
download: async (
stackId: string,
opts: { includeVolumes: boolean; stopFirst: boolean }
) => {
inventory: (stackId: string) =>
api
.get<BackupInventory>(`/api/stacks/${stackId}/backup/inventory`)
.then((r) => r.data),
download: async (stackId: string, opts: BackupOptions) => {
const res = await api.get(`/api/stacks/${stackId}/backup`, {
params: { include_volumes: opts.includeVolumes, stop_first: opts.stopFirst },
params: backupParams(opts),
responseType: "blob",
});
const cd = res.headers["content-disposition"] as string | undefined;
const match = cd?.match(/filename="?([^"]+)"?/);
const name = match?.[1] ?? `backup-${stackId}.tar.gz`;
triggerDownload(res.data as Blob, name);
return readReport(res.headers);
},
restore: async (
file: File,
opts: { targetId?: string; overwrite: boolean; restoreVolumes: boolean }
opts: {
targetId?: string;
overwrite: boolean;
restoreVolumes: boolean;
restoreBinds?: boolean;
}
) => {
const form = new FormData();
form.append("file", file);
if (opts.targetId) form.append("target_id", opts.targetId);
form.append("overwrite", String(opts.overwrite));
form.append("restore_volumes", String(opts.restoreVolumes));
const res = await api.post<{
stack_id: string;
name: string;
volumes_restored: number;
}>("/api/stacks/restore", form);
form.append("restore_binds", String(opts.restoreBinds ?? true));
const res = await api.post<RestoreResult>("/api/stacks/restore", form);
return res.data;
},
push: (
stackId: string,
body: { destination_id: number; include_volumes: boolean; stop_first: boolean }
body: {
destination_id: number;
include_volumes: boolean;
include_binds?: boolean;
stop_first: boolean;
binds?: string[];
volumes?: string[];
}
) =>
api
.post<{ ok: boolean; destination: string; name: string }>(
`/api/stacks/${stackId}/backup/push`,
body
)
.post<{
ok: boolean;
destination: string;
name: string;
report?: BackupReport;
}>(`/api/stacks/${stackId}/backup/push`, body)
.then((r) => r.data),
restoreFrom: (body: {
@@ -74,13 +169,8 @@ export const backupsApi = {
target_id?: string;
overwrite: boolean;
restore_volumes: boolean;
}) =>
api
.post<{ stack_id: string; name: string; volumes_restored: number }>(
"/api/stacks/restore-from",
body
)
.then((r) => r.data),
restore_binds?: boolean;
}) => api.post<RestoreResult>("/api/stacks/restore-from", body).then((r) => r.data),
};
export const destinationsApi = {
frontend/src/components/stacks/BackupRestore.tsx
+199 -22
View File
@@ -1,9 +1,10 @@
import { useState } from "react";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { Archive, Upload } from "lucide-react";
import { AlertTriangle, Archive, Upload } from "lucide-react";
import { toast } from "sonner";
import { Button } from "@/components/ui";
import { backupsApi, destinationsApi } from "@/api/backups";
import type { BackupReport } from "@/api/backups";
import { agentsApi } from "@/api/agents";
import { apiErrorMessage } from "@/api/client";
import { formatBytes } from "@/lib/utils";
@@ -38,11 +39,21 @@ function Checkbox({
);
}
function Modal({ children, onClose }: { children: React.ReactNode; onClose: () => void }) {
function Modal({
children,
onClose,
wide,
}: {
children: React.ReactNode;
onClose: () => void;
wide?: boolean;
}) {
return (
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/50 p-4" onClick={onClose}>
<div
className="w-full max-w-md rounded-xl border border-slate-200 bg-card p-5 shadow-xl dark:border-slate-700 dark:bg-card-dark"
className={`max-h-[85vh] w-full overflow-auto rounded-xl border border-slate-200 bg-card p-5 shadow-xl dark:border-slate-700 dark:bg-card-dark ${
wide ? "max-w-xl" : "max-w-md"
}`}
onClick={(e) => e.stopPropagation()}
>
{children}
@@ -51,6 +62,61 @@ function Modal({ children, onClose }: { children: React.ReactNode; onClose: () =
);
}
function AssetRow({
checked,
onChange,
title,
detail,
size,
badges,
disabled,
}: {
checked: boolean;
onChange: (v: boolean) => void;
title: string;
detail?: string;
size?: number | null;
badges?: { text: string; tone?: "muted" | "warn" }[];
disabled?: boolean;
}) {
return (
<label className="flex items-start gap-2 rounded-lg px-2 py-1.5 hover:bg-slate-50 dark:hover:bg-slate-800/60">
<input
type="checkbox"
checked={checked}
disabled={disabled}
onChange={(e) => onChange(e.target.checked)}
className="mt-1 h-4 w-4 shrink-0 rounded border-slate-300 text-accent disabled:opacity-40"
/>
<span className="min-w-0 flex-1">
<span className="flex items-baseline justify-between gap-2">
<span className="truncate font-mono text-xs text-slate-700 dark:text-slate-200">{title}</span>
<span className="shrink-0 text-[11px] tabular-nums text-slate-400">
{size != null ? formatBytes(size) : ""}
</span>
</span>
{detail && <span className="block truncate text-[11px] text-slate-500">{detail}</span>}
{badges && badges.length > 0 && (
<span className="mt-0.5 flex flex-wrap gap-1">
{badges.map((b) => (
<span
key={b.text}
className={`rounded-chip px-1.5 py-0.5 text-[10px] ${
b.tone === "warn"
? "bg-amber-100 text-amber-700 dark:bg-amber-900/40 dark:text-amber-300"
: "bg-slate-100 text-slate-500 dark:bg-slate-700 dark:text-slate-300"
}`}
>
{b.text}
</span>
))}
</span>
)}
</span>
</label>
);
}
export function BackupButton({
stackId,
agentId,
@@ -59,39 +125,75 @@ export function BackupButton({
agentId?: number;
}) {
const [open, setOpen] = useState(false);
const [includeVolumes, setIncludeVolumes] = useState(true);
const [stopFirst, setStopFirst] = useState(true);
const [target, setTarget] = useState("download"); // "download" | destination id
const [busy, setBusy] = useState(false);
// null = "use the inventory defaults" (until the user touches a checkbox).
const [pickedBinds, setPickedBinds] = useState<string[] | null>(null);
const [pickedVolumes, setPickedVolumes] = useState<string[] | null>(null);
const destinations = useQuery({
queryKey: ["destinations"],
queryFn: destinationsApi.list,
enabled: open,
});
const inventory = useQuery({
queryKey: ["backup-inventory", agentId ?? "local", stackId],
queryFn: () =>
agentId != null
? agentsApi.backupInventory(agentId, stackId)
: backupsApi.inventory(stackId),
enabled: open,
});
const binds = inventory.data?.binds ?? [];
const volumes = inventory.data?.volumes ?? [];
const bindSel = pickedBinds ?? binds.filter((b) => b.include_default).map((b) => b.source);
const volSel = pickedVolumes ?? volumes.filter((v) => v.include_default).map((v) => v.name);
const toggle = (list: string[], value: string, on: boolean) =>
on ? [...list, value] : list.filter((v) => v !== value);
const describe = (report: BackupReport | null | undefined) => {
if (!report) return "Backup created";
const parts = [`${report.binds.length} folder(s)`, `${report.volumes.length} volume(s)`];
if (report.size) parts.push(formatBytes(report.size));
const skipped = report.skipped.filter((s) => s.reason !== "not requested").length;
return `Backup: ${parts.join(" · ")}${skipped ? ` — ${skipped} skipped` : ""}`;
};
const run = async () => {
setBusy(true);
const tid = toast.loading("Creating backup…");
try {
const opts = {
includeVolumes: volSel.length > 0,
includeBinds: bindSel.length > 0,
stopFirst,
// Empty lists would be dropped from the query string and read as
// "use defaults", so the include_* flags carry that case.
binds: bindSel.length ? bindSel : undefined,
volumes: volSel.length ? volSel : undefined,
};
if (target === "download") {
if (agentId != null) {
await agentsApi.backupDownload(agentId, stackId, { includeVolumes, stopFirst });
} else {
await backupsApi.download(stackId, { includeVolumes, stopFirst });
}
toast.success("Backup downloaded", { id: tid });
const report =
agentId != null
? await agentsApi.backupDownload(agentId, stackId, opts)
: await backupsApi.download(stackId, opts);
toast.success(describe(report), { id: tid });
} else {
const body = {
destination_id: Number(target),
include_volumes: includeVolumes,
include_volumes: opts.includeVolumes,
include_binds: opts.includeBinds,
stop_first: stopFirst,
binds: opts.binds,
volumes: opts.volumes,
};
const res =
agentId != null
? await agentsApi.backupPush(agentId, stackId, body)
: await backupsApi.push(stackId, body);
toast.success(`Backup pushed to ${res.destination}`, { id: tid });
toast.success(`Pushed to ${res.destination}`, { id: tid });
}
setOpen(false);
} catch (e) {
@@ -107,9 +209,75 @@ export function BackupButton({
<Archive className="h-4 w-4" /> Backup
</Button>
{open && (
<Modal onClose={() => !busy && setOpen(false)}>
<Modal wide onClose={() => !busy && setOpen(false)}>
<h2 className="mb-3 sp-heading text-lg">Back up “{stackId}”</h2>
<div className="space-y-3">
{inventory.data?.path_mismatch && (
<div className="flex gap-2 rounded-lg bg-amber-50 p-3 text-xs text-amber-800 dark:bg-amber-900/30 dark:text-amber-200">
<AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" />
<span>
This stack's folder is <code>{inventory.data.path_mismatch.container}</code> inside
StackPilot but <code>{inventory.data.path_mismatch.host}</code> on the host, so the
containers' data directories are not visible here. They are read through a helper
container and are listed below.
</span>
</div>
)}
<div>
<div className="mb-1 text-xs font-medium text-slate-500">Contents</div>
<div className="rounded-lg border border-slate-200 p-1 dark:border-slate-700">
<AssetRow
checked
disabled
onChange={() => {}}
title={inventory.data?.stack_dir ?? "stack folder"}
detail="Compose file, .env and everything else in the stack folder"
badges={[{ text: "always included" }]}
/>
{inventory.isLoading && (
<div className="px-2 py-2 text-xs text-slate-500">Scanning stack data…</div>
)}
{binds.map((b) => (
<AssetRow
key={b.source}
checked={bindSel.includes(b.source)}
disabled={b.system || b.kind === "special"}
onChange={(v) => setPickedBinds(toggle(bindSel, b.source, v))}
title={b.source}
detail={b.mounts.map((m) => `${m.service}:${m.target}`).join(", ")}
size={b.size}
badges={[
...(b.via === "compose" ? [{ text: "in stack folder" as const }] : []),
...(b.reason ? [{ text: b.reason, tone: "warn" as const }] : []),
]}
/>
))}
{volumes.map((v) => (
<AssetRow
key={v.name}
checked={volSel.includes(v.name)}
onChange={(on) => setPickedVolumes(toggle(volSel, v.name, on))}
title={v.name}
detail={`named volume (${v.driver})`}
badges={[
...(v.remote ? [{ text: v.remote_type || "remote", tone: "warn" as const }] : []),
...(v.reason && !v.remote ? [{ text: v.reason, tone: "warn" as const }] : []),
]}
/>
))}
{!inventory.isLoading && binds.length === 0 && volumes.length === 0 && (
<div className="px-2 py-2 text-xs text-slate-500">
No bind mounts or named volumes found for this stack.
</div>
)}
</div>
<p className="mt-1 text-[11px] text-slate-500">
Remote storage (NFS/CIFS) is unchecked by default — it lives on your NAS and would be
overwritten on restore.
</p>
</div>
<label className="block space-y-1">
<span className="text-xs font-medium text-slate-500">Destination</span>
<select className={selectClass} value={target} onChange={(e) => setTarget(e.target.value)}>
@@ -121,17 +289,11 @@ export function BackupButton({
))}
</select>
</label>
<Checkbox
checked={includeVolumes}
onChange={setIncludeVolumes}
label="Include named volume data"
hint="Snapshots each compose-managed volume into the archive."
/>
<Checkbox
checked={stopFirst}
onChange={setStopFirst}
label="Stop the stack during backup"
hint="Recommended for a consistent volume snapshot; the stack is restarted afterwards."
hint="Recommended for a consistent snapshot; the stack is restarted afterwards."
/>
</div>
<div className="mt-4 flex justify-end gap-2">
@@ -158,6 +320,7 @@ export function RestoreButton({ agentId }: { agentId?: number }) {
const [targetId, setTargetId] = useState("");
const [overwrite, setOverwrite] = useState(false);
const [restoreVolumes, setRestoreVolumes] = useState(true);
const [restoreBinds, setRestoreBinds] = useState(true);
const [busy, setBusy] = useState(false);
const destinations = useQuery({
@@ -182,7 +345,12 @@ export function RestoreButton({ agentId }: { agentId?: number }) {
setBusy(false);
return;
}
const opts = { targetId: targetId.trim() || undefined, overwrite, restoreVolumes };
const opts = {
targetId: targetId.trim() || undefined,
overwrite,
restoreVolumes,
restoreBinds,
};
res =
agentId != null
? await agentsApi.restoreUpload(agentId, file, opts)
@@ -199,13 +367,16 @@ export function RestoreButton({ agentId }: { agentId?: number }) {
target_id: targetId.trim() || undefined,
overwrite,
restore_volumes: restoreVolumes,
restore_binds: restoreBinds,
};
res =
agentId != null
? await agentsApi.restoreFrom(agentId, body)
: await backupsApi.restoreFrom(body);
}
toast.success(`Restored '${res.stack_id}' (${res.volumes_restored} volume(s))`, { id: tid });
const bits = [`${res.volumes_restored} volume(s)`];
if (res.binds_restored) bits.push(`${res.binds_restored} folder(s)`);
toast.success(`Restored '${res.stack_id}' — ${bits.join(", ")}`, { id: tid });
qc.invalidateQueries({ queryKey: agentId != null ? ["agent-stacks", agentId] : ["stacks"] });
setOpen(false);
setFile(null);
@@ -305,6 +476,12 @@ export function RestoreButton({ agentId }: { agentId?: number }) {
/>
</label>
<Checkbox checked={restoreVolumes} onChange={setRestoreVolumes} label="Restore volume data" />
<Checkbox
checked={restoreBinds}
onChange={setRestoreBinds}
label="Restore bind-mounted folders"
hint="Writes the captured config folders back to their host paths."
/>
<Checkbox
checked={overwrite}
onChange={setOverwrite}