menzeljandClaude Opus 5
adfd77a983
Uploading a backup extracted the tar straight into the NFS mount via
put_archive, and the daemon chowns every entry while extracting — an export
with root_squash refuses that ("failed to Lchown ... for UID 0, GID 0:
operation not permitted"), so the upload died with a docker 500 even though
plain writes to the share work (which is why the destination test passed).
The helper container now unpacks into its own filesystem and copies the file
into the mount with cat, which never chowns. Restores hit the same wall when a
volume or bind folder lives on a squashed mount, so import_path/import_volume
fall back to a copy-through-staging when (and only when) the failure is a chown
denial — local restores keep preserving ownership. NFS file names are validated
against the same safe charset as the subdir parts, since both are interpolated
into the helper's shell commands.
Verified against a real root_squash NFS export: test/upload/list/download/delete
round trip, byte-identical download, restore into an NFS-backed volume via the
fallback, and ownership still preserved (1000:1000, 0600) on a local volume.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
549 lines
19 KiB
Python
549 lines
19 KiB
Python
"""Inventory of everything a stack's data actually lives in.
|
|
|
|
A stack is more than its compose file: bind-mounted directories (``./config``)
|
|
and named volumes hold the real state. StackPilot itself runs in a container, so
|
|
it can only *see* what is mounted into it — a bind source like
|
|
``/opt/stacks/arr-stack/gluetun`` may exist on the host and still be invisible
|
|
here (that happens whenever the stacks directory is mounted under a different
|
|
host path than ``STACKS_DIR``, because compose resolves ``./gluetun`` against the
|
|
path *inside* this container and the daemon then creates it at that same path on
|
|
the **host**).
|
|
|
|
Everything in this module therefore reads and writes host paths through a
|
|
throwaway helper container: the daemon does the mounting, so the data is
|
|
reachable regardless of what StackPilot has mounted. That is what makes backups
|
|
complete instead of "just the compose file".
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
from typing import Optional
|
|
|
|
import yaml
|
|
|
|
from config import settings
|
|
from docker_client import DockerError, get_client, safe_call
|
|
from services import compose_service
|
|
|
|
logger = logging.getLogger("stackpilot.assets")
|
|
|
|
COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
|
|
COMPOSE_SERVICE_LABEL = "com.docker.compose.service"
|
|
COMPOSE_VOLUME_LABEL = "com.docker.compose.volume"
|
|
|
|
# Paths that are plumbing, never stack data.
|
|
SYSTEM_PATHS = {
|
|
"/var/run/docker.sock",
|
|
"/run/docker.sock",
|
|
"/etc/localtime",
|
|
"/etc/timezone",
|
|
"/etc/hosts",
|
|
"/etc/resolv.conf",
|
|
}
|
|
SYSTEM_PREFIXES = ("/dev", "/proc", "/sys", "/run", "/var/run", "/var/lib/docker")
|
|
|
|
# Volume driver_opts types that point at storage which lives somewhere else
|
|
# entirely (a NAS). Pulling a media library through a tar.gz is never what the
|
|
# user wants, and *restoring* one would overwrite the share.
|
|
REMOTE_VOLUME_TYPES = {"nfs", "nfs4", "cifs", "smb", "smb3", "smbfs", "sshfs", "glusterfs"}
|
|
|
|
# Bind directories larger than this are listed but not selected by default.
|
|
DEFAULT_MAX_BIND_BYTES = 2 * 1024**3
|
|
|
|
_ENV_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::?-([^}]*))?\}|\$([A-Za-z_][A-Za-z0-9_]*)")
|
|
|
|
|
|
class AssetError(Exception):
|
|
pass
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Helper container primitives (host-path I/O)
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def ensure_helper_image(client) -> None:
|
|
image = settings.BACKUP_HELPER_IMAGE
|
|
try:
|
|
safe_call(client.images.get, image)
|
|
except DockerError:
|
|
logger.info("Pulling helper image %s", image)
|
|
safe_call(client.images.pull, image)
|
|
|
|
|
|
def _create_helper(client, volumes: dict):
|
|
return safe_call(
|
|
client.containers.create,
|
|
settings.BACKUP_HELPER_IMAGE,
|
|
command="true",
|
|
volumes=volumes,
|
|
)
|
|
|
|
|
|
def _remove(container) -> None:
|
|
try:
|
|
container.remove(force=True)
|
|
except Exception: # noqa: BLE001 - cleanup is best effort
|
|
pass
|
|
|
|
|
|
def _split(path: str) -> tuple[str, str]:
|
|
clean = path.rstrip("/") or "/"
|
|
return os.path.dirname(clean) or "/", os.path.basename(clean)
|
|
|
|
|
|
def _is_chown_error(exc: Exception) -> bool:
|
|
text = str(exc)
|
|
return "chown" in text.lower() and "not permitted" in text.lower()
|
|
|
|
|
|
def _put_archive(container, dest: str, src_file: str, volumes: dict) -> None:
|
|
"""Unpack an archive into a container path, coping with squashed mounts.
|
|
|
|
The daemon restores ownership while extracting, which an NFS/CIFS export
|
|
with ``root_squash`` refuses. In that case the archive is unpacked into a
|
|
throwaway container's own filesystem first and the files are then copied
|
|
across — ownership cannot be preserved there, but the restore completes
|
|
instead of failing outright.
|
|
"""
|
|
with open(src_file, "rb") as fh:
|
|
try:
|
|
container.put_archive(dest, fh)
|
|
return
|
|
except Exception as exc: # noqa: BLE001
|
|
if not _is_chown_error(exc):
|
|
raise
|
|
logger.warning("%s rejects ownership changes; restoring without it", dest)
|
|
client = get_client()
|
|
staging = _create_helper_with(
|
|
client, volumes, ["sh", "-c", f"cp -R /tmp/. '{dest}/'"]
|
|
)
|
|
try:
|
|
with open(src_file, "rb") as fh:
|
|
staging.put_archive("/tmp", fh)
|
|
staging.start()
|
|
status = staging.wait(timeout=3600).get("StatusCode", 1)
|
|
if status != 0:
|
|
err = (staging.logs(stdout=True, stderr=True) or b"").decode("utf-8", "replace")
|
|
raise AssetError(err.strip() or f"copy failed (exit {status})")
|
|
finally:
|
|
_remove(staging)
|
|
|
|
|
|
def _create_helper_with(client, volumes: dict, command: list[str]):
|
|
ensure_helper_image(client)
|
|
return safe_call(
|
|
client.containers.create,
|
|
settings.BACKUP_HELPER_IMAGE,
|
|
command=command,
|
|
volumes=volumes,
|
|
)
|
|
|
|
|
|
def inspect_paths(paths: list[str]) -> dict[str, dict]:
|
|
"""Classify host paths ({path: {"kind", "size"}}) via one helper container.
|
|
|
|
``kind`` is dir / file / special; ``size`` is bytes (best effort, the walk is
|
|
capped so a huge media share can't stall the request).
|
|
"""
|
|
unique = [p for p in dict.fromkeys(paths) if p]
|
|
if not unique:
|
|
return {}
|
|
client = get_client()
|
|
ensure_helper_image(client)
|
|
mounts = {p: {"bind": f"/m/{i}", "mode": "ro"} for i, p in enumerate(unique)}
|
|
script_parts = []
|
|
for i in range(len(unique)):
|
|
script_parts.append(
|
|
f'd=/m/{i}; '
|
|
f'if [ -d "$d" ]; then s=$(timeout 20 du -sk "$d" 2>/dev/null | cut -f1); '
|
|
f'echo "{i} dir ${{s:-}}"; '
|
|
f'elif [ -f "$d" ]; then echo "{i} file $(stat -c %s "$d" 2>/dev/null)"; '
|
|
f'else echo "{i} special"; fi'
|
|
)
|
|
script = "; ".join(script_parts)
|
|
try:
|
|
out = safe_call(
|
|
client.containers.run,
|
|
settings.BACKUP_HELPER_IMAGE,
|
|
["sh", "-c", script],
|
|
volumes=mounts,
|
|
remove=True,
|
|
stdout=True,
|
|
stderr=False,
|
|
)
|
|
except DockerError as exc:
|
|
logger.warning("Path inspection failed: %s", exc)
|
|
return {p: {"kind": "unknown", "size": None} for p in unique}
|
|
|
|
result: dict[str, dict] = {p: {"kind": "unknown", "size": None} for p in unique}
|
|
for line in (out or b"").decode("utf-8", "replace").splitlines():
|
|
parts = line.strip().split()
|
|
if len(parts) < 2 or not parts[0].isdigit():
|
|
continue
|
|
idx = int(parts[0])
|
|
if idx >= len(unique):
|
|
continue
|
|
kind = parts[1]
|
|
size: Optional[int] = None
|
|
if len(parts) > 2 and parts[2].isdigit():
|
|
size = int(parts[2]) * 1024 if kind == "dir" else int(parts[2])
|
|
result[unique[idx]] = {"kind": kind, "size": size}
|
|
return result
|
|
|
|
|
|
def export_path(source: str, kind: str, dest_file: str) -> int:
|
|
"""Tar a host path (dir contents, or a single file) into ``dest_file``."""
|
|
client = get_client()
|
|
ensure_helper_image(client)
|
|
if kind == "file":
|
|
parent, base = _split(source)
|
|
if not base:
|
|
raise AssetError(f"Cannot archive {source}")
|
|
container = _create_helper(client, {parent: {"bind": "/src", "mode": "ro"}})
|
|
member = f"/src/{base}"
|
|
else:
|
|
container = _create_helper(client, {source: {"bind": "/src", "mode": "ro"}})
|
|
# "/src/." archives the *contents*, so restore can unpack straight back
|
|
# into the directory without a stray prefix.
|
|
member = "/src/."
|
|
written = 0
|
|
try:
|
|
bits, _ = container.get_archive(member)
|
|
with open(dest_file, "wb") as fh:
|
|
for chunk in bits:
|
|
fh.write(chunk)
|
|
written += len(chunk)
|
|
finally:
|
|
_remove(container)
|
|
return written
|
|
|
|
|
|
def import_path(source: str, kind: str, src_file: str) -> None:
|
|
"""Unpack an archive produced by :func:`export_path` back to its host path."""
|
|
client = get_client()
|
|
ensure_helper_image(client)
|
|
if kind == "file":
|
|
parent, _base = _split(source)
|
|
mounts = {parent: {"bind": "/dst", "mode": "rw"}}
|
|
else:
|
|
mounts = {source: {"bind": "/dst", "mode": "rw"}}
|
|
container = _create_helper(client, mounts)
|
|
try:
|
|
_put_archive(container, "/dst", src_file, mounts)
|
|
finally:
|
|
_remove(container)
|
|
|
|
|
|
def export_volume(full_name: str, dest_file: str) -> int:
|
|
"""Stream a named volume's contents into ``dest_file`` (never into RAM)."""
|
|
client = get_client()
|
|
ensure_helper_image(client)
|
|
container = _create_helper(client, {full_name: {"bind": "/v", "mode": "ro"}})
|
|
written = 0
|
|
try:
|
|
bits, _ = container.get_archive("/v/.")
|
|
with open(dest_file, "wb") as fh:
|
|
for chunk in bits:
|
|
fh.write(chunk)
|
|
written += len(chunk)
|
|
finally:
|
|
_remove(container)
|
|
return written
|
|
|
|
|
|
def import_volume(full_name: str, labels: dict, src_file: str, wipe: bool = True) -> None:
|
|
"""Restore a volume from an archive, optionally clearing it first."""
|
|
client = get_client()
|
|
ensure_helper_image(client)
|
|
existed = True
|
|
try:
|
|
safe_call(client.volumes.get, full_name)
|
|
except DockerError:
|
|
existed = False
|
|
safe_call(client.volumes.create, name=full_name, labels=labels or {})
|
|
if existed and wipe:
|
|
# Restore means "back to the snapshot": drop files created since.
|
|
safe_call(
|
|
client.containers.run,
|
|
settings.BACKUP_HELPER_IMAGE,
|
|
["sh", "-c", "find /v -mindepth 1 -delete"],
|
|
volumes={full_name: {"bind": "/v", "mode": "rw"}},
|
|
remove=True,
|
|
)
|
|
mounts = {full_name: {"bind": "/v", "mode": "rw"}}
|
|
container = _create_helper(client, mounts)
|
|
try:
|
|
_put_archive(container, "/v", src_file, mounts)
|
|
finally:
|
|
_remove(container)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Where does STACKS_DIR really live on the host?
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def host_stacks_dir() -> Optional[str]:
|
|
"""Host path backing ``STACKS_DIR`` inside this container, if detectable.
|
|
|
|
Read from /proc/self/mountinfo (field 4 is the source subtree on the host
|
|
filesystem). Returns None when not running in a container / not bind-mounted.
|
|
"""
|
|
target = settings.STACKS_DIR.rstrip("/") or "/"
|
|
try:
|
|
with open("/proc/self/mountinfo", "r", encoding="utf-8") as fh:
|
|
for line in fh:
|
|
parts = line.split()
|
|
if len(parts) < 5:
|
|
continue
|
|
if parts[4].rstrip("/") == target:
|
|
return parts[3]
|
|
except OSError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def stacks_path_mismatch() -> Optional[dict]:
|
|
"""Report a host/container path mismatch for the stacks directory.
|
|
|
|
When they differ, compose resolves a stack's relative bind mounts against
|
|
the *container* path, so the daemon creates the data directories at that
|
|
path on the host — invisible to StackPilot. Backups then only find the
|
|
compose file unless bind sources are captured through a helper container.
|
|
"""
|
|
host = host_stacks_dir()
|
|
container = settings.STACKS_DIR.rstrip("/")
|
|
if not host or host.rstrip("/") == container:
|
|
return None
|
|
return {"host": host, "container": container}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Compose / container mount discovery
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _env_for_stack(stack_id: str) -> dict:
|
|
env: dict[str, str] = {}
|
|
path = os.path.join(compose_service.stack_dir(stack_id), ".env")
|
|
try:
|
|
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
|
for raw in fh:
|
|
line = raw.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
env[key.strip()] = value.strip().strip('"').strip("'")
|
|
except OSError:
|
|
pass
|
|
return env
|
|
|
|
|
|
def _interpolate(text: str, env: dict) -> str:
|
|
def repl(m: re.Match) -> str:
|
|
name = m.group(1) or m.group(3)
|
|
default = m.group(2) or ""
|
|
return env.get(name, default)
|
|
|
|
return _ENV_RE.sub(repl, text)
|
|
|
|
|
|
def _bind_specs_from_compose(stack_id: str) -> list[dict]:
|
|
"""Bind sources declared in the compose file (used when no containers exist)."""
|
|
directory = compose_service.stack_dir(stack_id)
|
|
compose_file = compose_service.find_compose_file(directory)
|
|
if not compose_file:
|
|
return []
|
|
try:
|
|
with open(compose_file, "r", encoding="utf-8", errors="replace") as fh:
|
|
data = yaml.safe_load(fh) or {}
|
|
except (OSError, yaml.YAMLError):
|
|
return []
|
|
env = _env_for_stack(stack_id)
|
|
out: list[dict] = []
|
|
for service, spec in (data.get("services") or {}).items():
|
|
if not isinstance(spec, dict):
|
|
continue
|
|
for entry in spec.get("volumes") or []:
|
|
source = target = None
|
|
if isinstance(entry, str):
|
|
parts = _interpolate(entry, env).split(":")
|
|
if len(parts) >= 2:
|
|
source, target = parts[0], parts[1]
|
|
elif isinstance(entry, dict):
|
|
if entry.get("type") not in (None, "bind"):
|
|
continue
|
|
source = _interpolate(str(entry.get("source") or ""), env)
|
|
target = _interpolate(str(entry.get("target") or ""), env)
|
|
if not source or not target:
|
|
continue
|
|
if not (source.startswith("/") or source.startswith(".") or source.startswith("~")):
|
|
continue # named volume
|
|
if source.startswith("~"):
|
|
continue # home-relative: resolved by the daemon's user, skip
|
|
resolved = source if source.startswith("/") else os.path.normpath(
|
|
os.path.join(directory, source)
|
|
)
|
|
out.append({"source": resolved, "service": str(service), "target": target})
|
|
return out
|
|
|
|
|
|
def _bind_specs_from_containers(stack_id: str) -> list[dict]:
|
|
"""Bind sources as the daemon actually mounted them (authoritative)."""
|
|
try:
|
|
client = get_client()
|
|
containers = safe_call(
|
|
client.containers.list,
|
|
all=True,
|
|
filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"},
|
|
)
|
|
except DockerError:
|
|
return []
|
|
out: list[dict] = []
|
|
for c in containers:
|
|
service = (c.labels or {}).get(COMPOSE_SERVICE_LABEL, c.name)
|
|
for mount in c.attrs.get("Mounts") or []:
|
|
if mount.get("Type") != "bind" or not mount.get("Source"):
|
|
continue
|
|
out.append(
|
|
{
|
|
"source": mount["Source"],
|
|
"service": service,
|
|
"target": mount.get("Destination") or "",
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def is_system_path(path: str) -> bool:
|
|
if path in SYSTEM_PATHS:
|
|
return True
|
|
return any(path == p or path.startswith(p + "/") for p in SYSTEM_PREFIXES)
|
|
|
|
|
|
def _inside(path: str, parent: str) -> bool:
|
|
parent = parent.rstrip("/")
|
|
return path == parent or path.startswith(parent + "/")
|
|
|
|
|
|
def compose_volumes(stack_id: str) -> list[dict]:
|
|
"""Compose-managed named volumes, with remote-storage detection."""
|
|
try:
|
|
client = get_client()
|
|
vols = safe_call(
|
|
client.volumes.list,
|
|
filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"},
|
|
)
|
|
except DockerError:
|
|
return []
|
|
out = []
|
|
for v in vols:
|
|
attrs = v.attrs or {}
|
|
labels = attrs.get("Labels") or {}
|
|
options = attrs.get("Options") or {}
|
|
driver = attrs.get("Driver", "local")
|
|
vtype = str(options.get("type") or "").lower()
|
|
device = str(options.get("device") or "")
|
|
remote = (
|
|
vtype in REMOTE_VOLUME_TYPES
|
|
or driver != "local"
|
|
or device.startswith("//")
|
|
or device.startswith(":")
|
|
)
|
|
out.append(
|
|
{
|
|
"name": v.name,
|
|
"short": labels.get(COMPOSE_VOLUME_LABEL, v.name),
|
|
"labels": labels,
|
|
"driver": driver,
|
|
"options": options,
|
|
"remote": remote,
|
|
"remote_type": vtype or (driver if driver != "local" else None),
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def inventory(stack_id: str, max_bind_bytes: int = DEFAULT_MAX_BIND_BYTES) -> dict:
|
|
"""What a backup of this stack would (and would not) capture.
|
|
|
|
Bind sources are merged from the running containers (authoritative) and the
|
|
compose file (covers stacks that were never started), classified through a
|
|
helper container so host-only paths are seen too.
|
|
"""
|
|
directory = compose_service.stack_dir(stack_id)
|
|
specs = _bind_specs_from_containers(stack_id) or []
|
|
seen = {(s["source"], s["service"], s["target"]) for s in specs}
|
|
for spec in _bind_specs_from_compose(stack_id):
|
|
if (spec["source"], spec["service"], spec["target"]) not in seen:
|
|
specs.append(spec)
|
|
|
|
grouped: dict[str, dict] = {}
|
|
for spec in specs:
|
|
entry = grouped.setdefault(spec["source"], {"source": spec["source"], "mounts": []})
|
|
mount = {"service": spec["service"], "target": spec["target"]}
|
|
if mount not in entry["mounts"]:
|
|
entry["mounts"].append(mount)
|
|
|
|
real_paths = [p for p in grouped if not is_system_path(p)]
|
|
stats = inspect_paths(real_paths)
|
|
|
|
binds = []
|
|
for path, entry in sorted(grouped.items()):
|
|
system = is_system_path(path)
|
|
info = stats.get(path, {"kind": "unknown", "size": None})
|
|
kind, size = info["kind"], info["size"]
|
|
inside = _inside(path, directory)
|
|
# A path inside the stack directory that this process can actually read
|
|
# is already covered by the compose/ tree in the archive.
|
|
visible = inside and os.path.exists(path)
|
|
include = True
|
|
reason = None
|
|
if system:
|
|
include, reason = False, "system path"
|
|
elif kind == "special":
|
|
include, reason = False, "not a regular file or directory"
|
|
elif kind == "unknown":
|
|
include, reason = False, "could not inspect path"
|
|
elif size is not None and size > max_bind_bytes:
|
|
include, reason = False, f"larger than {max_bind_bytes // 1024**3} GiB"
|
|
binds.append(
|
|
{
|
|
"source": path,
|
|
"mounts": entry["mounts"],
|
|
"kind": kind,
|
|
"size": size,
|
|
"inside_stack_dir": inside,
|
|
# Readable from here and inside the stack folder → the compose/
|
|
# tree already carries it, no separate archive needed.
|
|
"covered_by_compose": visible,
|
|
"via": "compose" if visible else "archive",
|
|
"system": system,
|
|
"include_default": include,
|
|
"reason": reason,
|
|
}
|
|
)
|
|
|
|
volumes = []
|
|
for vol in compose_volumes(stack_id):
|
|
include = not vol["remote"]
|
|
volumes.append(
|
|
{
|
|
**vol,
|
|
"include_default": include,
|
|
"reason": None if include else f"remote storage ({vol['remote_type']})",
|
|
}
|
|
)
|
|
|
|
return {
|
|
"stack_id": stack_id,
|
|
"stack_dir": directory,
|
|
"stack_dir_visible": os.path.isdir(directory),
|
|
"path_mismatch": stacks_path_mismatch(),
|
|
"binds": binds,
|
|
"volumes": volumes,
|
|
}
|