0.33.0: NFS share as backup destination
New destination type 'nfs' alongside SFTP and S3. The StackPilot container needs no mount privileges: the Docker daemon mounts the export as a named volume (stackpilot-nfs-dest-<id>, driver local/type nfs, recreated whenever server/path/options change) and all file I/O runs through throwaway helper containers (BACKUP_HELPER_IMAGE) — upload via put_archive, list via stat, download via get_archive, delete/test via short-lived runs. Config: server, export path, mount options (default rw), optional subdirectory (sanitized; shell-safe charset). Mount failures surface as clean destination errors. Settings UI gains the NFS form + summary; works everywhere destinations are used (push, restore-from, scheduled backups incl. retention). Verified live against a real kernel NFS server: test, push (file on the export), list, restore-from incl. volume data, remote delete, config change recreates the mount volume, unreachable server fails cleanly. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
79d82361d8
commit
786c346c40
@@ -108,9 +108,12 @@ as intuitive as Dockge, as capable as Portainer for Compose workflows.
|
|||||||
|
|
||||||
### Phase 6 — Backup destinations
|
### Phase 6 — Backup destinations
|
||||||
|
|
||||||
- **Off-box backups**: define **SFTP** or **S3-compatible** (MinIO, Backblaze B2,
|
- **Off-box backups**: define **SFTP**, **S3-compatible** (MinIO, Backblaze B2,
|
||||||
AWS S3, …) destinations under **Settings → Backup destinations** (with a Test
|
AWS S3, …) or **NFS share** (0.33.0) destinations under **Settings → Backup
|
||||||
button; secrets are masked in API responses).
|
destinations** (with a Test button; secrets are masked in API responses).
|
||||||
|
NFS needs no privileges in the StackPilot container: the Docker daemon mounts
|
||||||
|
the export as a named volume (`stackpilot-nfs-dest-<id>`, recreated when the
|
||||||
|
config changes) and file I/O runs through throwaway helper containers.
|
||||||
- **Push & restore**: the stack Backup dialog can push straight to a destination
|
- **Push & restore**: the stack Backup dialog can push straight to a destination
|
||||||
instead of downloading; the Restore dialog can browse a destination's backups
|
instead of downloading; the Restore dialog can browse a destination's backups
|
||||||
and restore (volumes included) directly from it. Remote backups can also be
|
and restore (volumes included) directly from it. Remote backups can also be
|
||||||
|
|||||||
@@ -10,14 +10,14 @@ def _now() -> datetime:
|
|||||||
return datetime.now(timezone.utc)
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
DESTINATION_TYPES = ["sftp", "s3"]
|
DESTINATION_TYPES = ["sftp", "s3", "nfs"]
|
||||||
|
|
||||||
# config keys that hold secrets — masked in API responses.
|
# config keys that hold secrets — masked in API responses.
|
||||||
SECRET_KEYS = {"password", "private_key", "secret_key"}
|
SECRET_KEYS = {"password", "private_key", "secret_key"}
|
||||||
|
|
||||||
|
|
||||||
class BackupDestination(SQLModel, table=True):
|
class BackupDestination(SQLModel, table=True):
|
||||||
"""A remote target for stack backups (SFTP or S3-compatible)."""
|
"""A remote target for stack backups (SFTP, S3-compatible or NFS)."""
|
||||||
|
|
||||||
id: Optional[int] = Field(default=None, primary_key=True)
|
id: Optional[int] = Field(default=None, primary_key=True)
|
||||||
name: str
|
name: str
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
"""Push/pull stack backups to remote destinations (SFTP or S3-compatible).
|
"""Push/pull stack backups to remote destinations (SFTP, S3-compatible or NFS).
|
||||||
|
|
||||||
All operations are synchronous (paramiko / boto3); async callers should wrap
|
All operations are synchronous (paramiko / boto3 / docker); async callers
|
||||||
them with ``asyncio.to_thread``. Destination config is a plain dict parsed from
|
should wrap them with ``asyncio.to_thread``. Destination config is a plain
|
||||||
the ``BackupDestination.config`` JSON column.
|
dict parsed from the ``BackupDestination.config`` JSON column.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -12,6 +12,8 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import posixpath
|
import posixpath
|
||||||
import stat
|
import stat
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from models.backup_destination import BackupDestination
|
from models.backup_destination import BackupDestination
|
||||||
@@ -209,6 +211,215 @@ def _s3_delete(cfg: dict, name: str) -> None:
|
|||||||
client.delete_object(Bucket=cfg["bucket"], Key=_s3_key(cfg, name))
|
client.delete_object(Bucket=cfg["bucket"], Key=_s3_key(cfg, name))
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# NFS — the Docker daemon mounts the export as a named volume; file I/O runs
|
||||||
|
# through a throwaway helper container (same pattern as volume backups), so
|
||||||
|
# the backend itself needs no mount privileges.
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
_NFS_VOLUME_PREFIX = "stackpilot-nfs-dest-"
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_check_name(name: str) -> None:
|
||||||
|
if not name or "/" in name or name in (".", "..") or name.startswith(".."):
|
||||||
|
raise DestinationError(f"Invalid backup file name '{name}'")
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_subdir(cfg: dict) -> str:
|
||||||
|
"""Sanitized relative directory inside the export ('' = export root).
|
||||||
|
|
||||||
|
Parts are restricted to a safe charset because the path is interpolated
|
||||||
|
into helper-container shell commands.
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
|
||||||
|
raw = (cfg.get("subdir") or "").strip().strip("/")
|
||||||
|
if not raw:
|
||||||
|
return ""
|
||||||
|
parts = [p for p in raw.split("/") if p]
|
||||||
|
for p in parts:
|
||||||
|
if p == ".." or not re.fullmatch(r"[A-Za-z0-9._-]+", p):
|
||||||
|
raise DestinationError(
|
||||||
|
"Subdirectory may only contain letters, digits, '.', '_' and '-'"
|
||||||
|
)
|
||||||
|
return "/".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_volume(dest: BackupDestination, cfg: dict) -> str:
|
||||||
|
"""Ensure the named volume describing this NFS mount exists; recreate it
|
||||||
|
when the destination's server/path/options changed (opts are immutable)."""
|
||||||
|
from docker_client import DockerError, get_client, safe_call
|
||||||
|
|
||||||
|
server = (cfg.get("server") or "").strip()
|
||||||
|
path = (cfg.get("path") or "").strip()
|
||||||
|
if not server:
|
||||||
|
raise DestinationError("NFS server is required")
|
||||||
|
if not path.startswith("/"):
|
||||||
|
raise DestinationError("NFS export path must be absolute (start with /)")
|
||||||
|
options = (cfg.get("options") or "rw").strip().strip(",")
|
||||||
|
driver_opts = {"type": "nfs", "o": f"addr={server},{options}", "device": f":{path}"}
|
||||||
|
|
||||||
|
name = f"{_NFS_VOLUME_PREFIX}{dest.id}"
|
||||||
|
client = get_client()
|
||||||
|
try:
|
||||||
|
vol = safe_call(client.volumes.get, name)
|
||||||
|
if (vol.attrs.get("Options") or {}) != driver_opts:
|
||||||
|
safe_call(vol.remove)
|
||||||
|
raise DockerError("recreate", "options changed")
|
||||||
|
except DockerError:
|
||||||
|
safe_call(
|
||||||
|
client.volumes.create,
|
||||||
|
name=name,
|
||||||
|
driver="local",
|
||||||
|
driver_opts=driver_opts,
|
||||||
|
labels={"stackpilot.nfs-destination": str(dest.id)},
|
||||||
|
)
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_target(cfg: dict) -> str:
|
||||||
|
sub = _nfs_subdir(cfg)
|
||||||
|
return f"/nfs/{sub}" if sub else "/nfs"
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_run(volume: str, command: list[str]) -> str:
|
||||||
|
"""Run a helper container with the NFS volume at /nfs; return stdout."""
|
||||||
|
import docker.errors
|
||||||
|
|
||||||
|
from config import settings
|
||||||
|
from docker_client import DockerError, get_client
|
||||||
|
from services.backup_service import _ensure_helper_image
|
||||||
|
|
||||||
|
client = get_client()
|
||||||
|
_ensure_helper_image(client)
|
||||||
|
try:
|
||||||
|
out = client.containers.run(
|
||||||
|
settings.BACKUP_HELPER_IMAGE,
|
||||||
|
command,
|
||||||
|
volumes={volume: {"bind": "/nfs", "mode": "rw"}},
|
||||||
|
remove=True,
|
||||||
|
)
|
||||||
|
return (out or b"").decode("utf-8", "replace")
|
||||||
|
except docker.errors.ContainerError as exc:
|
||||||
|
stderr = (exc.stderr or b"").decode("utf-8", "replace").strip()
|
||||||
|
raise DestinationError(f"NFS operation failed: {stderr or exc}") from exc
|
||||||
|
except (docker.errors.APIError, DockerError) as exc:
|
||||||
|
# Mount errors surface here (unreachable server, bad export, ...).
|
||||||
|
raise DestinationError(f"NFS mount failed: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_helper(volume: str):
|
||||||
|
"""A created (not started) helper container for archive I/O on /nfs."""
|
||||||
|
import docker.errors
|
||||||
|
|
||||||
|
from config import settings
|
||||||
|
from docker_client import DockerError, get_client, safe_call
|
||||||
|
from services.backup_service import _ensure_helper_image
|
||||||
|
|
||||||
|
client = get_client()
|
||||||
|
_ensure_helper_image(client)
|
||||||
|
try:
|
||||||
|
return safe_call(
|
||||||
|
client.containers.create,
|
||||||
|
settings.BACKUP_HELPER_IMAGE,
|
||||||
|
command="true",
|
||||||
|
volumes={volume: {"bind": "/nfs", "mode": "rw"}},
|
||||||
|
)
|
||||||
|
except (docker.errors.APIError, DockerError) as exc:
|
||||||
|
raise DestinationError(f"NFS mount failed: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_upload(dest: BackupDestination, cfg: dict, local_path: str, filename: str) -> str:
|
||||||
|
import docker.errors
|
||||||
|
|
||||||
|
_nfs_check_name(filename)
|
||||||
|
volume = _nfs_volume(dest, cfg)
|
||||||
|
target = _nfs_target(cfg)
|
||||||
|
# Creates the subdir if needed AND fails early with a clear mount error.
|
||||||
|
_nfs_run(volume, ["mkdir", "-p", target])
|
||||||
|
container = _nfs_helper(volume)
|
||||||
|
try:
|
||||||
|
with tempfile.TemporaryFile() as tmp:
|
||||||
|
with tarfile.open(fileobj=tmp, mode="w") as tar:
|
||||||
|
tar.add(local_path, arcname=filename)
|
||||||
|
tmp.seek(0)
|
||||||
|
container.put_archive(target, tmp)
|
||||||
|
except docker.errors.APIError as exc:
|
||||||
|
raise DestinationError(f"NFS upload failed: {exc}") from exc
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
container.remove(force=True)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
sub = _nfs_subdir(cfg)
|
||||||
|
return posixpath.join(sub, filename) if sub else filename
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_list(dest: BackupDestination, cfg: dict) -> list[dict]:
|
||||||
|
volume = _nfs_volume(dest, cfg)
|
||||||
|
target = _nfs_target(cfg)
|
||||||
|
out = _nfs_run(
|
||||||
|
volume,
|
||||||
|
["sh", "-c", f"cd {target} 2>/dev/null && stat -c '%n|%s|%Y' *.tar.gz 2>/dev/null; true"],
|
||||||
|
)
|
||||||
|
entries = []
|
||||||
|
for line in out.splitlines():
|
||||||
|
parts = line.strip().split("|")
|
||||||
|
if len(parts) != 3 or parts[0] == "*.tar.gz":
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
entries.append({"name": parts[0], "size": int(parts[1]), "modified": int(parts[2])})
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return sorted(entries, key=lambda x: x["modified"] or 0, reverse=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_download(dest: BackupDestination, cfg: dict, name: str, local_path: str) -> None:
|
||||||
|
import docker.errors
|
||||||
|
|
||||||
|
_nfs_check_name(name)
|
||||||
|
volume = _nfs_volume(dest, cfg)
|
||||||
|
target = _nfs_target(cfg)
|
||||||
|
container = _nfs_helper(volume)
|
||||||
|
try:
|
||||||
|
bits, _ = container.get_archive(f"{target}/{name}")
|
||||||
|
with tempfile.TemporaryFile() as tmp:
|
||||||
|
for chunk in bits:
|
||||||
|
tmp.write(chunk)
|
||||||
|
tmp.seek(0)
|
||||||
|
with tarfile.open(fileobj=tmp) as tar:
|
||||||
|
member = next((m for m in tar.getmembers() if m.isreg()), None)
|
||||||
|
fh = tar.extractfile(member) if member else None
|
||||||
|
if fh is None:
|
||||||
|
raise DestinationError(f"'{name}' not found on NFS destination")
|
||||||
|
with open(local_path, "wb") as out:
|
||||||
|
while chunk := fh.read(1024 * 1024):
|
||||||
|
out.write(chunk)
|
||||||
|
except docker.errors.APIError as exc:
|
||||||
|
raise DestinationError(f"NFS download failed (does '{name}' exist?): {exc}") from exc
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
container.remove(force=True)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_delete(dest: BackupDestination, cfg: dict, name: str) -> None:
|
||||||
|
_nfs_check_name(name)
|
||||||
|
volume = _nfs_volume(dest, cfg)
|
||||||
|
_nfs_run(volume, ["rm", "-f", f"{_nfs_target(cfg)}/{name}"])
|
||||||
|
|
||||||
|
|
||||||
|
def _nfs_test(dest: BackupDestination, cfg: dict) -> bool:
|
||||||
|
volume = _nfs_volume(dest, cfg)
|
||||||
|
target = _nfs_target(cfg)
|
||||||
|
_nfs_run(
|
||||||
|
volume,
|
||||||
|
["sh", "-c", f"mkdir -p {target} && touch {target}/.stackpilot-test && rm -f {target}/.stackpilot-test"],
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
# Dispatch
|
# Dispatch
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
@@ -220,6 +431,8 @@ def upload(dest: BackupDestination, local_path: str, filename: str) -> str:
|
|||||||
return _sftp_upload(cfg, local_path, filename)
|
return _sftp_upload(cfg, local_path, filename)
|
||||||
if dest.type == "s3":
|
if dest.type == "s3":
|
||||||
return _s3_upload(cfg, local_path, filename)
|
return _s3_upload(cfg, local_path, filename)
|
||||||
|
if dest.type == "nfs":
|
||||||
|
return _nfs_upload(dest, cfg, local_path, filename)
|
||||||
raise DestinationError(f"Unknown destination type '{dest.type}'")
|
raise DestinationError(f"Unknown destination type '{dest.type}'")
|
||||||
|
|
||||||
|
|
||||||
@@ -229,6 +442,8 @@ def list_backups(dest: BackupDestination) -> list[dict]:
|
|||||||
return _sftp_list(cfg)
|
return _sftp_list(cfg)
|
||||||
if dest.type == "s3":
|
if dest.type == "s3":
|
||||||
return _s3_list(cfg)
|
return _s3_list(cfg)
|
||||||
|
if dest.type == "nfs":
|
||||||
|
return _nfs_list(dest, cfg)
|
||||||
raise DestinationError(f"Unknown destination type '{dest.type}'")
|
raise DestinationError(f"Unknown destination type '{dest.type}'")
|
||||||
|
|
||||||
|
|
||||||
@@ -238,6 +453,8 @@ def download(dest: BackupDestination, name: str, local_path: str) -> None:
|
|||||||
_sftp_download(cfg, name, local_path)
|
_sftp_download(cfg, name, local_path)
|
||||||
elif dest.type == "s3":
|
elif dest.type == "s3":
|
||||||
_s3_download(cfg, name, local_path)
|
_s3_download(cfg, name, local_path)
|
||||||
|
elif dest.type == "nfs":
|
||||||
|
_nfs_download(dest, cfg, name, local_path)
|
||||||
else:
|
else:
|
||||||
raise DestinationError(f"Unknown destination type '{dest.type}'")
|
raise DestinationError(f"Unknown destination type '{dest.type}'")
|
||||||
|
|
||||||
@@ -248,11 +465,15 @@ def delete(dest: BackupDestination, name: str) -> None:
|
|||||||
_sftp_delete(cfg, name)
|
_sftp_delete(cfg, name)
|
||||||
elif dest.type == "s3":
|
elif dest.type == "s3":
|
||||||
_s3_delete(cfg, name)
|
_s3_delete(cfg, name)
|
||||||
|
elif dest.type == "nfs":
|
||||||
|
_nfs_delete(dest, cfg, name)
|
||||||
else:
|
else:
|
||||||
raise DestinationError(f"Unknown destination type '{dest.type}'")
|
raise DestinationError(f"Unknown destination type '{dest.type}'")
|
||||||
|
|
||||||
|
|
||||||
def test(dest: BackupDestination) -> bool:
|
def test(dest: BackupDestination) -> bool:
|
||||||
"""Connectivity check — lists the target (cheap, validates auth + path)."""
|
"""Connectivity check — validates reachability, auth and write access."""
|
||||||
|
if dest.type == "nfs":
|
||||||
|
return _nfs_test(dest, parse_config(dest))
|
||||||
list_backups(dest)
|
list_backups(dest)
|
||||||
return True
|
return True
|
||||||
|
|||||||
+1
-1
@@ -1,3 +1,3 @@
|
|||||||
"""Single source of truth for the StackPilot release version."""
|
"""Single source of truth for the StackPilot release version."""
|
||||||
|
|
||||||
APP_VERSION = "0.32.1"
|
APP_VERSION = "0.33.0"
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "stackpilot-frontend",
|
"name": "stackpilot-frontend",
|
||||||
"private": true,
|
"private": true,
|
||||||
"version": "0.32.1",
|
"version": "0.33.0",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "vite",
|
"dev": "vite",
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import api from "./client";
|
|||||||
export interface BackupDestination {
|
export interface BackupDestination {
|
||||||
id: number;
|
id: number;
|
||||||
name: string;
|
name: string;
|
||||||
type: "sftp" | "s3";
|
type: "sftp" | "s3" | "nfs";
|
||||||
config: Record<string, string>;
|
config: Record<string, string>;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -342,6 +342,12 @@ const FIELDS: Record<string, { key: string; label: string; secret?: boolean; are
|
|||||||
{ key: "secret_key", label: "Secret key", secret: true },
|
{ key: "secret_key", label: "Secret key", secret: true },
|
||||||
{ key: "prefix", label: "Key prefix (optional)", placeholder: "stackpilot/" },
|
{ key: "prefix", label: "Key prefix (optional)", placeholder: "stackpilot/" },
|
||||||
],
|
],
|
||||||
|
nfs: [
|
||||||
|
{ key: "server", label: "NFS server (host or IP)", placeholder: "10.0.0.5" },
|
||||||
|
{ key: "path", label: "Export path", placeholder: "/export/backups" },
|
||||||
|
{ key: "options", label: "Mount options", placeholder: "rw,nfsvers=4 (default: rw)" },
|
||||||
|
{ key: "subdir", label: "Subdirectory inside the export (optional)", placeholder: "stackpilot" },
|
||||||
|
],
|
||||||
};
|
};
|
||||||
|
|
||||||
function DestinationsSection() {
|
function DestinationsSection() {
|
||||||
@@ -362,8 +368,8 @@ function DestinationsSection() {
|
|||||||
{data?.length === 0 && !adding && (
|
{data?.length === 0 && !adding && (
|
||||||
<Card>
|
<Card>
|
||||||
<p className="text-sm text-slate-500">
|
<p className="text-sm text-slate-500">
|
||||||
No destinations. Add an SFTP server or S3-compatible bucket to push stack
|
No destinations. Add an SFTP server, S3-compatible bucket or NFS share to
|
||||||
backups off-box and restore from them.
|
push stack backups off-box and restore from them.
|
||||||
</p>
|
</p>
|
||||||
</Card>
|
</Card>
|
||||||
)}
|
)}
|
||||||
@@ -395,7 +401,9 @@ function DestinationRow({ dest, onChange }: { dest: BackupDestination; onChange:
|
|||||||
const summary =
|
const summary =
|
||||||
dest.type === "s3"
|
dest.type === "s3"
|
||||||
? `${dest.config.bucket}${dest.config.prefix ? "/" + dest.config.prefix : ""}`
|
? `${dest.config.bucket}${dest.config.prefix ? "/" + dest.config.prefix : ""}`
|
||||||
: `${dest.config.username}@${dest.config.host}:${dest.config.path || "."}`;
|
: dest.type === "nfs"
|
||||||
|
? `${dest.config.server}:${dest.config.path}${dest.config.subdir ? "/" + dest.config.subdir : ""}`
|
||||||
|
: `${dest.config.username}@${dest.config.host}:${dest.config.path || "."}`;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Card className="flex flex-wrap items-center justify-between gap-2">
|
<Card className="flex flex-wrap items-center justify-between gap-2">
|
||||||
@@ -443,6 +451,7 @@ function DestinationForm({ onDone, onCancel }: { onDone: () => void; onCancel: (
|
|||||||
<select className={selectClass} value={type} onChange={(e) => { setType(e.target.value); setConfig({}); }}>
|
<select className={selectClass} value={type} onChange={(e) => { setType(e.target.value); setConfig({}); }}>
|
||||||
<option value="sftp">SFTP</option>
|
<option value="sftp">SFTP</option>
|
||||||
<option value="s3">S3-compatible</option>
|
<option value="s3">S3-compatible</option>
|
||||||
|
<option value="nfs">NFS share</option>
|
||||||
</select>
|
</select>
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user