A review of the backup path found several ways a backup was not one, and a restore could destroy data it did not contain. Data loss: - Restore with overwrite rmtree'd the whole stack directory, including folders the backup had left out (over 2 GiB, deselected, an NFS share mounted inside). It now clears only what the backup holds, keeps the excluded paths (recorded in the manifest) and never crosses into another filesystem. - The stack was only stopped when named volumes were selected, so data folders inside the stack directory were archived while in use. Any selected data now stops it. - Retention matched by prefix: pruning "app" deleted "app-db"'s backups. It now matches backup-<id>-(full|config)-<date>.tar.gz exactly. - A restore under a new name wrote into the original's volumes when they had an explicit name, and into its folders outside the stack directory. Both are skipped now; a volume owned by another stack or mounted by a running container is never overwritten. - A truncated archive was only noticed after the old stack was gone. The whole archive is verified first, and the stack directory is unpacked next to the old one and swapped in. Remote storage: - Bind folders on a mounted share (fstab/autofs), bind-type volumes pointing at one and shares mounted inside the stack folder are detected by filesystem type and not selected; a folder du cannot measure in 20s is no longer selected by default either. Reliability: - After a backup exactly the containers that ran are started again with compose start (falling back to the Docker API), instead of up -d, which started services stopped on purpose and applied undeployed edits. A stack that cannot be stopped is not backed up. - Backup and restore hold the stack lock; a scheduled backup that finds the stack busy retries after 10 minutes. Restore stops the stack while replacing its data and brings it back up. - external: true volumes are backed up; anonymous ones are listed, not selected. Restored volumes get their original driver and options. - A backup with uncaptured data or a stack that did not restart is reported as incomplete: in the dialog, and for schedules as a warning with a notification and without rotating older backups out. - Archive writing and restore run off the event loop; downloads delete their temp file; helper containers use the long-timeout client; S3 listing is paginated; a file changing mid-read no longer corrupts the archive. Manifest format 3 records compose_binds/excluded and volume driver/options/external; format 1/2 backups still restore. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
280 lines
9.6 KiB
Python
280 lines
9.6 KiB
Python
"""Stack backup (incl. volumes) and restore."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import tempfile
|
|
|
|
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile
|
|
from fastapi.responses import FileResponse
|
|
from starlette.background import BackgroundTask
|
|
from pydantic import BaseModel
|
|
from sqlmodel import Session
|
|
|
|
from auth import require_admin
|
|
from database import get_session
|
|
from models.backup_destination import BackupDestination
|
|
from models.stack import Stack
|
|
from models.user import User
|
|
from services import (
|
|
audit_service,
|
|
backup_destination_service as dest_service,
|
|
backup_service,
|
|
compose_service,
|
|
)
|
|
|
|
router = APIRouter(prefix="/api/stacks", tags=["backups"])
|
|
|
|
|
|
def _ip(request: Request) -> str:
|
|
return request.client.host if request.client else "unknown"
|
|
|
|
|
|
_backup_filename = backup_service.backup_filename
|
|
|
|
|
|
def _compact(report: dict) -> dict:
|
|
"""The parts of a backup report worth showing the user."""
|
|
return {
|
|
"size": report.get("size"),
|
|
"binds": report.get("binds", []),
|
|
"volumes": report.get("volumes", []),
|
|
"skipped": report.get("skipped", []),
|
|
"path_mismatch": report.get("path_mismatch"),
|
|
"stopped": report.get("stopped", []),
|
|
"errors": report.get("errors", []),
|
|
# Capped: a busy log folder can produce a warning per file, and this
|
|
# travels in a response header for downloads.
|
|
"warnings": report.get("warnings", [])[:20],
|
|
"restart_error": report.get("restart_error"),
|
|
}
|
|
|
|
|
|
def _summary(report: dict) -> str:
|
|
text = (
|
|
f"binds={len(report.get('binds', []))} "
|
|
f"volumes={len(report.get('volumes', []))} "
|
|
f"skipped={len(report.get('skipped', []))}"
|
|
)
|
|
if report.get("errors"):
|
|
text += f" errors={len(report['errors'])}"
|
|
if report.get("restart_error"):
|
|
text += " restart_failed"
|
|
return text
|
|
|
|
|
|
def _backup_http_error(exc: backup_service.BackupError) -> HTTPException:
|
|
# 409 for a stack that is busy or already exists, 400 for the rest.
|
|
conflict = isinstance(exc, backup_service.StackBusyError) or "already exists" in str(exc)
|
|
return HTTPException(status_code=409 if conflict else 400, detail=str(exc))
|
|
|
|
|
|
@router.get("/{stack_id}/backup/inventory")
|
|
async def backup_inventory(
|
|
stack_id: str,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
"""What a backup of this stack would capture: bind-mount sources (with size
|
|
and whether they are reachable at all), named volumes, and anything that is
|
|
skipped by default with the reason why."""
|
|
if not session.get(Stack, stack_id):
|
|
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
|
|
return await asyncio.to_thread(backup_service.plan, stack_id)
|
|
|
|
|
|
@router.get("/{stack_id}/backup")
|
|
async def backup_stack(
|
|
stack_id: str,
|
|
request: Request,
|
|
include_volumes: bool = Query(True),
|
|
include_binds: bool = Query(True),
|
|
stop_first: bool = Query(True),
|
|
binds: list[str] | None = Query(None),
|
|
volumes: list[str] | None = Query(None),
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
):
|
|
stack = session.get(Stack, stack_id)
|
|
if not stack:
|
|
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
|
|
try:
|
|
path, report = await backup_service.create_backup_ex(
|
|
stack_id, stack.name, include_volumes=include_volumes,
|
|
stop_first=stop_first, include_binds=include_binds,
|
|
binds=binds, volumes=volumes,
|
|
)
|
|
except backup_service.BackupError as exc:
|
|
raise _backup_http_error(exc) from exc
|
|
audit_service.record(
|
|
session, user=user.username, action="stack.backup", target=stack_id,
|
|
detail=_summary(report), ip=_ip(request),
|
|
)
|
|
return FileResponse(
|
|
path,
|
|
media_type="application/gzip",
|
|
filename=_backup_filename(stack_id, include_volumes),
|
|
# The browser downloads a blob, so the summary of what actually made it
|
|
# into the archive rides along in a header.
|
|
headers={"X-Stackpilot-Backup": json.dumps(_compact(report))},
|
|
# The archive is a temp file; without this every download stayed in
|
|
# the container's /tmp until it filled the disk.
|
|
background=BackgroundTask(os.unlink, path),
|
|
)
|
|
|
|
|
|
@router.post("/restore")
|
|
async def restore_stack(
|
|
request: Request,
|
|
file: UploadFile = File(...),
|
|
target_id: str | None = Form(None),
|
|
overwrite: bool = Form(False),
|
|
restore_volumes: bool = Form(True),
|
|
restore_binds: bool = Form(True),
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
|
try:
|
|
while chunk := await file.read(1024 * 1024):
|
|
tmp.write(chunk)
|
|
tmp.close()
|
|
|
|
target = compose_service.slugify(target_id) if target_id else None
|
|
try:
|
|
result = await backup_service.restore(
|
|
tmp.name,
|
|
target_id=target,
|
|
overwrite=overwrite,
|
|
restore_volumes=restore_volumes,
|
|
restore_binds=restore_binds,
|
|
)
|
|
except backup_service.BackupError as exc:
|
|
raise _backup_http_error(exc) from exc
|
|
|
|
stack_id = result["stack_id"]
|
|
stack = session.get(Stack, stack_id)
|
|
if not stack:
|
|
session.add(Stack(id=stack_id, name=result.get("name", stack_id)))
|
|
session.commit()
|
|
audit_service.record(
|
|
session, user=user.username, action="stack.restore", target=stack_id,
|
|
detail=f"volumes={result['volumes_restored']} binds={result['binds_restored']}",
|
|
ip=_ip(request),
|
|
)
|
|
return result
|
|
finally:
|
|
if os.path.exists(tmp.name):
|
|
os.unlink(tmp.name)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Push to / restore from a remote destination
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
class PushBody(BaseModel):
|
|
destination_id: int
|
|
include_volumes: bool = True
|
|
include_binds: bool = True
|
|
stop_first: bool = True
|
|
binds: list[str] | None = None
|
|
volumes: list[str] | None = None
|
|
|
|
|
|
class RestoreFromBody(BaseModel):
|
|
destination_id: int
|
|
name: str
|
|
target_id: str | None = None
|
|
overwrite: bool = False
|
|
restore_volumes: bool = True
|
|
restore_binds: bool = True
|
|
|
|
|
|
def _get_dest(session: Session, dest_id: int) -> BackupDestination:
|
|
d = session.get(BackupDestination, dest_id)
|
|
if not d:
|
|
raise HTTPException(status_code=404, detail=f"Destination {dest_id} not found")
|
|
return d
|
|
|
|
|
|
@router.post("/{stack_id}/backup/push")
|
|
async def push_backup(
|
|
stack_id: str,
|
|
body: PushBody,
|
|
request: Request,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
stack = session.get(Stack, stack_id)
|
|
if not stack:
|
|
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
|
|
dest = _get_dest(session, body.destination_id)
|
|
try:
|
|
path, report = await backup_service.create_backup_ex(
|
|
stack_id, stack.name,
|
|
include_volumes=body.include_volumes, stop_first=body.stop_first,
|
|
include_binds=body.include_binds, binds=body.binds, volumes=body.volumes,
|
|
)
|
|
except backup_service.BackupError as exc:
|
|
raise _backup_http_error(exc) from exc
|
|
|
|
filename = _backup_filename(stack_id, body.include_volumes)
|
|
try:
|
|
remote = await asyncio.to_thread(dest_service.upload, dest, path, filename)
|
|
except dest_service.DestinationError as exc:
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
finally:
|
|
if os.path.exists(path):
|
|
os.unlink(path)
|
|
|
|
audit_service.record(
|
|
session, user=user.username, action="stack.backup.push",
|
|
target=stack_id, detail=f"{dest.name}:{filename} {_summary(report)}", ip=_ip(request),
|
|
)
|
|
return {
|
|
"ok": True, "destination": dest.name, "name": filename,
|
|
"remote": remote, "report": _compact(report),
|
|
}
|
|
|
|
|
|
@router.post("/restore-from")
|
|
async def restore_from_destination(
|
|
body: RestoreFromBody,
|
|
request: Request,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
dest = _get_dest(session, body.destination_id)
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
|
tmp.close()
|
|
try:
|
|
try:
|
|
await asyncio.to_thread(dest_service.download, dest, body.name, tmp.name)
|
|
except dest_service.DestinationError as exc:
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
|
|
target = compose_service.slugify(body.target_id) if body.target_id else None
|
|
try:
|
|
result = await backup_service.restore(
|
|
tmp.name, target_id=target,
|
|
overwrite=body.overwrite, restore_volumes=body.restore_volumes,
|
|
restore_binds=body.restore_binds,
|
|
)
|
|
except backup_service.BackupError as exc:
|
|
raise _backup_http_error(exc) from exc
|
|
|
|
stack_id = result["stack_id"]
|
|
if not session.get(Stack, stack_id):
|
|
session.add(Stack(id=stack_id, name=result.get("name", stack_id)))
|
|
session.commit()
|
|
audit_service.record(
|
|
session, user=user.username, action="stack.restore",
|
|
target=stack_id, detail=f"from {dest.name}:{body.name}", ip=_ip(request),
|
|
)
|
|
return result
|
|
finally:
|
|
if os.path.exists(tmp.name):
|
|
os.unlink(tmp.name)
|