menzeljandClaude Opus 5
270ddda51d
A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.
Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.
POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".
Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
283 lines
9.7 KiB
Python
283 lines
9.7 KiB
Python
"""Stack backup (incl. volumes) and restore."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import tempfile
|
|
|
|
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile
|
|
from fastapi.responses import FileResponse
|
|
from starlette.background import BackgroundTask
|
|
from pydantic import BaseModel
|
|
from sqlmodel import Session
|
|
|
|
from auth import require_admin
|
|
from database import get_session
|
|
from models.backup_destination import BackupDestination
|
|
from models.stack import Stack
|
|
from models.user import User
|
|
from services import (
|
|
audit_service,
|
|
backup_destination_service as dest_service,
|
|
backup_service,
|
|
compose_service,
|
|
)
|
|
|
|
router = APIRouter(prefix="/api/stacks", tags=["backups"])
|
|
|
|
|
|
def _ip(request: Request) -> str:
|
|
return request.client.host if request.client else "unknown"
|
|
|
|
|
|
_backup_filename = backup_service.backup_filename
|
|
|
|
|
|
def _compact(report: dict) -> dict:
|
|
"""The parts of a backup report worth showing the user."""
|
|
return {
|
|
"size": report.get("size"),
|
|
"binds": report.get("binds", []),
|
|
"volumes": report.get("volumes", []),
|
|
"skipped": report.get("skipped", []),
|
|
"path_mismatch": report.get("path_mismatch"),
|
|
"stopped": report.get("stopped", []),
|
|
"errors": report.get("errors", []),
|
|
# Capped: a busy log folder can produce a warning per file, and this
|
|
# travels in a response header for downloads.
|
|
"warnings": report.get("warnings", [])[:20],
|
|
"restart_error": report.get("restart_error"),
|
|
}
|
|
|
|
|
|
def _summary(report: dict) -> str:
|
|
text = (
|
|
f"binds={len(report.get('binds', []))} "
|
|
f"volumes={len(report.get('volumes', []))} "
|
|
f"skipped={len(report.get('skipped', []))}"
|
|
)
|
|
if report.get("errors"):
|
|
text += f" errors={len(report['errors'])}"
|
|
if report.get("restart_error"):
|
|
text += " restart_failed"
|
|
return text
|
|
|
|
|
|
def _backup_http_error(exc: backup_service.BackupError) -> HTTPException:
|
|
# 409 for a stack that is busy, a backup somebody stopped, or a target that
|
|
# already exists; 400 for the rest.
|
|
conflict = isinstance(
|
|
exc, (backup_service.StackBusyError, backup_service.BackupCancelled)
|
|
) or "already exists" in str(exc)
|
|
return HTTPException(status_code=409 if conflict else 400, detail=str(exc))
|
|
|
|
|
|
@router.get("/{stack_id}/backup/inventory")
|
|
async def backup_inventory(
|
|
stack_id: str,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
"""What a backup of this stack would capture: bind-mount sources (with size
|
|
and whether they are reachable at all), named volumes, and anything that is
|
|
skipped by default with the reason why."""
|
|
if not session.get(Stack, stack_id):
|
|
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
|
|
return await asyncio.to_thread(backup_service.plan, stack_id)
|
|
|
|
|
|
@router.get("/{stack_id}/backup")
|
|
async def backup_stack(
|
|
stack_id: str,
|
|
request: Request,
|
|
include_volumes: bool = Query(True),
|
|
include_binds: bool = Query(True),
|
|
stop_first: bool = Query(True),
|
|
binds: list[str] | None = Query(None),
|
|
volumes: list[str] | None = Query(None),
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
):
|
|
stack = session.get(Stack, stack_id)
|
|
if not stack:
|
|
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
|
|
try:
|
|
path, report = await backup_service.create_backup_ex(
|
|
stack_id, stack.name, include_volumes=include_volumes,
|
|
stop_first=stop_first, include_binds=include_binds,
|
|
binds=binds, volumes=volumes,
|
|
)
|
|
except backup_service.BackupError as exc:
|
|
raise _backup_http_error(exc) from exc
|
|
audit_service.record(
|
|
session, user=user.username, action="stack.backup", target=stack_id,
|
|
detail=_summary(report), ip=_ip(request),
|
|
)
|
|
return FileResponse(
|
|
path,
|
|
media_type="application/gzip",
|
|
filename=_backup_filename(stack_id, include_volumes),
|
|
# The browser downloads a blob, so the summary of what actually made it
|
|
# into the archive rides along in a header.
|
|
headers={"X-Stackpilot-Backup": json.dumps(_compact(report))},
|
|
# The archive is a temp file; without this every download stayed in
|
|
# the container's /tmp until it filled the disk.
|
|
background=BackgroundTask(os.unlink, path),
|
|
)
|
|
|
|
|
|
@router.post("/restore")
|
|
async def restore_stack(
|
|
request: Request,
|
|
file: UploadFile = File(...),
|
|
target_id: str | None = Form(None),
|
|
overwrite: bool = Form(False),
|
|
restore_volumes: bool = Form(True),
|
|
restore_binds: bool = Form(True),
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
|
try:
|
|
while chunk := await file.read(1024 * 1024):
|
|
tmp.write(chunk)
|
|
tmp.close()
|
|
|
|
target = compose_service.slugify(target_id) if target_id else None
|
|
try:
|
|
result = await backup_service.restore(
|
|
tmp.name,
|
|
target_id=target,
|
|
overwrite=overwrite,
|
|
restore_volumes=restore_volumes,
|
|
restore_binds=restore_binds,
|
|
)
|
|
except backup_service.BackupError as exc:
|
|
raise _backup_http_error(exc) from exc
|
|
|
|
stack_id = result["stack_id"]
|
|
stack = session.get(Stack, stack_id)
|
|
if not stack:
|
|
session.add(Stack(id=stack_id, name=result.get("name", stack_id)))
|
|
session.commit()
|
|
audit_service.record(
|
|
session, user=user.username, action="stack.restore", target=stack_id,
|
|
detail=f"volumes={result['volumes_restored']} binds={result['binds_restored']}",
|
|
ip=_ip(request),
|
|
)
|
|
return result
|
|
finally:
|
|
if os.path.exists(tmp.name):
|
|
os.unlink(tmp.name)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Push to / restore from a remote destination
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
class PushBody(BaseModel):
|
|
destination_id: int
|
|
include_volumes: bool = True
|
|
include_binds: bool = True
|
|
stop_first: bool = True
|
|
binds: list[str] | None = None
|
|
volumes: list[str] | None = None
|
|
|
|
|
|
class RestoreFromBody(BaseModel):
|
|
destination_id: int
|
|
name: str
|
|
target_id: str | None = None
|
|
overwrite: bool = False
|
|
restore_volumes: bool = True
|
|
restore_binds: bool = True
|
|
|
|
|
|
def _get_dest(session: Session, dest_id: int) -> BackupDestination:
|
|
d = session.get(BackupDestination, dest_id)
|
|
if not d:
|
|
raise HTTPException(status_code=404, detail=f"Destination {dest_id} not found")
|
|
return d
|
|
|
|
|
|
@router.post("/{stack_id}/backup/push")
|
|
async def push_backup(
|
|
stack_id: str,
|
|
body: PushBody,
|
|
request: Request,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
stack = session.get(Stack, stack_id)
|
|
if not stack:
|
|
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
|
|
dest = _get_dest(session, body.destination_id)
|
|
try:
|
|
path, report = await backup_service.create_backup_ex(
|
|
stack_id, stack.name,
|
|
include_volumes=body.include_volumes, stop_first=body.stop_first,
|
|
include_binds=body.include_binds, binds=body.binds, volumes=body.volumes,
|
|
)
|
|
except backup_service.BackupError as exc:
|
|
raise _backup_http_error(exc) from exc
|
|
|
|
filename = _backup_filename(stack_id, body.include_volumes)
|
|
try:
|
|
remote = await asyncio.to_thread(dest_service.upload, dest, path, filename)
|
|
except dest_service.DestinationError as exc:
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
finally:
|
|
if os.path.exists(path):
|
|
os.unlink(path)
|
|
|
|
audit_service.record(
|
|
session, user=user.username, action="stack.backup.push",
|
|
target=stack_id, detail=f"{dest.name}:{filename} {_summary(report)}", ip=_ip(request),
|
|
)
|
|
return {
|
|
"ok": True, "destination": dest.name, "name": filename,
|
|
"remote": remote, "report": _compact(report),
|
|
}
|
|
|
|
|
|
@router.post("/restore-from")
|
|
async def restore_from_destination(
|
|
body: RestoreFromBody,
|
|
request: Request,
|
|
session: Session = Depends(get_session),
|
|
user: User = Depends(require_admin),
|
|
) -> dict:
|
|
dest = _get_dest(session, body.destination_id)
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
|
tmp.close()
|
|
try:
|
|
try:
|
|
await asyncio.to_thread(dest_service.download, dest, body.name, tmp.name)
|
|
except dest_service.DestinationError as exc:
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
|
|
target = compose_service.slugify(body.target_id) if body.target_id else None
|
|
try:
|
|
result = await backup_service.restore(
|
|
tmp.name, target_id=target,
|
|
overwrite=body.overwrite, restore_volumes=body.restore_volumes,
|
|
restore_binds=body.restore_binds,
|
|
)
|
|
except backup_service.BackupError as exc:
|
|
raise _backup_http_error(exc) from exc
|
|
|
|
stack_id = result["stack_id"]
|
|
if not session.get(Stack, stack_id):
|
|
session.add(Stack(id=stack_id, name=result.get("name", stack_id)))
|
|
session.commit()
|
|
audit_service.record(
|
|
session, user=user.username, action="stack.restore",
|
|
target=stack_id, detail=f"from {dest.name}:{body.name}", ip=_ip(request),
|
|
)
|
|
return result
|
|
finally:
|
|
if os.path.exists(tmp.name):
|
|
os.unlink(tmp.name)
|