Phase 8: back up & restore remote (agent) stacks (0.8.0)

- Agent: GET /agent/stacks/{id}/backup + POST /agent/stacks/restore (reuse
  backup_service). backup_service gains backup_basename/backup_filename helpers.
- Main proxy streams agent <-> main <-> destination (creds stay central):
  agent_service download_to_file/upload_file; routers/agents.py backup download,
  backup/push, restore upload, restore-from.
- Schedules: BackupSchedule.agent_id; schedule_service downloads from the agent
  when set; per-host filename prefix isolates retention across hosts.
- Frontend: agents api backup/restore; BackupButton/RestoreButton agent-aware
  (Backup on remote stack detail, Restore per host section); schedule form host
  selector (local or an online agent) + host shown on schedule rows.

Rough-verified (per request): py_compile, frontend tsc build, image imports
(main 99 / agent 16 routes). Full live agent round-trip to be tested post-deploy.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/agent_app.py

@@ -14,17 +14,19 @@ import logging
 import os
 from dataclasses import asdict
 
-from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request
-from fastapi.responses import JSONResponse
+import tempfile
+
+from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, Request, UploadFile
+from fastapi.responses import FileResponse, JSONResponse
 from pydantic import BaseModel
 
 from config import settings
 from docker_client import DockerError, get_client, safe_call
-from services import compose_service
+from services import backup_service, compose_service
 
 logger = logging.getLogger("stackpilot.agent")
 
-AGENT_VERSION = "0.5.0"
+AGENT_VERSION = "0.8.0"
 
 
 # --------------------------------------------------------------------------- #
@@ -226,6 +228,52 @@ async def stack_logs(stack_id: str, tail: int = Query(200, le=2000)) -> dict:
     return {"logs": result.get("stdout", "") + result.get("stderr", "")}
 
 
+@app.get("/agent/stacks/{stack_id}/backup", dependencies=[Depends(verify_token)])
+async def backup_stack(
+    stack_id: str,
+    include_volumes: bool = Query(True),
+    stop_first: bool = Query(True),
+):
+    if not os.path.isdir(compose_service.stack_dir(stack_id)):
+        raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
+    try:
+        path = await backup_service.create_backup(
+            stack_id, stack_id, include_volumes=include_volumes, stop_first=stop_first,
+        )
+    except backup_service.BackupError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    return FileResponse(
+        path,
+        media_type="application/gzip",
+        filename=backup_service.backup_filename(stack_id, include_volumes),
+    )
+
+
+@app.post("/agent/stacks/restore", dependencies=[Depends(verify_token)])
+async def restore_stack(
+    file: UploadFile = File(...),
+    target_id: str | None = Form(None),
+    overwrite: bool = Form(False),
+    restore_volumes: bool = Form(True),
+) -> dict:
+    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
+    try:
+        while chunk := await file.read(1024 * 1024):
+            tmp.write(chunk)
+        tmp.close()
+        target = compose_service.slugify(target_id) if target_id else None
+        try:
+            return backup_service.restore_backup(
+                tmp.name, target_id=target, overwrite=overwrite, restore_volumes=restore_volumes,
+            )
+        except backup_service.BackupError as exc:
+            code = 409 if "already exists" in str(exc) else 400
+            raise HTTPException(status_code=code, detail=str(exc)) from exc
+    finally:
+        if os.path.exists(tmp.name):
+            os.unlink(tmp.name)
+
+
 @app.get("/agent/health")
 def health() -> dict:
     return {"status": "ok"}

backend/main.py

@@ -53,7 +53,7 @@ async def lifespan(app: FastAPI):
     schedule_task.cancel()
 
 
-app = FastAPI(title="StackPilot", version="0.7.0", lifespan=lifespan)
+app = FastAPI(title="StackPilot", version="0.8.0", lifespan=lifespan)
 
 app.add_middleware(
     CORSMiddleware,

backend/models/backup_schedule.py

@@ -19,6 +19,7 @@ class BackupSchedule(SQLModel, table=True):
     id: Optional[int] = Field(default=None, primary_key=True)
     stack_id: str
     destination_id: int
+    agent_id: Optional[int] = None  # None = local host; otherwise a remote agent
     frequency: str = "daily"  # one of FREQUENCIES
     hour: int = 3  # UTC, used for daily/weekly
     minute: int = 0
@@ -39,6 +40,7 @@ class BackupSchedule(SQLModel, table=True):
 class ScheduleCreate(SQLModel):
     stack_id: str
     destination_id: int
+    agent_id: Optional[int] = None
     frequency: str = "daily"
     hour: int = 3
     minute: int = 0
@@ -66,6 +68,8 @@ class ScheduleRead(SQLModel):
     stack_id: str
     destination_id: int
     destination_name: Optional[str]
+    agent_id: Optional[int]
+    agent_name: Optional[str]
     frequency: str
     hour: int
     minute: int

backend/routers/agents.py

@@ -1,15 +1,29 @@
 """Remote host (agent) management + proxied stack/system operations."""
 from __future__ import annotations
 
-from fastapi import APIRouter, Depends, HTTPException, Query, Request
+import asyncio
+import os
+import tempfile
+
+from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
 from sqlmodel import Session, select
+from starlette.background import BackgroundTask
 
 from auth import get_current_user, require_admin
 from database import get_session
 from models.agent import Agent, AgentCreate, AgentRead, AgentUpdate
+from models.backup_destination import BackupDestination
 from models.stack import StackCreate, StackUpdate
 from models.user import User
-from services import agent_service, audit_service
+from services import (
+    agent_service,
+    audit_service,
+    backup_destination_service as dest_service,
+    backup_service,
+    compose_service,
+)
 from services.agent_service import AgentError
 
 router = APIRouter(prefix="/api/agents", tags=["agents"])
@@ -17,6 +31,27 @@ router = APIRouter(prefix="/api/agents", tags=["agents"])
 _ACTIONS = {"start", "stop", "restart", "pull", "update", "down"}
 
 
+class AgentPushBody(BaseModel):
+    destination_id: int
+    include_volumes: bool = True
+    stop_first: bool = True
+
+
+class AgentRestoreFromBody(BaseModel):
+    destination_id: int
+    name: str
+    target_id: str | None = None
+    overwrite: bool = False
+    restore_volumes: bool = True
+
+
+def _get_dest(session: Session, dest_id: int) -> BackupDestination:
+    d = session.get(BackupDestination, dest_id)
+    if not d:
+        raise HTTPException(status_code=404, detail=f"Destination {dest_id} not found")
+    return d
+
+
 def _ip(request: Request) -> str:
     return request.client.host if request.client else "unknown"
 
@@ -284,3 +319,157 @@ async def agent_lifecycle(
         target=f"{agent.name}/{stack_id}", ip=_ip(request),
     )
     return result
+
+
+# --------------------------------------------------------------------------- #
+# Backup / restore of remote stacks (streamed agent <-> main <-> destination)
+# --------------------------------------------------------------------------- #
+
+
+@router.get("/{agent_id}/stacks/{stack_id}/backup")
+async def agent_backup_download(
+    agent_id: int,
+    stack_id: str,
+    request: Request,
+    include_volumes: bool = Query(True),
+    stop_first: bool = Query(True),
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+):
+    agent = _get_or_404(session, agent_id)
+    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
+    tmp.close()
+    try:
+        await agent_service.download_to_file(
+            session, agent, f"/agent/stacks/{stack_id}/backup", tmp.name,
+            params={"include_volumes": include_volumes, "stop_first": stop_first},
+        )
+    except AgentError as exc:
+        if os.path.exists(tmp.name):
+            os.unlink(tmp.name)
+        _raise(exc)
+    audit_service.record(
+        session, user=user.username, action="agent.stack.backup",
+        target=f"{agent.name}/{stack_id}", ip=_ip(request),
+    )
+    fname = backup_service.backup_filename(
+        stack_id, include_volumes, prefix=compose_service.slugify(agent.name)
+    )
+    return FileResponse(
+        tmp.name, media_type="application/gzip", filename=fname,
+        background=BackgroundTask(os.unlink, tmp.name),
+    )
+
+
+@router.post("/{agent_id}/stacks/{stack_id}/backup/push")
+async def agent_backup_push(
+    agent_id: int,
+    stack_id: str,
+    body: AgentPushBody,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    dest = _get_dest(session, body.destination_id)
+    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
+    tmp.close()
+    try:
+        try:
+            await agent_service.download_to_file(
+                session, agent, f"/agent/stacks/{stack_id}/backup", tmp.name,
+                params={"include_volumes": body.include_volumes, "stop_first": body.stop_first},
+            )
+        except AgentError as exc:
+            _raise(exc)
+        fname = backup_service.backup_filename(
+            stack_id, body.include_volumes, prefix=compose_service.slugify(agent.name)
+        )
+        try:
+            await asyncio.to_thread(dest_service.upload, dest, tmp.name, fname)
+        except dest_service.DestinationError as exc:
+            raise HTTPException(status_code=502, detail=str(exc)) from exc
+    finally:
+        if os.path.exists(tmp.name):
+            os.unlink(tmp.name)
+    audit_service.record(
+        session, user=user.username, action="agent.stack.backup.push",
+        target=f"{agent.name}/{stack_id}", detail=f"{dest.name}:{fname}", ip=_ip(request),
+    )
+    return {"ok": True, "destination": dest.name, "name": fname}
+
+
+@router.post("/{agent_id}/stacks/restore")
+async def agent_restore_upload(
+    agent_id: int,
+    request: Request,
+    file: UploadFile = File(...),
+    target_id: str | None = Form(None),
+    overwrite: bool = Form(False),
+    restore_volumes: bool = Form(True),
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
+    try:
+        while chunk := await file.read(1024 * 1024):
+            tmp.write(chunk)
+        tmp.close()
+        result = await agent_service.upload_file(
+            session, agent, "/agent/stacks/restore", tmp.name,
+            file.filename or "backup.tar.gz",
+            {
+                "target_id": target_id or "",
+                "overwrite": str(overwrite).lower(),
+                "restore_volumes": str(restore_volumes).lower(),
+            },
+        )
+    except AgentError as exc:
+        _raise(exc)
+    finally:
+        if os.path.exists(tmp.name):
+            os.unlink(tmp.name)
+    audit_service.record(
+        session, user=user.username, action="agent.stack.restore",
+        target=f"{agent.name}/{result.get('stack_id')}", ip=_ip(request),
+    )
+    return result
+
+
+@router.post("/{agent_id}/stacks/restore-from")
+async def agent_restore_from(
+    agent_id: int,
+    body: AgentRestoreFromBody,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    dest = _get_dest(session, body.destination_id)
+    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
+    tmp.close()
+    try:
+        try:
+            await asyncio.to_thread(dest_service.download, dest, body.name, tmp.name)
+        except dest_service.DestinationError as exc:
+            raise HTTPException(status_code=502, detail=str(exc)) from exc
+        result = await agent_service.upload_file(
+            session, agent, "/agent/stacks/restore", tmp.name, body.name,
+            {
+                "target_id": body.target_id or "",
+                "overwrite": str(body.overwrite).lower(),
+                "restore_volumes": str(body.restore_volumes).lower(),
+            },
+        )
+    except AgentError as exc:
+        _raise(exc)
+    finally:
+        if os.path.exists(tmp.name):
+            os.unlink(tmp.name)
+    audit_service.record(
+        session, user=user.username, action="agent.stack.restore",
+        target=f"{agent.name}/{result.get('stack_id')}", detail=f"from {dest.name}:{body.name}",
+        ip=_ip(request),
+    )
+    return result

backend/routers/schedules.py

@@ -6,6 +6,7 @@ from sqlmodel import Session, select
 
 from auth import require_admin
 from database import get_session
+from models.agent import Agent
 from models.backup_destination import BackupDestination
 from models.backup_schedule import (
     FREQUENCIES,
@@ -27,11 +28,14 @@ def _ip(request: Request) -> str:
 
 def _to_read(session: Session, s: BackupSchedule) -> ScheduleRead:
     dest = session.get(BackupDestination, s.destination_id)
+    agent = session.get(Agent, s.agent_id) if s.agent_id is not None else None
     return ScheduleRead(
         id=s.id,
         stack_id=s.stack_id,
         destination_id=s.destination_id,
         destination_name=dest.name if dest else None,
+        agent_id=s.agent_id,
+        agent_name=agent.name if agent else None,
         frequency=s.frequency,
         hour=s.hour,
         minute=s.minute,
@@ -54,13 +58,17 @@ def _get_or_404(session: Session, schedule_id: int) -> BackupSchedule:
     return s
 
 
-def _validate(session: Session, stack_id: str, destination_id: int, frequency: str) -> None:
-    if frequency not in FREQUENCIES:
-        raise HTTPException(status_code=400, detail=f"Unknown frequency '{frequency}'")
-    if not session.get(Stack, stack_id):
-        raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
-    if not session.get(BackupDestination, destination_id):
-        raise HTTPException(status_code=404, detail=f"Destination {destination_id} not found")
+def _validate(session: Session, schedule: BackupSchedule) -> None:
+    if schedule.frequency not in FREQUENCIES:
+        raise HTTPException(status_code=400, detail=f"Unknown frequency '{schedule.frequency}'")
+    if not session.get(BackupDestination, schedule.destination_id):
+        raise HTTPException(status_code=404, detail=f"Destination {schedule.destination_id} not found")
+    if schedule.agent_id is not None:
+        # Remote stack: validate the agent exists; the stack is checked at run time.
+        if not session.get(Agent, schedule.agent_id):
+            raise HTTPException(status_code=404, detail=f"Agent {schedule.agent_id} not found")
+    elif not session.get(Stack, schedule.stack_id):
+        raise HTTPException(status_code=404, detail=f"Stack '{schedule.stack_id}' not found")
 
 
 @router.get("", response_model=list[ScheduleRead])
@@ -79,8 +87,8 @@ def create_schedule(
     session: Session = Depends(get_session),
     user: User = Depends(require_admin),
 ) -> ScheduleRead:
-    _validate(session, body.stack_id, body.destination_id, body.frequency)
     s = BackupSchedule(**body.model_dump())
+    _validate(session, s)
     s.next_run = schedule_service.compute_next_run(
         s.frequency, s.hour, s.minute, s.weekday, schedule_service._now()
     )
@@ -106,7 +114,7 @@ def update_schedule(
     data = body.model_dump(exclude_unset=True)
     for k, v in data.items():
         setattr(s, k, v)
-    _validate(session, s.stack_id, s.destination_id, s.frequency)
+    _validate(session, s)
     # Recompute next run when timing fields change.
     if {"frequency", "hour", "minute", "weekday"} & set(data) or s.next_run is None:
         s.next_run = schedule_service.compute_next_run(

backend/services/agent_service.py

@@ -101,6 +101,73 @@ async def call(
     return None
 
 
+def _handle_status(session: Session, agent: Agent, status_code: int, body_text: str = "") -> None:
+    """Update agent status from a response code; raise AgentError on failure."""
+    if status_code in (401, 403):
+        _mark(session, agent, "unauthorized")
+        raise AgentError(status_code, "agent_unauthorized", "Invalid agent token")
+    _mark(session, agent, "online")
+    if status_code >= 400:
+        raise AgentError(status_code, "agent_error", body_text[:500])
+
+
+async def download_to_file(
+    session: Session,
+    agent: Agent,
+    path: str,
+    dest_path: str,
+    *,
+    params: Optional[dict] = None,
+) -> None:
+    """Stream a GET from the agent into ``dest_path``."""
+    url = agent.url.rstrip("/") + path
+    headers = {"Authorization": f"Bearer {agent.token}"}
+    try:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
+            async with client.stream("GET", url, headers=headers, params=params, timeout=None) as resp:
+                if resp.status_code >= 400:
+                    text = (await resp.aread()).decode("utf-8", "replace")
+                    _handle_status(session, agent, resp.status_code, text)
+                _handle_status(session, agent, resp.status_code)
+                with open(dest_path, "wb") as fh:
+                    async for chunk in resp.aiter_bytes(1024 * 256):
+                        fh.write(chunk)
+    except httpx.HTTPError as exc:
+        _mark(session, agent, "offline")
+        raise AgentError(502, "agent_unreachable", str(exc)) from exc
+
+
+async def upload_file(
+    session: Session,
+    agent: Agent,
+    path: str,
+    file_path: str,
+    filename: str,
+    data: dict,
+) -> Any:
+    """Stream a multipart POST (file + form fields) to the agent, returning JSON."""
+    url = agent.url.rstrip("/") + path
+    headers = {"Authorization": f"Bearer {agent.token}"}
+    try:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
+            with open(file_path, "rb") as fh:
+                files = {"file": (filename, fh, "application/gzip")}
+                resp = await client.post(url, headers=headers, files=files, data=data, timeout=None)
+    except httpx.HTTPError as exc:
+        _mark(session, agent, "offline")
+        raise AgentError(502, "agent_unreachable", str(exc)) from exc
+
+    detail = ""
+    if resp.status_code >= 400:
+        try:
+            body = resp.json()
+            detail = body.get("detail") if isinstance(body, dict) else str(body)
+        except ValueError:
+            detail = resp.text[:500]
+    _handle_status(session, agent, resp.status_code, str(detail))
+    return resp.json() if resp.content else None
+
+
 async def ping(session: Session, agent: Agent) -> dict:
     """Health-check an agent and refresh its status + hostname. Never raises."""
     try:

backend/services/backup_service.py

@@ -38,6 +38,26 @@ class BackupError(Exception):
     pass
 
 
+# --------------------------------------------------------------------------- #
+# Backup filename convention
+# --------------------------------------------------------------------------- #
+
+
+def backup_basename(stack_id: str, prefix: Optional[str] = None) -> str:
+    """Filename stem used to group a stack's backups (and match for retention).
+
+    ``prefix`` (e.g. a remote host slug) keeps backups of same-named stacks on
+    different hosts from colliding / pruning each other on a shared destination.
+    """
+    return f"backup-{prefix}-{stack_id}" if prefix else f"backup-{stack_id}"
+
+
+def backup_filename(stack_id: str, include_volumes: bool, prefix: Optional[str] = None) -> str:
+    date = now().strftime("%Y%m%d-%H%M%S")
+    suffix = "full" if include_volumes else "config"
+    return f"{backup_basename(stack_id, prefix)}-{suffix}-{date}.tar.gz"
+
+
 # --------------------------------------------------------------------------- #
 # Helper container for volume I/O
 # --------------------------------------------------------------------------- #

backend/services/schedule_service.py

@@ -11,16 +11,19 @@ from __future__ import annotations
 import asyncio
 import logging
 import os
+import tempfile
 from datetime import datetime, timedelta, timezone
 
 from sqlmodel import Session, select
 
 from database import engine
+from models.agent import Agent
 from models.backup_destination import BackupDestination
 from models.backup_schedule import BackupSchedule
 from models.setting import EVENT_BACKUP_FAILED
 from models.stack import Stack
 from services import (
+    agent_service,
     backup_destination_service as dest_service,
     backup_service,
     compose_service,
@@ -58,17 +61,11 @@ def compute_next_run(
     return base
 
 
-def _backup_filename(stack_id: str, include_volumes: bool) -> str:
-    date = compose_service.now().strftime("%Y%m%d-%H%M%S")
-    suffix = "full" if include_volumes else "config"
-    return f"backup-{stack_id}-{suffix}-{date}.tar.gz"
-
-
-def _prune(dest: BackupDestination, stack_id: str, keep: int) -> int:
+def _prune(dest: BackupDestination, basename: str, keep: int) -> int:
     if keep <= 0:
         return 0
     items = dest_service.list_backups(dest)
-    mine = [i for i in items if i["name"].startswith(f"backup-{stack_id}-")]
+    mine = [i for i in items if i["name"].startswith(f"{basename}-")]
     mine.sort(key=lambda x: x.get("modified") or 0, reverse=True)
     removed = 0
     for old in mine[keep:]:
@@ -83,22 +80,40 @@ def _prune(dest: BackupDestination, stack_id: str, keep: int) -> int:
 async def run_schedule(session: Session, schedule: BackupSchedule) -> dict:
     """Execute one schedule now. Updates status + next_run. Returns a summary."""
     now = _now()
-    stack = session.get(Stack, schedule.stack_id)
     dest = session.get(BackupDestination, schedule.destination_id)
     result: dict = {"ok": False}
     path = None
     try:
-        if not stack:
-            raise RuntimeError(f"stack '{schedule.stack_id}' not found")
         if not dest:
             raise RuntimeError(f"destination {schedule.destination_id} not found")
-        path = await backup_service.create_backup(
-            schedule.stack_id, stack.name,
-            include_volumes=schedule.include_volumes, stop_first=schedule.stop_first,
-        )
-        filename = _backup_filename(schedule.stack_id, schedule.include_volumes)
+
+        # Produce the backup archive — locally or by streaming it from an agent.
+        if schedule.agent_id is not None:
+            agent = session.get(Agent, schedule.agent_id)
+            if not agent:
+                raise RuntimeError(f"agent {schedule.agent_id} not found")
+            prefix = compose_service.slugify(agent.name)
+            tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
+            tmp.close()
+            path = tmp.name
+            await agent_service.download_to_file(
+                session, agent, f"/agent/stacks/{schedule.stack_id}/backup", path,
+                params={"include_volumes": schedule.include_volumes, "stop_first": schedule.stop_first},
+            )
+        else:
+            stack = session.get(Stack, schedule.stack_id)
+            if not stack:
+                raise RuntimeError(f"stack '{schedule.stack_id}' not found")
+            prefix = None
+            path = await backup_service.create_backup(
+                schedule.stack_id, stack.name,
+                include_volumes=schedule.include_volumes, stop_first=schedule.stop_first,
+            )
+
+        filename = backup_service.backup_filename(schedule.stack_id, schedule.include_volumes, prefix=prefix)
+        basename = backup_service.backup_basename(schedule.stack_id, prefix)
         await asyncio.to_thread(dest_service.upload, dest, path, filename)
-        pruned = await asyncio.to_thread(_prune, dest, schedule.stack_id, schedule.keep)
+        pruned = await asyncio.to_thread(_prune, dest, basename, schedule.keep)
         schedule.last_status = "ok"
         result = {"ok": True, "name": filename, "destination": dest.name, "pruned": pruned}
         logger.info("Scheduled backup %s → %s ok (pruned %d)", schedule.stack_id, dest.name, pruned)