Phase 8: back up & restore remote (agent) stacks (0.8.0)
- Agent: GET /agent/stacks/{id}/backup + POST /agent/stacks/restore (reuse
backup_service). backup_service gains backup_basename/backup_filename helpers.
- Main proxy streams agent <-> main <-> destination (creds stay central):
agent_service download_to_file/upload_file; routers/agents.py backup download,
backup/push, restore upload, restore-from.
- Schedules: BackupSchedule.agent_id; schedule_service downloads from the agent
when set; per-host filename prefix isolates retention across hosts.
- Frontend: agents api backup/restore; BackupButton/RestoreButton agent-aware
(Backup on remote stack detail, Restore per host section); schedule form host
selector (local or an online agent) + host shown on schedule rows.
Rough-verified (per request): py_compile, frontend tsc build, image imports
(main 99 / agent 16 routes). Full live agent round-trip to be tested post-deploy.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
84ef3df59e
commit
5cd55382ed
@@ -5,7 +5,8 @@ as intuitive as Dockge, as capable as Portainer for Compose workflows.
|
||||
|
||||
> **Status:** Phase 1 (Core) + Phase 2 (Volumes & GPU) + Phase 3 (Quality of
|
||||
> Life) + Phase 4 (Operations) + Phase 5 (Multi-host) + Phase 6 (Backup
|
||||
> destinations) + Phase 7 (Scheduled backups) complete.
|
||||
> destinations) + Phase 7 (Scheduled backups) + Phase 8 (Remote-stack backups)
|
||||
> complete.
|
||||
|
||||
## What works today (Phase 1)
|
||||
|
||||
@@ -101,6 +102,16 @@ as intuitive as Dockge, as capable as Portainer for Compose workflows.
|
||||
- **Run now** for an on-demand run, plus a `backup_failed` notification event
|
||||
wired into the webhook system.
|
||||
|
||||
### Phase 8 — Remote-stack backups
|
||||
|
||||
- **Back up agent stacks**: the agent exposes its own backup/restore endpoints,
|
||||
and the main app streams a remote stack's backup through to a destination
|
||||
(credentials stay central — agents never see them). Remote stack detail has a
|
||||
**Backup** button; each host section on the Stacks page has a **Restore** button.
|
||||
- **Schedule remote stacks**: a backup schedule can target a remote host; backups
|
||||
are namespaced per host (`backup-<host>-<stack>-…`) so retention never prunes
|
||||
across hosts sharing a destination.
|
||||
|
||||
## Deploying an agent on another host
|
||||
|
||||
```bash
|
||||
@@ -224,6 +235,7 @@ DELETE /api/agents/{id}/stacks/{sid} POST /api/agents/{id}/stacks/
|
||||
|
||||
agent (on the remote host, Bearer AGENT_TOKEN):
|
||||
GET /agent/ping | /system | /stacks | /stacks/{id} | /stacks/{id}/logs
|
||||
GET /agent/stacks/{id}/backup POST /agent/stacks/restore
|
||||
POST /agent/stacks | /stacks/{id}/{action} PUT/DELETE /agent/stacks/{id}
|
||||
```
|
||||
|
||||
@@ -245,6 +257,14 @@ PUT /api/backups/schedules/{id} DELETE /api/backups/schedules
|
||||
POST /api/backups/schedules/{id}/run
|
||||
```
|
||||
|
||||
### Phase 8 endpoints (remote-stack backups)
|
||||
|
||||
```
|
||||
GET /api/agents/{id}/stacks/{sid}/backup POST /api/agents/{id}/stacks/{sid}/backup/push
|
||||
POST /api/agents/{id}/stacks/restore POST /api/agents/{id}/stacks/restore-from
|
||||
backup schedules accept an optional agent_id to target a remote host.
|
||||
```
|
||||
|
||||
## Security notes
|
||||
|
||||
- The Docker socket is only ever touched by the backend process; it is never
|
||||
|
||||
+52
-4
@@ -14,17 +14,19 @@ import logging
|
||||
import os
|
||||
from dataclasses import asdict
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
import tempfile
|
||||
|
||||
from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from config import settings
|
||||
from docker_client import DockerError, get_client, safe_call
|
||||
from services import compose_service
|
||||
from services import backup_service, compose_service
|
||||
|
||||
logger = logging.getLogger("stackpilot.agent")
|
||||
|
||||
AGENT_VERSION = "0.5.0"
|
||||
AGENT_VERSION = "0.8.0"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -226,6 +228,52 @@ async def stack_logs(stack_id: str, tail: int = Query(200, le=2000)) -> dict:
|
||||
return {"logs": result.get("stdout", "") + result.get("stderr", "")}
|
||||
|
||||
|
||||
@app.get("/agent/stacks/{stack_id}/backup", dependencies=[Depends(verify_token)])
|
||||
async def backup_stack(
|
||||
stack_id: str,
|
||||
include_volumes: bool = Query(True),
|
||||
stop_first: bool = Query(True),
|
||||
):
|
||||
if not os.path.isdir(compose_service.stack_dir(stack_id)):
|
||||
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
|
||||
try:
|
||||
path = await backup_service.create_backup(
|
||||
stack_id, stack_id, include_volumes=include_volumes, stop_first=stop_first,
|
||||
)
|
||||
except backup_service.BackupError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type="application/gzip",
|
||||
filename=backup_service.backup_filename(stack_id, include_volumes),
|
||||
)
|
||||
|
||||
|
||||
@app.post("/agent/stacks/restore", dependencies=[Depends(verify_token)])
|
||||
async def restore_stack(
|
||||
file: UploadFile = File(...),
|
||||
target_id: str | None = Form(None),
|
||||
overwrite: bool = Form(False),
|
||||
restore_volumes: bool = Form(True),
|
||||
) -> dict:
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
||||
try:
|
||||
while chunk := await file.read(1024 * 1024):
|
||||
tmp.write(chunk)
|
||||
tmp.close()
|
||||
target = compose_service.slugify(target_id) if target_id else None
|
||||
try:
|
||||
return backup_service.restore_backup(
|
||||
tmp.name, target_id=target, overwrite=overwrite, restore_volumes=restore_volumes,
|
||||
)
|
||||
except backup_service.BackupError as exc:
|
||||
code = 409 if "already exists" in str(exc) else 400
|
||||
raise HTTPException(status_code=code, detail=str(exc)) from exc
|
||||
finally:
|
||||
if os.path.exists(tmp.name):
|
||||
os.unlink(tmp.name)
|
||||
|
||||
|
||||
@app.get("/agent/health")
|
||||
def health() -> dict:
|
||||
return {"status": "ok"}
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ async def lifespan(app: FastAPI):
|
||||
schedule_task.cancel()
|
||||
|
||||
|
||||
app = FastAPI(title="StackPilot", version="0.7.0", lifespan=lifespan)
|
||||
app = FastAPI(title="StackPilot", version="0.8.0", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
|
||||
@@ -19,6 +19,7 @@ class BackupSchedule(SQLModel, table=True):
|
||||
id: Optional[int] = Field(default=None, primary_key=True)
|
||||
stack_id: str
|
||||
destination_id: int
|
||||
agent_id: Optional[int] = None # None = local host; otherwise a remote agent
|
||||
frequency: str = "daily" # one of FREQUENCIES
|
||||
hour: int = 3 # UTC, used for daily/weekly
|
||||
minute: int = 0
|
||||
@@ -39,6 +40,7 @@ class BackupSchedule(SQLModel, table=True):
|
||||
class ScheduleCreate(SQLModel):
|
||||
stack_id: str
|
||||
destination_id: int
|
||||
agent_id: Optional[int] = None
|
||||
frequency: str = "daily"
|
||||
hour: int = 3
|
||||
minute: int = 0
|
||||
@@ -66,6 +68,8 @@ class ScheduleRead(SQLModel):
|
||||
stack_id: str
|
||||
destination_id: int
|
||||
destination_name: Optional[str]
|
||||
agent_id: Optional[int]
|
||||
agent_name: Optional[str]
|
||||
frequency: str
|
||||
hour: int
|
||||
minute: int
|
||||
|
||||
+191
-2
@@ -1,15 +1,29 @@
|
||||
"""Remote host (agent) management + proxied stack/system operations."""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlmodel import Session, select
|
||||
from starlette.background import BackgroundTask
|
||||
|
||||
from auth import get_current_user, require_admin
|
||||
from database import get_session
|
||||
from models.agent import Agent, AgentCreate, AgentRead, AgentUpdate
|
||||
from models.backup_destination import BackupDestination
|
||||
from models.stack import StackCreate, StackUpdate
|
||||
from models.user import User
|
||||
from services import agent_service, audit_service
|
||||
from services import (
|
||||
agent_service,
|
||||
audit_service,
|
||||
backup_destination_service as dest_service,
|
||||
backup_service,
|
||||
compose_service,
|
||||
)
|
||||
from services.agent_service import AgentError
|
||||
|
||||
router = APIRouter(prefix="/api/agents", tags=["agents"])
|
||||
@@ -17,6 +31,27 @@ router = APIRouter(prefix="/api/agents", tags=["agents"])
|
||||
_ACTIONS = {"start", "stop", "restart", "pull", "update", "down"}
|
||||
|
||||
|
||||
class AgentPushBody(BaseModel):
|
||||
destination_id: int
|
||||
include_volumes: bool = True
|
||||
stop_first: bool = True
|
||||
|
||||
|
||||
class AgentRestoreFromBody(BaseModel):
|
||||
destination_id: int
|
||||
name: str
|
||||
target_id: str | None = None
|
||||
overwrite: bool = False
|
||||
restore_volumes: bool = True
|
||||
|
||||
|
||||
def _get_dest(session: Session, dest_id: int) -> BackupDestination:
|
||||
d = session.get(BackupDestination, dest_id)
|
||||
if not d:
|
||||
raise HTTPException(status_code=404, detail=f"Destination {dest_id} not found")
|
||||
return d
|
||||
|
||||
|
||||
def _ip(request: Request) -> str:
|
||||
return request.client.host if request.client else "unknown"
|
||||
|
||||
@@ -284,3 +319,157 @@ async def agent_lifecycle(
|
||||
target=f"{agent.name}/{stack_id}", ip=_ip(request),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Backup / restore of remote stacks (streamed agent <-> main <-> destination)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@router.get("/{agent_id}/stacks/{stack_id}/backup")
|
||||
async def agent_backup_download(
|
||||
agent_id: int,
|
||||
stack_id: str,
|
||||
request: Request,
|
||||
include_volumes: bool = Query(True),
|
||||
stop_first: bool = Query(True),
|
||||
session: Session = Depends(get_session),
|
||||
user: User = Depends(require_admin),
|
||||
):
|
||||
agent = _get_or_404(session, agent_id)
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
||||
tmp.close()
|
||||
try:
|
||||
await agent_service.download_to_file(
|
||||
session, agent, f"/agent/stacks/{stack_id}/backup", tmp.name,
|
||||
params={"include_volumes": include_volumes, "stop_first": stop_first},
|
||||
)
|
||||
except AgentError as exc:
|
||||
if os.path.exists(tmp.name):
|
||||
os.unlink(tmp.name)
|
||||
_raise(exc)
|
||||
audit_service.record(
|
||||
session, user=user.username, action="agent.stack.backup",
|
||||
target=f"{agent.name}/{stack_id}", ip=_ip(request),
|
||||
)
|
||||
fname = backup_service.backup_filename(
|
||||
stack_id, include_volumes, prefix=compose_service.slugify(agent.name)
|
||||
)
|
||||
return FileResponse(
|
||||
tmp.name, media_type="application/gzip", filename=fname,
|
||||
background=BackgroundTask(os.unlink, tmp.name),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{agent_id}/stacks/{stack_id}/backup/push")
|
||||
async def agent_backup_push(
|
||||
agent_id: int,
|
||||
stack_id: str,
|
||||
body: AgentPushBody,
|
||||
request: Request,
|
||||
session: Session = Depends(get_session),
|
||||
user: User = Depends(require_admin),
|
||||
) -> dict:
|
||||
agent = _get_or_404(session, agent_id)
|
||||
dest = _get_dest(session, body.destination_id)
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
||||
tmp.close()
|
||||
try:
|
||||
try:
|
||||
await agent_service.download_to_file(
|
||||
session, agent, f"/agent/stacks/{stack_id}/backup", tmp.name,
|
||||
params={"include_volumes": body.include_volumes, "stop_first": body.stop_first},
|
||||
)
|
||||
except AgentError as exc:
|
||||
_raise(exc)
|
||||
fname = backup_service.backup_filename(
|
||||
stack_id, body.include_volumes, prefix=compose_service.slugify(agent.name)
|
||||
)
|
||||
try:
|
||||
await asyncio.to_thread(dest_service.upload, dest, tmp.name, fname)
|
||||
except dest_service.DestinationError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
finally:
|
||||
if os.path.exists(tmp.name):
|
||||
os.unlink(tmp.name)
|
||||
audit_service.record(
|
||||
session, user=user.username, action="agent.stack.backup.push",
|
||||
target=f"{agent.name}/{stack_id}", detail=f"{dest.name}:{fname}", ip=_ip(request),
|
||||
)
|
||||
return {"ok": True, "destination": dest.name, "name": fname}
|
||||
|
||||
|
||||
@router.post("/{agent_id}/stacks/restore")
|
||||
async def agent_restore_upload(
|
||||
agent_id: int,
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
target_id: str | None = Form(None),
|
||||
overwrite: bool = Form(False),
|
||||
restore_volumes: bool = Form(True),
|
||||
session: Session = Depends(get_session),
|
||||
user: User = Depends(require_admin),
|
||||
) -> dict:
|
||||
agent = _get_or_404(session, agent_id)
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
||||
try:
|
||||
while chunk := await file.read(1024 * 1024):
|
||||
tmp.write(chunk)
|
||||
tmp.close()
|
||||
result = await agent_service.upload_file(
|
||||
session, agent, "/agent/stacks/restore", tmp.name,
|
||||
file.filename or "backup.tar.gz",
|
||||
{
|
||||
"target_id": target_id or "",
|
||||
"overwrite": str(overwrite).lower(),
|
||||
"restore_volumes": str(restore_volumes).lower(),
|
||||
},
|
||||
)
|
||||
except AgentError as exc:
|
||||
_raise(exc)
|
||||
finally:
|
||||
if os.path.exists(tmp.name):
|
||||
os.unlink(tmp.name)
|
||||
audit_service.record(
|
||||
session, user=user.username, action="agent.stack.restore",
|
||||
target=f"{agent.name}/{result.get('stack_id')}", ip=_ip(request),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/{agent_id}/stacks/restore-from")
|
||||
async def agent_restore_from(
|
||||
agent_id: int,
|
||||
body: AgentRestoreFromBody,
|
||||
request: Request,
|
||||
session: Session = Depends(get_session),
|
||||
user: User = Depends(require_admin),
|
||||
) -> dict:
|
||||
agent = _get_or_404(session, agent_id)
|
||||
dest = _get_dest(session, body.destination_id)
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
||||
tmp.close()
|
||||
try:
|
||||
try:
|
||||
await asyncio.to_thread(dest_service.download, dest, body.name, tmp.name)
|
||||
except dest_service.DestinationError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
result = await agent_service.upload_file(
|
||||
session, agent, "/agent/stacks/restore", tmp.name, body.name,
|
||||
{
|
||||
"target_id": body.target_id or "",
|
||||
"overwrite": str(body.overwrite).lower(),
|
||||
"restore_volumes": str(body.restore_volumes).lower(),
|
||||
},
|
||||
)
|
||||
except AgentError as exc:
|
||||
_raise(exc)
|
||||
finally:
|
||||
if os.path.exists(tmp.name):
|
||||
os.unlink(tmp.name)
|
||||
audit_service.record(
|
||||
session, user=user.username, action="agent.stack.restore",
|
||||
target=f"{agent.name}/{result.get('stack_id')}", detail=f"from {dest.name}:{body.name}",
|
||||
ip=_ip(request),
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -6,6 +6,7 @@ from sqlmodel import Session, select
|
||||
|
||||
from auth import require_admin
|
||||
from database import get_session
|
||||
from models.agent import Agent
|
||||
from models.backup_destination import BackupDestination
|
||||
from models.backup_schedule import (
|
||||
FREQUENCIES,
|
||||
@@ -27,11 +28,14 @@ def _ip(request: Request) -> str:
|
||||
|
||||
def _to_read(session: Session, s: BackupSchedule) -> ScheduleRead:
|
||||
dest = session.get(BackupDestination, s.destination_id)
|
||||
agent = session.get(Agent, s.agent_id) if s.agent_id is not None else None
|
||||
return ScheduleRead(
|
||||
id=s.id,
|
||||
stack_id=s.stack_id,
|
||||
destination_id=s.destination_id,
|
||||
destination_name=dest.name if dest else None,
|
||||
agent_id=s.agent_id,
|
||||
agent_name=agent.name if agent else None,
|
||||
frequency=s.frequency,
|
||||
hour=s.hour,
|
||||
minute=s.minute,
|
||||
@@ -54,13 +58,17 @@ def _get_or_404(session: Session, schedule_id: int) -> BackupSchedule:
|
||||
return s
|
||||
|
||||
|
||||
def _validate(session: Session, stack_id: str, destination_id: int, frequency: str) -> None:
|
||||
if frequency not in FREQUENCIES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown frequency '{frequency}'")
|
||||
if not session.get(Stack, stack_id):
|
||||
raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
|
||||
if not session.get(BackupDestination, destination_id):
|
||||
raise HTTPException(status_code=404, detail=f"Destination {destination_id} not found")
|
||||
def _validate(session: Session, schedule: BackupSchedule) -> None:
|
||||
if schedule.frequency not in FREQUENCIES:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown frequency '{schedule.frequency}'")
|
||||
if not session.get(BackupDestination, schedule.destination_id):
|
||||
raise HTTPException(status_code=404, detail=f"Destination {schedule.destination_id} not found")
|
||||
if schedule.agent_id is not None:
|
||||
# Remote stack: validate the agent exists; the stack is checked at run time.
|
||||
if not session.get(Agent, schedule.agent_id):
|
||||
raise HTTPException(status_code=404, detail=f"Agent {schedule.agent_id} not found")
|
||||
elif not session.get(Stack, schedule.stack_id):
|
||||
raise HTTPException(status_code=404, detail=f"Stack '{schedule.stack_id}' not found")
|
||||
|
||||
|
||||
@router.get("", response_model=list[ScheduleRead])
|
||||
@@ -79,8 +87,8 @@ def create_schedule(
|
||||
session: Session = Depends(get_session),
|
||||
user: User = Depends(require_admin),
|
||||
) -> ScheduleRead:
|
||||
_validate(session, body.stack_id, body.destination_id, body.frequency)
|
||||
s = BackupSchedule(**body.model_dump())
|
||||
_validate(session, s)
|
||||
s.next_run = schedule_service.compute_next_run(
|
||||
s.frequency, s.hour, s.minute, s.weekday, schedule_service._now()
|
||||
)
|
||||
@@ -106,7 +114,7 @@ def update_schedule(
|
||||
data = body.model_dump(exclude_unset=True)
|
||||
for k, v in data.items():
|
||||
setattr(s, k, v)
|
||||
_validate(session, s.stack_id, s.destination_id, s.frequency)
|
||||
_validate(session, s)
|
||||
# Recompute next run when timing fields change.
|
||||
if {"frequency", "hour", "minute", "weekday"} & set(data) or s.next_run is None:
|
||||
s.next_run = schedule_service.compute_next_run(
|
||||
|
||||
@@ -101,6 +101,73 @@ async def call(
|
||||
return None
|
||||
|
||||
|
||||
def _handle_status(session: Session, agent: Agent, status_code: int, body_text: str = "") -> None:
|
||||
"""Update agent status from a response code; raise AgentError on failure."""
|
||||
if status_code in (401, 403):
|
||||
_mark(session, agent, "unauthorized")
|
||||
raise AgentError(status_code, "agent_unauthorized", "Invalid agent token")
|
||||
_mark(session, agent, "online")
|
||||
if status_code >= 400:
|
||||
raise AgentError(status_code, "agent_error", body_text[:500])
|
||||
|
||||
|
||||
async def download_to_file(
|
||||
session: Session,
|
||||
agent: Agent,
|
||||
path: str,
|
||||
dest_path: str,
|
||||
*,
|
||||
params: Optional[dict] = None,
|
||||
) -> None:
|
||||
"""Stream a GET from the agent into ``dest_path``."""
|
||||
url = agent.url.rstrip("/") + path
|
||||
headers = {"Authorization": f"Bearer {agent.token}"}
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
async with client.stream("GET", url, headers=headers, params=params, timeout=None) as resp:
|
||||
if resp.status_code >= 400:
|
||||
text = (await resp.aread()).decode("utf-8", "replace")
|
||||
_handle_status(session, agent, resp.status_code, text)
|
||||
_handle_status(session, agent, resp.status_code)
|
||||
with open(dest_path, "wb") as fh:
|
||||
async for chunk in resp.aiter_bytes(1024 * 256):
|
||||
fh.write(chunk)
|
||||
except httpx.HTTPError as exc:
|
||||
_mark(session, agent, "offline")
|
||||
raise AgentError(502, "agent_unreachable", str(exc)) from exc
|
||||
|
||||
|
||||
async def upload_file(
|
||||
session: Session,
|
||||
agent: Agent,
|
||||
path: str,
|
||||
file_path: str,
|
||||
filename: str,
|
||||
data: dict,
|
||||
) -> Any:
|
||||
"""Stream a multipart POST (file + form fields) to the agent, returning JSON."""
|
||||
url = agent.url.rstrip("/") + path
|
||||
headers = {"Authorization": f"Bearer {agent.token}"}
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
with open(file_path, "rb") as fh:
|
||||
files = {"file": (filename, fh, "application/gzip")}
|
||||
resp = await client.post(url, headers=headers, files=files, data=data, timeout=None)
|
||||
except httpx.HTTPError as exc:
|
||||
_mark(session, agent, "offline")
|
||||
raise AgentError(502, "agent_unreachable", str(exc)) from exc
|
||||
|
||||
detail = ""
|
||||
if resp.status_code >= 400:
|
||||
try:
|
||||
body = resp.json()
|
||||
detail = body.get("detail") if isinstance(body, dict) else str(body)
|
||||
except ValueError:
|
||||
detail = resp.text[:500]
|
||||
_handle_status(session, agent, resp.status_code, str(detail))
|
||||
return resp.json() if resp.content else None
|
||||
|
||||
|
||||
async def ping(session: Session, agent: Agent) -> dict:
|
||||
"""Health-check an agent and refresh its status + hostname. Never raises."""
|
||||
try:
|
||||
|
||||
@@ -38,6 +38,26 @@ class BackupError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Backup filename convention
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def backup_basename(stack_id: str, prefix: Optional[str] = None) -> str:
|
||||
"""Filename stem used to group a stack's backups (and match for retention).
|
||||
|
||||
``prefix`` (e.g. a remote host slug) keeps backups of same-named stacks on
|
||||
different hosts from colliding / pruning each other on a shared destination.
|
||||
"""
|
||||
return f"backup-{prefix}-{stack_id}" if prefix else f"backup-{stack_id}"
|
||||
|
||||
|
||||
def backup_filename(stack_id: str, include_volumes: bool, prefix: Optional[str] = None) -> str:
|
||||
date = now().strftime("%Y%m%d-%H%M%S")
|
||||
suffix = "full" if include_volumes else "config"
|
||||
return f"{backup_basename(stack_id, prefix)}-{suffix}-{date}.tar.gz"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Helper container for volume I/O
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
@@ -11,16 +11,19 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from database import engine
|
||||
from models.agent import Agent
|
||||
from models.backup_destination import BackupDestination
|
||||
from models.backup_schedule import BackupSchedule
|
||||
from models.setting import EVENT_BACKUP_FAILED
|
||||
from models.stack import Stack
|
||||
from services import (
|
||||
agent_service,
|
||||
backup_destination_service as dest_service,
|
||||
backup_service,
|
||||
compose_service,
|
||||
@@ -58,17 +61,11 @@ def compute_next_run(
|
||||
return base
|
||||
|
||||
|
||||
def _backup_filename(stack_id: str, include_volumes: bool) -> str:
|
||||
date = compose_service.now().strftime("%Y%m%d-%H%M%S")
|
||||
suffix = "full" if include_volumes else "config"
|
||||
return f"backup-{stack_id}-{suffix}-{date}.tar.gz"
|
||||
|
||||
|
||||
def _prune(dest: BackupDestination, stack_id: str, keep: int) -> int:
|
||||
def _prune(dest: BackupDestination, basename: str, keep: int) -> int:
|
||||
if keep <= 0:
|
||||
return 0
|
||||
items = dest_service.list_backups(dest)
|
||||
mine = [i for i in items if i["name"].startswith(f"backup-{stack_id}-")]
|
||||
mine = [i for i in items if i["name"].startswith(f"{basename}-")]
|
||||
mine.sort(key=lambda x: x.get("modified") or 0, reverse=True)
|
||||
removed = 0
|
||||
for old in mine[keep:]:
|
||||
@@ -83,22 +80,40 @@ def _prune(dest: BackupDestination, stack_id: str, keep: int) -> int:
|
||||
async def run_schedule(session: Session, schedule: BackupSchedule) -> dict:
|
||||
"""Execute one schedule now. Updates status + next_run. Returns a summary."""
|
||||
now = _now()
|
||||
stack = session.get(Stack, schedule.stack_id)
|
||||
dest = session.get(BackupDestination, schedule.destination_id)
|
||||
result: dict = {"ok": False}
|
||||
path = None
|
||||
try:
|
||||
if not stack:
|
||||
raise RuntimeError(f"stack '{schedule.stack_id}' not found")
|
||||
if not dest:
|
||||
raise RuntimeError(f"destination {schedule.destination_id} not found")
|
||||
path = await backup_service.create_backup(
|
||||
schedule.stack_id, stack.name,
|
||||
include_volumes=schedule.include_volumes, stop_first=schedule.stop_first,
|
||||
)
|
||||
filename = _backup_filename(schedule.stack_id, schedule.include_volumes)
|
||||
|
||||
# Produce the backup archive — locally or by streaming it from an agent.
|
||||
if schedule.agent_id is not None:
|
||||
agent = session.get(Agent, schedule.agent_id)
|
||||
if not agent:
|
||||
raise RuntimeError(f"agent {schedule.agent_id} not found")
|
||||
prefix = compose_service.slugify(agent.name)
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz")
|
||||
tmp.close()
|
||||
path = tmp.name
|
||||
await agent_service.download_to_file(
|
||||
session, agent, f"/agent/stacks/{schedule.stack_id}/backup", path,
|
||||
params={"include_volumes": schedule.include_volumes, "stop_first": schedule.stop_first},
|
||||
)
|
||||
else:
|
||||
stack = session.get(Stack, schedule.stack_id)
|
||||
if not stack:
|
||||
raise RuntimeError(f"stack '{schedule.stack_id}' not found")
|
||||
prefix = None
|
||||
path = await backup_service.create_backup(
|
||||
schedule.stack_id, stack.name,
|
||||
include_volumes=schedule.include_volumes, stop_first=schedule.stop_first,
|
||||
)
|
||||
|
||||
filename = backup_service.backup_filename(schedule.stack_id, schedule.include_volumes, prefix=prefix)
|
||||
basename = backup_service.backup_basename(schedule.stack_id, prefix)
|
||||
await asyncio.to_thread(dest_service.upload, dest, path, filename)
|
||||
pruned = await asyncio.to_thread(_prune, dest, schedule.stack_id, schedule.keep)
|
||||
pruned = await asyncio.to_thread(_prune, dest, basename, schedule.keep)
|
||||
schedule.last_status = "ok"
|
||||
result = {"ok": True, "name": filename, "destination": dest.name, "pruned": pruned}
|
||||
logger.info("Scheduled backup %s → %s ok (pruned %d)", schedule.stack_id, dest.name, pruned)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "stackpilot-frontend",
|
||||
"private": true,
|
||||
"version": "0.7.0",
|
||||
"version": "0.8.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
|
||||
@@ -27,4 +27,69 @@ export const agentsApi = {
|
||||
api.put(`/api/agents/${id}/stacks/${stackId}`, body).then((r) => r.data),
|
||||
action: (id: number, stackId: string, action: string) =>
|
||||
api.post(`/api/agents/${id}/stacks/${stackId}/${action}`).then((r) => r.data),
|
||||
|
||||
backupDownload: async (
|
||||
id: number,
|
||||
stackId: string,
|
||||
opts: { includeVolumes: boolean; stopFirst: boolean }
|
||||
) => {
|
||||
const res = await api.get(`/api/agents/${id}/stacks/${stackId}/backup`, {
|
||||
params: { include_volumes: opts.includeVolumes, stop_first: opts.stopFirst },
|
||||
responseType: "blob",
|
||||
});
|
||||
const cd = res.headers["content-disposition"] as string | undefined;
|
||||
const name = cd?.match(/filename="?([^"]+)"?/)?.[1] ?? `backup-${stackId}.tar.gz`;
|
||||
const url = URL.createObjectURL(res.data as Blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = name;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
URL.revokeObjectURL(url);
|
||||
},
|
||||
backupPush: (
|
||||
id: number,
|
||||
stackId: string,
|
||||
body: { destination_id: number; include_volumes: boolean; stop_first: boolean }
|
||||
) =>
|
||||
api
|
||||
.post<{ ok: boolean; destination: string; name: string }>(
|
||||
`/api/agents/${id}/stacks/${stackId}/backup/push`,
|
||||
body
|
||||
)
|
||||
.then((r) => r.data),
|
||||
restoreUpload: (
|
||||
id: number,
|
||||
file: File,
|
||||
opts: { targetId?: string; overwrite: boolean; restoreVolumes: boolean }
|
||||
) => {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
if (opts.targetId) form.append("target_id", opts.targetId);
|
||||
form.append("overwrite", String(opts.overwrite));
|
||||
form.append("restore_volumes", String(opts.restoreVolumes));
|
||||
return api
|
||||
.post<{ stack_id: string; name: string; volumes_restored: number }>(
|
||||
`/api/agents/${id}/stacks/restore`,
|
||||
form
|
||||
)
|
||||
.then((r) => r.data);
|
||||
},
|
||||
restoreFrom: (
|
||||
id: number,
|
||||
body: {
|
||||
destination_id: number;
|
||||
name: string;
|
||||
target_id?: string;
|
||||
overwrite: boolean;
|
||||
restore_volumes: boolean;
|
||||
}
|
||||
) =>
|
||||
api
|
||||
.post<{ stack_id: string; name: string; volumes_restored: number }>(
|
||||
`/api/agents/${id}/stacks/restore-from`,
|
||||
body
|
||||
)
|
||||
.then((r) => r.data),
|
||||
};
|
||||
|
||||
@@ -5,6 +5,8 @@ export interface BackupSchedule {
|
||||
stack_id: string;
|
||||
destination_id: number;
|
||||
destination_name: string | null;
|
||||
agent_id: number | null;
|
||||
agent_name: string | null;
|
||||
frequency: "hourly" | "daily" | "weekly";
|
||||
hour: number;
|
||||
minute: number;
|
||||
@@ -22,6 +24,7 @@ export interface BackupSchedule {
|
||||
export interface ScheduleInput {
|
||||
stack_id: string;
|
||||
destination_id: number;
|
||||
agent_id?: number | null;
|
||||
frequency: string;
|
||||
hour: number;
|
||||
minute: number;
|
||||
|
||||
@@ -4,6 +4,7 @@ import { Server } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { Card } from "@/components/ui";
|
||||
import { StackCard } from "@/components/stacks/StackCard";
|
||||
import { RestoreButton } from "@/components/stacks/BackupRestore";
|
||||
import { HostDot } from "@/components/hosts/HostDot";
|
||||
import { agentsApi } from "@/api/agents";
|
||||
import { apiErrorMessage } from "@/api/client";
|
||||
@@ -37,14 +38,17 @@ export function AgentStacksSection({ agent, isAdmin }: { agent: Agent; isAdmin:
|
||||
|
||||
return (
|
||||
<section>
|
||||
<h2 className="mb-3 flex items-center gap-2 text-sm font-semibold uppercase tracking-wide text-slate-500">
|
||||
<Server className="h-4 w-4" />
|
||||
{agent.name}
|
||||
<HostDot status={agent.status} />
|
||||
{agent.hostname && (
|
||||
<span className="font-mono text-xs normal-case text-slate-400">{agent.hostname}</span>
|
||||
)}
|
||||
</h2>
|
||||
<div className="mb-3 flex items-center justify-between gap-2">
|
||||
<h2 className="flex items-center gap-2 text-sm font-semibold uppercase tracking-wide text-slate-500">
|
||||
<Server className="h-4 w-4" />
|
||||
{agent.name}
|
||||
<HostDot status={agent.status} />
|
||||
{agent.hostname && (
|
||||
<span className="font-mono text-xs normal-case text-slate-400">{agent.hostname}</span>
|
||||
)}
|
||||
</h2>
|
||||
{isAdmin && online && <RestoreButton agentId={agent.id} />}
|
||||
</div>
|
||||
|
||||
{!online ? (
|
||||
<Card>
|
||||
|
||||
@@ -4,6 +4,7 @@ import { Archive, Upload } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { Button } from "@/components/ui";
|
||||
import { backupsApi, destinationsApi } from "@/api/backups";
|
||||
import { agentsApi } from "@/api/agents";
|
||||
import { apiErrorMessage } from "@/api/client";
|
||||
import { formatBytes } from "@/lib/utils";
|
||||
|
||||
@@ -50,7 +51,13 @@ function Modal({ children, onClose }: { children: React.ReactNode; onClose: () =
|
||||
);
|
||||
}
|
||||
|
||||
export function BackupButton({ stackId }: { stackId: string }) {
|
||||
export function BackupButton({
|
||||
stackId,
|
||||
agentId,
|
||||
}: {
|
||||
stackId: string;
|
||||
agentId?: number;
|
||||
}) {
|
||||
const [open, setOpen] = useState(false);
|
||||
const [includeVolumes, setIncludeVolumes] = useState(true);
|
||||
const [stopFirst, setStopFirst] = useState(true);
|
||||
@@ -68,14 +75,22 @@ export function BackupButton({ stackId }: { stackId: string }) {
|
||||
const tid = toast.loading("Creating backup…");
|
||||
try {
|
||||
if (target === "download") {
|
||||
await backupsApi.download(stackId, { includeVolumes, stopFirst });
|
||||
if (agentId != null) {
|
||||
await agentsApi.backupDownload(agentId, stackId, { includeVolumes, stopFirst });
|
||||
} else {
|
||||
await backupsApi.download(stackId, { includeVolumes, stopFirst });
|
||||
}
|
||||
toast.success("Backup downloaded", { id: tid });
|
||||
} else {
|
||||
const res = await backupsApi.push(stackId, {
|
||||
const body = {
|
||||
destination_id: Number(target),
|
||||
include_volumes: includeVolumes,
|
||||
stop_first: stopFirst,
|
||||
});
|
||||
};
|
||||
const res =
|
||||
agentId != null
|
||||
? await agentsApi.backupPush(agentId, stackId, body)
|
||||
: await backupsApi.push(stackId, body);
|
||||
toast.success(`Backup pushed to ${res.destination}`, { id: tid });
|
||||
}
|
||||
setOpen(false);
|
||||
@@ -133,7 +148,7 @@ export function BackupButton({ stackId }: { stackId: string }) {
|
||||
);
|
||||
}
|
||||
|
||||
export function RestoreButton() {
|
||||
export function RestoreButton({ agentId }: { agentId?: number }) {
|
||||
const qc = useQueryClient();
|
||||
const [open, setOpen] = useState(false);
|
||||
const [mode, setMode] = useState<"upload" | "destination">("upload");
|
||||
@@ -167,27 +182,31 @@ export function RestoreButton() {
|
||||
setBusy(false);
|
||||
return;
|
||||
}
|
||||
res = await backupsApi.restore(file, {
|
||||
targetId: targetId.trim() || undefined,
|
||||
overwrite,
|
||||
restoreVolumes,
|
||||
});
|
||||
const opts = { targetId: targetId.trim() || undefined, overwrite, restoreVolumes };
|
||||
res =
|
||||
agentId != null
|
||||
? await agentsApi.restoreUpload(agentId, file, opts)
|
||||
: await backupsApi.restore(file, opts);
|
||||
} else {
|
||||
if (!destId || !remoteName) {
|
||||
toast.error("Pick a destination and a backup", { id: tid });
|
||||
setBusy(false);
|
||||
return;
|
||||
}
|
||||
res = await backupsApi.restoreFrom({
|
||||
const body = {
|
||||
destination_id: Number(destId),
|
||||
name: remoteName,
|
||||
target_id: targetId.trim() || undefined,
|
||||
overwrite,
|
||||
restore_volumes: restoreVolumes,
|
||||
});
|
||||
};
|
||||
res =
|
||||
agentId != null
|
||||
? await agentsApi.restoreFrom(agentId, body)
|
||||
: await backupsApi.restoreFrom(body);
|
||||
}
|
||||
toast.success(`Restored '${res.stack_id}' (${res.volumes_restored} volume(s))`, { id: tid });
|
||||
qc.invalidateQueries({ queryKey: ["stacks"] });
|
||||
qc.invalidateQueries({ queryKey: agentId != null ? ["agent-stacks", agentId] : ["stacks"] });
|
||||
setOpen(false);
|
||||
setFile(null);
|
||||
setTargetId("");
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
import { toast } from "sonner";
|
||||
import { Badge, Button, Card, Spinner, StatusDot } from "@/components/ui";
|
||||
import { HostDot } from "@/components/hosts/HostDot";
|
||||
import { BackupButton } from "@/components/stacks/BackupRestore";
|
||||
import { agentsApi } from "@/api/agents";
|
||||
import { apiErrorMessage } from "@/api/client";
|
||||
import { useAuthStore } from "@/store/auth";
|
||||
@@ -92,6 +93,7 @@ export function RemoteStackDetail() {
|
||||
<Button variant="outline" onClick={() => run("down", "Tearing down")} loading={busy}>
|
||||
<Power className="h-4 w-4" /> Down
|
||||
</Button>
|
||||
<BackupButton stackId={id} agentId={aid} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -78,6 +78,7 @@ function SchedulesSection() {
|
||||
const { data, isLoading } = useQuery({ queryKey: ["schedules"], queryFn: schedulesApi.list });
|
||||
const destinations = useQuery({ queryKey: ["destinations"], queryFn: destinationsApi.list });
|
||||
const stacks = useQuery({ queryKey: ["stacks"], queryFn: stacksApi.list });
|
||||
const agents = useQuery({ queryKey: ["agents"], queryFn: () => agentsApi.list() });
|
||||
const [adding, setAdding] = useState(false);
|
||||
const invalidate = () => qc.invalidateQueries({ queryKey: ["schedules"] });
|
||||
const noDest = (destinations.data?.length ?? 0) === 0;
|
||||
@@ -103,6 +104,7 @@ function SchedulesSection() {
|
||||
<ScheduleForm
|
||||
stacks={stacks.data ?? []}
|
||||
destinations={destinations.data ?? []}
|
||||
agents={agents.data ?? []}
|
||||
onDone={() => { setAdding(false); invalidate(); }}
|
||||
onCancel={() => setAdding(false)}
|
||||
/>
|
||||
@@ -143,6 +145,7 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang
|
||||
<Card className="space-y-2">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<div className="flex items-center gap-2">
|
||||
{schedule.agent_name && <Badge>{schedule.agent_name}</Badge>}
|
||||
<span className="font-mono text-sm font-medium">{schedule.stack_id}</span>
|
||||
<span className="text-slate-400">→</span>
|
||||
<Badge>{schedule.destination_name ?? `dest ${schedule.destination_id}`}</Badge>
|
||||
@@ -178,14 +181,17 @@ function ScheduleRow({ schedule, onChange }: { schedule: BackupSchedule; onChang
|
||||
function ScheduleForm({
|
||||
stacks,
|
||||
destinations,
|
||||
agents,
|
||||
onDone,
|
||||
onCancel,
|
||||
}: {
|
||||
stacks: { id: string; name: string }[];
|
||||
destinations: BackupDestination[];
|
||||
agents: Agent[];
|
||||
onDone: () => void;
|
||||
onCancel: () => void;
|
||||
}) {
|
||||
const [host, setHost] = useState("local"); // "local" | agent id (string)
|
||||
const [form, setForm] = useState({
|
||||
stack_id: stacks[0]?.id ?? "",
|
||||
destination_id: destinations[0]?.id ?? 0,
|
||||
@@ -200,19 +206,51 @@ function ScheduleForm({
|
||||
});
|
||||
const set = (k: string, v: unknown) => setForm((f) => ({ ...f, [k]: v }));
|
||||
|
||||
const isRemote = host !== "local";
|
||||
const agentId = isRemote ? Number(host) : undefined;
|
||||
|
||||
// When a remote host is selected, pull its stacks for the picker.
|
||||
const remoteStacks = useQuery({
|
||||
queryKey: ["agent-stacks", agentId],
|
||||
queryFn: () => agentsApi.stacks(agentId!),
|
||||
enabled: isRemote,
|
||||
});
|
||||
const stackOptions = isRemote
|
||||
? (remoteStacks.data ?? []).map((s) => ({ id: s.id, name: s.name }))
|
||||
: stacks;
|
||||
|
||||
// Keep stack_id valid as host/options change.
|
||||
useEffect(() => {
|
||||
if (stackOptions.length && !stackOptions.some((s) => s.id === form.stack_id)) {
|
||||
set("stack_id", stackOptions[0].id);
|
||||
}
|
||||
}, [stackOptions]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
const onlineAgents = agents.filter((a) => a.status === "online");
|
||||
|
||||
const create = useMutation({
|
||||
mutationFn: () => schedulesApi.create(form),
|
||||
mutationFn: () => schedulesApi.create({ ...form, agent_id: agentId ?? null }),
|
||||
onSuccess: () => { toast.success("Schedule added"); onDone(); },
|
||||
onError: (e) => toast.error(apiErrorMessage(e)),
|
||||
});
|
||||
|
||||
return (
|
||||
<Card className="space-y-3">
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
<div className="grid gap-3 sm:grid-cols-3">
|
||||
<label className="space-y-1">
|
||||
<span className="text-xs font-medium text-slate-500">Host</span>
|
||||
<select className={selectClass} value={host} onChange={(e) => setHost(e.target.value)}>
|
||||
<option value="local">This host</option>
|
||||
{onlineAgents.map((a) => (
|
||||
<option key={a.id} value={String(a.id)}>{a.name}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label className="space-y-1">
|
||||
<span className="text-xs font-medium text-slate-500">Stack</span>
|
||||
<select className={selectClass} value={form.stack_id} onChange={(e) => set("stack_id", e.target.value)}>
|
||||
{stacks.map((s) => (
|
||||
{stackOptions.length === 0 && <option value="">{isRemote ? "no stacks" : "—"}</option>}
|
||||
{stackOptions.map((s) => (
|
||||
<option key={s.id} value={s.id}>{s.name}</option>
|
||||
))}
|
||||
</select>
|
||||
|
||||
Reference in New Issue
Block a user