Phase 5: multi-host agents (0.5.0)

- stackpilot-agent: slim token-guarded FastAPI (reuses compose_service) exposing
  stack CRUD/lifecycle/logs + system info; same image, different CMD. agent/
  Dockerfile + compose + .env.example.
- Central proxy: Agent model, agent_service (httpx ping/proxy + live status:
  online/offline/unauthorized + hostname/last_seen), routers/agents.py
  (CRUD + ping + proxied stacks/lifecycle/logs/system).
- Frontend: Settings → Remote hosts (add/check/remove, connectivity dot); Stacks
  grouped by host; remote stack detail with lifecycle, live logs, compose/.env edit.

Verified end-to-end: agent+main on a shared network — register (good/bad token),
list/create/start/logs/delete remote stacks, offline detection (502).

Remote backup destinations (SFTP/S3) deferred.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/agent_app.py

@@ -0,0 +1,231 @@
+"""StackPilot agent — a slim, token-guarded Docker Compose API for one host.
+
+The agent runs on each remote host (same image as the backend, different CMD).
+It has no users, no database and no UI: it exposes just enough of the stack /
+system surface for a central StackPilot to manage this host's compose stacks,
+authenticated by a single shared bearer token (``AGENT_TOKEN``).
+
+All compose/Docker logic is reused from the backend's ``compose_service`` and
+``docker_client`` so behaviour matches the local host exactly.
+"""
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import asdict
+
+from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+
+from config import settings
+from docker_client import DockerError, get_client, safe_call
+from services import compose_service
+
+logger = logging.getLogger("stackpilot.agent")
+
+AGENT_VERSION = "0.5.0"
+
+
+# --------------------------------------------------------------------------- #
+# Auth
+# --------------------------------------------------------------------------- #
+
+
+def verify_token(authorization: str = Header(default="")) -> None:
+    expected = settings.AGENT_TOKEN
+    if not expected:
+        raise HTTPException(status_code=503, detail="Agent token not configured")
+    if authorization != f"Bearer {expected}":
+        raise HTTPException(status_code=401, detail="Invalid agent token")
+
+
+# --------------------------------------------------------------------------- #
+# Schemas
+# --------------------------------------------------------------------------- #
+
+
+class StackBody(BaseModel):
+    name: str | None = None
+    yaml: str | None = None
+    env: str | None = None
+
+
+# --------------------------------------------------------------------------- #
+# Helpers
+# --------------------------------------------------------------------------- #
+
+
+def _summary(stack_id: str) -> dict:
+    try:
+        containers = compose_service.containers_for_stack(stack_id)
+        status = compose_service.compute_status(stack_id)
+    except DockerError:
+        containers = []
+        status = "unknown"
+    return {
+        "id": stack_id,
+        "name": stack_id,
+        "description": None,
+        "status": status,
+        "service_count": len(containers),
+        "running_count": sum(1 for c in containers if c.state == "running"),
+        "created_at": None,
+        "updated_at": None,
+    }
+
+
+def _hostname() -> str:
+    return os.uname().nodename
+
+
+def _system_info() -> dict:
+    docker_version = ""
+    host_os = ""
+    running = total = 0
+    try:
+        client = get_client()
+        docker_version = safe_call(client.version).get("Version", "")
+        info = safe_call(client.info)
+        host_os = info.get("OperatingSystem", "")
+        running = info.get("ContainersRunning", 0)
+        total = info.get("Containers", 0)
+    except DockerError as exc:
+        docker_version = f"unavailable ({exc.error})"
+    return {
+        "hostname": _hostname(),
+        "docker_version": docker_version,
+        "host_os": host_os,
+        "containers_running": running,
+        "containers_total": total,
+    }
+
+
+# --------------------------------------------------------------------------- #
+# App
+# --------------------------------------------------------------------------- #
+
+app = FastAPI(title="StackPilot Agent", version=AGENT_VERSION)
+
+
+@app.exception_handler(DockerError)
+async def _docker_error(_request: Request, exc: DockerError):
+    return JSONResponse(status_code=502, content={"error": exc.error, "detail": exc.detail})
+
+
+@app.get("/agent/ping", dependencies=[Depends(verify_token)])
+def ping() -> dict:
+    return {"ok": True, "hostname": _hostname(), "version": AGENT_VERSION}
+
+
+@app.get("/agent/system", dependencies=[Depends(verify_token)])
+def system() -> dict:
+    return _system_info()
+
+
+@app.get("/agent/stacks", dependencies=[Depends(verify_token)])
+def list_stacks() -> list[dict]:
+    return [_summary(sid) for sid in compose_service.discover_stacks()]
+
+
+@app.get("/agent/stacks/{stack_id}", dependencies=[Depends(verify_token)])
+def get_stack(stack_id: str) -> dict:
+    directory = compose_service.stack_dir(stack_id)
+    if not os.path.isdir(directory):
+        raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
+    try:
+        containers = [asdict(c) for c in compose_service.containers_for_stack(stack_id)]
+        status = compose_service.compute_status(stack_id)
+    except DockerError:
+        containers = []
+        status = "unknown"
+    return {
+        "id": stack_id,
+        "name": stack_id,
+        "description": None,
+        "status": status,
+        "yaml": compose_service.read_compose(stack_id),
+        "env": compose_service.read_env(stack_id),
+        "containers": containers,
+        "created_at": None,
+        "updated_at": None,
+    }
+
+
+@app.post("/agent/stacks", dependencies=[Depends(verify_token)], status_code=201)
+def create_stack(body: StackBody) -> dict:
+    if not body.name:
+        raise HTTPException(status_code=400, detail="name is required")
+    stack_id = compose_service.slugify(body.name)
+    if os.path.isdir(compose_service.stack_dir(stack_id)):
+        raise HTTPException(status_code=409, detail=f"Stack '{stack_id}' already exists")
+    compose_service.write_compose(stack_id, body.yaml or "services:\n")
+    if body.env:
+        compose_service.write_env(stack_id, body.env)
+    return _summary(stack_id)
+
+
+@app.put("/agent/stacks/{stack_id}", dependencies=[Depends(verify_token)])
+def update_stack(stack_id: str, body: StackBody) -> dict:
+    if not os.path.isdir(compose_service.stack_dir(stack_id)):
+        raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
+    if body.yaml is not None:
+        compose_service.write_compose(stack_id, body.yaml)
+    if body.env is not None:
+        compose_service.write_env(stack_id, body.env)
+    return _summary(stack_id)
+
+
+@app.delete("/agent/stacks/{stack_id}", dependencies=[Depends(verify_token)])
+async def delete_stack(stack_id: str, delete_files: bool = Query(True)) -> dict:
+    if not os.path.isdir(compose_service.stack_dir(stack_id)):
+        raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
+    try:
+        await compose_service.down(stack_id)
+    except Exception:  # noqa: BLE001 - best-effort teardown
+        pass
+    if delete_files:
+        compose_service.delete_stack_files(stack_id)
+    return {"ok": True}
+
+
+_ACTIONS = {
+    "start": compose_service.up,
+    "stop": compose_service.stop,
+    "restart": compose_service.restart,
+    "pull": compose_service.pull,
+    "update": compose_service.update,
+    "down": compose_service.down,
+}
+
+
+@app.post("/agent/stacks/{stack_id}/{action}", dependencies=[Depends(verify_token)])
+async def lifecycle(stack_id: str, action: str) -> dict:
+    fn = _ACTIONS.get(action)
+    if not fn:
+        raise HTTPException(status_code=400, detail=f"Unknown action '{action}'")
+    if not os.path.isdir(compose_service.stack_dir(stack_id)):
+        raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
+    result = await fn(stack_id)
+    if result.get("returncode") not in (0, None):
+        raise HTTPException(
+            status_code=500,
+            detail={
+                "error": f"compose {action} failed",
+                "detail": result.get("stderr", "").strip()[-2000:],
+            },
+        )
+    return result
+
+
+@app.get("/agent/stacks/{stack_id}/logs", dependencies=[Depends(verify_token)])
+async def stack_logs(stack_id: str, tail: int = Query(200, le=2000)) -> dict:
+    if not os.path.isdir(compose_service.stack_dir(stack_id)):
+        raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found")
+    result = await compose_service.logs(stack_id, tail=tail)
+    return {"logs": result.get("stdout", "") + result.get("stderr", "")}
+
+
+@app.get("/agent/health")
+def health() -> dict:
+    return {"status": "ok"}

backend/config.py

@@ -35,6 +35,10 @@ class Settings(BaseSettings):
     # Throwaway image used to read/write named-volume contents during backup.
     BACKUP_HELPER_IMAGE: str = "alpine:latest"
 
+    # Multi-host agent: shared bearer token the agent requires on every request.
+    # Only used when running the agent app (agent_app:app).
+    AGENT_TOKEN: str = ""
+
     # Host browser sandbox roots
     ALLOWED_BROWSE_ROOTS: Annotated[list[str], NoDecode] = [
         "/", "/mnt", "/media", "/srv", "/opt",

backend/main.py

@@ -14,6 +14,7 @@ from config import settings
 from database import engine, init_db
 from docker_client import DockerError
 from routers import (
+    agents,
     audit,
     auth,
     backups,
@@ -48,7 +49,7 @@ async def lifespan(app: FastAPI):
     update_task.cancel()
 
 
-app = FastAPI(title="StackPilot", version="0.4.0", lifespan=lifespan)
+app = FastAPI(title="StackPilot", version="0.5.0", lifespan=lifespan)
 
 app.add_middleware(
     CORSMiddleware,
@@ -78,6 +79,7 @@ app.include_router(templates.router)
 app.include_router(audit.router)
 app.include_router(settings_router.router)
 app.include_router(backups.router)
+app.include_router(agents.router)
 app.include_router(ws.router)
 
 

backend/models/__init__.py

@@ -1,8 +1,9 @@
 """SQLModel table models. Importing this package registers all tables."""
+from models.agent import Agent
 from models.audit import AuditLog
 from models.setting import Setting, Webhook
 from models.stack import Stack
 from models.template import Template
 from models.user import User
 
-__all__ = ["User", "Stack", "AuditLog", "Template", "Setting", "Webhook"]
+__all__ = ["User", "Stack", "AuditLog", "Template", "Setting", "Webhook", "Agent"]

backend/models/agent.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlmodel import Field, SQLModel
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+class Agent(SQLModel, table=True):
+    """A remote host running stackpilot-agent."""
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    name: str
+    url: str  # e.g. http://10.0.0.5:5010
+    token: str  # shared AGENT_TOKEN of that host
+    status: str = "unknown"  # online | offline | unauthorized | unknown
+    hostname: Optional[str] = None  # reported by the agent on ping
+    last_seen: Optional[datetime] = None
+    created_at: datetime = Field(default_factory=_now)
+
+
+# --- API schemas ---
+
+
+class AgentCreate(SQLModel):
+    name: str
+    url: str
+    token: str
+
+
+class AgentUpdate(SQLModel):
+    name: Optional[str] = None
+    url: Optional[str] = None
+    token: Optional[str] = None
+
+
+class AgentRead(SQLModel):
+    id: int
+    name: str
+    url: str
+    status: str
+    hostname: Optional[str]
+    last_seen: Optional[datetime]
+    created_at: datetime
+    token_set: bool

backend/routers/agents.py

@@ -0,0 +1,286 @@
+"""Remote host (agent) management + proxied stack/system operations."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from sqlmodel import Session, select
+
+from auth import get_current_user, require_admin
+from database import get_session
+from models.agent import Agent, AgentCreate, AgentRead, AgentUpdate
+from models.stack import StackCreate, StackUpdate
+from models.user import User
+from services import agent_service, audit_service
+from services.agent_service import AgentError
+
+router = APIRouter(prefix="/api/agents", tags=["agents"])
+
+_ACTIONS = {"start", "stop", "restart", "pull", "update", "down"}
+
+
+def _ip(request: Request) -> str:
+    return request.client.host if request.client else "unknown"
+
+
+def _to_read(a: Agent) -> AgentRead:
+    return AgentRead(
+        id=a.id,
+        name=a.name,
+        url=a.url,
+        status=a.status,
+        hostname=a.hostname,
+        last_seen=a.last_seen,
+        created_at=a.created_at,
+        token_set=bool(a.token),
+    )
+
+
+def _get_or_404(session: Session, agent_id: int) -> Agent:
+    agent = session.get(Agent, agent_id)
+    if not agent:
+        raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found")
+    return agent
+
+
+def _raise(exc: AgentError):
+    raise HTTPException(
+        status_code=exc.status if exc.status >= 400 else 502,
+        detail={"error": exc.error, "detail": exc.detail},
+    )
+
+
+async def _proxy(session: Session, agent: Agent, method: str, path: str, **kw):
+    try:
+        return await agent_service.call(session, agent, method, path, **kw)
+    except AgentError as exc:
+        _raise(exc)
+
+
+# --------------------------------------------------------------------------- #
+# CRUD
+# --------------------------------------------------------------------------- #
+
+
+@router.get("", response_model=list[AgentRead])
+async def list_agents(
+    refresh: bool = Query(True),
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+) -> list[AgentRead]:
+    agents = session.exec(select(Agent).order_by(Agent.id)).all()
+    if refresh:
+        for agent in agents:
+            await agent_service.ping(session, agent)
+    return [_to_read(a) for a in agents]
+
+
+@router.post("", response_model=AgentRead, status_code=201)
+async def create_agent(
+    body: AgentCreate,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> AgentRead:
+    agent = Agent(name=body.name, url=body.url.rstrip("/"), token=body.token)
+    session.add(agent)
+    session.commit()
+    session.refresh(agent)
+    # Validate connectivity immediately (best-effort; agent is saved regardless).
+    await agent_service.ping(session, agent)
+    audit_service.record(
+        session, user=user.username, action="agent.create", target=agent.name,
+        detail=agent.url, ip=_ip(request),
+    )
+    return _to_read(agent)
+
+
+@router.put("/{agent_id}", response_model=AgentRead)
+async def update_agent(
+    agent_id: int,
+    body: AgentUpdate,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> AgentRead:
+    agent = _get_or_404(session, agent_id)
+    if body.name is not None:
+        agent.name = body.name
+    if body.url is not None:
+        agent.url = body.url.rstrip("/")
+    if body.token:
+        agent.token = body.token
+    agent.status = "unknown"
+    session.add(agent)
+    session.commit()
+    session.refresh(agent)
+    await agent_service.ping(session, agent)
+    audit_service.record(
+        session, user=user.username, action="agent.update", target=agent.name,
+        ip=_ip(request),
+    )
+    return _to_read(agent)
+
+
+@router.delete("/{agent_id}")
+def delete_agent(
+    agent_id: int,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    name = agent.name
+    session.delete(agent)
+    session.commit()
+    audit_service.record(
+        session, user=user.username, action="agent.delete", target=name, ip=_ip(request),
+    )
+    return {"ok": True}
+
+
+@router.post("/{agent_id}/ping")
+async def ping_agent(
+    agent_id: int,
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    return await agent_service.ping(session, agent)
+
+
+# --------------------------------------------------------------------------- #
+# Proxied stack + system operations
+# --------------------------------------------------------------------------- #
+
+
+@router.get("/{agent_id}/system")
+async def agent_system(
+    agent_id: int,
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+):
+    agent = _get_or_404(session, agent_id)
+    return await _proxy(session, agent, "GET", "/agent/system")
+
+
+@router.get("/{agent_id}/stacks")
+async def agent_stacks(
+    agent_id: int,
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+) -> list[dict]:
+    agent = _get_or_404(session, agent_id)
+    stacks = await _proxy(session, agent, "GET", "/agent/stacks") or []
+    for s in stacks:
+        s["agent_id"] = agent.id
+        s["agent_name"] = agent.name
+    return stacks
+
+
+@router.get("/{agent_id}/stacks/{stack_id}")
+async def agent_stack_detail(
+    agent_id: int,
+    stack_id: str,
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    data = await _proxy(session, agent, "GET", f"/agent/stacks/{stack_id}")
+    data["agent_id"] = agent.id
+    data["agent_name"] = agent.name
+    return data
+
+
+@router.get("/{agent_id}/stacks/{stack_id}/logs")
+async def agent_stack_logs(
+    agent_id: int,
+    stack_id: str,
+    tail: int = Query(200, le=2000),
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    return await _proxy(
+        session, agent, "GET", f"/agent/stacks/{stack_id}/logs", params={"tail": tail}
+    )
+
+
+@router.post("/{agent_id}/stacks", status_code=201)
+async def agent_create_stack(
+    agent_id: int,
+    body: StackCreate,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    result = await _proxy(
+        session, agent, "POST", "/agent/stacks",
+        json={"name": body.name, "yaml": body.yaml, "env": body.env},
+    )
+    audit_service.record(
+        session, user=user.username, action="agent.stack.create",
+        target=f"{agent.name}/{result.get('id')}", ip=_ip(request),
+    )
+    return result
+
+
+@router.put("/{agent_id}/stacks/{stack_id}")
+async def agent_update_stack(
+    agent_id: int,
+    stack_id: str,
+    body: StackUpdate,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    result = await _proxy(
+        session, agent, "PUT", f"/agent/stacks/{stack_id}",
+        json={"yaml": body.yaml, "env": body.env},
+    )
+    audit_service.record(
+        session, user=user.username, action="agent.stack.update",
+        target=f"{agent.name}/{stack_id}", ip=_ip(request),
+    )
+    return result
+
+
+@router.delete("/{agent_id}/stacks/{stack_id}")
+async def agent_delete_stack(
+    agent_id: int,
+    stack_id: str,
+    request: Request,
+    delete_files: bool = Query(True),
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    result = await _proxy(
+        session, agent, "DELETE", f"/agent/stacks/{stack_id}",
+        params={"delete_files": delete_files},
+    )
+    audit_service.record(
+        session, user=user.username, action="agent.stack.delete",
+        target=f"{agent.name}/{stack_id}", ip=_ip(request),
+    )
+    return result
+
+
+@router.post("/{agent_id}/stacks/{stack_id}/{action}")
+async def agent_lifecycle(
+    agent_id: int,
+    stack_id: str,
+    action: str,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    if action not in _ACTIONS:
+        raise HTTPException(status_code=400, detail=f"Unknown action '{action}'")
+    agent = _get_or_404(session, agent_id)
+    result = await _proxy(session, agent, "POST", f"/agent/stacks/{stack_id}/{action}")
+    audit_service.record(
+        session, user=user.username, action=f"agent.stack.{action}",
+        target=f"{agent.name}/{stack_id}", ip=_ip(request),
+    )
+    return result

backend/services/agent_service.py

@@ -0,0 +1,115 @@
+"""Talk to remote stackpilot-agent hosts over HTTP.
+
+The central app stores an ``Agent`` row per remote host and proxies stack /
+system calls to it using the agent's shared token. Connectivity state
+(``status``, ``hostname``, ``last_seen``) is refreshed on every successful or
+failed call so the UI can show a live dot per host.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+import httpx
+from sqlmodel import Session
+
+from models.agent import Agent
+
+logger = logging.getLogger("stackpilot.agent_proxy")
+
+_TIMEOUT = 30.0
+
+
+class AgentError(Exception):
+    def __init__(self, status: int, error: str, detail: str = ""):
+        self.status = status
+        self.error = error
+        self.detail = detail
+        super().__init__(f"{error}: {detail}" if detail else error)
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _mark(session: Session, agent: Agent, status: str, hostname: Optional[str] = None) -> None:
+    agent.status = status
+    if status == "online":
+        agent.last_seen = _now()
+        if hostname:
+            agent.hostname = hostname
+    session.add(agent)
+    session.commit()
+    session.refresh(agent)
+
+
+async def _request(
+    agent: Agent,
+    method: str,
+    path: str,
+    *,
+    params: Optional[dict] = None,
+    json: Any = None,
+) -> httpx.Response:
+    url = agent.url.rstrip("/") + path
+    headers = {"Authorization": f"Bearer {agent.token}"}
+    async with httpx.AsyncClient(follow_redirects=True) as client:
+        return await client.request(
+            method, url, headers=headers, params=params, json=json, timeout=_TIMEOUT
+        )
+
+
+async def call(
+    session: Session,
+    agent: Agent,
+    method: str,
+    path: str,
+    *,
+    params: Optional[dict] = None,
+    json: Any = None,
+) -> Any:
+    """Proxy a request to the agent, updating its status, returning parsed JSON."""
+    try:
+        resp = await _request(agent, method, path, params=params, json=json)
+    except httpx.HTTPError as exc:
+        _mark(session, agent, "offline")
+        raise AgentError(502, "agent_unreachable", str(exc)) from exc
+
+    if resp.status_code in (401, 403):
+        _mark(session, agent, "unauthorized")
+        raise AgentError(resp.status_code, "agent_unauthorized", "Invalid agent token")
+
+    _mark(session, agent, "online")
+
+    if resp.status_code >= 400:
+        detail = ""
+        try:
+            body = resp.json()
+            detail = body.get("detail") if isinstance(body, dict) else str(body)
+            if isinstance(detail, dict):
+                detail = detail.get("detail") or detail.get("error") or str(detail)
+        except ValueError:
+            detail = resp.text[:500]
+        raise AgentError(resp.status_code, "agent_error", str(detail))
+
+    if resp.content:
+        try:
+            return resp.json()
+        except ValueError:
+            return resp.text
+    return None
+
+
+async def ping(session: Session, agent: Agent) -> dict:
+    """Health-check an agent and refresh its status + hostname. Never raises."""
+    try:
+        data = await call(session, agent, "GET", "/agent/ping")
+        if isinstance(data, dict) and data.get("hostname"):
+            agent.hostname = data["hostname"]
+            session.add(agent)
+            session.commit()
+            session.refresh(agent)
+        return {"status": agent.status, "hostname": agent.hostname, "data": data}
+    except AgentError:
+        return {"status": agent.status, "hostname": agent.hostname, "data": None}