Phase 22: auto-update (Watchtower-style), local + agent (0.28.0)

Per-stack auto-update policy on the stack Overview tab. When the background
image-update check finds a newer registry digest for one of a stack's images,
the stack is pulled + redeployed (or just flagged, "notify only"). Only running
stacks are auto-redeployed; a stopped stack is skipped, never silently started.

- models/auto_update.py: AutoUpdate(stack_id, agent_id, enabled, redeploy,
  last_run/status/result) + schemas; registered in models/__init__.py.
- update_service: DB-free stack_images/stack_updates helpers (agent reuses
  them); agent GET /agent/stacks/{id}/updates.
- services/auto_update_service.py: run_due/run_policy (local pull+up via
  compose_service, remote via agent_service POST /agent/stacks/{id}/update,
  notify-only with per-transition dedup); lazy-called from
  update_service.background_loop. New stack_auto_updated notify event.
- routers: GET/PUT/run /api/stacks/{id}/auto-update and the
  /api/agents/{id}/stacks/{sid}/auto-update variants (policy stored centrally).
- frontend: api/autoUpdate.ts + AutoUpdatePanel (enable, redeploy|notify-only,
  Check now, last-run status) on StackDetail + RemoteStackDetail; EVENT_LABELS
  gains stack_auto_updated + backup_failed.

Live-verified all four paths (updated / update-available / up-to-date /
skipped) against real compose.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/agent_app.py

@@ -64,7 +64,7 @@ def _map_docker(exc: DockerError):
         raise HTTPException(status_code=code, detail=exc.detail or exc.error)
     raise exc  # falls through to the global 502 DockerError handler
 
-AGENT_VERSION = "0.27.0"
+AGENT_VERSION = "0.28.0"
 
 
 # --------------------------------------------------------------------------- #
@@ -369,6 +369,12 @@ async def stack_logs(stack_id: str, tail: int = Query(200, le=2000)) -> dict:
     return {"logs": result.get("stdout", "") + result.get("stderr", "")}
 
 
+@app.get("/agent/stacks/{stack_id}/updates", dependencies=[Depends(verify_token)])
+async def stack_updates(stack_id: str, refresh: bool = Query(True)) -> dict:
+    """Update status for this stack's images (used by central auto-update)."""
+    return await update_service.stack_updates(stack_id, refresh=refresh)
+
+
 @app.get("/agent/stacks/{stack_id}/backup", dependencies=[Depends(verify_token)])
 async def backup_stack(
     stack_id: str,

backend/main.py

@@ -56,7 +56,7 @@ async def lifespan(app: FastAPI):
     schedule_task.cancel()
 
 
-app = FastAPI(title="StackPilot", version="0.27.0", lifespan=lifespan)
+app = FastAPI(title="StackPilot", version="0.28.0", lifespan=lifespan)
 
 app.add_middleware(
     CORSMiddleware,

backend/models/__init__.py

@@ -1,6 +1,7 @@
 """SQLModel table models. Importing this package registers all tables."""
 from models.agent import Agent
 from models.audit import AuditLog
+from models.auto_update import AutoUpdate
 from models.backup_destination import BackupDestination
 from models.backup_schedule import BackupSchedule
 from models.setting import Setting, Webhook
@@ -10,5 +11,5 @@ from models.user import User
 
 __all__ = [
     "User", "Stack", "AuditLog", "Template", "Setting", "Webhook", "Agent",
-    "BackupDestination", "BackupSchedule",
+    "BackupDestination", "BackupSchedule", "AutoUpdate",
 ]

backend/models/auto_update.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlmodel import Field, SQLModel
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+class AutoUpdate(SQLModel, table=True):
+    """Per-stack auto-update policy (Watchtower-style).
+
+    When the background image-update check finds a newer registry digest for one
+    of the stack's images, the stack is either pulled + redeployed
+    (``redeploy=True``) or merely notified about (``redeploy=False``).
+    ``agent_id`` None = local host, otherwise a remote agent's stack.
+    """
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    stack_id: str
+    agent_id: Optional[int] = None
+    enabled: bool = True
+    redeploy: bool = True  # True = pull + up -d; False = notify only
+    last_run: Optional[datetime] = None
+    last_status: Optional[str] = None  # "updated" | "up-to-date" | "update-available" | "skipped" | "error"
+    last_result: Optional[str] = None  # detail (stale images / error text)
+    created_at: datetime = Field(default_factory=_now)
+
+
+# --- API schemas ---
+
+
+class AutoUpdateWrite(SQLModel):
+    enabled: bool = True
+    redeploy: bool = True
+
+
+class AutoUpdateRead(SQLModel):
+    id: Optional[int]
+    stack_id: str
+    agent_id: Optional[int]
+    agent_name: Optional[str] = None
+    enabled: bool
+    redeploy: bool
+    last_run: Optional[datetime]
+    last_status: Optional[str]
+    last_result: Optional[str]

backend/models/setting.py

@@ -17,6 +17,7 @@ EVENT_STACK_STOP = "stack_stop"
 EVENT_STACK_ERROR = "stack_error"
 EVENT_PULL_FAILED = "pull_failed"
 EVENT_BACKUP_FAILED = "backup_failed"
+EVENT_STACK_AUTO_UPDATED = "stack_auto_updated"
 
 ALL_EVENTS = [
     EVENT_UPDATE_AVAILABLE,
@@ -25,6 +26,7 @@ ALL_EVENTS = [
     EVENT_STACK_ERROR,
     EVENT_PULL_FAILED,
     EVENT_BACKUP_FAILED,
+    EVENT_STACK_AUTO_UPDATED,
 ]
 
 WEBHOOK_TYPES = ["ntfy", "discord", "slack", "gotify", "generic"]

backend/routers/agents.py

@@ -19,9 +19,11 @@ from models.stack import StackCreate, StackUpdate
 from models.user import User
 from routers.files import NameBody, RenameBody, TransferBody, WriteBody
 from routers.networks import ContainerRef, NetworkCreate
+from models.auto_update import AutoUpdateRead, AutoUpdateWrite
 from services import (
     agent_service,
     audit_service,
+    auto_update_service,
     backup_destination_service as dest_service,
     backup_service,
     compose_service,
@@ -983,3 +985,57 @@ async def agent_container_action(
         target=f"{agent.name}/{container_id[:12]}", ip=_ip(request),
     )
     return result
+
+
+# --------------------------------------------------------------------------- #
+# Auto-update policy (Watchtower-style) — remote stacks (stored centrally)
+# --------------------------------------------------------------------------- #
+
+
+@router.get("/{agent_id}/stacks/{stack_id}/auto-update", response_model=AutoUpdateRead)
+def agent_get_auto_update(
+    agent_id: int,
+    stack_id: str,
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+) -> dict:
+    _get_or_404(session, agent_id)
+    policy = auto_update_service.get_policy(session, stack_id, agent_id)
+    return auto_update_service.to_read(session, policy, stack_id, agent_id)
+
+
+@router.put("/{agent_id}/stacks/{stack_id}/auto-update", response_model=AutoUpdateRead)
+def agent_set_auto_update(
+    agent_id: int,
+    stack_id: str,
+    body: AutoUpdateWrite,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    agent = _get_or_404(session, agent_id)
+    policy = auto_update_service.upsert_policy(
+        session, stack_id, body.enabled, body.redeploy, agent_id=agent_id
+    )
+    audit_service.record(
+        session, user=user.username, action="agent.stack.auto_update",
+        target=f"{agent.name}/{stack_id}",
+        detail=f"enabled={body.enabled} redeploy={body.redeploy}", ip=_ip(request),
+    )
+    return auto_update_service.to_read(session, policy, stack_id, agent_id)
+
+
+@router.post("/{agent_id}/stacks/{stack_id}/auto-update/run", response_model=AutoUpdateRead)
+async def agent_run_auto_update(
+    agent_id: int,
+    stack_id: str,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    _get_or_404(session, agent_id)
+    policy = auto_update_service.get_policy(session, stack_id, agent_id)
+    if policy is None:
+        raise HTTPException(status_code=404, detail="No auto-update policy for this stack")
+    await auto_update_service.run_policy(session, policy)
+    session.refresh(policy)
+    return auto_update_service.to_read(session, policy, stack_id, agent_id)

backend/routers/stacks.py

@@ -25,8 +25,9 @@ from models.setting import (
     EVENT_STACK_START,
     EVENT_STACK_STOP,
 )
+from models.auto_update import AutoUpdateRead, AutoUpdateWrite
 from models.user import User
-from services import audit_service, compose_service, notify_service, stats_service
+from services import audit_service, auto_update_service, compose_service, notify_service, stats_service
 from services.convert_service import convert_docker_run
 
 router = APIRouter(prefix="/api/stacks", tags=["stacks"])
@@ -398,3 +399,49 @@ def convert(
         return ConvertResponse(yaml=convert_docker_run(body.command))
     except ValueError as exc:
         raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
+# --------------------------------------------------------------------------- #
+# Auto-update policy (Watchtower-style) — local stacks
+# --------------------------------------------------------------------------- #
+
+
+@router.get("/{stack_id}/auto-update", response_model=AutoUpdateRead)
+def get_auto_update(
+    stack_id: str,
+    session: Session = Depends(get_session),
+    _user: User = Depends(get_current_user),
+) -> dict:
+    policy = auto_update_service.get_policy(session, stack_id)
+    return auto_update_service.to_read(session, policy, stack_id)
+
+
+@router.put("/{stack_id}/auto-update", response_model=AutoUpdateRead)
+def set_auto_update(
+    stack_id: str,
+    body: AutoUpdateWrite,
+    request: Request,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    policy = auto_update_service.upsert_policy(session, stack_id, body.enabled, body.redeploy)
+    audit_service.record(
+        session, user=user.username, action="stack.auto_update",
+        target=stack_id, detail=f"enabled={body.enabled} redeploy={body.redeploy}",
+        ip=_client_ip(request),
+    )
+    return auto_update_service.to_read(session, policy, stack_id)
+
+
+@router.post("/{stack_id}/auto-update/run", response_model=AutoUpdateRead)
+async def run_auto_update(
+    stack_id: str,
+    session: Session = Depends(get_session),
+    user: User = Depends(require_admin),
+) -> dict:
+    policy = auto_update_service.get_policy(session, stack_id)
+    if policy is None:
+        raise HTTPException(status_code=404, detail="No auto-update policy for this stack")
+    await auto_update_service.run_policy(session, policy)
+    session.refresh(policy)
+    return auto_update_service.to_read(session, policy, stack_id)

backend/services/auto_update_service.py

@@ -0,0 +1,194 @@
+"""Auto-update (Watchtower-style) orchestration.
+
+Runs once per image-update-check cycle (called from
+``update_service.background_loop`` so it reuses the freshly-populated digest
+cache). For each enabled policy whose stack has a newer image available, either
+pulls + redeploys the stack or just notifies, recording the outcome.
+
+Central-only / DB-aware. Image resolution + digest comparison live in the
+DB-free ``update_service`` so the agent can answer ``/agent/stacks/{id}/updates``
+with the same logic.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+
+from sqlmodel import Session, select
+
+from database import engine
+from models.agent import Agent
+from models.auto_update import AutoUpdate
+from models.setting import EVENT_PULL_FAILED, EVENT_STACK_AUTO_UPDATED
+from services import agent_service, compose_service, notify_service, update_service
+
+logger = logging.getLogger("stackpilot.autoupdate")
+
+# Stack states we'll act on. We never auto-start a stopped stack.
+_LIVE_STATES = {"running", "partial", "updating"}
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+# --------------------------------------------------------------------------- #
+# Policy CRUD helpers (shared by the stacks + agents routers)
+# --------------------------------------------------------------------------- #
+
+
+def get_policy(session: Session, stack_id: str, agent_id: int | None = None) -> AutoUpdate | None:
+    stmt = select(AutoUpdate).where(AutoUpdate.stack_id == stack_id)
+    stmt = stmt.where(AutoUpdate.agent_id == agent_id) if agent_id is not None \
+        else stmt.where(AutoUpdate.agent_id.is_(None))
+    return session.exec(stmt).first()
+
+
+def upsert_policy(
+    session: Session, stack_id: str, enabled: bool, redeploy: bool, agent_id: int | None = None
+) -> AutoUpdate:
+    policy = get_policy(session, stack_id, agent_id)
+    if policy is None:
+        policy = AutoUpdate(stack_id=stack_id, agent_id=agent_id)
+    policy.enabled = enabled
+    policy.redeploy = redeploy
+    session.add(policy)
+    session.commit()
+    session.refresh(policy)
+    return policy
+
+
+def to_read(session: Session, policy: AutoUpdate | None, stack_id: str, agent_id: int | None = None) -> dict:
+    """Build an AutoUpdateRead-shaped dict, defaulting to disabled when absent."""
+    agent_name = None
+    if agent_id is not None:
+        agent = session.get(Agent, agent_id)
+        agent_name = agent.name if agent else None
+    if policy is None:
+        return {
+            "id": None, "stack_id": stack_id, "agent_id": agent_id, "agent_name": agent_name,
+            "enabled": False, "redeploy": True,
+            "last_run": None, "last_status": None, "last_result": None,
+        }
+    return {
+        "id": policy.id, "stack_id": policy.stack_id, "agent_id": policy.agent_id,
+        "agent_name": agent_name, "enabled": policy.enabled, "redeploy": policy.redeploy,
+        "last_run": policy.last_run, "last_status": policy.last_status, "last_result": policy.last_result,
+    }
+
+
+def _record(session: Session, policy: AutoUpdate, status: str, result: str = "") -> None:
+    policy.last_run = _now()
+    policy.last_status = status
+    policy.last_result = result[:300] if result else None
+    session.add(policy)
+    session.commit()
+
+
+async def _run_local(session: Session, policy: AutoUpdate) -> None:
+    stack_id = policy.stack_id
+    try:
+        state = compose_service.compute_status(stack_id)
+    except Exception:  # noqa: BLE001
+        state = "unknown"
+    if state not in _LIVE_STATES:
+        _record(session, policy, "skipped", f"stack not running ({state})")
+        return
+
+    summary = await update_service.stack_updates(stack_id, refresh=False)
+    if not summary["update_available"]:
+        _record(session, policy, "up-to-date")
+        return
+
+    stale = ", ".join(summary["stale_images"])
+    prev = policy.last_status
+    if policy.redeploy:
+        try:
+            await compose_service.pull(stack_id)
+            await compose_service.up(stack_id)
+        except Exception as exc:  # noqa: BLE001
+            _record(session, policy, "error", str(exc))
+            await _safe_notify(EVENT_PULL_FAILED, f"Auto-update of '{stack_id}' failed", str(exc), session)
+            return
+        _record(session, policy, "updated", stale)
+        await _safe_notify(
+            EVENT_STACK_AUTO_UPDATED, f"Stack '{stack_id}' auto-updated",
+            f"Pulled and redeployed: {stale}.", session,
+        )
+    else:
+        _record(session, policy, "update-available", stale)
+        if prev != "update-available":  # notify once per transition, not every cycle
+            await _safe_notify(
+                EVENT_STACK_AUTO_UPDATED, f"Update available for '{stack_id}'",
+                f"Newer images: {stale} (auto-redeploy is off).", session,
+            )
+
+
+async def _run_remote(session: Session, policy: AutoUpdate) -> None:
+    agent = session.get(Agent, policy.agent_id)
+    if not agent:
+        _record(session, policy, "error", "agent not found")
+        return
+    stack_id = policy.stack_id
+    try:
+        summary = await agent_service.call(
+            session, agent, "GET", f"/agent/stacks/{stack_id}/updates",
+            params={"refresh": "true"},
+        )
+    except Exception as exc:  # noqa: BLE001
+        _record(session, policy, "error", f"agent check failed: {exc}")
+        return
+    if not summary or not summary.get("update_available"):
+        _record(session, policy, "up-to-date")
+        return
+
+    stale = ", ".join(summary.get("stale_images", []))
+    label = f"{agent.name}/{stack_id}"
+    prev = policy.last_status
+    if policy.redeploy:
+        try:
+            await agent_service.call(session, agent, "POST", f"/agent/stacks/{stack_id}/update")
+        except Exception as exc:  # noqa: BLE001
+            _record(session, policy, "error", str(exc))
+            await _safe_notify(EVENT_PULL_FAILED, f"Auto-update of '{label}' failed", str(exc), session)
+            return
+        _record(session, policy, "updated", stale)
+        await _safe_notify(
+            EVENT_STACK_AUTO_UPDATED, f"Stack '{label}' auto-updated",
+            f"Pulled and redeployed: {stale}.", session,
+        )
+    else:
+        _record(session, policy, "update-available", stale)
+        if prev != "update-available":
+            await _safe_notify(
+                EVENT_STACK_AUTO_UPDATED, f"Update available for '{label}'",
+                f"Newer images: {stale} (auto-redeploy is off).", session,
+            )
+
+
+async def _safe_notify(event: str, title: str, message: str, session: Session) -> None:
+    try:
+        await notify_service.notify(event, title, message, session)
+    except Exception as exc:  # noqa: BLE001 - notifications are best-effort
+        logger.debug("auto-update notify failed: %s", exc)
+
+
+async def run_policy(session: Session, policy: AutoUpdate) -> None:
+    if policy.agent_id is None:
+        await _run_local(session, policy)
+    else:
+        await _run_remote(session, policy)
+
+
+async def run_due() -> None:
+    """Process every enabled policy. Best-effort: one failure never aborts the rest."""
+    with Session(engine) as session:
+        policies = list(session.exec(select(AutoUpdate).where(AutoUpdate.enabled == True)))  # noqa: E712
+    for policy in policies:
+        try:
+            with Session(engine) as session:
+                fresh = session.get(AutoUpdate, policy.id)
+                if fresh and fresh.enabled:
+                    await run_policy(session, fresh)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Auto-update policy %s failed: %s", policy.id, exc)

backend/services/update_service.py

@@ -198,6 +198,45 @@ def _all_running_images() -> set[str]:
     return images
 
 
+def stack_images(stack_id: str) -> set[str]:
+    """Images used by the containers of one compose project (== stack id)."""
+    images: set[str] = set()
+    try:
+        client = get_client()
+        for c in safe_call(client.containers.list, all=True):
+            labels = c.labels or {}
+            if labels.get("com.docker.compose.project") != stack_id:
+                continue
+            cfg_image = c.attrs.get("Config", {}).get("Image")
+            if cfg_image:
+                images.add(cfg_image)
+    except DockerError:
+        pass
+    return images
+
+
+async def stack_updates(stack_id: str, refresh: bool = True) -> dict:
+    """Update status for one stack's images.
+
+    ``refresh=True`` queries the registry now; ``False`` reads the cache the
+    background loop already populated (so the auto-update pass adds no extra
+    registry round-trips). DB-free, so the agent can reuse it verbatim.
+    """
+    images = stack_images(stack_id)
+    result: dict[str, dict] = {}
+    for image in images:
+        status = await check_image(image) if refresh else _CACHE.get(image)
+        if status is not None:
+            result[image] = status.to_dict()
+    stale = [img for img, st in result.items() if st.get("update_available")]
+    return {
+        "stack_id": stack_id,
+        "update_available": bool(stale),
+        "stale_images": stale,
+        "images": result,
+    }
+
+
 async def check_all() -> dict[str, dict]:
     images = _all_running_images()
     for image in images:
@@ -218,6 +257,14 @@ async def background_loop():
             logger.info("Image update check complete (%d images)", len(_CACHE))
         except Exception as exc:  # noqa: BLE001
             logger.warning("Image update check failed: %s", exc)
+        # Apply auto-update policies using the digest cache we just refreshed.
+        # Lazy import avoids a circular import (auto_update_service imports us).
+        try:
+            from services import auto_update_service
+
+            await auto_update_service.run_due()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Auto-update pass failed: %s", exc)
         # Re-read the interval each cycle so Settings changes take effect.
         interval = max(settings_service.get_update_interval(), 5) * 60
         await asyncio.sleep(interval)