F16 — /ws/events was implemented and nothing consumed it, while thirty polling intervals across the pages asked for state that only changes when Docker does something. The endpoint was the answer; it just was not usable as it stood, so this is three fixes and a client, not a wiring job. The endpoint forwarded the whole firehose. Three exec_* events fire per web terminal session and top/attach fire whenever anything inspects a container, so a client invalidating on each would have been noisier than the polling it replaces. Now the daemon filters by resource type and the handler drops the actions that say nothing about rendered state — matching on the verb before the colon, since Docker reports these as "exec_create: /bin/sh". It never said *what* changed, so there was nothing to decide which caches to drop. The payload now carries the resource type. And it leaked its reader thread. Cancelling the executor future does not interrupt a thread already inside a blocking read; closing the underlying CancellableStream is what does. Every page load left one behind holding a socket open. A test asserts the close, because this is invisible until the process has been up for a week. Client side, useDockerEvents holds one connection for the session and maps resource types to query keys. Bursts are coalesced over 300ms — a ten-service compose up emits dozens of events in a second, and refetching per event would reintroduce exactly the load being removed. Reconnects back off to 30s, and any close reconnects including 4401, since the access token is short-lived and gets refreshed out from under the socket. Intervals drop from the mechanism to the safety net: 5s becomes 30-60s. Two deliberately stay fast. Live CPU/memory drifts continuously with no event to announce it, and that one is served from the 4s server-side cache added in 0.47.0, so it costs one sample per interval regardless of how many tabs are open. The audit feed polls because its entries come from people, not Docker. Net effect is both cheaper and faster: no fixed floor of requests per second against the daemon, and a stack that finishes starting shows up immediately rather than up to five seconds later. 23 new tests (758 total), driven against a fake daemon. Both nets were checked by reverting the fix: dropping the filter fails one, dropping the stream close fails the leak test. Not covered: the hook itself has no test — there is no frontend test runner yet. Its contract with the backend is tested; its own behaviour is only typechecked. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Dk43rmEeRfYi5wsLDbmfyG
407 lines
14 KiB
Python
407 lines
14 KiB
Python
"""WebSocket endpoints for real-time log streaming and Docker events."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
|
|
import contextlib
|
|
|
|
from fastapi import APIRouter, Query, WebSocket, WebSocketDisconnect
|
|
from jose import JWTError
|
|
from sqlmodel import Session
|
|
|
|
from auth import decode_token, resolve_token_user
|
|
from database import engine
|
|
from models.setting import EVENT_PULL_FAILED, EVENT_STACK_ERROR, EVENT_STACK_START
|
|
from services import (
|
|
audit_service,
|
|
compose_service,
|
|
exec_service,
|
|
notify_service,
|
|
stack_lock_service,
|
|
update_service,
|
|
)
|
|
|
|
logger = logging.getLogger("stackpilot.ws")
|
|
|
|
router = APIRouter(tags=["ws"])
|
|
|
|
|
|
def _user_for(token: str | None):
|
|
"""The live user behind a socket's token, or None.
|
|
|
|
Resolves against the database rather than reading the role straight off the
|
|
JWT: a socket can outlive a demotion, a disabled account or a password
|
|
reset, and the exec endpoint below is root-equivalent on the host. Same
|
|
check the HTTP routes make.
|
|
"""
|
|
if not token:
|
|
return None
|
|
try:
|
|
payload = decode_token(token, "access")
|
|
except (JWTError, Exception): # noqa: BLE001
|
|
return None
|
|
with Session(engine) as session:
|
|
user = resolve_token_user(session, payload)
|
|
if user:
|
|
session.expunge(user)
|
|
return user
|
|
|
|
|
|
async def _authorize(websocket: WebSocket, token: str | None) -> bool:
|
|
"""Validate the JWT supplied as a query param. Closes socket on failure."""
|
|
if _user_for(token) is None:
|
|
await websocket.close(code=4401)
|
|
return False
|
|
return True
|
|
|
|
|
|
async def _authorize_admin(websocket: WebSocket, token: str | None) -> bool:
|
|
"""Like _authorize but also requires the admin role (exec is root-equivalent).
|
|
|
|
Closes 4401 on a missing/invalid token, 4403 on a valid non-admin token."""
|
|
user = _user_for(token)
|
|
if user is None:
|
|
await websocket.close(code=4401)
|
|
return False
|
|
if user.role != "admin":
|
|
await websocket.close(code=4403)
|
|
return False
|
|
return True
|
|
|
|
|
|
async def _stream_logs(websocket: WebSocket, stack_id: str, service: str | None):
|
|
"""Stream `docker compose logs -f` output to the client."""
|
|
args = ["logs", "--no-color", "--tail", "200", "--timestamps", "-f"]
|
|
if service:
|
|
args.append(service)
|
|
try:
|
|
async for line in compose_service.stream_compose(stack_id, args):
|
|
await websocket.send_text(
|
|
json.dumps(
|
|
{
|
|
"type": "log",
|
|
"stack_id": stack_id,
|
|
"service": service,
|
|
"line": line,
|
|
}
|
|
)
|
|
)
|
|
except WebSocketDisconnect:
|
|
raise
|
|
except Exception as exc: # noqa: BLE001
|
|
await websocket.send_text(
|
|
json.dumps({"type": "error", "detail": str(exc)})
|
|
)
|
|
|
|
|
|
@router.websocket("/ws/logs/{stack_id}")
|
|
async def ws_stack_logs(
|
|
websocket: WebSocket,
|
|
stack_id: str,
|
|
token: str | None = Query(default=None),
|
|
):
|
|
await websocket.accept()
|
|
if not await _authorize(websocket, token):
|
|
return
|
|
try:
|
|
await _stream_logs(websocket, stack_id, None)
|
|
except WebSocketDisconnect:
|
|
pass
|
|
|
|
|
|
@router.websocket("/ws/logs/{stack_id}/{service}")
|
|
async def ws_service_logs(
|
|
websocket: WebSocket,
|
|
stack_id: str,
|
|
service: str,
|
|
token: str | None = Query(default=None),
|
|
):
|
|
await websocket.accept()
|
|
if not await _authorize(websocket, token):
|
|
return
|
|
try:
|
|
await _stream_logs(websocket, stack_id, service)
|
|
except WebSocketDisconnect:
|
|
pass
|
|
|
|
|
|
@router.websocket("/ws/deploy/{stack_id}")
|
|
async def ws_deploy(
|
|
websocket: WebSocket,
|
|
stack_id: str,
|
|
token: str | None = Query(default=None),
|
|
):
|
|
"""Run `docker compose up -d` and stream its output (image pulls, container
|
|
creation) to the browser so the user sees deploy progress live. Records the
|
|
same audit entry and notification as the REST `/start` endpoint."""
|
|
await websocket.accept()
|
|
if not await _authorize(websocket, token):
|
|
return
|
|
username = decode_token(token, "access").get("sub", "unknown") if token else "unknown"
|
|
|
|
rc: int | None = None
|
|
disconnected = False
|
|
# Same guard the REST lifecycle uses — the deploy console runs the very
|
|
# same `compose up`, so it has to queue behind an in-flight operation
|
|
# rather than race it.
|
|
lock_session = Session(engine)
|
|
try:
|
|
stack_lock_service.acquire(lock_session, stack_id, "start", username)
|
|
except stack_lock_service.StackBusy as exc:
|
|
lock_session.close()
|
|
with contextlib.suppress(Exception):
|
|
await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)}))
|
|
await websocket.close(code=4409)
|
|
return
|
|
try:
|
|
async for kind, payload in compose_service.stream_up(stack_id):
|
|
if kind == "log":
|
|
await websocket.send_text(json.dumps({"type": "log", "line": payload}))
|
|
else:
|
|
rc = payload
|
|
await websocket.send_text(json.dumps({"type": "done", "returncode": rc}))
|
|
except WebSocketDisconnect:
|
|
# Client navigated away; the compose subprocess keeps running so the
|
|
# deploy still completes in the background.
|
|
disconnected = True
|
|
except Exception as exc: # noqa: BLE001
|
|
with contextlib.suppress(Exception):
|
|
await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)}))
|
|
finally:
|
|
with contextlib.suppress(Exception):
|
|
stack_lock_service.release(lock_session, stack_id)
|
|
lock_session.close()
|
|
|
|
ok = rc in (0, None)
|
|
try:
|
|
with Session(engine) as session:
|
|
audit_service.record(
|
|
session, user=username, action="stack.start", target=stack_id,
|
|
detail=f"rc={rc} (deploy console)", ip="ws",
|
|
)
|
|
if ok:
|
|
await notify_service.notify(
|
|
EVENT_STACK_START, f"Stack '{stack_id}' started",
|
|
"compose up completed successfully.", session,
|
|
)
|
|
else:
|
|
await notify_service.notify(
|
|
EVENT_STACK_ERROR, f"Stack '{stack_id}' start failed",
|
|
"compose up returned a non-zero exit code.", session,
|
|
)
|
|
except Exception: # noqa: BLE001 - audit/notify are best-effort
|
|
pass
|
|
if not disconnected:
|
|
with contextlib.suppress(Exception):
|
|
await websocket.close()
|
|
|
|
|
|
@router.websocket("/ws/update/{stack_id}")
|
|
async def ws_update(
|
|
websocket: WebSocket,
|
|
stack_id: str,
|
|
token: str | None = Query(default=None),
|
|
):
|
|
"""Run `docker compose pull && up -d` and stream its output, so the stacks
|
|
list can render real update progress. Same audit/notify contract as the
|
|
REST `/update` endpoint, which stays for non-interactive callers."""
|
|
await websocket.accept()
|
|
if not await _authorize_admin(websocket, token):
|
|
return
|
|
username = decode_token(token, "access").get("sub", "unknown") if token else "unknown"
|
|
|
|
rc: int | None = None
|
|
disconnected = False
|
|
lock_session = Session(engine)
|
|
try:
|
|
stack_lock_service.acquire(lock_session, stack_id, "update", username)
|
|
except stack_lock_service.StackBusy as exc:
|
|
lock_session.close()
|
|
with contextlib.suppress(Exception):
|
|
await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)}))
|
|
await websocket.close(code=4409)
|
|
return
|
|
try:
|
|
async for kind, payload in compose_service.stream_update(stack_id):
|
|
if kind == "log":
|
|
await websocket.send_text(json.dumps({"type": "log", "line": payload}))
|
|
else:
|
|
rc = payload
|
|
await websocket.send_text(json.dumps({"type": "done", "returncode": rc}))
|
|
except WebSocketDisconnect:
|
|
# Client navigated away; compose keeps running so the update finishes.
|
|
disconnected = True
|
|
except Exception as exc: # noqa: BLE001
|
|
with contextlib.suppress(Exception):
|
|
await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)}))
|
|
finally:
|
|
with contextlib.suppress(Exception):
|
|
stack_lock_service.release(lock_session, stack_id)
|
|
lock_session.close()
|
|
|
|
ok = rc in (0, None)
|
|
try:
|
|
with Session(engine) as session:
|
|
audit_service.record(
|
|
session, user=username, action="stack.update", target=stack_id,
|
|
detail=f"rc={rc} (update stream)", ip="ws",
|
|
)
|
|
if ok:
|
|
await notify_service.notify(
|
|
EVENT_STACK_START, f"Stack '{stack_id}' updated",
|
|
"compose pull + up completed successfully.", session,
|
|
)
|
|
else:
|
|
await notify_service.notify(
|
|
EVENT_PULL_FAILED, f"Stack '{stack_id}' update failed",
|
|
"compose pull/up returned a non-zero exit code.", session,
|
|
)
|
|
except Exception: # noqa: BLE001 - audit/notify are best-effort
|
|
pass
|
|
if not disconnected:
|
|
with contextlib.suppress(Exception):
|
|
await websocket.close()
|
|
|
|
if ok:
|
|
update_service.refresh_stack_local(stack_id)
|
|
|
|
|
|
#: Docker event types worth telling the UI about. Filtered daemon-side, so the
|
|
#: bulk of the firehose never crosses the socket.
|
|
_EVENT_TYPES = ["container", "image", "network", "volume"]
|
|
|
|
#: Container actions that say nothing about state a page renders. exec_* alone
|
|
#: is three events per web-terminal keystroke session, and `top`/`attach` fire
|
|
#: whenever something inspects a container — invalidating queries on those would
|
|
#: make the stream noisier than the polling it replaces.
|
|
_IGNORED_ACTIONS = {
|
|
"exec_create", "exec_start", "exec_die", "exec_detach",
|
|
"attach", "top", "resize", "archive-path", "extract-to-dir",
|
|
}
|
|
|
|
|
|
@router.websocket("/ws/events")
|
|
async def ws_events(
|
|
websocket: WebSocket,
|
|
token: str | None = Query(default=None),
|
|
):
|
|
"""Stream Docker events so the UI can refresh on change instead of polling.
|
|
|
|
Every page used to poll its own endpoint every few seconds. Almost all of
|
|
that state only changes when Docker does something, which is exactly what
|
|
this reports — so the client refreshes on an event and keeps a slow poll as
|
|
a safety net.
|
|
|
|
Payload per event::
|
|
|
|
{"type": "event", "resource": "container", "action": "start",
|
|
"container": "jellyfin", "stack": "jellyfin"}
|
|
|
|
``resource`` is what the client needs to decide which queries to drop.
|
|
"""
|
|
await websocket.accept()
|
|
if not await _authorize(websocket, token):
|
|
return
|
|
from docker_client import get_client
|
|
|
|
loop = asyncio.get_event_loop()
|
|
queue: asyncio.Queue = asyncio.Queue()
|
|
stream = None
|
|
|
|
def reader():
|
|
"""Blocking read of the event stream, handed to the loop thread-safely."""
|
|
nonlocal stream
|
|
try:
|
|
stream = get_client().events(decode=True, filters={"type": _EVENT_TYPES})
|
|
for event in stream:
|
|
loop.call_soon_threadsafe(queue.put_nowait, event)
|
|
except Exception: # noqa: BLE001 - a closed stream lands here on teardown
|
|
pass
|
|
finally:
|
|
loop.call_soon_threadsafe(queue.put_nowait, None)
|
|
|
|
task = loop.run_in_executor(None, reader)
|
|
try:
|
|
await websocket.send_text(json.dumps({"type": "ready"}))
|
|
while True:
|
|
event = await queue.get()
|
|
if event is None: # reader finished — daemon gone or stream closed
|
|
await websocket.send_text(
|
|
json.dumps({"type": "error", "detail": "Docker event stream ended"})
|
|
)
|
|
break
|
|
action = (event.get("Action") or "").split(":")[0]
|
|
if action in _IGNORED_ACTIONS:
|
|
continue
|
|
actor = event.get("Actor", {}) or {}
|
|
attrs = actor.get("Attributes", {}) or {}
|
|
await websocket.send_text(
|
|
json.dumps(
|
|
{
|
|
"type": "event",
|
|
"resource": event.get("Type"),
|
|
"action": event.get("Action"),
|
|
"container": attrs.get("name"),
|
|
"stack": attrs.get("com.docker.compose.project"),
|
|
}
|
|
)
|
|
)
|
|
except WebSocketDisconnect:
|
|
pass
|
|
except Exception as exc: # noqa: BLE001
|
|
with contextlib.suppress(Exception):
|
|
await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)}))
|
|
finally:
|
|
# Closing the stream is what actually unblocks the reader thread.
|
|
# Cancelling the executor future does not: a thread already inside a
|
|
# blocking read keeps that read, and the thread leaks for the life of
|
|
# the process — once per page load, with the socket held open.
|
|
if stream is not None:
|
|
with contextlib.suppress(Exception):
|
|
stream.close()
|
|
task.cancel()
|
|
|
|
|
|
@router.websocket("/ws/exec/{container_id}")
|
|
async def ws_exec(
|
|
websocket: WebSocket,
|
|
container_id: str,
|
|
token: str | None = Query(default=None),
|
|
cmd: str | None = Query(default=None),
|
|
):
|
|
"""Interactive shell into a compose-managed container (admin only)."""
|
|
await websocket.accept()
|
|
if not await _authorize_admin(websocket, token):
|
|
return
|
|
username = decode_token(token, "access").get("sub", "unknown") if token else "unknown"
|
|
shell = cmd or exec_service.DEFAULT_SHELL
|
|
|
|
try:
|
|
exec_id = exec_service.create_exec(container_id, [shell])
|
|
holder, raw = exec_service.start_exec(exec_id)
|
|
except Exception as exc: # noqa: BLE001
|
|
with contextlib.suppress(Exception):
|
|
await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)}))
|
|
with contextlib.suppress(Exception):
|
|
await websocket.close()
|
|
return
|
|
|
|
with contextlib.suppress(Exception):
|
|
with Session(engine) as session:
|
|
audit_service.record(
|
|
session, user=username, action="container.exec",
|
|
target=container_id[:12], detail=shell, ip="ws",
|
|
)
|
|
|
|
try:
|
|
await exec_service.pump_exec(websocket, exec_id, holder, raw)
|
|
except WebSocketDisconnect:
|
|
pass
|
|
finally:
|
|
with contextlib.suppress(Exception):
|
|
await websocket.close()
|
|
|
|
|