menzeljonas/stackpilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dashboard: rebuild into an operator cockpit (0.38.0)

Browse Source 
Replace the analytics-style dashboard (stack-health funnel, uptime %,
operations/day grid, AI pill) with an attention-driven fleet cockpit:

- New /api/dashboard/fleet endpoint: server-side fan-out across the local
  host and every agent into one payload — a prioritized "needs attention"
  list, headline KPIs, an honest stack-status breakdown and a per-host
  resource rollup. Each agent uses its own DB session so the fan-out is
  concurrency-safe; failures degrade to "offline" instead of stalling.
- New frontend: AttentionStrip, FleetKpiRow, StackStatusBar and
  HostResourceTable; Dashboard.tsx rewritten around them.
- Remove the funnel/summary endpoints, the uptime sampler loop and the
  ops-activity machinery; delete the now-unused chart components.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
menzelj
2026-06-24 11:04:57 +00:00
co-authored by Claude Opus 4.8
parent 5c46e40866
commit c830d28b65
15 changed files with 771 additions and 1067 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/dashboard_service.py
+280 -231
View File
@@ -1,40 +1,38 @@
"""Aggregated dashboard data: stack-health funnel + summary widgets.
"""Fleet aggregate for the dashboard cockpit.
Everything here is read-only and cheap by construction: one container
*summary* list (no per-container inspect) feeds the whole funnel, image
freshness comes from the cache the update-service background loop already
maintains, and the daily uptime sample is appended lazily on read.
One read-only call rolls up every host (local + agents) into a "needs
attention" list, headline KPIs and a per-host resource view. It is cheap by
construction: a single container *summary* list (no per-container inspect)
drives the local figures, image freshness comes from the cache the
update-service background loop already maintains, and per agent it makes a
small bounded set of HTTP fetches that degrade gracefully on failure.
"""
from __future__ import annotations
import asyncio
import json
import os
import time
from datetime import datetime, timedelta, timezone
from typing import Optional
from sqlmodel import Session, select
from config import settings
from database import engine
from docker_client import DockerError, get_client, safe_call
from models.audit import AuditLog
from models.auto_update import AutoUpdate
from services import compose_service, update_service
from models.agent import Agent
from models.backup_schedule import BackupSchedule
from services import agent_service, compose_service, update_service
COMPOSE_LABEL = compose_service.COMPOSE_LABEL
DOCKER_TIMEOUT = 5.0 # seconds — a slow daemon must not stall the dashboard
FUNNEL_TTL = 30.0
UPTIME_FILE = os.path.join(settings.DATA_DIR, "uptime.jsonl")
UPTIME_DAYS = 30
UPTIME_SAMPLE_INTERVAL = 300.0 # seconds between uptime samples
# Fleet aggregate: cached briefly because each call may fan out to every agent.
FLEET_TTL = 25.0
DISK_PRESSURE = 0.85 # disk used fraction above which a host needs attention
MEM_PRESSURE = 0.90 # memory used fraction above which a host needs attention
_funnel_cache: dict = {"data": None, "ts": 0.0}
_fleet_cache: dict = {"data": None, "ts": 0.0}
# --------------------------------------------------------------------------- #
# Container summary (single Docker round-trip)
# Container summary (single Docker round-trip) + shared classifiers
# --------------------------------------------------------------------------- #
@@ -43,10 +41,6 @@ def _list_containers() -> list[dict]:
return safe_call(client.api.containers, all=True)
async def _containers_with_timeout() -> list[dict]:
return await asyncio.wait_for(asyncio.to_thread(_list_containers), timeout=DOCKER_TIMEOUT)
def _group_by_project(raw: list[dict]) -> dict[str, list[dict]]:
by_project: dict[str, list[dict]] = {}
for c in raw:
@@ -78,226 +72,281 @@ def _is_updated(containers: list[dict], cache: dict[str, dict]) -> bool:
return True
def _auto_managed_ids(session: Session) -> set[str]:
"""Local stacks with an enabled auto-update policy."""
rows = session.exec(
select(AutoUpdate.stack_id).where(
AutoUpdate.enabled == True, # noqa: E712 — SQL expression, not identity
AutoUpdate.agent_id == None, # noqa: E711
)
).all()
return set(rows)
# --------------------------------------------------------------------------- #
# Fleet aggregate (one call → "needs attention" + KPIs across every host)
#
# The dashboard used to poll each agent individually and recombine the numbers
# client-side. ``compute_fleet`` does the fan-out server-side instead: one local
# Docker pass plus, per online agent, a small set of system/stacks/updates
# fetches — all wrapped so a slow or broken agent degrades to "offline" rather
# than stalling the whole view. Cached for FLEET_TTL because the fan-out is not
# free.
# --------------------------------------------------------------------------- #
# Stack-status buckets the status bar / KPIs are built from. Anything reporting
# "error" or "dead" containers counts as a problem stack.
_PROBLEM_STATUSES = {"error", "dead"}
async def compute_funnel(session: Session, refresh: bool = False) -> dict:
now = time.time()
if not refresh and _funnel_cache["data"] and now - _funnel_cache["ts"] < FUNNEL_TTL:
return _funnel_cache["data"]
def _bucket_statuses(statuses: list[str]) -> dict[str, int]:
return {
"running": sum(1 for s in statuses if s == "running"),
"partial": sum(1 for s in statuses if s == "partial"),
"stopped": sum(1 for s in statuses if s in ("stopped", "exited")),
"error": sum(1 for s in statuses if s in _PROBLEM_STATUSES),
"total": len(statuses),
}
discovered_ids = compose_service.discover_stacks()
def _attn(severity: str, kind: str, host: str, title: str, detail: str, link: str) -> dict:
return {
"severity": severity,
"kind": kind,
"host": host,
"title": title,
"detail": detail,
"link": link,
}
def _resource_attention(name: str, link: str, mem_used: int, mem_total: int,
disk_used: int, disk_total: int) -> list[dict]:
items: list[dict] = []
if mem_total and mem_used / mem_total >= MEM_PRESSURE:
pct = round(mem_used / mem_total * 100)
items.append(_attn("warn", "mem_pressure", name,
f"{name}: memory at {pct}%", "Free memory or move stacks.", link))
if disk_total and disk_used / disk_total >= DISK_PRESSURE:
pct = round(disk_used / disk_total * 100)
items.append(_attn("warn", "disk_pressure", name,
f"{name}: disk at {pct}%", "Prune images/volumes or add capacity.", link))
return items
def _local_host() -> tuple[dict, list[dict]]:
"""Local host card + attention items from a single Docker container pass."""
discovered = compose_service.discover_stacks()
try:
raw = await _containers_with_timeout()
except (DockerError, asyncio.TimeoutError):
raw = _list_containers()
except DockerError:
raw = []
by_project = _group_by_project(raw)
update_cache = update_service.get_cache()
auto_managed = _auto_managed_ids(session)
running = healthy = updated = monitored = 0
for stack_id in discovered_ids:
statuses: list[str] = []
unhealthy: list[str] = []
updates = 0
for stack_id in discovered:
containers = by_project.get(stack_id, [])
states = [c.get("State", "") for c in containers]
if not states or any(s != "running" for s in states):
continue
running += 1
if not _is_healthy(containers):
continue
healthy += 1
status = compose_service._status_from_states([c.get("State", "") for c in containers])
statuses.append(status)
if status == "running" and not _is_healthy(containers):
unhealthy.append(stack_id)
if not _is_updated(containers, update_cache):
continue
updated += 1
# Final stage: the stack also keeps itself fresh (auto-update enabled).
if stack_id in auto_managed:
monitored += 1
updates += 1
buckets = _bucket_statuses(statuses)
labelled = [c for c in raw if (c.get("Labels") or {}).get(COMPOSE_LABEL)]
# Local resource figures (lazy import keeps dashboard_service free of a
# router dependency at module load time).
from routers.system import _cpu_count, _disk_usage, _mem_info
mem = _mem_info()
disk = _disk_usage()
host = {
"id": "local",
"name": "local",
"online": True,
"status": "online",
"cpu_cores": _cpu_count(),
"mem_used": mem["used"], "mem_total": mem["total"],
"disk_used": disk["used"], "disk_total": disk["total"],
"stacks": buckets,
"containers_running": sum(1 for c in labelled if c.get("State") == "running"),
"containers_total": len(labelled),
"unhealthy": len(unhealthy),
"updates_available": updates,
}
attention: list[dict] = []
for sid in unhealthy:
attention.append(_attn("error", "unhealthy", "local",
f"{sid} is unhealthy", "A container is failing its healthcheck.",
f"/stacks/{sid}"))
if buckets["error"]:
attention.append(_attn("error", "stack_error", "local",
f"{buckets['error']} stack(s) in error", "Containers are dead.", "/stacks"))
if buckets["partial"]:
attention.append(_attn("warn", "stack_partial", "local",
f"{buckets['partial']} stack(s) partially running",
"Some services are down.", "/stacks"))
if updates:
attention.append(_attn("warn", "updates", "local",
f"{updates} stack(s) have image updates", "Pull the newer images.", "/images"))
attention += _resource_attention("local", "/", mem["used"], mem["total"],
disk["used"], disk["total"])
return host, attention
def _offline_host(agent_id: int, name: str, status: str) -> dict:
return {"id": agent_id, "name": name, "online": False, "status": status,
"cpu_cores": 0, "mem_used": 0, "mem_total": 0, "disk_used": 0, "disk_total": 0,
"stacks": _bucket_statuses([]), "containers_running": 0, "containers_total": 0,
"unhealthy": 0, "updates_available": 0}
async def _agent_host(agent_id: int) -> tuple[dict, list[dict]]:
"""One agent's host card + attention items, tolerant of partial failure.
Opens its own Session so the fan-out across agents stays concurrency-safe
(a shared SQLModel session is not), and fetches sequentially within the
agent because :func:`agent_service.call` commits a status update each time.
"""
with Session(engine) as session:
agent = session.get(Agent, agent_id)
if agent is None:
return _offline_host(agent_id, str(agent_id), "unknown"), []
name = agent.name
# Agent stacks live in the host section of the dashboard, not a route
# of their own, so aggregate agent items deep-link back to it.
link = "/"
if agent.status != "online":
return _offline_host(agent_id, name, agent.status), [
_attn("error", "agent_offline", name,
f"{name} is {agent.status}", "Check it under Settings → Remote hosts.",
"/settings")]
async def _fetch(path: str):
try:
return await agent_service.call(session, agent, "GET", path)
except Exception: # AgentError or transport — degrade gracefully
return None
sys_data = await _fetch("/agent/system")
stacks = await _fetch("/agent/stacks")
updates = await _fetch("/agent/stacks/updates")
if sys_data is None and stacks is None:
# Couldn't reach it at all — agent_service.call already marked it offline.
return _offline_host(agent_id, name, "offline"), [
_attn("error", "agent_offline", name,
f"{name} is unreachable", "Check it under Settings → Remote hosts.",
"/settings")]
sys_data = sys_data or {}
statuses = [s.get("status", "") for s in (stacks or [])]
buckets = _bucket_statuses(statuses)
update_count = sum(1 for v in (updates or {}).values()
if isinstance(v, dict) and v.get("update_available"))
host = {
"id": agent_id,
"name": name,
"online": True,
"status": "online",
"cpu_cores": sys_data.get("cpu_cores", 0),
"mem_used": sys_data.get("mem_used", 0), "mem_total": sys_data.get("mem_total", 0),
"disk_used": sys_data.get("disk_used", 0), "disk_total": sys_data.get("disk_total", 0),
"stacks": buckets,
"containers_running": sys_data.get("compose_running", sys_data.get("containers_running", 0)),
"containers_total": sys_data.get("containers_total", 0),
"unhealthy": buckets["error"], # agents expose no healthcheck rollup; error is the proxy
"updates_available": update_count,
}
attention: list[dict] = []
if buckets["error"]:
attention.append(_attn("error", "stack_error", name,
f"{name}: {buckets['error']} stack(s) in error",
"Containers are dead.", link))
if buckets["partial"]:
attention.append(_attn("warn", "stack_partial", name,
f"{name}: {buckets['partial']} stack(s) partially running",
"Some services are down.", link))
if update_count:
attention.append(_attn("warn", "updates", name,
f"{name}: {update_count} stack(s) have image updates",
"Pull the newer images.", link))
attention += _resource_attention(name, link,
host["mem_used"], host["mem_total"],
host["disk_used"], host["disk_total"])
return host, attention
def _backup_attention(session: Session, agent_names: dict[int, str]) -> list[dict]:
"""Flag enabled backup schedules whose last run failed or is overdue."""
now = datetime.now(timezone.utc)
items: list[dict] = []
for sch in session.exec(select(BackupSchedule).where(BackupSchedule.enabled == True)).all(): # noqa: E712
host = agent_names.get(sch.agent_id, "local") if sch.agent_id else "local"
status = (sch.last_status or "").lower()
if status and not status.startswith("ok"):
items.append(_attn("error", "backup_failed", host,
f"Backup of {sch.stack_id} failed", sch.last_status or "",
"/settings"))
elif sch.next_run and _aware(sch.next_run) < now - timedelta(hours=1):
items.append(_attn("warn", "backup_overdue", host,
f"Backup of {sch.stack_id} is overdue",
"Scheduled run did not happen.", "/settings"))
return items
def _aware(dt: datetime) -> datetime:
"""Treat naive DB timestamps as UTC (they're stored that way)."""
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
_SEVERITY_ORDER = {"error": 0, "warn": 1}
async def compute_fleet(session: Session, refresh: bool = False) -> dict:
now = time.time()
if not refresh and _fleet_cache["data"] and now - _fleet_cache["ts"] < FLEET_TTL:
return _fleet_cache["data"]
local_host, attention = await asyncio.to_thread(_local_host)
agents = session.exec(select(Agent)).all()
agent_names = {a.id: a.name for a in agents}
agent_ids = [a.id for a in agents]
agent_results = await asyncio.gather(*[_agent_host(aid) for aid in agent_ids])
hosts = [local_host]
for host, items in agent_results:
hosts.append(host)
attention += items
attention += _backup_attention(session, agent_names)
attention.sort(key=lambda a: _SEVERITY_ORDER.get(a["severity"], 9))
kpis = {
"hosts_online": sum(1 for h in hosts if h["online"]),
"hosts_total": len(hosts),
"stacks_running": sum(h["stacks"]["running"] for h in hosts),
"stacks_partial": sum(h["stacks"]["partial"] for h in hosts),
"stacks_total": sum(h["stacks"]["total"] for h in hosts),
"containers_running": sum(h["containers_running"] for h in hosts),
"containers_total": sum(h["containers_total"] for h in hosts),
"unhealthy": sum(h["unhealthy"] for h in hosts),
"updates_available": sum(h["updates_available"] for h in hosts),
"backups_failing": sum(1 for a in attention if a["kind"] in ("backup_failed", "backup_overdue")),
}
status_totals = {
"running": kpis["stacks_running"],
"partial": kpis["stacks_partial"],
"stopped": sum(h["stacks"]["stopped"] for h in hosts),
"error": sum(h["stacks"]["error"] for h in hosts),
}
data = {
"discovered": len(discovered_ids),
"running": running,
"healthy": healthy,
"updated": updated,
"monitored": monitored,
"as_of": datetime.now(timezone.utc).isoformat(),
"hosts": hosts,
"kpis": kpis,
"status_totals": status_totals,
"attention": attention,
}
_funnel_cache["data"] = data
_funnel_cache["ts"] = now
_fleet_cache["data"] = data
_fleet_cache["ts"] = now
return data
# --------------------------------------------------------------------------- #
# Uptime series (sampled every few minutes, aggregated to a daily mean)
#
# Each sample is one JSONL line {"ts": iso, "value": pct}; a background loop
# samples every UPTIME_SAMPLE_INTERVAL and summary reads top up opportunistically,
# so the daily value approximates real uptime instead of a once-a-day snapshot.
# Legacy pre-0.31.1 lines {"date": d, "value": pct} still count as one sample.
# --------------------------------------------------------------------------- #
def _read_uptime() -> list[dict]:
if not os.path.isfile(UPTIME_FILE):
return []
entries = []
with open(UPTIME_FILE, "r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
entries.append(json.loads(line))
except json.JSONDecodeError:
continue
return entries
def _append_uptime(entry: dict) -> None:
os.makedirs(settings.DATA_DIR, exist_ok=True)
with open(UPTIME_FILE, "a", encoding="utf-8") as fh:
fh.write(json.dumps(entry) + "\n")
def _entry_date(e: dict) -> Optional[str]:
if "date" in e:
return e["date"]
ts = e.get("ts")
return ts[:10] if isinstance(ts, str) and len(ts) >= 10 else None
def _latest_sample_ts(entries: list[dict]) -> float:
latest = 0.0
for e in entries:
ts = e.get("ts")
if not isinstance(ts, str):
continue
try:
latest = max(latest, datetime.fromisoformat(ts).timestamp())
except ValueError:
continue
return latest
def _sample_uptime(raw: list[dict]) -> Optional[dict]:
"""Record a sample if the last one is older than the sample interval.
Uptime% = share of compose containers currently running. With no compose
containers at all there is nothing to be up — skip rather than fake 100%.
"""
entries = _read_uptime()
now = datetime.now(timezone.utc)
if now.timestamp() - _latest_sample_ts(entries) < UPTIME_SAMPLE_INTERVAL:
return None
labelled = [c for c in raw if (c.get("Labels") or {}).get(COMPOSE_LABEL)]
if not labelled:
return None
running = sum(1 for c in labelled if c.get("State") == "running")
value = round(running / len(labelled) * 100, 1)
entry = {"ts": now.isoformat(timespec="seconds"), "value": value}
_append_uptime(entry)
return entry
def prune_uptime_file() -> None:
"""Drop samples older than twice the chart window (called at startup)."""
entries = _read_uptime()
if not entries:
return
cutoff = (datetime.now(timezone.utc).date() - timedelta(days=UPTIME_DAYS * 2)).isoformat()
kept = [e for e in entries if (_entry_date(e) or cutoff) >= cutoff]
if len(kept) == len(entries):
return
os.makedirs(settings.DATA_DIR, exist_ok=True)
tmp = UPTIME_FILE + ".tmp"
with open(tmp, "w", encoding="utf-8") as fh:
for e in kept:
fh.write(json.dumps(e) + "\n")
os.replace(tmp, UPTIME_FILE)
def uptime_series(raw: list[dict]) -> list[dict]:
_sample_uptime(raw)
by_date: dict[str, list[float]] = {}
for e in _read_uptime():
day = _entry_date(e)
value = e.get("value")
if day and isinstance(value, (int, float)):
by_date.setdefault(day, []).append(float(value))
series = []
today = datetime.now(timezone.utc).date()
last_value: Optional[float] = None
for i in range(UPTIME_DAYS - 1, -1, -1):
day = (today - timedelta(days=i)).isoformat()
samples = by_date.get(day)
if samples:
last_value = round(sum(samples) / len(samples), 1)
# Days before monitoring started (or gaps) reuse the last known value
# so the chart doesn't show artificial dips.
series.append({"date": day, "value": last_value})
return series
async def uptime_sampler_loop() -> None:
"""Background task: keep the uptime series fed even when nobody is
looking at the dashboard."""
prune_uptime_file()
while True:
try:
raw = await _containers_with_timeout()
await asyncio.to_thread(_sample_uptime, raw)
except (DockerError, asyncio.TimeoutError, OSError):
pass
await asyncio.sleep(UPTIME_SAMPLE_INTERVAL)
# --------------------------------------------------------------------------- #
# Ops (audit-log) activity
# --------------------------------------------------------------------------- #
_WEEKDAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
def ops_activity(session: Session) -> tuple[list[dict], Optional[str]]:
today = datetime.now(timezone.utc).date()
cutoff = datetime.combine(today - timedelta(days=UPTIME_DAYS - 1), datetime.min.time(), timezone.utc)
timestamps = session.exec(
select(AuditLog.timestamp).where(AuditLog.timestamp >= cutoff)
).all()
per_day: dict[str, int] = {}
per_weekday = [0] * 7
for ts in timestamps:
per_day[ts.date().isoformat()] = per_day.get(ts.date().isoformat(), 0) + 1
per_weekday[ts.weekday()] += 1
series = []
for i in range(UPTIME_DAYS - 1, -1, -1):
day = (today - timedelta(days=i)).isoformat()
series.append({"date": day, "count": per_day.get(day, 0)})
peak = _WEEKDAYS[per_weekday.index(max(per_weekday))] if any(per_weekday) else None
return series, peak
async def compute_summary(session: Session) -> dict:
try:
raw = await _containers_with_timeout()
except (DockerError, asyncio.TimeoutError):
raw = []
labelled = [c for c in raw if (c.get("Labels") or {}).get(COMPOSE_LABEL)]
ops_series, peak = ops_activity(session)
return {
"total_containers": sum(1 for c in labelled if c.get("State") == "running"),
"containers_total": len(labelled),
"uptime_series": uptime_series(raw),
"ops_last_30d": ops_series,
"ops_peak_day": peak,
"as_of": datetime.now(timezone.utc).isoformat(),
}