"""Host aggregate for the dashboard cockpit. One read-only call rolls the host up into a "needs attention" list, headline KPIs and a resource view. It is cheap by construction: a single container *summary* list (no per-container inspect) drives the figures, and image freshness comes from the cache the update-service background loop already maintains. """ from __future__ import annotations import asyncio import time from datetime import datetime, timedelta, timezone from sqlmodel import Session, select from docker_client import DockerError, get_client, safe_call from models.backup_schedule import BackupSchedule from services import compose_service, update_service COMPOSE_LABEL = compose_service.COMPOSE_LABEL # Cached briefly: the dashboard polls this and the rollup is not free. FLEET_TTL = 25.0 DISK_PRESSURE = 0.85 # disk used fraction above which a host needs attention MEM_PRESSURE = 0.90 # memory used fraction above which a host needs attention _fleet_cache: dict = {"data": None, "ts": 0.0} # --------------------------------------------------------------------------- # # Container summary (single Docker round-trip) + shared classifiers # --------------------------------------------------------------------------- # def _list_containers() -> list[dict]: client = get_client() return safe_call(client.api.containers, all=True) def _group_by_project(raw: list[dict]) -> dict[str, list[dict]]: by_project: dict[str, list[dict]] = {} for c in raw: project = (c.get("Labels") or {}).get(COMPOSE_LABEL) if project: by_project.setdefault(project, []).append(c) return by_project def _is_healthy(containers: list[dict]) -> bool: """All containers that *have* a healthcheck report healthy. The summary ``Status`` string carries the health suffix — "(healthy)", "(unhealthy)" or "(health: starting)" — only for containers with a healthcheck configured, so its absence simply means "no healthcheck". """ for c in containers: status = c.get("Status", "") or "" if "(unhealthy)" in status or "(health:" in status: return False return True def _is_updated(containers: list[dict], cache: dict[str, dict]) -> bool: for c in containers: st = cache.get(c.get("Image", "")) if st and st.get("update_available"): return False return True # --------------------------------------------------------------------------- # # Host aggregate (one call → "needs attention" + KPIs) # # The dashboard used to fetch these numbers per stack and recombine them # client-side. ``compute_fleet`` rolls them up server-side instead, off one # Docker pass plus the update cache, and holds the result for FLEET_TTL because # the dashboard polls it. # --------------------------------------------------------------------------- # # Stack-status buckets the status bar / KPIs are built from. Anything reporting # "error" or "dead" containers counts as a problem stack. _PROBLEM_STATUSES = {"error", "dead"} def _bucket_statuses(statuses: list[str]) -> dict[str, int]: return { "running": sum(1 for s in statuses if s == "running"), "partial": sum(1 for s in statuses if s == "partial"), "stopped": sum(1 for s in statuses if s in ("stopped", "exited")), "error": sum(1 for s in statuses if s in _PROBLEM_STATUSES), "total": len(statuses), } def _attn(severity: str, kind: str, host: str, title: str, detail: str, link: str) -> dict: return { "severity": severity, "kind": kind, "host": host, "title": title, "detail": detail, "link": link, } def _resource_attention(name: str, link: str, mem_used: int, mem_total: int, disk_used: int, disk_total: int) -> list[dict]: items: list[dict] = [] if mem_total and mem_used / mem_total >= MEM_PRESSURE: pct = round(mem_used / mem_total * 100) items.append(_attn("warn", "mem_pressure", name, f"{name}: memory at {pct}%", "Free memory or move stacks.", link)) if disk_total and disk_used / disk_total >= DISK_PRESSURE: pct = round(disk_used / disk_total * 100) items.append(_attn("warn", "disk_pressure", name, f"{name}: disk at {pct}%", "Prune images/volumes or add capacity.", link)) return items def _local_host() -> tuple[dict, list[dict]]: """Local host card + attention items from a single Docker container pass.""" discovered = compose_service.discover_stacks() try: raw = _list_containers() except DockerError: raw = [] by_project = _group_by_project(raw) update_cache = update_service.get_cache() statuses: list[str] = [] unhealthy: list[str] = [] updates = 0 for stack_id in discovered: containers = by_project.get(stack_id, []) status = compose_service._status_from_states([c.get("State", "") for c in containers]) statuses.append(status) if status == "running" and not _is_healthy(containers): unhealthy.append(stack_id) if not _is_updated(containers, update_cache): updates += 1 buckets = _bucket_statuses(statuses) labelled = [c for c in raw if (c.get("Labels") or {}).get(COMPOSE_LABEL)] # Local resource figures (lazy import keeps dashboard_service free of a # router dependency at module load time). from routers.system import _cpu_count, _disk_usage, _mem_info mem = _mem_info() disk = _disk_usage() host = { "id": "local", "name": "local", "online": True, "status": "online", "cpu_cores": _cpu_count(), "mem_used": mem["used"], "mem_total": mem["total"], "disk_used": disk["used"], "disk_total": disk["total"], "stacks": buckets, "containers_running": sum(1 for c in labelled if c.get("State") == "running"), "containers_total": len(labelled), "unhealthy": len(unhealthy), "updates_available": updates, } attention: list[dict] = [] for sid in unhealthy: attention.append(_attn("error", "unhealthy", "local", f"{sid} is unhealthy", "A container is failing its healthcheck.", f"/stacks/{sid}")) if buckets["error"]: attention.append(_attn("error", "stack_error", "local", f"{buckets['error']} stack(s) in error", "Containers are dead.", "/stacks")) if buckets["partial"]: attention.append(_attn("warn", "stack_partial", "local", f"{buckets['partial']} stack(s) partially running", "Some services are down.", "/stacks")) if updates: attention.append(_attn("warn", "updates", "local", f"{updates} stack(s) have image updates", "Pull the newer images.", "/images")) attention += _resource_attention("local", "/", mem["used"], mem["total"], disk["used"], disk["total"]) return host, attention def _backup_attention(session: Session) -> list[dict]: """Flag enabled backup schedules whose last run failed or is overdue.""" now = datetime.now(timezone.utc) items: list[dict] = [] for sch in session.exec(select(BackupSchedule).where(BackupSchedule.enabled == True)).all(): # noqa: E712 host = "local" status = (sch.last_status or "").lower() if status and not status.startswith("ok"): items.append(_attn("error", "backup_failed", host, f"Backup of {sch.stack_id} failed", sch.last_status or "", "/settings")) elif sch.next_run and _aware(sch.next_run) < now - timedelta(hours=1): items.append(_attn("warn", "backup_overdue", host, f"Backup of {sch.stack_id} is overdue", "Scheduled run did not happen.", "/settings")) return items def _aware(dt: datetime) -> datetime: """Treat naive DB timestamps as UTC (they're stored that way).""" return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc) _SEVERITY_ORDER = {"error": 0, "warn": 1} async def compute_fleet(session: Session, refresh: bool = False) -> dict: now = time.time() if not refresh and _fleet_cache["data"] and now - _fleet_cache["ts"] < FLEET_TTL: return _fleet_cache["data"] local_host, attention = await asyncio.to_thread(_local_host) hosts = [local_host] attention += _backup_attention(session) attention.sort(key=lambda a: _SEVERITY_ORDER.get(a["severity"], 9)) kpis = { "stacks_running": sum(h["stacks"]["running"] for h in hosts), "stacks_partial": sum(h["stacks"]["partial"] for h in hosts), "stacks_total": sum(h["stacks"]["total"] for h in hosts), "containers_running": sum(h["containers_running"] for h in hosts), "containers_total": sum(h["containers_total"] for h in hosts), "unhealthy": sum(h["unhealthy"] for h in hosts), "updates_available": sum(h["updates_available"] for h in hosts), "backups_failing": sum(1 for a in attention if a["kind"] in ("backup_failed", "backup_overdue")), } status_totals = { "running": kpis["stacks_running"], "partial": kpis["stacks_partial"], "stopped": sum(h["stacks"]["stopped"] for h in hosts), "error": sum(h["stacks"]["error"] for h in hosts), } data = { "as_of": datetime.now(timezone.utc).isoformat(), "hosts": hosts, "kpis": kpis, "status_totals": status_totals, "attention": attention, } _fleet_cache["data"] = data _fleet_cache["ts"] = now return data