stackpilot/backend/services/dashboard_service.py

"""Host aggregate for the dashboard cockpit.

One read-only call rolls the host up into a "needs attention" list, headline
KPIs and a resource view. It is cheap by construction: a single container
*summary* list (no per-container inspect) drives the figures, and image
freshness comes from the cache the update-service background loop already
maintains.
"""
from __future__ import annotations

import asyncio
import time
from datetime import datetime, timedelta, timezone

from sqlmodel import Session, select

from docker_client import DockerError, get_client, safe_call
from models.backup_schedule import BackupSchedule, describe_stacks, status_of
from services import compose_service, docker_watch_service, update_service

COMPOSE_LABEL = compose_service.COMPOSE_LABEL

# Cached briefly: the dashboard polls this and the rollup is not free.
FLEET_TTL = 25.0
DISK_PRESSURE = 0.85  # disk used fraction above which a host needs attention
MEM_PRESSURE = 0.90   # memory used fraction above which a host needs attention

_fleet_cache: dict = {"data": None, "ts": 0.0, "gen": None}


# --------------------------------------------------------------------------- #
# Container summary (single Docker round-trip) + shared classifiers
# --------------------------------------------------------------------------- #


def _list_containers() -> list[dict]:
    client = get_client()
    return safe_call(client.api.containers, all=True)


def _group_by_project(raw: list[dict]) -> dict[str, list[dict]]:
    by_project: dict[str, list[dict]] = {}
    for c in raw:
        project = (c.get("Labels") or {}).get(COMPOSE_LABEL)
        if project:
            by_project.setdefault(project, []).append(c)
    return by_project


def _is_healthy(containers: list[dict]) -> bool:
    """All containers that *have* a healthcheck report healthy.

    The summary ``Status`` string carries the health suffix — "(healthy)",
    "(unhealthy)" or "(health: starting)" — only for containers with a
    healthcheck configured, so its absence simply means "no healthcheck".
    """
    for c in containers:
        status = c.get("Status", "") or ""
        if "(unhealthy)" in status or "(health:" in status:
            return False
    return True


def _is_updated(containers: list[dict], cache: dict[str, dict]) -> bool:
    for c in containers:
        st = cache.get(c.get("Image", ""))
        if st and st.get("update_available"):
            return False
    return True


# --------------------------------------------------------------------------- #
# Host aggregate (one call → "needs attention" + KPIs)
#
# The dashboard used to fetch these numbers per stack and recombine them
# client-side. ``compute_fleet`` rolls them up server-side instead, off one
# Docker pass plus the update cache, and holds the result for FLEET_TTL because
# the dashboard polls it.
# --------------------------------------------------------------------------- #

# Stack-status buckets the status bar / KPIs are built from. Anything reporting
# "error" or "dead" containers counts as a problem stack.
_PROBLEM_STATUSES = {"error", "dead"}


def _bucket_statuses(statuses: list[str]) -> dict[str, int]:
    return {
        "running": sum(1 for s in statuses if s == "running"),
        "partial": sum(1 for s in statuses if s == "partial"),
        "stopped": sum(1 for s in statuses if s in ("stopped", "exited")),
        "error": sum(1 for s in statuses if s in _PROBLEM_STATUSES),
        "total": len(statuses),
    }


def _attn(severity: str, kind: str, host: str, title: str, detail: str, link: str) -> dict:
    return {
        "severity": severity,
        "kind": kind,
        "host": host,
        "title": title,
        "detail": detail,
        "link": link,
    }


def _resource_attention(name: str, link: str, mem_used: int, mem_total: int,
                        disk_used: int, disk_total: int) -> list[dict]:
    items: list[dict] = []
    if mem_total and mem_used / mem_total >= MEM_PRESSURE:
        pct = round(mem_used / mem_total * 100)
        items.append(_attn("warn", "mem_pressure", name,
                            f"{name}: memory at {pct}%", "Free memory or move stacks.", link))
    if disk_total and disk_used / disk_total >= DISK_PRESSURE:
        pct = round(disk_used / disk_total * 100)
        items.append(_attn("warn", "disk_pressure", name,
                            f"{name}: disk at {pct}%", "Prune images/volumes or add capacity.", link))
    return items


def _local_host() -> tuple[dict, list[dict]]:
    """Local host card + attention items from a single Docker container pass."""
    discovered = compose_service.discover_stacks()
    try:
        raw = _list_containers()
    except DockerError:
        raw = []
    by_project = _group_by_project(raw)
    update_cache = update_service.get_cache()

    statuses: list[str] = []
    unhealthy: list[str] = []
    updates = 0
    for stack_id in discovered:
        containers = by_project.get(compose_service.project_name(stack_id), [])
        status = compose_service._status_from_states([c.get("State", "") for c in containers])
        statuses.append(status)
        if status == "running" and not _is_healthy(containers):
            unhealthy.append(stack_id)
        if not _is_updated(containers, update_cache):
            updates += 1

    buckets = _bucket_statuses(statuses)
    labelled = [c for c in raw if (c.get("Labels") or {}).get(COMPOSE_LABEL)]

    # Local resource figures (lazy import keeps dashboard_service free of a
    # router dependency at module load time).
    from routers.system import _cpu_count, _disk_usage, _mem_info
    mem = _mem_info()
    disk = _disk_usage()

    host = {
        "id": "local",
        "name": "local",
        "online": True,
        "status": "online",
        "cpu_cores": _cpu_count(),
        "mem_used": mem["used"], "mem_total": mem["total"],
        "disk_used": disk["used"], "disk_total": disk["total"],
        "stacks": buckets,
        "containers_running": sum(1 for c in labelled if c.get("State") == "running"),
        "containers_total": len(labelled),
        "unhealthy": len(unhealthy),
        "updates_available": updates,
    }

    attention: list[dict] = []
    for sid in unhealthy:
        attention.append(_attn("error", "unhealthy", "local",
                               f"{sid} is unhealthy", "A container is failing its healthcheck.",
                               f"/stacks/{sid}"))
    if buckets["error"]:
        attention.append(_attn("error", "stack_error", "local",
                               f"{buckets['error']} stack(s) in error", "Containers are dead.", "/stacks"))
    if buckets["partial"]:
        attention.append(_attn("warn", "stack_partial", "local",
                               f"{buckets['partial']} stack(s) partially running",
                               "Some services are down.", "/stacks"))
    if updates:
        attention.append(_attn("warn", "updates", "local",
                               f"{updates} stack(s) have image updates", "Pull the newer images.", "/images"))
    attention += _resource_attention("local", "/", mem["used"], mem["total"],
                                     disk["used"], disk["total"])
    return host, attention


def _backup_attention(session: Session) -> list[dict]:
    """Flag enabled backup schedules whose last run failed or is overdue."""
    now = datetime.now(timezone.utc)
    items: list[dict] = []
    for sch in session.exec(select(BackupSchedule).where(BackupSchedule.enabled == True)).all():  # noqa: E712
        host = "local"
        status = (sch.last_status or "").lower()
        # Name the stacks that actually had a problem, not the whole schedule.
        failing = sorted(
            sid for sid, entry in status_of(sch).items()
            if str((entry or {}).get("status", "")) != "ok"
        )
        what = ", ".join(failing) if failing else describe_stacks(sch)
        if status and not status.startswith("ok"):
            incomplete = status.startswith("warning")
            items.append(_attn("warn" if incomplete else "error", "backup_failed", host,
                               f"Backup of {what} {'is incomplete' if incomplete else 'failed'}",
                               sch.last_status or "", "/settings"))
        elif sch.next_run and _aware(sch.next_run) < now - timedelta(hours=1):
            items.append(_attn("warn", "backup_overdue", host,
                               f"Backup of {describe_stacks(sch)} is overdue",
                               "Scheduled run did not happen.", "/settings"))
    return items


def _aware(dt: datetime) -> datetime:
    """Treat naive DB timestamps as UTC (they're stored that way)."""
    return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)


_SEVERITY_ORDER = {"error": 0, "warn": 1}


async def compute_fleet(session: Session, refresh: bool = False) -> dict:
    now = time.time()
    # Stale as soon as Docker reported a change: the UI refetches on that very
    # event, and got the rollup from before it — a stack stopped in a terminal
    # stayed "running" in the tiles for up to FLEET_TTL. Read before the work,
    # so a change during it is not mistaken for one already counted.
    gen = docker_watch_service.generation()
    if (
        not refresh
        and _fleet_cache["data"]
        and now - _fleet_cache["ts"] < FLEET_TTL
        and _fleet_cache["gen"] == gen
    ):
        return _fleet_cache["data"]

    local_host, attention = await asyncio.to_thread(_local_host)
    hosts = [local_host]
    attention += _backup_attention(session)

    attention.sort(key=lambda a: _SEVERITY_ORDER.get(a["severity"], 9))

    kpis = {
        "stacks_running": sum(h["stacks"]["running"] for h in hosts),
        "stacks_partial": sum(h["stacks"]["partial"] for h in hosts),
        "stacks_total": sum(h["stacks"]["total"] for h in hosts),
        "containers_running": sum(h["containers_running"] for h in hosts),
        "containers_total": sum(h["containers_total"] for h in hosts),
        "unhealthy": sum(h["unhealthy"] for h in hosts),
        "updates_available": sum(h["updates_available"] for h in hosts),
        "backups_failing": sum(1 for a in attention if a["kind"] in ("backup_failed", "backup_overdue")),
    }
    status_totals = {
        "running": kpis["stacks_running"],
        "partial": kpis["stacks_partial"],
        "stopped": sum(h["stacks"]["stopped"] for h in hosts),
        "error": sum(h["stacks"]["error"] for h in hosts),
    }

    data = {
        "as_of": datetime.now(timezone.utc).isoformat(),
        "hosts": hosts,
        "kpis": kpis,
        "status_totals": status_totals,
        "attention": attention,
    }
    _fleet_cache["data"] = data
    _fleet_cache["ts"] = now
    _fleet_cache["gen"] = gen
    return data