From 5b59f5e8f9c5d0d12a1d013e26552340e3e8f0fe Mon Sep 17 00:00:00 2001 From: menzelj Date: Mon, 8 Jun 2026 18:53:12 +0000 Subject: [PATCH] =?UTF-8?q?Perf:=20serve=20stacks=20list=20from=20one=20Do?= =?UTF-8?q?cker=20call;=20registry=20host=20=E2=86=92=2010.10.6.10=20(0.21?= =?UTF-8?q?.6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stacks overview and detail were doing an N+1 inspect storm: list_stacks called containers_for_stack AND compute_status (which re-fetched) per stack, and containers.list(sparse=False) full-inspects every container plus c.image triggered an image-inspect each. For N stacks that was ~2N*(1 list + M inspects + M image-inspects) sequential socket round trips (~1s for just 2 stacks, growing linearly). - compose_service.stack_status_summaries(): one low-level api.containers(all=True) summary call grouped by compose project label → whole list served in a single Docker round-trip (~10x faster). - compute_status() takes optional pre-fetched containers; get_stack and _stack_summary no longer double-fetch. - containers_for_stack() reads the image name from the inspect it already has instead of c.image (drops the per-container image-inspect). - Same batching applied to the agent's stack list/detail. Also: Forgejo (registry + git) moved to 10.10.6.10:3020 — updated image refs in docker-compose.yml, agent/Dockerfile, agent/docker-compose.yml. Co-Authored-By: Claude Opus 4.8 --- agent/Dockerfile | 2 +- agent/docker-compose.yml | 4 +- backend/agent_app.py | 42 ++++++++++++------ backend/main.py | 2 +- backend/routers/stacks.py | 48 ++++++++++++++------ backend/services/compose_service.py | 68 +++++++++++++++++++++-------- docker-compose.yml | 4 +- frontend/package.json | 2 +- 8 files changed, 120 insertions(+), 52 deletions(-) diff --git a/agent/Dockerfile b/agent/Dockerfile index 0ade2b7..5845ec1 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -1,6 +1,6 @@ # The agent reuses the backend image (same compose/Docker code + deps) and # just runs a different ASGI app. Build the backend image first. -ARG BACKEND_IMAGE=10.10.5.10:3020/menzelj/stackpilot-backend:latest +ARG BACKEND_IMAGE=10.10.6.10:3020/menzelj/stackpilot-backend:latest FROM ${BACKEND_IMAGE} ENV STACKS_DIR=/opt/stacks \ diff --git a/agent/docker-compose.yml b/agent/docker-compose.yml index b401cad..b64fc16 100644 --- a/agent/docker-compose.yml +++ b/agent/docker-compose.yml @@ -3,11 +3,11 @@ # token you enter when adding this host in the central StackPilot UI). services: agent: - image: 10.10.5.10:3020/menzelj/stackpilot-agent:latest + image: 10.10.6.10:3020/menzelj/stackpilot-agent:latest build: context: . args: - BACKEND_IMAGE: 10.10.5.10:3020/menzelj/stackpilot-backend:latest + BACKEND_IMAGE: 10.10.6.10:3020/menzelj/stackpilot-backend:latest restart: unless-stopped environment: - AGENT_TOKEN=${AGENT_TOKEN:?set AGENT_TOKEN in .env} diff --git a/backend/agent_app.py b/backend/agent_app.py index 9a33b9e..bf4154e 100644 --- a/backend/agent_app.py +++ b/backend/agent_app.py @@ -62,7 +62,7 @@ def _map_docker(exc: DockerError): raise HTTPException(status_code=code, detail=exc.detail or exc.error) raise exc # falls through to the global 502 DockerError handler -AGENT_VERSION = "0.21.5" +AGENT_VERSION = "0.21.6" # --------------------------------------------------------------------------- # @@ -137,20 +137,31 @@ def _file_guard(fn, *args, **kwargs): # --------------------------------------------------------------------------- # -def _summary(stack_id: str) -> dict: - try: - containers = compose_service.containers_for_stack(stack_id) - status = compose_service.compute_status(stack_id) - except DockerError: - containers = [] - status = "unknown" +def _summary(stack_id: str, summaries: dict | None = None) -> dict: + if summaries is None: + try: + containers = compose_service.containers_for_stack(stack_id) + total = len(containers) + running = sum(1 for c in containers if c.state == "running") + status = compose_service.compute_status(stack_id, containers) + except DockerError: + total = running = 0 + status = "unknown" + else: + info = summaries.get(stack_id) + total = info["total"] if info else 0 + running = info["running"] if info else 0 + if compose_service.is_busy(stack_id): + status = "updating" + else: + status = info["status"] if info else "stopped" return { "id": stack_id, "name": stack_id, "description": None, "status": status, - "service_count": len(containers), - "running_count": sum(1 for c in containers if c.state == "running"), + "service_count": total, + "running_count": running, "created_at": None, "updated_at": None, } @@ -245,7 +256,11 @@ def system() -> dict: @app.get("/agent/stacks", dependencies=[Depends(verify_token)]) def list_stacks() -> list[dict]: - return [_summary(sid) for sid in compose_service.discover_stacks()] + try: + summaries = compose_service.stack_status_summaries() + except DockerError: + summaries = {} + return [_summary(sid, summaries) for sid in compose_service.discover_stacks()] @app.get("/agent/stacks/stats", dependencies=[Depends(verify_token)]) @@ -259,8 +274,9 @@ def get_stack(stack_id: str) -> dict: if not os.path.isdir(directory): raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found") try: - containers = [asdict(c) for c in compose_service.containers_for_stack(stack_id)] - status = compose_service.compute_status(stack_id) + raw = compose_service.containers_for_stack(stack_id) + containers = [asdict(c) for c in raw] + status = compose_service.compute_status(stack_id, raw) except DockerError: containers = [] status = "unknown" diff --git a/backend/main.py b/backend/main.py index 918d322..f4c38ba 100644 --- a/backend/main.py +++ b/backend/main.py @@ -55,7 +55,7 @@ async def lifespan(app: FastAPI): schedule_task.cancel() -app = FastAPI(title="StackPilot", version="0.21.5", lifespan=lifespan) +app = FastAPI(title="StackPilot", version="0.21.6", lifespan=lifespan) app.add_middleware( CORSMiddleware, diff --git a/backend/routers/stacks.py b/backend/routers/stacks.py index 169c2f8..61e26ef 100644 --- a/backend/routers/stacks.py +++ b/backend/routers/stacks.py @@ -58,20 +58,37 @@ def _get_stack_or_404(session: Session, stack_id: str) -> Stack: return stack -def _stack_summary(stack: Stack) -> dict: - try: - containers = compose_service.containers_for_stack(stack.id) - status = compose_service.compute_status(stack.id) - except DockerError: - containers = [] - status = "unknown" +def _stack_summary(stack: Stack, summaries: dict | None = None) -> dict: + """Build a list-row summary. + + Pass ``summaries`` (from :func:`compose_service.stack_status_summaries`) to + serve the whole stacks list from a single Docker call. Without it (single + create/update/clone responses), fall back to one direct query for this stack. + """ + if summaries is None: + try: + containers = compose_service.containers_for_stack(stack.id) + total = len(containers) + running = sum(1 for c in containers if c.state == "running") + status = compose_service.compute_status(stack.id, containers) + except DockerError: + total = running = 0 + status = "unknown" + else: + info = summaries.get(stack.id) + total = info["total"] if info else 0 + running = info["running"] if info else 0 + if compose_service.is_busy(stack.id): + status = "updating" + else: + status = info["status"] if info else "stopped" return { "id": stack.id, "name": stack.name, "description": stack.description, "status": status, - "service_count": len(containers), - "running_count": sum(1 for c in containers if c.state == "running"), + "service_count": total, + "running_count": running, "created_at": stack.created_at, "updated_at": stack.updated_at, } @@ -89,7 +106,11 @@ def list_stacks( ) -> list[dict]: sync_discovered_stacks(session) stacks = session.exec(select(Stack)).all() - return [_stack_summary(s) for s in stacks] + try: + summaries = compose_service.stack_status_summaries() + except DockerError: + summaries = {} + return [_stack_summary(s, summaries) for s in stacks] @router.post("", status_code=201) @@ -130,9 +151,10 @@ def get_stack( ) -> dict: stack = _get_stack_or_404(session, stack_id) try: - containers = [asdict(c) for c in compose_service.containers_for_stack(stack_id)] - status = compose_service.compute_status(stack_id) - except DockerError as exc: + raw = compose_service.containers_for_stack(stack_id) + containers = [asdict(c) for c in raw] + status = compose_service.compute_status(stack_id, raw) + except DockerError: containers = [] status = "unknown" return { diff --git a/backend/services/compose_service.py b/backend/services/compose_service.py index 32d9dc6..814db65 100644 --- a/backend/services/compose_service.py +++ b/backend/services/compose_service.py @@ -184,7 +184,10 @@ def containers_for_stack(stack_id: str) -> list[ContainerInfo]: id=c.id, name=c.name, service=c.labels.get(SERVICE_LABEL, c.name), - image=(c.image.tags[0] if c.image and c.image.tags else attrs.get("Config", {}).get("Image", "")), + # Use the configured image name from the inspect we already have; + # reading c.image.tags would trigger a separate image-inspect call + # per container. + image=attrs.get("Config", {}).get("Image", "") or attrs.get("Image", ""), state=state.get("Status", c.status), status=attrs.get("State", {}).get("Status", c.status), health=health, @@ -207,21 +210,14 @@ def clear_busy(stack_id: str) -> None: _BUSY.discard(stack_id) -def compute_status(stack_id: str) -> str: - if stack_id in _BUSY: - return "updating" - try: - containers = containers_for_stack(stack_id) - except DockerError: - return "unknown" - if not containers: +def is_busy(stack_id: str) -> bool: + return stack_id in _BUSY + + +def _status_from_states(states: list[str]) -> str: + if not states: return "stopped" - states = [c.state for c in containers] - if any(s in ("dead",) for s in states): - return "error" - if any( - c.state == "exited" and _nonzero_exit(c) for c in containers - ): + if any(s == "dead" for s in states): return "error" running = [s for s in states if s == "running"] if len(running) == len(states): @@ -231,10 +227,44 @@ def compute_status(stack_id: str) -> str: return "stopped" -def _nonzero_exit(c: ContainerInfo) -> bool: - # We only have the textual state here; treat plain "exited" as stopped, not - # an error unless health says otherwise. Detailed exit codes handled in detail view. - return False +def compute_status(stack_id: str, containers: Optional[list[ContainerInfo]] = None) -> str: + """Status for one stack. Pass already-fetched ``containers`` to avoid a + redundant Docker round-trip (the detail view already has them).""" + if stack_id in _BUSY: + return "updating" + try: + if containers is None: + containers = containers_for_stack(stack_id) + except DockerError: + return "unknown" + return _status_from_states([c.state for c in containers]) + + +def stack_status_summaries() -> dict[str, dict]: + """One cheap Docker call → ``{project: {status, total, running}}`` for every + stack at once. + + Uses the low-level container *summary* list (``GET /containers/json``, no + per-container inspect) grouped by the compose project label. This is far + cheaper than calling :func:`containers_for_stack` (which full-inspects every + container) once per stack for the overview — turning the stacks list from + O(stacks × containers) Docker round-trips into a single one. + """ + client = get_client() + raw = safe_call(client.api.containers, all=True) + by_project: dict[str, list[str]] = {} + for c in raw: + project = (c.get("Labels") or {}).get(COMPOSE_LABEL) + if project: + by_project.setdefault(project, []).append(c.get("State", "")) + return { + project: { + "status": _status_from_states(states), + "total": len(states), + "running": sum(1 for s in states if s == "running"), + } + for project, states in by_project.items() + } # --------------------------------------------------------------------------- # diff --git a/docker-compose.yml b/docker-compose.yml index c258bcb..ee3d0bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: backend: - image: 10.10.5.10:3020/menzelj/stackpilot-backend:latest + image: 10.10.6.10:3020/menzelj/stackpilot-backend:latest build: ./backend restart: unless-stopped environment: @@ -38,7 +38,7 @@ services: # - "5008:5008" frontend: - image: 10.10.5.10:3020/menzelj/stackpilot-frontend:latest + image: 10.10.6.10:3020/menzelj/stackpilot-frontend:latest build: ./frontend restart: unless-stopped depends_on: diff --git a/frontend/package.json b/frontend/package.json index f1746f1..651a571 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "stackpilot-frontend", "private": true, - "version": "0.21.5", + "version": "0.21.6", "type": "module", "scripts": { "dev": "vite",