Show live progress while pruning images (0.61.0)

docker image prune is a single blocking call that reports nothing until it
is done. With -a on a host with many images that meant minutes of a
spinner, and past the proxy timeout an error while the daemon carried on.

The prune now removes its candidates one by one and publishes how far it
is. The candidates are the ones Docker would pick: dangling only by
default, with -a every image no container (running or stopped) uses. The
UI starts the prune in the background (POST /api/images/prune?background=
true, 202) and polls GET /api/images/prune-status. A progress bar above
the image table shows x / y, the image being removed and any failures,
and a toast reports the outcome. It survives a reload and shows up in a
second tab.

- Multi-tag images lose their tags one by one instead of force=True, so a
  container created since the listing still makes the daemon refuse.
- One refused image does not stop the rest; up to ten errors are kept.
- Docker's own prune runs last to sweep up leftovers.
- Freed space is measured (image LayersSize before and after) rather than
  summed, which would count base layers shared with the images that stay.
- Without background=true the endpoint still waits and returns the old
  {ImagesDeleted, SpaceReclaimed} shape, so API-token scripts are
  unaffected. A second prune while one runs gets 409.
- The audit entry now counts deleted images, not Docker's untag entries.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/routers/images.py

@@ -3,11 +3,11 @@ from __future__ import annotations
 
 import json
 
-from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
 from sqlmodel import Session
 
 from auth import get_current_user, require_admin
-from database import get_session
+from database import engine, get_session
 from models.image_scan import ImageScan, ScanDetail, ScanRequest, ScanSummary
 from models.user import User
 from services import audit_service, image_service, scan_service, update_service
@@ -37,19 +37,50 @@ async def check(_user: User = Depends(require_admin)) -> dict:
 @router.post("/prune")
 def prune(
     request: Request,
+    response: Response,
     all_unused: bool = Query(False, alias="all"),
-    session: Session = Depends(get_session),
+    background: bool = Query(False),
     user: User = Depends(require_admin),
 ) -> dict:
-    result = image_service.prune_images(all_unused)
-    audit_service.record(
-        session, user=user.username, action="image.prune", target="*",
-        detail=f"all={all_unused} reclaimed={result.get('SpaceReclaimed')}",
-        ip=_ip(request),
-    )
+    """Remove unused images (``all=true``: also unused tagged ones, ``-a``).
+
+    Waits for the result by default. ``background=true`` returns 202 with the
+    initial status at once and leaves the caller to poll /prune-status, which
+    is what the UI does: a large prune can outlast the proxy's read timeout.
+    """
+    username, ip = user.username, _ip(request)
+
+    def audit(result: dict) -> None:
+        # Docker lists every untagging too; only "Deleted" entries are images.
+        deleted = sum(1 for entry in result["ImagesDeleted"] if "Deleted" in entry)
+        # Its own session: in the background case the request's is long closed.
+        with Session(engine) as own:
+            audit_service.record(
+                own, user=username, action="image.prune", target="*",
+                detail=(
+                    f"all={all_unused} deleted={deleted} "
+                    f"reclaimed={result['SpaceReclaimed']}"
+                ),
+                ip=ip,
+            )
+
+    try:
+        if background:
+            response.status_code = 202
+            return image_service.start_prune(all_unused, on_done=audit)
+        result = image_service.prune_images(all_unused)
+    except image_service.PruneBusy as exc:
+        raise HTTPException(status_code=409, detail=str(exc)) from exc
+    audit(result)
     return result
 
 
+@router.get("/prune-status")
+def prune_status(_admin: User = Depends(require_admin)) -> dict:
+    """Progress of a running (or the last) prune. Admin-only like the prune."""
+    return image_service.prune_status()
+
+
 # --------------------------------------------------------------------------- #
 # Vulnerability scanning
 # --------------------------------------------------------------------------- #

backend/services/image_service.py

@@ -1,9 +1,15 @@
-"""Image listing for the images router."""
+"""Image listing and pruning for the images router."""
 from __future__ import annotations
 
+import logging
+import threading
+from typing import Callable, Optional
+
 from docker_client import DockerError, get_client, get_long_client, safe_call
 from services import update_service
 
+logger = logging.getLogger(__name__)
+
 COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
 
 
@@ -47,15 +53,197 @@ def list_images() -> list[dict]:
     return result
 
 
-def prune_images(all_unused: bool = False) -> dict:
-    """Remove unused images. By default only dangling (untagged) images are
-    removed; ``all_unused=True`` removes every image not referenced by a
-    container (``docker image prune -a``)."""
+# --------------------------------------------------------------------------- #
+# Prune
+# --------------------------------------------------------------------------- #
+
+#: One prune at a time: a second pass over the same images would only collect
+#: errors for everything the first one already removed.
+_prune_lock = threading.Lock()
+_pruning: dict = {
+    "active": False,
+    "run": 0,
+    "all": False,
+    "total": 0,
+    "done": 0,
+    "current": None,
+    "deleted": 0,
+    "failed": 0,
+    "errors": [],
+    "reclaimed": None,
+    "error": None,
+}
+_MAX_ERRORS = 10
+
+
+class PruneBusy(Exception):
+    """A prune is already running."""
+
+
+def prune_status() -> dict:
+    return {**_pruning, "errors": list(_pruning["errors"])}
+
+
+def _layers_size(client) -> Optional[int]:
+    """Total size of all image layers on the host, or None if unavailable.
+
+    Taken before and after the prune, the difference is the space actually
+    freed. Summing the sizes of the deleted images would count base layers
+    they share with the images that stay. ``type=image`` keeps the daemon from
+    also walking every volume (API 1.42+; older daemons ignore it).
+    """
+    try:
+        api = client.api
+        response = api._get(api._url("/system/df"), params={"type": "image"})
+        return api._result(response, True).get("LayersSize")
+    except Exception as exc:  # noqa: BLE001 - the size is a nicety, not the job
+        logger.warning("Could not read image disk usage: %s", exc)
+        return None
+
+
+def _candidates(client, all_unused: bool) -> list[dict]:
+    """The images ``docker image prune [-a]`` would remove, largest first."""
+    in_use = {c.get("ImageID") for c in safe_call(client.api.containers, all=True)}
+    filters = None if all_unused else {"dangling": True}
+    images = safe_call(client.api.images, filters=filters)
+    unused = [img for img in images if img["Id"] not in in_use]
+    return sorted(unused, key=lambda img: img.get("Size", 0), reverse=True)
+
+
+def _label(image: dict) -> str:
+    tags = [t for t in image.get("RepoTags") or [] if t != "<none>:<none>"]
+    if tags:
+        return tags[0] if len(tags) == 1 else f"{tags[0]} (+{len(tags) - 1})"
+    return image["Id"].removeprefix("sha256:")[:12]
+
+
+def _remove(client, image: dict) -> list[dict]:
+    """Delete one image the way the prune would, returning Docker's entries.
+
+    Tags are removed one by one rather than with force=True: the last one takes
+    the image with it, and a container created since the candidates were listed
+    still makes the daemon refuse instead of losing its image.
+    """
+    tags = [t for t in image.get("RepoTags") or [] if t != "<none>:<none>"]
+    entries: list[dict] = []
+    for ref in tags or [image["Id"]]:
+        entries.extend(safe_call(client.api.remove_image, ref) or [])
+    return entries
+
+
+def _begin(all_unused: bool) -> tuple:
     client = get_long_client()
-    # dangling=false tells the engine to also consider tagged-but-unused images.
+    candidates = _candidates(client, all_unused)
+    _pruning.update(
+        {
+            "active": True,
+            "run": _pruning["run"] + 1,
+            "all": all_unused,
+            "total": len(candidates),
+            "done": 0,
+            "current": None,
+            "deleted": 0,
+            "failed": 0,
+            "errors": [],
+            "reclaimed": None,
+            "error": None,
+        }
+    )
+    return client, candidates
+
+
+def _execute(client, candidates: list[dict], all_unused: bool) -> dict:
+    before = _layers_size(client)
+    deleted: list[dict] = []
+    for image in candidates:
+        _pruning["current"] = _label(image)
+        try:
+            entries = _remove(client, image)
+            deleted.extend(entries)
+            if any("Deleted" in e for e in entries):
+                _pruning["deleted"] += 1
+        except DockerError as exc:
+            # One image must not stop the rest, e.g. a container started from
+            # it since the list was taken.
+            logger.warning("Prune: could not remove %s: %s", _pruning["current"], exc)
+            _pruning["failed"] += 1
+            if len(_pruning["errors"]) < _MAX_ERRORS:
+                _pruning["errors"].append(f"{_pruning['current']}: {exc.detail or exc.error}")
+        _pruning["done"] += 1
+    _pruning["current"] = None
+
+    # Let the daemon sweep up whatever the per-image pass left, such as
+    # untagged images referenced from several repositories.
     filters = {"dangling": False} if all_unused else {"dangling": True}
-    result = safe_call(client.images.prune, filters=filters)
-    return {
-        "ImagesDeleted": result.get("ImagesDeleted") or [],
-        "SpaceReclaimed": result.get("SpaceReclaimed", 0),
-    }
+    sweep = safe_call(client.images.prune, filters=filters)
+    deleted.extend(sweep.get("ImagesDeleted") or [])
+
+    after = _layers_size(client)
+    if before is not None and after is not None:
+        reclaimed = max(before - after, 0)
+    else:
+        reclaimed = sweep.get("SpaceReclaimed", 0)
+    _pruning["reclaimed"] = reclaimed
+    return {"ImagesDeleted": deleted, "SpaceReclaimed": reclaimed}
+
+
+def _finish(error: Optional[str] = None) -> None:
+    _pruning.update({"active": False, "current": None, "error": error})
+    _prune_lock.release()
+
+
+def prune_images(all_unused: bool = False) -> dict:
+    """Remove unused images and wait for the result.
+
+    By default only dangling (untagged) images are removed; ``all_unused=True``
+    removes every image not referenced by a container (``docker image prune
+    -a``). Progress is visible through prune_status() while it runs.
+    """
+    if not _prune_lock.acquire(blocking=False):
+        raise PruneBusy("An image prune is already running")
+    error = None
+    try:
+        client, candidates = _begin(all_unused)
+        return _execute(client, candidates, all_unused)
+    except DockerError as exc:
+        error = exc.detail or exc.error
+        raise
+    finally:
+        _finish(error)
+
+
+def start_prune(
+    all_unused: bool = False, on_done: Optional[Callable[[dict], None]] = None
+) -> dict:
+    """Start a prune in the background and return its initial status.
+
+    A prune removing gigabytes of layers can outlast the proxy's read timeout,
+    so the UI starts it this way and polls prune_status(). ``on_done`` gets
+    the same result prune_images() returns, from the worker thread.
+    """
+    if not _prune_lock.acquire(blocking=False):
+        raise PruneBusy("An image prune is already running")
+    try:
+        client, candidates = _begin(all_unused)
+    except BaseException:
+        _prune_lock.release()
+        raise
+
+    def work() -> None:
+        error = None
+        result = None
+        try:
+            result = _execute(client, candidates, all_unused)
+        except Exception as exc:  # noqa: BLE001 - reported through the status
+            logger.warning("Image prune failed: %s", exc)
+            error = exc.detail if isinstance(exc, DockerError) else str(exc)
+        finally:
+            _finish(error)
+        if result is not None and on_done:
+            try:
+                on_done(result)
+            except Exception as exc:  # noqa: BLE001 - the prune itself succeeded
+                logger.warning("Image prune follow-up failed: %s", exc)
+
+    threading.Thread(target=work, name="image-prune", daemon=True).start()
+    return prune_status()

backend/tests/test_docker_timeouts.py

@@ -54,7 +54,6 @@ def test_volume_prune_timeout_is_a_502_with_detail(as_admin, monkeypatch):
     "module, fn, args",
     [
         ("volume_service", "prune_volumes", ()),
-        ("image_service", "prune_images", (True,)),
         ("network_service", "prune_networks", ()),
     ],
 )
@@ -79,3 +78,7 @@ def test_prunes_use_the_long_timeout_client(monkeypatch, module, fn, args):
     monkeypatch.setattr(svc, "get_long_client", lambda: Client())
     getattr(svc, fn)(*args)
     assert used == ["long"]
+
+
+# The image prune goes image by image and has its own tests in
+# test_image_prune.py, including that it uses the long-timeout client.

backend/tests/test_image_prune.py

@@ -0,0 +1,242 @@
+"""Image prune with visible progress.
+
+``docker image prune -a`` is one blocking call that reports nothing until it
+is done; on a host with a lot of images that meant minutes of a spinner and
+then, past the proxy timeout, an error while the daemon carried on. The prune
+now removes the candidates one by one, publishes how far it is through
+/api/images/prune-status, and the UI starts it in the background and polls.
+
+What these tests pin: the candidate set matches Docker's (in-use images are
+never touched, dangling-only without -a), multi-tag images lose every tag,
+one failing image does not stop the rest, reclaimed space is measured rather
+than summed, and the endpoints keep the old synchronous contract for scripts.
+"""
+from __future__ import annotations
+
+import threading
+
+import pytest
+from docker.errors import APIError
+
+MB = 1024 * 1024
+
+
+class FakeApi:
+    """Just enough of docker-py's low-level APIClient for the prune."""
+
+    def __init__(self, images, containers, layers=500 * MB):
+        self._images = images
+        self._containers = containers
+        self.layers = layers
+        self.removed: list[str] = []
+        self.refuse: set[str] = set()
+        self.gate: threading.Event | None = None
+
+    def containers(self, all=False):
+        return self._containers
+
+    def images(self, filters=None):
+        if filters and filters.get("dangling"):
+            return [i for i in self._images if not i.get("RepoTags")]
+        return list(self._images)
+
+    def remove_image(self, ref, force=False, noprune=False):
+        if self.gate:
+            self.gate.wait(5)
+        if ref in self.refuse:
+            raise APIError(f"conflict: unable to remove {ref}")
+        self.removed.append(ref)
+        for img in self._images:
+            if ref in (img.get("RepoTags") or []):
+                img["RepoTags"].remove(ref)
+                if img["RepoTags"]:
+                    return [{"Untagged": ref}]
+                self._images.remove(img)
+                self.layers -= img["Size"]
+                return [{"Untagged": ref}, {"Deleted": img["Id"]}]
+            if img["Id"] == ref:
+                self._images.remove(img)
+                self.layers -= img["Size"]
+                return [{"Deleted": img["Id"]}]
+        raise APIError(f"No such image: {ref}")
+
+    # /system/df?type=image
+    def _url(self, path):
+        return path
+
+    def _get(self, url, params=None):
+        assert params == {"type": "image"}
+        return self
+
+    def _result(self, response, json=False):
+        return {"LayersSize": self.layers}
+
+
+class FakeImages:
+    def __init__(self, api):
+        self.api = api
+        self.prune_filters = None
+
+    def prune(self, filters=None):
+        self.prune_filters = filters
+        return {"ImagesDeleted": None, "SpaceReclaimed": 0}
+
+
+class FakeClient:
+    def __init__(self, api):
+        self.api = api
+        self.images = FakeImages(api)
+
+
+def _host():
+    images = [
+        {"Id": "sha256:aaa", "RepoTags": ["nginx:1.25"], "Size": 100 * MB},
+        {"Id": "sha256:bbb", "RepoTags": ["redis:7", "redis:latest"], "Size": 50 * MB},
+        {"Id": "sha256:ccc", "RepoTags": [], "Size": 20 * MB},
+        {"Id": "sha256:ddd", "RepoTags": ["postgres:16"], "Size": 200 * MB},
+    ]
+    # postgres is used by a (possibly stopped) container and must survive.
+    containers = [{"ImageID": "sha256:ddd"}]
+    return FakeApi(images, containers)
+
+
+@pytest.fixture
+def svc(monkeypatch):
+    from services import image_service
+
+    api = _host()
+    client = FakeClient(api)
+
+    def short():
+        raise AssertionError("prune must not use the 60s client")
+
+    monkeypatch.setattr(image_service, "get_client", short)
+    monkeypatch.setattr(image_service, "get_long_client", lambda: client)
+    image_service._pruning.update({"active": False, "run": 0})
+    yield image_service, api, client
+    if image_service._prune_lock.locked():  # a test that failed mid-prune
+        image_service._prune_lock.release()
+
+
+def test_all_unused_removes_everything_no_container_uses(svc):
+    image_service, api, client = svc
+
+    result = image_service.prune_images(all_unused=True)
+
+    assert {i["Id"] for i in api._images} == {"sha256:ddd"}
+    # Every tag of a multi-tag image is removed, without force.
+    assert set(api.removed) == {"nginx:1.25", "redis:7", "redis:latest", "sha256:ccc"}
+    assert {e["Deleted"] for e in result["ImagesDeleted"] if "Deleted" in e} == {
+        "sha256:aaa", "sha256:bbb", "sha256:ccc",
+    }
+    assert result["SpaceReclaimed"] == 170 * MB
+    # The daemon's own prune still runs afterwards to sweep up leftovers.
+    assert client.images.prune_filters == {"dangling": False}
+
+    status = image_service.prune_status()
+    assert status["active"] is False
+    assert (status["total"], status["done"], status["deleted"]) == (3, 3, 3)
+    assert status["reclaimed"] == 170 * MB
+
+
+def test_default_only_touches_dangling_images(svc):
+    image_service, api, client = svc
+
+    image_service.prune_images(all_unused=False)
+
+    assert api.removed == ["sha256:ccc"]
+    assert client.images.prune_filters == {"dangling": True}
+
+
+def test_one_refused_image_does_not_stop_the_rest(svc):
+    image_service, api, _ = svc
+    api.refuse.add("nginx:1.25")
+
+    image_service.prune_images(all_unused=True)
+
+    status = image_service.prune_status()
+    assert status["failed"] == 1 and status["done"] == 3 and status["deleted"] == 2
+    assert status["errors"][0].startswith("nginx:1.25: ")
+    assert any(i["Id"] == "sha256:aaa" for i in api._images)
+
+
+def test_background_prune_reports_progress_and_blocks_a_second_one(svc):
+    image_service, api, _ = svc
+    api.gate = threading.Event()
+    finished = threading.Event()
+
+    started = image_service.start_prune(True, on_done=lambda r: finished.set())
+    assert started["active"] is True and started["total"] == 3
+    with pytest.raises(image_service.PruneBusy):
+        image_service.start_prune(True)
+
+    api.gate.set()
+    assert finished.wait(5)
+    status = image_service.prune_status()
+    assert status["run"] == started["run"]
+    assert status["done"] == 3 and status["reclaimed"] == 170 * MB
+
+
+def test_background_prune_error_lands_in_the_status(svc):
+    from docker_client import DockerError
+
+    image_service, _, client = svc
+
+    def broken(filters=None):
+        raise DockerError("docker_timeout", "Docker did not answer in time")
+
+    client.images.prune = broken
+    image_service.start_prune(True)
+    for _ in range(100):
+        if not image_service.prune_status()["active"]:
+            break
+        threading.Event().wait(0.02)
+    status = image_service.prune_status()
+    assert status["active"] is False
+    assert status["error"] == "Docker did not answer in time"
+    assert not image_service._prune_lock.locked()
+
+
+def test_endpoint_waits_by_default_and_is_audited(svc, as_admin):
+    from sqlmodel import Session, select
+
+    from database import engine
+    from models.audit import AuditLog
+
+    r = as_admin.post("/api/images/prune", params={"all": True})
+    assert r.status_code == 200
+    assert r.json()["SpaceReclaimed"] == 170 * MB
+
+    with Session(engine) as session:
+        row = session.exec(
+            select(AuditLog).where(AuditLog.action == "image.prune").order_by(AuditLog.id.desc())
+        ).first()
+    assert row.detail == f"all=True deleted=3 reclaimed={170 * MB}"
+
+
+def test_endpoint_background_returns_202_and_status_is_pollable(svc, as_admin):
+    r = as_admin.post("/api/images/prune", params={"all": True, "background": True})
+    assert r.status_code == 202
+    assert r.json()["total"] == 3
+
+    for _ in range(100):
+        s = as_admin.get("/api/images/prune-status").json()
+        if not s["active"]:
+            break
+        threading.Event().wait(0.02)
+    assert s["done"] == 3 and s["reclaimed"] == 170 * MB
+
+
+def test_endpoint_rejects_a_second_prune(svc, as_admin):
+    image_service, _, _ = svc
+    image_service._prune_lock.acquire()
+    try:
+        r = as_admin.post("/api/images/prune", params={"background": True})
+        assert r.status_code == 409
+    finally:
+        image_service._prune_lock.release()
+
+
+def test_prune_is_admin_only(svc, as_user):
+    assert as_user.post("/api/images/prune").status_code == 403
+    assert as_user.get("/api/images/prune-status").status_code == 403

backend/version.py

@@ -1,3 +1,3 @@
 """Single source of truth for the StackPilot release version."""
 
-APP_VERSION = "0.60.1"
+APP_VERSION = "0.61.0"