diff --git a/README.md b/README.md index 19107a2..1713df4 100644 --- a/README.md +++ b/README.md @@ -804,7 +804,12 @@ GET /api/images/prune-status (progress of the running / last prune still labelled with a stack that has been deleted form their own group, marked *stack removed* — that is where forgotten data collects. - Admin actions: delete a volume (with an in-use warning + force option) and - **Prune unused**; an *Only unused* filter. + **Prune unused**; an *Only unused* filter. The prune asks first and removes + only unused **anonymous** volumes unless *Also remove unused named volumes + (`-a`)* is ticked, which also takes the data of every stack that is down. It + runs in the background with the same **live progress** as the image prune and + reports the space freed (`POST /api/volumes/prune?all=…&background=true`, + `GET /api/volumes/prune-status`). - **Volume sizes** are loaded on demand via a *Compute sizes* button (runs `docker system df`, which walks volume contents and can take a few seconds); results are cached ~60s. Endpoint `GET /api/volumes/sizes`. @@ -1010,7 +1015,8 @@ WS /ws/deploy/{stack_id} WS /ws/update/{stack_id} ``` GET /api/volumes | /orphaned DELETE /api/volumes/{name} -POST /api/volumes/prune POST /api/volumes/generate-yaml +POST /api/volumes/prune?all=&background= POST /api/volumes/generate-yaml +GET /api/volumes/prune-status GET /api/host/paths?path=&show_hidden= (sandboxed browser) POST /api/editor/services | add-volume | set-gpu | add-device | remove-device | set-privileged ``` diff --git a/backend/routers/images.py b/backend/routers/images.py index 024728a..901840d 100644 --- a/backend/routers/images.py +++ b/backend/routers/images.py @@ -11,6 +11,7 @@ from database import engine, get_session from models.image_scan import ImageScan, ScanDetail, ScanRequest, ScanSummary from models.user import User from services import audit_service, image_service, scan_service, update_service +from services.prune_job import PruneBusy router = APIRouter(prefix="/api/images", tags=["images"]) @@ -69,7 +70,7 @@ def prune( response.status_code = 202 return image_service.start_prune(all_unused, on_done=audit) result = image_service.prune_images(all_unused) - except image_service.PruneBusy as exc: + except PruneBusy as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc audit(result) return result diff --git a/backend/routers/volumes.py b/backend/routers/volumes.py index 8165f33..4a2b6ab 100644 --- a/backend/routers/volumes.py +++ b/backend/routers/volumes.py @@ -1,13 +1,14 @@ """Volume management, NFS/SMB YAML generation, and host path browser.""" from __future__ import annotations -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from sqlmodel import Session from auth import get_current_user, require_admin -from database import get_session +from database import engine, get_session from models.user import User from services import audit_service, device_service, volume_service +from services.prune_job import PruneBusy router = APIRouter(tags=["volumes"]) @@ -67,17 +68,45 @@ def delete_volume( @router.post("/api/volumes/prune") def prune( request: Request, - session: Session = Depends(get_session), + response: Response, + all_unused: bool = Query(False, alias="all"), + background: bool = Query(False), user: User = Depends(require_admin), ) -> dict: - result = volume_service.prune_volumes() - audit_service.record( - session, user=user.username, action="volume.prune", target="*", - detail=str(result.get("VolumesDeleted")), ip=_ip(request), - ) + """Remove unused volumes: anonymous ones, or with ``all=true`` named ones + too (``-a``). + + Waits for the result by default. ``background=true`` returns 202 with the + initial status at once and leaves the caller to poll /prune-status, which + is what the UI does: a large prune can outlast the proxy's read timeout. + """ + username, ip = user.username, _ip(request) + + def audit(result: dict) -> None: + # Its own session: in the background case the request's is long closed. + with Session(engine) as own: + audit_service.record( + own, user=username, action="volume.prune", target="*", + detail=f"all={all_unused} deleted={result['VolumesDeleted']}", ip=ip, + ) + + try: + if background: + response.status_code = 202 + return volume_service.start_prune(all_unused, on_done=audit) + result = volume_service.prune_volumes(all_unused) + except PruneBusy as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + audit(result) return result +@router.get("/api/volumes/prune-status") +def prune_status(_admin: User = Depends(require_admin)) -> dict: + """Progress of a running (or the last) prune. Admin-only like the prune.""" + return volume_service.prune_status() + + @router.post("/api/volumes/generate-yaml") def generate_yaml( spec: dict, diff --git a/backend/services/image_service.py b/backend/services/image_service.py index 610316d..234303a 100644 --- a/backend/services/image_service.py +++ b/backend/services/image_service.py @@ -2,11 +2,11 @@ from __future__ import annotations import logging -import threading from typing import Callable, Optional from docker_client import DockerError, get_client, get_long_client, safe_call from services import update_service +from services.prune_job import PruneJob, PrunePlan logger = logging.getLogger(__name__) @@ -57,31 +57,11 @@ def list_images() -> list[dict]: # Prune # --------------------------------------------------------------------------- # -#: One prune at a time: a second pass over the same images would only collect -#: errors for everything the first one already removed. -_prune_lock = threading.Lock() -_pruning: dict = { - "active": False, - "run": 0, - "all": False, - "total": 0, - "done": 0, - "current": None, - "deleted": 0, - "failed": 0, - "errors": [], - "reclaimed": None, - "error": None, -} -_MAX_ERRORS = 10 - - -class PruneBusy(Exception): - """A prune is already running.""" +_prune = PruneJob("image") def prune_status() -> dict: - return {**_pruning, "errors": list(_pruning["errors"])} + return _prune.status() def _layers_size(client) -> Optional[int]: @@ -131,65 +111,38 @@ def _remove(client, image: dict) -> list[dict]: return entries -def _begin(all_unused: bool) -> tuple: +def _plan(all_unused: bool) -> PrunePlan: client = get_long_client() - candidates = _candidates(client, all_unused) - _pruning.update( - { - "active": True, - "run": _pruning["run"] + 1, - "all": all_unused, - "total": len(candidates), - "done": 0, - "current": None, - "deleted": 0, - "failed": 0, - "errors": [], - "reclaimed": None, - "error": None, - } + measured: dict = {} + + def prepare() -> None: + measured["before"] = _layers_size(client) + + def remove(image: dict) -> tuple[list, bool]: + entries = _remove(client, image) + return entries, any("Deleted" in e for e in entries) + + def finish(entries: list) -> tuple[dict, Optional[int]]: + # Let the daemon sweep up whatever the per-image pass left, such as + # untagged images referenced from several repositories. + filters = {"dangling": False} if all_unused else {"dangling": True} + sweep = safe_call(client.images.prune, filters=filters) + entries = entries + (sweep.get("ImagesDeleted") or []) + before, after = measured.get("before"), _layers_size(client) + if before is not None and after is not None: + reclaimed = max(before - after, 0) + else: + reclaimed = sweep.get("SpaceReclaimed", 0) + return {"ImagesDeleted": entries, "SpaceReclaimed": reclaimed}, reclaimed + + return PrunePlan( + all=all_unused, + items=_candidates(client, all_unused), + label=_label, + remove=remove, + finish=finish, + prepare=prepare, ) - return client, candidates - - -def _execute(client, candidates: list[dict], all_unused: bool) -> dict: - before = _layers_size(client) - deleted: list[dict] = [] - for image in candidates: - _pruning["current"] = _label(image) - try: - entries = _remove(client, image) - deleted.extend(entries) - if any("Deleted" in e for e in entries): - _pruning["deleted"] += 1 - except DockerError as exc: - # One image must not stop the rest, e.g. a container started from - # it since the list was taken. - logger.warning("Prune: could not remove %s: %s", _pruning["current"], exc) - _pruning["failed"] += 1 - if len(_pruning["errors"]) < _MAX_ERRORS: - _pruning["errors"].append(f"{_pruning['current']}: {exc.detail or exc.error}") - _pruning["done"] += 1 - _pruning["current"] = None - - # Let the daemon sweep up whatever the per-image pass left, such as - # untagged images referenced from several repositories. - filters = {"dangling": False} if all_unused else {"dangling": True} - sweep = safe_call(client.images.prune, filters=filters) - deleted.extend(sweep.get("ImagesDeleted") or []) - - after = _layers_size(client) - if before is not None and after is not None: - reclaimed = max(before - after, 0) - else: - reclaimed = sweep.get("SpaceReclaimed", 0) - _pruning["reclaimed"] = reclaimed - return {"ImagesDeleted": deleted, "SpaceReclaimed": reclaimed} - - -def _finish(error: Optional[str] = None) -> None: - _pruning.update({"active": False, "current": None, "error": error}) - _prune_lock.release() def prune_images(all_unused: bool = False) -> dict: @@ -199,51 +152,12 @@ def prune_images(all_unused: bool = False) -> dict: removes every image not referenced by a container (``docker image prune -a``). Progress is visible through prune_status() while it runs. """ - if not _prune_lock.acquire(blocking=False): - raise PruneBusy("An image prune is already running") - error = None - try: - client, candidates = _begin(all_unused) - return _execute(client, candidates, all_unused) - except DockerError as exc: - error = exc.detail or exc.error - raise - finally: - _finish(error) + return _prune.run(lambda: _plan(all_unused)) def start_prune( all_unused: bool = False, on_done: Optional[Callable[[dict], None]] = None ) -> dict: - """Start a prune in the background and return its initial status. - - A prune removing gigabytes of layers can outlast the proxy's read timeout, - so the UI starts it this way and polls prune_status(). ``on_done`` gets - the same result prune_images() returns, from the worker thread. - """ - if not _prune_lock.acquire(blocking=False): - raise PruneBusy("An image prune is already running") - try: - client, candidates = _begin(all_unused) - except BaseException: - _prune_lock.release() - raise - - def work() -> None: - error = None - result = None - try: - result = _execute(client, candidates, all_unused) - except Exception as exc: # noqa: BLE001 - reported through the status - logger.warning("Image prune failed: %s", exc) - error = exc.detail if isinstance(exc, DockerError) else str(exc) - finally: - _finish(error) - if result is not None and on_done: - try: - on_done(result) - except Exception as exc: # noqa: BLE001 - the prune itself succeeded - logger.warning("Image prune follow-up failed: %s", exc) - - threading.Thread(target=work, name="image-prune", daemon=True).start() - return prune_status() + """Start a prune in the background and return its initial status; the UI + polls prune_status(). ``on_done`` gets what prune_images() would return.""" + return _prune.start(lambda: _plan(all_unused), on_done) diff --git a/backend/services/prune_job.py b/backend/services/prune_job.py new file mode 100644 index 0000000..d7b565b --- /dev/null +++ b/backend/services/prune_job.py @@ -0,0 +1,170 @@ +"""Prunes with visible progress, shared by images and volumes. + +``docker image prune`` / ``docker volume prune`` are single blocking calls that +report nothing until they are done, which on a host with a lot to delete means +minutes of a spinner and, past the proxy's read timeout, an error while the +daemon carries on. Instead, a service lists what the prune would remove and a +PruneJob deletes it one item at a time, publishing how far it is so the UI can +poll it. One prune of each kind runs at a time. +""" +from __future__ import annotations + +import logging +import threading +from dataclasses import dataclass +from typing import Any, Callable, Optional + +from docker_client import DockerError + +logger = logging.getLogger(__name__) + +_MAX_ERRORS = 10 + + +class PruneBusy(Exception): + """A prune of this kind is already running.""" + + +@dataclass +class PrunePlan: + """What one prune will do, built by the owning service. + + ``remove`` deletes one item and returns the entries it contributes to the + result plus whether the item is actually gone (removing one of several + image tags is not). It raises DockerError to mark just that item failed. + ``prepare`` runs before the first removal (e.g. measuring disk usage) and + ``finish`` after the last; it gets every collected entry and returns the + API result together with the space reclaimed, or None if unknown. + """ + + all: bool + items: list + label: Callable[[Any], str] + remove: Callable[[Any], tuple[list, bool]] + finish: Callable[[list], tuple[dict, Optional[int]]] + prepare: Optional[Callable[[], None]] = None + + +class PruneJob: + def __init__(self, what: str): + self._what = what + self._lock = threading.Lock() + self._status: dict = { + "active": False, + "run": 0, + "all": False, + # preparing -> removing -> finishing, so the UI can say what the + # parts without a count are doing. + "phase": None, + "total": 0, + "done": 0, + "current": None, + "deleted": 0, + "failed": 0, + "errors": [], + "reclaimed": None, + "error": None, + } + + def status(self) -> dict: + return {**self._status, "errors": list(self._status["errors"])} + + def run(self, build: Callable[[], PrunePlan]) -> dict: + """Prune and wait for the result.""" + plan = self._begin(build) + error = None + try: + return self._execute(plan) + except DockerError as exc: + error = exc.detail or exc.error + raise + finally: + self._finish(error) + + def start( + self, + build: Callable[[], PrunePlan], + on_done: Optional[Callable[[dict], None]] = None, + ) -> dict: + """Prune in a background thread and return the initial status. + + The plan is still built here, so a daemon that cannot even list the + candidates fails the request instead of a status nobody polls yet. + ``on_done`` gets the result from the worker thread. + """ + plan = self._begin(build) + + def work() -> None: + error = None + result = None + try: + result = self._execute(plan) + except Exception as exc: # noqa: BLE001 - reported through the status + logger.warning("%s prune failed: %s", self._what.capitalize(), exc) + error = exc.detail if isinstance(exc, DockerError) else str(exc) + finally: + self._finish(error) + if result is not None and on_done: + try: + on_done(result) + except Exception as exc: # noqa: BLE001 - the prune itself succeeded + logger.warning("%s prune follow-up failed: %s", self._what, exc) + + threading.Thread(target=work, name=f"{self._what}-prune", daemon=True).start() + return self.status() + + def _begin(self, build: Callable[[], PrunePlan]) -> PrunePlan: + if not self._lock.acquire(blocking=False): + raise PruneBusy(f"{self._what.capitalize()} prune is already running") + try: + plan = build() + except BaseException: + self._lock.release() + raise + self._status.update( + { + "active": True, + "run": self._status["run"] + 1, + "all": plan.all, + "phase": "preparing", + "total": len(plan.items), + "done": 0, + "current": None, + "deleted": 0, + "failed": 0, + "errors": [], + "reclaimed": None, + "error": None, + } + ) + return plan + + def _execute(self, plan: PrunePlan) -> dict: + status = self._status + if plan.prepare: + plan.prepare() + status["phase"] = "removing" + entries: list = [] + for item in plan.items: + status["current"] = plan.label(item) + try: + produced, gone = plan.remove(item) + entries.extend(produced) + if gone: + status["deleted"] += 1 + except DockerError as exc: + # One item must not stop the rest, e.g. something started + # using it since the candidates were listed. + logger.warning("Prune: could not remove %s: %s", status["current"], exc) + status["failed"] += 1 + if len(status["errors"]) < _MAX_ERRORS: + status["errors"].append(f"{status['current']}: {exc.detail or exc.error}") + status["done"] += 1 + status.update({"phase": "finishing", "current": None}) + result, reclaimed = plan.finish(entries) + status["reclaimed"] = reclaimed + return result + + def _finish(self, error: Optional[str]) -> None: + self._status.update({"active": False, "phase": None, "current": None, "error": error}) + self._lock.release() diff --git a/backend/services/volume_service.py b/backend/services/volume_service.py index 05ce48b..8cfd04f 100644 --- a/backend/services/volume_service.py +++ b/backend/services/volume_service.py @@ -1,15 +1,22 @@ """Docker volume management + Compose volume YAML generation.""" from __future__ import annotations +import logging import threading import time -from typing import Literal, Optional +from typing import Callable, Literal, Optional import yaml from docker_client import get_client, get_long_client, safe_call +from services.prune_job import PruneJob, PrunePlan + +logger = logging.getLogger(__name__) COMPOSE_PROJECT_LABEL = "com.docker.compose.project" +#: Set by the daemon (Engine 23+) on volumes created without a name. A plain +#: `docker volume prune` only removes these; `-a` removes named ones too. +ANONYMOUS_LABEL = "com.docker.volume.anonymous" # `docker system df -v` walks every volume's contents, so it can take many # seconds. Cache the result so the (polled) UI and repeated requests reuse it. @@ -25,7 +32,9 @@ def volume_sizes(force: bool = False) -> dict: if not force and _size_cache["data"] and now - _size_cache["at"] < _SIZE_TTL: return _size_cache["data"] - client = get_client() + # The slow call this cache exists for; the 60s client would time out on a + # host with large volumes. + client = get_long_client() df = safe_call(client.df) sizes: dict[str, Optional[int]] = {} for v in df.get("Volumes") or []: @@ -85,9 +94,98 @@ def remove_volume(name: str, force: bool = False) -> None: safe_call(volume.remove, force=force) -def prune_volumes() -> dict: +_prune = PruneJob("volume") + + +def prune_status() -> dict: + return _prune.status() + + +def _candidates(client, all_unused: bool) -> list[dict]: + """The volumes ``docker volume prune [-a]`` would remove. + + Unused means no container, running or stopped, mounts it. Without -a only + anonymous volumes qualify. Volumes created before Engine 23 carry no + anonymous label and so are only removed with -a, which errs on the side of + keeping data. + """ + in_use = set() + for c in safe_call(client.api.containers, all=True): + for mount in c.get("Mounts") or []: + if mount.get("Type") == "volume" and mount.get("Name"): + in_use.add(mount["Name"]) + volumes = (safe_call(client.api.volumes) or {}).get("Volumes") or [] + return [ + v + for v in volumes + if v["Name"] not in in_use + # Cluster (swarm) volumes are not the local daemon's to prune. + and v.get("Scope", "local") == "local" + and (all_unused or ANONYMOUS_LABEL in (v.get("Labels") or {})) + ] + + +def _label(volume: dict) -> str: + name = volume["Name"] + # Anonymous volumes are 64 hex characters; the first 12 identify them. + return name[:12] if ANONYMOUS_LABEL in (volume.get("Labels") or {}) else name + + +def _plan(all_unused: bool) -> PrunePlan: client = get_long_client() - return safe_call(client.volumes.prune) + measured: dict = {} + + def prepare() -> None: + # Volumes share no data, so the freed space is simply the sum of what + # was deleted, provided their sizes are known up front. Unused volumes + # cannot change, so the cached figures are good enough. + try: + measured["sizes"] = volume_sizes() + except Exception as exc: # noqa: BLE001 - the size is a nicety, not the job + logger.warning("Could not measure volume sizes: %s", exc) + + def remove(volume: dict) -> tuple[list, bool]: + # No force: a volume mounted since the listing makes the daemon refuse. + safe_call(client.api.remove_volume, volume["Name"]) + return [volume["Name"]], True + + def finish(names: list) -> tuple[dict, Optional[int]]: + # No closing daemon prune as for images: nothing is left behind, and + # on a pre-23 daemon an unfiltered one would delete named volumes too. + sizes = measured.get("sizes") + reclaimed = ( + sum(sizes.get(name) or 0 for name in names) if sizes is not None else None + ) + with _size_lock: + _size_cache["at"] = 0.0 # deleted volumes must not linger in it + return {"VolumesDeleted": names, "SpaceReclaimed": reclaimed or 0}, reclaimed + + return PrunePlan( + all=all_unused, + items=_candidates(client, all_unused), + label=_label, + remove=remove, + finish=finish, + prepare=prepare, + ) + + +def prune_volumes(all_unused: bool = False) -> dict: + """Remove unused volumes and wait for the result. + + By default only anonymous volumes; ``all_unused=True`` also removes named + ones (``docker volume prune -a``). Progress is visible through + prune_status() while it runs. + """ + return _prune.run(lambda: _plan(all_unused)) + + +def start_prune( + all_unused: bool = False, on_done: Optional[Callable[[dict], None]] = None +) -> dict: + """Start a prune in the background and return its initial status; the UI + polls prune_status(). ``on_done`` gets what prune_volumes() would return.""" + return _prune.start(lambda: _plan(all_unused), on_done) # --------------------------------------------------------------------------- # diff --git a/backend/tests/test_docker_timeouts.py b/backend/tests/test_docker_timeouts.py index 57e2ff0..3f28ad0 100644 --- a/backend/tests/test_docker_timeouts.py +++ b/backend/tests/test_docker_timeouts.py @@ -37,12 +37,12 @@ def test_safe_call_wraps_connection_error(): def test_volume_prune_timeout_is_a_502_with_detail(as_admin, monkeypatch): from services import volume_service - class Volumes: - def prune(self, filters=None): + class Api: + def containers(self, all=False): raise requests.exceptions.ReadTimeout("Read timed out.") class Client: - volumes = Volumes() + api = Api() monkeypatch.setattr(volume_service, "get_long_client", lambda: Client()) r = as_admin.post("/api/volumes/prune") @@ -53,7 +53,6 @@ def test_volume_prune_timeout_is_a_502_with_detail(as_admin, monkeypatch): @pytest.mark.parametrize( "module, fn, args", [ - ("volume_service", "prune_volumes", ()), ("network_service", "prune_networks", ()), ], ) @@ -80,5 +79,6 @@ def test_prunes_use_the_long_timeout_client(monkeypatch, module, fn, args): assert used == ["long"] -# The image prune goes image by image and has its own tests in -# test_image_prune.py, including that it uses the long-timeout client. +# The image and volume prunes go item by item and have their own tests in +# test_image_prune.py / test_volume_prune.py, including that they use the +# long-timeout client. diff --git a/backend/tests/test_image_prune.py b/backend/tests/test_image_prune.py index 5add72d..3b50887 100644 --- a/backend/tests/test_image_prune.py +++ b/backend/tests/test_image_prune.py @@ -112,10 +112,9 @@ def svc(monkeypatch): monkeypatch.setattr(image_service, "get_client", short) monkeypatch.setattr(image_service, "get_long_client", lambda: client) - image_service._pruning.update({"active": False, "run": 0}) yield image_service, api, client - if image_service._prune_lock.locked(): # a test that failed mid-prune - image_service._prune_lock.release() + if image_service._prune._lock.locked(): # a test that failed mid-prune + image_service._prune._lock.release() def test_all_unused_removes_everything_no_container_uses(svc): @@ -167,7 +166,9 @@ def test_background_prune_reports_progress_and_blocks_a_second_one(svc): started = image_service.start_prune(True, on_done=lambda r: finished.set()) assert started["active"] is True and started["total"] == 3 - with pytest.raises(image_service.PruneBusy): + from services.prune_job import PruneBusy + + with pytest.raises(PruneBusy): image_service.start_prune(True) api.gate.set() @@ -194,7 +195,7 @@ def test_background_prune_error_lands_in_the_status(svc): status = image_service.prune_status() assert status["active"] is False assert status["error"] == "Docker did not answer in time" - assert not image_service._prune_lock.locked() + assert not image_service._prune._lock.locked() def test_endpoint_waits_by_default_and_is_audited(svc, as_admin): @@ -229,12 +230,12 @@ def test_endpoint_background_returns_202_and_status_is_pollable(svc, as_admin): def test_endpoint_rejects_a_second_prune(svc, as_admin): image_service, _, _ = svc - image_service._prune_lock.acquire() + image_service._prune._lock.acquire() try: r = as_admin.post("/api/images/prune", params={"background": True}) assert r.status_code == 409 finally: - image_service._prune_lock.release() + image_service._prune._lock.release() def test_prune_is_admin_only(svc, as_user): diff --git a/backend/tests/test_volume_prune.py b/backend/tests/test_volume_prune.py new file mode 100644 index 0000000..f922148 --- /dev/null +++ b/backend/tests/test_volume_prune.py @@ -0,0 +1,189 @@ +"""Volume prune with visible progress, and an explicit -a. + +Same mechanism as the image prune (services/prune_job.py): the candidates are +listed, removed one by one and the progress is published through +/api/volumes/prune-status. On top of that the button used to call a bare +``docker volume prune``, which on Engine 23+ only removes anonymous volumes and +offered no way to include named ones. + +Pinned here: nothing a container mounts is touched, named volumes only go with +-a, a refusal does not stop the rest, and the freed space is the sum of what +was actually deleted. +""" +from __future__ import annotations + +import threading + +import pytest +from docker.errors import APIError + +MB = 1024 * 1024 +ANON = {"com.docker.volume.anonymous": ""} +ANON_ID = "3f1c" + "0" * 60 + + +class FakeApi: + def __init__(self): + self._volumes = [ + {"Name": ANON_ID, "Labels": ANON, "Scope": "local"}, + {"Name": "media_cache", "Labels": {"com.docker.compose.project": "media"}, "Scope": "local"}, + {"Name": "old_data", "Labels": None, "Scope": "local"}, + {"Name": "pg_data", "Labels": {}, "Scope": "local"}, + {"Name": "cluster_vol", "Labels": {}, "Scope": "global"}, + ] + # pg_data is mounted by a (possibly stopped) container. + self._containers = [ + {"Mounts": [{"Type": "volume", "Name": "pg_data"}, {"Type": "bind", "Source": "/srv"}]} + ] + self.removed: list[str] = [] + self.refuse: set[str] = set() + self.gate: threading.Event | None = None + + def containers(self, all=False): + return self._containers + + def volumes(self, filters=None): + return {"Volumes": list(self._volumes), "Warnings": None} + + def remove_volume(self, name, force=False): + assert force is False + if self.gate: + self.gate.wait(5) + if name in self.refuse: + raise APIError(f"remove {name}: volume is in use") + self.removed.append(name) + self._volumes = [v for v in self._volumes if v["Name"] != name] + + +class FakeClient: + def __init__(self, api): + self.api = api + + +@pytest.fixture +def svc(monkeypatch): + from services import volume_service + + api = FakeApi() + client = FakeClient(api) + + def short(): + raise AssertionError("prune must not use the 60s client") + + monkeypatch.setattr(volume_service, "get_client", short) + monkeypatch.setattr(volume_service, "get_long_client", lambda: client) + monkeypatch.setattr( + volume_service, + "volume_sizes", + lambda force=False: {ANON_ID: 5 * MB, "media_cache": 300 * MB, "old_data": None, "pg_data": 900 * MB}, + ) + yield volume_service, api + if volume_service._prune._lock.locked(): + volume_service._prune._lock.release() + + +def test_default_only_removes_unused_anonymous_volumes(svc): + volume_service, api = svc + + result = volume_service.prune_volumes() + + assert api.removed == [ANON_ID] + assert result == {"VolumesDeleted": [ANON_ID], "SpaceReclaimed": 5 * MB} + + +def test_all_also_removes_named_but_never_mounted_or_cluster_volumes(svc): + volume_service, api = svc + + result = volume_service.prune_volumes(all_unused=True) + + assert set(api.removed) == {ANON_ID, "media_cache", "old_data"} + # A volume of unknown size counts as nothing rather than failing the sum. + assert result["SpaceReclaimed"] == 305 * MB + status = volume_service.prune_status() + assert (status["total"], status["done"], status["deleted"]) == (3, 3, 3) + assert status["reclaimed"] == 305 * MB and status["active"] is False + + +def test_refused_volume_does_not_stop_the_rest(svc): + volume_service, api = svc + api.refuse.add("media_cache") + + result = volume_service.prune_volumes(all_unused=True) + + assert set(result["VolumesDeleted"]) == {ANON_ID, "old_data"} + status = volume_service.prune_status() + assert status["failed"] == 1 and status["errors"][0].startswith("media_cache: ") + assert result["SpaceReclaimed"] == 5 * MB + + +def test_unknown_sizes_leave_reclaimed_unknown(svc, monkeypatch): + volume_service, _ = svc + + def broken(force=False): + raise RuntimeError("df failed") + + monkeypatch.setattr(volume_service, "volume_sizes", broken) + result = volume_service.prune_volumes(all_unused=True) + + assert len(result["VolumesDeleted"]) == 3 + assert volume_service.prune_status()["reclaimed"] is None + + +def test_anonymous_volumes_are_labelled_by_short_id(svc): + volume_service, api = svc + api.gate = threading.Event() + + finished = threading.Event() + + volume_service.start_prune(all_unused=False, on_done=lambda r: finished.set()) + for _ in range(100): + status = volume_service.prune_status() + if status["current"]: + break + threading.Event().wait(0.02) + api.gate.set() + assert finished.wait(5) + assert status["current"] == ANON_ID[:12] + + +def test_endpoint_background_then_poll(svc, as_admin): + r = as_admin.post("/api/volumes/prune", params={"all": True, "background": True}) + assert r.status_code == 202 and r.json()["total"] == 3 + + for _ in range(100): + s = as_admin.get("/api/volumes/prune-status").json() + if not s["active"]: + break + threading.Event().wait(0.02) + assert s["deleted"] == 3 and s["reclaimed"] == 305 * MB + + +def test_endpoint_waits_by_default_and_is_audited(svc, as_admin): + from sqlmodel import Session, select + + from database import engine + from models.audit import AuditLog + + r = as_admin.post("/api/volumes/prune") + assert r.status_code == 200 + assert r.json()["VolumesDeleted"] == [ANON_ID] + + with Session(engine) as session: + row = session.exec( + select(AuditLog).where(AuditLog.action == "volume.prune").order_by(AuditLog.id.desc()) + ).first() + assert row.detail == f"all=False deleted=['{ANON_ID}']" + + +def test_endpoint_rejects_a_second_prune(svc, as_admin): + volume_service, _ = svc + volume_service._prune._lock.acquire() + try: + assert as_admin.post("/api/volumes/prune").status_code == 409 + finally: + volume_service._prune._lock.release() + + +def test_prune_is_admin_only(svc, as_user): + assert as_user.post("/api/volumes/prune").status_code == 403 + assert as_user.get("/api/volumes/prune-status").status_code == 403 diff --git a/backend/version.py b/backend/version.py index f86877c..c37f516 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1,3 +1,3 @@ """Single source of truth for the StackPilot release version.""" -APP_VERSION = "0.61.0" +APP_VERSION = "0.62.0" diff --git a/frontend/package.json b/frontend/package.json index a2f2db3..df5ff6a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "stackpilot-frontend", "private": true, - "version": "0.61.0", + "version": "0.62.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/api/images.ts b/frontend/src/api/images.ts index 5a41582..35c7628 100644 --- a/frontend/src/api/images.ts +++ b/frontend/src/api/images.ts @@ -1,4 +1,5 @@ import api from "./client"; +import type { PruneStatus } from "@/types"; export interface UpdateStatus { image: string; @@ -54,23 +55,6 @@ export interface ScanDetail extends ScanSummary { const base = "/api/images"; -/** Progress of the running image prune, or the outcome of the last one. */ -export interface PruneStatus { - active: boolean; - /** Increments per prune, so a client can tell "its" run from an older one. */ - run: number; - all: boolean; - total: number; - done: number; - current: string | null; - deleted: number; - failed: number; - errors: string[]; - /** Measured once the prune is through; null while it runs. */ - reclaimed: number | null; - error: string | null; -} - export const imagesApi = { list: () => api.get(base).then((r) => r.data), updates: () => api.get>(`${base}/updates`).then((r) => r.data), diff --git a/frontend/src/api/volumes.ts b/frontend/src/api/volumes.ts index c2d6247..e14daf1 100644 --- a/frontend/src/api/volumes.ts +++ b/frontend/src/api/volumes.ts @@ -1,5 +1,5 @@ import api from "./client"; -import type { HostPathResult, VolumeInfo } from "@/types"; +import type { HostPathResult, PruneStatus, VolumeInfo } from "@/types"; const base = "/api/volumes"; @@ -11,8 +11,14 @@ export const volumesApi = { .then((r) => r.data), remove: (name: string, force = false) => api.delete(`${base}/${name}?force=${force}`).then((r) => r.data), - prune: () => - api.post<{ VolumesDeleted: string[] | null }>(`${base}/prune`).then((r) => r.data), + // Runs in the background on the server; poll pruneStatus() for progress. + startPrune: (allUnused: boolean) => + api + .post(`${base}/prune`, null, { + params: { all: allUnused, background: true }, + }) + .then((r) => r.data), + pruneStatus: () => api.get(`${base}/prune-status`).then((r) => r.data), generateYaml: (spec: Record) => api .post<{ yaml: string }>("/api/volumes/generate-yaml", spec) diff --git a/frontend/src/components/ui/PruneProgress.tsx b/frontend/src/components/ui/PruneProgress.tsx new file mode 100644 index 0000000..30e9119 --- /dev/null +++ b/frontend/src/components/ui/PruneProgress.tsx @@ -0,0 +1,53 @@ +import { Card } from "@/components/ui"; +import type { PruneStatus } from "@/types"; + +const PHASE_DETAIL = { + preparing: "Measuring disk usage…", + finishing: "Finishing up…", +} as const; + +/** Live progress of a running prune, shown above the page's table. */ +export function PruneProgress({ status, title }: { status: PruneStatus; title: string }) { + const measurable = status.total > 0; + const pct = measurable ? Math.min((status.done / status.total) * 100, 100) : 0; + const detail = + status.phase === "removing" + ? status.current + ? `Removing ${status.current}` + : "" + : status.phase + ? PHASE_DETAIL[status.phase] + : ""; + + return ( + +
+ {title} + {measurable && ( + + {status.done} / {status.total} + + )} +
+
+ {measurable && status.phase === "removing" ? ( +
+ ) : ( + // Measuring and finishing have nothing to count — slide instead. +
+ )} +
+
+ {detail} + {status.failed > 0 && ( + + {status.failed} failed + + )} +
+ + ); +} diff --git a/frontend/src/hooks/usePruneJob.ts b/frontend/src/hooks/usePruneJob.ts new file mode 100644 index 0000000..1e14fe5 --- /dev/null +++ b/frontend/src/hooks/usePruneJob.ts @@ -0,0 +1,91 @@ +import { useEffect, useRef } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { apiErrorMessage } from "@/api/client"; +import { formatBytes } from "@/lib/utils"; +import type { PruneStatus } from "@/types"; + +/** + * A server-side prune that runs in the background (images, volumes). + * + * Polls the status while a prune is active. It is fetched once on mount too, + * so a prune started in another tab, or before a reload, still shows up. The + * outcome of the run this page saw active is announced exactly once, and + * never the stale result of an older run on first load. + */ +export function usePruneJob({ + noun, + queryKey, + fetchStatus, + start, + enabled, + onFinished, +}: { + /** Singular, for the toast: "image", "volume". */ + noun: string; + queryKey: string; + fetchStatus: () => Promise; + start: (allUnused: boolean) => Promise; + enabled: boolean; + onFinished: () => void; +}) { + const qc = useQueryClient(); + const status = useQuery({ + queryKey: [queryKey], + queryFn: fetchStatus, + enabled, + refetchInterval: (q) => (q.state.data?.active ? 1000 : false), + }); + + const watchedRun = useRef(null); + const finished = useRef(onFinished); + finished.current = onFinished; + useEffect(() => { + const s = status.data; + if (!s) return; + if (s.active) { + watchedRun.current = s.run; + return; + } + if (watchedRun.current !== s.run) return; + watchedRun.current = null; + finished.current(); + announcePrune(s, noun); + }, [status.data, noun]); + + const mutation = useMutation({ + mutationFn: start, + onSuccess: (s) => { + watchedRun.current = s.run; + qc.setQueryData([queryKey], s); + }, + onError: (e) => toast.error(apiErrorMessage(e)), + }); + + return { + /** The running prune, or null. */ + running: status.data?.active ? status.data : null, + start: mutation, + }; +} + +/** Toast for a finished prune: what went, what it freed, what refused to go. */ +function announcePrune(s: PruneStatus, noun: string) { + if (s.error) { + toast.error(`Prune failed: ${s.error}`); + return; + } + if (!s.deleted && !s.failed) { + toast.success(`No unused ${noun}s`); + return; + } + const freed = s.reclaimed ? `, freed ${formatBytes(s.reclaimed)}` : ""; + const summary = `Removed ${s.deleted} ${noun}${s.deleted === 1 ? "" : "s"}${freed}`; + if (s.failed) { + toast.warning(`${summary}; ${s.failed} could not be removed`, { + description: s.errors.slice(0, 3).join("\n"), + }); + } else { + toast.success(summary); + } +} diff --git a/frontend/src/pages/Images.tsx b/frontend/src/pages/Images.tsx index 94cf8c7..305a934 100644 --- a/frontend/src/pages/Images.tsx +++ b/frontend/src/pages/Images.tsx @@ -1,10 +1,12 @@ -import { Fragment, useEffect, useMemo, useRef, useState } from "react"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Fragment, useMemo, useState } from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; import { RefreshCw, ArrowUpCircle, CheckCircle2, HelpCircle, Eraser, ShieldAlert } from "lucide-react"; import { toast } from "sonner"; import { Button, Card, Spinner } from "@/components/ui"; import { ConfirmDialog } from "@/components/ui/ConfirmDialog"; -import { imagesApi, type ImageRow, type PruneStatus } from "@/api/images"; +import { imagesApi, type ImageRow } from "@/api/images"; +import { PruneProgress } from "@/components/ui/PruneProgress"; +import { usePruneJob } from "@/hooks/usePruneJob"; import { ScanBadge, ScanDetailDialog } from "@/components/stacks/ScanFindings"; import { stacksApi } from "@/api/stacks"; import { groupByStack, type StackGroup } from "@/lib/stackGroups"; @@ -108,48 +110,20 @@ function ImagesSection({ isAdmin }: { isAdmin: boolean }) { } }; - // The prune runs on the server in the background; this polls its progress - // while it is active. Fetched once on mount too, so a prune started in - // another tab, or before a reload, still shows up here. - const pruneStatus = useQuery({ - queryKey: ["image-prune-status"], - queryFn: imagesApi.pruneStatus, + const prune = usePruneJob({ + noun: "image", + queryKey: "image-prune-status", + fetchStatus: imagesApi.pruneStatus, + start: imagesApi.startPrune, enabled: isAdmin, - refetchInterval: (q) => (q.state.data?.active ? 1000 : false), + onFinished: () => qc.invalidateQueries({ queryKey: ["images"] }), }); - // The run this page has seen active, so its outcome is announced exactly - // once — and never the stale result of an older run on first load. - const watchedRun = useRef(null); - useEffect(() => { - const s = pruneStatus.data; - if (!s) return; - if (s.active) { - watchedRun.current = s.run; - return; - } - if (watchedRun.current !== s.run) return; - watchedRun.current = null; - qc.invalidateQueries({ queryKey: ["images"] }); - announcePrune(s); - }, [pruneStatus.data, qc]); - - const prune = useMutation({ - mutationFn: () => imagesApi.startPrune(pruneAll), - onSuccess: (s) => { - watchedRun.current = s.run; - qc.setQueryData(["image-prune-status"], s); - setPruneOpen(false); - setPruneAll(false); - }, - onError: (e) => toast.error(apiErrorMessage(e)), - }); - const pruning = pruneStatus.data?.active ? pruneStatus.data : null; return (
{isAdmin && (
-
)} - {pruning && } + {prune.running && ( + + )} {isLoading ? ( @@ -236,8 +215,15 @@ function ImagesSection({ isAdmin }: { isAdmin: boolean }) { } confirmLabel="Prune images" danger - busy={prune.isPending} - onConfirm={() => prune.mutate()} + busy={prune.start.isPending} + onConfirm={() => + prune.start.mutate(pruneAll, { + onSuccess: () => { + setPruneOpen(false); + setPruneAll(false); + }, + }) + } onCancel={() => { setPruneOpen(false); setPruneAll(false); @@ -257,71 +243,6 @@ function ImagesSection({ isAdmin }: { isAdmin: boolean }) { ); } -/** Toast for a finished prune: what went, what it freed, what refused to go. */ -function announcePrune(s: PruneStatus) { - if (s.error) { - toast.error(`Prune failed: ${s.error}`); - return; - } - if (!s.deleted && !s.failed) { - toast.success("No unused images"); - return; - } - const freed = s.reclaimed ? `, freed ${formatBytes(s.reclaimed)}` : ""; - const summary = `Removed ${s.deleted} image${s.deleted === 1 ? "" : "s"}${freed}`; - if (s.failed) { - toast.warning(`${summary}; ${s.failed} could not be removed`, { - description: s.errors.slice(0, 3).join("\n"), - }); - } else { - toast.success(summary); - } -} - -/** Live progress of a running prune, above the image table. */ -function PruneProgress({ status }: { status: PruneStatus }) { - const measurable = status.total > 0; - const pct = measurable ? Math.min((status.done / status.total) * 100, 100) : 0; - const detail = status.current - ? `Removing ${status.current}` - : status.done < status.total - ? "Starting…" - : "Cleaning up and measuring freed space…"; - - return ( - -
- - Pruning {status.all ? "unused" : "dangling"} images… - - {measurable && ( - - {status.done} / {status.total} - - )} -
-
- {measurable ? ( -
- ) : ( -
- )} -
-
- {detail} - {status.failed > 0 && ( - - {status.failed} failed - - )} -
- - ); -} - /** What an image group can say for itself: how many, how big, how stale. */ function groupMeta(group: StackGroup): string { const parts = [`${group.items.length} image${group.items.length > 1 ? "s" : ""}`]; diff --git a/frontend/src/pages/Volumes.tsx b/frontend/src/pages/Volumes.tsx index 8c8ab41..6f1acb0 100644 --- a/frontend/src/pages/Volumes.tsx +++ b/frontend/src/pages/Volumes.tsx @@ -5,6 +5,8 @@ import { toast } from "sonner"; import { Button, Card, Spinner } from "@/components/ui"; import { ConfirmDialog } from "@/components/ui/ConfirmDialog"; import { volumesApi } from "@/api/volumes"; +import { PruneProgress } from "@/components/ui/PruneProgress"; +import { usePruneJob } from "@/hooks/usePruneJob"; import { stacksApi } from "@/api/stacks"; import { apiErrorMessage } from "@/api/client"; import { useAuthStore } from "@/store/auth"; @@ -23,6 +25,8 @@ function VolumesSection({ isAdmin }: { isAdmin: boolean }) { const [onlyUnused, setOnlyUnused] = useState(false); const [toDelete, setToDelete] = useState(null); const [force, setForce] = useState(false); + const [pruneOpen, setPruneOpen] = useState(false); + const [pruneAll, setPruneAll] = useState(false); const { data, isLoading } = useQuery({ queryKey: ["volumes"], @@ -38,14 +42,17 @@ function VolumesSection({ isAdmin }: { isAdmin: boolean }) { }); const invalidate = () => qc.invalidateQueries({ queryKey: ["volumes"] }); - const prune = useMutation({ - mutationFn: () => volumesApi.prune(), - onSuccess: (r) => { - const n = r.VolumesDeleted?.length ?? 0; - toast.success(n ? `Pruned ${n} volume(s)` : "No unused volumes"); + const prune = usePruneJob({ + noun: "volume", + queryKey: "volume-prune-status", + fetchStatus: volumesApi.pruneStatus, + start: volumesApi.startPrune, + enabled: isAdmin, + onFinished: () => { invalidate(); + // Whatever was measured before includes volumes that are gone now. + if (sizes.data) sizes.refetch(); }, - onError: (e) => toast.error(apiErrorMessage(e)), }); const remove = useMutation({ mutationFn: (v: VolumeInfo) => volumesApi.remove(v.name, force), @@ -104,12 +111,19 @@ function VolumesSection({ isAdmin }: { isAdmin: boolean }) { Compute sizes {isAdmin && ( - )}
+ {prune.running && ( + + )} + {isLoading ? ( ) : ( @@ -161,6 +175,41 @@ function VolumesSection({ isAdmin }: { isAdmin: boolean }) { )} + {pruneOpen && ( + + prune.start.mutate(pruneAll, { + onSuccess: () => { + setPruneOpen(false); + setPruneAll(false); + }, + }) + } + onCancel={() => { + setPruneOpen(false); + setPruneAll(false); + }} + > + + + )} + {toDelete && (