Stop leaking helper containers that fail to start (0.66.1)

docker-py's containers.run(remove=True) only removes a container after it
ran. A helper whose start failed - an NFS destination refusing the mount -
stayed behind in "Created", holding the destination's volume. Every Test
click and scheduled backup added one (a user found 18), the volume could
not be deleted even with everything stopped, and changed mount options
never took effect because the volume could not be recreated with them.

- docker_client.run_container: create, start, wait, collect output and
  always remove, also when the start fails; pulls a missing image. Used
  for NFS destination I/O, bind-path inspection, the volume wipe before a
  restore and the Trivy scanner.
- Every helper container is labelled stackpilot.helper; on startup the
  stopped ones are removed, together with anything left on an NFS
  destination volume by older versions.
- When an NFS destination's options change, stopped containers holding
  its volume are removed before it is recreated; a running one is now a
  clear error instead of silently keeping the old options.
- A refused mount explains itself: NFS enabled and reachable from the
  Docker host, NFSv3 needing rpcbind and mountd, and nfsvers=4 needing
  only port 2049.
- The scanner waits for Trivy's own 10-minute timeout instead of giving
  up after the default client's 60 seconds.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/docker_client.py

@@ -6,11 +6,11 @@ to the frontend.
 from __future__ import annotations
 
 import functools
-from typing import Any
+from typing import Any, Optional
 
 import docker
 import requests
-from docker.errors import APIError, DockerException, NotFound
+from docker.errors import APIError, DockerException, ImageNotFound, NotFound
 
 from config import settings
 
@@ -57,6 +57,70 @@ def reset_client() -> None:
     get_long_client.cache_clear()
 
 
+#: Label on every throwaway container StackPilot starts (backup helpers, NFS
+#: I/O, the scanner), so leftovers can be found and removed.
+HELPER_LABEL = "stackpilot.helper"
+
+
+def run_container(
+    client: docker.DockerClient,
+    image: str,
+    command: Any,
+    *,
+    volumes: Optional[dict] = None,
+    timeout: Optional[float] = None,
+) -> bytes:
+    """``docker run --rm`` that also cleans up a container that cannot start.
+
+    docker-py's ``containers.run(remove=True)`` only removes a container after
+    it ran. One whose start fails — a volume that cannot be mounted, say an
+    NFS share that refuses the connection — stays behind in "Created" and
+    keeps its volumes in use; every retry adds another. Returns stdout; a
+    non-zero exit raises ``DockerError("container_failed", <stderr>)``.
+    """
+    kwargs = {"command": command, "volumes": volumes or {}, "labels": {HELPER_LABEL: "1"}}
+    try:
+        container = client.containers.create(image, **kwargs)
+    except ImageNotFound:
+        safe_call(client.images.pull, image)
+        container = safe_call(client.containers.create, image, **kwargs)
+    except DockerException as exc:
+        raise DockerError("docker_api_error", str(exc)) from exc
+    try:
+        safe_call(container.start)
+        code = safe_call(container.wait, timeout=timeout).get("StatusCode", 1)
+        out = container.logs(stdout=True, stderr=False) or b""
+        if code != 0:
+            err = (container.logs(stdout=False, stderr=True) or b"").decode("utf-8", "replace")
+            raise DockerError("container_failed", err.strip() or f"exit status {code}")
+        return out
+    finally:
+        try:
+            container.remove(force=True)
+        except Exception:  # noqa: BLE001 - best effort, never mask the real error
+            pass
+
+
+def remove_stale_helpers(client: docker.DockerClient) -> int:
+    """Remove StackPilot's helper containers that are not running.
+
+    Called at startup, when nothing of ours can be in flight. Returns how many
+    were removed.
+    """
+    removed = 0
+    for container in safe_call(
+        client.containers.list, all=True, filters={"label": HELPER_LABEL}
+    ):
+        if container.status == "running":
+            continue
+        try:
+            container.remove(force=True)
+            removed += 1
+        except Exception:  # noqa: BLE001
+            pass
+    return removed
+
+
 def safe_call(fn, *args, **kwargs) -> Any:
     """Run a docker-py call, converting exceptions into DockerError."""
     try: