Stop leaking helper containers that fail to start (0.66.1)

docker-py's containers.run(remove=True) only removes a container after it
ran. A helper whose start failed - an NFS destination refusing the mount -
stayed behind in "Created", holding the destination's volume. Every Test
click and scheduled backup added one (a user found 18), the volume could
not be deleted even with everything stopped, and changed mount options
never took effect because the volume could not be recreated with them.

- docker_client.run_container: create, start, wait, collect output and
  always remove, also when the start fails; pulls a missing image. Used
  for NFS destination I/O, bind-path inspection, the volume wipe before a
  restore and the Trivy scanner.
- Every helper container is labelled stackpilot.helper; on startup the
  stopped ones are removed, together with anything left on an NFS
  destination volume by older versions.
- When an NFS destination's options change, stopped containers holding
  its volume are removed before it is recreated; a running one is now a
  clear error instead of silently keeping the old options.
- A refused mount explains itself: NFS enabled and reachable from the
  Docker host, NFSv3 needing rpcbind and mountd, and nfsvers=4 needing
  only port 2049.
- The scanner waits for Trivy's own 10-minute timeout instead of giving
  up after the default client's 60 seconds.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/services/backup_destination_service.py

@@ -16,7 +16,7 @@ import posixpath
 import stat
 import tarfile
 import tempfile
-from typing import Any
+from typing import Any, Optional
 
 from models.backup_destination import BackupDestination
 from services import crypto_service
@@ -309,10 +309,21 @@ def _nfs_volume(dest: BackupDestination, cfg: dict) -> str:
     client = get_client()
     try:
         vol = safe_call(client.volumes.get, name)
-        if (vol.attrs.get("Options") or {}) != driver_opts:
-            safe_call(vol.remove)
-            raise DockerError("recreate", "options changed")
     except DockerError:
+        vol = None
+    if vol is not None and (vol.attrs.get("Options") or {}) != driver_opts:
+        # Stopped containers still hold the volume; before 0.66.1 every failed
+        # mount left one behind, and then new settings never took effect.
+        clear_nfs_leftovers(client, name)
+        try:
+            safe_call(vol.remove)
+        except DockerError as exc:
+            raise DestinationError(
+                f"The NFS settings changed, but the Docker volume '{name}' is still in use "
+                f"and cannot be recreated with them: {exc.detail or exc.error}"
+            ) from exc
+        vol = None
+    if vol is None:
         safe_call(
             client.volumes.create,
             name=name,
@@ -323,35 +334,81 @@ def _nfs_volume(dest: BackupDestination, cfg: dict) -> str:
     return name
 
 
+def clear_nfs_leftovers(client, volume: Optional[str] = None) -> int:
+    """Remove stopped containers holding an NFS destination volume.
+
+    Only StackPilot's helpers ever mount these volumes, so any container on
+    one that is not running is a leftover (older versions left one behind per
+    failed mount). ``volume=None`` covers every NFS destination.
+    """
+    from docker_client import DockerError, safe_call
+
+    if volume:
+        names = [volume]
+    else:
+        try:
+            vols = safe_call(
+                client.volumes.list, filters={"label": "stackpilot.nfs-destination"}
+            )
+        except DockerError:
+            return 0
+        names = [v.name for v in vols]
+    removed = 0
+    for name in names:
+        try:
+            containers = safe_call(client.containers.list, all=True, filters={"volume": name})
+        except DockerError:
+            continue
+        for container in containers:
+            if container.status == "running":
+                continue
+            try:
+                container.remove(force=True)
+                removed += 1
+            except Exception:  # noqa: BLE001
+                pass
+    return removed
+
+
 def _nfs_target(cfg: dict) -> str:
     sub = _nfs_subdir(cfg)
     return f"/nfs/{sub}" if sub else "/nfs"
 
 
+def _mount_error(exc: Exception) -> DestinationError:
+    """A mount failure, with what to do about the common one."""
+    text = str(exc)
+    message = f"NFS mount failed: {text}"
+    if "connection refused" in text.lower():
+        message += (
+            " — the NFS server refused the Docker host's connection. Check that NFS is "
+            "enabled on the server and reachable from the Docker host. NFSv3 also needs "
+            "rpcbind (port 111) and mountd; mount options 'rw,nfsvers=4' need only port 2049."
+        )
+    return DestinationError(message)
+
+
 def _nfs_run(volume: str, command: list[str]) -> str:
     """Run a helper container with the NFS volume at /nfs; return stdout."""
-    import docker.errors
-
     from config import settings
-    from docker_client import DockerError, get_client
+    from docker_client import DockerError, get_long_client, run_container
     from services.stack_assets_service import ensure_helper_image
 
-    client = get_client()
+    client = get_long_client()
     ensure_helper_image(client)
     try:
-        out = client.containers.run(
+        out = run_container(
+            client,
             settings.BACKUP_HELPER_IMAGE,
             command,
             volumes={volume: {"bind": "/nfs", "mode": "rw"}},
-            remove=True,
         )
-        return (out or b"").decode("utf-8", "replace")
-    except docker.errors.ContainerError as exc:
-        stderr = (exc.stderr or b"").decode("utf-8", "replace").strip()
-        raise DestinationError(f"NFS operation failed: {stderr or exc}") from exc
-    except (docker.errors.APIError, DockerError) as exc:
+        return out.decode("utf-8", "replace")
+    except DockerError as exc:
+        if exc.error == "container_failed":
+            raise DestinationError(f"NFS operation failed: {exc.detail}") from exc
         # Mount errors surface here (unreachable server, bad export, ...).
-        raise DestinationError(f"NFS mount failed: {exc}") from exc
+        raise _mount_error(exc) from exc
 
 
 def _nfs_helper(volume: str, command: list[str] | str = "true"):
@@ -359,7 +416,7 @@ def _nfs_helper(volume: str, command: list[str] | str = "true"):
     import docker.errors
 
     from config import settings
-    from docker_client import DockerError, get_client, safe_call
+    from docker_client import HELPER_LABEL, DockerError, get_client, safe_call
     from services.stack_assets_service import ensure_helper_image
 
     client = get_client()
@@ -370,9 +427,10 @@ def _nfs_helper(volume: str, command: list[str] | str = "true"):
             settings.BACKUP_HELPER_IMAGE,
             command=command,
             volumes={volume: {"bind": "/nfs", "mode": "rw"}},
+            labels={HELPER_LABEL: "1"},
         )
     except (docker.errors.APIError, DockerError) as exc:
-        raise DestinationError(f"NFS mount failed: {exc}") from exc
+        raise _mount_error(exc) from exc
 
 
 def _nfs_upload(dest: BackupDestination, cfg: dict, local_path: str, filename: str) -> str: