"""Throwaway containers must not outlive their job. docker-py's ``containers.run(remove=True)`` only removes a container after it ran. A helper whose start failed — an NFS destination that refuses the mount — stayed behind in "Created", holding the destination's volume. Every retry of "Test" or of a scheduled backup added one (a user found 18), and because the volume stayed in use, changing the destination's mount options never took effect: the volume could not be recreated with them. """ from __future__ import annotations import pytest from docker_client import HELPER_LABEL, DockerError class FakeContainer: def __init__(self, client, image, command=None, volumes=None, labels=None, status="created"): self.client, self.image, self.command = client, image, command self.volumes, self.labels, self.status = volumes or {}, labels or {}, status self.removed = False def start(self): if self.client.start_error: raise DockerError("docker_api_error", self.client.start_error) self.status = "exited" def wait(self, timeout=None): self.client.wait_timeouts.append(timeout) return {"StatusCode": self.client.exit_code} def logs(self, stdout=True, stderr=True): return b"out" if stdout else b"boom" def remove(self, force=False): self.removed = True self.client.all.remove(self) class FakeClient: def __init__(self, start_error=None, exit_code=0): self.start_error, self.exit_code = start_error, exit_code self.all: list[FakeContainer] = [] self.wait_timeouts: list = [] self.containers = self self.volumes = FakeVolumes(self) self.images = self def create(self, image, command=None, volumes=None, labels=None): c = FakeContainer(self, image, command, volumes, labels) self.all.append(c) return c def get(self, image): return object() def list(self, all=False, filters=None): filters = filters or {} out = list(self.all) if "label" in filters: out = [c for c in out if filters["label"] in c.labels] if "volume" in filters: out = [c for c in out if filters["volume"] in c.volumes] return out class FakeVolume: def __init__(self, client, name, options, labels): self.client, self.name = client, name self.attrs = {"Options": options, "Labels": labels} def remove(self): if any(self.name in c.volumes for c in self.client.all): raise DockerError("docker_api_error", "volume is in use") del self.client.volumes.items[self.name] class FakeVolumes: def __init__(self, client): self.client = client self.items: dict[str, FakeVolume] = {} def get(self, name): if name not in self.items: raise DockerError("not_found", name) return self.items[name] def create(self, name, driver, driver_opts, labels): self.items[name] = FakeVolume(self.client, name, driver_opts, labels) return self.items[name] def list(self, filters=None): label = (filters or {}).get("label") return [v for v in self.items.values() if not label or label in v.attrs["Labels"]] def test_container_that_cannot_start_is_removed(): from docker_client import run_container client = FakeClient(start_error="failed to mount local volume: connection refused") with pytest.raises(DockerError, match="connection refused"): run_container(client, "alpine", ["true"], volumes={"nfs": {"bind": "/nfs"}}) assert client.all == [] def test_failed_command_is_removed_and_reports_stderr(): from docker_client import run_container client = FakeClient(exit_code=2) with pytest.raises(DockerError) as info: run_container(client, "alpine", ["false"]) assert info.value.error == "container_failed" and info.value.detail == "boom" assert client.all == [] def test_successful_run_returns_stdout_and_is_labelled(): from docker_client import run_container client = FakeClient() seen = [] original = client.create client.create = lambda *a, **k: seen.append(k["labels"]) or original(*a, **k) assert run_container(client, "alpine", ["echo"], timeout=900) == b"out" assert seen == [{HELPER_LABEL: "1"}] and client.all == [] assert client.wait_timeouts == [900] def test_startup_removes_stopped_helpers_only(): from docker_client import remove_stale_helpers client = FakeClient() stale = client.create("alpine", labels={HELPER_LABEL: "1"}) busy = client.create("alpine", labels={HELPER_LABEL: "1"}) busy.status = "running" foreign = client.create("nginx", labels={}) assert remove_stale_helpers(client) == 1 assert stale.removed and not busy.removed and not foreign.removed def _nfs_dest(options="rw"): from models.backup_destination import BackupDestination return BackupDestination(id=1, name="nas", type="nfs", config="{}"), { "server": "10.10.1.80", "path": "/mnt/user/Share", "options": options, } def test_changed_nfs_options_take_effect_despite_leftovers(monkeypatch): from services import backup_destination_service as svc client = FakeClient() monkeypatch.setattr("docker_client.get_client", lambda: client) dest, cfg = _nfs_dest("rw") name = svc._nfs_volume(dest, cfg) # 18 helpers left behind by failed mounts under the old version. for _ in range(18): client.create("alpine", volumes={name: {"bind": "/nfs"}}) dest, cfg = _nfs_dest("rw,nfsvers=4") assert svc._nfs_volume(dest, cfg) == name assert client.volumes.items[name].attrs["Options"]["o"] == "addr=10.10.1.80,rw,nfsvers=4" assert client.all == [] def test_volume_held_by_a_running_container_is_a_clear_error(monkeypatch): from services import backup_destination_service as svc client = FakeClient() monkeypatch.setattr("docker_client.get_client", lambda: client) dest, cfg = _nfs_dest("rw") name = svc._nfs_volume(dest, cfg) client.create("alpine", volumes={name: {"bind": "/nfs"}}).status = "running" dest, cfg = _nfs_dest("rw,nfsvers=4") with pytest.raises(svc.DestinationError, match="still in use"): svc._nfs_volume(dest, cfg) def test_clear_nfs_leftovers_covers_every_destination(): from services import backup_destination_service as svc client = FakeClient() client.volumes.create("stackpilot-nfs-dest-1", "local", {}, {"stackpilot.nfs-destination": "1"}) client.volumes.create("stackpilot-nfs-dest-2", "local", {}, {"stackpilot.nfs-destination": "2"}) client.volumes.create("app_data", "local", {}, {}) client.create("alpine", volumes={"stackpilot-nfs-dest-1": {}}) client.create("alpine", volumes={"stackpilot-nfs-dest-2": {}}) keep = client.create("postgres", volumes={"app_data": {}}) assert svc.clear_nfs_leftovers(client) == 2 assert client.all == [keep] def test_refused_mount_says_what_to_do(monkeypatch): from services import backup_destination_service as svc client = FakeClient(start_error=( "error while mounting volume: failed to mount local volume: mount " ":/mnt/user/Share: data: addr=10.10.1.80: connection refused" )) monkeypatch.setattr("docker_client.get_long_client", lambda: client) monkeypatch.setattr("services.stack_assets_service.ensure_helper_image", lambda c: None) with pytest.raises(svc.DestinationError) as info: svc._nfs_run("stackpilot-nfs-dest-1", ["mkdir", "-p", "/nfs"]) assert "nfsvers=4" in str(info.value) and "port 2049" in str(info.value) assert client.all == [] # and nothing is left behind