"""Throwaway containers must not outlive their job. docker-py's ``containers.run(remove=True)`` only removes a container after it ran. A helper whose start failed — an NFS destination that refuses the mount — stayed behind in "Created", holding the destination's volume. Every retry of "Test" or of a scheduled backup added one (a user found 18), and because the volume stayed in use, changing the destination's mount options never took effect: the volume could not be recreated with them. """ from __future__ import annotations import pytest from docker_client import HELPER_LABEL, DockerError class FakeContainer: def __init__(self, client, image, command=None, volumes=None, labels=None, status="created"): self.client, self.image, self.command = client, image, command self.volumes, self.labels, self.status = volumes or {}, labels or {}, status self.removed = False def start(self): if self.client.start_error: raise DockerError("docker_api_error", self.client.start_error) self.status = "exited" def wait(self, timeout=None): self.client.wait_timeouts.append(timeout) return {"StatusCode": self.client.exit_code} def logs(self, stdout=True, stderr=True): return b"out" if stdout else b"boom" def remove(self, force=False): self.removed = True self.client.all.remove(self) class FakeClient: def __init__(self, start_error=None, exit_code=0): self.start_error, self.exit_code = start_error, exit_code self.all: list[FakeContainer] = [] self.wait_timeouts: list = [] self.containers = self self.volumes = FakeVolumes(self) self.images = self def create(self, image, command=None, volumes=None, labels=None): c = FakeContainer(self, image, command, volumes, labels) self.all.append(c) return c def get(self, image): return object() def list(self, all=False, filters=None): filters = filters or {} out = list(self.all) if "label" in filters: out = [c for c in out if filters["label"] in c.labels] if "volume" in filters: out = [c for c in out if filters["volume"] in c.volumes] return out class FakeVolume: def __init__(self, client, name, options, labels): self.client, self.name = client, name self.attrs = {"Options": options, "Labels": labels} def remove(self): if any(self.name in c.volumes for c in self.client.all): raise DockerError("docker_api_error", "volume is in use") del self.client.volumes.items[self.name] class FakeVolumes: def __init__(self, client): self.client = client self.items: dict[str, FakeVolume] = {} def get(self, name): if name not in self.items: raise DockerError("not_found", name) return self.items[name] def create(self, name, driver, driver_opts, labels): self.items[name] = FakeVolume(self.client, name, driver_opts, labels) return self.items[name] def list(self, filters=None): label = (filters or {}).get("label") return [v for v in self.items.values() if not label or label in v.attrs["Labels"]] def test_container_that_cannot_start_is_removed(): from docker_client import run_container client = FakeClient(start_error="failed to mount local volume: connection refused") with pytest.raises(DockerError, match="connection refused"): run_container(client, "alpine", ["true"], volumes={"nfs": {"bind": "/nfs"}}) assert client.all == [] def test_failed_command_is_removed_and_reports_stderr(): from docker_client import run_container client = FakeClient(exit_code=2) with pytest.raises(DockerError) as info: run_container(client, "alpine", ["false"]) assert info.value.error == "container_failed" and info.value.detail == "boom" assert client.all == [] def test_successful_run_returns_stdout_and_is_labelled(): from docker_client import run_container client = FakeClient() seen = [] original = client.create client.create = lambda *a, **k: seen.append(k["labels"]) or original(*a, **k) assert run_container(client, "alpine", ["echo"], timeout=900) == b"out" assert seen == [{HELPER_LABEL: "1"}] and client.all == [] assert client.wait_timeouts == [900] def test_startup_removes_stopped_helpers_only(): from docker_client import remove_stale_helpers client = FakeClient() stale = client.create("alpine", labels={HELPER_LABEL: "1"}) busy = client.create("alpine", labels={HELPER_LABEL: "1"}) busy.status = "running" foreign = client.create("nginx", labels={}) assert remove_stale_helpers(client) == 1 assert stale.removed and not busy.removed and not foreign.removed def _nfs_dest(options="rw"): from models.backup_destination import BackupDestination return BackupDestination(id=1, name="nas", type="nfs", config="{}"), { "server": "10.10.1.80", "path": "/mnt/user/Share", "options": options, } def test_changed_nfs_options_take_effect_despite_leftovers(monkeypatch): from services import backup_destination_service as svc client = FakeClient() monkeypatch.setattr("docker_client.get_client", lambda: client) dest, cfg = _nfs_dest("rw") name = svc._nfs_volume(dest, cfg) # 18 helpers left behind by failed mounts under the old version. for _ in range(18): client.create("alpine", volumes={name: {"bind": "/nfs"}}) dest, cfg = _nfs_dest("rw,nfsvers=4") assert svc._nfs_volume(dest, cfg) == name assert client.volumes.items[name].attrs["Options"]["o"] == "addr=10.10.1.80,rw,nfsvers=4" assert client.all == [] def test_volume_held_by_a_running_container_is_a_clear_error(monkeypatch): from services import backup_destination_service as svc client = FakeClient() monkeypatch.setattr("docker_client.get_client", lambda: client) dest, cfg = _nfs_dest("rw") name = svc._nfs_volume(dest, cfg) client.create("alpine", volumes={name: {"bind": "/nfs"}}).status = "running" dest, cfg = _nfs_dest("rw,nfsvers=4") with pytest.raises(svc.DestinationError, match="still in use"): svc._nfs_volume(dest, cfg) def test_clear_nfs_leftovers_covers_every_destination(): from services import backup_destination_service as svc client = FakeClient() client.volumes.create("stackpilot-nfs-dest-1", "local", {}, {"stackpilot.nfs-destination": "1"}) client.volumes.create("stackpilot-nfs-dest-2", "local", {}, {"stackpilot.nfs-destination": "2"}) client.volumes.create("app_data", "local", {}, {}) client.create("alpine", volumes={"stackpilot-nfs-dest-1": {}}) client.create("alpine", volumes={"stackpilot-nfs-dest-2": {}}) keep = client.create("postgres", volumes={"app_data": {}}) assert svc.clear_nfs_leftovers(client) == 2 assert client.all == [keep] def test_refused_mount_says_what_to_do(monkeypatch): from services import backup_destination_service as svc client = FakeClient(start_error=( "error while mounting volume: failed to mount local volume: mount " ":/mnt/user/Share: data: addr=10.10.1.80: connection refused" )) monkeypatch.setattr("docker_client.get_long_client", lambda: client) monkeypatch.setattr("services.stack_assets_service.ensure_helper_image", lambda c: None) with pytest.raises(svc.DestinationError) as info: svc._nfs_run("stackpilot-nfs-dest-1", ["mkdir", "-p", "/nfs"]) assert "rpcbind" in str(info.value) and "2049" in str(info.value) assert client.all == [] # and nothing is left behind # --------------------------------------------------------------------------- # # nolock: NFS destinations that work without nfs-common on the Docker host # --------------------------------------------------------------------------- # @pytest.mark.parametrize( "options, expected", [ (None, "rw,nolock"), ("", "rw,nolock"), ("rw", "rw,nolock"), ("rw,soft,", "rw,soft,nolock"), ("rw,nfsvers=3", "rw,nfsvers=3,nolock"), ("rw,lock", "rw,lock"), # asked for locking: respected ("rw,nolock", "rw,nolock"), ("rw,local_lock=all", "rw,local_lock=all"), ("rw,nfsvers=4", "rw,nfsvers=4"), # v4 locks in-protocol, no rpcbind ("rw,vers=4.1", "rw,vers=4.1"), ], ) def test_nolock_is_added_for_nfs_v3_only(options, expected): from services.backup_destination_service import mount_options assert mount_options({"options": options}) == expected def test_existing_destination_is_remounted_with_nolock(monkeypatch): """A destination saved with plain "rw" gets its volume recreated once.""" from services import backup_destination_service as svc client = FakeClient() monkeypatch.setattr("docker_client.get_client", lambda: client) dest, cfg = _nfs_dest("rw") name = f"{svc._NFS_VOLUME_PREFIX}{dest.id}" # As 0.66.1 and earlier created it. client.volumes.create(name, "local", { "type": "nfs", "o": "addr=10.10.1.80,rw", "device": ":/mnt/user/Share", }, {"stackpilot.nfs-destination": "1"}) svc._nfs_volume(dest, cfg) assert client.volumes.items[name].attrs["Options"]["o"] == "addr=10.10.1.80,rw,nolock" def test_refused_mount_points_at_rpcbind_on_the_docker_host(monkeypatch): from services import backup_destination_service as svc client = FakeClient(start_error="failed to mount local volume: connection refused") monkeypatch.setattr("docker_client.get_long_client", lambda: client) monkeypatch.setattr("services.stack_assets_service.ensure_helper_image", lambda c: None) with pytest.raises(svc.DestinationError) as info: svc._nfs_run("stackpilot-nfs-dest-1", ["true"]) assert "nfs-common" in str(info.value)