stackpilot/backend/tests/test_helper_containers.py

"""Throwaway containers must not outlive their job.

docker-py's ``containers.run(remove=True)`` only removes a container after it
ran. A helper whose start failed — an NFS destination that refuses the mount —
stayed behind in "Created", holding the destination's volume. Every retry of
"Test" or of a scheduled backup added one (a user found 18), and because the
volume stayed in use, changing the destination's mount options never took
effect: the volume could not be recreated with them.
"""
from __future__ import annotations

import pytest

from docker_client import HELPER_LABEL, DockerError


class FakeContainer:
    def __init__(self, client, image, command=None, volumes=None, labels=None, status="created"):
        self.client, self.image, self.command = client, image, command
        self.volumes, self.labels, self.status = volumes or {}, labels or {}, status
        self.removed = False

    def start(self):
        if self.client.start_error:
            raise DockerError("docker_api_error", self.client.start_error)
        self.status = "exited"

    def wait(self, timeout=None):
        self.client.wait_timeouts.append(timeout)
        return {"StatusCode": self.client.exit_code}

    def logs(self, stdout=True, stderr=True):
        return b"out" if stdout else b"boom"

    def remove(self, force=False):
        self.removed = True
        self.client.all.remove(self)


class FakeClient:
    def __init__(self, start_error=None, exit_code=0):
        self.start_error, self.exit_code = start_error, exit_code
        self.all: list[FakeContainer] = []
        self.wait_timeouts: list = []
        self.containers = self
        self.volumes = FakeVolumes(self)
        self.images = self

    def create(self, image, command=None, volumes=None, labels=None):
        c = FakeContainer(self, image, command, volumes, labels)
        self.all.append(c)
        return c

    def get(self, image):
        return object()

    def list(self, all=False, filters=None):
        filters = filters or {}
        out = list(self.all)
        if "label" in filters:
            out = [c for c in out if filters["label"] in c.labels]
        if "volume" in filters:
            out = [c for c in out if filters["volume"] in c.volumes]
        return out


class FakeVolume:
    def __init__(self, client, name, options, labels):
        self.client, self.name = client, name
        self.attrs = {"Options": options, "Labels": labels}

    def remove(self):
        if any(self.name in c.volumes for c in self.client.all):
            raise DockerError("docker_api_error", "volume is in use")
        del self.client.volumes.items[self.name]


class FakeVolumes:
    def __init__(self, client):
        self.client = client
        self.items: dict[str, FakeVolume] = {}

    def get(self, name):
        if name not in self.items:
            raise DockerError("not_found", name)
        return self.items[name]

    def create(self, name, driver, driver_opts, labels):
        self.items[name] = FakeVolume(self.client, name, driver_opts, labels)
        return self.items[name]

    def list(self, filters=None):
        label = (filters or {}).get("label")
        return [v for v in self.items.values() if not label or label in v.attrs["Labels"]]


def test_container_that_cannot_start_is_removed():
    from docker_client import run_container

    client = FakeClient(start_error="failed to mount local volume: connection refused")
    with pytest.raises(DockerError, match="connection refused"):
        run_container(client, "alpine", ["true"], volumes={"nfs": {"bind": "/nfs"}})
    assert client.all == []


def test_failed_command_is_removed_and_reports_stderr():
    from docker_client import run_container

    client = FakeClient(exit_code=2)
    with pytest.raises(DockerError) as info:
        run_container(client, "alpine", ["false"])
    assert info.value.error == "container_failed" and info.value.detail == "boom"
    assert client.all == []


def test_successful_run_returns_stdout_and_is_labelled():
    from docker_client import run_container

    client = FakeClient()
    seen = []
    original = client.create
    client.create = lambda *a, **k: seen.append(k["labels"]) or original(*a, **k)
    assert run_container(client, "alpine", ["echo"], timeout=900) == b"out"
    assert seen == [{HELPER_LABEL: "1"}] and client.all == []
    assert client.wait_timeouts == [900]


def test_startup_removes_stopped_helpers_only():
    from docker_client import remove_stale_helpers

    client = FakeClient()
    stale = client.create("alpine", labels={HELPER_LABEL: "1"})
    busy = client.create("alpine", labels={HELPER_LABEL: "1"})
    busy.status = "running"
    foreign = client.create("nginx", labels={})

    assert remove_stale_helpers(client) == 1
    assert stale.removed and not busy.removed and not foreign.removed


def _nfs_dest(options="rw"):
    from models.backup_destination import BackupDestination

    return BackupDestination(id=1, name="nas", type="nfs", config="{}"), {
        "server": "10.10.1.80", "path": "/mnt/user/Share", "options": options,
    }


def test_changed_nfs_options_take_effect_despite_leftovers(monkeypatch):
    from services import backup_destination_service as svc

    client = FakeClient()
    monkeypatch.setattr("docker_client.get_client", lambda: client)
    dest, cfg = _nfs_dest("rw")
    name = svc._nfs_volume(dest, cfg)
    # 18 helpers left behind by failed mounts under the old version.
    for _ in range(18):
        client.create("alpine", volumes={name: {"bind": "/nfs"}})

    dest, cfg = _nfs_dest("rw,nfsvers=4")
    assert svc._nfs_volume(dest, cfg) == name
    assert client.volumes.items[name].attrs["Options"]["o"] == "addr=10.10.1.80,rw,nfsvers=4"
    assert client.all == []


def test_volume_held_by_a_running_container_is_a_clear_error(monkeypatch):
    from services import backup_destination_service as svc

    client = FakeClient()
    monkeypatch.setattr("docker_client.get_client", lambda: client)
    dest, cfg = _nfs_dest("rw")
    name = svc._nfs_volume(dest, cfg)
    client.create("alpine", volumes={name: {"bind": "/nfs"}}).status = "running"

    dest, cfg = _nfs_dest("rw,nfsvers=4")
    with pytest.raises(svc.DestinationError, match="still in use"):
        svc._nfs_volume(dest, cfg)


def test_clear_nfs_leftovers_covers_every_destination():
    from services import backup_destination_service as svc

    client = FakeClient()
    client.volumes.create("stackpilot-nfs-dest-1", "local", {}, {"stackpilot.nfs-destination": "1"})
    client.volumes.create("stackpilot-nfs-dest-2", "local", {}, {"stackpilot.nfs-destination": "2"})
    client.volumes.create("app_data", "local", {}, {})
    client.create("alpine", volumes={"stackpilot-nfs-dest-1": {}})
    client.create("alpine", volumes={"stackpilot-nfs-dest-2": {}})
    keep = client.create("postgres", volumes={"app_data": {}})

    assert svc.clear_nfs_leftovers(client) == 2
    assert client.all == [keep]


def test_refused_mount_says_what_to_do(monkeypatch):
    from services import backup_destination_service as svc

    client = FakeClient(start_error=(
        "error while mounting volume: failed to mount local volume: mount "
        ":/mnt/user/Share: data: addr=10.10.1.80: connection refused"
    ))
    monkeypatch.setattr("docker_client.get_long_client", lambda: client)
    monkeypatch.setattr("services.stack_assets_service.ensure_helper_image", lambda c: None)

    with pytest.raises(svc.DestinationError) as info:
        svc._nfs_run("stackpilot-nfs-dest-1", ["mkdir", "-p", "/nfs"])
    assert "rpcbind" in str(info.value) and "2049" in str(info.value)
    assert client.all == []  # and nothing is left behind


# --------------------------------------------------------------------------- #
# nolock: NFS destinations that work without nfs-common on the Docker host
# --------------------------------------------------------------------------- #


@pytest.mark.parametrize(
    "options, expected",
    [
        (None, "rw,nolock"),
        ("", "rw,nolock"),
        ("rw", "rw,nolock"),
        ("rw,soft,", "rw,soft,nolock"),
        ("rw,nfsvers=3", "rw,nfsvers=3,nolock"),
        ("rw,lock", "rw,lock"),  # asked for locking: respected
        ("rw,nolock", "rw,nolock"),
        ("rw,local_lock=all", "rw,local_lock=all"),
        ("rw,nfsvers=4", "rw,nfsvers=4"),  # v4 locks in-protocol, no rpcbind
        ("rw,vers=4.1", "rw,vers=4.1"),
    ],
)
def test_nolock_is_added_for_nfs_v3_only(options, expected):
    from services.backup_destination_service import mount_options

    assert mount_options({"options": options}) == expected


def test_existing_destination_is_remounted_with_nolock(monkeypatch):
    """A destination saved with plain "rw" gets its volume recreated once."""
    from services import backup_destination_service as svc

    client = FakeClient()
    monkeypatch.setattr("docker_client.get_client", lambda: client)
    dest, cfg = _nfs_dest("rw")
    name = f"{svc._NFS_VOLUME_PREFIX}{dest.id}"
    # As 0.66.1 and earlier created it.
    client.volumes.create(name, "local", {
        "type": "nfs", "o": "addr=10.10.1.80,rw", "device": ":/mnt/user/Share",
    }, {"stackpilot.nfs-destination": "1"})

    svc._nfs_volume(dest, cfg)

    assert client.volumes.items[name].attrs["Options"]["o"] == "addr=10.10.1.80,rw,nolock"


def test_refused_mount_points_at_rpcbind_on_the_docker_host(monkeypatch):
    from services import backup_destination_service as svc

    client = FakeClient(start_error="failed to mount local volume: connection refused")
    monkeypatch.setattr("docker_client.get_long_client", lambda: client)
    monkeypatch.setattr("services.stack_assets_service.ensure_helper_image", lambda c: None)
    with pytest.raises(svc.DestinationError) as info:
        svc._nfs_run("stackpilot-nfs-dest-1", ["true"])
    assert "nfs-common" in str(info.value)