stackpilot/backend/services/backup_destination_service.py

"""Push/pull stack backups to remote destinations (SFTP, S3-compatible or NFS).

All operations are synchronous (paramiko / boto3 / docker); async callers
should wrap them with ``asyncio.to_thread``. Destination config is a dict
stored in the ``BackupDestination.config`` column as JSON, encrypted at rest
(:mod:`services.crypto_service`) because it carries SFTP passwords, SSH keys
and S3 secret keys. Always go through :func:`parse_config` / :func:`dump_config`
— never touch the column directly.
"""
from __future__ import annotations

import io
import json
import logging
import posixpath
import stat
import tarfile
import tempfile
from typing import Any, Optional

from models.backup_destination import BackupDestination
from services import crypto_service

logger = logging.getLogger("stackpilot.backup_dest")


class DestinationError(Exception):
    pass


def parse_config(dest: BackupDestination) -> dict:
    """Decrypt and parse a destination's config.

    Tolerates plaintext (pre-encryption rows) and returns ``{}`` rather than
    raising if the value can't be decrypted — a destination whose key is gone
    should show up as unconfigured in the UI, not take the whole list down with
    a 500. The failure is logged with the destination name so it's findable.
    """
    try:
        raw = crypto_service.decrypt(dest.config or "{}")
    except crypto_service.DecryptError as exc:
        logger.error("Destination '%s': %s", dest.name, exc)
        return {}
    try:
        return json.loads(raw or "{}")
    except json.JSONDecodeError:
        return {}


def dump_config(config: dict) -> str:
    """Serialise and encrypt a config dict for storage."""
    return crypto_service.encrypt(json.dumps(config or {}))


def migrate_plaintext_configs(session) -> int:
    """Encrypt destination configs written before encryption existed.

    Runs once at startup. Returns how many rows were rewritten.
    """
    from sqlmodel import select

    migrated = 0
    for dest in session.exec(select(BackupDestination)).all():
        if crypto_service.is_encrypted(dest.config):
            continue
        dest.config = crypto_service.encrypt(dest.config or "{}")
        session.add(dest)
        migrated += 1
    if migrated:
        session.commit()
    return migrated


# --------------------------------------------------------------------------- #
# SFTP (paramiko)
# --------------------------------------------------------------------------- #


#: Seconds to wait for the SFTP server to accept the TCP connection.
SFTP_CONNECT_TIMEOUT = 20


def _check_file_name(name: str) -> None:
    """A backup is addressed by its bare file name, never a path.

    The name comes from the API (restore from / delete on a destination); with
    ``../`` in it a delete reached files outside the backup folder — on SFTP
    anything the account may write. NFS has always checked this.
    """
    if (
        not name
        or name in (".", "..")
        or "/" in name
        or "\\" in name
        or "\0" in name
    ):
        raise DestinationError(f"Invalid backup file name '{name}'")


def host_key_fingerprint(key) -> str:
    """OpenSSH's form: ``SHA256:`` + unpadded base64 of the key's SHA-256."""
    import base64
    import hashlib

    digest = hashlib.sha256(key.asbytes()).digest()
    return "SHA256:" + base64.b64encode(digest).decode().rstrip("=")


def fingerprint_of(pinned: str) -> str:
    """The ``SHA256:`` fingerprint of a stored host key (as given, if it is
    one already) — what the UI shows and ``ssh-keygen -lf`` prints."""
    import base64
    import hashlib

    pinned = pinned.strip()
    if pinned.startswith("SHA256:"):
        return pinned
    parts = pinned.split()
    for i, part in enumerate(parts[:-1]):
        if part.startswith(("ssh-", "ecdsa-", "sk-")):
            try:
                blob = base64.b64decode(parts[i + 1])
            except ValueError:
                break
            return "SHA256:" + base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip("=")
    return pinned


def _pinned_type(pinned: str) -> Optional[str]:
    """The key type of a pinned ``<type> <base64>`` line (a known_hosts line
    with the host in front works too); None for a bare fingerprint."""
    parts = pinned.split()
    for part in parts[:-1]:
        if part.startswith(("ssh-", "ecdsa-", "sk-")):
            return part
    return None


#: What paramiko negotiates for a key of each type. An RSA key is offered
#: under the SHA-2 signature names, not under its own.
_HOST_KEY_ALGORITHMS = {
    "ssh-rsa": ("rsa-sha2-512", "rsa-sha2-256", "ssh-rsa"),
}


def _host_key_matches(pinned: str, key) -> bool:
    pinned = pinned.strip()
    if pinned.startswith("SHA256:"):
        return pinned.rstrip("=") == host_key_fingerprint(key)
    parts = pinned.split()
    for i, part in enumerate(parts[:-1]):
        if part == key.get_name() and parts[i + 1] == key.get_base64():
            return True
    return False


def _sftp_connect(cfg: dict, pin=None):
    """Connect and authenticate, checking the server's host key.

    Before 0.67.1 any host key was accepted, so whoever could answer on the
    server's address (a spoofed DNS entry, a box that took its IP) received
    the password and every backup. Now the key is pinned in the destination's
    ``host_key``: entered by hand (a key line or its ``SHA256:`` fingerprint),
    or — when empty — the one seen on the first connection, handed to ``pin``
    to be stored. After that a different key refuses the connection.
    """
    import socket

    import paramiko

    host = cfg.get("host")
    if not host:
        raise DestinationError("SFTP host is required")
    try:
        port = int(cfg.get("port") or 22)
    except (TypeError, ValueError) as exc:
        raise DestinationError(f"Invalid SFTP port '{cfg.get('port')}'") from exc
    username = cfg.get("username")
    pinned = (cfg.get("host_key") or "").strip()
    # Connected here, with a timeout, rather than by Transport((host, port)):
    # that connects inside the constructor, outside any handler, so an
    # unreachable host was a bare 500 instead of a message — and with no
    # timeout it hung for the kernel's two minutes first.
    try:
        sock = socket.create_connection((host, port), timeout=SFTP_CONNECT_TIMEOUT)
    except OSError as exc:
        raise DestinationError(f"SFTP connection to {host}:{port} failed: {exc}") from exc
    transport = paramiko.Transport(sock)
    try:
        key_type = _pinned_type(pinned) if pinned else None
        if key_type:
            # Ask for the pinned kind of key: a server has several, and the
            # one negotiated by default need not be the one that was pinned.
            wanted = _HOST_KEY_ALGORITHMS.get(key_type, (key_type,))
            options = transport.get_security_options()
            usable = tuple(k for k in wanted if k in options.key_types)
            if usable:
                options.key_types = usable
        transport.start_client(timeout=SFTP_CONNECT_TIMEOUT)
        server_key = transport.get_remote_server_key()
        if pinned:
            if not _host_key_matches(pinned, server_key):
                raise DestinationError(
                    f"SFTP host key of {host}:{port} does not match the stored one: the "
                    f"server now presents {server_key.get_name()} "
                    f"{host_key_fingerprint(server_key)}. If the server was reinstalled or "
                    "its keys were rotated, clear the destination's host key to trust the "
                    "new one — otherwise something else is answering on that address."
                )
        elif pin is not None:
            pin(f"{server_key.get_name()} {server_key.get_base64()}")
        pkey = None
        if cfg.get("private_key"):
            pkey = _load_key(cfg["private_key"])
        # What Transport.connect did: the key when there is one, else the password.
        if pkey is not None:
            transport.auth_publickey(username, pkey)
        elif cfg.get("password"):
            transport.auth_password(username, cfg["password"])
        return paramiko.SFTPClient.from_transport(transport), transport
    except Exception as exc:  # noqa: BLE001
        transport.close()
        if isinstance(exc, DestinationError):
            raise
        raise DestinationError(f"SFTP connection failed: {exc}") from exc


def _sftp_io(what: str):
    """Turn what paramiko raises mid-transfer into a DestinationError."""
    import contextlib

    @contextlib.contextmanager
    def wrapper():
        try:
            yield
        except DestinationError:
            raise
        except Exception as exc:  # noqa: BLE001 - IOError, SSHException, EOFError …
            raise DestinationError(f"SFTP {what} failed: {exc}") from exc

    return wrapper()


def _load_key(key_str: str):
    import paramiko

    for cls in (paramiko.Ed25519Key, paramiko.ECDSAKey, paramiko.RSAKey):
        try:
            return cls.from_private_key(io.StringIO(key_str))
        except Exception:  # noqa: BLE001
            continue
    raise DestinationError("Could not parse SFTP private key")


def _sftp_makedirs(sftp, path: str) -> None:
    if not path or path in (".", "/"):
        return
    parts = path.strip("/").split("/")
    cur = "/" if path.startswith("/") else ""
    for p in parts:
        cur = posixpath.join(cur, p) if cur else p
        try:
            sftp.stat(cur)
        except IOError:
            sftp.mkdir(cur)


def _sftp_upload(cfg: dict, local_path: str, filename: str, pin=None) -> str:
    _check_file_name(filename)
    sftp, transport = _sftp_connect(cfg, pin)
    try:
        with _sftp_io("upload"):
            base = cfg.get("path") or "."
            if base not in (".", ""):
                _sftp_makedirs(sftp, base)
            remote = posixpath.join(base, filename) if base not in (".", "") else filename
            sftp.put(local_path, remote)
            return remote
    finally:
        sftp.close()
        transport.close()


def _sftp_list(cfg: dict, pin=None) -> list[dict]:
    sftp, transport = _sftp_connect(cfg, pin)
    try:
        base = cfg.get("path") or "."
        out = []
        try:
            entries = sftp.listdir_attr(base)
        except IOError:
            return []
        for e in entries:
            if stat.S_ISDIR(e.st_mode):
                continue
            if not e.filename.endswith(".tar.gz"):
                continue
            out.append({"name": e.filename, "size": e.st_size, "modified": e.st_mtime})
        return sorted(out, key=lambda x: x["modified"] or 0, reverse=True)
    finally:
        sftp.close()
        transport.close()


def _sftp_download(cfg: dict, name: str, local_path: str, pin=None) -> None:
    _check_file_name(name)
    sftp, transport = _sftp_connect(cfg, pin)
    try:
        with _sftp_io(f"download of '{name}'"):
            base = cfg.get("path") or "."
            remote = posixpath.join(base, name) if base not in (".", "") else name
            sftp.get(remote, local_path)
    finally:
        sftp.close()
        transport.close()


def _sftp_delete(cfg: dict, name: str, pin=None) -> None:
    _check_file_name(name)
    sftp, transport = _sftp_connect(cfg, pin)
    try:
        with _sftp_io(f"delete of '{name}'"):
            base = cfg.get("path") or "."
            remote = posixpath.join(base, name) if base not in (".", "") else name
            sftp.remove(remote)
    finally:
        sftp.close()
        transport.close()


_TEST_FILE = ".stackpilot-test"


def _sftp_test(cfg: dict, pin=None) -> None:
    """Connect, create the folder if needed, write and remove a file.

    Listing alone said "OK" for a folder that does not exist or cannot be
    written to — the first scheduled backup was then the one to find out.
    """
    sftp, transport = _sftp_connect(cfg, pin)
    try:
        with _sftp_io("write test"):
            base = cfg.get("path") or "."
            if base not in (".", ""):
                _sftp_makedirs(sftp, base)
            remote = posixpath.join(base, _TEST_FILE) if base not in (".", "") else _TEST_FILE
            with sftp.open(remote, "w") as fh:
                fh.write(b"ok")
            sftp.remove(remote)
    finally:
        sftp.close()
        transport.close()


# --------------------------------------------------------------------------- #
# S3-compatible (boto3)
# --------------------------------------------------------------------------- #


def _s3_client(cfg: dict):
    import boto3

    bucket = cfg.get("bucket")
    if not bucket:
        raise DestinationError("S3 bucket is required")
    try:
        return boto3.client(
            "s3",
            endpoint_url=cfg.get("endpoint_url") or None,
            region_name=cfg.get("region") or None,
            aws_access_key_id=cfg.get("access_key") or None,
            aws_secret_access_key=cfg.get("secret_key") or None,
        )
    except Exception as exc:  # noqa: BLE001 - e.g. a malformed endpoint URL
        raise DestinationError(f"Invalid S3 settings: {exc}") from exc


def _s3_key(cfg: dict, filename: str) -> str:
    prefix = (cfg.get("prefix") or "").strip("/")
    return f"{prefix}/{filename}" if prefix else filename


def _s3_upload(cfg: dict, local_path: str, filename: str) -> str:
    _check_file_name(filename)
    client = _s3_client(cfg)
    key = _s3_key(cfg, filename)
    try:
        client.upload_file(local_path, cfg["bucket"], key)
    except Exception as exc:  # noqa: BLE001
        raise DestinationError(f"S3 upload failed: {exc}") from exc
    return key


def _s3_list(cfg: dict) -> list[dict]:
    client = _s3_client(cfg)
    prefix = (cfg.get("prefix") or "").strip("/")
    kwargs: dict[str, Any] = {"Bucket": cfg["bucket"]}
    if prefix:
        kwargs["Prefix"] = prefix + "/"
    try:
        # Paginated: one call returns at most 1000 keys, and retention sorts
        # by age, so the oldest backups were the ones it never saw.
        pages = client.get_paginator("list_objects_v2").paginate(**kwargs)
        objects = [obj for page in pages for obj in page.get("Contents", [])]
    except Exception as exc:  # noqa: BLE001
        raise DestinationError(f"S3 list failed: {exc}") from exc
    out = []
    for obj in objects:
        name = obj["Key"].split("/")[-1]
        if not name.endswith(".tar.gz"):
            continue
        out.append(
            {
                "name": name,
                "size": obj.get("Size", 0),
                "modified": obj["LastModified"].timestamp() if obj.get("LastModified") else None,
            }
        )
    return sorted(out, key=lambda x: x["modified"] or 0, reverse=True)


def _s3_download(cfg: dict, name: str, local_path: str) -> None:
    _check_file_name(name)
    client = _s3_client(cfg)
    try:
        client.download_file(cfg["bucket"], _s3_key(cfg, name), local_path)
    except Exception as exc:  # noqa: BLE001
        raise DestinationError(f"S3 download failed: {exc}") from exc


def _s3_delete(cfg: dict, name: str) -> None:
    _check_file_name(name)
    client = _s3_client(cfg)
    try:
        client.delete_object(Bucket=cfg["bucket"], Key=_s3_key(cfg, name))
    except Exception as exc:  # noqa: BLE001
        raise DestinationError(f"S3 delete failed: {exc}") from exc


def _s3_test(cfg: dict) -> None:
    """List, then write and delete a small object: proves the keys may write."""
    _s3_list(cfg)
    client = _s3_client(cfg)
    key = _s3_key(cfg, _TEST_FILE)
    try:
        client.put_object(Bucket=cfg["bucket"], Key=key, Body=b"ok")
        client.delete_object(Bucket=cfg["bucket"], Key=key)
    except Exception as exc:  # noqa: BLE001
        raise DestinationError(f"S3 write test failed: {exc}") from exc


# --------------------------------------------------------------------------- #
# NFS — the Docker daemon mounts the export as a named volume; file I/O runs
# through a throwaway helper container (same pattern as volume backups), so
# the backend itself needs no mount privileges.
# --------------------------------------------------------------------------- #

_NFS_VOLUME_PREFIX = "stackpilot-nfs-dest-"


def _nfs_check_name(name: str) -> None:
    import re

    # Same safe charset as the subdir parts: the name is interpolated into the
    # helper container's shell commands.
    if not name or not re.fullmatch(r"[A-Za-z0-9._-]+", name) or name.startswith("."):
        raise DestinationError(f"Invalid backup file name '{name}'")


def _nfs_subdir(cfg: dict) -> str:
    """Sanitized relative directory inside the export ('' = export root).

    Parts are restricted to a safe charset because the path is interpolated
    into helper-container shell commands.
    """
    import re

    raw = (cfg.get("subdir") or "").strip().strip("/")
    if not raw:
        return ""
    parts = [p for p in raw.split("/") if p]
    for p in parts:
        if p == ".." or not re.fullmatch(r"[A-Za-z0-9._-]+", p):
            raise DestinationError(
                "Subdirectory may only contain letters, digits, '.', '_' and '-'"
            )
    return "/".join(parts)


def mount_options(cfg: dict) -> str:
    """The destination's mount options, with ``nolock`` added for NFSv2/3.

    An NFSv3 mount with locking makes the kernel register its lock manager
    with rpcbind on the *Docker host*, which is only there when nfs-common (or
    the distribution's equivalent) is installed; without it the mount fails
    with a bare "connection refused". Only StackPilot ever writes these backup
    files, one writer per file, so there is nothing to lock. An explicit
    lock/nolock/local_lock option is left as given, and NFSv4 has locking
    built into the protocol and needs no rpcbind at all.
    """
    raw = (cfg.get("options") or "rw").strip().strip(",")
    parts = [p.strip() for p in raw.split(",") if p.strip()]
    chosen = any(p in ("lock", "nolock") or p.startswith("local_lock=") for p in parts)
    v4 = any(p.split("=", 1)[0] in ("nfsvers", "vers") and p.split("=", 1)[-1].startswith("4")
             for p in parts)
    if not chosen and not v4:
        parts.append("nolock")
    return ",".join(parts)


def _nfs_volume(dest: BackupDestination, cfg: dict) -> str:
    """Ensure the named volume describing this NFS mount exists; recreate it
    when the destination's server/path/options changed (opts are immutable)."""
    from docker_client import DockerError, get_client, safe_call

    server = (cfg.get("server") or "").strip()
    path = (cfg.get("path") or "").strip()
    if not server:
        raise DestinationError("NFS server is required")
    if not path.startswith("/"):
        raise DestinationError("NFS export path must be absolute (start with /)")
    driver_opts = {"type": "nfs", "o": f"addr={server},{mount_options(cfg)}", "device": f":{path}"}

    name = f"{_NFS_VOLUME_PREFIX}{dest.id}"
    client = get_client()
    try:
        vol = safe_call(client.volumes.get, name)
    except DockerError:
        vol = None
    if vol is not None and (vol.attrs.get("Options") or {}) != driver_opts:
        # Stopped containers still hold the volume; before 0.66.1 every failed
        # mount left one behind, and then new settings never took effect.
        clear_nfs_leftovers(client, name)
        try:
            safe_call(vol.remove)
        except DockerError as exc:
            raise DestinationError(
                f"The NFS settings changed, but the Docker volume '{name}' is still in use "
                f"and cannot be recreated with them: {exc.detail or exc.error}"
            ) from exc
        vol = None
    if vol is None:
        safe_call(
            client.volumes.create,
            name=name,
            driver="local",
            driver_opts=driver_opts,
            labels={"stackpilot.nfs-destination": str(dest.id)},
        )
    return name


def forget_nfs_volume(dest_id: int) -> None:
    """Remove the Docker volume of a deleted NFS destination (best effort).

    Left behind, it kept the old mount options under a name the next
    destination with the same id would pick up again.
    """
    from docker_client import DockerError, get_client, safe_call

    name = f"{_NFS_VOLUME_PREFIX}{dest_id}"
    try:
        client = get_client()
        clear_nfs_leftovers(client, name)
        safe_call(safe_call(client.volumes.get, name).remove)
    except DockerError as exc:
        logger.info("NFS volume %s not removed: %s", name, exc.detail or exc.error)


def clear_nfs_leftovers(client, volume: Optional[str] = None) -> int:
    """Remove stopped containers holding an NFS destination volume.

    Only StackPilot's helpers ever mount these volumes, so any container on
    one that is not running is a leftover (older versions left one behind per
    failed mount). ``volume=None`` covers every NFS destination.
    """
    from docker_client import DockerError, safe_call

    if volume:
        names = [volume]
    else:
        try:
            vols = safe_call(
                client.volumes.list, filters={"label": "stackpilot.nfs-destination"}
            )
        except DockerError:
            return 0
        names = [v.name for v in vols]
    removed = 0
    for name in names:
        try:
            containers = safe_call(client.containers.list, all=True, filters={"volume": name})
        except DockerError:
            continue
        for container in containers:
            if container.status == "running":
                continue
            try:
                container.remove(force=True)
                removed += 1
            except Exception:  # noqa: BLE001
                pass
    return removed


def _nfs_target(cfg: dict) -> str:
    sub = _nfs_subdir(cfg)
    return f"/nfs/{sub}" if sub else "/nfs"


def _mount_error(exc: Exception) -> DestinationError:
    """A mount failure, with what to do about the common one."""
    text = str(exc)
    message = f"NFS mount failed: {text}"
    lowered = text.lower()
    if "connection refused" in lowered:
        message += (
            " — a connection was refused. With locking enabled ('lock' in the mount "
            "options) NFSv3 needs rpcbind running on the Docker host itself: install "
            "nfs-common there, or drop 'lock'. Otherwise check that NFS is enabled on the "
            "server and that ports 111, 2049 and mountd are reachable from the Docker host."
        )
    elif "protocol not supported" in lowered:
        message += (
            " — the Docker host's kernel cannot speak this NFS version: load the module "
            "(e.g. 'modprobe nfsv4') or remove nfsvers from the mount options."
        )
    return DestinationError(message)


def _nfs_run(volume: str, command: list[str]) -> str:
    """Run a helper container with the NFS volume at /nfs; return stdout."""
    from config import settings
    from docker_client import DockerError, get_long_client, run_container
    from services.stack_assets_service import ensure_helper_image

    client = get_long_client()
    ensure_helper_image(client)
    try:
        out = run_container(
            client,
            settings.BACKUP_HELPER_IMAGE,
            command,
            volumes={volume: {"bind": "/nfs", "mode": "rw"}},
        )
        return out.decode("utf-8", "replace")
    except DockerError as exc:
        if exc.error == "container_failed":
            raise DestinationError(f"NFS operation failed: {exc.detail}") from exc
        # Mount errors surface here (unreachable server, bad export, ...).
        raise _mount_error(exc) from exc


def _nfs_helper(volume: str, command: list[str] | str = "true"):
    """A created (not started) helper container for archive I/O on /nfs."""
    import docker.errors

    from config import settings
    from docker_client import HELPER_LABEL, DockerError, get_client, safe_call
    from services.stack_assets_service import ensure_helper_image

    client = get_client()
    ensure_helper_image(client)
    try:
        return safe_call(
            client.containers.create,
            settings.BACKUP_HELPER_IMAGE,
            command=command,
            volumes={volume: {"bind": "/nfs", "mode": "rw"}},
            labels={HELPER_LABEL: "1"},
        )
    except (docker.errors.APIError, DockerError) as exc:
        raise _mount_error(exc) from exc


def _nfs_upload(dest: BackupDestination, cfg: dict, local_path: str, filename: str) -> str:
    import docker.errors

    _nfs_check_name(filename)
    volume = _nfs_volume(dest, cfg)
    target = _nfs_target(cfg)
    # Creates the subdir if needed AND fails early with a clear mount error.
    _nfs_run(volume, ["mkdir", "-p", target])
    # Unpack into the container's own filesystem, then copy the file across:
    # extracting straight into the NFS mount makes the daemon chown the file,
    # which a root_squash export refuses ("failed to Lchown ... for UID 0").
    container = _nfs_helper(
        volume, ["sh", "-c", f"cat '/tmp/{filename}' > '{target}/{filename}'"]
    )
    try:
        with tempfile.TemporaryFile() as tmp:
            with tarfile.open(fileobj=tmp, mode="w") as tar:
                tar.add(local_path, arcname=filename)
            tmp.seek(0)
            container.put_archive("/tmp", tmp)
        container.start()
        status = container.wait(timeout=3600).get("StatusCode", 1)
        if status != 0:
            err = (container.logs(stdout=True, stderr=True) or b"").decode("utf-8", "replace")
            raise DestinationError(f"NFS upload failed: {err.strip() or f'exit {status}'}")
    except docker.errors.APIError as exc:
        raise DestinationError(f"NFS upload failed: {exc}") from exc
    finally:
        try:
            container.remove(force=True)
        except Exception:  # noqa: BLE001
            pass
    sub = _nfs_subdir(cfg)
    return posixpath.join(sub, filename) if sub else filename


def _nfs_list(dest: BackupDestination, cfg: dict) -> list[dict]:
    volume = _nfs_volume(dest, cfg)
    target = _nfs_target(cfg)
    out = _nfs_run(
        volume,
        ["sh", "-c", f"cd {target} 2>/dev/null && stat -c '%n|%s|%Y' *.tar.gz 2>/dev/null; true"],
    )
    entries = []
    for line in out.splitlines():
        parts = line.strip().split("|")
        if len(parts) != 3 or parts[0] == "*.tar.gz":
            continue
        try:
            entries.append({"name": parts[0], "size": int(parts[1]), "modified": int(parts[2])})
        except ValueError:
            continue
    return sorted(entries, key=lambda x: x["modified"] or 0, reverse=True)


def _nfs_download(dest: BackupDestination, cfg: dict, name: str, local_path: str) -> None:
    import docker.errors

    _nfs_check_name(name)
    volume = _nfs_volume(dest, cfg)
    target = _nfs_target(cfg)
    container = _nfs_helper(volume)
    try:
        bits, _ = container.get_archive(f"{target}/{name}")
        with tempfile.TemporaryFile() as tmp:
            for chunk in bits:
                tmp.write(chunk)
            tmp.seek(0)
            with tarfile.open(fileobj=tmp) as tar:
                member = next((m for m in tar.getmembers() if m.isreg()), None)
                fh = tar.extractfile(member) if member else None
                if fh is None:
                    raise DestinationError(f"'{name}' not found on NFS destination")
                with open(local_path, "wb") as out:
                    while chunk := fh.read(1024 * 1024):
                        out.write(chunk)
    except docker.errors.APIError as exc:
        raise DestinationError(f"NFS download failed (does '{name}' exist?): {exc}") from exc
    finally:
        try:
            container.remove(force=True)
        except Exception:  # noqa: BLE001
            pass


def _nfs_delete(dest: BackupDestination, cfg: dict, name: str) -> None:
    _nfs_check_name(name)
    volume = _nfs_volume(dest, cfg)
    _nfs_run(volume, ["rm", "-f", f"{_nfs_target(cfg)}/{name}"])


def _nfs_test(dest: BackupDestination, cfg: dict) -> bool:
    volume = _nfs_volume(dest, cfg)
    target = _nfs_target(cfg)
    _nfs_run(
        volume,
        ["sh", "-c", f"mkdir -p {target} && touch {target}/.stackpilot-test && rm -f {target}/.stackpilot-test"],
    )
    return True


# --------------------------------------------------------------------------- #
# Dispatch
# --------------------------------------------------------------------------- #


def _pinner(dest: BackupDestination):
    """Store the first host key an SFTP destination presents (trust on first
    use). Written straight to the row — the callers are a scheduled run and a
    "Test" click, and neither holds a session that will commit."""

    def pin(host_key: str) -> None:
        if dest.id is None:
            return  # not saved yet: nothing to pin it to
        from sqlmodel import Session

        from database import engine

        with Session(engine) as session:
            row = session.get(BackupDestination, dest.id)
            if row is None:
                return
            cfg = parse_config(row)
            if (cfg.get("host_key") or "").strip():
                return  # pinned meanwhile; the next connect compares against it
            cfg["host_key"] = host_key
            row.config = dump_config(cfg)
            session.add(row)
            session.commit()
        logger.info("Destination '%s': pinned SFTP host key %s", dest.name, host_key.split()[0])

    return pin


def upload(dest: BackupDestination, local_path: str, filename: str) -> str:
    cfg = parse_config(dest)
    if dest.type == "sftp":
        return _sftp_upload(cfg, local_path, filename, _pinner(dest))
    if dest.type == "s3":
        return _s3_upload(cfg, local_path, filename)
    if dest.type == "nfs":
        return _nfs_upload(dest, cfg, local_path, filename)
    raise DestinationError(f"Unknown destination type '{dest.type}'")


def list_backups(dest: BackupDestination) -> list[dict]:
    cfg = parse_config(dest)
    if dest.type == "sftp":
        return _sftp_list(cfg, _pinner(dest))
    if dest.type == "s3":
        return _s3_list(cfg)
    if dest.type == "nfs":
        return _nfs_list(dest, cfg)
    raise DestinationError(f"Unknown destination type '{dest.type}'")


def download(dest: BackupDestination, name: str, local_path: str) -> None:
    cfg = parse_config(dest)
    if dest.type == "sftp":
        _sftp_download(cfg, name, local_path, _pinner(dest))
    elif dest.type == "s3":
        _s3_download(cfg, name, local_path)
    elif dest.type == "nfs":
        _nfs_download(dest, cfg, name, local_path)
    else:
        raise DestinationError(f"Unknown destination type '{dest.type}'")


def delete(dest: BackupDestination, name: str) -> None:
    cfg = parse_config(dest)
    if dest.type == "sftp":
        _sftp_delete(cfg, name, _pinner(dest))
    elif dest.type == "s3":
        _s3_delete(cfg, name)
    elif dest.type == "nfs":
        _nfs_delete(dest, cfg, name)
    else:
        raise DestinationError(f"Unknown destination type '{dest.type}'")


def test(dest: BackupDestination) -> bool:
    """Connectivity check — validates reachability, auth and write access."""
    cfg = parse_config(dest)
    if dest.type == "nfs":
        return _nfs_test(dest, cfg)
    if dest.type == "sftp":
        _sftp_test(cfg, _pinner(dest))
        return True
    if dest.type == "s3":
        _s3_test(cfg)
        return True
    raise DestinationError(f"Unknown destination type '{dest.type}'")