stackpilot/backend/tests/test_stack_locking.py

"""One compose operation per stack (F7), and the lock surviving a restart (F11).

``docker compose`` does no locking of its own. Before 0.47.0 nothing stopped two
``update`` calls landing on the same project at once — two browser tabs, or the
auto-update pass picking up a stack somebody had just clicked — and both would
run ``pull`` then ``up -d`` and fight over recreating the same containers.

There was a busy flag, but it only fed the status column: no handler consulted
it before acting. The lock is a table now, so it also holds across workers and
across a restart.
"""
from __future__ import annotations

from datetime import datetime, timedelta, timezone

import pytest
from sqlmodel import Session


@pytest.fixture
def lock_session(db):
    from database import engine

    with Session(engine) as session:
        yield session


@pytest.fixture(autouse=True)
def clean_locks(db):
    """Locks are global state; don't let one test leak into the next."""
    from database import engine
    from models.runtime_state import StackLock
    from sqlmodel import delete

    with Session(engine) as session:
        session.exec(delete(StackLock))
        session.commit()
    yield


@pytest.fixture
def svc():
    from services import stack_lock_service

    return stack_lock_service


# --------------------------------------------------------------------------- #
# The lock itself
# --------------------------------------------------------------------------- #


def test_a_second_caller_is_refused(svc, lock_session):
    svc.acquire(lock_session, "demo", "update", "alice")
    with pytest.raises(svc.StackBusy) as excinfo:
        svc.acquire(lock_session, "demo", "start", "bob")
    assert excinfo.value.action == "update", "the error names the operation in flight"


def test_releasing_frees_it(svc, lock_session):
    svc.acquire(lock_session, "demo", "update")
    svc.release(lock_session, "demo")
    svc.acquire(lock_session, "demo", "start")  # must not raise


def test_releasing_a_lock_nobody_holds_is_harmless(svc, lock_session):
    svc.release(lock_session, "never-locked")


def test_different_stacks_do_not_block_each_other(svc, lock_session):
    svc.acquire(lock_session, "one", "update")
    svc.acquire(lock_session, "two", "update")


def test_hold_releases_even_when_the_operation_fails(svc, lock_session):
    """A failed deploy must not leave the stack locked."""
    with pytest.raises(RuntimeError):
        with svc.hold(lock_session, "demo", "update"):
            raise RuntimeError("compose blew up")
    assert not svc.is_busy(lock_session, "demo")


def test_an_expired_lock_is_taken_over(svc, lock_session):
    """The recovery path for a worker killed mid-deploy.

    Without it the stack would stay locked forever and the only fix would be
    editing the database by hand.
    """
    from models.runtime_state import StackLock

    past = datetime.now(timezone.utc) - timedelta(minutes=5)
    lock_session.add(
        StackLock(
            stack_id="orphaned",
            action="update",
            owner="a worker that died",
            acquired_at=past - timedelta(minutes=30),
            expires_at=past,
        )
    )
    lock_session.commit()

    assert not svc.is_busy(lock_session, "orphaned")
    svc.acquire(lock_session, "orphaned", "start")  # must not raise


def test_active_lists_current_locks_and_skips_expired(svc, lock_session):
    from models.runtime_state import StackLock

    svc.acquire(lock_session, "live", "update")
    lock_session.add(
        StackLock(
            stack_id="dead",
            action="start",
            expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
        )
    )
    lock_session.commit()

    assert svc.active(lock_session) == {"live": "update"}


def test_prune_removes_only_expired_locks(svc, lock_session):
    from models.runtime_state import StackLock

    svc.acquire(lock_session, "live", "update")
    lock_session.add(
        StackLock(
            stack_id="dead",
            action="start",
            expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
        )
    )
    lock_session.commit()

    assert svc.prune_expired(lock_session) == 1
    assert svc.active(lock_session) == {"live": "update"}


def test_the_lock_outlives_the_session_that_took_it(svc, db):
    """It is a row, not a set in one worker's memory — that is the point."""
    from database import engine

    with Session(engine) as first:
        svc.acquire(first, "persisted", "update")

    with Session(engine) as second:
        assert svc.is_busy(second, "persisted")
        with pytest.raises(svc.StackBusy):
            svc.acquire(second, "persisted", "start")


# --------------------------------------------------------------------------- #
# Enforcement at the HTTP entry point
# --------------------------------------------------------------------------- #


@pytest.fixture
def locked_stack(db):
    """A registered stack that is already mid-operation."""
    from database import engine
    from models.stack import Stack
    from services import compose_service, stack_lock_service

    stack_id = "lock-fixture"
    compose_service.write_compose(stack_id, "services:\n  a:\n    image: alpine\n")
    with Session(engine) as session:
        if not session.get(Stack, stack_id):
            session.add(Stack(id=stack_id, name="lock fixture"))
            session.commit()
        stack_lock_service.acquire(session, stack_id, "update", "someone-else")
    return stack_id


@pytest.mark.parametrize("action", ["start", "stop", "restart", "pull", "update", "down"])
def test_lifecycle_calls_are_refused_while_a_stack_is_busy(as_admin, locked_stack, action):
    """409, and crucially *without* reaching Docker — the guard is in front."""
    response = as_admin.post(f"/api/stacks/{locked_stack}/{action}")
    assert response.status_code == 409
    assert "busy" in response.json()["detail"].lower()
    assert "update in progress" in response.json()["detail"]


def test_the_stacks_list_shows_a_busy_stack_as_updating(as_admin, locked_stack):
    rows = {row["id"]: row for row in as_admin.get("/api/stacks").json()}
    assert rows[locked_stack]["status"] == "updating"


def test_an_unlocked_stack_is_not_refused(as_admin, db):
    """The guard must not block ordinary use.

    Without a Docker daemon the call fails further in — anything but 409 means
    it got past the lock, which is what this asserts.
    """
    from database import engine
    from models.stack import Stack
    from services import compose_service

    stack_id = "unlocked-fixture"
    compose_service.write_compose(stack_id, "services:\n  a:\n    image: alpine\n")
    with Session(engine) as session:
        if not session.get(Stack, stack_id):
            session.add(Stack(id=stack_id, name="unlocked fixture"))
            session.commit()

    assert as_admin.post(f"/api/stacks/{stack_id}/start").status_code != 409


# --------------------------------------------------------------------------- #
# Getting out from under a lock
# --------------------------------------------------------------------------- #
#
# The lock made a stuck backup fatal: it is held for twelve hours (a large
# volume legitimately takes that long), and while it is held the stack can be
# neither started, stopped nor restarted — "Stack 'x' is busy: backup in
# progress", with nothing to click. These are the three ways out: the job is
# told to stop, the lock is dropped by hand, or a restart clears what is left
# of a run that is over.


def test_clear_all_drops_locks_that_outlived_their_run(svc, lock_session):
    """What startup does.

    A lock means "an operation is running". After a restart none is, so an
    unexpired one is a leftover — and before this it kept its stack unusable
    for the rest of a backup's twelve-hour TTL.
    """
    svc.acquire(lock_session, "alpha", "backup")
    svc.acquire(lock_session, "beta", "update")

    assert sorted(svc.clear_all(lock_session)) == ["alpha", "beta"]
    assert svc.active(lock_session) == {}


def test_info_says_what_holds_the_stack(svc, lock_session):
    svc.acquire(lock_session, "demo", "backup", "scheduled")

    info = svc.info(lock_session, "demo")
    assert info["action"] == "backup"
    assert info["owner"] == "scheduled"
    assert info["held_for"] >= 0
    assert svc.info(lock_session, "not-locked-at-all") is None


def test_info_ignores_an_expired_lock(svc, lock_session):
    from models.runtime_state import StackLock

    lock_session.add(
        StackLock(
            stack_id="dead",
            action="backup",
            expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
        )
    )
    lock_session.commit()
    assert svc.info(lock_session, "dead") is None


def test_the_lock_endpoint_says_what_holds_the_stack(as_admin, locked_stack):
    body = as_admin.get(f"/api/stacks/{locked_stack}/lock").json()
    assert body["locked"] is True
    assert body["lock"]["action"] == "update"
    assert body["lock"]["cancellable"] is False
    # A deploy runs as a subprocess that outlives its request: no job here
    # does not mean nothing is running.
    assert body["lock"]["stale"] is False


def test_a_backup_lock_with_no_backup_behind_it_is_a_leftover(as_admin, locked_stack):
    """This is the state a crashed backup leaves: the twelve-hour lock stays,
    the backup is long gone, and the stack refuses everything. One click has
    to be enough — there is nothing left to stop."""
    from database import engine
    from services import stack_lock_service

    with Session(engine) as session:
        stack_lock_service.release(session, locked_stack)
        stack_lock_service.acquire(session, locked_stack, "backup", "backup")

    assert as_admin.get(f"/api/stacks/{locked_stack}/lock").json()["lock"]["stale"] is True

    body = as_admin.post(f"/api/stacks/{locked_stack}/unlock").json()
    assert body == {"ok": True, "action": "backup", "cancelled": False,
                    "released": True, "forced": False}
    assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code != 409


def test_unlocking_a_stack_that_nothing_holds_is_refused(as_admin, db):
    from models.stack import Stack
    from database import engine
    from services import compose_service

    stack_id = "not-locked-fixture"
    compose_service.write_compose(stack_id, "services:\n  a:\n    image: alpine\n")
    with Session(engine) as session:
        if not session.get(Stack, stack_id):
            session.add(Stack(id=stack_id, name="not locked"))
            session.commit()

    response = as_admin.post(f"/api/stacks/{stack_id}/unlock")
    assert response.status_code == 409
    assert "not locked" in response.json()["detail"]


def test_a_lock_nobody_can_stop_needs_force(as_admin, locked_stack):
    """The deploy behind this lock is not something we can interrupt, so the
    plain call refuses and says what the second step is — rather than claiming
    to have stopped something."""
    response = as_admin.post(f"/api/stacks/{locked_stack}/unlock")
    assert response.status_code == 409
    assert "force" in response.json()["detail"]

    forced = as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true")
    assert forced.status_code == 200
    assert forced.json() == {
        "ok": True, "action": "update", "cancelled": False,
        "released": True, "forced": True,
    }
    assert as_admin.get(f"/api/stacks/{locked_stack}/lock").json()["locked"] is False


def test_the_stack_can_be_used_again_after_the_lock_is_released(as_admin, locked_stack):
    """The whole point: the actions that were refused go through afterwards."""
    assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code == 409
    as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true")
    assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code != 409


def test_releasing_a_lock_is_written_to_the_audit_log(as_admin, locked_stack):
    as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true")
    rows = as_admin.get(f"/api/audit?stack_id={locked_stack}&limit=20").json()
    assert any(r["action"] == "stack.unlock" and r["target"] == locked_stack for r in rows)


# --------------------------------------------------------------------------- #
# The job behind the lock
# --------------------------------------------------------------------------- #


@pytest.fixture
def jobs():
    from services import stack_job_service

    return stack_job_service


def test_a_backup_can_be_cancelled_and_a_restore_cannot(jobs):
    """A backup only reads, so dropping it costs nothing. A restore is halfway
    through writing a stack's data back; interrupting it would leave the stack
    in a state no one asked for, so it registers itself to be seen, not
    stopped."""
    with jobs.track("demo", "backup"):
        assert jobs.cancel("demo", "alice").cancelled is True

    with jobs.track("demo", "restore"):
        assert jobs.cancel("demo", "alice").cancelled is False


def test_cancelling_a_stack_nobody_is_working_on_says_so(jobs):
    assert jobs.cancel("nothing-running") is None


def test_the_job_is_gone_once_its_operation_ends(jobs):
    """A leftover entry would make a free stack look like it is still busy."""
    with pytest.raises(RuntimeError):
        with jobs.track("demo", "backup"):
            raise RuntimeError("backup blew up")
    assert jobs.get("demo") is None


def test_check_only_raises_inside_a_cancelled_job(jobs):
    jobs.check()  # no job at all: the backup code calls this unconditionally
    with jobs.track("demo", "backup"):
        jobs.check()
        jobs.cancel("demo", "alice")
        with pytest.raises(jobs.JobCancelled):
            jobs.check()


def test_a_cancelled_job_that_will_not_stop_can_still_be_forced(
    as_admin, locked_stack, monkeypatch, jobs
):
    """The order the UI offers: ask the backup to stop, and only when it does
    not, drop the lock without it."""
    import routers.stacks as stacks_router

    monkeypatch.setattr(stacks_router, "CANCEL_WAIT_SECONDS", 0.5)
    with jobs.track(locked_stack, "backup", "scheduled") as job:
        response = as_admin.post(f"/api/stacks/{locked_stack}/unlock")
        assert response.status_code == 409
        assert "did not stop in time" in response.json()["detail"]
        assert job.cancelled is True, "it was still asked to stop"

        forced = as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true").json()
        assert forced["cancelled"] is True and forced["forced"] is True