"""One compose operation per stack (F7), and the lock surviving a restart (F11). ``docker compose`` does no locking of its own. Before 0.47.0 nothing stopped two ``update`` calls landing on the same project at once — two browser tabs, or the auto-update pass picking up a stack somebody had just clicked — and both would run ``pull`` then ``up -d`` and fight over recreating the same containers. There was a busy flag, but it only fed the status column: no handler consulted it before acting. The lock is a table now, so it also holds across workers and across a restart. """ from __future__ import annotations from datetime import datetime, timedelta, timezone import pytest from sqlmodel import Session @pytest.fixture def lock_session(db): from database import engine with Session(engine) as session: yield session @pytest.fixture(autouse=True) def clean_locks(db): """Locks are global state; don't let one test leak into the next.""" from database import engine from models.runtime_state import StackLock from sqlmodel import delete with Session(engine) as session: session.exec(delete(StackLock)) session.commit() yield @pytest.fixture def svc(): from services import stack_lock_service return stack_lock_service # --------------------------------------------------------------------------- # # The lock itself # --------------------------------------------------------------------------- # def test_a_second_caller_is_refused(svc, lock_session): svc.acquire(lock_session, "demo", "update", "alice") with pytest.raises(svc.StackBusy) as excinfo: svc.acquire(lock_session, "demo", "start", "bob") assert excinfo.value.action == "update", "the error names the operation in flight" def test_releasing_frees_it(svc, lock_session): svc.acquire(lock_session, "demo", "update") svc.release(lock_session, "demo") svc.acquire(lock_session, "demo", "start") # must not raise def test_releasing_a_lock_nobody_holds_is_harmless(svc, lock_session): svc.release(lock_session, "never-locked") def test_different_stacks_do_not_block_each_other(svc, lock_session): svc.acquire(lock_session, "one", "update") svc.acquire(lock_session, "two", "update") def test_hold_releases_even_when_the_operation_fails(svc, lock_session): """A failed deploy must not leave the stack locked.""" with pytest.raises(RuntimeError): with svc.hold(lock_session, "demo", "update"): raise RuntimeError("compose blew up") assert not svc.is_busy(lock_session, "demo") def test_an_expired_lock_is_taken_over(svc, lock_session): """The recovery path for a worker killed mid-deploy. Without it the stack would stay locked forever and the only fix would be editing the database by hand. """ from models.runtime_state import StackLock past = datetime.now(timezone.utc) - timedelta(minutes=5) lock_session.add( StackLock( stack_id="orphaned", action="update", owner="a worker that died", acquired_at=past - timedelta(minutes=30), expires_at=past, ) ) lock_session.commit() assert not svc.is_busy(lock_session, "orphaned") svc.acquire(lock_session, "orphaned", "start") # must not raise def test_active_lists_current_locks_and_skips_expired(svc, lock_session): from models.runtime_state import StackLock svc.acquire(lock_session, "live", "update") lock_session.add( StackLock( stack_id="dead", action="start", expires_at=datetime.now(timezone.utc) - timedelta(seconds=1), ) ) lock_session.commit() assert svc.active(lock_session) == {"live": "update"} def test_prune_removes_only_expired_locks(svc, lock_session): from models.runtime_state import StackLock svc.acquire(lock_session, "live", "update") lock_session.add( StackLock( stack_id="dead", action="start", expires_at=datetime.now(timezone.utc) - timedelta(seconds=1), ) ) lock_session.commit() assert svc.prune_expired(lock_session) == 1 assert svc.active(lock_session) == {"live": "update"} def test_the_lock_outlives_the_session_that_took_it(svc, db): """It is a row, not a set in one worker's memory — that is the point.""" from database import engine with Session(engine) as first: svc.acquire(first, "persisted", "update") with Session(engine) as second: assert svc.is_busy(second, "persisted") with pytest.raises(svc.StackBusy): svc.acquire(second, "persisted", "start") # --------------------------------------------------------------------------- # # Enforcement at the HTTP entry point # --------------------------------------------------------------------------- # @pytest.fixture def locked_stack(db): """A registered stack that is already mid-operation.""" from database import engine from models.stack import Stack from services import compose_service, stack_lock_service stack_id = "lock-fixture" compose_service.write_compose(stack_id, "services:\n a:\n image: alpine\n") with Session(engine) as session: if not session.get(Stack, stack_id): session.add(Stack(id=stack_id, name="lock fixture")) session.commit() stack_lock_service.acquire(session, stack_id, "update", "someone-else") return stack_id @pytest.mark.parametrize("action", ["start", "stop", "restart", "pull", "update", "down"]) def test_lifecycle_calls_are_refused_while_a_stack_is_busy(as_admin, locked_stack, action): """409, and crucially *without* reaching Docker — the guard is in front.""" response = as_admin.post(f"/api/stacks/{locked_stack}/{action}") assert response.status_code == 409 assert "busy" in response.json()["detail"].lower() assert "update in progress" in response.json()["detail"] def test_the_stacks_list_shows_a_busy_stack_as_updating(as_admin, locked_stack): rows = {row["id"]: row for row in as_admin.get("/api/stacks").json()} assert rows[locked_stack]["status"] == "updating" def test_an_unlocked_stack_is_not_refused(as_admin, db): """The guard must not block ordinary use. Without a Docker daemon the call fails further in — anything but 409 means it got past the lock, which is what this asserts. """ from database import engine from models.stack import Stack from services import compose_service stack_id = "unlocked-fixture" compose_service.write_compose(stack_id, "services:\n a:\n image: alpine\n") with Session(engine) as session: if not session.get(Stack, stack_id): session.add(Stack(id=stack_id, name="unlocked fixture")) session.commit() assert as_admin.post(f"/api/stacks/{stack_id}/start").status_code != 409 # --------------------------------------------------------------------------- # # Getting out from under a lock # --------------------------------------------------------------------------- # # # The lock made a stuck backup fatal: it is held for twelve hours (a large # volume legitimately takes that long), and while it is held the stack can be # neither started, stopped nor restarted — "Stack 'x' is busy: backup in # progress", with nothing to click. These are the three ways out: the job is # told to stop, the lock is dropped by hand, or a restart clears what is left # of a run that is over. def test_clear_all_drops_locks_that_outlived_their_run(svc, lock_session): """What startup does. A lock means "an operation is running". After a restart none is, so an unexpired one is a leftover — and before this it kept its stack unusable for the rest of a backup's twelve-hour TTL. """ svc.acquire(lock_session, "alpha", "backup") svc.acquire(lock_session, "beta", "update") assert sorted(svc.clear_all(lock_session)) == ["alpha", "beta"] assert svc.active(lock_session) == {} def test_info_says_what_holds_the_stack(svc, lock_session): svc.acquire(lock_session, "demo", "backup", "scheduled") info = svc.info(lock_session, "demo") assert info["action"] == "backup" assert info["owner"] == "scheduled" assert info["held_for"] >= 0 assert svc.info(lock_session, "not-locked-at-all") is None def test_info_ignores_an_expired_lock(svc, lock_session): from models.runtime_state import StackLock lock_session.add( StackLock( stack_id="dead", action="backup", expires_at=datetime.now(timezone.utc) - timedelta(seconds=1), ) ) lock_session.commit() assert svc.info(lock_session, "dead") is None def test_the_lock_endpoint_says_what_holds_the_stack(as_admin, locked_stack): body = as_admin.get(f"/api/stacks/{locked_stack}/lock").json() assert body["locked"] is True assert body["lock"]["action"] == "update" assert body["lock"]["cancellable"] is False # A deploy runs as a subprocess that outlives its request: no job here # does not mean nothing is running. assert body["lock"]["stale"] is False def test_a_backup_lock_with_no_backup_behind_it_is_a_leftover(as_admin, locked_stack): """This is the state a crashed backup leaves: the twelve-hour lock stays, the backup is long gone, and the stack refuses everything. One click has to be enough — there is nothing left to stop.""" from database import engine from services import stack_lock_service with Session(engine) as session: stack_lock_service.release(session, locked_stack) stack_lock_service.acquire(session, locked_stack, "backup", "backup") assert as_admin.get(f"/api/stacks/{locked_stack}/lock").json()["lock"]["stale"] is True body = as_admin.post(f"/api/stacks/{locked_stack}/unlock").json() assert body == {"ok": True, "action": "backup", "cancelled": False, "released": True, "forced": False} assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code != 409 def test_unlocking_a_stack_that_nothing_holds_is_refused(as_admin, db): from models.stack import Stack from database import engine from services import compose_service stack_id = "not-locked-fixture" compose_service.write_compose(stack_id, "services:\n a:\n image: alpine\n") with Session(engine) as session: if not session.get(Stack, stack_id): session.add(Stack(id=stack_id, name="not locked")) session.commit() response = as_admin.post(f"/api/stacks/{stack_id}/unlock") assert response.status_code == 409 assert "not locked" in response.json()["detail"] def test_a_lock_nobody_can_stop_needs_force(as_admin, locked_stack): """The deploy behind this lock is not something we can interrupt, so the plain call refuses and says what the second step is — rather than claiming to have stopped something.""" response = as_admin.post(f"/api/stacks/{locked_stack}/unlock") assert response.status_code == 409 assert "force" in response.json()["detail"] forced = as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true") assert forced.status_code == 200 assert forced.json() == { "ok": True, "action": "update", "cancelled": False, "released": True, "forced": True, } assert as_admin.get(f"/api/stacks/{locked_stack}/lock").json()["locked"] is False def test_the_stack_can_be_used_again_after_the_lock_is_released(as_admin, locked_stack): """The whole point: the actions that were refused go through afterwards.""" assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code == 409 as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true") assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code != 409 def test_releasing_a_lock_is_written_to_the_audit_log(as_admin, locked_stack): as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true") rows = as_admin.get(f"/api/audit?stack_id={locked_stack}&limit=20").json() assert any(r["action"] == "stack.unlock" and r["target"] == locked_stack for r in rows) # --------------------------------------------------------------------------- # # The job behind the lock # --------------------------------------------------------------------------- # @pytest.fixture def jobs(): from services import stack_job_service return stack_job_service def test_a_backup_can_be_cancelled_and_a_restore_cannot(jobs): """A backup only reads, so dropping it costs nothing. A restore is halfway through writing a stack's data back; interrupting it would leave the stack in a state no one asked for, so it registers itself to be seen, not stopped.""" with jobs.track("demo", "backup"): assert jobs.cancel("demo", "alice").cancelled is True with jobs.track("demo", "restore"): assert jobs.cancel("demo", "alice").cancelled is False def test_cancelling_a_stack_nobody_is_working_on_says_so(jobs): assert jobs.cancel("nothing-running") is None def test_the_job_is_gone_once_its_operation_ends(jobs): """A leftover entry would make a free stack look like it is still busy.""" with pytest.raises(RuntimeError): with jobs.track("demo", "backup"): raise RuntimeError("backup blew up") assert jobs.get("demo") is None def test_check_only_raises_inside_a_cancelled_job(jobs): jobs.check() # no job at all: the backup code calls this unconditionally with jobs.track("demo", "backup"): jobs.check() jobs.cancel("demo", "alice") with pytest.raises(jobs.JobCancelled): jobs.check() def test_a_cancelled_job_that_will_not_stop_can_still_be_forced( as_admin, locked_stack, monkeypatch, jobs ): """The order the UI offers: ask the backup to stop, and only when it does not, drop the lock without it.""" import routers.stacks as stacks_router monkeypatch.setattr(stacks_router, "CANCEL_WAIT_SECONDS", 0.5) with jobs.track(locked_stack, "backup", "scheduled") as job: response = as_admin.post(f"/api/stacks/{locked_stack}/unlock") assert response.status_code == 409 assert "did not stop in time" in response.json()["detail"] assert job.cancelled is True, "it was still asked to stop" forced = as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true").json() assert forced["cancelled"] is True and forced["forced"] is True