Let a stuck backup be stopped instead of stranding the stack (0.67.0)

A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.

Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.

POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".

Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/tests/test_stack_locking.py

@@ -204,3 +204,192 @@ def test_an_unlocked_stack_is_not_refused(as_admin, db):
             session.commit()
 
     assert as_admin.post(f"/api/stacks/{stack_id}/start").status_code != 409
+
+
+# --------------------------------------------------------------------------- #
+# Getting out from under a lock
+# --------------------------------------------------------------------------- #
+#
+# The lock made a stuck backup fatal: it is held for twelve hours (a large
+# volume legitimately takes that long), and while it is held the stack can be
+# neither started, stopped nor restarted — "Stack 'x' is busy: backup in
+# progress", with nothing to click. These are the three ways out: the job is
+# told to stop, the lock is dropped by hand, or a restart clears what is left
+# of a run that is over.
+
+
+def test_clear_all_drops_locks_that_outlived_their_run(svc, lock_session):
+    """What startup does.
+
+    A lock means "an operation is running". After a restart none is, so an
+    unexpired one is a leftover — and before this it kept its stack unusable
+    for the rest of a backup's twelve-hour TTL.
+    """
+    svc.acquire(lock_session, "alpha", "backup")
+    svc.acquire(lock_session, "beta", "update")
+
+    assert sorted(svc.clear_all(lock_session)) == ["alpha", "beta"]
+    assert svc.active(lock_session) == {}
+
+
+def test_info_says_what_holds_the_stack(svc, lock_session):
+    svc.acquire(lock_session, "demo", "backup", "scheduled")
+
+    info = svc.info(lock_session, "demo")
+    assert info["action"] == "backup"
+    assert info["owner"] == "scheduled"
+    assert info["held_for"] >= 0
+    assert svc.info(lock_session, "not-locked-at-all") is None
+
+
+def test_info_ignores_an_expired_lock(svc, lock_session):
+    from models.runtime_state import StackLock
+
+    lock_session.add(
+        StackLock(
+            stack_id="dead",
+            action="backup",
+            expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
+        )
+    )
+    lock_session.commit()
+    assert svc.info(lock_session, "dead") is None
+
+
+def test_the_lock_endpoint_says_what_holds_the_stack(as_admin, locked_stack):
+    body = as_admin.get(f"/api/stacks/{locked_stack}/lock").json()
+    assert body["locked"] is True
+    assert body["lock"]["action"] == "update"
+    assert body["lock"]["cancellable"] is False
+    # A deploy runs as a subprocess that outlives its request: no job here
+    # does not mean nothing is running.
+    assert body["lock"]["stale"] is False
+
+
+def test_a_backup_lock_with_no_backup_behind_it_is_a_leftover(as_admin, locked_stack):
+    """This is the state a crashed backup leaves: the twelve-hour lock stays,
+    the backup is long gone, and the stack refuses everything. One click has
+    to be enough — there is nothing left to stop."""
+    from database import engine
+    from services import stack_lock_service
+
+    with Session(engine) as session:
+        stack_lock_service.release(session, locked_stack)
+        stack_lock_service.acquire(session, locked_stack, "backup", "backup")
+
+    assert as_admin.get(f"/api/stacks/{locked_stack}/lock").json()["lock"]["stale"] is True
+
+    body = as_admin.post(f"/api/stacks/{locked_stack}/unlock").json()
+    assert body == {"ok": True, "action": "backup", "cancelled": False,
+                    "released": True, "forced": False}
+    assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code != 409
+
+
+def test_unlocking_a_stack_that_nothing_holds_is_refused(as_admin, db):
+    from models.stack import Stack
+    from database import engine
+    from services import compose_service
+
+    stack_id = "not-locked-fixture"
+    compose_service.write_compose(stack_id, "services:\n  a:\n    image: alpine\n")
+    with Session(engine) as session:
+        if not session.get(Stack, stack_id):
+            session.add(Stack(id=stack_id, name="not locked"))
+            session.commit()
+
+    response = as_admin.post(f"/api/stacks/{stack_id}/unlock")
+    assert response.status_code == 409
+    assert "not locked" in response.json()["detail"]
+
+
+def test_a_lock_nobody_can_stop_needs_force(as_admin, locked_stack):
+    """The deploy behind this lock is not something we can interrupt, so the
+    plain call refuses and says what the second step is — rather than claiming
+    to have stopped something."""
+    response = as_admin.post(f"/api/stacks/{locked_stack}/unlock")
+    assert response.status_code == 409
+    assert "force" in response.json()["detail"]
+
+    forced = as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true")
+    assert forced.status_code == 200
+    assert forced.json() == {
+        "ok": True, "action": "update", "cancelled": False,
+        "released": True, "forced": True,
+    }
+    assert as_admin.get(f"/api/stacks/{locked_stack}/lock").json()["locked"] is False
+
+
+def test_the_stack_can_be_used_again_after_the_lock_is_released(as_admin, locked_stack):
+    """The whole point: the actions that were refused go through afterwards."""
+    assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code == 409
+    as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true")
+    assert as_admin.post(f"/api/stacks/{locked_stack}/restart").status_code != 409
+
+
+def test_releasing_a_lock_is_written_to_the_audit_log(as_admin, locked_stack):
+    as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true")
+    rows = as_admin.get(f"/api/audit?stack_id={locked_stack}&limit=20").json()
+    assert any(r["action"] == "stack.unlock" and r["target"] == locked_stack for r in rows)
+
+
+# --------------------------------------------------------------------------- #
+# The job behind the lock
+# --------------------------------------------------------------------------- #
+
+
+@pytest.fixture
+def jobs():
+    from services import stack_job_service
+
+    return stack_job_service
+
+
+def test_a_backup_can_be_cancelled_and_a_restore_cannot(jobs):
+    """A backup only reads, so dropping it costs nothing. A restore is halfway
+    through writing a stack's data back; interrupting it would leave the stack
+    in a state no one asked for, so it registers itself to be seen, not
+    stopped."""
+    with jobs.track("demo", "backup"):
+        assert jobs.cancel("demo", "alice").cancelled is True
+
+    with jobs.track("demo", "restore"):
+        assert jobs.cancel("demo", "alice").cancelled is False
+
+
+def test_cancelling_a_stack_nobody_is_working_on_says_so(jobs):
+    assert jobs.cancel("nothing-running") is None
+
+
+def test_the_job_is_gone_once_its_operation_ends(jobs):
+    """A leftover entry would make a free stack look like it is still busy."""
+    with pytest.raises(RuntimeError):
+        with jobs.track("demo", "backup"):
+            raise RuntimeError("backup blew up")
+    assert jobs.get("demo") is None
+
+
+def test_check_only_raises_inside_a_cancelled_job(jobs):
+    jobs.check()  # no job at all: the backup code calls this unconditionally
+    with jobs.track("demo", "backup"):
+        jobs.check()
+        jobs.cancel("demo", "alice")
+        with pytest.raises(jobs.JobCancelled):
+            jobs.check()
+
+
+def test_a_cancelled_job_that_will_not_stop_can_still_be_forced(
+    as_admin, locked_stack, monkeypatch, jobs
+):
+    """The order the UI offers: ask the backup to stop, and only when it does
+    not, drop the lock without it."""
+    import routers.stacks as stacks_router
+
+    monkeypatch.setattr(stacks_router, "CANCEL_WAIT_SECONDS", 0.5)
+    with jobs.track(locked_stack, "backup", "scheduled") as job:
+        response = as_admin.post(f"/api/stacks/{locked_stack}/unlock")
+        assert response.status_code == 409
+        assert "did not stop in time" in response.json()["detail"]
+        assert job.cancelled is True, "it was still asked to stop"
+
+        forced = as_admin.post(f"/api/stacks/{locked_stack}/unlock?force=true").json()
+        assert forced["cancelled"] is True and forced["forced"] is True