Files
stackpilot/backend/main.py
T
menzeljandClaude Opus 5 270ddda51d
CI / check (push) Successful in 13m33s
CI / build-and-push (push) Successful in 1m51s
Let a stuck backup be stopped instead of stranding the stack (0.67.0)
A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.

Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.

POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".

Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-21 13:39:17 +02:00

188 lines
6.3 KiB
Python

"""StackPilot backend — FastAPI application entry point."""
from __future__ import annotations
import asyncio
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from sqlmodel import Session
from config import settings
from version import APP_VERSION
from database import engine, init_db
import docker_client
from docker_client import DockerError
from routers import (
audit,
auth,
backups,
containers,
dashboard,
destinations,
editor,
files,
git,
images,
networks,
oidc,
ports,
registries,
schedules,
secrets,
settings as settings_router,
stacks,
system,
templates,
tokens,
volumes,
ws,
)
from services import (
backup_destination_service,
git_service,
image_status_store,
logo_service,
registry_service,
schedule_service,
stack_lock_service,
template_service,
update_service,
)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("stackpilot")
@asynccontextmanager
async def lifespan(app: FastAPI):
init_db()
# Register stacks that already exist on disk.
try:
with Session(engine) as session:
stacks.sync_discovered_stacks(session)
except Exception as exc: # noqa: BLE001
logger.warning("Stack discovery failed: %s", exc)
# One-off: encrypt backup-destination credentials written before they were
# stored encrypted (see services/crypto_service.py).
try:
with Session(engine) as session:
encrypted = backup_destination_service.migrate_plaintext_configs(session)
if encrypted:
logger.info("Encrypted %d backup destination config(s) at rest", encrypted)
except Exception as exc: # noqa: BLE001
logger.warning("Destination config encryption migration failed: %s", exc)
try:
moved = template_service.migrate_legacy_db_templates()
if moved:
logger.info("Migrated %d custom template(s) from the database to folders", moved)
except Exception as exc: # noqa: BLE001
logger.warning("Legacy template migration failed: %s", exc)
# Throwaway containers a previous run left behind (a helper whose start
# failed, e.g. on an NFS mount, used to stay in "Created" and keep its
# volume in use). Nothing of ours is in flight yet, so all can go.
try:
client = docker_client.get_client()
cleared = docker_client.remove_stale_helpers(client)
cleared += backup_destination_service.clear_nfs_leftovers(client)
if cleared:
logger.info("Removed %d leftover helper container(s)", cleared)
except Exception as exc: # noqa: BLE001 - Docker may not be up yet
logger.warning("Could not clean up helper containers: %s", exc)
# Runtime state that used to live in module dicts and was lost on restart.
# Stack locks mean "an operation is running"; after a restart none is, so
# all of them go — not just the expired ones. A backup holds its lock for
# twelve hours, and one that died with its process used to leave the stack
# un-startable for the rest of them.
try:
with Session(engine) as session:
stale = stack_lock_service.clear_all(session)
if stale:
logger.info(
"Cleared %d stack lock(s) left over from a previous run: %s",
len(stale), ", ".join(sorted(stale)),
)
except Exception as exc: # noqa: BLE001
logger.warning("Could not clear stack locks: %s", exc)
try:
restored = image_status_store.install()
logger.info("Restored %d cached image update status(es)", restored)
except Exception as exc: # noqa: BLE001
logger.warning("Could not restore the image update cache: %s", exc)
# Private registry credentials: into the in-memory cache the update checker
# reads, and into the config.json the Docker CLI reads.
try:
with Session(engine) as session:
known = registry_service.reload(session)
if known:
logger.info("Loaded credentials for %d registr%s", known, "y" if known == 1 else "ies")
except Exception as exc: # noqa: BLE001
logger.warning("Could not load registry credentials: %s", exc)
update_task = asyncio.create_task(update_service.background_loop())
schedule_task = asyncio.create_task(schedule_service.scheduler_loop())
# App logos. Deliberately a task and not awaited: the catalog is a network
# download, and a box with no outbound internet must still start instantly
# (it just keeps the built-in glyphs).
logo_task = asyncio.create_task(logo_service.catalog_loop())
git_service.ensure_cache_root()
git_task = asyncio.create_task(git_service.poll_loop())
logger.info("StackPilot backend ready on port %s", settings.PORT)
yield
update_task.cancel()
schedule_task.cancel()
logo_task.cancel()
git_task.cancel()
app = FastAPI(title="StackPilot", version=APP_VERSION, lifespan=lifespan)
app.add_middleware(
CORSMiddleware,
allow_origins=settings.CORS_ORIGINS,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
@app.exception_handler(DockerError)
async def docker_error_handler(_request: Request, exc: DockerError):
return JSONResponse(
status_code=502,
content={"error": exc.error, "detail": exc.detail},
)
app.include_router(auth.router)
app.include_router(stacks.router)
app.include_router(git.router)
app.include_router(git.hook_router)
app.include_router(oidc.router)
app.include_router(tokens.router)
app.include_router(registries.router)
app.include_router(secrets.router)
app.include_router(containers.router)
app.include_router(dashboard.router)
app.include_router(system.router)
app.include_router(volumes.router)
app.include_router(editor.router)
app.include_router(files.router)
app.include_router(images.router)
app.include_router(ports.router)
app.include_router(templates.router)
app.include_router(audit.router)
app.include_router(settings_router.router)
app.include_router(backups.router)
app.include_router(destinations.router)
app.include_router(schedules.router)
app.include_router(networks.router)
app.include_router(ws.router)
@app.get("/api/health")
def health() -> dict:
return {"status": "ok", "version": APP_VERSION}