A backup holds the stack lock for up to twelve hours — a large volume
legitimately takes that long — and stops the stack's containers for a
consistent snapshot. One that died without unwinding therefore left the
stack stopped and refusing every start, stop, restart and deploy with
"Stack 'npmplus' is busy: backup in progress", with nothing to click
until the TTL ran out.
Backups and restores now register themselves in stack_job_service while
they run, with their phase and the helper containers they stream
through. Cancelling sets a flag and removes those containers, which is
what actually ends the long read: the backup then unwinds through its
own finally — the half-written archive goes, the containers it stopped
are started again, the lock is released. Checks sit between two files
and two volumes, never inside one. A restore is not cancellable at all;
it is halfway through writing a stack's data back.
POST /api/stacks/{id}/unlock does that and waits for it. A backup lock
with no job behind it is a leftover, so it is released on one click; a
deploy lock takes an explicit force, because compose keeps running past
its request. GET /api/stacks/{id}/lock and the stack detail response say
what holds the stack and since when, and the stack page turns that into
"Stop backup" / "Release lock".
Startup now clears every lock rather than only expired ones: after a
restart nothing of ours is in flight, the same reasoning the leftover
helper containers already follow.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
188 lines
6.3 KiB
Python
188 lines
6.3 KiB
Python
"""StackPilot backend — FastAPI application entry point."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from contextlib import asynccontextmanager
|
|
|
|
from fastapi import FastAPI, Request
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.responses import JSONResponse
|
|
from sqlmodel import Session
|
|
|
|
from config import settings
|
|
from version import APP_VERSION
|
|
from database import engine, init_db
|
|
import docker_client
|
|
from docker_client import DockerError
|
|
from routers import (
|
|
audit,
|
|
auth,
|
|
backups,
|
|
containers,
|
|
dashboard,
|
|
destinations,
|
|
editor,
|
|
files,
|
|
git,
|
|
images,
|
|
networks,
|
|
oidc,
|
|
ports,
|
|
registries,
|
|
schedules,
|
|
secrets,
|
|
settings as settings_router,
|
|
stacks,
|
|
system,
|
|
templates,
|
|
tokens,
|
|
volumes,
|
|
ws,
|
|
)
|
|
from services import (
|
|
backup_destination_service,
|
|
git_service,
|
|
image_status_store,
|
|
logo_service,
|
|
registry_service,
|
|
schedule_service,
|
|
stack_lock_service,
|
|
template_service,
|
|
update_service,
|
|
)
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger("stackpilot")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
init_db()
|
|
# Register stacks that already exist on disk.
|
|
try:
|
|
with Session(engine) as session:
|
|
stacks.sync_discovered_stacks(session)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Stack discovery failed: %s", exc)
|
|
# One-off: encrypt backup-destination credentials written before they were
|
|
# stored encrypted (see services/crypto_service.py).
|
|
try:
|
|
with Session(engine) as session:
|
|
encrypted = backup_destination_service.migrate_plaintext_configs(session)
|
|
if encrypted:
|
|
logger.info("Encrypted %d backup destination config(s) at rest", encrypted)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Destination config encryption migration failed: %s", exc)
|
|
try:
|
|
moved = template_service.migrate_legacy_db_templates()
|
|
if moved:
|
|
logger.info("Migrated %d custom template(s) from the database to folders", moved)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Legacy template migration failed: %s", exc)
|
|
# Throwaway containers a previous run left behind (a helper whose start
|
|
# failed, e.g. on an NFS mount, used to stay in "Created" and keep its
|
|
# volume in use). Nothing of ours is in flight yet, so all can go.
|
|
try:
|
|
client = docker_client.get_client()
|
|
cleared = docker_client.remove_stale_helpers(client)
|
|
cleared += backup_destination_service.clear_nfs_leftovers(client)
|
|
if cleared:
|
|
logger.info("Removed %d leftover helper container(s)", cleared)
|
|
except Exception as exc: # noqa: BLE001 - Docker may not be up yet
|
|
logger.warning("Could not clean up helper containers: %s", exc)
|
|
# Runtime state that used to live in module dicts and was lost on restart.
|
|
# Stack locks mean "an operation is running"; after a restart none is, so
|
|
# all of them go — not just the expired ones. A backup holds its lock for
|
|
# twelve hours, and one that died with its process used to leave the stack
|
|
# un-startable for the rest of them.
|
|
try:
|
|
with Session(engine) as session:
|
|
stale = stack_lock_service.clear_all(session)
|
|
if stale:
|
|
logger.info(
|
|
"Cleared %d stack lock(s) left over from a previous run: %s",
|
|
len(stale), ", ".join(sorted(stale)),
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Could not clear stack locks: %s", exc)
|
|
try:
|
|
restored = image_status_store.install()
|
|
logger.info("Restored %d cached image update status(es)", restored)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Could not restore the image update cache: %s", exc)
|
|
# Private registry credentials: into the in-memory cache the update checker
|
|
# reads, and into the config.json the Docker CLI reads.
|
|
try:
|
|
with Session(engine) as session:
|
|
known = registry_service.reload(session)
|
|
if known:
|
|
logger.info("Loaded credentials for %d registr%s", known, "y" if known == 1 else "ies")
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("Could not load registry credentials: %s", exc)
|
|
|
|
update_task = asyncio.create_task(update_service.background_loop())
|
|
schedule_task = asyncio.create_task(schedule_service.scheduler_loop())
|
|
# App logos. Deliberately a task and not awaited: the catalog is a network
|
|
# download, and a box with no outbound internet must still start instantly
|
|
# (it just keeps the built-in glyphs).
|
|
logo_task = asyncio.create_task(logo_service.catalog_loop())
|
|
git_service.ensure_cache_root()
|
|
git_task = asyncio.create_task(git_service.poll_loop())
|
|
logger.info("StackPilot backend ready on port %s", settings.PORT)
|
|
yield
|
|
update_task.cancel()
|
|
schedule_task.cancel()
|
|
logo_task.cancel()
|
|
git_task.cancel()
|
|
|
|
|
|
app = FastAPI(title="StackPilot", version=APP_VERSION, lifespan=lifespan)
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=settings.CORS_ORIGINS,
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
|
|
@app.exception_handler(DockerError)
|
|
async def docker_error_handler(_request: Request, exc: DockerError):
|
|
return JSONResponse(
|
|
status_code=502,
|
|
content={"error": exc.error, "detail": exc.detail},
|
|
)
|
|
|
|
|
|
app.include_router(auth.router)
|
|
app.include_router(stacks.router)
|
|
app.include_router(git.router)
|
|
app.include_router(git.hook_router)
|
|
app.include_router(oidc.router)
|
|
app.include_router(tokens.router)
|
|
app.include_router(registries.router)
|
|
app.include_router(secrets.router)
|
|
app.include_router(containers.router)
|
|
app.include_router(dashboard.router)
|
|
app.include_router(system.router)
|
|
app.include_router(volumes.router)
|
|
app.include_router(editor.router)
|
|
app.include_router(files.router)
|
|
app.include_router(images.router)
|
|
app.include_router(ports.router)
|
|
app.include_router(templates.router)
|
|
app.include_router(audit.router)
|
|
app.include_router(settings_router.router)
|
|
app.include_router(backups.router)
|
|
app.include_router(destinations.router)
|
|
app.include_router(schedules.router)
|
|
app.include_router(networks.router)
|
|
app.include_router(ws.router)
|
|
|
|
|
|
@app.get("/api/health")
|
|
def health() -> dict:
|
|
return {"status": "ok", "version": APP_VERSION}
|