diff --git a/README.md b/README.md index f90f7d1..b97979d 100644 --- a/README.md +++ b/README.md @@ -652,6 +652,13 @@ it is what your saved destination credentials are encrypted with. `local_lock=` yourself: with locking the kernel needs rpcbind on the Docker host (nfs-common), and without it the mount fails with "connection refused". Only StackPilot writes the backup files, so nothing needs locking (0.66.2). +- **SFTP host keys** (0.67.1): the server's host key is checked on every + connection. Paste it into the destination's *Host key* field (a key line or + its `SHA256:` fingerprint, e.g. from `ssh-keyscan HOST | ssh-keygen -lf -`), + or leave the field empty and the key seen on the first connection is pinned + — "Test" shows its fingerprint. A different key is refused before the + password is sent; clear the field after a server reinstall to trust the new + one. Changing the host or port forgets the pinned key. - **Push & restore**: the stack Backup dialog can push straight to a destination instead of downloading; the Restore dialog can browse a destination's backups and restore (volumes included) directly from it. Backups can also be @@ -665,7 +672,9 @@ it is what your saved destination credentials are encrypted with. - **Local time** (0.66.0): each schedule has a time zone, preset to the browser's (e.g. `Europe/Berlin`), and its time is wall-clock time there — a 03:00 backup stays at 03:00 across daylight saving changes. Schedules made - before were in UTC and stay so until edited. + before were in UTC and stay so until edited. Backup file names carry the time + on that clock too (`backup-app-full-20261015-122154.tar.gz` for 12:21:54 in + Berlin); a backup made by hand uses the browser's time zone (0.67.1). - **Several stacks per schedule** (0.65.0): tick any number of stacks, or *All stacks* to include stacks added later without editing the schedule. They are backed up one after another, each to its own file with its own retention diff --git a/backend/Dockerfile b/backend/Dockerfile index 911ccb9..17cd475 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -26,7 +26,11 @@ ENV STACKS_DIR=/opt/stacks \ EXPOSE 5008 -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \ +# --start-interval: probed every 2s while starting, so "healthy" — which the +# frontend waits for (docker-compose.yml) — comes as soon as the API answers +# instead of after the first 30s interval. Engines before 25.0 ignore it and +# take the 30s. +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --start-interval=2s \ CMD curl -fsS http://localhost:5008/api/health || exit 1 # --proxy-headers: nginx setzt X-Forwarded-For, ohne dieses Flag ignoriert diff --git a/backend/main.py b/backend/main.py index c34e46a..6f65b26 100644 --- a/backend/main.py +++ b/backend/main.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging +from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager from fastapi import FastAPI, Request @@ -47,6 +48,8 @@ from services import ( logo_service, registry_service, schedule_service, + secret_service, + stack_cleanup_service, stack_lock_service, template_service, update_service, @@ -56,13 +59,27 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger("stackpilot") +#: Workers for ``asyncio.to_thread``. The default (CPU count + 4, so six on a +#: two-core VM) is shared by everything that leaves the event loop — a backup +#: streaming several volumes, the dashboard, NFS and SFTP destinations — and +#: running out of it does not fail, it waits. Long-lived readers (the event +#: stream, terminals) have threads of their own; this is headroom for the rest. +DEFAULT_EXECUTOR_WORKERS = 64 + + @asynccontextmanager async def lifespan(app: FastAPI): + asyncio.get_running_loop().set_default_executor( + ThreadPoolExecutor(max_workers=DEFAULT_EXECUTOR_WORKERS, thread_name_prefix="sp-worker") + ) init_db() # Register stacks that already exist on disk. try: with Session(engine) as session: stacks.sync_discovered_stacks(session) + orphans = stack_cleanup_service.prune_orphans(session) + if orphans: + logger.info("Forgot Git sources/policies of deleted stack(s): %s", ", ".join(orphans)) except Exception as exc: # noqa: BLE001 logger.warning("Stack discovery failed: %s", exc) # One-off: encrypt backup-destination credentials written before they were @@ -74,6 +91,12 @@ async def lifespan(app: FastAPI): logger.info("Encrypted %d backup destination config(s) at rest", encrypted) except Exception as exc: # noqa: BLE001 logger.warning("Destination config encryption migration failed: %s", exc) + try: + fixed = secret_service.fix_permissions() + if fixed: + logger.info("Made %d stack secret file(s) readable for non-root containers", fixed) + except Exception as exc: # noqa: BLE001 + logger.warning("Could not fix secret permissions: %s", exc) try: moved = template_service.migrate_legacy_db_templates() if moved: diff --git a/backend/models/runtime_state.py b/backend/models/runtime_state.py index 3f5faab..dc9e39f 100644 --- a/backend/models/runtime_state.py +++ b/backend/models/runtime_state.py @@ -55,6 +55,8 @@ class ImageStatus(SQLModel, table=True): remote_digest: Optional[str] = None checked_at: float = 0.0 error: Optional[str] = None + #: Built on the host, so there is nothing in a registry to compare with. + local: bool = False #: Whether an "update available" notification already went out for this #: image at its current state. notified: bool = False diff --git a/backend/routers/auth.py b/backend/routers/auth.py index 50ac0a0..d864baf 100644 --- a/backend/routers/auth.py +++ b/backend/routers/auth.py @@ -114,8 +114,10 @@ def setup( ) -> TokenPair: if auth_mod.users_exist(session): raise HTTPException(status_code=400, detail="Setup already completed") + if not body.username.strip() or not body.password: + raise HTTPException(status_code=400, detail="Username and password required") user = User( - username=body.username, + username=body.username.strip(), hashed_password=auth_mod.hash_password(body.password), role="admin", ) diff --git a/backend/routers/backups.py b/backend/routers/backups.py index d6fad00..8f9dfa7 100644 --- a/backend/routers/backups.py +++ b/backend/routers/backups.py @@ -9,7 +9,7 @@ import tempfile from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import FileResponse from starlette.background import BackgroundTask -from pydantic import BaseModel +from pydantic import BaseModel, Field from sqlmodel import Session from auth import require_admin @@ -51,6 +51,23 @@ def _compact(report: dict) -> dict: } +#: The report rides in a response header, which a proxy refuses past its +#: header buffer (nginx: 4–8 KiB by default, then a 502 for a backup that +#: worked). Lists are cut until it fits; the counts stay in the audit log. +MAX_REPORT_HEADER = 6 * 1024 + + +def _report_header(report: dict) -> str: + compact = _compact(report) + text = json.dumps(compact) + for key in ("warnings", "skipped", "binds", "volumes", "errors"): + while len(text) > MAX_REPORT_HEADER and compact.get(key): + compact[key] = compact[key][: len(compact[key]) // 2] + compact["truncated"] = True + text = json.dumps(compact) + return text + + def _summary(report: dict) -> str: text = ( f"binds={len(report.get('binds', []))} " @@ -96,6 +113,8 @@ async def backup_stack( stop_first: bool = Query(True), binds: list[str] | None = Query(None), volumes: list[str] | None = Query(None), + # The browser's zone, for the time in the file name. + tz: str | None = Query(None, max_length=64), session: Session = Depends(get_session), user: User = Depends(require_admin), ): @@ -117,10 +136,10 @@ async def backup_stack( return FileResponse( path, media_type="application/gzip", - filename=_backup_filename(stack_id, include_volumes), + filename=_backup_filename(stack_id, include_volumes, tz=tz), # The browser downloads a blob, so the summary of what actually made it # into the archive rides along in a header. - headers={"X-Stackpilot-Backup": json.dumps(_compact(report))}, + headers={"X-Stackpilot-Backup": _report_header(report)}, # The archive is a temp file; without this every download stayed in # the container's /tmp until it filled the disk. background=BackgroundTask(os.unlink, path), @@ -184,6 +203,8 @@ class PushBody(BaseModel): stop_first: bool = True binds: list[str] | None = None volumes: list[str] | None = None + #: The browser's zone, for the time in the file name. + tz: str | None = Field(None, max_length=64) class RestoreFromBody(BaseModel): @@ -223,7 +244,7 @@ async def push_backup( except backup_service.BackupError as exc: raise _backup_http_error(exc) from exc - filename = _backup_filename(stack_id, body.include_volumes) + filename = _backup_filename(stack_id, body.include_volumes, tz=body.tz) try: remote = await asyncio.to_thread(dest_service.upload, dest, path, filename) except dest_service.DestinationError as exc: diff --git a/backend/routers/containers.py b/backend/routers/containers.py index 1f567fa..9d7b36c 100644 --- a/backend/routers/containers.py +++ b/backend/routers/containers.py @@ -1,24 +1,46 @@ """Single-container inspect + lifecycle for compose-managed containers.""" from __future__ import annotations -from fastapi import APIRouter, Depends, Request +from fastapi import APIRouter, Depends, HTTPException, Request from sqlmodel import Session from auth import get_current_user, require_admin from database import get_session +from docker_client import DockerError from models.user import User from services import audit_service, container_service router = APIRouter(prefix="/api/containers", tags=["containers"]) +#: Docker errors that are the caller's to fix, not a daemon failure (the global +#: handler turns every DockerError into a 502). +_STATUS = {"not_found": 404, "not_managed": 404, "bad_action": 400} + def _ip(request: Request) -> str: return request.client.host if request.client else "unknown" +def _raise(exc: DockerError): + status = _STATUS.get(exc.error) + if status: + raise HTTPException(status_code=status, detail=exc.detail or exc.error) from exc + raise exc + + @router.get("/{container_id}") -def inspect(container_id: str, _user: User = Depends(get_current_user)) -> dict: - return container_service.inspect_container(container_id) +def inspect(container_id: str, user: User = Depends(get_current_user)) -> dict: + try: + data = container_service.inspect_container(container_id) + except DockerError as exc: + _raise(exc) + if user.role != "admin": + # The environment is where a container's credentials live — the same + # values the stack's .env holds, which the read-only role does not get + # either. Names stay visible, values do not. + data["env"] = [entry.split("=", 1)[0] + "=••••••" if "=" in entry else entry + for entry in data.get("env") or []] + return data @router.post("/{container_id}/{action}") @@ -29,7 +51,10 @@ def action( session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: - result = container_service.container_action(container_id, action) + try: + result = container_service.container_action(container_id, action) + except DockerError as exc: + _raise(exc) audit_service.record( session, user=user.username, action=f"container.{action}", target=container_id[:12], ip=_ip(request), diff --git a/backend/routers/destinations.py b/backend/routers/destinations.py index a364344..00e5aae 100644 --- a/backend/routers/destinations.py +++ b/backend/routers/destinations.py @@ -16,6 +16,7 @@ from models.backup_destination import ( DestinationRead, DestinationUpdate, ) +from models.backup_schedule import BackupSchedule from models.user import User from services import audit_service, backup_destination_service as dest_service @@ -92,10 +93,16 @@ def update_destination( if body.config is not None: # Merge so masked/blank secrets don't wipe stored ones. existing = dest_service.parse_config(d) + before = dict(existing) for k, v in body.config.items(): if k in SECRET_KEYS and (v == "" or v == "••••••"): continue # keep existing secret existing[k] = v + # A host key pinned for the old address says nothing about a new one; + # kept, the first connection to the new server would be refused. + moved = any(str(existing.get(k) or "") != str(before.get(k) or "") for k in ("host", "port")) + if moved and existing.get("host_key") == before.get("host_key"): + existing.pop("host_key", None) d.config = dest_service.dump_config(existing) session.add(d) session.commit() @@ -115,9 +122,24 @@ def delete_destination( user: User = Depends(require_admin), ) -> dict: d = _get_or_404(session, dest_id) - name = d.name + users = session.exec( + select(BackupSchedule).where(BackupSchedule.destination_id == dest_id) + ).all() + if users: + # Deleted from under them, every run of these failed with + # "destination not found" and paged somebody about it. + raise HTTPException( + status_code=409, + detail=( + f"Destination '{d.name}' is used by {len(users)} backup schedule(s) — " + "point them at another destination or delete them first" + ), + ) + name, kind = d.name, d.type session.delete(d) session.commit() + if kind == "nfs": + dest_service.forget_nfs_volume(dest_id) audit_service.record( session, user=user.username, action="destination.delete", target=name, ip=_ip(request), @@ -134,9 +156,16 @@ async def test_destination( d = _get_or_404(session, dest_id) try: await asyncio.to_thread(dest_service.test, d) - return {"ok": True} except dest_service.DestinationError as exc: return {"ok": False, "error": str(exc)} + result: dict = {"ok": True} + if d.type == "sftp": + # Re-read: a first connection has just pinned the server's key. + session.refresh(d) + pinned = (dest_service.parse_config(d).get("host_key") or "").strip() + if pinned: + result["host_key"] = dest_service.fingerprint_of(pinned) + return result @router.get("/{dest_id}/backups") diff --git a/backend/routers/editor.py b/backend/routers/editor.py index b595790..1838263 100644 --- a/backend/routers/editor.py +++ b/backend/routers/editor.py @@ -1,6 +1,8 @@ """Editor helper endpoints — merge wizard fragments into compose YAML.""" from __future__ import annotations +import os + from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel @@ -15,6 +17,8 @@ router = APIRouter(prefix="/api/editor", tags=["editor"]) class ValidateBody(BaseModel): yaml: str env: str = "" + #: An existing stack: relative paths are then resolved in its folder. + stack_id: str | None = None class AddVolumeBody(BaseModel): @@ -61,7 +65,18 @@ def _run(fn, *args) -> dict: @router.post("/validate") async def validate(body: ValidateBody, _user: User = Depends(get_current_user)) -> dict: """Run `docker compose config -q` on the supplied YAML (+ optional .env).""" - return await compose_service.validate_yaml(body.yaml, body.env) + project_dir = None + if body.stack_id: + candidate = compose_service.stack_dir(body.stack_id) + # Only a real stack folder under STACKS_DIR, never a path out of it. + if os.path.dirname(os.path.normpath(candidate)) == os.path.normpath( + compose_service.stacks_root() + ): + project_dir = candidate + try: + return await compose_service.validate_yaml(body.yaml, body.env, project_dir) + except compose_service.StackFileError as exc: + raise HTTPException(status_code=504, detail=str(exc)) from exc @router.post("/services") diff --git a/backend/routers/files.py b/backend/routers/files.py index 9bb7479..af9cdc0 100644 --- a/backend/routers/files.py +++ b/backend/routers/files.py @@ -10,7 +10,9 @@ All paths are sandboxed by :mod:`services.file_service`. from __future__ import annotations import os +import stat import tempfile +from urllib.parse import quote from fastapi import ( APIRouter, @@ -35,9 +37,19 @@ router = APIRouter(prefix="/api/files", tags=["files"]) def _attachment(filename: str) -> str: - """A safe ``Content-Disposition`` value for an arbitrary filename.""" + """A safe ``Content-Disposition`` value for an arbitrary filename. + + Headers are latin-1; a folder called "Fotos – 2024" (en dash) raised + UnicodeEncodeError and the download died with a 500. Non-ASCII names go + out as RFC 5987 ``filename*``, with an ASCII fallback beside it. + """ safe = filename.replace("\\", "_").replace('"', "_") - return f'attachment; filename="{safe}"' + try: + safe.encode("ascii") + return f'attachment; filename="{safe}"' + except UnicodeEncodeError: + fallback = safe.encode("ascii", "replace").decode("ascii").replace("?", "_") + return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename)}" def _ip(request: Request) -> str: @@ -230,6 +242,27 @@ def delete( return result +def _settle_mode(tmp: str, target: str) -> None: + """Give an upload the permissions a normal file there would have. + + A temp file is created 0600, and ``os.replace`` kept that — so a config + uploaded into a stack folder was unreadable for any container not running + as root. An overwritten file keeps its mode and owner; a new one gets the + usual 0644 (minus umask). + """ + try: + st = os.stat(target) + os.chmod(tmp, stat.S_IMODE(st.st_mode)) + try: + os.chown(tmp, st.st_uid, st.st_gid) + except OSError: + pass + except FileNotFoundError: + umask = os.umask(0) + os.umask(umask) + os.chmod(tmp, 0o666 & ~umask) + + @router.post("/upload") async def upload( request: Request, @@ -249,6 +282,7 @@ async def upload( while chunk := await file.read(1024 * 1024): tmp.write(chunk) tmp.close() + _settle_mode(tmp.name, real) os.replace(tmp.name, real) except OSError as exc: if os.path.exists(tmp.name): diff --git a/backend/routers/git.py b/backend/routers/git.py index 7d0f4f0..0829900 100644 --- a/backend/routers/git.py +++ b/backend/routers/git.py @@ -97,6 +97,11 @@ def connect( raise HTTPException(status_code=400, detail=f"Unknown auth type '{body.auth_type}'") if not (body.url or "").strip(): raise HTTPException(status_code=400, detail="A repository URL is required") + try: + git_service.validate_url(body.url) + git_service.validate_branch(body.branch or "main") + except git_service.GitError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc row = session.exec(select(GitSource).where(GitSource.stack_id == stack_id)).first() if row is None: @@ -221,6 +226,9 @@ async def webhook( body = await request.body() if not row or not git_service.verify_webhook(row, body, request.headers): raise HTTPException(status_code=404, detail="Not found") + if not session.get(Stack, stack_id): + # Signed, but for a stack that was deleted: never bring it back. + raise HTTPException(status_code=404, detail="Not found") try: result = await git_service.sync(session, row, actor="webhook") diff --git a/backend/routers/images.py b/backend/routers/images.py index 901840d..d5be8c6 100644 --- a/backend/routers/images.py +++ b/backend/routers/images.py @@ -10,7 +10,7 @@ from auth import get_current_user, require_admin from database import engine, get_session from models.image_scan import ImageScan, ScanDetail, ScanRequest, ScanSummary from models.user import User -from services import audit_service, image_service, scan_service, update_service +from services import audit_service, compose_service, image_service, scan_service, update_service from services.prune_job import PruneBusy router = APIRouter(prefix="/api/images", tags=["images"]) @@ -21,8 +21,18 @@ def _ip(request: Request) -> str: @router.get("") -def list_images(_user: User = Depends(get_current_user)) -> list[dict]: - return image_service.list_images() +def list_images( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> list[dict]: + rows = image_service.list_images() + # Keyed by stack id like the stack list, or a stack whose folder name + # compose had to normalise ("Nextcloud" -> "nextcloud") showed up as a + # stranger whose stack had been removed. + ids = compose_service.stack_ids_by_project_in(session) + for row in rows: + row["stacks"] = sorted({compose_service.owner_stack(p, ids) for p in row["stacks"]}) + return rows @router.get("/updates") @@ -160,20 +170,40 @@ async def scan_image( @router.post("/scan-all") async def scan_all( request: Request, + response: Response, + background: bool = Query(False), session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: - """Scan every image a stack is using. Serial, and one sweep at a time.""" - images = sorted({row["tag"] for row in image_service.list_images() if row["stacks"]}) + """Scan every image a stack is using. Serial, and one sweep at a time. + + ``background=true`` returns 202 with the sweep's status at once; poll + ``/scan-status`` for progress and the result. That is what the UI does — + waiting on the response outlived the proxy's timeout. + """ + # Images of StackPilot's own stacks — not of every compose project on the + # host, which "a stack is using" used to include (other tools' projects, + # leftovers of deleted stacks). + ids = compose_service.stack_ids_by_project_in(session) + images = sorted( + {row["tag"] for row in image_service.list_images() if set(row["stacks"]) & ids.keys()} + ) + username, ip = user.username, _ip(request) + + def audit(own: Session, result: dict) -> None: + audit_service.record( + own, user=username, action="image.scan-all", target="*", + detail=f"{result['scanned']} scanned, {result['failed']} failed", ip=ip, + ) + try: + if background: + response.status_code = 202 + return scan_service.start_sweep(images, on_done=audit) result = await scan_service.sweep(session, images) except scan_service.ScanError as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc - audit_service.record( - session, user=user.username, action="image.scan-all", target="*", - detail=f"{result['scanned']} scanned, {result['failed']} failed", - ip=_ip(request), - ) + audit(session, result) return result diff --git a/backend/routers/networks.py b/backend/routers/networks.py index 62df9da..5c872b0 100644 --- a/backend/routers/networks.py +++ b/backend/routers/networks.py @@ -9,7 +9,7 @@ from auth import get_current_user, require_admin from database import engine, get_session from docker_client import DockerError from models.user import User -from services import audit_service, network_service +from services import audit_service, compose_service, network_service from services.prune_job import PruneBusy router = APIRouter(prefix="/api/networks", tags=["networks"]) @@ -44,8 +44,15 @@ class ContainerRef(BaseModel): @router.get("") -def list_networks(_user: User = Depends(get_current_user)) -> list[dict]: - return network_service.list_networks() +def list_networks( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> list[dict]: + rows = network_service.list_networks() + ids = compose_service.stack_ids_by_project_in(session) + for row in rows: + row["stack"] = compose_service.owner_stack(row["stack"], ids) + return rows # Before /{network_id}, which would otherwise take "prune-status" for an id. diff --git a/backend/routers/secrets.py b/backend/routers/secrets.py index 118ed83..a6ffcc6 100644 --- a/backend/routers/secrets.py +++ b/backend/routers/secrets.py @@ -13,6 +13,7 @@ from sqlmodel import Session from auth import require_admin from database import get_session +from models.stack import Stack from models.user import User from services import audit_service, secret_service from services.compose_edit_service import EditError @@ -51,6 +52,16 @@ def _guard(fn, *args, **kwargs): raise HTTPException(status_code=400, detail=str(exc)) from exc +def _stack(session: Session, stack_id: str) -> None: + """404 for a stack that does not exist. + + Without it a typo in the URL created ``//.secrets`` on + disk, and ``..`` reached the directory above the stacks. + """ + if not session.get(Stack, stack_id): + raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found") + + def list_secrets(stack_id: str) -> list[dict]: @@ -82,7 +93,12 @@ def detach_secret(stack_id: str, body: DetachBody) -> dict: @router.get("") -def http_list(stack_id: str, _user: User = Depends(require_admin)) -> list[dict]: +def http_list( + stack_id: str, + session: Session = Depends(get_session), + _user: User = Depends(require_admin), +) -> list[dict]: + _stack(session, stack_id) return list_secrets(stack_id) @@ -91,6 +107,7 @@ def http_write( stack_id: str, body: SecretWrite, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _stack(session, stack_id) result = write_secret(stack_id, body) audit_service.record( session, user=user.username, action="secret.write", @@ -104,6 +121,7 @@ def http_delete( stack_id: str, kind: str, name: str, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _stack(session, stack_id) result = delete_secret(stack_id, kind, name) audit_service.record( session, user=user.username, action="secret.delete", @@ -117,6 +135,7 @@ def http_attach( stack_id: str, body: AttachBody, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _stack(session, stack_id) result = attach_secret(stack_id, body) audit_service.record( session, user=user.username, action="secret.attach", @@ -130,6 +149,7 @@ def http_detach( stack_id: str, body: DetachBody, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _stack(session, stack_id) result = detach_secret(stack_id, body) audit_service.record( session, user=user.username, action="secret.detach", diff --git a/backend/routers/stacks.py b/backend/routers/stacks.py index c1588d8..1d0d56a 100644 --- a/backend/routers/stacks.py +++ b/backend/routers/stacks.py @@ -9,6 +9,7 @@ from dataclasses import asdict from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile from fastapi.responses import FileResponse from sqlmodel import Session, select +from starlette.background import BackgroundTask from auth import get_current_user, require_admin from database import get_session @@ -36,6 +37,7 @@ from services import ( icon_service, logo_service, notify_service, + stack_cleanup_service, stack_job_service, stack_lock_service, stats_service, @@ -131,7 +133,7 @@ def _stack_summary( total = running = 0 status = "unknown" else: - info = summaries.get(stack.id) + info = summaries.get(compose_service.project_name(stack.id)) total = info["total"] if info else 0 running = info["running"] if info else 0 if stack.id in busy: @@ -204,17 +206,30 @@ def create_stack( return _stack_summary(stack) +def _by_stack_id(session: Session, per_project: dict) -> dict: + """Re-key per-compose-project data by stack id (they differ only for a + folder whose name compose had to normalise, see project_name).""" + ids = compose_service.stack_ids_by_project_in(session) + return {ids.get(project, project): value for project, value in per_project.items()} + + @router.get("/stats") -def stacks_stats(_user: User = Depends(get_current_user)) -> dict: +def stacks_stats( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> dict: """Live CPU (cores) and memory usage per stack, with assigned limits.""" - return stats_service.stack_stats() + return _by_stack_id(session, stats_service.stack_stats()) @router.get("/updates") -def stacks_updates(_user: User = Depends(get_current_user)) -> dict: +def stacks_updates( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> dict: """Per-stack image-update availability, read from the cached registry digests (no live registry calls — safe for the list to poll).""" - return update_service.stacks_update_summary() + return _by_stack_id(session, update_service.stacks_update_summary()) @router.get("/icons/search") @@ -275,7 +290,12 @@ def get_stack( # What is holding the stack, so the page can offer to stop it instead # of only showing a spinner that never ends. "lock": _lock_view(lock, stack_id), - "yaml": compose_service.read_compose(stack_id), + # Inline credentials masked for the read-only role, like the .env. + "yaml": ( + compose_service.read_compose(stack_id) + if user.role == "admin" + else compose_service.mask_secrets(compose_service.read_compose(stack_id)) + ), # The .env is where credentials live by convention, so it is withheld # from the read-only role — same reasoning as the admin-only file # browser. Non-admins still get status, services and the compose file. @@ -333,21 +353,49 @@ async def delete_stack( user: User = Depends(require_admin), ) -> dict: stack = _get_stack_or_404(session, stack_id) + # Deleting a stack in the middle of a backup, restore or deploy pulls the + # folder out from under it — and a backup then starts the containers it + # stopped again, for a stack that no longer exists. + lock = stack_lock_service.info(session, stack_id) + if lock: + raise HTTPException( + status_code=409, + detail=f"Stack '{stack_id}' is busy: {lock['action']} in progress", + ) try: - await compose_service.down(stack_id) - except Exception: # noqa: BLE001 - best-effort teardown - pass + result = await compose_service.down(stack_id) + down_ok = result.get("returncode") == 0 + except Exception: # noqa: BLE001 - fall back to the API below + down_ok = False + if not down_ok: + # A compose file that no longer parses makes `down` fail, and the + # containers would keep running with nothing left here to stop them. + try: + removed = await asyncio.to_thread(compose_service.force_remove_project, stack_id) + logger.info("compose down failed for '%s'; removed %d container(s) directly", + stack_id, removed) + except DockerError as exc: + logger.warning("Could not remove the containers of '%s': %s", stack_id, exc) + kept: list[str] = [] if delete_files: - compose_service.delete_stack_files(stack_id) + try: + kept = compose_service.delete_stack_files(stack_id) + except OSError as exc: + raise HTTPException( + status_code=500, + detail=f"The stack was stopped, but its folder could not be deleted: {exc}", + ) from exc icon_service.remove(stack_id) logo_service.forget(stack_id) + stack_cleanup_service.forget_stack(session, stack_id) session.delete(stack) session.commit() audit_service.record( session, user=user.username, action="stack.delete", target=stack_id, - detail=f"delete_files={delete_files}", ip=_client_ip(request), + detail=f"delete_files={delete_files}" + (f" kept={len(kept)} mount(s)" if kept else ""), + ip=_client_ip(request), ) - return {"ok": True} + return {"ok": True, "kept": kept} @router.post("/{stack_id}/clone") @@ -365,7 +413,8 @@ def clone_stack( try: compose_service.clone_stack_files(stack_id, new_id) except compose_service.StackFileError as exc: - raise HTTPException(status_code=409, detail=str(exc)) from exc + status = 409 if "already exists" in str(exc) else 500 + raise HTTPException(status_code=status, detail=str(exc)) from exc stack = Stack( id=new_id, name=body.name, @@ -412,6 +461,8 @@ async def get_stack_icon( def _icon_response(path: str, media_type: str, filename: str) -> FileResponse: + # A folder found on disk can be named anything; headers are latin-1. + filename = filename.encode("ascii", "replace").decode("ascii").replace("?", "_") return FileResponse( path, media_type=media_type, @@ -516,6 +567,9 @@ async def _lifecycle(action_fn, action_name, stack_id, request, session, user): status_code=409, detail=f"Stack '{stack_id}' is busy: {exc.action} in progress", ) from exc + except compose_service.StackFileError as exc: + # The compose call timed out; it may still be finishing on the host. + raise HTTPException(status_code=504, detail=str(exc)) from exc audit_service.record( session, user=user.username, action=f"stack.{action_name}", target=stack_id, detail=f"rc={result.get('returncode')}", ip=_client_ip(request), @@ -683,7 +737,7 @@ async def unlock_stack( @router.get("/{stack_id}/logs") async def stack_logs( stack_id: str, - tail: int = Query(200, le=2000), + tail: int = Query(200, ge=0, le=2000), session: Session = Depends(get_session), _user: User = Depends(get_current_user), ) -> dict: @@ -696,7 +750,7 @@ async def stack_logs( async def service_logs( stack_id: str, service: str, - tail: int = Query(200, le=2000), + tail: int = Query(200, ge=0, le=2000), session: Session = Depends(get_session), _user: User = Depends(get_current_user), ) -> dict: @@ -721,13 +775,20 @@ def export_stack( if not os.path.isdir(directory): raise HTTPException(status_code=404, detail="Stack directory missing") tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz") - with tarfile.open(tmp.name, "w:gz") as tar: - tar.add(directory, arcname=stack_id) + tmp.close() + try: + with tarfile.open(tmp.name, "w:gz") as tar: + tar.add(directory, arcname=stack_id) + except BaseException: + os.unlink(tmp.name) + raise date = compose_service.now().strftime("%Y%m%d") return FileResponse( tmp.name, media_type="application/gzip", filename=f"stack-{stack_id}-{date}.tar.gz", + # The archive is a temp file; without this every export stayed in /tmp. + background=BackgroundTask(os.unlink, tmp.name), ) @@ -765,6 +826,7 @@ def set_auto_update( session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _get_stack_or_404(session, stack_id) policy = auto_update_service.upsert_policy(session, stack_id, body.enabled, body.redeploy) audit_service.record( session, user=user.username, action="stack.auto_update", diff --git a/backend/routers/volumes.py b/backend/routers/volumes.py index 4a2b6ab..2fa965d 100644 --- a/backend/routers/volumes.py +++ b/backend/routers/volumes.py @@ -7,7 +7,7 @@ from sqlmodel import Session from auth import get_current_user, require_admin from database import engine, get_session from models.user import User -from services import audit_service, device_service, volume_service +from services import audit_service, compose_service, device_service, volume_service from services.prune_job import PruneBusy router = APIRouter(tags=["volumes"]) @@ -23,8 +23,15 @@ def _ip(request: Request) -> str: @router.get("/api/volumes") -def list_volumes(_user: User = Depends(get_current_user)) -> list[dict]: - return volume_service.list_volumes() +def list_volumes( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> list[dict]: + rows = volume_service.list_volumes() + ids = compose_service.stack_ids_by_project_in(session) + for row in rows: + row["stack"] = compose_service.owner_stack(row["stack"], ids) + return rows @router.get("/api/volumes/orphaned") diff --git a/backend/routers/ws.py b/backend/routers/ws.py index 0f48708..44d50d4 100644 --- a/backend/routers/ws.py +++ b/backend/routers/ws.py @@ -2,10 +2,11 @@ from __future__ import annotations import asyncio +import contextlib import json import logging - -import contextlib +import threading +from collections.abc import Awaitable, Callable from fastapi import APIRouter, Query, WebSocket, WebSocketDisconnect from jose import JWTError @@ -14,12 +15,13 @@ from sqlmodel import Session from auth import decode_token, resolve_token_user from database import engine from models.setting import EVENT_PULL_FAILED, EVENT_STACK_ERROR, EVENT_STACK_START +from models.stack import Stack from services import ( audit_service, + compose_run_service, compose_service, exec_service, notify_service, - stack_lock_service, update_service, ) @@ -49,52 +51,93 @@ def _user_for(token: str | None): return user -async def _authorize(websocket: WebSocket, token: str | None) -> bool: - """Validate the JWT supplied as a query param. Closes socket on failure.""" - if _user_for(token) is None: - await websocket.close(code=4401) - return False - return True - - -async def _authorize_admin(websocket: WebSocket, token: str | None) -> bool: - """Like _authorize but also requires the admin role (exec is root-equivalent). - - Closes 4401 on a missing/invalid token, 4403 on a valid non-admin token.""" +async def _authorize(websocket: WebSocket, token: str | None): + """The user behind the token, or None after closing the socket with 4401.""" user = _user_for(token) if user is None: await websocket.close(code=4401) - return False + return user + + +async def _authorize_admin(websocket: WebSocket, token: str | None): + """Like _authorize but also requires the admin role. + + Anything that changes what runs on the host is admin-only over HTTP, so it + has to be over a socket too — the deploy console runs the same + ``compose up`` the REST ``/start`` does. Closes 4401 on a missing/invalid + token, 4403 on a valid non-admin one.""" + user = _user_for(token) + if user is None: + await websocket.close(code=4401) + return None if user.role != "admin": await websocket.close(code=4403) - return False - return True + return None + return user + + +async def _wait_disconnect(websocket: WebSocket) -> None: + """Return once the client has gone. Whatever it sends is ignored. + + A stream that only ever *sends* notices a closed socket on its next send — + which for a quiet log or an idle Docker host can be hours away, and until + then everything behind the socket (a ``compose logs -f``, an event reader + thread) stays alive for nobody. + """ + try: + while True: + message = await websocket.receive() + if message.get("type") == "websocket.disconnect": + return + except Exception: # noqa: BLE001 - a broken socket is a gone client + return + + +async def _until_gone(websocket: WebSocket, work: Awaitable[None]) -> None: + """Run ``work`` until it finishes or the client disconnects, whichever is first. + + Cancelling ``work`` is what stops whatever it was reading from; see + :func:`compose_service.stream_compose`. + """ + task = asyncio.ensure_future(work) + gone = asyncio.create_task(_wait_disconnect(websocket)) + try: + await asyncio.wait({task, gone}, return_when=asyncio.FIRST_COMPLETED) + finally: + for pending in (task, gone): + pending.cancel() + await asyncio.gather(task, gone, return_exceptions=True) + if task.done() and not task.cancelled() and task.exception() is not None: + exc = task.exception() + if not isinstance(exc, WebSocketDisconnect): + with contextlib.suppress(Exception): + await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) async def _stream_logs(websocket: WebSocket, stack_id: str, service: str | None): """Stream `docker compose logs -f` output to the client.""" args = ["logs", "--no-color", "--tail", "200", "--timestamps", "-f"] if service: - args.append(service) - try: - async for line in compose_service.stream_compose(stack_id, args): - await websocket.send_text( - json.dumps( - { - "type": "log", - "stack_id": stack_id, - "service": service, - "line": line, - } - ) - ) - except WebSocketDisconnect: - raise - except Exception as exc: # noqa: BLE001 + # "--" so a service name can never be read as a compose option. + args += ["--", service] + + async def pump() -> None: + # aclosing: cancelled while *sending*, the loop is left by an exception + # outside the generator, which would then only be closed — and its + # `compose logs -f` killed — whenever garbage collection got to it. + async with contextlib.aclosing( + compose_service.stream_compose(stack_id, args) + ) as lines: + async for line in lines: + await _send_log(line) + + async def _send_log(line: str) -> None: await websocket.send_text( - json.dumps({"type": "error", "detail": str(exc)}) + json.dumps({"type": "log", "stack_id": stack_id, "service": service, "line": line}) ) + await _until_gone(websocket, pump()) + @router.websocket("/ws/logs/{stack_id}") async def ws_stack_logs( @@ -105,10 +148,7 @@ async def ws_stack_logs( await websocket.accept() if not await _authorize(websocket, token): return - try: - await _stream_logs(websocket, stack_id, None) - except WebSocketDisconnect: - pass + await _stream_logs(websocket, stack_id, None) @router.websocket("/ws/logs/{stack_id}/{service}") @@ -121,10 +161,67 @@ async def ws_service_logs( await websocket.accept() if not await _authorize(websocket, token): return + await _stream_logs(websocket, stack_id, service) + + +# --------------------------------------------------------------------------- # +# Deploy / update: a compose run the socket only watches +# --------------------------------------------------------------------------- # + + +def _stack_known(stack_id: str) -> bool: + with Session(engine) as session: + return session.get(Stack, stack_id) is not None + + +async def _watch_run( + websocket: WebSocket, + stack_id: str, + action: str, + username: str, + stream: Callable, + on_finish: Callable[[int | None], Awaitable[None]], +) -> None: + """Start (or attach to) the compose run and relay it until it ends. + + The run belongs to :mod:`services.compose_run_service`, not to this socket: + closing the deploy console no longer releases the stack lock under a + running compose, and the audit entry carries the real exit code. + """ + if not _stack_known(stack_id): + with contextlib.suppress(Exception): + await websocket.send_text( + json.dumps({"type": "error", "detail": f"Stack '{stack_id}' not found"}) + ) + await websocket.close(code=4404) + return try: - await _stream_logs(websocket, stack_id, service) - except WebSocketDisconnect: - pass + run, queue, _attached = compose_run_service.start( + stack_id, action, username, stream, on_finish + ) + except compose_run_service.RunBusy as exc: + with contextlib.suppress(Exception): + await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) + await websocket.close(code=4409) + return + + async def relay() -> None: + while True: + item = await queue.get() + if item is None: + return + kind, payload = item + if kind == "log": + await websocket.send_text(json.dumps({"type": "log", "line": payload})) + else: + await websocket.send_text(json.dumps({"type": "done", "returncode": payload})) + + try: + await _until_gone(websocket, relay()) + finally: + run.unsubscribe(queue) + with contextlib.suppress(Exception): + await websocket.close() @router.websocket("/ws/deploy/{stack_id}") @@ -135,53 +232,21 @@ async def ws_deploy( ): """Run `docker compose up -d` and stream its output (image pulls, container creation) to the browser so the user sees deploy progress live. Records the - same audit entry and notification as the REST `/start` endpoint.""" + same audit entry and notification as the REST `/start` endpoint. Admin-only, + like that endpoint.""" await websocket.accept() - if not await _authorize(websocket, token): + user = await _authorize_admin(websocket, token) + if not user: return - username = decode_token(token, "access").get("sub", "unknown") if token else "unknown" + username = user.username - rc: int | None = None - disconnected = False - # Same guard the REST lifecycle uses — the deploy console runs the very - # same `compose up`, so it has to queue behind an in-flight operation - # rather than race it. - lock_session = Session(engine) - try: - stack_lock_service.acquire(lock_session, stack_id, "start", username) - except stack_lock_service.StackBusy as exc: - lock_session.close() - with contextlib.suppress(Exception): - await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) - await websocket.close(code=4409) - return - try: - async for kind, payload in compose_service.stream_up(stack_id): - if kind == "log": - await websocket.send_text(json.dumps({"type": "log", "line": payload})) - else: - rc = payload - await websocket.send_text(json.dumps({"type": "done", "returncode": rc})) - except WebSocketDisconnect: - # Client navigated away; the compose subprocess keeps running so the - # deploy still completes in the background. - disconnected = True - except Exception as exc: # noqa: BLE001 - with contextlib.suppress(Exception): - await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) - finally: - with contextlib.suppress(Exception): - stack_lock_service.release(lock_session, stack_id) - lock_session.close() - - ok = rc in (0, None) - try: + async def finish(rc: int | None) -> None: with Session(engine) as session: audit_service.record( session, user=username, action="stack.start", target=stack_id, detail=f"rc={rc} (deploy console)", ip="ws", ) - if ok: + if rc == 0: await notify_service.notify( EVENT_STACK_START, f"Stack '{stack_id}' started", "compose up completed successfully.", session, @@ -191,11 +256,11 @@ async def ws_deploy( EVENT_STACK_ERROR, f"Stack '{stack_id}' start failed", "compose up returned a non-zero exit code.", session, ) - except Exception: # noqa: BLE001 - audit/notify are best-effort - pass - if not disconnected: - with contextlib.suppress(Exception): - await websocket.close() + + await _watch_run( + websocket, stack_id, "start", username, + lambda: compose_service.stream_up(stack_id), finish, + ) @router.websocket("/ws/update/{stack_id}") @@ -208,47 +273,18 @@ async def ws_update( list can render real update progress. Same audit/notify contract as the REST `/update` endpoint, which stays for non-interactive callers.""" await websocket.accept() - if not await _authorize_admin(websocket, token): + user = await _authorize_admin(websocket, token) + if not user: return - username = decode_token(token, "access").get("sub", "unknown") if token else "unknown" + username = user.username - rc: int | None = None - disconnected = False - lock_session = Session(engine) - try: - stack_lock_service.acquire(lock_session, stack_id, "update", username) - except stack_lock_service.StackBusy as exc: - lock_session.close() - with contextlib.suppress(Exception): - await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) - await websocket.close(code=4409) - return - try: - async for kind, payload in compose_service.stream_update(stack_id): - if kind == "log": - await websocket.send_text(json.dumps({"type": "log", "line": payload})) - else: - rc = payload - await websocket.send_text(json.dumps({"type": "done", "returncode": rc})) - except WebSocketDisconnect: - # Client navigated away; compose keeps running so the update finishes. - disconnected = True - except Exception as exc: # noqa: BLE001 - with contextlib.suppress(Exception): - await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) - finally: - with contextlib.suppress(Exception): - stack_lock_service.release(lock_session, stack_id) - lock_session.close() - - ok = rc in (0, None) - try: + async def finish(rc: int | None) -> None: with Session(engine) as session: audit_service.record( session, user=username, action="stack.update", target=stack_id, detail=f"rc={rc} (update stream)", ip="ws", ) - if ok: + if rc == 0: await notify_service.notify( EVENT_STACK_START, f"Stack '{stack_id}' updated", "compose pull + up completed successfully.", session, @@ -258,16 +294,19 @@ async def ws_update( EVENT_PULL_FAILED, f"Stack '{stack_id}' update failed", "compose pull/up returned a non-zero exit code.", session, ) - except Exception: # noqa: BLE001 - audit/notify are best-effort - pass - if not disconnected: - with contextlib.suppress(Exception): - await websocket.close() + if rc == 0: + await asyncio.to_thread(update_service.refresh_stack_local, stack_id) - if ok: - update_service.refresh_stack_local(stack_id) + await _watch_run( + websocket, stack_id, "update", username, + lambda: compose_service.stream_update(stack_id), finish, + ) +# --------------------------------------------------------------------------- # +# Docker events +# --------------------------------------------------------------------------- # + #: Docker event types worth telling the UI about. Filtered daemon-side, so the #: bulk of the firehose never crosses the socket. _EVENT_TYPES = ["container", "image", "network", "volume"] @@ -300,33 +339,58 @@ async def ws_events( "container": "jellyfin", "stack": "jellyfin"} ``resource`` is what the client needs to decide which queries to drop. + + The blocking read runs on a thread of its own, not on the event loop's + shared executor: every open tab holds one reader for as long as it is open, + and on the executor a handful of tabs used up every worker — after which + each ``asyncio.to_thread`` (backups, the dashboard, NFS destinations) waited + forever. """ await websocket.accept() if not await _authorize(websocket, token): return from docker_client import get_client - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() queue: asyncio.Queue = asyncio.Queue() - stream = None + guard = threading.Lock() + state: dict = {"stream": None, "stop": False} - def reader(): + def post(item) -> None: + try: + loop.call_soon_threadsafe(queue.put_nowait, item) + except RuntimeError: # the loop is gone (shutdown) + pass + + def reader() -> None: """Blocking read of the event stream, handed to the loop thread-safely.""" - nonlocal stream try: stream = get_client().events(decode=True, filters={"type": _EVENT_TYPES}) + with guard: + if state["stop"]: + # The client left while we were connecting. + with contextlib.suppress(Exception): + stream.close() + return + state["stream"] = stream for event in stream: - loop.call_soon_threadsafe(queue.put_nowait, event) + post(event) except Exception: # noqa: BLE001 - a closed stream lands here on teardown pass finally: - loop.call_soon_threadsafe(queue.put_nowait, None) + post(None) - task = loop.run_in_executor(None, reader) + threading.Thread(target=reader, name="docker-events", daemon=True).start() + gone = asyncio.create_task(_wait_disconnect(websocket)) try: await websocket.send_text(json.dumps({"type": "ready"})) while True: - event = await queue.get() + getter = asyncio.ensure_future(queue.get()) + done, _ = await asyncio.wait({getter, gone}, return_when=asyncio.FIRST_COMPLETED) + if getter not in done: + getter.cancel() + break # the client disconnected + event = getter.result() if event is None: # reader finished — daemon gone or stream closed await websocket.send_text( json.dumps({"type": "error", "detail": "Docker event stream ended"}) @@ -354,14 +418,14 @@ async def ws_events( with contextlib.suppress(Exception): await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) finally: + gone.cancel() # Closing the stream is what actually unblocks the reader thread. - # Cancelling the executor future does not: a thread already inside a - # blocking read keeps that read, and the thread leaks for the life of - # the process — once per page load, with the socket held open. + with guard: + state["stop"] = True + stream = state["stream"] if stream is not None: with contextlib.suppress(Exception): stream.close() - task.cancel() @router.websocket("/ws/exec/{container_id}") @@ -373,9 +437,10 @@ async def ws_exec( ): """Interactive shell into a compose-managed container (admin only).""" await websocket.accept() - if not await _authorize_admin(websocket, token): + user = await _authorize_admin(websocket, token) + if not user: return - username = decode_token(token, "access").get("sub", "unknown") if token else "unknown" + username = user.username shell = cmd or exec_service.DEFAULT_SHELL try: @@ -402,5 +467,3 @@ async def ws_exec( finally: with contextlib.suppress(Exception): await websocket.close() - - diff --git a/backend/services/auto_update_service.py b/backend/services/auto_update_service.py index c39250d..424b9a4 100644 --- a/backend/services/auto_update_service.py +++ b/backend/services/auto_update_service.py @@ -103,8 +103,12 @@ async def _run_local(session: Session, policy: AutoUpdate) -> None: # this stack, skip and pick it up next cycle rather than racing # them over the same containers. with stack_lock_service.hold(session, stack_id, "auto-update", "auto-update"): - await compose_service.pull(stack_id) - await compose_service.up(stack_id) + # pull + up, stopping at a failed pull. The exit codes used to + # be ignored entirely: a registry outage or a failing `up` was + # recorded as "updated" and announced as a success. + result = await compose_service.update(stack_id) + if result.get("returncode") != 0: + raise RuntimeError(_failure(result)) except stack_lock_service.StackBusy as exc: _record(session, policy, "skipped", str(exc)) return @@ -127,6 +131,13 @@ async def _run_local(session: Session, policy: AutoUpdate) -> None: ) +def _failure(result: dict) -> str: + """The last meaningful line of a failed compose run.""" + text = (result.get("stderr") or result.get("stdout") or "").strip() + tail = text.splitlines()[-1] if text else "" + return f"compose exited with {result.get('returncode')}" + (f": {tail}" if tail else "") + + async def _safe_notify(event: str, title: str, message: str, session: Session) -> None: try: await notify_service.notify(event, title, message, session) diff --git a/backend/services/backup_destination_service.py b/backend/services/backup_destination_service.py index b181ada..7c34dcc 100644 --- a/backend/services/backup_destination_service.py +++ b/backend/services/backup_destination_service.py @@ -76,24 +76,169 @@ def migrate_plaintext_configs(session) -> int: # --------------------------------------------------------------------------- # -def _sftp_connect(cfg: dict): +#: Seconds to wait for the SFTP server to accept the TCP connection. +SFTP_CONNECT_TIMEOUT = 20 + + +def _check_file_name(name: str) -> None: + """A backup is addressed by its bare file name, never a path. + + The name comes from the API (restore from / delete on a destination); with + ``../`` in it a delete reached files outside the backup folder — on SFTP + anything the account may write. NFS has always checked this. + """ + if ( + not name + or name in (".", "..") + or "/" in name + or "\\" in name + or "\0" in name + ): + raise DestinationError(f"Invalid backup file name '{name}'") + + +def host_key_fingerprint(key) -> str: + """OpenSSH's form: ``SHA256:`` + unpadded base64 of the key's SHA-256.""" + import base64 + import hashlib + + digest = hashlib.sha256(key.asbytes()).digest() + return "SHA256:" + base64.b64encode(digest).decode().rstrip("=") + + +def fingerprint_of(pinned: str) -> str: + """The ``SHA256:`` fingerprint of a stored host key (as given, if it is + one already) — what the UI shows and ``ssh-keygen -lf`` prints.""" + import base64 + import hashlib + + pinned = pinned.strip() + if pinned.startswith("SHA256:"): + return pinned + parts = pinned.split() + for i, part in enumerate(parts[:-1]): + if part.startswith(("ssh-", "ecdsa-", "sk-")): + try: + blob = base64.b64decode(parts[i + 1]) + except ValueError: + break + return "SHA256:" + base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip("=") + return pinned + + +def _pinned_type(pinned: str) -> Optional[str]: + """The key type of a pinned `` `` line (a known_hosts line + with the host in front works too); None for a bare fingerprint.""" + parts = pinned.split() + for part in parts[:-1]: + if part.startswith(("ssh-", "ecdsa-", "sk-")): + return part + return None + + +#: What paramiko negotiates for a key of each type. An RSA key is offered +#: under the SHA-2 signature names, not under its own. +_HOST_KEY_ALGORITHMS = { + "ssh-rsa": ("rsa-sha2-512", "rsa-sha2-256", "ssh-rsa"), +} + + +def _host_key_matches(pinned: str, key) -> bool: + pinned = pinned.strip() + if pinned.startswith("SHA256:"): + return pinned.rstrip("=") == host_key_fingerprint(key) + parts = pinned.split() + for i, part in enumerate(parts[:-1]): + if part == key.get_name() and parts[i + 1] == key.get_base64(): + return True + return False + + +def _sftp_connect(cfg: dict, pin=None): + """Connect and authenticate, checking the server's host key. + + Before 0.67.1 any host key was accepted, so whoever could answer on the + server's address (a spoofed DNS entry, a box that took its IP) received + the password and every backup. Now the key is pinned in the destination's + ``host_key``: entered by hand (a key line or its ``SHA256:`` fingerprint), + or — when empty — the one seen on the first connection, handed to ``pin`` + to be stored. After that a different key refuses the connection. + """ + import socket + import paramiko host = cfg.get("host") if not host: raise DestinationError("SFTP host is required") - port = int(cfg.get("port") or 22) - username = cfg.get("username") - transport = paramiko.Transport((host, port)) try: + port = int(cfg.get("port") or 22) + except (TypeError, ValueError) as exc: + raise DestinationError(f"Invalid SFTP port '{cfg.get('port')}'") from exc + username = cfg.get("username") + pinned = (cfg.get("host_key") or "").strip() + # Connected here, with a timeout, rather than by Transport((host, port)): + # that connects inside the constructor, outside any handler, so an + # unreachable host was a bare 500 instead of a message — and with no + # timeout it hung for the kernel's two minutes first. + try: + sock = socket.create_connection((host, port), timeout=SFTP_CONNECT_TIMEOUT) + except OSError as exc: + raise DestinationError(f"SFTP connection to {host}:{port} failed: {exc}") from exc + transport = paramiko.Transport(sock) + try: + key_type = _pinned_type(pinned) if pinned else None + if key_type: + # Ask for the pinned kind of key: a server has several, and the + # one negotiated by default need not be the one that was pinned. + wanted = _HOST_KEY_ALGORITHMS.get(key_type, (key_type,)) + options = transport.get_security_options() + usable = tuple(k for k in wanted if k in options.key_types) + if usable: + options.key_types = usable + transport.start_client(timeout=SFTP_CONNECT_TIMEOUT) + server_key = transport.get_remote_server_key() + if pinned: + if not _host_key_matches(pinned, server_key): + raise DestinationError( + f"SFTP host key of {host}:{port} does not match the stored one: the " + f"server now presents {server_key.get_name()} " + f"{host_key_fingerprint(server_key)}. If the server was reinstalled or " + "its keys were rotated, clear the destination's host key to trust the " + "new one — otherwise something else is answering on that address." + ) + elif pin is not None: + pin(f"{server_key.get_name()} {server_key.get_base64()}") pkey = None if cfg.get("private_key"): pkey = _load_key(cfg["private_key"]) - transport.connect(username=username, password=cfg.get("password") or None, pkey=pkey) + # What Transport.connect did: the key when there is one, else the password. + if pkey is not None: + transport.auth_publickey(username, pkey) + elif cfg.get("password"): + transport.auth_password(username, cfg["password"]) + return paramiko.SFTPClient.from_transport(transport), transport except Exception as exc: # noqa: BLE001 transport.close() + if isinstance(exc, DestinationError): + raise raise DestinationError(f"SFTP connection failed: {exc}") from exc - return paramiko.SFTPClient.from_transport(transport), transport + + +def _sftp_io(what: str): + """Turn what paramiko raises mid-transfer into a DestinationError.""" + import contextlib + + @contextlib.contextmanager + def wrapper(): + try: + yield + except DestinationError: + raise + except Exception as exc: # noqa: BLE001 - IOError, SSHException, EOFError … + raise DestinationError(f"SFTP {what} failed: {exc}") from exc + + return wrapper() def _load_key(key_str: str): @@ -120,22 +265,24 @@ def _sftp_makedirs(sftp, path: str) -> None: sftp.mkdir(cur) -def _sftp_upload(cfg: dict, local_path: str, filename: str) -> str: - sftp, transport = _sftp_connect(cfg) +def _sftp_upload(cfg: dict, local_path: str, filename: str, pin=None) -> str: + _check_file_name(filename) + sftp, transport = _sftp_connect(cfg, pin) try: - base = cfg.get("path") or "." - if base not in (".", ""): - _sftp_makedirs(sftp, base) - remote = posixpath.join(base, filename) if base not in (".", "") else filename - sftp.put(local_path, remote) - return remote + with _sftp_io("upload"): + base = cfg.get("path") or "." + if base not in (".", ""): + _sftp_makedirs(sftp, base) + remote = posixpath.join(base, filename) if base not in (".", "") else filename + sftp.put(local_path, remote) + return remote finally: sftp.close() transport.close() -def _sftp_list(cfg: dict) -> list[dict]: - sftp, transport = _sftp_connect(cfg) +def _sftp_list(cfg: dict, pin=None) -> list[dict]: + sftp, transport = _sftp_connect(cfg, pin) try: base = cfg.get("path") or "." out = [] @@ -155,23 +302,51 @@ def _sftp_list(cfg: dict) -> list[dict]: transport.close() -def _sftp_download(cfg: dict, name: str, local_path: str) -> None: - sftp, transport = _sftp_connect(cfg) +def _sftp_download(cfg: dict, name: str, local_path: str, pin=None) -> None: + _check_file_name(name) + sftp, transport = _sftp_connect(cfg, pin) try: - base = cfg.get("path") or "." - remote = posixpath.join(base, name) if base not in (".", "") else name - sftp.get(remote, local_path) + with _sftp_io(f"download of '{name}'"): + base = cfg.get("path") or "." + remote = posixpath.join(base, name) if base not in (".", "") else name + sftp.get(remote, local_path) finally: sftp.close() transport.close() -def _sftp_delete(cfg: dict, name: str) -> None: - sftp, transport = _sftp_connect(cfg) +def _sftp_delete(cfg: dict, name: str, pin=None) -> None: + _check_file_name(name) + sftp, transport = _sftp_connect(cfg, pin) try: - base = cfg.get("path") or "." - remote = posixpath.join(base, name) if base not in (".", "") else name - sftp.remove(remote) + with _sftp_io(f"delete of '{name}'"): + base = cfg.get("path") or "." + remote = posixpath.join(base, name) if base not in (".", "") else name + sftp.remove(remote) + finally: + sftp.close() + transport.close() + + +_TEST_FILE = ".stackpilot-test" + + +def _sftp_test(cfg: dict, pin=None) -> None: + """Connect, create the folder if needed, write and remove a file. + + Listing alone said "OK" for a folder that does not exist or cannot be + written to — the first scheduled backup was then the one to find out. + """ + sftp, transport = _sftp_connect(cfg, pin) + try: + with _sftp_io("write test"): + base = cfg.get("path") or "." + if base not in (".", ""): + _sftp_makedirs(sftp, base) + remote = posixpath.join(base, _TEST_FILE) if base not in (".", "") else _TEST_FILE + with sftp.open(remote, "w") as fh: + fh.write(b"ok") + sftp.remove(remote) finally: sftp.close() transport.close() @@ -188,13 +363,16 @@ def _s3_client(cfg: dict): bucket = cfg.get("bucket") if not bucket: raise DestinationError("S3 bucket is required") - return boto3.client( - "s3", - endpoint_url=cfg.get("endpoint_url") or None, - region_name=cfg.get("region") or None, - aws_access_key_id=cfg.get("access_key") or None, - aws_secret_access_key=cfg.get("secret_key") or None, - ) + try: + return boto3.client( + "s3", + endpoint_url=cfg.get("endpoint_url") or None, + region_name=cfg.get("region") or None, + aws_access_key_id=cfg.get("access_key") or None, + aws_secret_access_key=cfg.get("secret_key") or None, + ) + except Exception as exc: # noqa: BLE001 - e.g. a malformed endpoint URL + raise DestinationError(f"Invalid S3 settings: {exc}") from exc def _s3_key(cfg: dict, filename: str) -> str: @@ -203,6 +381,7 @@ def _s3_key(cfg: dict, filename: str) -> str: def _s3_upload(cfg: dict, local_path: str, filename: str) -> str: + _check_file_name(filename) client = _s3_client(cfg) key = _s3_key(cfg, filename) try: @@ -241,6 +420,7 @@ def _s3_list(cfg: dict) -> list[dict]: def _s3_download(cfg: dict, name: str, local_path: str) -> None: + _check_file_name(name) client = _s3_client(cfg) try: client.download_file(cfg["bucket"], _s3_key(cfg, name), local_path) @@ -249,8 +429,24 @@ def _s3_download(cfg: dict, name: str, local_path: str) -> None: def _s3_delete(cfg: dict, name: str) -> None: + _check_file_name(name) client = _s3_client(cfg) - client.delete_object(Bucket=cfg["bucket"], Key=_s3_key(cfg, name)) + try: + client.delete_object(Bucket=cfg["bucket"], Key=_s3_key(cfg, name)) + except Exception as exc: # noqa: BLE001 + raise DestinationError(f"S3 delete failed: {exc}") from exc + + +def _s3_test(cfg: dict) -> None: + """List, then write and delete a small object: proves the keys may write.""" + _s3_list(cfg) + client = _s3_client(cfg) + key = _s3_key(cfg, _TEST_FILE) + try: + client.put_object(Bucket=cfg["bucket"], Key=key, Body=b"ok") + client.delete_object(Bucket=cfg["bucket"], Key=key) + except Exception as exc: # noqa: BLE001 + raise DestinationError(f"S3 write test failed: {exc}") from exc # --------------------------------------------------------------------------- # @@ -354,6 +550,23 @@ def _nfs_volume(dest: BackupDestination, cfg: dict) -> str: return name +def forget_nfs_volume(dest_id: int) -> None: + """Remove the Docker volume of a deleted NFS destination (best effort). + + Left behind, it kept the old mount options under a name the next + destination with the same id would pick up again. + """ + from docker_client import DockerError, get_client, safe_call + + name = f"{_NFS_VOLUME_PREFIX}{dest_id}" + try: + client = get_client() + clear_nfs_leftovers(client, name) + safe_call(safe_call(client.volumes.get, name).remove) + except DockerError as exc: + logger.info("NFS volume %s not removed: %s", name, exc.detail or exc.error) + + def clear_nfs_leftovers(client, volume: Optional[str] = None) -> int: """Remove stopped containers holding an NFS destination volume. @@ -566,10 +779,38 @@ def _nfs_test(dest: BackupDestination, cfg: dict) -> bool: # --------------------------------------------------------------------------- # +def _pinner(dest: BackupDestination): + """Store the first host key an SFTP destination presents (trust on first + use). Written straight to the row — the callers are a scheduled run and a + "Test" click, and neither holds a session that will commit.""" + + def pin(host_key: str) -> None: + if dest.id is None: + return # not saved yet: nothing to pin it to + from sqlmodel import Session + + from database import engine + + with Session(engine) as session: + row = session.get(BackupDestination, dest.id) + if row is None: + return + cfg = parse_config(row) + if (cfg.get("host_key") or "").strip(): + return # pinned meanwhile; the next connect compares against it + cfg["host_key"] = host_key + row.config = dump_config(cfg) + session.add(row) + session.commit() + logger.info("Destination '%s': pinned SFTP host key %s", dest.name, host_key.split()[0]) + + return pin + + def upload(dest: BackupDestination, local_path: str, filename: str) -> str: cfg = parse_config(dest) if dest.type == "sftp": - return _sftp_upload(cfg, local_path, filename) + return _sftp_upload(cfg, local_path, filename, _pinner(dest)) if dest.type == "s3": return _s3_upload(cfg, local_path, filename) if dest.type == "nfs": @@ -580,7 +821,7 @@ def upload(dest: BackupDestination, local_path: str, filename: str) -> str: def list_backups(dest: BackupDestination) -> list[dict]: cfg = parse_config(dest) if dest.type == "sftp": - return _sftp_list(cfg) + return _sftp_list(cfg, _pinner(dest)) if dest.type == "s3": return _s3_list(cfg) if dest.type == "nfs": @@ -591,7 +832,7 @@ def list_backups(dest: BackupDestination) -> list[dict]: def download(dest: BackupDestination, name: str, local_path: str) -> None: cfg = parse_config(dest) if dest.type == "sftp": - _sftp_download(cfg, name, local_path) + _sftp_download(cfg, name, local_path, _pinner(dest)) elif dest.type == "s3": _s3_download(cfg, name, local_path) elif dest.type == "nfs": @@ -603,7 +844,7 @@ def download(dest: BackupDestination, name: str, local_path: str) -> None: def delete(dest: BackupDestination, name: str) -> None: cfg = parse_config(dest) if dest.type == "sftp": - _sftp_delete(cfg, name) + _sftp_delete(cfg, name, _pinner(dest)) elif dest.type == "s3": _s3_delete(cfg, name) elif dest.type == "nfs": @@ -614,7 +855,13 @@ def delete(dest: BackupDestination, name: str) -> None: def test(dest: BackupDestination) -> bool: """Connectivity check — validates reachability, auth and write access.""" + cfg = parse_config(dest) if dest.type == "nfs": - return _nfs_test(dest, parse_config(dest)) - list_backups(dest) - return True + return _nfs_test(dest, cfg) + if dest.type == "sftp": + _sftp_test(cfg, _pinner(dest)) + return True + if dest.type == "s3": + _s3_test(cfg) + return True + raise DestinationError(f"Unknown destination type '{dest.type}'") diff --git a/backend/services/backup_service.py b/backend/services/backup_service.py index 85ae3f0..bded79e 100644 --- a/backend/services/backup_service.py +++ b/backend/services/backup_service.py @@ -104,8 +104,20 @@ def backup_basename(stack_id: str, prefix: Optional[str] = None) -> str: return f"backup-{prefix}-{stack_id}" if prefix else f"backup-{stack_id}" -def backup_filename(stack_id: str, include_volumes: bool, prefix: Optional[str] = None) -> str: - date = compose_service.now().strftime("%Y%m%d-%H%M%S") +def backup_filename( + stack_id: str, + include_volumes: bool, + prefix: Optional[str] = None, + tz: Optional[str] = None, +) -> str: + """``backup---YYYYmmdd-HHMMSS.tar.gz``, the time on the + wall clock of ``tz`` — the schedule's zone, or the browser's for a backup + made by hand. It used to be UTC, so a backup taken at 12:21 in Berlin was + called …-102154 and seemed to be from two hours earlier. The format is + unchanged, so retention still matches older files (it orders by mtime).""" + from services.schedule_service import zone + + date = compose_service.now().astimezone(zone(tz)).strftime("%Y%m%d-%H%M%S") suffix = "full" if include_volumes else "config" return f"{backup_basename(stack_id, prefix)}-{suffix}-{date}.tar.gz" @@ -907,6 +919,10 @@ def restore_backup( stack_id = _target_id(manifest, target_id) old_id = compose_service.slugify(manifest.get("stack_id") or stack_id) renamed = stack_id != old_id + # Volumes are named after the compose *project*, which differs from the id + # for a folder compose had to normalise ("my.app" -> "myapp_data"). + old_project = compose_service.project_name(manifest.get("stack_id") or stack_id) + new_project = compose_service.project_name(stack_id) old_dir = manifest.get("stack_dir") or "" directory = compose_service.stack_dir(stack_id) @@ -969,14 +985,14 @@ def restore_backup( skipped.append({"kind": "volume", "name": full, "reason": "not requested"}) continue retargeted = False - if renamed and full.startswith(old_id + "_"): - full = stack_id + full[len(old_id) :] + if old_project != new_project and full.startswith(old_project + "_"): + full = new_project + full[len(old_project) :] retargeted = True labels = dict(v.get("labels") or {}) if not v.get("external"): - # Re-target compose's labels to the (possibly new) stack id. - labels[COMPOSE_PROJECT_LABEL] = stack_id - reason = _volume_conflict(full, stack_id, old_id, renamed and not retargeted) + # Re-target compose's labels to the (possibly new) project. + labels[COMPOSE_PROJECT_LABEL] = new_project + reason = _volume_conflict(full, new_project, old_id, renamed and not retargeted) if reason: skipped.append({"kind": "volume", "name": full, "reason": reason}) continue @@ -1012,7 +1028,10 @@ def restore_backup( skipped += manifest.get("skipped", []) return { "stack_id": stack_id, - "name": manifest.get("name", stack_id), + # Restored under a new id, the copy must not carry the original's + # display name — two stacks called the same in the list is how the + # wrong one gets stopped. + "name": stack_id if renamed else manifest.get("name", stack_id), "volumes_restored": volumes_restored, "binds_restored": binds_restored, "skipped": skipped, @@ -1021,13 +1040,13 @@ def restore_backup( } -def _volume_conflict(full: str, stack_id: str, old_id: str, shared_with_original: bool) -> Optional[str]: +def _volume_conflict(full: str, project: str, old_id: str, shared_with_original: bool) -> Optional[str]: """Why a volume must not be overwritten, or None if it may be.""" attrs = assets.volume_attrs(full) if attrs is None: return None # created fresh owner = (attrs.get("Labels") or {}).get(COMPOSE_PROJECT_LABEL) - if owner and owner != stack_id: + if owner and owner != project: return f"belongs to stack '{owner}'; not overwritten" if shared_with_original: return ( diff --git a/backend/services/compose_run_service.py b/backend/services/compose_run_service.py new file mode 100644 index 0000000..26868be --- /dev/null +++ b/backend/services/compose_run_service.py @@ -0,0 +1,174 @@ +"""Streamed compose runs (deploy console, update progress) that outlive their socket. + +The deploy console and the stacks list's update bar watch ``compose up`` / +``compose pull && up`` over a WebSocket. That run used to *be* the socket +handler, which went wrong the moment somebody closed the modal or navigated +away halfway through a deploy: + +* the stack lock was released in the handler's ``finally`` at once, while + compose was still running — so the next deploy, update or backup could start + on top of it; +* nobody read compose's output any more, so a chatty pull filled the pipe and + compose blocked on it; +* the audit entry and the "stack started" notification went out with + ``rc=None`` before anything had finished, as a success. + +Here the run is its own task. It holds the lock and drains the output until +compose exits, then records the real exit code. A socket only *watches*: it can +leave at any time, and one that connects while the same operation is already in +flight on that stack is attached to it (backlog first, then live) instead of +being turned away or starting a second compose. +""" +from __future__ import annotations + +import asyncio +import collections +import logging +from collections.abc import AsyncIterator, Awaitable, Callable +from dataclasses import dataclass, field +from datetime import timedelta +from typing import Optional + +from sqlmodel import Session + +from database import engine +from services import stack_lock_service + +logger = logging.getLogger("stackpilot.compose_run") + +#: Lines kept for a socket that attaches to a run already in progress. +BACKLOG_LINES = 2000 + +#: A streamed run has no timeout of its own (a large pull legitimately takes a +#: while), so its lock may live longer than a REST call's. The unlock endpoint +#: is the way out of one that hangs. +LOCK_TTL = timedelta(hours=2) + +#: (kind, payload): ("log", line) or ("done", returncode). +Item = tuple[str, object] + + +@dataclass +class ComposeRun: + stack_id: str + action: str + owner: str + returncode: Optional[int] = None + finished: bool = False + backlog: collections.deque = field( + default_factory=lambda: collections.deque(maxlen=BACKLOG_LINES) + ) + _subscribers: list[asyncio.Queue] = field(default_factory=list) + + def subscribe(self) -> asyncio.Queue: + """A queue that gets the backlog, every later item, then ``None``.""" + queue: asyncio.Queue = asyncio.Queue() + for line in self.backlog: + queue.put_nowait(("log", line)) + if self.finished: + queue.put_nowait(("done", self.returncode)) + queue.put_nowait(None) + else: + self._subscribers.append(queue) + return queue + + def unsubscribe(self, queue: asyncio.Queue) -> None: + if queue in self._subscribers: + self._subscribers.remove(queue) + + def _publish(self, item: Optional[Item]) -> None: + for queue in list(self._subscribers): + queue.put_nowait(item) + + +_runs: dict[str, ComposeRun] = {} +#: asyncio keeps only weak references to tasks; these must not be collected +#: while compose is still running. +_tasks: set[asyncio.Task] = set() + + +class RunBusy(Exception): + """The stack is held by a different operation.""" + + def __init__(self, stack_id: str, action: str): + self.stack_id = stack_id + self.action = action + super().__init__(f"Stack '{stack_id}' is busy: {action} in progress") + + +def get(stack_id: str) -> Optional[ComposeRun]: + return _runs.get(stack_id) + + +def start( + stack_id: str, + action: str, + owner: str, + stream: Callable[[], AsyncIterator[Item]], + on_finish: Optional[Callable[[Optional[int]], Awaitable[None]]] = None, +) -> tuple[ComposeRun, asyncio.Queue, bool]: + """Start ``stream`` under the stack lock, or attach to the same run in flight. + + Returns ``(run, queue, attached)``. Raises :class:`RunBusy` when another + operation holds the stack. Must be called from the event loop; the queue is + subscribed before the task gets to run, so not a line is missed. + """ + current = _runs.get(stack_id) + if current is not None and not current.finished: + if current.action != action: + raise RunBusy(stack_id, current.action) + return current, current.subscribe(), True + + with Session(engine) as session: + try: + stack_lock_service.acquire(session, stack_id, action, owner, ttl=LOCK_TTL) + except stack_lock_service.StackBusy as exc: + raise RunBusy(stack_id, exc.action) from exc + + run = ComposeRun(stack_id=stack_id, action=action, owner=owner) + _runs[stack_id] = run + queue = run.subscribe() + task = asyncio.create_task(_drive(run, stream, on_finish), name=f"compose-{action}-{stack_id}") + _tasks.add(task) + task.add_done_callback(_tasks.discard) + return run, queue, False + + +async def _drive( + run: ComposeRun, + stream: Callable[[], AsyncIterator[Item]], + on_finish: Optional[Callable[[Optional[int]], Awaitable[None]]], +) -> None: + rc: Optional[int] = None + try: + async for kind, payload in stream(): + if kind == "log": + run.backlog.append(payload) + run._publish(("log", payload)) + else: + rc = payload # type: ignore[assignment] + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - reported to whoever is watching + logger.warning("compose %s of '%s' failed: %s", run.action, run.stack_id, exc) + line = f"error: {exc}" + run.backlog.append(line) + run._publish(("log", line)) + rc = rc if rc is not None else -1 + finally: + run.returncode = rc + run.finished = True + try: + with Session(engine) as session: + stack_lock_service.release(session, run.stack_id) + except Exception: # noqa: BLE001 - never mask the outcome + logger.exception("Failed to release the lock on '%s'", run.stack_id) + run._publish(("done", rc)) + run._publish(None) + if _runs.get(run.stack_id) is run: + del _runs[run.stack_id] + if on_finish is not None: + try: + await on_finish(rc) + except Exception: # noqa: BLE001 - audit/notify are best-effort + logger.exception("Post-run hook for '%s' failed", run.stack_id) diff --git a/backend/services/compose_service.py b/backend/services/compose_service.py index 237ac18..0bc16d5 100644 --- a/backend/services/compose_service.py +++ b/backend/services/compose_service.py @@ -9,6 +9,7 @@ import asyncio import os import re import shutil +import stat import tempfile from dataclasses import dataclass, field from datetime import datetime, timezone @@ -33,6 +34,42 @@ class StackFileError(Exception): # --------------------------------------------------------------------------- # +_PROJECT_INVALID_RE = re.compile(r"[^a-z0-9_-]+") + + +def project_name(stack_id: str) -> str: + """The compose project name for a stack. + + The same normalisation compose applies to a directory name when it picks + the project name itself (lowercase, only ``a-z0-9_-``, no leading ``-``/``_``). + For every stack StackPilot creates that is the id unchanged. A folder found + on disk as ``Nextcloud`` or ``my.app`` used to be passed as ``-p Nextcloud``, + which compose refuses — so every action on such a stack failed, while + ``docker compose up`` in that folder worked. + """ + return _PROJECT_INVALID_RE.sub("", stack_id.lower()).lstrip("_-") or "stack" + + +def stack_ids_by_project(stack_ids) -> dict[str, str]: + """``{project: stack_id}``, to key per-project Docker data by stack id.""" + return {project_name(sid): sid for sid in stack_ids} + + +def stack_ids_by_project_in(session) -> dict[str, str]: + """:func:`stack_ids_by_project` for every stack in the database.""" + from sqlmodel import select + + from models.stack import Stack + + return stack_ids_by_project(s.id for s in session.exec(select(Stack)).all()) + + +def owner_stack(project: Optional[str], ids: dict[str, str]) -> Optional[str]: + """The stack id behind a ``com.docker.compose.project`` label, else the + label itself (a project StackPilot does not manage).""" + return ids.get(project, project) if project else project + + def slugify(name: str) -> str: slug = re.sub(r"[^a-z0-9_-]+", "-", name.strip().lower()) slug = re.sub(r"-{2,}", "-", slug).strip("-_") @@ -85,6 +122,57 @@ def read_env(stack_id: str, override: Optional[str] = None) -> str: return fh.read() +#: Name parts that mark a value as a credential (DB_PASSWORD, JWT_SECRET, +#: CF_API_KEY, TS_AUTHKEY …), matched per ``_``/``.``/``-`` separated part so +#: that KEYCLOAK_ADMIN or a "monkey" service stay readable. +_SECRET_PARTS = { + "pass", "passwd", "password", "passphrase", "pwd", "secret", "token", "key", + "apikey", "authkey", "credential", "credentials", "private", +} +#: ``KEY: value`` and ``- KEY=value``, keeping indentation and quotes. +_MAPPING_LINE = re.compile(r"^(\s*)([\"']?)([A-Za-z0-9_.-]+)\2(\s*:\s+)(.+?)\s*$") +_LIST_LINE = re.compile(r"^(\s*-\s*)([\"']?)([A-Za-z0-9_.-]+)=(.*?)\2\s*$") +#: ``${VAR}``, ``$VAR`` or ``${VAR:?message}`` — but not ``${VAR:-default}``, +#: whose default is a value like any other. +_ONLY_A_VARIABLE = re.compile( + r"^[\"']?\$(?:\{[A-Za-z_][A-Za-z0-9_]*(?::?\?[^}]*)?\}|[A-Za-z_][A-Za-z0-9_]*)[\"']?$" +) +MASK = "••••••" + + +def _secret_name(name: str) -> bool: + return any(part in _SECRET_PARTS for part in re.split(r"[_.-]", name.lower())) + + +def mask_secrets(compose: str) -> str: + """The compose file with credential values replaced by ``••••••``. + + For the read-only role, which is not given the ``.env`` — but plenty of + compose files carry the password inline (``POSTGRES_PASSWORD: hunter2``), + and the Compose tab showed it. A reference to a variable (``${DB_PASS}``) + is not a secret and stays. Line-based on purpose: the file keeps its + layout and comments, and a file that does not parse is masked all the same. + """ + out = [] + for line in compose.splitlines(keepends=True): + body = line.rstrip("\r\n") + ending = line[len(body):] + m = _LIST_LINE.match(body) + if m and _secret_name(m.group(3)) and m.group(4) and not _ONLY_A_VARIABLE.match(m.group(4)): + body = f"{m.group(1)}{m.group(2)}{m.group(3)}={MASK}{m.group(2)}" + else: + m = _MAPPING_LINE.match(body) + if ( + m + and _secret_name(m.group(3)) + and not m.group(5).startswith(("|", ">", "{", "[", "&", "*", "#")) + and not _ONLY_A_VARIABLE.match(m.group(5)) + ): + body = f"{m.group(1)}{m.group(2)}{m.group(3)}{m.group(2)}{m.group(4)}{MASK}" + out.append(body + ending) + return "".join(out) + + def write_compose(stack_id: str, content: str, override: Optional[str] = None) -> None: directory = stack_dir(stack_id, override) os.makedirs(directory, exist_ok=True) @@ -106,10 +194,71 @@ def write_env(stack_id: str, content: str, override: Optional[str] = None) -> No fh.write(content) -def delete_stack_files(stack_id: str, override: Optional[str] = None) -> None: +def _mount_points() -> set[str]: + """Every mount point this process sees (from /proc/self/mountinfo).""" + points: set[str] = set() + try: + with open("/proc/self/mountinfo", "r", encoding="utf-8") as fh: + for line in fh: + fields = line.split() + if len(fields) >= 5: + # Spaces etc. are octal-escaped in mountinfo. + points.add(fields[4].encode().decode("unicode_escape")) + except OSError: + pass + return points + + +def delete_stack_files(stack_id: str, override: Optional[str] = None) -> list[str]: + """Remove the stack folder, but never what is mounted inside it. + + A stack folder can have a NAS share or a second disk mounted in it (a media + library at ``./media``). ``rmtree`` would have walked straight into it and + deleted the share's contents along with the compose file. Directories on + another filesystem, or that are mount points of their own (a bind mount of + the same disk), are left exactly as they are. Returns those paths. + """ directory = stack_dir(stack_id, override) - if os.path.isdir(directory): - shutil.rmtree(directory) + if not os.path.isdir(directory): + return [] + mounts = _mount_points() + root_dev = os.lstat(directory).st_dev + kept: list[str] = [] + + def remove(path: str) -> None: + with os.scandir(path) as it: + entries = list(it) + for entry in entries: + if entry.is_dir(follow_symlinks=False): + if ( + entry.stat(follow_symlinks=False).st_dev != root_dev + or entry.path in mounts + ): + kept.append(entry.path) + continue + remove(entry.path) + else: + os.unlink(entry.path) + try: + os.rmdir(path) + except OSError: + pass # still holds something that was kept + + remove(directory) + return kept + + +def _skip_special_files(directory: str, names: list[str]) -> list[str]: + """copytree ignore hook: sockets, FIFOs and devices cannot be copied.""" + skipped = [] + for name in names: + try: + mode = os.lstat(os.path.join(directory, name)).st_mode + except OSError: + continue + if not (stat.S_ISDIR(mode) or stat.S_ISREG(mode) or stat.S_ISLNK(mode)): + skipped.append(name) + return skipped def clone_stack_files(src_id: str, dst_id: str, override: Optional[str] = None) -> None: @@ -117,7 +266,42 @@ def clone_stack_files(src_id: str, dst_id: str, override: Optional[str] = None) dst = stack_dir(dst_id, override) if os.path.isdir(dst): raise StackFileError(f"Target stack '{dst_id}' already exists") - shutil.copytree(src, dst) + try: + # symlinks=True copies links as links: following them would duplicate + # whatever they point at (or fail on a dangling one). A container's + # runtime socket in a data folder would fail the whole copy. + shutil.copytree(src, dst, symlinks=True, ignore=_skip_special_files) + except (OSError, shutil.Error) as exc: + shutil.rmtree(dst, ignore_errors=True) + raise StackFileError(f"Could not copy the stack folder: {exc}") from exc + + +def force_remove_project(stack_id: str) -> int: + """Remove a project's containers and networks through the Docker API. + + The fallback for deleting a stack whose ``compose down`` failed — typically + because the compose file no longer parses. Without it the containers kept + running with nothing left in StackPilot to stop them. Returns how many + containers went. + """ + client = get_client() + label = f"{COMPOSE_LABEL}={project_name(stack_id)}" + removed = 0 + for container in safe_call(client.containers.list, all=True, filters={"label": label}): + try: + container.remove(force=True) + removed += 1 + except Exception: # noqa: BLE001 - best effort, report what went + pass + try: + for network in safe_call(client.networks.list, filters={"label": label}): + try: + network.remove() + except Exception: # noqa: BLE001 + pass + except DockerError: + pass + return removed def discover_stacks(override: Optional[str] = None) -> list[str]: @@ -178,7 +362,7 @@ def containers_for_stack(stack_id: str) -> list[ContainerInfo]: raw = safe_call( client.containers.list, all=True, - filters={"label": f"{COMPOSE_LABEL}={stack_id}"}, + filters={"label": f"{COMPOSE_LABEL}={project_name(stack_id)}"}, ) result = [] for c in raw: @@ -272,7 +456,7 @@ def _compose_base_cmd(stack_id: str, override: Optional[str] = None) -> list[str "docker", "compose", "-p", - stack_id, + project_name(stack_id), "--project-directory", directory, "-f", @@ -298,6 +482,7 @@ async def run_compose( stdout_b, stderr_b = await asyncio.wait_for(proc.communicate(), timeout=timeout) except asyncio.TimeoutError as exc: proc.kill() + await proc.wait() raise StackFileError(f"compose command timed out: {' '.join(args)}") from exc return { "returncode": proc.returncode, @@ -307,42 +492,69 @@ async def run_compose( } -async def validate_yaml(content: str, env_content: str = "") -> dict: - """Validate compose YAML by running ``docker compose config -q`` against it - in a throwaway directory. Returns ``{"ok": bool, "errors": str}`` — the - errors string carries compose's own diagnostics (parse errors, unknown - keys, bad interpolation) so the editor can surface them before saving.""" +async def validate_yaml( + content: str, env_content: str = "", project_dir: Optional[str] = None +) -> dict: + """Validate compose YAML by running ``docker compose config -q`` against it. + Returns ``{"ok": bool, "errors": str}`` — the errors string carries + compose's own diagnostics (parse errors, unknown keys, bad interpolation) + so the editor can surface them before saving. + + With ``project_dir`` (an existing stack's folder) relative paths resolve + against that folder, as they will on deploy. Validated from a bare temp + directory, a stack using ``env_file: ./app.env`` — or an ``include``, a + ``secrets: file:`` — was reported broken although it deploys fine. The + edited ``.env`` is still the one used for interpolation. + """ with tempfile.TemporaryDirectory(prefix="sp-validate-") as tmp: compose_file = os.path.join(tmp, DEFAULT_COMPOSE_NAME) with open(compose_file, "w", encoding="utf-8") as fh: fh.write(content) - if env_content: - with open(os.path.join(tmp, ".env"), "w", encoding="utf-8") as fh: - fh.write(env_content) + env_file = os.path.join(tmp, ".env") + with open(env_file, "w", encoding="utf-8") as fh: + fh.write(env_content or "") + directory = project_dir if project_dir and os.path.isdir(project_dir) else tmp cmd = [ "docker", "compose", - "--project-directory", tmp, + "--project-directory", directory, "-f", compose_file, + "--env-file", env_file, "config", "-q", ] + if directory != tmp: + # The folder's name may not be a valid project name; config does + # not care what the project is called. + cmd[2:2] = ["-p", "sp-validate"] + proc = await asyncio.create_subprocess_exec( + *cmd, + env=registry_service.cli_env(), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) try: - proc = await asyncio.create_subprocess_exec( - *cmd, - env=registry_service.cli_env(), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) _out, err = await asyncio.wait_for(proc.communicate(), timeout=30.0) except asyncio.TimeoutError as exc: + proc.kill() + await proc.wait() raise StackFileError("compose config validation timed out") from exc ok = proc.returncode == 0 - return {"ok": ok, "errors": "" if ok else err.decode("utf-8", "replace").strip()} + errors = "" if ok else err.decode("utf-8", "replace").strip() + # Name the file the user is editing, not our scratch copy of it. + errors = errors.replace(compose_file, DEFAULT_COMPOSE_NAME).replace(env_file, ".env") + errors = errors.replace(tmp + os.sep, "./") + return {"ok": ok, "errors": errors} async def stream_compose( stack_id: str, args: list[str], override: Optional[str] = None ): - """Yield lines from a `docker compose` subcommand as they are produced.""" + """Yield lines from a `docker compose` subcommand as they are produced. + + The subprocess is killed when the consumer stops early (the generator is + closed or its task cancelled). For ``logs -f`` that is the only way it ever + ends: without this every log view that was closed left a ``compose logs`` + process behind for as long as the containers lived. + """ cmd = _compose_base_cmd(stack_id, override) + args proc = await asyncio.create_subprocess_exec( *cmd, @@ -351,9 +563,22 @@ async def stream_compose( stderr=asyncio.subprocess.STDOUT, ) assert proc.stdout is not None - async for raw in proc.stdout: - yield raw.decode("utf-8", "replace").rstrip("\n") - await proc.wait() + try: + async for raw in proc.stdout: + yield raw.decode("utf-8", "replace").rstrip("\n") + await proc.wait() + finally: + if proc.returncode is None: + try: + proc.kill() + except ProcessLookupError: + pass + # Reap it, so it does not linger as a zombie. Shielded: this runs + # while the consuming task is being cancelled. + try: + await asyncio.shield(proc.wait()) + except (asyncio.CancelledError, ProcessLookupError): + pass _json_progress: Optional[bool] = None @@ -468,8 +693,15 @@ async def pull(stack_id: str, override: Optional[str] = None) -> dict: async def update(stack_id: str, override: Optional[str] = None) -> dict: - """Pull then up -d.""" + """Pull then up -d. + + A failed pull stops here, like the streamed update does. Running ``up`` + anyway recreated nothing new — the old images were still there — and its + exit code 0 then reported the update as a success. + """ pull_res = await run_compose(stack_id, ["pull"], override) + if pull_res["returncode"] != 0: + return {**pull_res, "command": "pull + up -d"} up_res = await run_compose(stack_id, ["up", "-d", "--remove-orphans"], override) return { "returncode": up_res["returncode"], @@ -487,7 +719,8 @@ async def logs( ) -> dict: args = ["logs", "--no-color", "--tail", str(tail), "--timestamps"] if service: - args.append(service) + # "--" so a service name can never be read as a compose option. + args += ["--", service] return await run_compose(stack_id, args, override, timeout=60.0) diff --git a/backend/services/convert_service.py b/backend/services/convert_service.py index f4f3ba1..1065bc7 100644 --- a/backend/services/convert_service.py +++ b/backend/services/convert_service.py @@ -1,135 +1,357 @@ -"""Convert a `docker run ...` command string into a Compose YAML fragment.""" +"""Convert a `docker run ...` command string into a Compose YAML fragment. + +The result has to be a compose file that ``docker compose config`` accepts and +that means what the command meant. Three things got that wrong before: + +* every flag the converter did not know was assumed to take a value, so a + boolean like ``--read-only`` or ``-P`` swallowed the image name, and + ``-dit ubuntu bash`` came out as a service running the image ``bash``; +* ``--network host`` became ``networks: [host]`` and ``-v data:/data`` a + reference to a volume nobody declared — both of which compose rejects; +* flags with real effect (``--gpus``, ``--mount``, ``--entrypoint``, + ``--cap-drop``, ``--shm-size`` …) were dropped without a word. + +``docker run``'s boolean flags are a short, closed list; everything else takes a +value. That is what the parser goes by now. +""" from __future__ import annotations +import re import shlex import yaml +#: docker run flags that take no value. +_BOOL_LONG = { + "--detach", "--interactive", "--tty", "--rm", "--init", "--privileged", + "--read-only", "--publish-all", "--oom-kill-disable", "--no-healthcheck", + "--disable-content-trust", "--quiet", "--sig-proxy", "--use-api-socket", +} +#: Single-letter booleans, which may be bundled ("-dit"). +_BOOL_SHORT = set("ditPq") +#: Single-letter flags that take a value ("-p 80:80", "-p80:80"). +_VALUE_SHORT = { + "p": "--publish", "v": "--volume", "e": "--env", "l": "--label", + "u": "--user", "w": "--workdir", "h": "--hostname", "m": "--memory", + "a": "--attach", "c": "--cpu-shares", +} +_SHORT_TO_LONG = {"d": "--detach", "i": "--interactive", "t": "--tty", + "P": "--publish-all", "q": "--quiet"} + +#: network_mode values, as opposed to networks a service joins. +_NETWORK_MODES = {"host", "none", "bridge"} + +_SERVICE_NAME_RE = re.compile(r"[^a-zA-Z0-9._-]+") + def _service_name_from_image(image: str) -> str: - name = image.split("/")[-1].split(":")[0] - return name or "app" + name = image.split("@", 1)[0].split("/")[-1].split(":")[0] + return _SERVICE_NAME_RE.sub("-", name).strip("-.") or "app" + + +#: A line continuation whose newline is already gone. Pasted into a one-line +#: field, backslash-newline arrives as a backslash followed by the next line's +#: indentation, which shlex read as an escaped space: a lone " " token that +#: became the image. A backslash followed by an option, or by more than one +#: blank, is such a continuation; a single escaped space inside a word +#: (``My\ Files``) is left alone. +_FLATTENED_CONTINUATION = re.compile(r"\\[ \t]+(?=-)|\\[ \t]{2,}") + + +def _tokens(command: str) -> list[str]: + # Shell line continuations, as pasted from a README. + command = command.replace("\\\r\n", " ").replace("\\\n", " ") + tokens = shlex.split(_FLATTENED_CONTINUATION.sub(" ", command)) + # Drop leading "sudo docker run" / "docker container run". + while tokens and tokens[0] in ("docker", "run", "container", "sudo", "podman"): + tokens.pop(0) + return tokens + + +def _parse(tokens: list[str]) -> tuple[list[tuple[str, str | None]], str | None, list[str]]: + """Split into ([(long_flag, value)], image, command).""" + flags: list[tuple[str, str | None]] = [] + i, n = 0, len(tokens) + while i < n: + tok = tokens[i] + if tok == "--": + i += 1 + break + if not tok.startswith("-") or tok == "-": + break # the image + if tok.startswith("--"): + name, eq, value = tok.partition("=") + if eq: + flags.append((name, value)) + i += 1 + elif name in _BOOL_LONG: + flags.append((name, None)) + i += 1 + else: + flags.append((name, tokens[i + 1] if i + 1 < n else "")) + i += 2 + continue + # Short form: bundled booleans ("-dit"), or one value flag with its + # value attached ("-p8080:80") or following ("-p 8080:80"). + body = tok[1:] + j = 0 + while j < len(body) and body[j] in _BOOL_SHORT: + flags.append((_SHORT_TO_LONG[body[j]], None)) + j += 1 + if j == len(body): + i += 1 + continue + letter = body[j] + long_name = _VALUE_SHORT.get(letter, f"-{letter}") + attached = body[j + 1 :] + if attached.startswith("="): + attached = attached[1:] + if attached: + flags.append((long_name, attached)) + i += 1 + else: + flags.append((long_name, tokens[i + 1] if i + 1 < n else "")) + i += 2 + image = tokens[i] if i < n else None + command = tokens[i + 1 :] if image is not None else [] + return flags, image, command + + +def _is_named_volume(source: str) -> bool: + return bool(source) and not source.startswith(("/", ".", "~", "$")) + + +def _mount(value: str) -> dict: + """``--mount type=bind,source=/a,target=/b,readonly`` as compose long syntax.""" + spec: dict = {} + for part in value.split(","): + key, eq, val = part.partition("=") + key = key.strip() + if key in ("source", "src"): + spec["source"] = val + elif key in ("target", "destination", "dst"): + spec["target"] = val + elif key == "type": + spec["type"] = val + elif key in ("readonly", "ro"): + spec["read_only"] = (not eq) or val.lower() in ("1", "true") + elif key == "bind-propagation": + spec.setdefault("bind", {})["propagation"] = val + elif key == "volume-nocopy": + spec.setdefault("volume", {})["nocopy"] = (not eq) or val.lower() in ("1", "true") + elif key == "tmpfs-size": + spec.setdefault("tmpfs", {})["size"] = val + spec.setdefault("type", "volume") + return spec + + +def _gpus(value: str) -> dict: + """``--gpus all`` / ``--gpus 2`` / ``--gpus device=0,1`` as a device reservation.""" + reservation: dict = {"driver": "nvidia", "capabilities": ["gpu"]} + value = value.strip().strip('"').strip("'") + if value.startswith("device="): + reservation["device_ids"] = [d for d in value[len("device="):].split(",") if d] + elif value.isdigit(): + reservation["count"] = int(value) + else: + reservation["count"] = "all" + return reservation + + +def _ulimit(value: str, ulimits: dict) -> None: + name, _, limits = value.partition("=") + soft, _, hard = limits.partition(":") + try: + ulimits[name] = ( + {"soft": int(soft), "hard": int(hard)} if hard else int(soft) + ) + except ValueError: + pass def convert_docker_run(command: str) -> str: - tokens = shlex.split(command) - # Drop leading "docker run" / "docker container run". - while tokens and tokens[0] in ("docker", "run", "container", "sudo"): - tokens.pop(0) - - service: dict = {} - name = None - image = None - ports: list[str] = [] - volumes: list[str] = [] - environment: list[str] = [] - env_file: list[str] = [] - devices: list[str] = [] - cap_add: list[str] = [] - labels: list[str] = [] - networks: list[str] = [] - command_args: list[str] = [] - - i = 0 - n = len(tokens) - - def take_value(idx: int, tok: str): - if "=" in tok and tok.startswith("--") and not tok.endswith("="): - return tok.split("=", 1)[1], idx + 1 - return tokens[idx + 1], idx + 2 - - while i < n: - tok = tokens[i] - if image is not None: - # Everything after the image is the container command. - command_args = tokens[i:] - break - if not tok.startswith("-"): - image = tok - i += 1 - continue - - flag = tok.split("=", 1)[0] - if flag in ("-d", "--detach", "-i", "--interactive", "-t", "--tty", "--rm", "--init"): - i += 1 - elif flag in ("--name",): - name, i = take_value(i, tok) - elif flag in ("-p", "--publish"): - val, i = take_value(i, tok) - ports.append(val) - elif flag in ("-v", "--volume", "--mount"): - val, i = take_value(i, tok) - volumes.append(val) - elif flag in ("-e", "--env"): - val, i = take_value(i, tok) - environment.append(val) - elif flag in ("--env-file",): - val, i = take_value(i, tok) - env_file.append(val) - elif flag in ("--device",): - val, i = take_value(i, tok) - devices.append(val) - elif flag in ("--cap-add",): - val, i = take_value(i, tok) - cap_add.append(val) - elif flag in ("-l", "--label"): - val, i = take_value(i, tok) - labels.append(val) - elif flag in ("--network", "--net"): - val, i = take_value(i, tok) - networks.append(val) - elif flag in ("--restart",): - val, i = take_value(i, tok) - service["restart"] = val - elif flag in ("--privileged",): - service["privileged"] = True - i += 1 - elif flag in ("--hostname", "-h"): - val, i = take_value(i, tok) - service["hostname"] = val - elif flag in ("-u", "--user"): - val, i = take_value(i, tok) - service["user"] = val - elif flag in ("-w", "--workdir"): - val, i = take_value(i, tok) - service["working_dir"] = val - else: - # Unknown flag — try to consume a value if it looks like it takes one. - if "=" in tok: - i += 1 - elif i + 1 < n and not tokens[i + 1].startswith("-"): - i += 2 - else: - i += 1 - + tokens = _tokens(command) + flags, image, command_args = _parse(tokens) if not image: raise ValueError("Could not find an image in the docker run command") - service["image"] = image - if "restart" not in service: - service["restart"] = "unless-stopped" - if ports: - service["ports"] = ports - if volumes: - service["volumes"] = volumes - if environment: - service["environment"] = environment - if env_file: - service["env_file"] = env_file - if devices: - service["devices"] = devices - if cap_add: - service["cap_add"] = cap_add - if labels: - service["labels"] = labels - if networks: - service["networks"] = networks + service: dict = {"image": image} + top_volumes: dict = {} + top_networks: dict = {} + name = None + lists: dict[str, list] = {} + healthcheck: dict = {} + logging: dict = {} + ulimits: dict = {} + gpus: list[dict] = [] + + def add(key: str, value) -> None: + lists.setdefault(key, []).append(value) + + for flag, value in flags: + value = value if value is not None else "" + if flag == "--name": + name = value + elif flag == "--publish": + add("ports", value) + elif flag == "--volume": + add("volumes", value) + source = value.split(":", 1)[0] if ":" in value else "" + if _is_named_volume(source): + top_volumes[source] = None + elif flag == "--mount": + spec = _mount(value) + add("volumes", spec) + if spec.get("type") == "volume" and _is_named_volume(spec.get("source", "")): + top_volumes[spec["source"]] = None + elif flag == "--tmpfs": + add("tmpfs", value) + elif flag == "--env": + add("environment", value) + elif flag == "--env-file": + add("env_file", value) + elif flag == "--device": + add("devices", value) + elif flag == "--cap-add": + add("cap_add", value) + elif flag == "--cap-drop": + add("cap_drop", value) + elif flag == "--label": + add("labels", value) + elif flag in ("--network", "--net"): + if value in _NETWORK_MODES or value.startswith(("container:", "service:")): + service["network_mode"] = value + elif value: + add("networks", value) + # `docker run --network x` needs x to exist already. + top_networks[value] = {"external": True} + elif flag == "--network-alias": + add("_aliases", value) + elif flag == "--add-host": + add("extra_hosts", value) + elif flag == "--dns": + add("dns", value) + elif flag == "--dns-search": + add("dns_search", value) + elif flag == "--security-opt": + add("security_opt", value) + elif flag == "--sysctl": + add("sysctls", value) + elif flag == "--group-add": + add("group_add", value) + elif flag == "--expose": + add("expose", value) + elif flag == "--gpus": + gpus.append(_gpus(value)) + elif flag == "--ulimit": + _ulimit(value, ulimits) + elif flag == "--restart": + service["restart"] = value + elif flag == "--privileged": + service["privileged"] = True + elif flag == "--read-only": + service["read_only"] = True + elif flag == "--init": + service["init"] = True + elif flag == "--interactive": + service["stdin_open"] = True + elif flag == "--tty": + service["tty"] = True + elif flag == "--hostname": + service["hostname"] = value + elif flag == "--user": + service["user"] = value + elif flag == "--workdir": + service["working_dir"] = value + elif flag == "--entrypoint": + service["entrypoint"] = value + elif flag == "--shm-size": + service["shm_size"] = value + elif flag == "--memory": + service["mem_limit"] = value + elif flag == "--memory-reservation": + service["mem_reservation"] = value + elif flag == "--cpus": + try: + service["cpus"] = float(value) + except ValueError: + pass + elif flag == "--cpu-shares": + try: + service["cpu_shares"] = int(value) + except ValueError: + pass + elif flag == "--pids-limit": + try: + service["pids_limit"] = int(value) + except ValueError: + pass + elif flag == "--pid": + service["pid"] = value + elif flag == "--ipc": + service["ipc"] = value + elif flag == "--stop-signal": + service["stop_signal"] = value + elif flag in ("--stop-timeout", "--time"): + service["stop_grace_period"] = f"{value}s" if value.isdigit() else value + elif flag == "--platform": + service["platform"] = value + elif flag == "--pull": + service["pull_policy"] = value + elif flag == "--log-driver": + logging["driver"] = value + elif flag == "--log-opt": + key, _, val = value.partition("=") + logging.setdefault("options", {})[key] = val + elif flag == "--health-cmd": + healthcheck["test"] = ["CMD-SHELL", value] + elif flag == "--health-interval": + healthcheck["interval"] = value + elif flag == "--health-timeout": + healthcheck["timeout"] = value + elif flag == "--health-start-period": + healthcheck["start_period"] = value + elif flag == "--health-retries": + try: + healthcheck["retries"] = int(value) + except ValueError: + pass + elif flag == "--no-healthcheck": + healthcheck = {"disable": True} + # Everything else (--detach, --rm, --publish-all, --cidfile, …) has no + # compose equivalent worth writing down. + + aliases = lists.pop("_aliases", []) + if aliases: + nets = lists.get("networks") or [] + if nets: + # Aliases apply to the networks given on the command line. + lists["networks"] = {net: {"aliases": aliases} for net in nets} + service.setdefault("restart", "unless-stopped") + for key in ( + "ports", "expose", "volumes", "tmpfs", "environment", "env_file", "devices", + "cap_add", "cap_drop", "security_opt", "sysctls", "group_add", "extra_hosts", + "dns", "dns_search", "labels", "networks", + ): + if key in lists: + service[key] = lists[key] + if ulimits: + service["ulimits"] = ulimits + if logging: + service["logging"] = logging + if healthcheck: + service["healthcheck"] = healthcheck + if gpus: + service["deploy"] = {"resources": {"reservations": {"devices": gpus}}} if command_args: service["command"] = command_args - - svc_name = name or _service_name_from_image(image) if name: service["container_name"] = name - doc = {"services": {svc_name: service}} + svc_name = _SERVICE_NAME_RE.sub("-", name).strip("-.") if name else "" + doc: dict = {"services": {svc_name or _service_name_from_image(image): service}} + if top_volumes: + doc["volumes"] = top_volumes + if top_networks: + doc["networks"] = top_networks return yaml.safe_dump(doc, sort_keys=False, default_flow_style=False) diff --git a/backend/services/dashboard_service.py b/backend/services/dashboard_service.py index 404f505..debeaba 100644 --- a/backend/services/dashboard_service.py +++ b/backend/services/dashboard_service.py @@ -132,7 +132,7 @@ def _local_host() -> tuple[dict, list[dict]]: unhealthy: list[str] = [] updates = 0 for stack_id in discovered: - containers = by_project.get(stack_id, []) + containers = by_project.get(compose_service.project_name(stack_id), []) status = compose_service._status_from_states([c.get("State", "") for c in containers]) statuses.append(status) if status == "running" and not _is_healthy(containers): diff --git a/backend/services/device_service.py b/backend/services/device_service.py index ce1bb13..dc9b700 100644 --- a/backend/services/device_service.py +++ b/backend/services/device_service.py @@ -118,12 +118,61 @@ def _is_allowed(path: str) -> bool: return False +def _toward_roots(path: str) -> list[dict]: + """For a path *above* the allowed roots: the folders below it that lead to + one (``/`` -> ``mnt``, ``opt``; ``/srv`` -> ``data`` for ``/srv/data``). + + The browser opens at ``/``, which the default roots do not include, and the + breadcrumbs and "up" lead there too — so the page used to open on an error + with no way into any root at all. + """ + prefix = "/" if path == "/" else path + os.sep + names: set[str] = set() + for root in settings.ALLOWED_BROWSE_ROOTS: + root = os.path.normpath(root) + if not root.startswith(prefix): + continue + try: + if not os.path.isdir(_real_root(root)): + continue + except BrowseError: + continue + names.add(root[len(prefix):].split(os.sep, 1)[0]) + entries = [] + for name in sorted(names, key=str.lower): + try: + st = os.stat(_real_root(os.path.join(path, name))) + except (OSError, BrowseError): + continue + entries.append( + { + "name": name, + "type": "dir", + "size": None, + "permissions": oct(st.st_mode & 0o777), + "mtime": st.st_mtime, + "symlink": False, + } + ) + return entries + + def browse(path: str = "/", show_hidden: bool = False) -> dict: path = os.path.normpath(path or "/") if not path.startswith("/"): raise BrowseError("Path must be absolute") if not _is_allowed(path): - raise BrowseError("Path is outside the allowed browse roots") + entries = _toward_roots(path) + if not entries: + raise BrowseError("Path is outside the allowed browse roots") + return { + "path": path, + "parent": os.path.dirname(path) if path != "/" else None, + "roots": settings.ALLOWED_BROWSE_ROOTS, + "entries": entries, + # Only a way through: nothing can be created or changed here. + "virtual": True, + } real = _real_root(path) if not os.path.isdir(real): diff --git a/backend/services/exec_service.py b/backend/services/exec_service.py index e68efd4..c609639 100644 --- a/backend/services/exec_service.py +++ b/backend/services/exec_service.py @@ -8,9 +8,12 @@ stdout/stderr stream), suitable for piping straight to xterm.js. from __future__ import annotations import asyncio +import codecs import contextlib import json +import queue import socket as _socket +import threading from starlette.websockets import WebSocketDisconnect @@ -71,22 +74,64 @@ async def pump_exec(websocket, exec_id: str, holder, raw) -> None: accepted as keystrokes). Container -> browser: ``{"type":"data","data":...}`` then a final ``{"type":"exit","code":...}``. - The blocking ``recv`` runs in the default executor; on teardown we shut the - socket down so that orphaned recv thread unblocks and exits. + The blocking ``recv`` and ``sendall`` run on two threads of this session's + own. They used to run on the event loop's default executor, where an open + terminal held a worker for its whole lifetime — shared with every + ``asyncio.to_thread`` in the app. On teardown the socket is shut down, which + is what unblocks the reader. """ raw.setblocking(True) - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() closed = asyncio.Event() + incoming: asyncio.Queue = asyncio.Queue() + outgoing: queue.Queue = queue.Queue() - async def to_browser() -> None: + def post(item) -> None: + try: + loop.call_soon_threadsafe(incoming.put_nowait, item) + except RuntimeError: # loop closed + pass + + def reader() -> None: try: while True: - data = await loop.run_in_executor(None, raw.recv, 4096) + data = raw.recv(4096) if not data: break - await websocket.send_text( - json.dumps({"type": "data", "data": data.decode("utf-8", "replace")}) - ) + post(data) + except Exception: # noqa: BLE001 - shutdown on teardown lands here + pass + finally: + post(None) + + def writer() -> None: + while True: + data = outgoing.get() + if data is None: + return + try: + raw.sendall(data) + except Exception: # noqa: BLE001 + return + + name = exec_id[:12] + threading.Thread(target=reader, name=f"exec-read-{name}", daemon=True).start() + threading.Thread(target=writer, name=f"exec-write-{name}", daemon=True).start() + + async def to_browser() -> None: + # Incremental: a multi-byte character (an umlaut, a box-drawing glyph in + # htop) split across two reads would otherwise turn into two U+FFFD. + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + try: + while True: + data = await incoming.get() + if data is None: + break + text = decoder.decode(data) + if text: + await websocket.send_text(json.dumps({"type": "data", "data": text})) + if tail := decoder.decode(b"", final=True): + await websocket.send_text(json.dumps({"type": "data", "data": tail})) except Exception: # noqa: BLE001 pass finally: @@ -105,9 +150,9 @@ async def pump_exec(websocket, exec_id: str, holder, raw) -> None: with contextlib.suppress(Exception): resize_exec(exec_id, int(obj.get("rows", 24)), int(obj.get("cols", 80))) elif isinstance(obj, dict) and "data" in obj: - await loop.run_in_executor(None, raw.sendall, str(obj["data"]).encode()) + outgoing.put(str(obj["data"]).encode()) else: - await loop.run_in_executor(None, raw.sendall, msg.encode()) + outgoing.put(msg.encode()) except WebSocketDisconnect: pass except Exception: # noqa: BLE001 @@ -119,6 +164,7 @@ async def pump_exec(websocket, exec_id: str, holder, raw) -> None: in_task = asyncio.create_task(from_browser()) await closed.wait() + outgoing.put(None) with contextlib.suppress(Exception): raw.shutdown(_socket.SHUT_RDWR) with contextlib.suppress(Exception): diff --git a/backend/services/file_service.py b/backend/services/file_service.py index 27becfb..b15cb53 100644 --- a/backend/services/file_service.py +++ b/backend/services/file_service.py @@ -124,6 +124,8 @@ def create_file(path: str, name: str) -> dict: def rename(path: str, new_name: str) -> dict: real = _safe_real(path) + if _is_root(path): + raise BrowseError("Refusing to rename a browse root") if not os.path.lexists(real): raise BrowseError(f"No such path: {path}") parent = os.path.dirname(path) or "/" @@ -138,10 +140,14 @@ def rename(path: str, new_name: str) -> dict: return {"path": dest} +def _is_root(path: str) -> bool: + norm = os.path.normpath(path) + return norm == "/" or norm in {os.path.normpath(r) for r in _root_paths()} + + def delete(path: str, recursive: bool = False) -> dict: real = _safe_real(path) - norm = os.path.normpath(path) - if norm == "/" or norm in {os.path.normpath(r) for r in _root_paths()}: + if _is_root(path): raise BrowseError("Refusing to delete a browse root") if not os.path.lexists(real): raise BrowseError(f"No such path: {path}") @@ -324,6 +330,10 @@ def _transfer_dest(src: str, dest_dir: str, overwrite: bool) -> tuple[str, str, name = os.path.basename(src.rstrip("/")) dest_logical = _child(dest_dir, name) dest_real = _safe_real(dest_logical) + # Overwriting removes the target first — which, for nested roots + # (/mnt and /mnt/user), could be a root. Same rule as delete. + if overwrite and _is_root(dest_logical): + raise BrowseError("Refusing to replace a browse root") # Refuse to copy/move a directory into itself or its own subtree. src_norm = os.path.normpath(src_real) @@ -350,6 +360,10 @@ def copy(src: str, dest_dir: str, overwrite: bool = False) -> dict: def move(src: str, dest_dir: str, overwrite: bool = False) -> dict: + # Moving a root away takes the whole tree with it — a delete by another + # name, and delete refuses roots. + if _is_root(src): + raise BrowseError("Refusing to move a browse root") src_real, dest_real, dest_logical = _transfer_dest(src, dest_dir, overwrite) try: if os.path.exists(dest_real) and overwrite: diff --git a/backend/services/git_service.py b/backend/services/git_service.py index 1c42832..1063928 100644 --- a/backend/services/git_service.py +++ b/backend/services/git_service.py @@ -53,6 +53,29 @@ class GitError(Exception): """A repository that cannot be reached, or a sync that cannot be completed.""" +#: What a branch name may look like here. git itself allows more, but not a +#: leading "-": as the refspec of `git fetch origin ` that would be read +#: as an option (`--upload-pack=…` runs a command). +_BRANCH_RE = re.compile(r"^[A-Za-z0-9._/][A-Za-z0-9._/+-]{0,199}$") + + +def validate_branch(branch: str) -> str: + branch = (branch or "").strip() + if not _BRANCH_RE.match(branch) or ".." in branch or branch.endswith((".lock", "/")): + raise GitError(f"Invalid branch name '{branch}'") + return branch + + +def validate_url(url: str) -> str: + url = (url or "").strip() + if not url or url.startswith("-") or any(c.isspace() for c in url): + raise GitError(f"Invalid repository URL '{url}'") + # git's ext:: and fd:: transports run commands or read file descriptors. + if url.lower().startswith(("ext::", "fd::")): + raise GitError("ext:: and fd:: repository URLs are not supported") + return url + + # --------------------------------------------------------------------------- # # Paths # --------------------------------------------------------------------------- # @@ -179,7 +202,8 @@ def _cleanup_auth(source: GitSource) -> None: async def _fetch(source: GitSource, env: dict, secret: Optional[str]) -> str: """Bring the cached clone to the tip of the configured branch. Returns the commit.""" directory = repo_dir(source.stack_id) - branch = source.branch or "main" + branch = validate_branch(source.branch or "main") + validate_url(source.url) os.makedirs(cache_root(), mode=0o700, exist_ok=True) if os.path.isdir(os.path.join(directory, ".git")): @@ -193,8 +217,10 @@ async def _fetch(source: GitSource, env: dict, secret: Optional[str]) -> str: shutil.rmtree(directory, ignore_errors=True) if not os.path.isdir(os.path.join(directory, ".git")): + # "--" ends the options: a URL that starts with "-" is a URL. await _git( - ["clone", "--depth", "1", "--branch", branch, source.url, directory], env, secret + ["clone", "--depth", "1", "--branch", branch, "--", source.url, directory], + env, secret, ) else: await _git(["-C", directory, "fetch", "--depth", "1", "origin", branch], env, secret) @@ -294,6 +320,12 @@ def _prune_empty_dirs(stack_dir: str, removed: list[str]) -> None: async def sync(session: Session, source: GitSource, actor: str = "system") -> SyncResult: """Fetch, copy into the stack, and deploy when something changed.""" + from models.stack import Stack + + if session.get(Stack, source.stack_id) is None: + # A source left behind by a deleted stack: syncing it would recreate + # the folder and, with auto deploy, start the stack again. + raise GitError(f"Stack '{source.stack_id}' no longer exists") env, secret = _auth_env(source) try: commit = await _fetch(source, env, secret) diff --git a/backend/services/gpu_service.py b/backend/services/gpu_service.py index 060fe3d..06395fe 100644 --- a/backend/services/gpu_service.py +++ b/backend/services/gpu_service.py @@ -8,6 +8,7 @@ from __future__ import annotations import glob import os +import re import subprocess from dataclasses import asdict, dataclass from typing import Literal, Optional @@ -83,6 +84,49 @@ def _detect_nvidia() -> list[GPUInfo]: return gpus +def _detect_nvidia_procfs() -> list[GPUInfo]: + """NVIDIA GPUs from the kernel driver's /proc entries. + + StackPilot's image has no ``nvidia-smi`` (and the container runs without + the NVIDIA runtime), and the DRI scan below skips NVIDIA on purpose — so on + a normal deployment an NVIDIA card was never detected at all. The driver + publishes every GPU under ``/proc/driver/nvidia/gpus//information``, + which the host /proc mount (``HOST_PROC_PATH``) makes readable here. + """ + from config import settings + + for proc in (settings.HOST_PROC_PATH, "/proc"): + base = os.path.join(proc, "driver", "nvidia", "gpus") + if not os.path.isdir(base): + continue + version = _read(os.path.join(proc, "driver", "nvidia", "version")) + match = re.search(r"Kernel Module\s+([\d.]+)", version) + driver = match.group(1) if match else "" + gpus: list[GPUInfo] = [] + for position, bus in enumerate(sorted(os.listdir(base))): + info: dict[str, str] = {} + for line in _read(os.path.join(base, bus, "information")).splitlines(): + key, sep, value = line.partition(":") + if sep: + info[key.strip()] = value.strip() + try: + index = int(info.get("Device Minor", position)) + except ValueError: + index = position + gpus.append( + GPUInfo( + vendor="nvidia", + index=index, + name=info.get("Model") or "NVIDIA GPU", + uuid=info.get("GPU UUID") or None, + driver=driver, + ) + ) + if gpus: + return sorted(gpus, key=lambda g: g.index) + return [] + + def _read(path: str) -> str: try: with open(path, "r", encoding="utf-8") as fh: @@ -151,7 +195,7 @@ def _detect_dri() -> list[GPUInfo]: def detect_gpus() -> list[GPUInfo]: - return _detect_nvidia() + _detect_dri() + return (_detect_nvidia() or _detect_nvidia_procfs()) + _detect_dri() def dri_group_gids() -> set[int]: diff --git a/backend/services/image_service.py b/backend/services/image_service.py index 234303a..9396f9e 100644 --- a/backend/services/image_service.py +++ b/backend/services/image_service.py @@ -14,23 +14,33 @@ COMPOSE_PROJECT_LABEL = "com.docker.compose.project" def list_images() -> list[dict]: - """Return one row per image tag, annotated with using-stacks + update status.""" + """Return one row per image tag, annotated with using-stacks + update status. + + Usage is matched on the image *id* the containers run, not on the name: + a compose file saying ``image: nginx`` or ``docker.io/library/nginx:1`` + produces containers whose configured image never equals the tag Docker + lists (``nginx:latest``, ``nginx:1``), so those images showed up as used + by no stack — and were left out of "scan all". + """ try: client = get_client() images = safe_call(client.images.list) - containers = safe_call(client.containers.list, all=True) + containers = safe_call(client.api.containers, all=True) except DockerError: return [] - # Map image ref -> stacks using it. - usage: dict[str, set[str]] = {} + # image id -> stacks using it, and the references they were created from + # (which is what the update checker caches its results under). + stacks_by_id: dict[str, set[str]] = {} + refs_by_id: dict[str, set[str]] = {} for c in containers: - ref = c.attrs.get("Config", {}).get("Image") - stack = c.labels.get(COMPOSE_PROJECT_LABEL) - if ref: - usage.setdefault(ref, set()) - if stack: - usage[ref].add(stack) + image_id = c.get("ImageID") or "" + stack = (c.get("Labels") or {}).get(COMPOSE_PROJECT_LABEL) + stacks_by_id.setdefault(image_id, set()) + if stack: + stacks_by_id[image_id].add(stack) + if (ref := c.get("Image")) and not ref.startswith("sha256:"): + refs_by_id.setdefault(image_id, set()).add(ref) cache = update_service.get_cache() result = [] @@ -38,15 +48,20 @@ def list_images() -> list[dict]: tags = img.tags or [] if not tags: continue + refs = refs_by_id.get(img.id, set()) for tag in tags: + update = cache.get(tag) + if update is None: + # Cached under the name the compose file used for it. + update = next((cache[r] for r in sorted(refs) if r in cache), None) result.append( { "id": img.short_id, "tag": tag, "size": img.attrs.get("Size", 0), "created": img.attrs.get("Created"), - "stacks": sorted(usage.get(tag, set())), - "update": cache.get(tag), + "stacks": sorted(stacks_by_id.get(img.id, set())), + "update": update, } ) result.sort(key=lambda r: r["tag"]) diff --git a/backend/services/image_status_store.py b/backend/services/image_status_store.py index 3deebbf..2822894 100644 --- a/backend/services/image_status_store.py +++ b/backend/services/image_status_store.py @@ -31,6 +31,7 @@ def _to_status(row: ImageStatus) -> update_service.UpdateStatus: remote_digest=row.remote_digest, checked_at=row.checked_at, error=row.error, + local=row.local, ) @@ -46,6 +47,7 @@ def save(status: update_service.UpdateStatus, notified: bool) -> None: row.remote_digest = status.remote_digest row.checked_at = status.checked_at row.error = status.error + row.local = status.local row.notified = notified session.add(row) session.commit() diff --git a/backend/services/notify_service.py b/backend/services/notify_service.py index f70327c..34916c4 100644 --- a/backend/services/notify_service.py +++ b/backend/services/notify_service.py @@ -11,6 +11,7 @@ All delivery is best-effort: failures are logged, never raised to the caller. """ from __future__ import annotations +import base64 import logging from typing import Optional @@ -37,14 +38,14 @@ def _build_request(wtype: str, url: str, event: str, title: str, message: str): return { "url": url, "content": message.encode("utf-8"), - "headers": {"Title": title, "Tags": _ntfy_tag(event)}, + "headers": {"Title": _header_value(title), "Tags": _ntfy_tag(event)}, } if wtype == "discord": return {"url": url, "json": {"content": f"**{title}**\n{message}"}} if wtype == "slack": return {"url": url, "json": {"text": f"*{title}*\n{message}"}} if wtype == "gotify": - priority = 8 if event in ("stack_error", "pull_failed") else 5 + priority = 8 if event in _URGENT_EVENTS else 5 return { "url": url, "json": {"title": title, "message": message, "priority": priority}, @@ -56,6 +57,24 @@ def _build_request(wtype: str, url: str, event: str, title: str, message: str): } +_URGENT_EVENTS = {"stack_error", "pull_failed", "backup_failed"} + + +def _header_value(text: str) -> str: + """A header value httpx can send, keeping non-ASCII readable. + + HTTP headers are ASCII; httpx raises UnicodeEncodeError on anything else, + so a title with an umlaut or a dash ("Backup to NAS Büro failed") never + reached ntfy at all. RFC 2047 encoded-words are what ntfy decodes. + """ + try: + text.encode("ascii") + return text + except UnicodeEncodeError: + encoded = base64.b64encode(text.encode("utf-8")).decode("ascii") + return f"=?UTF-8?B?{encoded}?=" + + def _ntfy_tag(event: str) -> str: return { "update_available": "arrow_up", @@ -63,17 +82,21 @@ def _ntfy_tag(event: str) -> str: "stack_stop": "stop_button", "stack_error": "rotating_light", "pull_failed": "warning", + "backup_failed": "floppy_disk", + "stack_auto_updated": "arrows_counterclockwise", }.get(event, "bell") async def _deliver(client: httpx.AsyncClient, wtype: str, url: str, event: str, title: str, message: str) -> bool: - kwargs = _build_request(wtype, url, event, title, message) - target = kwargs.pop("url") try: + kwargs = _build_request(wtype, url, event, title, message) + target = kwargs.pop("url") resp = await client.post(target, timeout=_TIMEOUT, **kwargs) resp.raise_for_status() return True - except httpx.HTTPError as exc: + except Exception as exc: # noqa: BLE001 - one bad webhook must not stop the rest + # Not just httpx.HTTPError: an invalid URL or an unencodable header + # raised past this and aborted delivery to every webhook after it. logger.warning("Webhook delivery failed (%s): %s", wtype, exc) return False diff --git a/backend/services/port_service.py b/backend/services/port_service.py index fb205c3..99d157b 100644 --- a/backend/services/port_service.py +++ b/backend/services/port_service.py @@ -8,6 +8,7 @@ import yaml from config import settings from docker_client import DockerError, get_client, safe_call +from services.compose_service import project_name # --------------------------------------------------------------------------- # @@ -21,30 +22,47 @@ def parse_compose_ports(yaml_str: str) -> list[dict]: data = yaml.safe_load(yaml_str) or {} except yaml.YAMLError: return [] + if not isinstance(data, dict): + return [] services = data.get("services") or {} + if not isinstance(services, dict): + return [] out: list[dict] = [] for svc_name, svc in services.items(): if not isinstance(svc, dict): continue for entry in svc.get("ports", []) or []: - parsed = _parse_port_entry(entry) - if parsed: + for parsed in _parse_port_entry(entry): parsed["service"] = svc_name out.append(parsed) return out -def _parse_port_entry(entry) -> Optional[dict]: +#: A published range is checked port by port, up to this many. +_MAX_RANGE = 1024 + + +def _host_ports(spec: str) -> list[int]: + """"8080" -> [8080]; "8080-8082" -> [8080, 8081, 8082]; junk -> [].""" + try: + if "-" in spec: + low, high = (int(p) for p in spec.split("-", 1)) + if low > high or high - low >= _MAX_RANGE: + return [low] + return list(range(low, high + 1)) + return [int(spec)] + except ValueError: + return [] + + +def _parse_port_entry(entry) -> list[dict]: # Long form: {target, published, protocol} if isinstance(entry, dict): published = entry.get("published") if published is None: - return None - try: - host_port = int(str(published).split("-")[0]) - except ValueError: - return None - return {"host_port": host_port, "protocol": entry.get("protocol", "tcp")} + return [] + proto = str(entry.get("protocol") or "tcp") + return [{"host_port": p, "protocol": proto} for p in _host_ports(str(published))] # Short form string: "[ip:]host:container[/proto]" or "container" s = str(entry) @@ -54,14 +72,10 @@ def _parse_port_entry(entry) -> Optional[dict]: parts = s.split(":") # No host mapping (only container port) -> random host port, no conflict. if len(parts) == 1: - return None - # host:container or ip:host:container - host = parts[-2] - try: - host_port = int(host.split("-")[0]) - except ValueError: - return None - return {"host_port": host_port, "protocol": proto} + return [] + # host:container or ip:host:container (an IPv6 address has more colons; + # the host port is always the second to last part) + return [{"host_port": p, "protocol": proto} for p in _host_ports(parts[-2])] # --------------------------------------------------------------------------- # @@ -69,11 +83,27 @@ def _parse_port_entry(entry) -> Optional[dict]: # --------------------------------------------------------------------------- # +def _proc_net_path(name: str) -> str: + """The host's socket table, not this container's. + + ``/proc/net`` is a link to ``self/net`` — the network namespace of whoever + reads it — so ``${HOST_PROC_PATH}/net/tcp`` read from inside the StackPilot + container lists the container's own sockets (just its API port), and every + port a host process listens on went unnoticed. PID 1 of the host's /proc + lives in the host's namespace. + """ + for candidate in ( + os.path.join(settings.HOST_PROC_PATH, "1", "net", name), + os.path.join(settings.HOST_PROC_PATH, "net", name), + ): + if os.path.isfile(candidate): + return candidate + return os.path.join("/proc/net", name) + + def _read_proc_net(name: str) -> set[int]: ports: set[int] = set() - path = os.path.join(settings.HOST_PROC_PATH, "net", name) - if not os.path.isfile(path): - path = os.path.join("/proc/net", name) + path = _proc_net_path(name) try: with open(path, "r", encoding="utf-8") as fh: lines = fh.readlines()[1:] @@ -154,7 +184,7 @@ def detect_conflicts(yaml_str: str, ignore_stack: Optional[str] = None) -> list[ # Match on the compose project label, falling back to the legacy # name-prefix check for containers without the label. if ignore_stack and ( - owner_project == ignore_stack + owner_project in (ignore_stack, project_name(ignore_stack)) or owner_name.startswith(f"{ignore_stack}-") ): continue diff --git a/backend/services/scan_service.py b/backend/services/scan_service.py index 360f945..15accc3 100644 --- a/backend/services/scan_service.py +++ b/backend/services/scan_service.py @@ -43,7 +43,7 @@ SCAN_TIMEOUT = 900.0 #: One sweep at a time. Scanning is CPU- and IO-heavy, and two passes over the #: same images would just fight each other for the daemon. _sweep_lock = threading.Lock() -_sweeping = {"active": False, "done": 0, "total": 0} +_sweeping = {"active": False, "done": 0, "total": 0, "scanned": 0, "failed": 0, "run": 0} class ScanError(Exception): @@ -241,15 +241,16 @@ def sweep_status() -> dict: return dict(_sweeping) -async def sweep(session: Session, images: list[str]) -> dict: - """Scan a list of images one after another. - - Deliberately serial: each scan is already CPU-hungry, and running several at - once on a homelab box would starve the very containers being protected. - """ +def _begin(total: int) -> None: if not _sweep_lock.acquire(blocking=False): raise ScanError("A scan is already running") - _sweeping.update({"active": True, "done": 0, "total": len(images)}) + _sweeping.update( + {"active": True, "done": 0, "total": total, "scanned": 0, "failed": 0, + "run": _sweeping.get("run", 0) + 1} + ) + + +async def _run(session: Session, images: list[str]) -> dict: failed = 0 try: for image in images: @@ -262,6 +263,46 @@ async def sweep(session: Session, images: list[str]) -> dict: failed += 1 _sweeping["done"] += 1 finally: - _sweeping.update({"active": False}) + _sweeping.update({"active": False, "scanned": len(images), "failed": failed}) _sweep_lock.release() return {"scanned": len(images), "failed": failed} + + +async def sweep(session: Session, images: list[str]) -> dict: + """Scan a list of images one after another. + + Deliberately serial: each scan is already CPU-hungry, and running several at + once on a homelab box would starve the very containers being protected. + """ + _begin(len(images)) + return await _run(session, images) + + +#: Background sweeps, kept referenced until they finish. +_tasks: set[asyncio.Task] = set() + + +def start_sweep(images: list[str], on_done=None) -> dict: + """Start a sweep in the background and return its status at once. + + A sweep over a dozen images takes far longer than the proxy waits for a + response (600s), so the button used to report a 504 while the scans went + on. Marked active before this returns, so a poll right after sees it. + """ + from database import engine + + _begin(len(images)) + + async def work() -> None: + with Session(engine) as session: + result = await _run(session, images) + if on_done is not None: + try: + on_done(session, result) + except Exception: # noqa: BLE001 - the sweep itself is done + logger.exception("Post-sweep hook failed") + + task = asyncio.create_task(work(), name="image-scan-sweep") + _tasks.add(task) + task.add_done_callback(_tasks.discard) + return sweep_status() diff --git a/backend/services/schedule_service.py b/backend/services/schedule_service.py index a5f4fe4..135a135 100644 --- a/backend/services/schedule_service.py +++ b/backend/services/schedule_service.py @@ -159,7 +159,9 @@ async def _backup_stack( stack_id, stack.name, include_volumes=schedule.include_volumes, stop_first=schedule.stop_first, ) - filename = backup_service.backup_filename(stack_id, schedule.include_volumes) + filename = backup_service.backup_filename( + stack_id, schedule.include_volumes, tz=schedule.timezone + ) await asyncio.to_thread(dest_service.upload, dest, path, filename) problems = _problems(report) if problems: diff --git a/backend/services/secret_service.py b/backend/services/secret_service.py index 039bbb1..f71a480 100644 --- a/backend/services/secret_service.py +++ b/backend/services/secret_service.py @@ -19,6 +19,17 @@ from typing import Optional from services import compose_edit_service, compose_service KIND_DIR = {"secret": ".secrets", "config": ".configs"} + +#: Who can read a secret is decided by the directory, not the file. +#: +#: Outside swarm, compose hands a file-based secret to the container as a bind +#: mount of the file itself, with the file's own owner and mode. At 0600 root a +#: container running as anything but root (postgres, every PUID/PGID image) got +#: "permission denied" on /run/secrets/. The bind mount reaches the file +#: without passing through its directory, so a 0700 directory keeps other users +#: on the host out while the file stays readable inside the container. +DIR_MODE = 0o700 +FILE_MODE = 0o644 _NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$") MAX_SECRET_BYTES = 1 * 1024 * 1024 # 1 MiB @@ -83,11 +94,11 @@ def write_secret(stack_id: str, kind: str, name: str, content: str, override: Op raise SecretError("content exceeds 1 MiB limit") directory = kind_dir(stack_id, kind, override) os.makedirs(directory, exist_ok=True) - os.chmod(directory, 0o700) + os.chmod(directory, DIR_MODE) path = _path(stack_id, kind, name, override) with open(path, "wb") as fh: fh.write(data) - os.chmod(path, 0o600) + os.chmod(path, FILE_MODE) return {"name": name, "kind": kind, "size": len(data)} @@ -107,6 +118,9 @@ def attach(stack_id: str, kind: str, name: str, service: str, file (relative ``file:`` path) and persist it. Returns the new YAML.""" _check_kind(kind) _check_name(name) + stored = _path(stack_id, kind, name, override) + if os.path.isfile(stored): + os.chmod(stored, FILE_MODE) # one written by an older version yaml_str = compose_service.read_compose(stack_id, override) file_path = rel_path(kind, name) if kind == "config": @@ -126,3 +140,33 @@ def detach(stack_id: str, kind: str, name: str, service: str, override: Optional new_yaml = compose_edit_service.remove_secret(yaml_str, service, name) compose_service.write_compose(stack_id, new_yaml, override) return new_yaml + + +def fix_permissions(root: Optional[str] = None) -> int: + """Make secrets written by older versions (0600) readable by containers. + + Runs at startup over every stack's ``.secrets`` and ``.configs`` — folders + only StackPilot writes. Returns how many files were changed. + """ + base = compose_service.stacks_root(root) + changed = 0 + try: + stacks = os.listdir(base) + except OSError: + return 0 + for stack in stacks: + for folder in KIND_DIR.values(): + directory = os.path.join(base, stack, folder) + if not os.path.isdir(directory) or os.path.islink(directory): + continue + try: + os.chmod(directory, DIR_MODE) + for name in os.listdir(directory): + path = os.path.join(directory, name) + if os.path.isfile(path) and not os.path.islink(path): + if os.stat(path).st_mode & 0o777 == 0o600: + os.chmod(path, FILE_MODE) + changed += 1 + except OSError: + continue + return changed diff --git a/backend/services/stack_assets_service.py b/backend/services/stack_assets_service.py index fb0c400..40eaea2 100644 --- a/backend/services/stack_assets_service.py +++ b/backend/services/stack_assets_service.py @@ -371,24 +371,48 @@ def import_volume( # --------------------------------------------------------------------------- # +#: Found once and kept: a running container's mounts do not change. +_HOST_STACKS_DIR: Optional[str] = None + + def host_stacks_dir() -> Optional[str]: """Host path backing ``STACKS_DIR`` inside this container, if detectable. - Read from /proc/self/mountinfo (field 4 is the source subtree on the host - filesystem). Returns None when not running in a container / not bind-mounted. + Taken from Docker's record of this container's mounts. Not from + /proc/self/mountinfo, as it once was: its 4th field is the path *inside the + mounted filesystem*, which is the host path only when that filesystem is the + host's root. A stacks folder on its own disk, a ZFS dataset or a btrfs + subvolume read as ``/stacks`` or ``/@/opt/stacks``, and every backup dialog + warned about a path mismatch that was not there. Returns None when not + running in a container or when the mount cannot be found. """ - target = settings.STACKS_DIR.rstrip("/") or "/" + global _HOST_STACKS_DIR + if _HOST_STACKS_DIR is not None: + return _HOST_STACKS_DIR + from services.self_update_service import _own_container + + target = os.path.normpath(settings.STACKS_DIR) try: - with open("/proc/self/mountinfo", "r", encoding="utf-8") as fh: - for line in fh: - parts = line.split() - if len(parts) < 5: - continue - if parts[4].rstrip("/") == target: - return parts[3] - except OSError: + container = _own_container() + except Exception: # noqa: BLE001 - no Docker, no answer return None - return None + if container is None: + return None + # The mount whose destination is STACKS_DIR or the closest folder above it + # (a whole /opt mounted, STACKS_DIR=/opt/stacks). + best = None + for mount in container.attrs.get("Mounts") or []: + dest = os.path.normpath(mount.get("Destination") or "") + if mount.get("Type") != "bind" or not mount.get("Source"): + continue + if target == dest or target.startswith(dest.rstrip("/") + "/"): + if best is None or len(dest) > len(best[0]): + best = (dest, mount["Source"]) + if best is None: + return None + dest, source = best + _HOST_STACKS_DIR = os.path.normpath(source + target[len(dest):]) + return _HOST_STACKS_DIR def stacks_path_mismatch() -> Optional[dict]: @@ -483,7 +507,7 @@ def _bind_specs_from_containers(stack_id: str) -> list[dict]: containers = safe_call( client.containers.list, all=True, - filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"}, + filters={"label": _project_label(stack_id)}, ) except DockerError: return [] @@ -503,6 +527,10 @@ def _bind_specs_from_containers(stack_id: str) -> list[dict]: return out +def _project_label(stack_id: str) -> str: + return f"{COMPOSE_PROJECT_LABEL}={compose_service.project_name(stack_id)}" + + def is_system_path(path: str) -> bool: if path in SYSTEM_PATHS: return True @@ -555,7 +583,7 @@ def _external_volume_names(stack_id: str) -> set[str]: containers = safe_call( client.containers.list, all=True, - filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"}, + filters={"label": _project_label(stack_id)}, ) for c in containers: for mount in c.attrs.get("Mounts") or []: @@ -589,7 +617,7 @@ def compose_volumes(stack_id: str) -> list[dict]: client = get_client() vols = safe_call( client.volumes.list, - filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"}, + filters={"label": _project_label(stack_id)}, ) except DockerError: return [] diff --git a/backend/services/stack_cleanup_service.py b/backend/services/stack_cleanup_service.py new file mode 100644 index 0000000..b3eda1c --- /dev/null +++ b/backend/services/stack_cleanup_service.py @@ -0,0 +1,86 @@ +"""What has to go with a deleted stack, besides its containers and files. + +Several features keep rows keyed by stack id, and deleting the stack used to +leave all of them behind: + +* a **Git source**: the poll loop and the webhook kept syncing it — which + recreated the deleted stack's folder from the repository and, with auto + deploy on, started it again; +* an **auto-update policy**, which a new stack of the same name then silently + inherited; +* **backup schedules** naming it, which from then on failed on every run with + "stack not found" and paged somebody each time. +""" +from __future__ import annotations + +import logging + +from sqlmodel import Session, select + +from models.auto_update import AutoUpdate +from models.backup_schedule import BackupSchedule, set_stacks, stacks_of +from models.git_source import GitSource +from services import git_service + +logger = logging.getLogger("stackpilot.stack_cleanup") + + +def forget_stack(session: Session, stack_id: str) -> dict: + """Drop everything that refers to ``stack_id``. The caller deletes the Stack. + + A schedule that covered only this stack is disabled rather than deleted: + it may carry a destination and retention somebody wants to point at + another stack. All-stacks schedules need nothing — they resolve the stack + list when they run. + """ + report = {"git": False, "auto_update": 0, "schedules_changed": [], "schedules_disabled": []} + + source = session.exec(select(GitSource).where(GitSource.stack_id == stack_id)).first() + if source is not None: + session.delete(source) + report["git"] = True + + for policy in session.exec(select(AutoUpdate).where(AutoUpdate.stack_id == stack_id)).all(): + session.delete(policy) + report["auto_update"] += 1 + + for schedule in session.exec(select(BackupSchedule)).all(): + if schedule.all_stacks: + continue + ids = stacks_of(schedule) + if stack_id not in ids: + continue + remaining = [sid for sid in ids if sid != stack_id] + set_stacks(schedule, remaining) + if remaining: + report["schedules_changed"].append(schedule.id) + else: + schedule.enabled = False + schedule.last_status = f"disabled: its stack '{stack_id}' was deleted" + report["schedules_disabled"].append(schedule.id) + session.add(schedule) + + session.commit() + if report["git"]: + git_service.forget(stack_id) + if any(report[k] for k in report): + logger.info("Cleaned up after deleted stack '%s': %s", stack_id, report) + return report + + +def prune_orphans(session: Session) -> list[str]: + """Forget every stack that has rows here but no Stack any more. + + Run at startup, after on-disk stacks have been registered: installs that + deleted stacks before this existed still carry their Git sources and + policies. Returns the stack ids cleaned up. + """ + from models.stack import Stack + + known = {s.id for s in session.exec(select(Stack)).all()} + referenced = {g.stack_id for g in session.exec(select(GitSource)).all()} + referenced |= {p.stack_id for p in session.exec(select(AutoUpdate)).all()} + orphans = sorted(referenced - known) + for stack_id in orphans: + forget_stack(session, stack_id) + return orphans diff --git a/backend/services/update_service.py b/backend/services/update_service.py index 34ac461..ece13c0 100644 --- a/backend/services/update_service.py +++ b/backend/services/update_service.py @@ -39,6 +39,11 @@ class UpdateStatus: remote_digest: Optional[str] checked_at: float error: Optional[str] = None + #: Built on this host (``docker build``, compose ``build:``) rather than + #: pulled: there is no registry copy to compare against, so it is never + #: "out of date" — and asking Docker Hub about it only got a 401 that read + #: as "needs credentials". + local: bool = False def to_dict(self) -> dict: return asdict(self) @@ -100,6 +105,14 @@ def _store(status: UpdateStatus, notified: bool) -> None: # --------------------------------------------------------------------------- # +#: Docker Hub under the names people write in compose files. Only +#: registry-1.docker.io serves the v2 API; https://docker.io/v2/ redirects to +#: the marketing site, so a `docker.io/library/postgres` image could never be +#: checked and sat at "could not reach registry" forever. +_DOCKER_HUB = "registry-1.docker.io" +_DOCKER_HUB_ALIASES = {"docker.io", "index.docker.io", "registry.docker.io", _DOCKER_HUB} + + def parse_ref(image: str) -> tuple[str, str, str]: """Return (registry_host, repository, tag/digest).""" ref = image @@ -107,37 +120,68 @@ def parse_ref(image: str) -> tuple[str, str, str]: # split tag (but not the registry port colon) if "@" in ref: ref, tag = ref.split("@", 1) - else: - # find last colon after last slash - slash = ref.rfind("/") - colon = ref.rfind(":") - if colon > slash: + # find last colon after last slash. With a digest present this is the tag + # in "name:tag@sha256:…", which the digest outranks — but it must not stay + # glued to the repository name. + slash = ref.rfind("/") + colon = ref.rfind(":") + if colon > slash: + if "@" not in image: tag = ref[colon + 1 :] - ref = ref[:colon] + ref = ref[:colon] parts = ref.split("/", 1) if len(parts) == 2 and ("." in parts[0] or ":" in parts[0] or parts[0] == "localhost"): registry = parts[0] repo = parts[1] + if registry.lower() in _DOCKER_HUB_ALIASES: + registry = _DOCKER_HUB else: - registry = "registry-1.docker.io" + registry = _DOCKER_HUB repo = ref - if "/" not in repo: - repo = "library/" + repo + if registry == _DOCKER_HUB and "/" not in repo: + repo = "library/" + repo return registry, repo, tag -def _local_digest(image: str) -> Optional[str]: +def _local_attrs(image: str) -> Optional[dict]: try: - client = get_client() - img = safe_call(client.images.get, image) + return safe_call(get_client().images.get, image).attrs except DockerError: return None - repo_digests = img.attrs.get("RepoDigests") or [] + + +def _built_locally(attrs: Optional[dict]) -> bool: + """Whether the image came from a build on this host, not from a registry. + + Docker 29+ records where an image came from (``Identity.Pull`` / + ``Identity.Build``). Older daemons do not; there a built image is the one + without RepoDigests. Unknown means "pulled" — skipping the check for a + pulled image would hide its updates for good. + """ + if attrs is None: + return False + identity = attrs.get("Identity") + if isinstance(identity, dict) and identity: + return bool(identity.get("Build")) and not identity.get("Pull") + return not (attrs.get("RepoDigests") or []) + + +def _local_digest(image: str, attrs: Optional[dict] = None) -> Optional[str]: + if attrs is None: + attrs = _local_attrs(image) + if attrs is None: + return None + repo_digests = [rd for rd in attrs.get("RepoDigests") or [] if "@" in rd] + # An image can carry digests for several repositories (the same image + # pulled as lscr.io/... and as docker.io/...); compare against the one + # this reference is checked against, not whichever comes first. + wanted = parse_ref(image)[:2] for rd in repo_digests: - if "@" in rd: - return rd.split("@", 1)[1] - return None + name, digest = rd.split("@", 1) + if parse_ref(name)[:2] == wanted: + return digest + return repo_digests[0].split("@", 1)[1] if repo_digests else None # --------------------------------------------------------------------------- # @@ -236,7 +280,20 @@ async def remote_digest(image: str) -> Optional[str]: async def check_image(image: str) -> UpdateStatus: - local = _local_digest(image) + attrs = _local_attrs(image) + if _built_locally(attrs): + status = UpdateStatus( + image=image, + update_available=False, + current_digest=None, + remote_digest=None, + checked_at=time.time(), + local=True, + ) + _NOTIFIED.discard(image) + _store(status, False) + return status + local = _local_digest(image, attrs) error = None try: remote = await remote_digest(image) @@ -294,12 +351,15 @@ def _all_running_images() -> set[str]: def stack_images(stack_id: str) -> set[str]: """Images used by the containers of one compose project (== stack id).""" + from services.compose_service import project_name + + project = project_name(stack_id) images: set[str] = set() try: client = get_client() for c in safe_call(client.containers.list, all=True): labels = c.labels or {} - if labels.get("com.docker.compose.project") != stack_id: + if labels.get("com.docker.compose.project") != project: continue cfg_image = c.attrs.get("Config", {}).get("Image") if cfg_image: @@ -380,6 +440,9 @@ def refresh_stack_local(stack_id: str) -> None: status.checked_at = time.time() if not status.update_available: _NOTIFIED.discard(image) + # Persisted too, or a restart brought the cleared badge back until the + # next background check. + _store(status, image in _NOTIFIED) async def check_all() -> dict[str, dict]: diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 9c178a2..4b462df 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -30,6 +30,14 @@ os.environ["STACKS_DIR"] = os.path.join(_TMP, "stacks") os.environ["SECRET_KEY"] = "test-secret-key-not-used-anywhere-real" os.environ["ALLOWED_BROWSE_ROOTS"] = "/mnt,/media,/srv,/opt,/home" os.environ["HOST_ROOT_PREFIX"] = "" +# No Docker, even where there is one. The suite is written for a host without a +# daemon, and where one *is* reachable — a dev box, or the CI runner, which gets +# the host's socket — tests that go "far enough in" really ran `compose up` and +# left containers running on that host after every run. docker-py reads +# DOCKER_SOCKET, the docker CLI (compose) reads DOCKER_HOST; both point nowhere. +_NO_DOCKER = os.path.join(_TMP, "no-docker.sock") +os.environ["DOCKER_SOCKET"] = _NO_DOCKER +os.environ["DOCKER_HOST"] = f"unix://{_NO_DOCKER}" # The backend runs from its own root at runtime (`uvicorn main:app` with # /app as the workdir), so make the same layout importable here. diff --git a/backend/tests/test_compose_runs.py b/backend/tests/test_compose_runs.py new file mode 100644 index 0000000..5d0e74c --- /dev/null +++ b/backend/tests/test_compose_runs.py @@ -0,0 +1,226 @@ +"""Streamed compose runs: the deploy console and the update bar. + +Until 0.67.1 the run *was* the WebSocket handler, so closing the console in the +middle of a deploy released the stack lock at once — while compose went on — +and logged the deploy as a success with ``rc=None``. The run is its own task +now; a socket only watches it. And the deploy socket is admin-only, like the +REST ``/start`` it mirrors: before, the read-only role could run ``compose up`` +through it. +""" +from __future__ import annotations + +import asyncio +import json + +import pytest +from sqlmodel import Session + + +@pytest.fixture(autouse=True) +def clean_locks(db): + from sqlmodel import delete + + from database import engine + from models.runtime_state import StackLock + + with Session(engine) as session: + session.exec(delete(StackLock)) + session.commit() + yield + + +def _locked(stack_id: str) -> bool: + from database import engine + from services import stack_lock_service + + with Session(engine) as session: + return stack_lock_service.is_busy(session, stack_id) + + +def test_the_lock_is_held_until_compose_finishes(db): + from services import compose_run_service as runs + + async def scenario(): + release = asyncio.Event() + finished: list = [] + + async def stream(): + yield ("log", "pulling") + await release.wait() + yield ("log", "started") + yield ("done", 0) + + async def on_finish(rc): + finished.append(rc) + + run, queue, attached = runs.start("rx", "start", "alice", stream, on_finish) + assert not attached + assert await queue.get() == ("log", "pulling") + # Nobody is watching any more — the console was closed. + run.unsubscribe(queue) + await asyncio.sleep(0.05) + assert _locked("rx"), "the lock must outlive the socket" + assert finished == [] + + release.set() + for _ in range(50): + if finished: + break + await asyncio.sleep(0.02) + assert finished == [0], "the real exit code, after the fact" + assert not _locked("rx") + + asyncio.run(scenario()) + + +def test_a_second_socket_attaches_to_the_run_in_flight(db): + from services import compose_run_service as runs + + async def scenario(): + release = asyncio.Event() + + async def stream(): + yield ("log", "one") + await release.wait() + yield ("done", 0) + + run, first, _ = runs.start("ry", "start", "alice", stream) + assert await first.get() == ("log", "one") + + again, second, attached = runs.start("ry", "start", "bob", stream) + assert attached and again is run + assert await second.get() == ("log", "one"), "backlog first" + + with pytest.raises(runs.RunBusy): + runs.start("ry", "update", "carol", stream) + + release.set() + items = [] + while (item := await second.get()) is not None: + items.append(item) + assert items == [("done", 0)] + + asyncio.run(scenario()) + + +def test_a_run_that_cannot_get_the_lock_is_busy(db): + from database import engine + from services import compose_run_service as runs + from services import stack_lock_service + + with Session(engine) as session: + stack_lock_service.acquire(session, "rz", "backup", "scheduled") + + async def stream(): + yield ("done", 0) + + async def scenario(): + with pytest.raises(runs.RunBusy) as excinfo: + runs.start("rz", "start", "alice", stream) + assert excinfo.value.action == "backup" + + asyncio.run(scenario()) + + +def test_a_failing_stream_still_releases_the_lock(db): + from services import compose_run_service as runs + + async def scenario(): + async def stream(): + yield ("log", "about to break") + raise RuntimeError("compose binary missing") + + finished = [] + + async def on_finish(rc): + finished.append(rc) + + _run, queue, _ = runs.start("rf", "update", "alice", stream, on_finish) + items = [] + while (item := await queue.get()) is not None: + items.append(item) + await asyncio.sleep(0.05) + assert items[-1][0] == "done" and items[-1][1] != 0 + assert finished and finished[0] != 0 + assert not _locked("rf") + + asyncio.run(scenario()) + + +def test_the_deploy_socket_is_admin_only(client, user_token): + from starlette.websockets import WebSocketDisconnect + + with pytest.raises(WebSocketDisconnect) as excinfo: + with client.websocket_connect(f"/ws/deploy/anything?token={user_token}") as socket: + socket.receive_text() + assert excinfo.value.code == 4403 + + +def test_deploying_an_unknown_stack_is_refused(client, admin_token): + from starlette.websockets import WebSocketDisconnect + + with pytest.raises(WebSocketDisconnect) as excinfo: + with client.websocket_connect(f"/ws/deploy/no-such-stack?token={admin_token}") as socket: + frame = json.loads(socket.receive_text()) + assert frame["type"] == "error" + socket.receive_text() + assert excinfo.value.code == 4404 + + +# --------------------------------------------------------------------------- # +# The REST paths +# --------------------------------------------------------------------------- # + + +def test_update_stops_at_a_failed_pull(monkeypatch): + """`up` after a failed pull recreated nothing and reported success.""" + from services import compose_service + + calls = [] + + async def fake(stack_id, args, override=None, timeout=600.0): + calls.append(args[0]) + rc = 1 if args[0] == "pull" else 0 + return {"returncode": rc, "stdout": "", "stderr": "denied", "command": args[0]} + + monkeypatch.setattr(compose_service, "run_compose", fake) + result = asyncio.run(compose_service.update("s")) + assert calls == ["pull"] + assert result["returncode"] == 1 + + +def test_auto_update_reports_a_failed_compose_run(db, monkeypatch): + """The exit codes used to be ignored: a failure was recorded as "updated".""" + from database import engine + from models.auto_update import AutoUpdate + from services import auto_update_service, compose_service, update_service + + monkeypatch.setattr(compose_service, "compute_status", lambda stack_id: "running") + + async def stale(stack_id, refresh=False): + return {"update_available": True, "stale_images": ["nginx:latest"]} + + async def failing_update(stack_id, override=None): + return {"returncode": 1, "stdout": "", "stderr": "Error: pull access denied"} + + notified = [] + + async def notify(event, title, message, session=None): + notified.append(title) + + monkeypatch.setattr(update_service, "stack_updates", stale) + monkeypatch.setattr(compose_service, "update", failing_update) + monkeypatch.setattr(auto_update_service.notify_service, "notify", notify) + + with Session(engine) as session: + policy = AutoUpdate(stack_id="au-fail", enabled=True, redeploy=True) + session.add(policy) + session.commit() + session.refresh(policy) + asyncio.run(auto_update_service.run_policy(session, policy)) + session.refresh(policy) + assert policy.last_status == "error" + assert "pull access denied" in (policy.last_result or "") + session.delete(policy) + session.commit() + assert notified and "failed" in notified[0] diff --git a/backend/tests/test_compose_service.py b/backend/tests/test_compose_service.py index d211c90..f0d942e 100644 --- a/backend/tests/test_compose_service.py +++ b/backend/tests/test_compose_service.py @@ -137,16 +137,38 @@ def test_secret_names_reject_traversal_and_hidden_files(name): secret_service._check_name(name) -def test_secret_files_are_written_owner_only(tmp_path): +def test_secret_files_are_guarded_by_their_directory(tmp_path): + """The 0700 directory keeps other host users out; the file itself has to be + world-readable, because compose bind-mounts it into the container as it is + and a container running as postgres or PUID 1000 could not open a 0600 + root file.""" from services import secret_service secret_service.write_secret("s", "secret", "db_password", "hunter2", override=str(tmp_path)) path = tmp_path / "s" / ".secrets" / "db_password" assert path.read_text() == "hunter2" - assert oct(path.stat().st_mode)[-3:] == "600" + assert oct(path.stat().st_mode)[-3:] == "644" assert oct(path.parent.stat().st_mode)[-3:] == "700" +def test_secrets_from_older_versions_are_made_readable(tmp_path): + import os + + from services import secret_service + + folder = tmp_path / "old" / ".secrets" + folder.mkdir(parents=True) + old = folder / "pw" + old.write_text("x") + os.chmod(old, 0o600) + os.chmod(folder, 0o755) + + assert secret_service.fix_permissions(str(tmp_path)) == 1 + assert oct(old.stat().st_mode)[-3:] == "644" + assert oct(folder.stat().st_mode)[-3:] == "700" + assert secret_service.fix_permissions(str(tmp_path)) == 0, "idempotent" + + def test_listing_secrets_never_returns_their_content(tmp_path): from services import secret_service diff --git a/backend/tests/test_convert_service.py b/backend/tests/test_convert_service.py new file mode 100644 index 0000000..c778293 --- /dev/null +++ b/backend/tests/test_convert_service.py @@ -0,0 +1,167 @@ +"""Converting `docker run` into compose. + +The output has to be a compose file compose accepts and that means what the +command meant. Before 0.67.1 a boolean flag the converter did not know swallowed +the image name (``-dit ubuntu bash`` became a service running ``bash``), +``--network host`` and ``-v data:/data`` produced files compose rejects, and +``--gpus`` or ``--mount`` were dropped without a word. +""" +from __future__ import annotations + +import pytest +import yaml + + +def _service(command: str) -> tuple[str, dict, dict]: + from services.convert_service import convert_docker_run + + doc = yaml.safe_load(convert_docker_run(command)) + ((name, service),) = doc["services"].items() + return name, service, doc + + +@pytest.mark.parametrize( + "command, image", + [ + ("docker run -dit ubuntu bash", "ubuntu"), + ("docker run -d --read-only nginx", "nginx"), + ("docker run -P nginx", "nginx"), + ("docker run --rm --init --privileged alpine", "alpine"), + ("docker run -d --oom-kill-disable --no-healthcheck redis:7", "redis:7"), + ], +) +def test_boolean_flags_never_swallow_the_image(command, image): + _name, service, _doc = _service(command) + assert service["image"] == image + + +def test_the_command_after_the_image_is_kept(): + name, service, _doc = _service("docker run -it --rm ubuntu bash -lc 'echo hi'") + assert name == "ubuntu" + assert service["command"] == ["bash", "-lc", "echo hi"] + assert service["stdin_open"] is True and service["tty"] is True + + +@pytest.mark.parametrize("mode", ["host", "none", "bridge", "container:vpn"]) +def test_special_networks_become_network_mode(mode): + _name, service, doc = _service(f"docker run -d --network {mode} alpine") + assert service["network_mode"] == mode + assert "networks" not in service and "networks" not in doc + + +def test_a_named_network_is_declared_external(): + _name, service, doc = _service("docker run -d --net=proxy --network-alias web alpine") + assert service["networks"] == {"proxy": {"aliases": ["web"]}} + assert doc["networks"] == {"proxy": {"external": True}} + + +def test_named_volumes_are_declared_and_paths_are_not(): + _name, service, doc = _service( + "docker run -d -v data:/data -v /srv/media:/media:ro -v ./conf:/conf alpine" + ) + assert service["volumes"] == ["data:/data", "/srv/media:/media:ro", "./conf:/conf"] + assert doc["volumes"] == {"data": None} + + +def test_mount_becomes_long_syntax(): + _name, service, doc = _service( + "docker run --mount type=bind,source=/srv,target=/srv,readonly " + "--mount type=volume,src=cache,dst=/cache alpine" + ) + assert service["volumes"][0] == { + "type": "bind", "source": "/srv", "target": "/srv", "read_only": True, + } + assert service["volumes"][1] == {"type": "volume", "source": "cache", "target": "/cache"} + assert doc["volumes"] == {"cache": None} + + +@pytest.mark.parametrize( + "value, expected", + [ + ("all", {"count": "all"}), + ("2", {"count": 2}), + ('"device=0,1"', {"device_ids": ["0", "1"]}), + ], +) +def test_gpus_become_a_device_reservation(value, expected): + _name, service, _doc = _service(f"docker run -d --gpus {value} ollama/ollama") + (device,) = service["deploy"]["resources"]["reservations"]["devices"] + assert device["driver"] == "nvidia" + assert device["capabilities"] == ["gpu"] + for key, val in expected.items(): + assert device[key] == val + + +def test_attached_short_values_and_equals_forms(): + _name, service, _doc = _service( + "docker run -p8080:80 -eTZ=Europe/Berlin --restart=always --name=web nginx" + ) + assert service["ports"] == ["8080:80"] + assert service["environment"] == ["TZ=Europe/Berlin"] + assert service["restart"] == "always" + assert service["container_name"] == "web" + + +def test_common_runtime_flags_are_kept(): + _name, service, _doc = _service( + "sudo docker run -d --name plex --entrypoint /init --shm-size 1g -m 2g --cpus 1.5 " + "--cap-add NET_ADMIN --cap-drop ALL --security-opt no-new-privileges " + "--sysctl net.ipv4.ip_forward=1 --add-host nas:10.0.0.2 --dns 1.1.1.1 " + "--ulimit nofile=1024:2048 --log-opt max-size=10m --tmpfs /run " + "--health-cmd 'curl -f http://localhost' --health-interval 30s --stop-timeout 30 " + "lscr.io/linuxserver/plex:latest" + ) + assert service["entrypoint"] == "/init" + assert service["shm_size"] == "1g" + assert service["mem_limit"] == "2g" + assert service["cpus"] == 1.5 + assert service["cap_add"] == ["NET_ADMIN"] and service["cap_drop"] == ["ALL"] + assert service["security_opt"] == ["no-new-privileges"] + assert service["sysctls"] == ["net.ipv4.ip_forward=1"] + assert service["extra_hosts"] == ["nas:10.0.0.2"] + assert service["dns"] == ["1.1.1.1"] + assert service["ulimits"] == {"nofile": {"soft": 1024, "hard": 2048}} + assert service["logging"] == {"options": {"max-size": "10m"}} + assert service["tmpfs"] == ["/run"] + assert service["healthcheck"] == { + "test": ["CMD-SHELL", "curl -f http://localhost"], "interval": "30s", + } + assert service["stop_grace_period"] == "30s" + + +def test_line_continuations_from_a_readme(): + name, service, _doc = _service("docker run -d \\\n --name x \\\r\n -p 1:1 \\\n alpine") + assert name == "x" + assert service["ports"] == ["1:1"] + + +def test_the_service_name_comes_from_the_image_without_registry_or_tag(): + name, _service_, _doc = _service("docker run -d ghcr.io/home-assistant/home-assistant:stable") + assert name == "home-assistant" + + +def test_no_image_is_an_error(): + from services.convert_service import convert_docker_run + + with pytest.raises(ValueError): + convert_docker_run("docker run -d --name x") + + +@pytest.mark.parametrize( + "command", + [ + # Pasted into a one-line field: the newlines are gone, the backslashes + # and indentation are not. + "docker run -d \\ --name web \\ -p 8081:80 \\ -v data:/data \\ nginx:alpine", + "docker run -d \\ --name web \\ -p 8081:80 \\\n nginx:alpine", + ], +) +def test_continuations_that_lost_their_newline(command): + name, web, _ = _service(command) + assert name == "web" and web["image"] == "nginx:alpine" + assert web["ports"] == ["8081:80"] + + +def test_an_escaped_space_in_a_path_is_still_one(): + _, service, _ = _service("docker run -v /mnt/My\\ Files:/data nginx") + assert service["volumes"] == ["/mnt/My Files:/data"] diff --git a/backend/tests/test_destinations.py b/backend/tests/test_destinations.py new file mode 100644 index 0000000..e3d8ab2 --- /dev/null +++ b/backend/tests/test_destinations.py @@ -0,0 +1,276 @@ +"""Backup destinations: errors that say what happened, and names that stay names. + +An SFTP host that refused the connection was a bare 500 (paramiko connects in +the Transport constructor, outside every handler), so the "Test" button showed +nothing useful. And a backup file name from the API went straight into +``posixpath.join`` — ``../`` let a delete reach outside the backup folder. +""" +from __future__ import annotations + +import socket + +import pytest + + +@pytest.mark.parametrize("name", ["../etc/passwd", "a/b", "..", "", "x\\y", "a\0b"]) +def test_backup_names_must_be_bare_file_names(name): + from services import backup_destination_service as dest + + with pytest.raises(dest.DestinationError): + dest._check_file_name(name) + + +def test_a_real_backup_name_passes(): + from services import backup_destination_service as dest + + dest._check_file_name("backup-immich-full-20260101-030000.tar.gz") + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _destination(kind: str, config: dict): + from models.backup_destination import BackupDestination + from services import backup_destination_service as dest + + return BackupDestination(id=99, name="t", type=kind, config=dest.dump_config(config)) + + +def test_an_unreachable_sftp_host_is_a_destination_error(): + from services import backup_destination_service as dest + + d = _destination( + "sftp", {"host": "127.0.0.1", "port": _free_port(), "username": "u", "password": "p"} + ) + with pytest.raises(dest.DestinationError, match="SFTP connection"): + dest.test(d) + + +def test_sftp_delete_refuses_a_path_before_connecting(): + from services import backup_destination_service as dest + + d = _destination("sftp", {"host": "198.51.100.1", "username": "u"}) + with pytest.raises(dest.DestinationError, match="Invalid backup file name"): + dest.delete(d, "../../important") + + +def test_a_malformed_s3_endpoint_is_a_destination_error(): + from services import backup_destination_service as dest + + d = _destination("s3", {"bucket": "b", "endpoint_url": "not a url"}) + with pytest.raises(dest.DestinationError, match="Invalid S3 settings"): + dest.test(d) + + +def test_the_test_endpoint_reports_instead_of_failing(as_admin, db): + from sqlmodel import Session + + from database import engine + from models.backup_destination import BackupDestination + from services import backup_destination_service as dest + + with Session(engine) as session: + row = BackupDestination( + name="refusing", + type="sftp", + config=dest.dump_config( + {"host": "127.0.0.1", "port": _free_port(), "username": "u", "password": "p"} + ), + ) + session.add(row) + session.commit() + session.refresh(row) + dest_id = row.id + try: + response = as_admin.post(f"/api/backups/destinations/{dest_id}/test") + assert response.status_code == 200 + assert response.json()["ok"] is False + assert "SFTP connection" in response.json()["error"] + finally: + with Session(engine) as session: + session.delete(session.get(BackupDestination, dest_id)) + session.commit() + + +def test_a_destination_in_use_by_a_schedule_cannot_be_deleted(as_admin, db): + """Deleted from under it, the schedule failed on every run from then on.""" + from sqlmodel import Session + + from database import engine + from models.backup_destination import BackupDestination + from models.backup_schedule import BackupSchedule + from services import backup_destination_service as dest + + with Session(engine) as session: + row = BackupDestination(name="nas", type="sftp", config=dest.dump_config({"host": "x"})) + session.add(row) + session.commit() + session.refresh(row) + schedule = BackupSchedule(destination_id=row.id, all_stacks=True) + session.add(schedule) + session.commit() + session.refresh(schedule) + dest_id, schedule_id = row.id, schedule.id + try: + response = as_admin.delete(f"/api/backups/destinations/{dest_id}") + assert response.status_code == 409 + assert "schedule" in response.json()["detail"] + finally: + with Session(engine) as session: + session.delete(session.get(BackupSchedule, schedule_id)) + session.commit() + assert as_admin.delete(f"/api/backups/destinations/{dest_id}").status_code == 200 + + +# --------------------------------------------------------------------------- # +# SFTP host keys: a real SSH handshake against an in-process paramiko server. +# --------------------------------------------------------------------------- # + + +class _SshServer: + """One-connection SSH server with an SFTP subsystem and a known host key.""" + + def __init__(self, host_key): + import threading + + import paramiko + + self.host_key = host_key + self.passwords: list[str] = [] + outer = self + + class Iface(paramiko.ServerInterface): + def get_allowed_auths(self, username): + return "password" + + def check_auth_password(self, username, password): + outer.passwords.append(password) + return paramiko.AUTH_SUCCESSFUL + + def check_channel_request(self, kind, chanid): + return paramiko.OPEN_SUCCEEDED + + self._iface = Iface + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(1) + self.port = self.sock.getsockname()[1] + threading.Thread(target=self._run, daemon=True).start() + + def _run(self): + import paramiko + + conn, _ = self.sock.accept() + transport = paramiko.Transport(conn) + transport.add_server_key(self.host_key) + transport.set_subsystem_handler("sftp", paramiko.SFTPServer, paramiko.SFTPServerInterface) + try: + transport.start_server(server=self._iface()) + # Kept: a channel that is garbage collected closes itself, and the + # client saw "Channel closed" in the middle of the SFTP handshake. + self.channel = transport.accept(10) + while transport.is_active(): + transport.join(0.1) + except Exception: # noqa: BLE001 - the client hung up on purpose + pass + finally: + self.sock.close() + + def cfg(self, **extra): + return {"host": "127.0.0.1", "port": self.port, "username": "u", "password": "pw", **extra} + + +@pytest.fixture +def host_key(): + import paramiko + + return paramiko.ECDSAKey.generate() + + +def _connect(cfg, pin=None): + from services import backup_destination_service as dest + + sftp, transport = dest._sftp_connect(cfg, pin) + sftp.close() + transport.close() + + +def test_the_first_host_key_is_pinned(host_key): + server = _SshServer(host_key) + pinned = [] + _connect(server.cfg(), pinned.append) + assert pinned == [f"{host_key.get_name()} {host_key.get_base64()}"] + + +@pytest.mark.parametrize("form", ["line", "known_hosts", "fingerprint"]) +def test_a_pinned_key_lets_its_server_in(host_key, form): + from services import backup_destination_service as dest + + line = f"{host_key.get_name()} {host_key.get_base64()}" + stored = { + "line": line, + "known_hosts": f"[nas]:2222 {line}", + "fingerprint": dest.host_key_fingerprint(host_key), + }[form] + server = _SshServer(host_key) + pinned = [] + _connect(server.cfg(host_key=stored), pinned.append) + assert pinned == [] # nothing new to pin + assert server.passwords == ["pw"] + + +def test_another_key_is_refused_before_the_password_is_sent(host_key): + import paramiko + + from services import backup_destination_service as dest + + impostor = _SshServer(host_key) + real = paramiko.ECDSAKey.generate() + with pytest.raises(dest.DestinationError, match="does not match"): + _connect(impostor.cfg(host_key=f"{real.get_name()} {real.get_base64()}")) + assert impostor.passwords == [] + + +def test_the_fingerprint_of_a_stored_key_is_the_one_ssh_keygen_prints(host_key): + from services import backup_destination_service as dest + + line = f"{host_key.get_name()} {host_key.get_base64()}" + assert dest.fingerprint_of(line) == dest.host_key_fingerprint(host_key) + assert dest.fingerprint_of("SHA256:abc") == "SHA256:abc" + + +def test_moving_a_destination_to_another_host_forgets_its_key(as_admin, db): + from sqlmodel import Session + + from database import engine + from models.backup_destination import BackupDestination + from services import backup_destination_service as dest + + with Session(engine) as session: + row = BackupDestination( + name="moving", + type="sftp", + config=dest.dump_config({"host": "old", "username": "u", "host_key": "ssh-ed25519 AAAA"}), + ) + session.add(row) + session.commit() + session.refresh(row) + dest_id = row.id + try: + # The form sends every field back, the shown host key included. + same = {"host": "old", "username": "u", "host_key": "ssh-ed25519 AAAA", "path": "/b"} + as_admin.put(f"/api/backups/destinations/{dest_id}", json={"config": same}) + moved = {**same, "host": "new"} + assert as_admin.put( + f"/api/backups/destinations/{dest_id}", json={"config": moved} + ).status_code == 200 + with Session(engine) as session: + cfg = dest.parse_config(session.get(BackupDestination, dest_id)) + assert cfg["host"] == "new" and "host_key" not in cfg + finally: + with Session(engine) as session: + session.delete(session.get(BackupDestination, dest_id)) + session.commit() diff --git a/backend/tests/test_image_refs_and_ports.py b/backend/tests/test_image_refs_and_ports.py new file mode 100644 index 0000000..da84982 --- /dev/null +++ b/backend/tests/test_image_refs_and_ports.py @@ -0,0 +1,123 @@ +"""Image references, compose project names and published ports. + +Small parsers with large consequences: + +* ``docker.io/library/postgres`` is how many compose files name an image, and + https://docker.io/v2/ is not a registry — so every such image sat at "could + not reach registry" forever and an update never showed up. +* compose refuses ``-p`` with anything but ``[a-z0-9_-]``, so a stack folder + found on disk as ``Nextcloud`` could not be started, stopped or deleted. +* the host's listening ports live in the host's network namespace, which + ``/proc/net`` read from inside the container is not. +""" +from __future__ import annotations + +import pytest + + +@pytest.mark.parametrize( + "image, expected", + [ + ("nginx", ("registry-1.docker.io", "library/nginx", "latest")), + ("docker.io/nginx:1", ("registry-1.docker.io", "library/nginx", "1")), + ("docker.io/library/postgres:16", ("registry-1.docker.io", "library/postgres", "16")), + ("index.docker.io/foo/bar:2", ("registry-1.docker.io", "foo/bar", "2")), + ("ghcr.io/a/b:1@sha256:abc", ("ghcr.io", "a/b", "sha256:abc")), + ("localhost:5000/x", ("localhost:5000", "x", "latest")), + ("registry.example:5000/a/b:tag", ("registry.example:5000", "a/b", "tag")), + ], +) +def test_parse_ref(image, expected): + from services.update_service import parse_ref + + assert parse_ref(image) == expected + + +def test_the_local_digest_is_the_one_for_this_repository(monkeypatch): + """An image pulled under two names carries a digest per repository.""" + from services import update_service + + class Image: + attrs = {"RepoDigests": [ + "lscr.io/linuxserver/radarr@sha256:aaa", + "linuxserver/radarr@sha256:bbb", + ]} + + class Images: + def get(self, name): + return Image() + + class Client: + images = Images() + + monkeypatch.setattr(update_service, "get_client", lambda: Client()) + assert update_service._local_digest("linuxserver/radarr:latest") == "sha256:bbb" + assert update_service._local_digest("lscr.io/linuxserver/radarr:latest") == "sha256:aaa" + assert update_service._local_digest("docker.io/linuxserver/radarr") == "sha256:bbb" + + +@pytest.mark.parametrize( + "stack_id, project", + [ + ("immich", "immich"), + ("arr_stack-2", "arr_stack-2"), + ("Nextcloud", "nextcloud"), + ("my.app", "myapp"), + ("_hidden-ish", "hidden-ish"), + ("...", "stack"), + ], +) +def test_project_name_matches_compose(stack_id, project): + from services.compose_service import project_name + + assert project_name(stack_id) == project + + +def test_compose_is_called_with_the_normalised_project(): + from services import compose_service + + cmd = compose_service._compose_base_cmd("Nextcloud") + assert cmd[cmd.index("-p") + 1] == "nextcloud" + + +def test_published_ports_including_ranges_and_long_syntax(): + from services.port_service import parse_compose_ports + + ports = parse_compose_ports( + "services:\n a:\n ports:\n" + ' - "8080:80"\n' + ' - "127.0.0.1:9000:9000/udp"\n' + ' - "7000-7002:7000-7002"\n' + ' - "80"\n' + ' - {target: 80, published: "8443"}\n' + ' - "[::1]:5353:53/udp"\n' + ) + assert [(p["host_port"], p["protocol"]) for p in ports] == [ + (8080, "tcp"), (9000, "udp"), (7000, "tcp"), (7001, "tcp"), (7002, "tcp"), + (8443, "tcp"), (5353, "udp"), + ] + + +def test_garbage_compose_yields_no_ports(): + from services.port_service import parse_compose_ports + + assert parse_compose_ports("just a string") == [] + assert parse_compose_ports("services: [1, 2]") == [] + assert parse_compose_ports(": : :") == [] + + +def test_host_ports_are_read_from_the_hosts_namespace(tmp_path, monkeypatch): + """/proc/net is self/net; PID 1 of the host /proc is the host's namespace.""" + from config import settings + from services import port_service + + (tmp_path / "net").mkdir() + (tmp_path / "1" / "net").mkdir(parents=True) + header = " sl local_address rem_address st\n" + # The container's own table: just its API port. + (tmp_path / "net" / "tcp").write_text(header + " 0: 00000000:1390 00000000:0000 0A\n") + # The host's: ssh on 22 (0x16). + (tmp_path / "1" / "net" / "tcp").write_text(header + " 0: 00000000:0016 00000000:0000 0A\n") + monkeypatch.setattr(settings, "HOST_PROC_PATH", str(tmp_path)) + + assert port_service._read_proc_net("tcp") == {22} diff --git a/backend/tests/test_notifications.py b/backend/tests/test_notifications.py new file mode 100644 index 0000000..f32dd9a --- /dev/null +++ b/backend/tests/test_notifications.py @@ -0,0 +1,58 @@ +"""Webhook delivery. + +Two ways a notification silently went nowhere before 0.67.1: an ntfy title with +a non-ASCII character (a destination called "NAS Büro", an em dash) raised +UnicodeEncodeError inside httpx, and any error that was not an +``httpx.HTTPError`` escaped the per-webhook handler and aborted delivery to +every webhook after it. +""" +from __future__ import annotations + +import asyncio +import email.header + +import httpx + + +def test_a_non_ascii_ntfy_title_is_rfc2047_encoded(): + from services.notify_service import _build_request + + kwargs = _build_request("ntfy", "http://ntfy/t", "backup_failed", "Backup to NAS Büro — failed", "m") + request = httpx.Request("POST", kwargs["url"], headers=kwargs["headers"], content=kwargs["content"]) + (decoded, charset), = email.header.decode_header(request.headers["title"]) + assert decoded.decode(charset) == "Backup to NAS Büro — failed" + + +def test_an_ascii_title_is_sent_as_is(): + from services.notify_service import _header_value + + assert _header_value("Stack 'x' started") == "Stack 'x' started" + + +def test_one_broken_webhook_does_not_stop_the_others(monkeypatch): + from services import notify_service + + delivered = [] + + class Client: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def post(self, url, timeout=None, **kwargs): + if "broken" in url: + raise ValueError("not an httpx error") + delivered.append(url) + return httpx.Response(200, request=httpx.Request("POST", url)) + + monkeypatch.setattr(notify_service.httpx, "AsyncClient", lambda **kw: Client()) + monkeypatch.setattr( + notify_service, + "_targets_for_event", + lambda session, event: [("generic", "http://broken/"), ("generic", "http://fine/")], + ) + count = asyncio.run(notify_service.notify("stack_start", "t", "m", session=object())) + assert count == 1 + assert delivered == ["http://fine/"] diff --git a/backend/tests/test_registries.py b/backend/tests/test_registries.py index 29cace7..e30cdf5 100644 --- a/backend/tests/test_registries.py +++ b/backend/tests/test_registries.py @@ -331,7 +331,7 @@ def _refusing(monkeypatch): raise update_service.AuthRequired("ghcr.io") monkeypatch.setattr(update_service, "remote_digest", refuse) - monkeypatch.setattr(update_service, "_local_digest", lambda image: "sha256:local") + monkeypatch.setattr(update_service, "_local_digest", lambda image, attrs=None: "sha256:local") return update_service @@ -370,6 +370,6 @@ def test_a_registry_we_simply_cannot_reach_still_says_that(monkeypatch): return None monkeypatch.setattr(update_service, "remote_digest", unreachable) - monkeypatch.setattr(update_service, "_local_digest", lambda image: "sha256:local") + monkeypatch.setattr(update_service, "_local_digest", lambda image, attrs=None: "sha256:local") result = asyncio.run(update_service.check_image("ghcr.io/acme/app:1.0")) assert result.error == "could not reach registry" diff --git a/backend/tests/test_review_regressions.py b/backend/tests/test_review_regressions.py new file mode 100644 index 0000000..36d64fc --- /dev/null +++ b/backend/tests/test_review_regressions.py @@ -0,0 +1,137 @@ +"""Smaller regressions found in the 0.67 review, one test each. + +* NVIDIA cards were never detected: the image has no nvidia-smi and the DRI + scan skips NVIDIA on purpose. The driver's /proc entries are read instead. +* A container's inspect view handed its environment — credentials, by + convention — to the read-only role, which does not get the stack's .env. +* Git branch names and URLs go to git's command line; a leading "-" made them + options. +* "Scan all" ran for longer than the proxy waits; it now runs in the + background and reports through /scan-status. +""" +from __future__ import annotations + +import asyncio + +import pytest + + +def test_nvidia_gpus_are_read_from_the_driver_procfs(tmp_path, monkeypatch): + from config import settings + from services import gpu_service + + gpus = tmp_path / "driver" / "nvidia" / "gpus" + (gpus / "0000:01:00.0").mkdir(parents=True) + (gpus / "0000:01:00.0" / "information").write_text( + "Model: \t\t NVIDIA GeForce RTX 3060\n" + "IRQ: \t\t 150\n" + "GPU UUID: \t GPU-1234-abcd\n" + "Device Minor: \t 0\n" + ) + (tmp_path / "driver" / "nvidia" / "version").write_text( + "NVRM version: NVIDIA UNIX x86_64 Kernel Module 550.120 Fri Sep 13 2024\n" + ) + monkeypatch.setattr(settings, "HOST_PROC_PATH", str(tmp_path)) + monkeypatch.setattr(gpu_service, "_detect_nvidia", lambda: []) + monkeypatch.setattr(gpu_service, "_detect_dri", lambda: []) + + (gpu,) = gpu_service.detect_gpus() + assert gpu.vendor == "nvidia" + assert gpu.name == "NVIDIA GeForce RTX 3060" + assert gpu.uuid == "GPU-1234-abcd" + assert gpu.index == 0 + assert gpu.driver == "550.120" + + +@pytest.fixture +def fake_container(monkeypatch): + from services import container_service + + monkeypatch.setattr( + container_service, + "inspect_container", + lambda container_id: {"id": container_id, "env": ["DB_PASSWORD=hunter2", "NOVALUE"]}, + ) + + +def test_container_env_values_are_hidden_from_the_read_only_role(as_user, fake_container): + env = as_user.get("/api/containers/abc").json()["env"] + assert env == ["DB_PASSWORD=••••••", "NOVALUE"] + + +def test_admins_still_see_the_environment(as_admin, fake_container): + assert as_admin.get("/api/containers/abc").json()["env"][0] == "DB_PASSWORD=hunter2" + + +def test_a_missing_container_is_a_404(as_admin, monkeypatch): + from docker_client import DockerError + from services import container_service + + def missing(container_id): + raise DockerError("not_found", "No such container") + + monkeypatch.setattr(container_service, "inspect_container", missing) + assert as_admin.get("/api/containers/nope").status_code == 404 + + +@pytest.mark.parametrize("branch", ["-x", "--upload-pack=sh", "a..b", "x.lock", ""]) +def test_branch_names_that_git_would_read_as_options_are_refused(branch): + from services import git_service + + with pytest.raises(git_service.GitError): + git_service.validate_branch(branch) + + +@pytest.mark.parametrize("url", ["-oProxyCommand=x", "ext::sh -c id", "fd::3", ""]) +def test_repository_urls_that_run_commands_are_refused(url): + from services import git_service + + with pytest.raises(git_service.GitError): + git_service.validate_url(url) + + +def test_ordinary_branches_and_urls_pass(): + from services import git_service + + assert git_service.validate_branch("release/1.2") == "release/1.2" + assert git_service.validate_url("git@forge:me/stack.git") == "git@forge:me/stack.git" + + +def test_scan_all_in_the_background_is_active_before_the_request_returns(monkeypatch, db): + from services import scan_service + + release = asyncio.Event() + + async def slow_scan(session, image): + await release.wait() + + class Row: + error = None + + return Row() + + monkeypatch.setattr(scan_service, "scan", slow_scan) + + async def scenario(): + status = scan_service.start_sweep(["a", "b"]) + assert status["active"] is True and status["total"] == 2 + with pytest.raises(scan_service.ScanError): + scan_service.start_sweep(["c"]) + release.set() + for _ in range(100): + if not scan_service.sweep_status()["active"]: + break + await asyncio.sleep(0.01) + final = scan_service.sweep_status() + assert final["active"] is False + assert final["scanned"] == 2 and final["failed"] == 0 + + asyncio.run(scenario()) + + +def test_setup_refuses_an_empty_account(client, db, monkeypatch): + import auth as auth_mod + + monkeypatch.setattr(auth_mod, "users_exist", lambda session: False) + response = client.post("/api/auth/setup", json={"username": " ", "password": ""}) + assert response.status_code == 400 diff --git a/backend/tests/test_stack_delete.py b/backend/tests/test_stack_delete.py new file mode 100644 index 0000000..b7b9139 --- /dev/null +++ b/backend/tests/test_stack_delete.py @@ -0,0 +1,186 @@ +"""Deleting a stack: what goes with it, and what must not. + +Before 0.67.1 a deleted stack left its Git source behind — and the poll loop or +the next push webhook recreated the folder from the repository and started the +stack again. Its auto-update policy was inherited by the next stack of the same +name, and every backup schedule naming it failed from then on. Deleting the +folder also walked into whatever was mounted inside it (a NAS share at +``./media``) and deleted that too. +""" +from __future__ import annotations + +import asyncio +import json +import os + +import pytest +from sqlmodel import Session, delete, select + + +@pytest.fixture(autouse=True) +def clean(db): + from database import engine + from models.auto_update import AutoUpdate + from models.backup_schedule import BackupSchedule + from models.git_source import GitSource + from models.runtime_state import StackLock + + def wipe(): + with Session(engine) as session: + for model in (GitSource, AutoUpdate, BackupSchedule, StackLock): + session.exec(delete(model)) + session.commit() + + wipe() + yield + wipe() + + +def _schedule(session, stack_ids, all_stacks=False): + from models.backup_schedule import BackupSchedule, set_stacks + + schedule = BackupSchedule(destination_id=1, all_stacks=all_stacks) + set_stacks(schedule, stack_ids) + session.add(schedule) + session.commit() + session.refresh(schedule) + return schedule.id + + +def test_forget_stack_drops_what_refers_to_it(db): + from database import engine + from models.auto_update import AutoUpdate + from models.backup_schedule import BackupSchedule, stacks_of + from models.git_source import GitSource + from services import stack_cleanup_service + + with Session(engine) as session: + session.add(GitSource(stack_id="gone", url="https://example.invalid/x.git")) + session.add(AutoUpdate(stack_id="gone")) + session.add(AutoUpdate(stack_id="stays")) + both = _schedule(session, ["gone", "stays"]) + only = _schedule(session, ["gone"]) + everything = _schedule(session, [], all_stacks=True) + + report = stack_cleanup_service.forget_stack(session, "gone") + + assert report["git"] is True and report["auto_update"] == 1 + assert session.exec(select(GitSource)).all() == [] + assert [p.stack_id for p in session.exec(select(AutoUpdate)).all()] == ["stays"] + assert stacks_of(session.get(BackupSchedule, both)) == ["stays"] + lonely = session.get(BackupSchedule, only) + assert stacks_of(lonely) == [] and lonely.enabled is False + assert "was deleted" in lonely.last_status + assert session.get(BackupSchedule, everything).enabled is True + + +def test_orphans_from_older_versions_are_forgotten_at_startup(db): + from database import engine + from models.git_source import GitSource + from models.stack import Stack + from services import stack_cleanup_service + + with Session(engine) as session: + if not session.get(Stack, "alive"): + session.add(Stack(id="alive", name="alive")) + session.add(GitSource(stack_id="alive", url="https://example.invalid/a.git")) + session.add(GitSource(stack_id="long-deleted", url="https://example.invalid/b.git")) + session.commit() + + assert stack_cleanup_service.prune_orphans(session) == ["long-deleted"] + assert [g.stack_id for g in session.exec(select(GitSource)).all()] == ["alive"] + + +def test_git_never_syncs_a_stack_that_no_longer_exists(db): + from database import engine + from models.git_source import GitSource + from services import git_service + + with Session(engine) as session: + source = GitSource(stack_id="ghost", url="https://example.invalid/g.git") + session.add(source) + session.commit() + session.refresh(source) + with pytest.raises(git_service.GitError, match="no longer exists"): + asyncio.run(git_service.sync(session, source)) + + +def test_deleting_a_busy_stack_is_refused(as_admin, db): + from database import engine + from models.stack import Stack + from services import stack_lock_service + + with Session(engine) as session: + if not session.get(Stack, "busy-del"): + session.add(Stack(id="busy-del", name="busy")) + session.commit() + stack_lock_service.acquire(session, "busy-del", "backup", "scheduled") + + response = as_admin.delete("/api/stacks/busy-del") + assert response.status_code == 409 + assert "backup" in response.json()["detail"] + + +def test_folder_removal_stops_at_mount_points(tmp_path, monkeypatch): + from services import compose_service + + stack = tmp_path / "media-stack" + (stack / "config").mkdir(parents=True) + (stack / "config" / "app.db").write_text("x") + (stack / "compose.yaml").write_text("services: {}\n") + share = stack / "media" + share.mkdir() + (share / "movie.mkv").write_text("precious") + + monkeypatch.setattr(compose_service, "_mount_points", lambda: {str(share)}) + kept = compose_service.delete_stack_files("media-stack", override=str(tmp_path)) + + assert kept == [str(share)] + assert (share / "movie.mkv").read_text() == "precious" + assert not (stack / "config").exists() and not (stack / "compose.yaml").exists() + + +def test_cloning_copies_links_as_links_and_skips_sockets(tmp_path): + import socket + + from services import compose_service + + src = tmp_path / "orig" + src.mkdir() + (src / "compose.yaml").write_text("services: {}\n") + os.symlink("/definitely/not/there", src / "dangling") + sock = socket.socket(socket.AF_UNIX) + sock.bind(str(src / "app.sock")) + try: + compose_service.clone_stack_files("orig", "copy", override=str(tmp_path)) + finally: + sock.close() + + dst = tmp_path / "copy" + assert (dst / "compose.yaml").exists() + assert os.readlink(dst / "dangling") == "/definitely/not/there" + assert not os.path.lexists(dst / "app.sock") + + +def test_a_deleted_stack_name_does_not_inherit_its_schedules(as_admin, db, monkeypatch): + """End to end through the API, with Docker stubbed out.""" + from database import engine + from models.backup_schedule import BackupSchedule, stacks_of + from services import compose_service + + async def down(stack_id, override=None): + return {"returncode": 0, "stdout": "", "stderr": ""} + + monkeypatch.setattr(compose_service, "down", down) + assert as_admin.post("/api/stacks", json={"name": "del-me"}).status_code == 201 + with Session(engine) as session: + schedule_id = _schedule(session, ["del-me"]) + + response = as_admin.delete("/api/stacks/del-me") + assert response.status_code == 200, response.text + assert response.json()["kept"] == [] + with Session(engine) as session: + schedule = session.get(BackupSchedule, schedule_id) + assert stacks_of(schedule) == [] and not schedule.enabled + assert not os.path.isdir(compose_service.stack_dir("del-me")) + assert json.loads(as_admin.get("/api/stacks").text) is not None diff --git a/backend/tests/test_walkthrough_fixes.py b/backend/tests/test_walkthrough_fixes.py new file mode 100644 index 0000000..70aa546 --- /dev/null +++ b/backend/tests/test_walkthrough_fixes.py @@ -0,0 +1,242 @@ +"""Found clicking through the UI for 0.67.1. + +* The file browser opens at ``/``, which the default roots (/mnt, /opt, …) + do not include: the page opened on "outside the allowed browse roots" and + offered no way into any root. Paths above the roots now list the way to them. +* A root could not be deleted, but it could be moved — the whole tree with it. +* Locally built images (compose ``build:``) were checked against Docker Hub, + which answers 401 for a repository that does not exist: "needs credentials". +* Volumes, networks and images of a stack whose folder name compose had to + normalise ("Nextcloud" -> project "nextcloud") were grouped as leftovers of a + removed stack. +""" +from __future__ import annotations + +import asyncio + +import pytest + + +@pytest.fixture +def roots(tmp_path, monkeypatch): + from config import settings + + (tmp_path / "srv" / "data").mkdir(parents=True) + (tmp_path / "opt").mkdir() + monkeypatch.setattr(settings, "HOST_ROOT_PREFIX", str(tmp_path), raising=False) + monkeypatch.setattr( + settings, "ALLOWED_BROWSE_ROOTS", ["/opt", "/srv/data", "/media"], raising=False + ) + return tmp_path + + +def test_the_top_of_the_tree_lists_the_way_to_the_roots(roots): + from services import device_service + + top = device_service.browse("/") + assert top["virtual"] is True + # /media does not exist on this host, so it is not offered. + assert [e["name"] for e in top["entries"]] == ["opt", "srv"] + assert [e["name"] for e in device_service.browse("/srv")["entries"]] == ["data"] + inside = device_service.browse("/srv/data") + assert "virtual" not in inside and inside["parent"] == "/srv" + + +def test_a_path_leading_nowhere_is_still_refused(roots): + from services import device_service + + with pytest.raises(device_service.BrowseError): + device_service.browse("/etc") + + +def test_a_root_cannot_be_moved_or_renamed(roots): + from services import file_service + + with pytest.raises(file_service.BrowseError, match="browse root"): + file_service.move("/srv/data", "/opt") + with pytest.raises(file_service.BrowseError, match="browse root"): + file_service.rename("/opt", "opt2") + assert (roots / "srv" / "data").is_dir() + + +def test_moving_inside_a_root_still_works(roots): + from services import file_service + + (roots / "opt" / "a.txt").write_text("x") + file_service.move("/opt/a.txt", "/srv/data") + assert (roots / "srv" / "data" / "a.txt").read_text() == "x" + + +@pytest.mark.parametrize( + "attrs, local", + [ + ({"Identity": {"Build": [{"Ref": "x"}]}, "RepoDigests": ["app@sha256:1"]}, True), + ({"Identity": {"Pull": [{"Repository": "docker.io/library/nginx"}]}}, False), + ({"RepoDigests": []}, True), # older daemon, never pulled + ({"RepoDigests": ["nginx@sha256:1"]}, False), + ({"Identity": {}, "RepoDigests": ["nginx@sha256:1"]}, False), + (None, False), + ], +) +def test_where_an_image_came_from(attrs, local): + from services import update_service + + assert update_service._built_locally(attrs) is local + + +def test_a_local_build_is_not_asked_about_at_the_registry(monkeypatch): + from services import update_service + + monkeypatch.setattr( + update_service, "_local_attrs", lambda image: {"Identity": {"Build": [{"Ref": "x"}]}} + ) + + async def no_registry(image): + raise AssertionError("registry contacted for a local build") + + monkeypatch.setattr(update_service, "remote_digest", no_registry) + status = asyncio.run(update_service.check_image("myapp-web:latest")) + assert status.local is True and status.error is None and not status.update_available + + +def test_resources_of_a_normalised_stack_are_grouped_under_it(as_admin, db, monkeypatch): + from sqlmodel import Session + + from database import engine + from models.stack import Stack + from services import network_service, volume_service + + with Session(engine) as session: + if not session.get(Stack, "Nextcloud"): + session.add(Stack(id="Nextcloud", name="Nextcloud")) + session.commit() + monkeypatch.setattr( + volume_service, "list_volumes", lambda: [{"name": "nextcloud_db", "stack": "nextcloud"}] + ) + monkeypatch.setattr( + network_service, + "list_networks", + lambda: [{"name": "x", "stack": "elsewhere"}, {"name": "bridge", "stack": None}], + ) + assert as_admin.get("/api/volumes").json()[0]["stack"] == "Nextcloud" + assert [n["stack"] for n in as_admin.get("/api/networks").json()] == ["elsewhere", None] + + +class _Container: + def __init__(self, mounts): + self.attrs = {"Mounts": mounts} + + +@pytest.mark.parametrize( + "mounts, host", + [ + # The usual: same path on both sides — no warning. + ([{"Type": "bind", "Source": "/opt/stacks", "Destination": "/opt/stacks"}], None), + # A real mismatch. + ( + [{"Type": "bind", "Source": "/srv/compose", "Destination": "/opt/stacks"}], + "/srv/compose", + ), + # The folder above STACKS_DIR is what is mounted. + ([{"Type": "bind", "Source": "/srv", "Destination": "/opt"}], "/srv/stacks"), + # Not mounted at all. + ([{"Type": "volume", "Source": "/var/lib/docker/volumes/x", "Destination": "/data"}], None), + ], +) +def test_the_host_stacks_path_comes_from_docker_not_mountinfo(monkeypatch, mounts, host): + """mountinfo's root field is relative to the filesystem: a stacks folder on + its own disk or btrfs subvolume showed a mismatch that was not there.""" + from config import settings + from services import self_update_service, stack_assets_service + + monkeypatch.setattr(settings, "STACKS_DIR", "/opt/stacks") + monkeypatch.setattr(stack_assets_service, "_HOST_STACKS_DIR", None) + monkeypatch.setattr(self_update_service, "_own_container", lambda: _Container(mounts)) + mismatch = stack_assets_service.stacks_path_mismatch() + assert (mismatch["host"] if mismatch else None) == host + + +def test_backup_names_carry_the_time_on_the_users_clock(monkeypatch): + """Named in UTC, a backup taken at 12:21 in Berlin read as …-102154.""" + from datetime import datetime, timezone + + from services import backup_service, compose_service + + monkeypatch.setattr( + compose_service, "now", lambda: datetime(2026, 10, 15, 10, 21, 54, tzinfo=timezone.utc) + ) + berlin = backup_service.backup_filename("app", True, tz="Europe/Berlin") + assert berlin == "backup-app-full-20261015-122154.tar.gz" + # No zone, or one that does not exist: UTC, as before. + assert backup_service.backup_filename("app", False).endswith("-20261015-102154.tar.gz") + assert backup_service.backup_filename("app", False, tz="Mars/Olympus").endswith("-102154.tar.gz") + # Retention still recognises the files. + assert backup_service.backup_name_pattern("app").match(berlin) + + +COMPOSE_WITH_SECRETS = """services: + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: hunter2 + POSTGRES_USER: app + API_KEY: ${API_KEY:-changeme} + DB_PASS: ${DB_PASS} + KEYCLOAK_ADMIN: admin + app: + environment: + - "TS_AUTHKEY=tskey-123" + - TZ=Europe/Berlin +""" + + +def test_inline_credentials_are_masked_and_the_rest_left_alone(): + from services.compose_service import MASK, mask_secrets + + masked = mask_secrets(COMPOSE_WITH_SECRETS) + assert "hunter2" not in masked and "tskey-123" not in masked and "changeme" not in masked + assert f"POSTGRES_PASSWORD: {MASK}" in masked + assert f'- "TS_AUTHKEY={MASK}"' in masked + # A bare variable is no secret; names that merely contain "key" are not either. + for kept in ("DB_PASS: ${DB_PASS}", "KEYCLOAK_ADMIN: admin", "POSTGRES_USER: app", "TZ=Europe/Berlin"): + assert kept in masked + assert masked.count("\n") == COMPOSE_WITH_SECRETS.count("\n") + + +def test_the_read_only_role_gets_the_masked_compose_file(as_admin, as_user, db): + from services import compose_service + + assert as_admin.post("/api/stacks", json={"name": "masked", "yaml": COMPOSE_WITH_SECRETS}).status_code == 201 + try: + assert "hunter2" in as_admin.get("/api/stacks/masked").json()["yaml"] + assert "hunter2" not in as_user.get("/api/stacks/masked").json()["yaml"] + finally: + compose_service.delete_stack_files("masked") + + +def test_scan_all_covers_only_stackpilots_own_stacks(as_admin, db, monkeypatch): + from sqlmodel import Session + + from database import engine + from models.stack import Stack + from services import image_service, scan_service + + with Session(engine) as session: + if not session.get(Stack, "Mine"): + session.add(Stack(id="Mine", name="Mine")) + session.commit() + monkeypatch.setattr( + image_service, + "list_images", + lambda: [ + {"tag": "mine:1", "stacks": ["mine"]}, + {"tag": "theirs:1", "stacks": ["some-other-project"]}, + {"tag": "loose:1", "stacks": []}, + ], + ) + swept = [] + monkeypatch.setattr( + scan_service, "start_sweep", lambda images, on_done=None: swept.append(images) or {"active": True} + ) + assert as_admin.post("/api/images/scan-all?background=true").status_code == 202 + assert swept == [["mine:1"]] diff --git a/backend/version.py b/backend/version.py index 1f3b5c5..5cc1588 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1,3 +1,3 @@ """Single source of truth for the StackPilot release version.""" -APP_VERSION = "0.67.0" +APP_VERSION = "0.67.1" diff --git a/docker-compose.yml b/docker-compose.yml index 08f2ccc..ac8f847 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,10 @@ services: image: git.menzel.center/menzeljonas/stackpilot-frontend:latest build: ./frontend restart: unless-stopped + # Started once the API answers: before, nginx answered 502 for the first + # seconds after every (re)start. depends_on: - - backend + backend: + condition: service_healthy ports: - "5009:80" diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 05b1a88..9779e68 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,7 +1,10 @@ FROM node:20-alpine AS build WORKDIR /app -COPY package.json ./ -RUN npm install +# The lockfile, and `npm ci`: `npm install` against package.json alone +# resolved whatever the caret ranges allowed on the day of the build, so the +# image could ship dependency versions CI never typechecked or tested. +COPY package.json package-lock.json ./ +RUN npm ci COPY . . RUN npm run build diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 3e419b9..1680f05 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,3 +1,11 @@ +# Behind a TLS-terminating proxy (Traefik, NPM, Caddy) this nginx itself only +# sees plain http: passing $scheme on would tell the backend "http" for an +# https site. Keep what the outer proxy said, and fall back to our own scheme. +map $http_x_forwarded_proto $sp_forwarded_proto { + default $http_x_forwarded_proto; + "" $scheme; +} + server { listen 80; server_name _; @@ -12,17 +20,24 @@ server { # API proxy location /api/ { proxy_pass http://backend:5008; - proxy_set_header Host $host; + # $http_host, not $host: $host drops the port, so with the default + # "5009:80" mapping the OIDC redirect URI the backend derives from it + # pointed at port 80. + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # Without this uvicorn sees plain http even behind TLS, which would make # the OIDC redirect URI it derives wrong and the refresh cookie miss its # Secure flag. - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $sp_forwarded_proto; proxy_read_timeout 600s; # Stack backups (incl. volume data) can be large in both directions. client_max_body_size 0; proxy_request_buffering off; + # A backup download carries its report in a response header; past the + # default 4-8k header buffer nginx answers 502 for a backup that worked. + proxy_buffer_size 16k; + proxy_buffers 8 16k; } # WebSocket proxy @@ -31,7 +46,10 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $sp_forwarded_proto; proxy_read_timeout 86400s; } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index edf9d9a..b29f0ad 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "stackpilot-frontend", - "version": "0.49.0", + "version": "0.67.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "stackpilot-frontend", - "version": "0.49.0", + "version": "0.67.0", "dependencies": { "@fontsource-variable/schibsted-grotesk": "^5.2.8", "@monaco-editor/react": "^4.6.0", @@ -16,6 +16,7 @@ "axios": "^1.7.9", "clsx": "^2.1.1", "lucide-react": "^0.468.0", + "monaco-editor": "^0.57.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.28.0", @@ -618,9 +619,9 @@ } }, "node_modules/@remix-run/router": { - "version": "1.23.3", - "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.3.tgz", - "integrity": "sha512-4An71tdz9X8+3sI4Qqqd2LWd9vS39J7sqd9EU4Scw7TJE/qB10Flv/UuqbPVgfQV9XoK8Np6jNquZitnZq5i+Q==", + "version": "1.23.4", + "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.4.tgz", + "integrity": "sha512-q7j5geK7xs3UJSdm9/iytUNclBnLmYx1EnSeCFXHPeutdqgIMeFeHtUZgS3EhlKxdBEAu8OwtJCwmLrEzpSs7Q==", "license": "MIT", "engines": { "node": ">=14.0.0" @@ -1145,8 +1146,7 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true, - "peer": true + "optional": true }, "node_modules/@vitejs/plugin-react": { "version": "4.7.0", @@ -1170,14 +1170,14 @@ } }, "node_modules/@vitest/expect": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.8.tgz", - "integrity": "sha512-8ytZ/fFHq2g4PJVAtDX57mayemKgDR6X3Oa2Foro+EygiOJHUXhCqBAAKQYYajZpFoIfvBCF1j6R6IYRSIUFuw==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", + "integrity": "sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "2.1.8", - "@vitest/utils": "2.1.8", + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", "chai": "^5.1.2", "tinyrainbow": "^1.2.0" }, @@ -1186,13 +1186,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.8.tgz", - "integrity": "sha512-7guJ/47I6uqfttp33mgo6ga5Gr1VnL58rcqYKyShoRK9ebu8T5Rs6HN3s1NABiBeVTdWNrwUMcHH54uXZBN4zA==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.9.tgz", + "integrity": "sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "2.1.8", + "@vitest/spy": "2.1.9", "estree-walker": "^3.0.3", "magic-string": "^0.30.12" }, @@ -1226,13 +1226,13 @@ } }, "node_modules/@vitest/runner": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.8.tgz", - "integrity": "sha512-17ub8vQstRnRlIU5k50bG+QOMLHRhYPAna5tw8tYbj+jzjcspnwnwtPtiOlkuKC4+ixDPTuLZiqiWWQ2PSXHVg==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.9.tgz", + "integrity": "sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "2.1.8", + "@vitest/utils": "2.1.9", "pathe": "^1.1.2" }, "funding": { @@ -1240,13 +1240,13 @@ } }, "node_modules/@vitest/snapshot": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.8.tgz", - "integrity": "sha512-20T7xRFbmnkfcmgVEz+z3AU/3b0cEzZOt/zmnvZEctg64/QZbSDJEVm9fLnnlSi74KibmRsO9/Qabi+t0vCRPg==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.9.tgz", + "integrity": "sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.1.8", + "@vitest/pretty-format": "2.1.9", "magic-string": "^0.30.12", "pathe": "^1.1.2" }, @@ -1254,23 +1254,10 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/snapshot/node_modules/@vitest/pretty-format": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.8.tgz", - "integrity": "sha512-9HiSZ9zpqNLKlbIDRWOnAWqgcA7xu+8YxXSekhr0Ykab7PAYFkhkwoqVArPOtJhPmYeE2YHgKZlj3CP36z2AJQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^1.2.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/@vitest/spy": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.8.tgz", - "integrity": "sha512-5swjf2q95gXeYPevtW0BLk6H8+bPlMb4Vw/9Em4hFxDcaOxS+e0LOX4yqNxoHzMR2akEB2xfpnWUzkZokmgWDg==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.9.tgz", + "integrity": "sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1281,13 +1268,13 @@ } }, "node_modules/@vitest/utils": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.8.tgz", - "integrity": "sha512-dwSoui6djdwbfFmIgbIjX2ZhIoG7Ex/+xpxyiEgIGzjliY8xGkcpITKTlp6B4MgtGkF2ilvm97cPM96XZaAgcA==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.9.tgz", + "integrity": "sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.1.8", + "@vitest/pretty-format": "2.1.9", "loupe": "^3.1.2", "tinyrainbow": "^1.2.0" }, @@ -1295,19 +1282,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/utils/node_modules/@vitest/pretty-format": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.8.tgz", - "integrity": "sha512-9HiSZ9zpqNLKlbIDRWOnAWqgcA7xu+8YxXSekhr0Ykab7PAYFkhkwoqVArPOtJhPmYeE2YHgKZlj3CP36z2AJQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^1.2.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/@xterm/addon-fit": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.10.0.tgz", @@ -1453,21 +1427,21 @@ } }, "node_modules/axios": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.17.0.tgz", - "integrity": "sha512-J8SwNxprqqpbfenehxWYXE7CW+wM1BB4w3+N+g+/Wx40xM4rsLrfPmHHxSWIxJLYDgSY/HqlFPIYb2/S3rxafw==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", + "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.16.0", - "form-data": "^4.0.5", + "form-data": "^4.0.6", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, "node_modules/baseline-browser-mapping": { - "version": "2.10.34", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.34.tgz", - "integrity": "sha512-IMDedajPifLnHNY0X9n8hKxRTQ6/eTHwr5bDo04WnuqxyKw6LYtQywCuuqPZwhl3aBXMvQpJov42GLCwRRdQzw==", + "version": "2.11.28", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.28.tgz", + "integrity": "sha512-br0LOmjJqr6Rt4AEkadbcNf67Zpo7EPwKsWbjnRrJIa1QYFHQBqM4C7F2WTbV8ahTEcg1Eo+Y0aWjjORInYFyA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1504,9 +1478,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "version": "4.29.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", + "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", "dev": true, "funding": [ { @@ -1524,11 +1498,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.10.12", - "caniuse-lite": "^1.0.30001782", - "electron-to-chromium": "^1.5.328", - "node-releases": "^2.0.36", - "update-browserslist-db": "^1.2.3" + "baseline-browser-mapping": "^2.11.26", + "caniuse-lite": "^1.0.30001813", + "electron-to-chromium": "^1.5.439", + "node-releases": "^2.0.57", + "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" @@ -1571,9 +1545,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001797", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001797.tgz", - "integrity": "sha512-l8xKG+gwAIExZGl9FrF7KUwuOmk6wbEPC9Xoy/RtnWv1XG0Q4LFlagaLpUv3Kiza3W/wm27zy0yWJEieYKAP6w==", + "version": "1.0.30001816", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001816.tgz", + "integrity": "sha512-+8Yx7BXCoqk6IF8yTicjfUR6D/ldJzdlsodlEOiOSPmhK844WIgk67/zIQaGAOxNkZyyvBlrRFnLvqKIoqbAFw==", "dev": true, "funding": [ { @@ -1861,11 +1835,10 @@ "license": "MIT" }, "node_modules/dompurify": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.7.tgz", - "integrity": "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==", + "version": "3.4.15", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.15.tgz", + "integrity": "sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw==", "license": "(MPL-2.0 OR Apache-2.0)", - "peer": true, "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -1885,9 +1858,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.368", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.368.tgz", - "integrity": "sha512-7RckJJK4uESJF9PxvfMWd3TGqIiieUTG4HxnKaKuIpGbcr+r2ZEB3g2gAhCP3Fqm42vJSzLfgab9eva/C4/XVw==", + "version": "1.5.456", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.456.tgz", + "integrity": "sha512-8hboqI6xesozteFb8aa7OGknvxUtTeRJLz0FoePysUEh/oWbddykIGdgpEJRbPRg/nIMfBf2DFos+H2MtvJl2A==", "dev": true, "license": "ISC" }, @@ -2099,16 +2072,16 @@ } }, "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" @@ -2595,7 +2568,6 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -2658,13 +2630,12 @@ } }, "node_modules/monaco-editor": { - "version": "0.55.1", - "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.55.1.tgz", - "integrity": "sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.57.0.tgz", + "integrity": "sha512-5BkI9KGoqrNvBGUe15/QlZq3OooZ8WLg1AxTpaqHRCP3HNpzPPZKE2EDz8M7c+VRmCeUw1Brp4cx/PWm3kI/5A==", "license": "MIT", - "peer": true, "dependencies": { - "dompurify": "3.2.7", + "dompurify": "3.4.15", "marked": "14.0.0" } }, @@ -2687,9 +2658,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", "dev": true, "funding": [ { @@ -2706,9 +2677,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.47", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.47.tgz", - "integrity": "sha512-Uzmd6LXpouKo8EUK68IjH4+E01w/hXyV3R3g/geCJo+rXLNfh1xucB+LOzYEOQPSiUK3h/xZf0cQGcSsmyL2Og==", + "version": "2.0.59", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.59.tgz", + "integrity": "sha512-5qr2okC4ItAI0hG5SaV0HoYMpnM/Y6+Oqr7AmPYy/2uEoGvS+GbSKLeRxSS1uAdu59yz0ouzcuXievSgg///WA==", "dev": true, "license": "MIT", "engines": { @@ -2830,9 +2801,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.30", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.30.tgz", + "integrity": "sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA==", "dev": true, "funding": [ { @@ -2850,9 +2821,9 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.20", "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" + "source-map-js": "^1.2.2" }, "engines": { "node": "^10 || ^12 || >=14" @@ -2972,9 +2943,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3103,12 +3074,12 @@ } }, "node_modules/react-router": { - "version": "6.30.4", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.4.tgz", - "integrity": "sha512-SVUsDe+DybHM/WmYKIVYhZh1o5Dcuf16yM6WjG02Q9XVFMZIJyHYhwrr6bFBXZkVP6z69kNkMyBCujt8FaFLJA==", + "version": "6.30.6", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.6.tgz", + "integrity": "sha512-5HfK7k5im7LTOB0EqCQmfvy4C13G92Ssj1VTmouTK3AJvyjKTnFuCV0vcMAD/JS+JC4DvDIBRrlAeJIFjh5VWg==", "license": "MIT", "dependencies": { - "@remix-run/router": "1.23.3" + "@remix-run/router": "1.23.4" }, "engines": { "node": ">=14.0.0" @@ -3118,13 +3089,13 @@ } }, "node_modules/react-router-dom": { - "version": "6.30.4", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.4.tgz", - "integrity": "sha512-q4HvNl+mmDdkS0g+MqiBZNteQJCuimWoOyHMy4T/RQLAn9Z29+E91QXRaxOujeMl2HTzRSS0KFPd7lxX3PjV0Q==", + "version": "6.30.6", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.6.tgz", + "integrity": "sha512-0RHKZz7wwffvkU+2MFVT2NnjK44ssLEV+m0CAJaS2Ksmorrwj7WxH00jO0SOCW26/tINUnJHToXblDs33I38YQ==", "license": "MIT", "dependencies": { - "@remix-run/router": "1.23.3", - "react-router": "6.30.4" + "@remix-run/router": "1.23.4", + "react-router": "6.30.6" }, "engines": { "node": ">=14.0.0" @@ -3337,9 +3308,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -3673,9 +3644,9 @@ "license": "MIT" }, "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.4.tgz", + "integrity": "sha512-t8wonIOdQyOjtPd2VktlBu6QjTVTks6v8bSi7kd0r2Vll8TK1JYDZtRRdbmCHOAMphkZgrzK+L+vXK4pyZ8t4A==", "dev": true, "funding": [ { @@ -3771,9 +3742,9 @@ } }, "node_modules/vite-node": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.8.tgz", - "integrity": "sha512-uPAwSr57kYjAUux+8E2j0q0Fxpn8M9VoyfGiRI8Kfktz9NcYMCenwY5RnZxnF1WTu3TGiYipirIzacLL3VVGFg==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.9.tgz", + "integrity": "sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==", "dev": true, "license": "MIT", "dependencies": { @@ -3794,19 +3765,19 @@ } }, "node_modules/vitest": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.8.tgz", - "integrity": "sha512-1vBKTZskHw/aosXqQUlVWWlGUxSJR8YtiyZDJAFeW2kPAeX6S3Sool0mjspO+kXLuxVWlEDDowBAeqeAQefqLQ==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.9.tgz", + "integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "2.1.8", - "@vitest/mocker": "2.1.8", - "@vitest/pretty-format": "^2.1.8", - "@vitest/runner": "2.1.8", - "@vitest/snapshot": "2.1.8", - "@vitest/spy": "2.1.8", - "@vitest/utils": "2.1.8", + "@vitest/expect": "2.1.9", + "@vitest/mocker": "2.1.9", + "@vitest/pretty-format": "^2.1.9", + "@vitest/runner": "2.1.9", + "@vitest/snapshot": "2.1.9", + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", "chai": "^5.1.2", "debug": "^4.3.7", "expect-type": "^1.1.0", @@ -3818,7 +3789,7 @@ "tinypool": "^1.0.1", "tinyrainbow": "^1.2.0", "vite": "^5.0.0", - "vite-node": "2.1.8", + "vite-node": "2.1.9", "why-is-node-running": "^2.3.0" }, "bin": { @@ -3833,8 +3804,8 @@ "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "2.1.8", - "@vitest/ui": "2.1.8", + "@vitest/browser": "2.1.9", + "@vitest/ui": "2.1.9", "happy-dom": "*", "jsdom": "*" }, diff --git a/frontend/package.json b/frontend/package.json index deca6d3..c199f5e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "stackpilot-frontend", "private": true, - "version": "0.67.0", + "version": "0.67.1", "type": "module", "scripts": { "dev": "vite", @@ -18,6 +18,7 @@ "axios": "^1.7.9", "clsx": "^2.1.1", "lucide-react": "^0.468.0", + "monaco-editor": "^0.57.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.28.0", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b12b3c9..6d527c2 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,5 +1,12 @@ import { lazy, useEffect } from "react"; -import { BrowserRouter, Navigate, Outlet, Route, Routes } from "react-router-dom"; +import { + Navigate, + Outlet, + Route, + RouterProvider, + createBrowserRouter, + createRoutesFromElements, +} from "react-router-dom"; import { AppShell } from "@/components/layout/AppShell"; import { Login } from "@/pages/Login"; import { Dashboard } from "@/pages/Dashboard"; @@ -46,17 +53,52 @@ function RequireAuth() { } function BootSplash() { + // Set while the backend does not answer yet (starting, restarting after an + // update) — the session is kept and the restore keeps trying. + const waiting = useAuthStore((s) => s.waiting); return ( -
+
Restoring your session… ); } +// A data router rather than : only it supports useBlocker, which +// is what keeps the stack editor from dropping unsaved edits on a stray click. +const router = createBrowserRouter( + createRoutesFromElements( + <> + } /> + }> + }> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + } /> + + ) +); + export default function App() { const applyTheme = useThemeStore((s) => s.apply); const restore = useAuthStore((s) => s.restore); @@ -70,28 +112,5 @@ export default function App() { restore(); }, [restore]); - return ( - - - } /> - }> - }> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - - - } /> - - - ); + return ; } diff --git a/frontend/src/api/backups.ts b/frontend/src/api/backups.ts index bc6997d..b4756f7 100644 --- a/frontend/src/api/backups.ts +++ b/frontend/src/api/backups.ts @@ -1,4 +1,4 @@ -import api from "./client"; +import api, { withBlobError } from "./client"; export interface BackupDestination { id: number; @@ -83,7 +83,17 @@ export interface BackupOptions { } /** Query params for the backup endpoints. */ -export function backupParams(opts: BackupOptions) { +export /** The browser's time zone: the server names backup files in it, so the time + * in the name is the one on the user's clock rather than UTC. */ +function browserZone(): string | undefined { + try { + return Intl.DateTimeFormat().resolvedOptions().timeZone || undefined; + } catch { + return undefined; + } +} + +function backupParams(opts: BackupOptions) { return { include_volumes: opts.includeVolumes, include_binds: opts.includeBinds ?? true, @@ -128,10 +138,12 @@ export const backupsApi = { .then((r) => r.data), download: async (stackId: string, opts: BackupOptions) => { - const res = await api.get(`/api/stacks/${stackId}/backup`, { - params: backupParams(opts), - responseType: "blob", - }); + const res = await withBlobError( + api.get(`/api/stacks/${stackId}/backup`, { + params: { ...backupParams(opts), tz: browserZone() }, + responseType: "blob", + }) + ); const cd = res.headers["content-disposition"] as string | undefined; const match = cd?.match(/filename="?([^"]+)"?/); const name = match?.[1] ?? `backup-${stackId}.tar.gz`; @@ -175,7 +187,7 @@ export const backupsApi = { destination: string; name: string; report?: BackupReport; - }>(`/api/stacks/${stackId}/backup/push`, body) + }>(`/api/stacks/${stackId}/backup/push`, { ...body, tz: browserZone() }) .then((r) => r.data), restoreFrom: (body: { @@ -199,7 +211,8 @@ export const destinationsApi = { api.delete(`/api/backups/destinations/${id}`).then((r) => r.data), test: (id: number) => api - .post<{ ok: boolean; error?: string }>(`/api/backups/destinations/${id}/test`) + // host_key: the SFTP server's pinned key, as a SHA256 fingerprint. + .post<{ ok: boolean; error?: string; host_key?: string }>(`/api/backups/destinations/${id}/test`) .then((r) => r.data), backups: (id: number) => api.get(`/api/backups/destinations/${id}/backups`).then((r) => r.data), diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 8d7a859..36bde92 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -2,7 +2,14 @@ import axios, { AxiosError } from "axios"; import { useAuthStore } from "@/store/auth"; // withCredentials so the httpOnly refresh cookie rides along on /api/auth/refresh. -const api = axios.create({ baseURL: "/", withCredentials: true }); +const api = axios.create({ + baseURL: "/", + withCredentials: true, + // Arrays as `binds=a&binds=b`, which is what FastAPI reads as a list. Axios' + // default `binds[]=a` is a different parameter name to it, so the folders and + // volumes picked in the backup dialog were silently ignored on download. + paramsSerializer: { indexes: null }, +}); api.interceptors.request.use((config) => { const token = useAuthStore.getState().accessToken; @@ -40,6 +47,29 @@ api.interceptors.response.use( } ); +/** + * The server's JSON error body behind a `responseType: "blob"` request (a + * backup download), which axios hands back as a Blob. Without this such a + * failure read "Request failed with status code 409" instead of saying the + * stack was busy. + */ +export async function withBlobError(request: Promise): Promise { + try { + return await request; + } catch (err) { + const e = err as AxiosError; + const data = e?.response?.data; + if (e?.response && typeof Blob !== "undefined" && data instanceof Blob) { + try { + e.response.data = JSON.parse(await data.text()); + } catch { + /* not JSON — keep the generic message */ + } + } + throw err; + } +} + export function apiErrorMessage(err: unknown): string { const e = err as AxiosError; const detail = e?.response?.data?.detail; diff --git a/frontend/src/api/editor.ts b/frontend/src/api/editor.ts index 4fa6cb1..7a638b0 100644 --- a/frontend/src/api/editor.ts +++ b/frontend/src/api/editor.ts @@ -1,9 +1,15 @@ import api from "./client"; export const editorApi = { - validate: (yaml: string, env: string) => + /** With `stackId`, relative paths (env_file, include, secrets) resolve in + * that stack's folder, as they will when it is deployed. */ + validate: (yaml: string, env: string, stackId?: string) => api - .post<{ ok: boolean; errors: string }>("/api/editor/validate", { yaml, env }) + .post<{ ok: boolean; errors: string }>("/api/editor/validate", { + yaml, + env, + stack_id: stackId ?? null, + }) .then((r) => r.data), services: (yaml: string) => api diff --git a/frontend/src/api/files.ts b/frontend/src/api/files.ts index 1c1963d..bc16136 100644 --- a/frontend/src/api/files.ts +++ b/frontend/src/api/files.ts @@ -1,4 +1,4 @@ -import api from "./client"; +import api, { withBlobError } from "./client"; import type { HostPathResult } from "@/types"; export interface FileContent { @@ -61,13 +61,15 @@ export const filesApi = { filename: string, onProgress?: (loaded: number, total: number | undefined) => void, ) => { - const res = await api.get(`${base}/download`, { - params: { path }, - responseType: "blob", - onDownloadProgress: onProgress - ? (e) => onProgress(e.loaded, e.total || undefined) - : undefined, - }); + const res = await withBlobError( + api.get(`${base}/download`, { + params: { path }, + responseType: "blob", + onDownloadProgress: onProgress + ? (e) => onProgress(e.loaded, e.total || undefined) + : undefined, + }) + ); triggerDownload(res.data as Blob, filename); }, diff --git a/frontend/src/api/images.ts b/frontend/src/api/images.ts index 35c7628..28db1e5 100644 --- a/frontend/src/api/images.ts +++ b/frontend/src/api/images.ts @@ -8,6 +8,8 @@ export interface UpdateStatus { remote_digest: string | null; checked_at: number; error: string | null; + /** Built on this host: there is no registry copy to compare against. */ + local?: boolean; } export interface ImageRow { @@ -53,6 +55,17 @@ export interface ScanDetail extends ScanSummary { findings: Finding[]; } +export interface ScanSweepStatus { + active: boolean; + done: number; + total: number; + /** Set once a sweep has finished. */ + scanned: number; + failed: number; + /** Increments per sweep, to tell the one just started from the last. */ + run: number; +} + const base = "/api/images"; export const imagesApi = { @@ -64,12 +77,13 @@ export const imagesApi = { api.get(`${base}/scan`, { params: { image } }).then((r) => r.data), scan: (image: string) => api.post(`${base}/scan`, { image }).then((r) => r.data), - scanAll: () => - api.post<{ scanned: number; failed: number }>(`${base}/scan-all`).then((r) => r.data), - scanStatus: () => + /** Starts the sweep on the server and returns at once; poll scanStatus(). + * Waiting on the response outlived the proxy's 600s timeout. */ + startScanAll: () => api - .get<{ active: boolean; done: number; total: number }>(`${base}/scan-status`) + .post(`${base}/scan-all`, null, { params: { background: true } }) .then((r) => r.data), + scanStatus: () => api.get(`${base}/scan-status`).then((r) => r.data), // Runs in the background on the server; poll pruneStatus() for progress. startPrune: (allUnused: boolean) => api diff --git a/frontend/src/api/stacks.ts b/frontend/src/api/stacks.ts index 1b53599..7c87015 100644 --- a/frontend/src/api/stacks.ts +++ b/frontend/src/api/stacks.ts @@ -33,8 +33,12 @@ export const stacksApi = { env?: string; } ) => api.put(`/api/stacks/${id}`, body).then((r) => r.data), + /** `kept` lists folders left in place because another filesystem (a share, + * a second disk) is mounted there. */ remove: (id: string, deleteFiles = true) => - api.delete(`/api/stacks/${id}?delete_files=${deleteFiles}`).then((r) => r.data), + api + .delete<{ ok: boolean; kept?: string[] }>(`/api/stacks/${id}?delete_files=${deleteFiles}`) + .then((r) => r.data), clone: (id: string, name: string) => api.post(`/api/stacks/${id}/clone`, { name }).then((r) => r.data), diff --git a/frontend/src/components/stacks/AutoUpdatePanel.tsx b/frontend/src/components/stacks/AutoUpdatePanel.tsx index d30aaab..d628ee5 100644 --- a/frontend/src/components/stacks/AutoUpdatePanel.tsx +++ b/frontend/src/components/stacks/AutoUpdatePanel.tsx @@ -4,6 +4,7 @@ import { toast } from "sonner"; import { Badge, Button, Card } from "@/components/ui"; import { autoUpdateApi, type AutoUpdatePolicy } from "@/api/autoUpdate"; import { apiErrorMessage } from "@/api/client"; +import { parseServerTime } from "@/lib/utils"; const STATUS_TONE: Record = { updated: "text-green-500", @@ -110,7 +111,7 @@ export function AutoUpdatePanel({ {p.last_run && (

- Last run {new Date(p.last_run).toLocaleString()} —{" "} + Last run {parseServerTime(p.last_run).toLocaleString()} —{" "} {p.last_status} diff --git a/frontend/src/components/stacks/ContainerTerminal.tsx b/frontend/src/components/stacks/ContainerTerminal.tsx index 395a5a3..e03fbfe 100644 --- a/frontend/src/components/stacks/ContainerTerminal.tsx +++ b/frontend/src/components/stacks/ContainerTerminal.tsx @@ -25,16 +25,20 @@ export function ContainerTerminal({ service: string; onClose: () => void; }) { - const token = useAuthStore((s) => s.accessToken); const [shell, setShell] = useState(SHELLS[0]); const [status, setStatus] = useState("connecting"); const [detail, setDetail] = useState(null); const boxRef = useRef(null); + // Keyed on the container and shell only. With the access token as a + // dependency, every background token refresh tore the session down and + // opened a new shell — whatever was running in the terminal was gone. useEffect(() => { - if (!token || !boxRef.current) return; + if (!boxRef.current) return; setStatus("connecting"); setDetail(null); + let cancelled = false; + let ws: WebSocket | null = null; const term = new Terminal({ fontSize: 13, @@ -47,55 +51,68 @@ export function ContainerTerminal({ term.open(boxRef.current); fit.fit(); - const proto = window.location.protocol === "https:" ? "wss" : "ws"; - const url = - `${proto}://${window.location.host}/ws/exec/${containerId}` + - `?token=${encodeURIComponent(token)}&cmd=${encodeURIComponent(shell)}`; - const ws = new WebSocket(url); - const sendResize = () => { - if (ws.readyState === WebSocket.OPEN) { + if (ws?.readyState === WebSocket.OPEN) { ws.send(JSON.stringify({ type: "resize", rows: term.rows, cols: term.cols })); } }; - ws.onopen = () => { - setStatus("open"); - term.focus(); - sendResize(); - }; - ws.onmessage = (ev) => { - try { - const msg = JSON.parse(ev.data); - if (msg.type === "data") term.write(msg.data as string); - else if (msg.type === "exit") { - term.write(`\r\n\x1b[90m[process exited${msg.code != null ? ` with code ${msg.code}` : ""}]\x1b[0m\r\n`); - setStatus("closed"); - } else if (msg.type === "error") { - term.write(`\r\n\x1b[31m${msg.detail || "error"}\x1b[0m\r\n`); - setStatus("error"); - setDetail(msg.detail || "error"); - } - } catch { - /* non-JSON: write raw */ - term.write(typeof ev.data === "string" ? ev.data : ""); - } - }; - ws.onclose = (ev) => { - if (ev.code === 4403) { + const connect = async () => { + const token = await useAuthStore.getState().freshAccessToken(); + if (cancelled) return; + if (!token) { setStatus("error"); - setDetail("Terminal access is admin-only."); - term.write("\r\n\x1b[31mTerminal access is admin-only.\x1b[0m\r\n"); - } else if (ev.code === 4401) { - setStatus("error"); - setDetail("Authentication failed."); - } else { - setStatus((s) => (s === "open" ? "closed" : s)); + setDetail("Your session has ended — sign in again."); + return; } + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + ws = new WebSocket( + `${proto}://${window.location.host}/ws/exec/${encodeURIComponent(containerId)}` + + `?token=${encodeURIComponent(token)}&cmd=${encodeURIComponent(shell)}` + ); + wire(ws); }; + const wire = (ws: WebSocket) => { + ws.onopen = () => { + setStatus("open"); + term.focus(); + sendResize(); + }; + ws.onmessage = (ev) => { + try { + const msg = JSON.parse(ev.data); + if (msg.type === "data") term.write(msg.data as string); + else if (msg.type === "exit") { + term.write(`\r\n\x1b[90m[process exited${msg.code != null ? ` with code ${msg.code}` : ""}]\x1b[0m\r\n`); + setStatus("closed"); + } else if (msg.type === "error") { + term.write(`\r\n\x1b[31m${msg.detail || "error"}\x1b[0m\r\n`); + setStatus("error"); + setDetail(msg.detail || "error"); + } + } catch { + /* non-JSON: write raw */ + term.write(typeof ev.data === "string" ? ev.data : ""); + } + }; + ws.onclose = (ev) => { + if (ev.code === 4403) { + setStatus("error"); + setDetail("Terminal access is admin-only."); + term.write("\r\n\x1b[31mTerminal access is admin-only.\x1b[0m\r\n"); + } else if (ev.code === 4401) { + setStatus("error"); + setDetail("Authentication failed."); + } else { + setStatus((s) => (s === "open" ? "closed" : s)); + } + }; + }; + void connect(); + const onData = term.onData((d) => { - if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "data", data: d })); + if (ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "data", data: d })); }); const ro = new ResizeObserver(() => { @@ -109,12 +126,13 @@ export function ContainerTerminal({ ro.observe(boxRef.current); return () => { + cancelled = true; ro.disconnect(); onData.dispose(); - ws.close(); + ws?.close(); term.dispose(); }; - }, [containerId, shell, token]); + }, [containerId, shell]); return (

diff --git a/frontend/src/components/stacks/DeployConsole.tsx b/frontend/src/components/stacks/DeployConsole.tsx index e64405a..d7fa3d3 100644 --- a/frontend/src/components/stacks/DeployConsole.tsx +++ b/frontend/src/components/stacks/DeployConsole.tsx @@ -40,15 +40,19 @@ export function DeployConsole({ const [errorDetail, setErrorDetail] = useState(null); const [showLog, setShowLog] = useState(true); const boxRef = useRef(null); - const token = useAuthStore((s) => s.accessToken); const tracker = useMemo(() => new DeployTracker(), []); const bufferRef = useRef([]); const indexRef = useRef(new Map()); const dirtyRef = useRef(false); + // Connected once per stack — deliberately not keyed on the access token. + // Re-running this effect opens a *new* /ws/deploy, i.e. starts another + // `compose up`; with the token as a dependency, every background token + // refresh during (or after) a deploy did exactly that. useEffect(() => { - if (!token) return; + let ws: WebSocket | null = null; + let cancelled = false; // One line per layer/container, updated in place — the same way compose // renders on a TTY — so status changes replace their line instead of @@ -86,38 +90,63 @@ export function DeployConsole({ push(ev.id, `${ev.id} ${ev.text}${details}`, ev.status === "error" ? "error" : "plain"); }; - const proto = window.location.protocol === "https:" ? "wss" : "ws"; - const url = `${proto}://${window.location.host}/ws/deploy/${stackId}?token=${token}`; - const ws = new WebSocket(url); - ws.onmessage = (ev) => { - try { - const msg = JSON.parse(ev.data); - if (msg.type === "log") { - handleLine(msg.line as string); - } else if (msg.type === "done") { - const ok = msg.returncode === 0; - tracker.finish(ok); - dirtyRef.current = true; - setPhase(ok ? "success" : "failed"); - } else if (msg.type === "error") { - tracker.finish(false); - dirtyRef.current = true; - setPhase("error"); - setErrorDetail(msg.detail || "Deploy error"); - } - } catch { - /* ignore */ + const fail = (detail: string) => { + tracker.finish(false); + dirtyRef.current = true; + setPhase("error"); + setErrorDetail(detail); + }; + + // Set once a done/error frame arrived, so the close that follows it is + // not mistaken for a dropped connection. + let settled = false; + + const connect = async () => { + const token = await useAuthStore.getState().freshAccessToken(); + if (cancelled) return; + if (!token) { + settled = true; + fail("Your session has ended — sign in again."); + return; } + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + ws = new WebSocket( + `${proto}://${window.location.host}/ws/deploy/${encodeURIComponent(stackId)}` + + `?token=${encodeURIComponent(token)}` + ); + ws.onmessage = (ev) => { + try { + const msg = JSON.parse(ev.data); + if (msg.type === "log") { + handleLine(msg.line as string); + } else if (msg.type === "done") { + const ok = msg.returncode === 0; + settled = true; + tracker.finish(ok); + dirtyRef.current = true; + setPhase(ok ? "success" : "failed"); + } else if (msg.type === "error") { + settled = true; + fail(msg.detail || "Deploy error"); + } + } catch { + /* ignore */ + } + }; + ws.onclose = (ev) => { + // A socket that dropped before a done/error frame is surfaced. + if (settled || cancelled) return; + settled = true; + fail( + ev.code === 4403 + ? "Deploying is admin-only." + : ev.code === 4401 + ? "Your session has ended — sign in again." + : "Lost the connection to the server. The deploy carries on there." + ); + }; }; - ws.onclose = () => { - // If the socket dropped before a done/error frame, surface it. - setPhase((p) => { - if (p !== "running") return p; - tracker.finish(false); - dirtyRef.current = true; - return "error"; - }); - }; + void connect(); const timer = window.setInterval(() => { if (!dirtyRef.current) return; @@ -127,10 +156,11 @@ export function DeployConsole({ }, FLUSH_MS); return () => { + cancelled = true; window.clearInterval(timer); - ws.close(); + ws?.close(); }; - }, [stackId, token, tracker]); + }, [stackId, tracker]); useEffect(() => { if (boxRef.current) boxRef.current.scrollTop = boxRef.current.scrollHeight; diff --git a/frontend/src/components/stacks/EditorHelperPanel.tsx b/frontend/src/components/stacks/EditorHelperPanel.tsx index 80b84c9..e0e1f26 100644 --- a/frontend/src/components/stacks/EditorHelperPanel.tsx +++ b/frontend/src/components/stacks/EditorHelperPanel.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { RefreshCw, HardDrive, Cpu, Plug, Gauge } from "lucide-react"; import { VolumeWizard } from "@/components/volumes/VolumeWizard"; import { GPUSelector } from "@/components/gpu/GPUSelector"; @@ -22,9 +22,14 @@ export function EditorHelperPanel({ const [tab, setTab] = useState("volumes"); const [privileged, setPrivileged] = useState(false); + // Only the newest request may answer: an older one finishing late would put + // back the services of a YAML that has since changed. + const request = useRef(0); const loadServices = async () => { + const mine = ++request.current; try { const svc = await editorApi.services(yaml); + if (mine !== request.current) return; setServices(svc); setService((cur) => (cur && svc.includes(cur) ? cur : svc[0] ?? "")); } catch (e) { @@ -32,10 +37,15 @@ export function EditorHelperPanel({ } }; + // Follows the YAML (debounced, it is a server round trip). Read once on mount + // it listed the editor's starter service "app" for every existing stack — + // its file only arrives after the panel mounted — and the helpers then + // wrote into a service that did not exist. useEffect(() => { - loadServices(); + const t = setTimeout(loadServices, 500); + return () => clearTimeout(t); // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); + }, [yaml]); const guard = (): string | null => { if (!service) { diff --git a/frontend/src/components/stacks/LogViewer.tsx b/frontend/src/components/stacks/LogViewer.tsx index 805d99f..8f6bb58 100644 --- a/frontend/src/components/stacks/LogViewer.tsx +++ b/frontend/src/components/stacks/LogViewer.tsx @@ -95,45 +95,64 @@ export function LogViewer({ stackId }: { stackId: string }) { const [serviceFilter, setServiceFilter] = useState("__all__"); const [search, setSearch] = useState(""); const containerRef = useRef(null); - const token = useAuthStore((s) => s.accessToken); + // One stream per stack. Keyed on the access token, the stream reconnected on + // every background token refresh and the server replayed its 200-line tail + // into the view a second time. useEffect(() => { - if (!token) return; setError(null); + setLines([]); let gotError = false; - const proto = window.location.protocol === "https:" ? "wss" : "ws"; - const url = `${proto}://${window.location.host}/ws/logs/${stackId}?token=${token}`; - const ws = new WebSocket(url); - ws.onopen = () => setConnected(true); - ws.onclose = (ev) => { - setConnected(false); - // An expired or revoked session closes 4401. - if (!gotError && ev.code === 4401) { + let cancelled = false; + let ws: WebSocket | null = null; + + const connect = async () => { + const token = await useAuthStore.getState().freshAccessToken(); + if (cancelled) return; + if (!token) { setError("Not authorized to stream logs — sign in again."); + return; } - }; - ws.onmessage = (ev) => { - try { - const msg = JSON.parse(ev.data); - if (msg.type === "log") { - const { service, time, msg: body } = parseLine(msg.service, msg.line); - setLines((prev) => { - const next = [ - ...prev, - { service, time, msg: body, raw: msg.line, level: levelFor(body) }, - ]; - return next.length > MAX_LINES ? next.slice(-MAX_LINES) : next; - }); - } else if (msg.type === "error") { - gotError = true; - setError(msg.detail || "Log stream error"); + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + ws = new WebSocket( + `${proto}://${window.location.host}/ws/logs/${encodeURIComponent(stackId)}` + + `?token=${encodeURIComponent(token)}` + ); + ws.onopen = () => setConnected(true); + ws.onclose = (ev) => { + setConnected(false); + // An expired or revoked session closes 4401. + if (!gotError && ev.code === 4401) { + setError("Not authorized to stream logs — sign in again."); } - } catch { - /* ignore */ - } + }; + ws.onmessage = (ev) => { + try { + const msg = JSON.parse(ev.data); + if (msg.type === "log") { + const { service, time, msg: body } = parseLine(msg.service, msg.line); + setLines((prev) => { + const next = [ + ...prev, + { service, time, msg: body, raw: msg.line, level: levelFor(body) }, + ]; + return next.length > MAX_LINES ? next.slice(-MAX_LINES) : next; + }); + } else if (msg.type === "error") { + gotError = true; + setError(msg.detail || "Log stream error"); + } + } catch { + /* ignore */ + } + }; }; - return () => ws.close(); - }, [stackId, token]); + void connect(); + return () => { + cancelled = true; + ws?.close(); + }; + }, [stackId]); // Unique services seen so far, for the container filter. const services = useMemo(() => { diff --git a/frontend/src/components/stacks/StacksTable.tsx b/frontend/src/components/stacks/StacksTable.tsx index 0872dab..f5e66fc 100644 --- a/frontend/src/components/stacks/StacksTable.tsx +++ b/frontend/src/components/stacks/StacksTable.tsx @@ -67,8 +67,10 @@ export function StacksTable({ Stack - CPU - Memory + {/* Left out on a phone: next to the name block there was no room, + and the meters were drawn over the service count. */} + CPU + Memory {isAdmin && } @@ -188,7 +190,7 @@ function StackRow({ {status && }
- + {running && stats ? ( — )} - + {running && stats ? ( + +

Admin only

+

{children}

+ + ); +} diff --git a/frontend/src/components/ui/StackGroupHeader.tsx b/frontend/src/components/ui/StackGroupHeader.tsx index eaaca41..7200bcc 100644 --- a/frontend/src/components/ui/StackGroupHeader.tsx +++ b/frontend/src/components/ui/StackGroupHeader.tsx @@ -70,14 +70,18 @@ function Label({ group }: { group: StackGroup }) { ); } - // Labelled with a compose project that is no longer a stack: exactly where - // things left behind by a deleted stack collect. + // Labelled with a compose project that is no StackPilot stack: left behind + // by a deleted stack, or started outside StackPilot (`docker compose up` in + // some other folder) — which of the two cannot be told from the label. return ( <> {group.stackId} - - stack removed + + not a stack ); diff --git a/frontend/src/hooks/useDockerEvents.ts b/frontend/src/hooks/useDockerEvents.ts index 782d7b9..bd25545 100644 --- a/frontend/src/hooks/useDockerEvents.ts +++ b/frontend/src/hooks/useDockerEvents.ts @@ -67,10 +67,14 @@ export function useDockerEvents() { } }; - const connect = () => { + const connect = async () => { + // The token this effect started with may have expired by the time a + // dropped socket reconnects; a refused one would just retry forever. + const fresh = await useAuthStore.getState().freshAccessToken(); + if (closed || !fresh) return; const proto = window.location.protocol === "https:" ? "wss" : "ws"; socket = new WebSocket( - `${proto}://${window.location.host}/ws/events?token=${encodeURIComponent(token)}` + `${proto}://${window.location.host}/ws/events?token=${encodeURIComponent(fresh)}` ); socket.onopen = () => { @@ -94,12 +98,12 @@ export function useDockerEvents() { // token is short-lived and the API client refreshes it out from under us. socket.onclose = () => { if (closed) return; - retry = window.setTimeout(connect, delay); + retry = window.setTimeout(() => void connect(), delay); delay = Math.min(delay * 2, RECONNECT_MAX_MS); }; }; - connect(); + void connect(); return () => { closed = true; diff --git a/frontend/src/hooks/useStackActions.ts b/frontend/src/hooks/useStackActions.ts index f29cd18..a341673 100644 --- a/frontend/src/hooks/useStackActions.ts +++ b/frontend/src/hooks/useStackActions.ts @@ -107,7 +107,7 @@ export function useStackActions() { begin(id, label); const t = toast.loading(`${label} ${id}…`); try { - await streamCompose(path, token!, (progress) => + await streamCompose(path, (progress) => setStatusMap((s) => (s[id] ? { ...s, [id]: { ...s[id], progress } } : s)) ); toast.success(`${label} ${id} ✓`, { id: t }); diff --git a/frontend/src/index.css b/frontend/src/index.css index a4bdfae..af45398 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -19,6 +19,10 @@ body { margin: 0; font-family: var(--sp-font); background: var(--sp-bg); + /* The same text colour AppShell sets: dialogs are portalled onto , + outside it, and inherited the browser's black there — unreadable on the + dark theme's dialog background. */ + @apply text-slate-900 dark:text-slate-100; } /* Custom scrollbar */ diff --git a/frontend/src/lib/composeStream.ts b/frontend/src/lib/composeStream.ts index 9159ced..a4ff812 100644 --- a/frontend/src/lib/composeStream.ts +++ b/frontend/src/lib/composeStream.ts @@ -7,6 +7,7 @@ * stacks list, where an update needs to be legible at a glance in one row. */ import { DeployTracker, parseDeployLine } from "./deployProgress"; +import { useAuthStore } from "@/store/auth"; /** Compose emits status changes far faster than the UI needs to repaint. */ const FLUSH_MS = 150; @@ -27,12 +28,16 @@ function snapshot(tracker: DeployTracker): ProgressSnapshot { * Opens `path`, reporting progress until compose finishes. Resolves on exit * code 0, rejects with the failure reason otherwise. The compose subprocess * keeps running server-side even if this socket is closed early. + * + * Fetches a fresh access token itself: an expired one is refused with 4401, + * which used to surface as "connection closed before the update finished". */ -export function streamCompose( +export async function streamCompose( path: string, - token: string, onProgress: (p: ProgressSnapshot) => void ): Promise { + const token = await useAuthStore.getState().freshAccessToken(); + if (!token) throw new Error("your session has ended — sign in again"); return new Promise((resolve, reject) => { const tracker = new DeployTracker(); const proto = window.location.protocol === "https:" ? "wss" : "ws"; @@ -82,6 +87,15 @@ export function streamCompose( ws.onerror = () => settle(new Error("connection to the server failed")); // Reached normally after settle() closes the socket; the guard makes this // a no-op then, so it only fires for a stream that dropped mid-update. - ws.onclose = () => settle(new Error("connection closed before the update finished")); + ws.onclose = (ev) => + settle( + new Error( + ev.code === 4403 + ? "admin rights required" + : ev.code === 4401 + ? "your session has ended — sign in again" + : "connection closed before the update finished" + ) + ); }); } diff --git a/frontend/src/lib/monaco.ts b/frontend/src/lib/monaco.ts new file mode 100644 index 0000000..cc832b6 --- /dev/null +++ b/frontend/src/lib/monaco.ts @@ -0,0 +1,41 @@ +/** + * Monaco from the bundle instead of a CDN. + * + * `@monaco-editor/react` fetches the editor from cdn.jsdelivr.net by default, + * at run time: on a host without internet access, or behind a DNS filter that + * blocks jsdelivr, the compose editor never got past "Loading…". Here it is + * built into the chunks of the two pages that edit text (both lazy-loaded, so + * nobody pays for it elsewhere): the API, the editor features (find, folding, + * the diff view …) and the languages the file browser picks by extension. + * Each language's tokenizer is itself a separate chunk, fetched on first use. + * + * Imported for its side effect, before any mounts. + */ +import { loader } from "@monaco-editor/react"; +import * as monaco from "monaco-editor/editor"; +import "monaco-editor/features/register.all"; +import "monaco-editor/languages/definitions/yaml/register"; +import "monaco-editor/languages/definitions/ini/register"; +import "monaco-editor/languages/definitions/shell/register"; +import "monaco-editor/languages/definitions/dockerfile/register"; +import "monaco-editor/languages/definitions/python/register"; +import "monaco-editor/languages/definitions/javascript/register"; +import "monaco-editor/languages/definitions/typescript/register"; +import "monaco-editor/languages/definitions/markdown/register"; +import "monaco-editor/languages/definitions/html/register"; +import "monaco-editor/languages/definitions/css/register"; +import "monaco-editor/languages/definitions/xml/register"; +import "monaco-editor/languages/definitions/sql/register"; +// JSON is the one with a language service (validation), and its own worker. +// Both JSON paths spell out ".js": without it Vite fails to resolve them +// through the package's "./*" export, the others resolve either way. +import "monaco-editor/languages/features/json/register.js"; +import EditorWorker from "monaco-editor/editor/editor.worker?worker"; +import JsonWorker from "monaco-editor/language/json/json.worker.js?worker"; + +self.MonacoEnvironment = { + getWorker: (_moduleId: string, label: string) => + label === "json" ? new JsonWorker() : new EditorWorker(), +}; + +loader.config({ monaco }); diff --git a/frontend/src/lib/utils.test.ts b/frontend/src/lib/utils.test.ts new file mode 100644 index 0000000..11a68f8 --- /dev/null +++ b/frontend/src/lib/utils.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { parseServerTime, relativeTime } from "./utils"; + +describe("parseServerTime", () => { + it("reads an offset-less API timestamp as UTC", () => { + // What SQLite hands back for a UTC datetime: no zone at all. + expect(parseServerTime("2026-10-15T09:56:04.321963").toISOString()).toBe( + "2026-10-15T09:56:04.321Z" + ); + }); + + it("leaves a timestamp that carries its zone alone", () => { + expect(parseServerTime("2026-10-15T11:56:04+02:00").toISOString()).toBe( + "2026-10-15T09:56:04.000Z" + ); + expect(parseServerTime("2026-10-15T09:56:04.123456789Z").toISOString()).toBe( + "2026-10-15T09:56:04.123Z" + ); + }); +}); + +describe("relativeTime", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-15T10:00:00Z")); + }); + afterEach(() => vi.useRealTimers()); + + it("is not hours off for a UTC timestamp without an offset", () => { + expect(relativeTime("2026-10-15T09:59:30")).toBe("just now"); + expect(relativeTime("2026-10-15T09:55:00")).toBe("5m ago"); + }); + + it("says when a time is still ahead", () => { + // A schedule's next run used to read "next just now", whenever it was. + expect(relativeTime("2026-10-15T13:00:00")).toBe("in 3h"); + expect(relativeTime("2026-10-25T10:00:00Z")).toBe("in 10d"); + }); + + it("does not print NaN for something unparseable", () => { + expect(relativeTime("not a date")).toBe("—"); + }); +}); diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 747f847..9e04022 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -21,15 +21,32 @@ export function formatUptime(seconds: number): string { return `${m}m`; } +/** + * A timestamp from the API as a Date. + * + * StackPilot's own times are UTC, but SQLite hands them back without an + * offset, and JavaScript reads an offset-less date-time as *local* time — which + * put every audit entry, sync and backup two hours in the past in German summer + * time. Docker's own timestamps carry a zone and pass through unchanged. + */ +export function parseServerTime(iso: string): Date { + const zoned = /(?:Z|[+-]\d{2}:?\d{2})$/i.test(iso); + return new Date(zoned || !iso.includes("T") ? iso : `${iso}Z`); +} + +/** "5m ago", or "in 5m" for a time still ahead (a schedule's next run). */ export function relativeTime(iso: string): string { - const then = new Date(iso).getTime(); + const then = parseServerTime(iso).getTime(); + if (Number.isNaN(then)) return "—"; const diff = Date.now() - then; - const s = Math.floor(diff / 1000); - if (s < 60) return "just now"; + const ahead = diff < 0; + const say = (v: string) => (ahead ? `in ${v}` : `${v} ago`); + const s = Math.floor(Math.abs(diff) / 1000); + if (s < 60) return ahead ? "in a moment" : "just now"; const m = Math.floor(s / 60); - if (m < 60) return `${m}m ago`; + if (m < 60) return say(`${m}m`); const h = Math.floor(m / 60); - if (h < 24) return `${h}h ago`; + if (h < 24) return say(`${h}h`); const d = Math.floor(h / 24); - return `${d}d ago`; + return say(`${d}d`); } diff --git a/frontend/src/pages/Audit.tsx b/frontend/src/pages/Audit.tsx index 186147a..f1d23be 100644 --- a/frontend/src/pages/Audit.tsx +++ b/frontend/src/pages/Audit.tsx @@ -5,10 +5,18 @@ import { Card, Input, Spinner } from "@/components/ui"; import api from "@/api/client"; import type { AuditEntry } from "@/types"; import { relativeTime } from "@/lib/utils"; +import { useAuthStore } from "@/store/auth"; +import { AdminOnly } from "@/components/ui/AdminOnly"; const PAGE = 100; export function Audit() { + const isAdmin = useAuthStore((s) => s.user?.role === "admin"); + if (!isAdmin) return The audit log is available to administrators only.; + return ; +} + +function AuditLog() { const [offset, setOffset] = useState(0); const [filter, setFilter] = useState(""); diff --git a/frontend/src/pages/Files.tsx b/frontend/src/pages/Files.tsx index 1554598..50ff26f 100644 --- a/frontend/src/pages/Files.tsx +++ b/frontend/src/pages/Files.tsx @@ -24,11 +24,14 @@ import { } from "lucide-react"; import { toast } from "sonner"; import Editor from "@monaco-editor/react"; +// Before any editor mounts: points the wrapper at the bundled Monaco. +import "@/lib/monaco"; import { Button, Card, Input, Spinner } from "@/components/ui"; import { ConfirmDialog } from "@/components/ui/ConfirmDialog"; import { filesApi } from "@/api/files"; import { apiErrorMessage } from "@/api/client"; import { useAuthStore } from "@/store/auth"; +import { AdminOnly } from "@/components/ui/AdminOnly"; import { useThemeStore } from "@/store/theme"; import { formatBytes, relativeTime } from "@/lib/utils"; import type { HostPathEntry } from "@/types"; @@ -71,6 +74,11 @@ function crumbs(path: string): { label: string; path: string }[] { export function Files() { const isAdmin = useAuthStore((s) => s.user?.role === "admin"); + if (!isAdmin) return The file browser is available to administrators only.; + return ; +} + +function FileBrowser({ isAdmin }: { isAdmin: boolean }) { const qc = useQueryClient(); const [path, setPath] = useState("/"); // Persist "Show hidden" across reloads — otherwise an uploaded dotfile (.env) @@ -293,7 +301,7 @@ export function Files() { - {isAdmin && ( + {isAdmin && !data?.virtual && ( <> - {isAdmin && ( - <> - - - - - - )} -
+ {/* A folder on the way to a root is only a way through. */} + {!data?.virtual && ( +
+ + {isAdmin && ( + <> + + + + + + )} +
+ )} ))} diff --git a/frontend/src/pages/Images.tsx b/frontend/src/pages/Images.tsx index 305a934..06709d3 100644 --- a/frontend/src/pages/Images.tsx +++ b/frontend/src/pages/Images.tsx @@ -18,6 +18,12 @@ import { useAuthStore } from "@/store/auth"; function UpdateBadge({ row }: { row: ImageRow }) { const u = row.update; if (!u) return not checked; + if (u.local) + return ( + + local build + + ); if (u.error) return ⚠ {u.error}; if (u.update_available) return ( @@ -83,10 +89,19 @@ function ImagesSection({ isAdmin }: { isAdmin: boolean }) { setScanning(true); const t = toast.loading("Scanning images… this runs the scanner per image"); try { - const r = await imagesApi.scanAll(); + const started = await imagesApi.startScanAll(); + // The sweep runs on the server; follow it until it is done. + let status = started; + while (status.active) { + await new Promise((r) => window.setTimeout(r, 2000)); + status = await imagesApi.scanStatus(); + if (status.run !== started.run) break; // a newer sweep took over + toast.loading(`Scanning images… ${status.done}/${status.total}`, { id: t }); + qc.invalidateQueries({ queryKey: ["image-scans"] }); + } await qc.invalidateQueries({ queryKey: ["image-scans"] }); toast.success( - `Scanned ${r.scanned} image(s)${r.failed ? `, ${r.failed} failed` : ""}`, + `Scanned ${status.scanned} image(s)${status.failed ? `, ${status.failed} failed` : ""}`, { id: t } ); } catch (e) { diff --git a/frontend/src/pages/Networks.tsx b/frontend/src/pages/Networks.tsx index bdb373f..12c47d3 100644 --- a/frontend/src/pages/Networks.tsx +++ b/frontend/src/pages/Networks.tsx @@ -250,15 +250,18 @@ function NetworkRows({ {isAdmin && ( {!n.is_default && ( + // Disabled while in use: Docker refuses to remove it anyway, and + // the red bin offered a delete that could only fail. )} diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index 812da11..2404b7d 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -7,7 +7,6 @@ import { Send, Trash2, Users as UsersIcon, - ShieldCheck, Power, RefreshCw, HardDrive, @@ -40,21 +39,14 @@ import { stacksApi } from "@/api/stacks"; import { apiErrorMessage } from "@/api/client"; import { relativeTime } from "@/lib/utils"; import { useAuthStore } from "@/store/auth"; +import { AdminOnly } from "@/components/ui/AdminOnly"; import type { User } from "@/types"; export function Settings() { const isAdmin = useAuthStore((s) => s.user?.role === "admin"); if (!isAdmin) { - return ( - - -

Admin only

-

- Settings are available to administrators only. -

- - ); + return Settings are available to administrators only.; } return ( @@ -514,6 +506,11 @@ const FIELDS: Record destinationsApi.test(dest.id), - onSuccess: (r) => - r.ok ? toast.success("Reachable") : toast.error(r.error || "Connection failed"), + onSuccess: (r) => { + if (!r.ok) return toast.error(r.error || "Connection failed"); + toast.success("Reachable", r.host_key ? { description: `Host key ${r.host_key}` } : undefined); + // A first SFTP connection pins the server's key; show it in the form. + onChange(); + }, onError: (e) => toast.error(apiErrorMessage(e)), }); const remove = useMutation({ diff --git a/frontend/src/pages/StackDetail.tsx b/frontend/src/pages/StackDetail.tsx index eb93ad0..3d60f19 100644 --- a/frontend/src/pages/StackDetail.tsx +++ b/frontend/src/pages/StackDetail.tsx @@ -107,15 +107,15 @@ export function StackDetail() { {/* Tabs */} -
+
{TABS.map((t) => (