From d6cebb337109616dab2d42955f43633c02479073 Mon Sep 17 00:00:00 2001 From: menzelj Date: Thu, 15 Oct 2026 13:51:26 +0200 Subject: [PATCH] Fix what a review of the whole app found, end to end (0.67.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pass over every feature against a real Docker daemon, then clicking through the UI as admin and read-only user, on a desktop and a phone. Security. /ws/deploy ran compose up for the read-only role; it is admin-only now. A container's inspect view, a stack's compose file (POSTGRES_PASSWORD: hunter2) and the viewer's .env tab no longer show credentials to that role. Git branch and URL values that git would read as options (-x, ext::) are refused. SFTP host keys are checked: pinned on first use or entered by hand (key line or SHA256 fingerprint), a different key is refused before the password is sent. A backup name from the API can no longer reach outside the destination folder. The file browser refuses to move or rename a browse root, as delete did. Docker lifecycle. Closing the deploy console released the stack lock while compose was still running and logged rc=None as success; deploys and updates are now background runs that hold the lock to the end, report the real exit code and can be re-attached to. Every closed log view left a `compose logs -f` behind. A dozen open tabs exhausted the default thread pool and hung the API; event and exec streams have their own threads. Auto-update and the update endpoint ignored failed pulls. A deleted git stack was recreated and started by the next webhook or poll; deleting a stack now drops its git source, auto-update policy and schedule entries. Deleting no longer walks into mounted shares, is refused while a backup runs, and removes the containers even when the compose file is broken. Stack folders compose has to normalise (Nextcloud -> nextcloud) work everywhere and group correctly on the volumes, networks and images pages. Detection. Host port conflicts read the container's own network namespace; /proc/1/net is read instead. Images referenced as docker.io/… were never update-checked; local builds no longer report "needs credentials". NVIDIA GPUs are read from the driver's procfs. The backup dialog's "path mismatch" warning came from mountinfo's filesystem-relative root and fired for any stacks folder on its own disk, ZFS dataset or btrfs subvolume; the host path now comes from Docker's mounts. Backups and destinations. The download ignored the folder/volume selection (axios sent binds[]). Restoring under a new name kept the old one. A report header past nginx's buffer turned a good backup into a 502. SFTP/S3 errors were bare 500s and "Test" did not try to write. A destination used by a schedule could be deleted. Backup file names are in the schedule's or the browser's time zone instead of UTC. Editor and converter. `docker run` conversion produced invalid compose for --network host, named volumes and -dit; --gpus and --mount were dropped; a multi-line command pasted into the one-line field became `image: ' '` (now a textarea, and flattened continuations are read). Validation failed for env_file stacks. The editor overwrote unsaved edits on every container event, lost them on a stray click (now asks), threw when closing the diff, and its helper panel listed the starter service "app" for every existing stack. Monaco is bundled instead of loaded from jsdelivr, so the editor works offline (and monaco-editor 0.57 drops the dompurify advisory; axios and react-router updated). UI. All times were two hours off: offset-less UTC from SQLite was read as local time, and "next run" always said "just now". A token refresh mid-session duplicated deploys, reset terminals and doubled logs. A backend restart (update, self-update) signed everyone out on reload; the page now waits for it. The editor was unusable on a phone, tables and tabs overflowed, dialog text was black on the dark theme, and the file browser opened on an error with the default roots. Read-only users get "Admin only" pages instead of 403s; in-use networks offer no delete; "Scan all" covers StackPilot's own stacks only and runs in the background instead of timing out at 600s. Deployment. nginx passes the port in Host (the OIDC redirect URI lost :5009) and keeps an outer proxy's X-Forwarded-Proto. The frontend waits for a healthy backend (healthcheck probes every 2s while starting). The frontend image builds with npm ci. Secrets are 0644 in a 0700 folder so non-root containers can read them; existing files are fixed at startup. The test suite no longer starts real containers on a host with Docker. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Fx7gcuy8d6nPd95g43rGHu --- README.md | 11 +- backend/Dockerfile | 6 +- backend/main.py | 23 + backend/models/runtime_state.py | 2 + backend/routers/auth.py | 4 +- backend/routers/backups.py | 29 +- backend/routers/containers.py | 33 +- backend/routers/destinations.py | 33 +- backend/routers/editor.py | 17 +- backend/routers/files.py | 38 +- backend/routers/git.py | 8 + backend/routers/images.py | 50 +- backend/routers/networks.py | 13 +- backend/routers/secrets.py | 22 +- backend/routers/stacks.py | 96 +++- backend/routers/volumes.py | 13 +- backend/routers/ws.py | 349 ++++++++------ backend/services/auto_update_service.py | 15 +- .../services/backup_destination_service.py | 329 +++++++++++-- backend/services/backup_service.py | 39 +- backend/services/compose_run_service.py | 174 +++++++ backend/services/compose_service.py | 289 +++++++++-- backend/services/convert_service.py | 452 +++++++++++++----- backend/services/dashboard_service.py | 2 +- backend/services/device_service.py | 51 +- backend/services/exec_service.py | 66 ++- backend/services/file_service.py | 18 +- backend/services/git_service.py | 36 +- backend/services/gpu_service.py | 46 +- backend/services/image_service.py | 39 +- backend/services/image_status_store.py | 2 + backend/services/notify_service.py | 33 +- backend/services/port_service.py | 72 ++- backend/services/scan_service.py | 59 ++- backend/services/schedule_service.py | 4 +- backend/services/secret_service.py | 48 +- backend/services/stack_assets_service.py | 58 ++- backend/services/stack_cleanup_service.py | 86 ++++ backend/services/update_service.py | 99 +++- backend/tests/conftest.py | 8 + backend/tests/test_compose_runs.py | 226 +++++++++ backend/tests/test_compose_service.py | 26 +- backend/tests/test_convert_service.py | 167 +++++++ backend/tests/test_destinations.py | 276 +++++++++++ backend/tests/test_image_refs_and_ports.py | 123 +++++ backend/tests/test_notifications.py | 58 +++ backend/tests/test_registries.py | 4 +- backend/tests/test_review_regressions.py | 137 ++++++ backend/tests/test_stack_delete.py | 186 +++++++ backend/tests/test_walkthrough_fixes.py | 242 ++++++++++ backend/version.py | 2 +- docker-compose.yml | 5 +- frontend/Dockerfile | 7 +- frontend/nginx.conf | 24 +- frontend/package-lock.json | 247 +++++----- frontend/package.json | 3 +- frontend/src/App.tsx | 71 ++- frontend/src/api/backups.ts | 29 +- frontend/src/api/client.ts | 32 +- frontend/src/api/editor.ts | 10 +- frontend/src/api/files.ts | 18 +- frontend/src/api/images.ts | 22 +- frontend/src/api/stacks.ts | 6 +- .../src/components/stacks/AutoUpdatePanel.tsx | 3 +- .../components/stacks/ContainerTerminal.tsx | 104 ++-- .../src/components/stacks/DeployConsole.tsx | 98 ++-- .../components/stacks/EditorHelperPanel.tsx | 16 +- frontend/src/components/stacks/LogViewer.tsx | 81 ++-- .../src/components/stacks/StacksTable.tsx | 12 +- frontend/src/components/ui/AdminOnly.tsx | 15 + .../src/components/ui/StackGroupHeader.tsx | 12 +- frontend/src/hooks/useDockerEvents.ts | 12 +- frontend/src/hooks/useStackActions.ts | 2 +- frontend/src/index.css | 4 + frontend/src/lib/composeStream.ts | 20 +- frontend/src/lib/monaco.ts | 41 ++ frontend/src/lib/utils.test.ts | 43 ++ frontend/src/lib/utils.ts | 29 +- frontend/src/pages/Audit.tsx | 8 + frontend/src/pages/Files.tsx | 103 ++-- frontend/src/pages/Images.tsx | 19 +- frontend/src/pages/Networks.tsx | 9 +- frontend/src/pages/Settings.tsx | 25 +- frontend/src/pages/StackDetail.tsx | 35 +- frontend/src/pages/StackEditor.tsx | 118 ++++- frontend/src/store/auth.test.ts | 88 ++++ frontend/src/store/auth.ts | 84 +++- frontend/src/types/index.ts | 3 + 88 files changed, 4752 insertions(+), 925 deletions(-) create mode 100644 backend/services/compose_run_service.py create mode 100644 backend/services/stack_cleanup_service.py create mode 100644 backend/tests/test_compose_runs.py create mode 100644 backend/tests/test_convert_service.py create mode 100644 backend/tests/test_destinations.py create mode 100644 backend/tests/test_image_refs_and_ports.py create mode 100644 backend/tests/test_notifications.py create mode 100644 backend/tests/test_review_regressions.py create mode 100644 backend/tests/test_stack_delete.py create mode 100644 backend/tests/test_walkthrough_fixes.py create mode 100644 frontend/src/components/ui/AdminOnly.tsx create mode 100644 frontend/src/lib/monaco.ts create mode 100644 frontend/src/lib/utils.test.ts create mode 100644 frontend/src/store/auth.test.ts diff --git a/README.md b/README.md index f90f7d1..b97979d 100644 --- a/README.md +++ b/README.md @@ -652,6 +652,13 @@ it is what your saved destination credentials are encrypted with. `local_lock=` yourself: with locking the kernel needs rpcbind on the Docker host (nfs-common), and without it the mount fails with "connection refused". Only StackPilot writes the backup files, so nothing needs locking (0.66.2). +- **SFTP host keys** (0.67.1): the server's host key is checked on every + connection. Paste it into the destination's *Host key* field (a key line or + its `SHA256:` fingerprint, e.g. from `ssh-keyscan HOST | ssh-keygen -lf -`), + or leave the field empty and the key seen on the first connection is pinned + — "Test" shows its fingerprint. A different key is refused before the + password is sent; clear the field after a server reinstall to trust the new + one. Changing the host or port forgets the pinned key. - **Push & restore**: the stack Backup dialog can push straight to a destination instead of downloading; the Restore dialog can browse a destination's backups and restore (volumes included) directly from it. Backups can also be @@ -665,7 +672,9 @@ it is what your saved destination credentials are encrypted with. - **Local time** (0.66.0): each schedule has a time zone, preset to the browser's (e.g. `Europe/Berlin`), and its time is wall-clock time there — a 03:00 backup stays at 03:00 across daylight saving changes. Schedules made - before were in UTC and stay so until edited. + before were in UTC and stay so until edited. Backup file names carry the time + on that clock too (`backup-app-full-20261015-122154.tar.gz` for 12:21:54 in + Berlin); a backup made by hand uses the browser's time zone (0.67.1). - **Several stacks per schedule** (0.65.0): tick any number of stacks, or *All stacks* to include stacks added later without editing the schedule. They are backed up one after another, each to its own file with its own retention diff --git a/backend/Dockerfile b/backend/Dockerfile index 911ccb9..17cd475 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -26,7 +26,11 @@ ENV STACKS_DIR=/opt/stacks \ EXPOSE 5008 -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \ +# --start-interval: probed every 2s while starting, so "healthy" — which the +# frontend waits for (docker-compose.yml) — comes as soon as the API answers +# instead of after the first 30s interval. Engines before 25.0 ignore it and +# take the 30s. +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --start-interval=2s \ CMD curl -fsS http://localhost:5008/api/health || exit 1 # --proxy-headers: nginx setzt X-Forwarded-For, ohne dieses Flag ignoriert diff --git a/backend/main.py b/backend/main.py index c34e46a..6f65b26 100644 --- a/backend/main.py +++ b/backend/main.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging +from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager from fastapi import FastAPI, Request @@ -47,6 +48,8 @@ from services import ( logo_service, registry_service, schedule_service, + secret_service, + stack_cleanup_service, stack_lock_service, template_service, update_service, @@ -56,13 +59,27 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger("stackpilot") +#: Workers for ``asyncio.to_thread``. The default (CPU count + 4, so six on a +#: two-core VM) is shared by everything that leaves the event loop — a backup +#: streaming several volumes, the dashboard, NFS and SFTP destinations — and +#: running out of it does not fail, it waits. Long-lived readers (the event +#: stream, terminals) have threads of their own; this is headroom for the rest. +DEFAULT_EXECUTOR_WORKERS = 64 + + @asynccontextmanager async def lifespan(app: FastAPI): + asyncio.get_running_loop().set_default_executor( + ThreadPoolExecutor(max_workers=DEFAULT_EXECUTOR_WORKERS, thread_name_prefix="sp-worker") + ) init_db() # Register stacks that already exist on disk. try: with Session(engine) as session: stacks.sync_discovered_stacks(session) + orphans = stack_cleanup_service.prune_orphans(session) + if orphans: + logger.info("Forgot Git sources/policies of deleted stack(s): %s", ", ".join(orphans)) except Exception as exc: # noqa: BLE001 logger.warning("Stack discovery failed: %s", exc) # One-off: encrypt backup-destination credentials written before they were @@ -74,6 +91,12 @@ async def lifespan(app: FastAPI): logger.info("Encrypted %d backup destination config(s) at rest", encrypted) except Exception as exc: # noqa: BLE001 logger.warning("Destination config encryption migration failed: %s", exc) + try: + fixed = secret_service.fix_permissions() + if fixed: + logger.info("Made %d stack secret file(s) readable for non-root containers", fixed) + except Exception as exc: # noqa: BLE001 + logger.warning("Could not fix secret permissions: %s", exc) try: moved = template_service.migrate_legacy_db_templates() if moved: diff --git a/backend/models/runtime_state.py b/backend/models/runtime_state.py index 3f5faab..dc9e39f 100644 --- a/backend/models/runtime_state.py +++ b/backend/models/runtime_state.py @@ -55,6 +55,8 @@ class ImageStatus(SQLModel, table=True): remote_digest: Optional[str] = None checked_at: float = 0.0 error: Optional[str] = None + #: Built on the host, so there is nothing in a registry to compare with. + local: bool = False #: Whether an "update available" notification already went out for this #: image at its current state. notified: bool = False diff --git a/backend/routers/auth.py b/backend/routers/auth.py index 50ac0a0..d864baf 100644 --- a/backend/routers/auth.py +++ b/backend/routers/auth.py @@ -114,8 +114,10 @@ def setup( ) -> TokenPair: if auth_mod.users_exist(session): raise HTTPException(status_code=400, detail="Setup already completed") + if not body.username.strip() or not body.password: + raise HTTPException(status_code=400, detail="Username and password required") user = User( - username=body.username, + username=body.username.strip(), hashed_password=auth_mod.hash_password(body.password), role="admin", ) diff --git a/backend/routers/backups.py b/backend/routers/backups.py index d6fad00..8f9dfa7 100644 --- a/backend/routers/backups.py +++ b/backend/routers/backups.py @@ -9,7 +9,7 @@ import tempfile from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import FileResponse from starlette.background import BackgroundTask -from pydantic import BaseModel +from pydantic import BaseModel, Field from sqlmodel import Session from auth import require_admin @@ -51,6 +51,23 @@ def _compact(report: dict) -> dict: } +#: The report rides in a response header, which a proxy refuses past its +#: header buffer (nginx: 4–8 KiB by default, then a 502 for a backup that +#: worked). Lists are cut until it fits; the counts stay in the audit log. +MAX_REPORT_HEADER = 6 * 1024 + + +def _report_header(report: dict) -> str: + compact = _compact(report) + text = json.dumps(compact) + for key in ("warnings", "skipped", "binds", "volumes", "errors"): + while len(text) > MAX_REPORT_HEADER and compact.get(key): + compact[key] = compact[key][: len(compact[key]) // 2] + compact["truncated"] = True + text = json.dumps(compact) + return text + + def _summary(report: dict) -> str: text = ( f"binds={len(report.get('binds', []))} " @@ -96,6 +113,8 @@ async def backup_stack( stop_first: bool = Query(True), binds: list[str] | None = Query(None), volumes: list[str] | None = Query(None), + # The browser's zone, for the time in the file name. + tz: str | None = Query(None, max_length=64), session: Session = Depends(get_session), user: User = Depends(require_admin), ): @@ -117,10 +136,10 @@ async def backup_stack( return FileResponse( path, media_type="application/gzip", - filename=_backup_filename(stack_id, include_volumes), + filename=_backup_filename(stack_id, include_volumes, tz=tz), # The browser downloads a blob, so the summary of what actually made it # into the archive rides along in a header. - headers={"X-Stackpilot-Backup": json.dumps(_compact(report))}, + headers={"X-Stackpilot-Backup": _report_header(report)}, # The archive is a temp file; without this every download stayed in # the container's /tmp until it filled the disk. background=BackgroundTask(os.unlink, path), @@ -184,6 +203,8 @@ class PushBody(BaseModel): stop_first: bool = True binds: list[str] | None = None volumes: list[str] | None = None + #: The browser's zone, for the time in the file name. + tz: str | None = Field(None, max_length=64) class RestoreFromBody(BaseModel): @@ -223,7 +244,7 @@ async def push_backup( except backup_service.BackupError as exc: raise _backup_http_error(exc) from exc - filename = _backup_filename(stack_id, body.include_volumes) + filename = _backup_filename(stack_id, body.include_volumes, tz=body.tz) try: remote = await asyncio.to_thread(dest_service.upload, dest, path, filename) except dest_service.DestinationError as exc: diff --git a/backend/routers/containers.py b/backend/routers/containers.py index 1f567fa..9d7b36c 100644 --- a/backend/routers/containers.py +++ b/backend/routers/containers.py @@ -1,24 +1,46 @@ """Single-container inspect + lifecycle for compose-managed containers.""" from __future__ import annotations -from fastapi import APIRouter, Depends, Request +from fastapi import APIRouter, Depends, HTTPException, Request from sqlmodel import Session from auth import get_current_user, require_admin from database import get_session +from docker_client import DockerError from models.user import User from services import audit_service, container_service router = APIRouter(prefix="/api/containers", tags=["containers"]) +#: Docker errors that are the caller's to fix, not a daemon failure (the global +#: handler turns every DockerError into a 502). +_STATUS = {"not_found": 404, "not_managed": 404, "bad_action": 400} + def _ip(request: Request) -> str: return request.client.host if request.client else "unknown" +def _raise(exc: DockerError): + status = _STATUS.get(exc.error) + if status: + raise HTTPException(status_code=status, detail=exc.detail or exc.error) from exc + raise exc + + @router.get("/{container_id}") -def inspect(container_id: str, _user: User = Depends(get_current_user)) -> dict: - return container_service.inspect_container(container_id) +def inspect(container_id: str, user: User = Depends(get_current_user)) -> dict: + try: + data = container_service.inspect_container(container_id) + except DockerError as exc: + _raise(exc) + if user.role != "admin": + # The environment is where a container's credentials live — the same + # values the stack's .env holds, which the read-only role does not get + # either. Names stay visible, values do not. + data["env"] = [entry.split("=", 1)[0] + "=••••••" if "=" in entry else entry + for entry in data.get("env") or []] + return data @router.post("/{container_id}/{action}") @@ -29,7 +51,10 @@ def action( session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: - result = container_service.container_action(container_id, action) + try: + result = container_service.container_action(container_id, action) + except DockerError as exc: + _raise(exc) audit_service.record( session, user=user.username, action=f"container.{action}", target=container_id[:12], ip=_ip(request), diff --git a/backend/routers/destinations.py b/backend/routers/destinations.py index a364344..00e5aae 100644 --- a/backend/routers/destinations.py +++ b/backend/routers/destinations.py @@ -16,6 +16,7 @@ from models.backup_destination import ( DestinationRead, DestinationUpdate, ) +from models.backup_schedule import BackupSchedule from models.user import User from services import audit_service, backup_destination_service as dest_service @@ -92,10 +93,16 @@ def update_destination( if body.config is not None: # Merge so masked/blank secrets don't wipe stored ones. existing = dest_service.parse_config(d) + before = dict(existing) for k, v in body.config.items(): if k in SECRET_KEYS and (v == "" or v == "••••••"): continue # keep existing secret existing[k] = v + # A host key pinned for the old address says nothing about a new one; + # kept, the first connection to the new server would be refused. + moved = any(str(existing.get(k) or "") != str(before.get(k) or "") for k in ("host", "port")) + if moved and existing.get("host_key") == before.get("host_key"): + existing.pop("host_key", None) d.config = dest_service.dump_config(existing) session.add(d) session.commit() @@ -115,9 +122,24 @@ def delete_destination( user: User = Depends(require_admin), ) -> dict: d = _get_or_404(session, dest_id) - name = d.name + users = session.exec( + select(BackupSchedule).where(BackupSchedule.destination_id == dest_id) + ).all() + if users: + # Deleted from under them, every run of these failed with + # "destination not found" and paged somebody about it. + raise HTTPException( + status_code=409, + detail=( + f"Destination '{d.name}' is used by {len(users)} backup schedule(s) — " + "point them at another destination or delete them first" + ), + ) + name, kind = d.name, d.type session.delete(d) session.commit() + if kind == "nfs": + dest_service.forget_nfs_volume(dest_id) audit_service.record( session, user=user.username, action="destination.delete", target=name, ip=_ip(request), @@ -134,9 +156,16 @@ async def test_destination( d = _get_or_404(session, dest_id) try: await asyncio.to_thread(dest_service.test, d) - return {"ok": True} except dest_service.DestinationError as exc: return {"ok": False, "error": str(exc)} + result: dict = {"ok": True} + if d.type == "sftp": + # Re-read: a first connection has just pinned the server's key. + session.refresh(d) + pinned = (dest_service.parse_config(d).get("host_key") or "").strip() + if pinned: + result["host_key"] = dest_service.fingerprint_of(pinned) + return result @router.get("/{dest_id}/backups") diff --git a/backend/routers/editor.py b/backend/routers/editor.py index b595790..1838263 100644 --- a/backend/routers/editor.py +++ b/backend/routers/editor.py @@ -1,6 +1,8 @@ """Editor helper endpoints — merge wizard fragments into compose YAML.""" from __future__ import annotations +import os + from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel @@ -15,6 +17,8 @@ router = APIRouter(prefix="/api/editor", tags=["editor"]) class ValidateBody(BaseModel): yaml: str env: str = "" + #: An existing stack: relative paths are then resolved in its folder. + stack_id: str | None = None class AddVolumeBody(BaseModel): @@ -61,7 +65,18 @@ def _run(fn, *args) -> dict: @router.post("/validate") async def validate(body: ValidateBody, _user: User = Depends(get_current_user)) -> dict: """Run `docker compose config -q` on the supplied YAML (+ optional .env).""" - return await compose_service.validate_yaml(body.yaml, body.env) + project_dir = None + if body.stack_id: + candidate = compose_service.stack_dir(body.stack_id) + # Only a real stack folder under STACKS_DIR, never a path out of it. + if os.path.dirname(os.path.normpath(candidate)) == os.path.normpath( + compose_service.stacks_root() + ): + project_dir = candidate + try: + return await compose_service.validate_yaml(body.yaml, body.env, project_dir) + except compose_service.StackFileError as exc: + raise HTTPException(status_code=504, detail=str(exc)) from exc @router.post("/services") diff --git a/backend/routers/files.py b/backend/routers/files.py index 9bb7479..af9cdc0 100644 --- a/backend/routers/files.py +++ b/backend/routers/files.py @@ -10,7 +10,9 @@ All paths are sandboxed by :mod:`services.file_service`. from __future__ import annotations import os +import stat import tempfile +from urllib.parse import quote from fastapi import ( APIRouter, @@ -35,9 +37,19 @@ router = APIRouter(prefix="/api/files", tags=["files"]) def _attachment(filename: str) -> str: - """A safe ``Content-Disposition`` value for an arbitrary filename.""" + """A safe ``Content-Disposition`` value for an arbitrary filename. + + Headers are latin-1; a folder called "Fotos – 2024" (en dash) raised + UnicodeEncodeError and the download died with a 500. Non-ASCII names go + out as RFC 5987 ``filename*``, with an ASCII fallback beside it. + """ safe = filename.replace("\\", "_").replace('"', "_") - return f'attachment; filename="{safe}"' + try: + safe.encode("ascii") + return f'attachment; filename="{safe}"' + except UnicodeEncodeError: + fallback = safe.encode("ascii", "replace").decode("ascii").replace("?", "_") + return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename)}" def _ip(request: Request) -> str: @@ -230,6 +242,27 @@ def delete( return result +def _settle_mode(tmp: str, target: str) -> None: + """Give an upload the permissions a normal file there would have. + + A temp file is created 0600, and ``os.replace`` kept that — so a config + uploaded into a stack folder was unreadable for any container not running + as root. An overwritten file keeps its mode and owner; a new one gets the + usual 0644 (minus umask). + """ + try: + st = os.stat(target) + os.chmod(tmp, stat.S_IMODE(st.st_mode)) + try: + os.chown(tmp, st.st_uid, st.st_gid) + except OSError: + pass + except FileNotFoundError: + umask = os.umask(0) + os.umask(umask) + os.chmod(tmp, 0o666 & ~umask) + + @router.post("/upload") async def upload( request: Request, @@ -249,6 +282,7 @@ async def upload( while chunk := await file.read(1024 * 1024): tmp.write(chunk) tmp.close() + _settle_mode(tmp.name, real) os.replace(tmp.name, real) except OSError as exc: if os.path.exists(tmp.name): diff --git a/backend/routers/git.py b/backend/routers/git.py index 7d0f4f0..0829900 100644 --- a/backend/routers/git.py +++ b/backend/routers/git.py @@ -97,6 +97,11 @@ def connect( raise HTTPException(status_code=400, detail=f"Unknown auth type '{body.auth_type}'") if not (body.url or "").strip(): raise HTTPException(status_code=400, detail="A repository URL is required") + try: + git_service.validate_url(body.url) + git_service.validate_branch(body.branch or "main") + except git_service.GitError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc row = session.exec(select(GitSource).where(GitSource.stack_id == stack_id)).first() if row is None: @@ -221,6 +226,9 @@ async def webhook( body = await request.body() if not row or not git_service.verify_webhook(row, body, request.headers): raise HTTPException(status_code=404, detail="Not found") + if not session.get(Stack, stack_id): + # Signed, but for a stack that was deleted: never bring it back. + raise HTTPException(status_code=404, detail="Not found") try: result = await git_service.sync(session, row, actor="webhook") diff --git a/backend/routers/images.py b/backend/routers/images.py index 901840d..d5be8c6 100644 --- a/backend/routers/images.py +++ b/backend/routers/images.py @@ -10,7 +10,7 @@ from auth import get_current_user, require_admin from database import engine, get_session from models.image_scan import ImageScan, ScanDetail, ScanRequest, ScanSummary from models.user import User -from services import audit_service, image_service, scan_service, update_service +from services import audit_service, compose_service, image_service, scan_service, update_service from services.prune_job import PruneBusy router = APIRouter(prefix="/api/images", tags=["images"]) @@ -21,8 +21,18 @@ def _ip(request: Request) -> str: @router.get("") -def list_images(_user: User = Depends(get_current_user)) -> list[dict]: - return image_service.list_images() +def list_images( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> list[dict]: + rows = image_service.list_images() + # Keyed by stack id like the stack list, or a stack whose folder name + # compose had to normalise ("Nextcloud" -> "nextcloud") showed up as a + # stranger whose stack had been removed. + ids = compose_service.stack_ids_by_project_in(session) + for row in rows: + row["stacks"] = sorted({compose_service.owner_stack(p, ids) for p in row["stacks"]}) + return rows @router.get("/updates") @@ -160,20 +170,40 @@ async def scan_image( @router.post("/scan-all") async def scan_all( request: Request, + response: Response, + background: bool = Query(False), session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: - """Scan every image a stack is using. Serial, and one sweep at a time.""" - images = sorted({row["tag"] for row in image_service.list_images() if row["stacks"]}) + """Scan every image a stack is using. Serial, and one sweep at a time. + + ``background=true`` returns 202 with the sweep's status at once; poll + ``/scan-status`` for progress and the result. That is what the UI does — + waiting on the response outlived the proxy's timeout. + """ + # Images of StackPilot's own stacks — not of every compose project on the + # host, which "a stack is using" used to include (other tools' projects, + # leftovers of deleted stacks). + ids = compose_service.stack_ids_by_project_in(session) + images = sorted( + {row["tag"] for row in image_service.list_images() if set(row["stacks"]) & ids.keys()} + ) + username, ip = user.username, _ip(request) + + def audit(own: Session, result: dict) -> None: + audit_service.record( + own, user=username, action="image.scan-all", target="*", + detail=f"{result['scanned']} scanned, {result['failed']} failed", ip=ip, + ) + try: + if background: + response.status_code = 202 + return scan_service.start_sweep(images, on_done=audit) result = await scan_service.sweep(session, images) except scan_service.ScanError as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc - audit_service.record( - session, user=user.username, action="image.scan-all", target="*", - detail=f"{result['scanned']} scanned, {result['failed']} failed", - ip=_ip(request), - ) + audit(session, result) return result diff --git a/backend/routers/networks.py b/backend/routers/networks.py index 62df9da..5c872b0 100644 --- a/backend/routers/networks.py +++ b/backend/routers/networks.py @@ -9,7 +9,7 @@ from auth import get_current_user, require_admin from database import engine, get_session from docker_client import DockerError from models.user import User -from services import audit_service, network_service +from services import audit_service, compose_service, network_service from services.prune_job import PruneBusy router = APIRouter(prefix="/api/networks", tags=["networks"]) @@ -44,8 +44,15 @@ class ContainerRef(BaseModel): @router.get("") -def list_networks(_user: User = Depends(get_current_user)) -> list[dict]: - return network_service.list_networks() +def list_networks( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> list[dict]: + rows = network_service.list_networks() + ids = compose_service.stack_ids_by_project_in(session) + for row in rows: + row["stack"] = compose_service.owner_stack(row["stack"], ids) + return rows # Before /{network_id}, which would otherwise take "prune-status" for an id. diff --git a/backend/routers/secrets.py b/backend/routers/secrets.py index 118ed83..a6ffcc6 100644 --- a/backend/routers/secrets.py +++ b/backend/routers/secrets.py @@ -13,6 +13,7 @@ from sqlmodel import Session from auth import require_admin from database import get_session +from models.stack import Stack from models.user import User from services import audit_service, secret_service from services.compose_edit_service import EditError @@ -51,6 +52,16 @@ def _guard(fn, *args, **kwargs): raise HTTPException(status_code=400, detail=str(exc)) from exc +def _stack(session: Session, stack_id: str) -> None: + """404 for a stack that does not exist. + + Without it a typo in the URL created ``//.secrets`` on + disk, and ``..`` reached the directory above the stacks. + """ + if not session.get(Stack, stack_id): + raise HTTPException(status_code=404, detail=f"Stack '{stack_id}' not found") + + def list_secrets(stack_id: str) -> list[dict]: @@ -82,7 +93,12 @@ def detach_secret(stack_id: str, body: DetachBody) -> dict: @router.get("") -def http_list(stack_id: str, _user: User = Depends(require_admin)) -> list[dict]: +def http_list( + stack_id: str, + session: Session = Depends(get_session), + _user: User = Depends(require_admin), +) -> list[dict]: + _stack(session, stack_id) return list_secrets(stack_id) @@ -91,6 +107,7 @@ def http_write( stack_id: str, body: SecretWrite, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _stack(session, stack_id) result = write_secret(stack_id, body) audit_service.record( session, user=user.username, action="secret.write", @@ -104,6 +121,7 @@ def http_delete( stack_id: str, kind: str, name: str, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _stack(session, stack_id) result = delete_secret(stack_id, kind, name) audit_service.record( session, user=user.username, action="secret.delete", @@ -117,6 +135,7 @@ def http_attach( stack_id: str, body: AttachBody, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _stack(session, stack_id) result = attach_secret(stack_id, body) audit_service.record( session, user=user.username, action="secret.attach", @@ -130,6 +149,7 @@ def http_detach( stack_id: str, body: DetachBody, request: Request, session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _stack(session, stack_id) result = detach_secret(stack_id, body) audit_service.record( session, user=user.username, action="secret.detach", diff --git a/backend/routers/stacks.py b/backend/routers/stacks.py index c1588d8..1d0d56a 100644 --- a/backend/routers/stacks.py +++ b/backend/routers/stacks.py @@ -9,6 +9,7 @@ from dataclasses import asdict from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile from fastapi.responses import FileResponse from sqlmodel import Session, select +from starlette.background import BackgroundTask from auth import get_current_user, require_admin from database import get_session @@ -36,6 +37,7 @@ from services import ( icon_service, logo_service, notify_service, + stack_cleanup_service, stack_job_service, stack_lock_service, stats_service, @@ -131,7 +133,7 @@ def _stack_summary( total = running = 0 status = "unknown" else: - info = summaries.get(stack.id) + info = summaries.get(compose_service.project_name(stack.id)) total = info["total"] if info else 0 running = info["running"] if info else 0 if stack.id in busy: @@ -204,17 +206,30 @@ def create_stack( return _stack_summary(stack) +def _by_stack_id(session: Session, per_project: dict) -> dict: + """Re-key per-compose-project data by stack id (they differ only for a + folder whose name compose had to normalise, see project_name).""" + ids = compose_service.stack_ids_by_project_in(session) + return {ids.get(project, project): value for project, value in per_project.items()} + + @router.get("/stats") -def stacks_stats(_user: User = Depends(get_current_user)) -> dict: +def stacks_stats( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> dict: """Live CPU (cores) and memory usage per stack, with assigned limits.""" - return stats_service.stack_stats() + return _by_stack_id(session, stats_service.stack_stats()) @router.get("/updates") -def stacks_updates(_user: User = Depends(get_current_user)) -> dict: +def stacks_updates( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> dict: """Per-stack image-update availability, read from the cached registry digests (no live registry calls — safe for the list to poll).""" - return update_service.stacks_update_summary() + return _by_stack_id(session, update_service.stacks_update_summary()) @router.get("/icons/search") @@ -275,7 +290,12 @@ def get_stack( # What is holding the stack, so the page can offer to stop it instead # of only showing a spinner that never ends. "lock": _lock_view(lock, stack_id), - "yaml": compose_service.read_compose(stack_id), + # Inline credentials masked for the read-only role, like the .env. + "yaml": ( + compose_service.read_compose(stack_id) + if user.role == "admin" + else compose_service.mask_secrets(compose_service.read_compose(stack_id)) + ), # The .env is where credentials live by convention, so it is withheld # from the read-only role — same reasoning as the admin-only file # browser. Non-admins still get status, services and the compose file. @@ -333,21 +353,49 @@ async def delete_stack( user: User = Depends(require_admin), ) -> dict: stack = _get_stack_or_404(session, stack_id) + # Deleting a stack in the middle of a backup, restore or deploy pulls the + # folder out from under it — and a backup then starts the containers it + # stopped again, for a stack that no longer exists. + lock = stack_lock_service.info(session, stack_id) + if lock: + raise HTTPException( + status_code=409, + detail=f"Stack '{stack_id}' is busy: {lock['action']} in progress", + ) try: - await compose_service.down(stack_id) - except Exception: # noqa: BLE001 - best-effort teardown - pass + result = await compose_service.down(stack_id) + down_ok = result.get("returncode") == 0 + except Exception: # noqa: BLE001 - fall back to the API below + down_ok = False + if not down_ok: + # A compose file that no longer parses makes `down` fail, and the + # containers would keep running with nothing left here to stop them. + try: + removed = await asyncio.to_thread(compose_service.force_remove_project, stack_id) + logger.info("compose down failed for '%s'; removed %d container(s) directly", + stack_id, removed) + except DockerError as exc: + logger.warning("Could not remove the containers of '%s': %s", stack_id, exc) + kept: list[str] = [] if delete_files: - compose_service.delete_stack_files(stack_id) + try: + kept = compose_service.delete_stack_files(stack_id) + except OSError as exc: + raise HTTPException( + status_code=500, + detail=f"The stack was stopped, but its folder could not be deleted: {exc}", + ) from exc icon_service.remove(stack_id) logo_service.forget(stack_id) + stack_cleanup_service.forget_stack(session, stack_id) session.delete(stack) session.commit() audit_service.record( session, user=user.username, action="stack.delete", target=stack_id, - detail=f"delete_files={delete_files}", ip=_client_ip(request), + detail=f"delete_files={delete_files}" + (f" kept={len(kept)} mount(s)" if kept else ""), + ip=_client_ip(request), ) - return {"ok": True} + return {"ok": True, "kept": kept} @router.post("/{stack_id}/clone") @@ -365,7 +413,8 @@ def clone_stack( try: compose_service.clone_stack_files(stack_id, new_id) except compose_service.StackFileError as exc: - raise HTTPException(status_code=409, detail=str(exc)) from exc + status = 409 if "already exists" in str(exc) else 500 + raise HTTPException(status_code=status, detail=str(exc)) from exc stack = Stack( id=new_id, name=body.name, @@ -412,6 +461,8 @@ async def get_stack_icon( def _icon_response(path: str, media_type: str, filename: str) -> FileResponse: + # A folder found on disk can be named anything; headers are latin-1. + filename = filename.encode("ascii", "replace").decode("ascii").replace("?", "_") return FileResponse( path, media_type=media_type, @@ -516,6 +567,9 @@ async def _lifecycle(action_fn, action_name, stack_id, request, session, user): status_code=409, detail=f"Stack '{stack_id}' is busy: {exc.action} in progress", ) from exc + except compose_service.StackFileError as exc: + # The compose call timed out; it may still be finishing on the host. + raise HTTPException(status_code=504, detail=str(exc)) from exc audit_service.record( session, user=user.username, action=f"stack.{action_name}", target=stack_id, detail=f"rc={result.get('returncode')}", ip=_client_ip(request), @@ -683,7 +737,7 @@ async def unlock_stack( @router.get("/{stack_id}/logs") async def stack_logs( stack_id: str, - tail: int = Query(200, le=2000), + tail: int = Query(200, ge=0, le=2000), session: Session = Depends(get_session), _user: User = Depends(get_current_user), ) -> dict: @@ -696,7 +750,7 @@ async def stack_logs( async def service_logs( stack_id: str, service: str, - tail: int = Query(200, le=2000), + tail: int = Query(200, ge=0, le=2000), session: Session = Depends(get_session), _user: User = Depends(get_current_user), ) -> dict: @@ -721,13 +775,20 @@ def export_stack( if not os.path.isdir(directory): raise HTTPException(status_code=404, detail="Stack directory missing") tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz") - with tarfile.open(tmp.name, "w:gz") as tar: - tar.add(directory, arcname=stack_id) + tmp.close() + try: + with tarfile.open(tmp.name, "w:gz") as tar: + tar.add(directory, arcname=stack_id) + except BaseException: + os.unlink(tmp.name) + raise date = compose_service.now().strftime("%Y%m%d") return FileResponse( tmp.name, media_type="application/gzip", filename=f"stack-{stack_id}-{date}.tar.gz", + # The archive is a temp file; without this every export stayed in /tmp. + background=BackgroundTask(os.unlink, tmp.name), ) @@ -765,6 +826,7 @@ def set_auto_update( session: Session = Depends(get_session), user: User = Depends(require_admin), ) -> dict: + _get_stack_or_404(session, stack_id) policy = auto_update_service.upsert_policy(session, stack_id, body.enabled, body.redeploy) audit_service.record( session, user=user.username, action="stack.auto_update", diff --git a/backend/routers/volumes.py b/backend/routers/volumes.py index 4a2b6ab..2fa965d 100644 --- a/backend/routers/volumes.py +++ b/backend/routers/volumes.py @@ -7,7 +7,7 @@ from sqlmodel import Session from auth import get_current_user, require_admin from database import engine, get_session from models.user import User -from services import audit_service, device_service, volume_service +from services import audit_service, compose_service, device_service, volume_service from services.prune_job import PruneBusy router = APIRouter(tags=["volumes"]) @@ -23,8 +23,15 @@ def _ip(request: Request) -> str: @router.get("/api/volumes") -def list_volumes(_user: User = Depends(get_current_user)) -> list[dict]: - return volume_service.list_volumes() +def list_volumes( + session: Session = Depends(get_session), + _user: User = Depends(get_current_user), +) -> list[dict]: + rows = volume_service.list_volumes() + ids = compose_service.stack_ids_by_project_in(session) + for row in rows: + row["stack"] = compose_service.owner_stack(row["stack"], ids) + return rows @router.get("/api/volumes/orphaned") diff --git a/backend/routers/ws.py b/backend/routers/ws.py index 0f48708..44d50d4 100644 --- a/backend/routers/ws.py +++ b/backend/routers/ws.py @@ -2,10 +2,11 @@ from __future__ import annotations import asyncio +import contextlib import json import logging - -import contextlib +import threading +from collections.abc import Awaitable, Callable from fastapi import APIRouter, Query, WebSocket, WebSocketDisconnect from jose import JWTError @@ -14,12 +15,13 @@ from sqlmodel import Session from auth import decode_token, resolve_token_user from database import engine from models.setting import EVENT_PULL_FAILED, EVENT_STACK_ERROR, EVENT_STACK_START +from models.stack import Stack from services import ( audit_service, + compose_run_service, compose_service, exec_service, notify_service, - stack_lock_service, update_service, ) @@ -49,52 +51,93 @@ def _user_for(token: str | None): return user -async def _authorize(websocket: WebSocket, token: str | None) -> bool: - """Validate the JWT supplied as a query param. Closes socket on failure.""" - if _user_for(token) is None: - await websocket.close(code=4401) - return False - return True - - -async def _authorize_admin(websocket: WebSocket, token: str | None) -> bool: - """Like _authorize but also requires the admin role (exec is root-equivalent). - - Closes 4401 on a missing/invalid token, 4403 on a valid non-admin token.""" +async def _authorize(websocket: WebSocket, token: str | None): + """The user behind the token, or None after closing the socket with 4401.""" user = _user_for(token) if user is None: await websocket.close(code=4401) - return False + return user + + +async def _authorize_admin(websocket: WebSocket, token: str | None): + """Like _authorize but also requires the admin role. + + Anything that changes what runs on the host is admin-only over HTTP, so it + has to be over a socket too — the deploy console runs the same + ``compose up`` the REST ``/start`` does. Closes 4401 on a missing/invalid + token, 4403 on a valid non-admin one.""" + user = _user_for(token) + if user is None: + await websocket.close(code=4401) + return None if user.role != "admin": await websocket.close(code=4403) - return False - return True + return None + return user + + +async def _wait_disconnect(websocket: WebSocket) -> None: + """Return once the client has gone. Whatever it sends is ignored. + + A stream that only ever *sends* notices a closed socket on its next send — + which for a quiet log or an idle Docker host can be hours away, and until + then everything behind the socket (a ``compose logs -f``, an event reader + thread) stays alive for nobody. + """ + try: + while True: + message = await websocket.receive() + if message.get("type") == "websocket.disconnect": + return + except Exception: # noqa: BLE001 - a broken socket is a gone client + return + + +async def _until_gone(websocket: WebSocket, work: Awaitable[None]) -> None: + """Run ``work`` until it finishes or the client disconnects, whichever is first. + + Cancelling ``work`` is what stops whatever it was reading from; see + :func:`compose_service.stream_compose`. + """ + task = asyncio.ensure_future(work) + gone = asyncio.create_task(_wait_disconnect(websocket)) + try: + await asyncio.wait({task, gone}, return_when=asyncio.FIRST_COMPLETED) + finally: + for pending in (task, gone): + pending.cancel() + await asyncio.gather(task, gone, return_exceptions=True) + if task.done() and not task.cancelled() and task.exception() is not None: + exc = task.exception() + if not isinstance(exc, WebSocketDisconnect): + with contextlib.suppress(Exception): + await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) async def _stream_logs(websocket: WebSocket, stack_id: str, service: str | None): """Stream `docker compose logs -f` output to the client.""" args = ["logs", "--no-color", "--tail", "200", "--timestamps", "-f"] if service: - args.append(service) - try: - async for line in compose_service.stream_compose(stack_id, args): - await websocket.send_text( - json.dumps( - { - "type": "log", - "stack_id": stack_id, - "service": service, - "line": line, - } - ) - ) - except WebSocketDisconnect: - raise - except Exception as exc: # noqa: BLE001 + # "--" so a service name can never be read as a compose option. + args += ["--", service] + + async def pump() -> None: + # aclosing: cancelled while *sending*, the loop is left by an exception + # outside the generator, which would then only be closed — and its + # `compose logs -f` killed — whenever garbage collection got to it. + async with contextlib.aclosing( + compose_service.stream_compose(stack_id, args) + ) as lines: + async for line in lines: + await _send_log(line) + + async def _send_log(line: str) -> None: await websocket.send_text( - json.dumps({"type": "error", "detail": str(exc)}) + json.dumps({"type": "log", "stack_id": stack_id, "service": service, "line": line}) ) + await _until_gone(websocket, pump()) + @router.websocket("/ws/logs/{stack_id}") async def ws_stack_logs( @@ -105,10 +148,7 @@ async def ws_stack_logs( await websocket.accept() if not await _authorize(websocket, token): return - try: - await _stream_logs(websocket, stack_id, None) - except WebSocketDisconnect: - pass + await _stream_logs(websocket, stack_id, None) @router.websocket("/ws/logs/{stack_id}/{service}") @@ -121,10 +161,67 @@ async def ws_service_logs( await websocket.accept() if not await _authorize(websocket, token): return + await _stream_logs(websocket, stack_id, service) + + +# --------------------------------------------------------------------------- # +# Deploy / update: a compose run the socket only watches +# --------------------------------------------------------------------------- # + + +def _stack_known(stack_id: str) -> bool: + with Session(engine) as session: + return session.get(Stack, stack_id) is not None + + +async def _watch_run( + websocket: WebSocket, + stack_id: str, + action: str, + username: str, + stream: Callable, + on_finish: Callable[[int | None], Awaitable[None]], +) -> None: + """Start (or attach to) the compose run and relay it until it ends. + + The run belongs to :mod:`services.compose_run_service`, not to this socket: + closing the deploy console no longer releases the stack lock under a + running compose, and the audit entry carries the real exit code. + """ + if not _stack_known(stack_id): + with contextlib.suppress(Exception): + await websocket.send_text( + json.dumps({"type": "error", "detail": f"Stack '{stack_id}' not found"}) + ) + await websocket.close(code=4404) + return try: - await _stream_logs(websocket, stack_id, service) - except WebSocketDisconnect: - pass + run, queue, _attached = compose_run_service.start( + stack_id, action, username, stream, on_finish + ) + except compose_run_service.RunBusy as exc: + with contextlib.suppress(Exception): + await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) + await websocket.close(code=4409) + return + + async def relay() -> None: + while True: + item = await queue.get() + if item is None: + return + kind, payload = item + if kind == "log": + await websocket.send_text(json.dumps({"type": "log", "line": payload})) + else: + await websocket.send_text(json.dumps({"type": "done", "returncode": payload})) + + try: + await _until_gone(websocket, relay()) + finally: + run.unsubscribe(queue) + with contextlib.suppress(Exception): + await websocket.close() @router.websocket("/ws/deploy/{stack_id}") @@ -135,53 +232,21 @@ async def ws_deploy( ): """Run `docker compose up -d` and stream its output (image pulls, container creation) to the browser so the user sees deploy progress live. Records the - same audit entry and notification as the REST `/start` endpoint.""" + same audit entry and notification as the REST `/start` endpoint. Admin-only, + like that endpoint.""" await websocket.accept() - if not await _authorize(websocket, token): + user = await _authorize_admin(websocket, token) + if not user: return - username = decode_token(token, "access").get("sub", "unknown") if token else "unknown" + username = user.username - rc: int | None = None - disconnected = False - # Same guard the REST lifecycle uses — the deploy console runs the very - # same `compose up`, so it has to queue behind an in-flight operation - # rather than race it. - lock_session = Session(engine) - try: - stack_lock_service.acquire(lock_session, stack_id, "start", username) - except stack_lock_service.StackBusy as exc: - lock_session.close() - with contextlib.suppress(Exception): - await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) - await websocket.close(code=4409) - return - try: - async for kind, payload in compose_service.stream_up(stack_id): - if kind == "log": - await websocket.send_text(json.dumps({"type": "log", "line": payload})) - else: - rc = payload - await websocket.send_text(json.dumps({"type": "done", "returncode": rc})) - except WebSocketDisconnect: - # Client navigated away; the compose subprocess keeps running so the - # deploy still completes in the background. - disconnected = True - except Exception as exc: # noqa: BLE001 - with contextlib.suppress(Exception): - await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) - finally: - with contextlib.suppress(Exception): - stack_lock_service.release(lock_session, stack_id) - lock_session.close() - - ok = rc in (0, None) - try: + async def finish(rc: int | None) -> None: with Session(engine) as session: audit_service.record( session, user=username, action="stack.start", target=stack_id, detail=f"rc={rc} (deploy console)", ip="ws", ) - if ok: + if rc == 0: await notify_service.notify( EVENT_STACK_START, f"Stack '{stack_id}' started", "compose up completed successfully.", session, @@ -191,11 +256,11 @@ async def ws_deploy( EVENT_STACK_ERROR, f"Stack '{stack_id}' start failed", "compose up returned a non-zero exit code.", session, ) - except Exception: # noqa: BLE001 - audit/notify are best-effort - pass - if not disconnected: - with contextlib.suppress(Exception): - await websocket.close() + + await _watch_run( + websocket, stack_id, "start", username, + lambda: compose_service.stream_up(stack_id), finish, + ) @router.websocket("/ws/update/{stack_id}") @@ -208,47 +273,18 @@ async def ws_update( list can render real update progress. Same audit/notify contract as the REST `/update` endpoint, which stays for non-interactive callers.""" await websocket.accept() - if not await _authorize_admin(websocket, token): + user = await _authorize_admin(websocket, token) + if not user: return - username = decode_token(token, "access").get("sub", "unknown") if token else "unknown" + username = user.username - rc: int | None = None - disconnected = False - lock_session = Session(engine) - try: - stack_lock_service.acquire(lock_session, stack_id, "update", username) - except stack_lock_service.StackBusy as exc: - lock_session.close() - with contextlib.suppress(Exception): - await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) - await websocket.close(code=4409) - return - try: - async for kind, payload in compose_service.stream_update(stack_id): - if kind == "log": - await websocket.send_text(json.dumps({"type": "log", "line": payload})) - else: - rc = payload - await websocket.send_text(json.dumps({"type": "done", "returncode": rc})) - except WebSocketDisconnect: - # Client navigated away; compose keeps running so the update finishes. - disconnected = True - except Exception as exc: # noqa: BLE001 - with contextlib.suppress(Exception): - await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) - finally: - with contextlib.suppress(Exception): - stack_lock_service.release(lock_session, stack_id) - lock_session.close() - - ok = rc in (0, None) - try: + async def finish(rc: int | None) -> None: with Session(engine) as session: audit_service.record( session, user=username, action="stack.update", target=stack_id, detail=f"rc={rc} (update stream)", ip="ws", ) - if ok: + if rc == 0: await notify_service.notify( EVENT_STACK_START, f"Stack '{stack_id}' updated", "compose pull + up completed successfully.", session, @@ -258,16 +294,19 @@ async def ws_update( EVENT_PULL_FAILED, f"Stack '{stack_id}' update failed", "compose pull/up returned a non-zero exit code.", session, ) - except Exception: # noqa: BLE001 - audit/notify are best-effort - pass - if not disconnected: - with contextlib.suppress(Exception): - await websocket.close() + if rc == 0: + await asyncio.to_thread(update_service.refresh_stack_local, stack_id) - if ok: - update_service.refresh_stack_local(stack_id) + await _watch_run( + websocket, stack_id, "update", username, + lambda: compose_service.stream_update(stack_id), finish, + ) +# --------------------------------------------------------------------------- # +# Docker events +# --------------------------------------------------------------------------- # + #: Docker event types worth telling the UI about. Filtered daemon-side, so the #: bulk of the firehose never crosses the socket. _EVENT_TYPES = ["container", "image", "network", "volume"] @@ -300,33 +339,58 @@ async def ws_events( "container": "jellyfin", "stack": "jellyfin"} ``resource`` is what the client needs to decide which queries to drop. + + The blocking read runs on a thread of its own, not on the event loop's + shared executor: every open tab holds one reader for as long as it is open, + and on the executor a handful of tabs used up every worker — after which + each ``asyncio.to_thread`` (backups, the dashboard, NFS destinations) waited + forever. """ await websocket.accept() if not await _authorize(websocket, token): return from docker_client import get_client - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() queue: asyncio.Queue = asyncio.Queue() - stream = None + guard = threading.Lock() + state: dict = {"stream": None, "stop": False} - def reader(): + def post(item) -> None: + try: + loop.call_soon_threadsafe(queue.put_nowait, item) + except RuntimeError: # the loop is gone (shutdown) + pass + + def reader() -> None: """Blocking read of the event stream, handed to the loop thread-safely.""" - nonlocal stream try: stream = get_client().events(decode=True, filters={"type": _EVENT_TYPES}) + with guard: + if state["stop"]: + # The client left while we were connecting. + with contextlib.suppress(Exception): + stream.close() + return + state["stream"] = stream for event in stream: - loop.call_soon_threadsafe(queue.put_nowait, event) + post(event) except Exception: # noqa: BLE001 - a closed stream lands here on teardown pass finally: - loop.call_soon_threadsafe(queue.put_nowait, None) + post(None) - task = loop.run_in_executor(None, reader) + threading.Thread(target=reader, name="docker-events", daemon=True).start() + gone = asyncio.create_task(_wait_disconnect(websocket)) try: await websocket.send_text(json.dumps({"type": "ready"})) while True: - event = await queue.get() + getter = asyncio.ensure_future(queue.get()) + done, _ = await asyncio.wait({getter, gone}, return_when=asyncio.FIRST_COMPLETED) + if getter not in done: + getter.cancel() + break # the client disconnected + event = getter.result() if event is None: # reader finished — daemon gone or stream closed await websocket.send_text( json.dumps({"type": "error", "detail": "Docker event stream ended"}) @@ -354,14 +418,14 @@ async def ws_events( with contextlib.suppress(Exception): await websocket.send_text(json.dumps({"type": "error", "detail": str(exc)})) finally: + gone.cancel() # Closing the stream is what actually unblocks the reader thread. - # Cancelling the executor future does not: a thread already inside a - # blocking read keeps that read, and the thread leaks for the life of - # the process — once per page load, with the socket held open. + with guard: + state["stop"] = True + stream = state["stream"] if stream is not None: with contextlib.suppress(Exception): stream.close() - task.cancel() @router.websocket("/ws/exec/{container_id}") @@ -373,9 +437,10 @@ async def ws_exec( ): """Interactive shell into a compose-managed container (admin only).""" await websocket.accept() - if not await _authorize_admin(websocket, token): + user = await _authorize_admin(websocket, token) + if not user: return - username = decode_token(token, "access").get("sub", "unknown") if token else "unknown" + username = user.username shell = cmd or exec_service.DEFAULT_SHELL try: @@ -402,5 +467,3 @@ async def ws_exec( finally: with contextlib.suppress(Exception): await websocket.close() - - diff --git a/backend/services/auto_update_service.py b/backend/services/auto_update_service.py index c39250d..424b9a4 100644 --- a/backend/services/auto_update_service.py +++ b/backend/services/auto_update_service.py @@ -103,8 +103,12 @@ async def _run_local(session: Session, policy: AutoUpdate) -> None: # this stack, skip and pick it up next cycle rather than racing # them over the same containers. with stack_lock_service.hold(session, stack_id, "auto-update", "auto-update"): - await compose_service.pull(stack_id) - await compose_service.up(stack_id) + # pull + up, stopping at a failed pull. The exit codes used to + # be ignored entirely: a registry outage or a failing `up` was + # recorded as "updated" and announced as a success. + result = await compose_service.update(stack_id) + if result.get("returncode") != 0: + raise RuntimeError(_failure(result)) except stack_lock_service.StackBusy as exc: _record(session, policy, "skipped", str(exc)) return @@ -127,6 +131,13 @@ async def _run_local(session: Session, policy: AutoUpdate) -> None: ) +def _failure(result: dict) -> str: + """The last meaningful line of a failed compose run.""" + text = (result.get("stderr") or result.get("stdout") or "").strip() + tail = text.splitlines()[-1] if text else "" + return f"compose exited with {result.get('returncode')}" + (f": {tail}" if tail else "") + + async def _safe_notify(event: str, title: str, message: str, session: Session) -> None: try: await notify_service.notify(event, title, message, session) diff --git a/backend/services/backup_destination_service.py b/backend/services/backup_destination_service.py index b181ada..7c34dcc 100644 --- a/backend/services/backup_destination_service.py +++ b/backend/services/backup_destination_service.py @@ -76,24 +76,169 @@ def migrate_plaintext_configs(session) -> int: # --------------------------------------------------------------------------- # -def _sftp_connect(cfg: dict): +#: Seconds to wait for the SFTP server to accept the TCP connection. +SFTP_CONNECT_TIMEOUT = 20 + + +def _check_file_name(name: str) -> None: + """A backup is addressed by its bare file name, never a path. + + The name comes from the API (restore from / delete on a destination); with + ``../`` in it a delete reached files outside the backup folder — on SFTP + anything the account may write. NFS has always checked this. + """ + if ( + not name + or name in (".", "..") + or "/" in name + or "\\" in name + or "\0" in name + ): + raise DestinationError(f"Invalid backup file name '{name}'") + + +def host_key_fingerprint(key) -> str: + """OpenSSH's form: ``SHA256:`` + unpadded base64 of the key's SHA-256.""" + import base64 + import hashlib + + digest = hashlib.sha256(key.asbytes()).digest() + return "SHA256:" + base64.b64encode(digest).decode().rstrip("=") + + +def fingerprint_of(pinned: str) -> str: + """The ``SHA256:`` fingerprint of a stored host key (as given, if it is + one already) — what the UI shows and ``ssh-keygen -lf`` prints.""" + import base64 + import hashlib + + pinned = pinned.strip() + if pinned.startswith("SHA256:"): + return pinned + parts = pinned.split() + for i, part in enumerate(parts[:-1]): + if part.startswith(("ssh-", "ecdsa-", "sk-")): + try: + blob = base64.b64decode(parts[i + 1]) + except ValueError: + break + return "SHA256:" + base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip("=") + return pinned + + +def _pinned_type(pinned: str) -> Optional[str]: + """The key type of a pinned `` `` line (a known_hosts line + with the host in front works too); None for a bare fingerprint.""" + parts = pinned.split() + for part in parts[:-1]: + if part.startswith(("ssh-", "ecdsa-", "sk-")): + return part + return None + + +#: What paramiko negotiates for a key of each type. An RSA key is offered +#: under the SHA-2 signature names, not under its own. +_HOST_KEY_ALGORITHMS = { + "ssh-rsa": ("rsa-sha2-512", "rsa-sha2-256", "ssh-rsa"), +} + + +def _host_key_matches(pinned: str, key) -> bool: + pinned = pinned.strip() + if pinned.startswith("SHA256:"): + return pinned.rstrip("=") == host_key_fingerprint(key) + parts = pinned.split() + for i, part in enumerate(parts[:-1]): + if part == key.get_name() and parts[i + 1] == key.get_base64(): + return True + return False + + +def _sftp_connect(cfg: dict, pin=None): + """Connect and authenticate, checking the server's host key. + + Before 0.67.1 any host key was accepted, so whoever could answer on the + server's address (a spoofed DNS entry, a box that took its IP) received + the password and every backup. Now the key is pinned in the destination's + ``host_key``: entered by hand (a key line or its ``SHA256:`` fingerprint), + or — when empty — the one seen on the first connection, handed to ``pin`` + to be stored. After that a different key refuses the connection. + """ + import socket + import paramiko host = cfg.get("host") if not host: raise DestinationError("SFTP host is required") - port = int(cfg.get("port") or 22) - username = cfg.get("username") - transport = paramiko.Transport((host, port)) try: + port = int(cfg.get("port") or 22) + except (TypeError, ValueError) as exc: + raise DestinationError(f"Invalid SFTP port '{cfg.get('port')}'") from exc + username = cfg.get("username") + pinned = (cfg.get("host_key") or "").strip() + # Connected here, with a timeout, rather than by Transport((host, port)): + # that connects inside the constructor, outside any handler, so an + # unreachable host was a bare 500 instead of a message — and with no + # timeout it hung for the kernel's two minutes first. + try: + sock = socket.create_connection((host, port), timeout=SFTP_CONNECT_TIMEOUT) + except OSError as exc: + raise DestinationError(f"SFTP connection to {host}:{port} failed: {exc}") from exc + transport = paramiko.Transport(sock) + try: + key_type = _pinned_type(pinned) if pinned else None + if key_type: + # Ask for the pinned kind of key: a server has several, and the + # one negotiated by default need not be the one that was pinned. + wanted = _HOST_KEY_ALGORITHMS.get(key_type, (key_type,)) + options = transport.get_security_options() + usable = tuple(k for k in wanted if k in options.key_types) + if usable: + options.key_types = usable + transport.start_client(timeout=SFTP_CONNECT_TIMEOUT) + server_key = transport.get_remote_server_key() + if pinned: + if not _host_key_matches(pinned, server_key): + raise DestinationError( + f"SFTP host key of {host}:{port} does not match the stored one: the " + f"server now presents {server_key.get_name()} " + f"{host_key_fingerprint(server_key)}. If the server was reinstalled or " + "its keys were rotated, clear the destination's host key to trust the " + "new one — otherwise something else is answering on that address." + ) + elif pin is not None: + pin(f"{server_key.get_name()} {server_key.get_base64()}") pkey = None if cfg.get("private_key"): pkey = _load_key(cfg["private_key"]) - transport.connect(username=username, password=cfg.get("password") or None, pkey=pkey) + # What Transport.connect did: the key when there is one, else the password. + if pkey is not None: + transport.auth_publickey(username, pkey) + elif cfg.get("password"): + transport.auth_password(username, cfg["password"]) + return paramiko.SFTPClient.from_transport(transport), transport except Exception as exc: # noqa: BLE001 transport.close() + if isinstance(exc, DestinationError): + raise raise DestinationError(f"SFTP connection failed: {exc}") from exc - return paramiko.SFTPClient.from_transport(transport), transport + + +def _sftp_io(what: str): + """Turn what paramiko raises mid-transfer into a DestinationError.""" + import contextlib + + @contextlib.contextmanager + def wrapper(): + try: + yield + except DestinationError: + raise + except Exception as exc: # noqa: BLE001 - IOError, SSHException, EOFError … + raise DestinationError(f"SFTP {what} failed: {exc}") from exc + + return wrapper() def _load_key(key_str: str): @@ -120,22 +265,24 @@ def _sftp_makedirs(sftp, path: str) -> None: sftp.mkdir(cur) -def _sftp_upload(cfg: dict, local_path: str, filename: str) -> str: - sftp, transport = _sftp_connect(cfg) +def _sftp_upload(cfg: dict, local_path: str, filename: str, pin=None) -> str: + _check_file_name(filename) + sftp, transport = _sftp_connect(cfg, pin) try: - base = cfg.get("path") or "." - if base not in (".", ""): - _sftp_makedirs(sftp, base) - remote = posixpath.join(base, filename) if base not in (".", "") else filename - sftp.put(local_path, remote) - return remote + with _sftp_io("upload"): + base = cfg.get("path") or "." + if base not in (".", ""): + _sftp_makedirs(sftp, base) + remote = posixpath.join(base, filename) if base not in (".", "") else filename + sftp.put(local_path, remote) + return remote finally: sftp.close() transport.close() -def _sftp_list(cfg: dict) -> list[dict]: - sftp, transport = _sftp_connect(cfg) +def _sftp_list(cfg: dict, pin=None) -> list[dict]: + sftp, transport = _sftp_connect(cfg, pin) try: base = cfg.get("path") or "." out = [] @@ -155,23 +302,51 @@ def _sftp_list(cfg: dict) -> list[dict]: transport.close() -def _sftp_download(cfg: dict, name: str, local_path: str) -> None: - sftp, transport = _sftp_connect(cfg) +def _sftp_download(cfg: dict, name: str, local_path: str, pin=None) -> None: + _check_file_name(name) + sftp, transport = _sftp_connect(cfg, pin) try: - base = cfg.get("path") or "." - remote = posixpath.join(base, name) if base not in (".", "") else name - sftp.get(remote, local_path) + with _sftp_io(f"download of '{name}'"): + base = cfg.get("path") or "." + remote = posixpath.join(base, name) if base not in (".", "") else name + sftp.get(remote, local_path) finally: sftp.close() transport.close() -def _sftp_delete(cfg: dict, name: str) -> None: - sftp, transport = _sftp_connect(cfg) +def _sftp_delete(cfg: dict, name: str, pin=None) -> None: + _check_file_name(name) + sftp, transport = _sftp_connect(cfg, pin) try: - base = cfg.get("path") or "." - remote = posixpath.join(base, name) if base not in (".", "") else name - sftp.remove(remote) + with _sftp_io(f"delete of '{name}'"): + base = cfg.get("path") or "." + remote = posixpath.join(base, name) if base not in (".", "") else name + sftp.remove(remote) + finally: + sftp.close() + transport.close() + + +_TEST_FILE = ".stackpilot-test" + + +def _sftp_test(cfg: dict, pin=None) -> None: + """Connect, create the folder if needed, write and remove a file. + + Listing alone said "OK" for a folder that does not exist or cannot be + written to — the first scheduled backup was then the one to find out. + """ + sftp, transport = _sftp_connect(cfg, pin) + try: + with _sftp_io("write test"): + base = cfg.get("path") or "." + if base not in (".", ""): + _sftp_makedirs(sftp, base) + remote = posixpath.join(base, _TEST_FILE) if base not in (".", "") else _TEST_FILE + with sftp.open(remote, "w") as fh: + fh.write(b"ok") + sftp.remove(remote) finally: sftp.close() transport.close() @@ -188,13 +363,16 @@ def _s3_client(cfg: dict): bucket = cfg.get("bucket") if not bucket: raise DestinationError("S3 bucket is required") - return boto3.client( - "s3", - endpoint_url=cfg.get("endpoint_url") or None, - region_name=cfg.get("region") or None, - aws_access_key_id=cfg.get("access_key") or None, - aws_secret_access_key=cfg.get("secret_key") or None, - ) + try: + return boto3.client( + "s3", + endpoint_url=cfg.get("endpoint_url") or None, + region_name=cfg.get("region") or None, + aws_access_key_id=cfg.get("access_key") or None, + aws_secret_access_key=cfg.get("secret_key") or None, + ) + except Exception as exc: # noqa: BLE001 - e.g. a malformed endpoint URL + raise DestinationError(f"Invalid S3 settings: {exc}") from exc def _s3_key(cfg: dict, filename: str) -> str: @@ -203,6 +381,7 @@ def _s3_key(cfg: dict, filename: str) -> str: def _s3_upload(cfg: dict, local_path: str, filename: str) -> str: + _check_file_name(filename) client = _s3_client(cfg) key = _s3_key(cfg, filename) try: @@ -241,6 +420,7 @@ def _s3_list(cfg: dict) -> list[dict]: def _s3_download(cfg: dict, name: str, local_path: str) -> None: + _check_file_name(name) client = _s3_client(cfg) try: client.download_file(cfg["bucket"], _s3_key(cfg, name), local_path) @@ -249,8 +429,24 @@ def _s3_download(cfg: dict, name: str, local_path: str) -> None: def _s3_delete(cfg: dict, name: str) -> None: + _check_file_name(name) client = _s3_client(cfg) - client.delete_object(Bucket=cfg["bucket"], Key=_s3_key(cfg, name)) + try: + client.delete_object(Bucket=cfg["bucket"], Key=_s3_key(cfg, name)) + except Exception as exc: # noqa: BLE001 + raise DestinationError(f"S3 delete failed: {exc}") from exc + + +def _s3_test(cfg: dict) -> None: + """List, then write and delete a small object: proves the keys may write.""" + _s3_list(cfg) + client = _s3_client(cfg) + key = _s3_key(cfg, _TEST_FILE) + try: + client.put_object(Bucket=cfg["bucket"], Key=key, Body=b"ok") + client.delete_object(Bucket=cfg["bucket"], Key=key) + except Exception as exc: # noqa: BLE001 + raise DestinationError(f"S3 write test failed: {exc}") from exc # --------------------------------------------------------------------------- # @@ -354,6 +550,23 @@ def _nfs_volume(dest: BackupDestination, cfg: dict) -> str: return name +def forget_nfs_volume(dest_id: int) -> None: + """Remove the Docker volume of a deleted NFS destination (best effort). + + Left behind, it kept the old mount options under a name the next + destination with the same id would pick up again. + """ + from docker_client import DockerError, get_client, safe_call + + name = f"{_NFS_VOLUME_PREFIX}{dest_id}" + try: + client = get_client() + clear_nfs_leftovers(client, name) + safe_call(safe_call(client.volumes.get, name).remove) + except DockerError as exc: + logger.info("NFS volume %s not removed: %s", name, exc.detail or exc.error) + + def clear_nfs_leftovers(client, volume: Optional[str] = None) -> int: """Remove stopped containers holding an NFS destination volume. @@ -566,10 +779,38 @@ def _nfs_test(dest: BackupDestination, cfg: dict) -> bool: # --------------------------------------------------------------------------- # +def _pinner(dest: BackupDestination): + """Store the first host key an SFTP destination presents (trust on first + use). Written straight to the row — the callers are a scheduled run and a + "Test" click, and neither holds a session that will commit.""" + + def pin(host_key: str) -> None: + if dest.id is None: + return # not saved yet: nothing to pin it to + from sqlmodel import Session + + from database import engine + + with Session(engine) as session: + row = session.get(BackupDestination, dest.id) + if row is None: + return + cfg = parse_config(row) + if (cfg.get("host_key") or "").strip(): + return # pinned meanwhile; the next connect compares against it + cfg["host_key"] = host_key + row.config = dump_config(cfg) + session.add(row) + session.commit() + logger.info("Destination '%s': pinned SFTP host key %s", dest.name, host_key.split()[0]) + + return pin + + def upload(dest: BackupDestination, local_path: str, filename: str) -> str: cfg = parse_config(dest) if dest.type == "sftp": - return _sftp_upload(cfg, local_path, filename) + return _sftp_upload(cfg, local_path, filename, _pinner(dest)) if dest.type == "s3": return _s3_upload(cfg, local_path, filename) if dest.type == "nfs": @@ -580,7 +821,7 @@ def upload(dest: BackupDestination, local_path: str, filename: str) -> str: def list_backups(dest: BackupDestination) -> list[dict]: cfg = parse_config(dest) if dest.type == "sftp": - return _sftp_list(cfg) + return _sftp_list(cfg, _pinner(dest)) if dest.type == "s3": return _s3_list(cfg) if dest.type == "nfs": @@ -591,7 +832,7 @@ def list_backups(dest: BackupDestination) -> list[dict]: def download(dest: BackupDestination, name: str, local_path: str) -> None: cfg = parse_config(dest) if dest.type == "sftp": - _sftp_download(cfg, name, local_path) + _sftp_download(cfg, name, local_path, _pinner(dest)) elif dest.type == "s3": _s3_download(cfg, name, local_path) elif dest.type == "nfs": @@ -603,7 +844,7 @@ def download(dest: BackupDestination, name: str, local_path: str) -> None: def delete(dest: BackupDestination, name: str) -> None: cfg = parse_config(dest) if dest.type == "sftp": - _sftp_delete(cfg, name) + _sftp_delete(cfg, name, _pinner(dest)) elif dest.type == "s3": _s3_delete(cfg, name) elif dest.type == "nfs": @@ -614,7 +855,13 @@ def delete(dest: BackupDestination, name: str) -> None: def test(dest: BackupDestination) -> bool: """Connectivity check — validates reachability, auth and write access.""" + cfg = parse_config(dest) if dest.type == "nfs": - return _nfs_test(dest, parse_config(dest)) - list_backups(dest) - return True + return _nfs_test(dest, cfg) + if dest.type == "sftp": + _sftp_test(cfg, _pinner(dest)) + return True + if dest.type == "s3": + _s3_test(cfg) + return True + raise DestinationError(f"Unknown destination type '{dest.type}'") diff --git a/backend/services/backup_service.py b/backend/services/backup_service.py index 85ae3f0..bded79e 100644 --- a/backend/services/backup_service.py +++ b/backend/services/backup_service.py @@ -104,8 +104,20 @@ def backup_basename(stack_id: str, prefix: Optional[str] = None) -> str: return f"backup-{prefix}-{stack_id}" if prefix else f"backup-{stack_id}" -def backup_filename(stack_id: str, include_volumes: bool, prefix: Optional[str] = None) -> str: - date = compose_service.now().strftime("%Y%m%d-%H%M%S") +def backup_filename( + stack_id: str, + include_volumes: bool, + prefix: Optional[str] = None, + tz: Optional[str] = None, +) -> str: + """``backup---YYYYmmdd-HHMMSS.tar.gz``, the time on the + wall clock of ``tz`` — the schedule's zone, or the browser's for a backup + made by hand. It used to be UTC, so a backup taken at 12:21 in Berlin was + called …-102154 and seemed to be from two hours earlier. The format is + unchanged, so retention still matches older files (it orders by mtime).""" + from services.schedule_service import zone + + date = compose_service.now().astimezone(zone(tz)).strftime("%Y%m%d-%H%M%S") suffix = "full" if include_volumes else "config" return f"{backup_basename(stack_id, prefix)}-{suffix}-{date}.tar.gz" @@ -907,6 +919,10 @@ def restore_backup( stack_id = _target_id(manifest, target_id) old_id = compose_service.slugify(manifest.get("stack_id") or stack_id) renamed = stack_id != old_id + # Volumes are named after the compose *project*, which differs from the id + # for a folder compose had to normalise ("my.app" -> "myapp_data"). + old_project = compose_service.project_name(manifest.get("stack_id") or stack_id) + new_project = compose_service.project_name(stack_id) old_dir = manifest.get("stack_dir") or "" directory = compose_service.stack_dir(stack_id) @@ -969,14 +985,14 @@ def restore_backup( skipped.append({"kind": "volume", "name": full, "reason": "not requested"}) continue retargeted = False - if renamed and full.startswith(old_id + "_"): - full = stack_id + full[len(old_id) :] + if old_project != new_project and full.startswith(old_project + "_"): + full = new_project + full[len(old_project) :] retargeted = True labels = dict(v.get("labels") or {}) if not v.get("external"): - # Re-target compose's labels to the (possibly new) stack id. - labels[COMPOSE_PROJECT_LABEL] = stack_id - reason = _volume_conflict(full, stack_id, old_id, renamed and not retargeted) + # Re-target compose's labels to the (possibly new) project. + labels[COMPOSE_PROJECT_LABEL] = new_project + reason = _volume_conflict(full, new_project, old_id, renamed and not retargeted) if reason: skipped.append({"kind": "volume", "name": full, "reason": reason}) continue @@ -1012,7 +1028,10 @@ def restore_backup( skipped += manifest.get("skipped", []) return { "stack_id": stack_id, - "name": manifest.get("name", stack_id), + # Restored under a new id, the copy must not carry the original's + # display name — two stacks called the same in the list is how the + # wrong one gets stopped. + "name": stack_id if renamed else manifest.get("name", stack_id), "volumes_restored": volumes_restored, "binds_restored": binds_restored, "skipped": skipped, @@ -1021,13 +1040,13 @@ def restore_backup( } -def _volume_conflict(full: str, stack_id: str, old_id: str, shared_with_original: bool) -> Optional[str]: +def _volume_conflict(full: str, project: str, old_id: str, shared_with_original: bool) -> Optional[str]: """Why a volume must not be overwritten, or None if it may be.""" attrs = assets.volume_attrs(full) if attrs is None: return None # created fresh owner = (attrs.get("Labels") or {}).get(COMPOSE_PROJECT_LABEL) - if owner and owner != stack_id: + if owner and owner != project: return f"belongs to stack '{owner}'; not overwritten" if shared_with_original: return ( diff --git a/backend/services/compose_run_service.py b/backend/services/compose_run_service.py new file mode 100644 index 0000000..26868be --- /dev/null +++ b/backend/services/compose_run_service.py @@ -0,0 +1,174 @@ +"""Streamed compose runs (deploy console, update progress) that outlive their socket. + +The deploy console and the stacks list's update bar watch ``compose up`` / +``compose pull && up`` over a WebSocket. That run used to *be* the socket +handler, which went wrong the moment somebody closed the modal or navigated +away halfway through a deploy: + +* the stack lock was released in the handler's ``finally`` at once, while + compose was still running — so the next deploy, update or backup could start + on top of it; +* nobody read compose's output any more, so a chatty pull filled the pipe and + compose blocked on it; +* the audit entry and the "stack started" notification went out with + ``rc=None`` before anything had finished, as a success. + +Here the run is its own task. It holds the lock and drains the output until +compose exits, then records the real exit code. A socket only *watches*: it can +leave at any time, and one that connects while the same operation is already in +flight on that stack is attached to it (backlog first, then live) instead of +being turned away or starting a second compose. +""" +from __future__ import annotations + +import asyncio +import collections +import logging +from collections.abc import AsyncIterator, Awaitable, Callable +from dataclasses import dataclass, field +from datetime import timedelta +from typing import Optional + +from sqlmodel import Session + +from database import engine +from services import stack_lock_service + +logger = logging.getLogger("stackpilot.compose_run") + +#: Lines kept for a socket that attaches to a run already in progress. +BACKLOG_LINES = 2000 + +#: A streamed run has no timeout of its own (a large pull legitimately takes a +#: while), so its lock may live longer than a REST call's. The unlock endpoint +#: is the way out of one that hangs. +LOCK_TTL = timedelta(hours=2) + +#: (kind, payload): ("log", line) or ("done", returncode). +Item = tuple[str, object] + + +@dataclass +class ComposeRun: + stack_id: str + action: str + owner: str + returncode: Optional[int] = None + finished: bool = False + backlog: collections.deque = field( + default_factory=lambda: collections.deque(maxlen=BACKLOG_LINES) + ) + _subscribers: list[asyncio.Queue] = field(default_factory=list) + + def subscribe(self) -> asyncio.Queue: + """A queue that gets the backlog, every later item, then ``None``.""" + queue: asyncio.Queue = asyncio.Queue() + for line in self.backlog: + queue.put_nowait(("log", line)) + if self.finished: + queue.put_nowait(("done", self.returncode)) + queue.put_nowait(None) + else: + self._subscribers.append(queue) + return queue + + def unsubscribe(self, queue: asyncio.Queue) -> None: + if queue in self._subscribers: + self._subscribers.remove(queue) + + def _publish(self, item: Optional[Item]) -> None: + for queue in list(self._subscribers): + queue.put_nowait(item) + + +_runs: dict[str, ComposeRun] = {} +#: asyncio keeps only weak references to tasks; these must not be collected +#: while compose is still running. +_tasks: set[asyncio.Task] = set() + + +class RunBusy(Exception): + """The stack is held by a different operation.""" + + def __init__(self, stack_id: str, action: str): + self.stack_id = stack_id + self.action = action + super().__init__(f"Stack '{stack_id}' is busy: {action} in progress") + + +def get(stack_id: str) -> Optional[ComposeRun]: + return _runs.get(stack_id) + + +def start( + stack_id: str, + action: str, + owner: str, + stream: Callable[[], AsyncIterator[Item]], + on_finish: Optional[Callable[[Optional[int]], Awaitable[None]]] = None, +) -> tuple[ComposeRun, asyncio.Queue, bool]: + """Start ``stream`` under the stack lock, or attach to the same run in flight. + + Returns ``(run, queue, attached)``. Raises :class:`RunBusy` when another + operation holds the stack. Must be called from the event loop; the queue is + subscribed before the task gets to run, so not a line is missed. + """ + current = _runs.get(stack_id) + if current is not None and not current.finished: + if current.action != action: + raise RunBusy(stack_id, current.action) + return current, current.subscribe(), True + + with Session(engine) as session: + try: + stack_lock_service.acquire(session, stack_id, action, owner, ttl=LOCK_TTL) + except stack_lock_service.StackBusy as exc: + raise RunBusy(stack_id, exc.action) from exc + + run = ComposeRun(stack_id=stack_id, action=action, owner=owner) + _runs[stack_id] = run + queue = run.subscribe() + task = asyncio.create_task(_drive(run, stream, on_finish), name=f"compose-{action}-{stack_id}") + _tasks.add(task) + task.add_done_callback(_tasks.discard) + return run, queue, False + + +async def _drive( + run: ComposeRun, + stream: Callable[[], AsyncIterator[Item]], + on_finish: Optional[Callable[[Optional[int]], Awaitable[None]]], +) -> None: + rc: Optional[int] = None + try: + async for kind, payload in stream(): + if kind == "log": + run.backlog.append(payload) + run._publish(("log", payload)) + else: + rc = payload # type: ignore[assignment] + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - reported to whoever is watching + logger.warning("compose %s of '%s' failed: %s", run.action, run.stack_id, exc) + line = f"error: {exc}" + run.backlog.append(line) + run._publish(("log", line)) + rc = rc if rc is not None else -1 + finally: + run.returncode = rc + run.finished = True + try: + with Session(engine) as session: + stack_lock_service.release(session, run.stack_id) + except Exception: # noqa: BLE001 - never mask the outcome + logger.exception("Failed to release the lock on '%s'", run.stack_id) + run._publish(("done", rc)) + run._publish(None) + if _runs.get(run.stack_id) is run: + del _runs[run.stack_id] + if on_finish is not None: + try: + await on_finish(rc) + except Exception: # noqa: BLE001 - audit/notify are best-effort + logger.exception("Post-run hook for '%s' failed", run.stack_id) diff --git a/backend/services/compose_service.py b/backend/services/compose_service.py index 237ac18..0bc16d5 100644 --- a/backend/services/compose_service.py +++ b/backend/services/compose_service.py @@ -9,6 +9,7 @@ import asyncio import os import re import shutil +import stat import tempfile from dataclasses import dataclass, field from datetime import datetime, timezone @@ -33,6 +34,42 @@ class StackFileError(Exception): # --------------------------------------------------------------------------- # +_PROJECT_INVALID_RE = re.compile(r"[^a-z0-9_-]+") + + +def project_name(stack_id: str) -> str: + """The compose project name for a stack. + + The same normalisation compose applies to a directory name when it picks + the project name itself (lowercase, only ``a-z0-9_-``, no leading ``-``/``_``). + For every stack StackPilot creates that is the id unchanged. A folder found + on disk as ``Nextcloud`` or ``my.app`` used to be passed as ``-p Nextcloud``, + which compose refuses — so every action on such a stack failed, while + ``docker compose up`` in that folder worked. + """ + return _PROJECT_INVALID_RE.sub("", stack_id.lower()).lstrip("_-") or "stack" + + +def stack_ids_by_project(stack_ids) -> dict[str, str]: + """``{project: stack_id}``, to key per-project Docker data by stack id.""" + return {project_name(sid): sid for sid in stack_ids} + + +def stack_ids_by_project_in(session) -> dict[str, str]: + """:func:`stack_ids_by_project` for every stack in the database.""" + from sqlmodel import select + + from models.stack import Stack + + return stack_ids_by_project(s.id for s in session.exec(select(Stack)).all()) + + +def owner_stack(project: Optional[str], ids: dict[str, str]) -> Optional[str]: + """The stack id behind a ``com.docker.compose.project`` label, else the + label itself (a project StackPilot does not manage).""" + return ids.get(project, project) if project else project + + def slugify(name: str) -> str: slug = re.sub(r"[^a-z0-9_-]+", "-", name.strip().lower()) slug = re.sub(r"-{2,}", "-", slug).strip("-_") @@ -85,6 +122,57 @@ def read_env(stack_id: str, override: Optional[str] = None) -> str: return fh.read() +#: Name parts that mark a value as a credential (DB_PASSWORD, JWT_SECRET, +#: CF_API_KEY, TS_AUTHKEY …), matched per ``_``/``.``/``-`` separated part so +#: that KEYCLOAK_ADMIN or a "monkey" service stay readable. +_SECRET_PARTS = { + "pass", "passwd", "password", "passphrase", "pwd", "secret", "token", "key", + "apikey", "authkey", "credential", "credentials", "private", +} +#: ``KEY: value`` and ``- KEY=value``, keeping indentation and quotes. +_MAPPING_LINE = re.compile(r"^(\s*)([\"']?)([A-Za-z0-9_.-]+)\2(\s*:\s+)(.+?)\s*$") +_LIST_LINE = re.compile(r"^(\s*-\s*)([\"']?)([A-Za-z0-9_.-]+)=(.*?)\2\s*$") +#: ``${VAR}``, ``$VAR`` or ``${VAR:?message}`` — but not ``${VAR:-default}``, +#: whose default is a value like any other. +_ONLY_A_VARIABLE = re.compile( + r"^[\"']?\$(?:\{[A-Za-z_][A-Za-z0-9_]*(?::?\?[^}]*)?\}|[A-Za-z_][A-Za-z0-9_]*)[\"']?$" +) +MASK = "••••••" + + +def _secret_name(name: str) -> bool: + return any(part in _SECRET_PARTS for part in re.split(r"[_.-]", name.lower())) + + +def mask_secrets(compose: str) -> str: + """The compose file with credential values replaced by ``••••••``. + + For the read-only role, which is not given the ``.env`` — but plenty of + compose files carry the password inline (``POSTGRES_PASSWORD: hunter2``), + and the Compose tab showed it. A reference to a variable (``${DB_PASS}``) + is not a secret and stays. Line-based on purpose: the file keeps its + layout and comments, and a file that does not parse is masked all the same. + """ + out = [] + for line in compose.splitlines(keepends=True): + body = line.rstrip("\r\n") + ending = line[len(body):] + m = _LIST_LINE.match(body) + if m and _secret_name(m.group(3)) and m.group(4) and not _ONLY_A_VARIABLE.match(m.group(4)): + body = f"{m.group(1)}{m.group(2)}{m.group(3)}={MASK}{m.group(2)}" + else: + m = _MAPPING_LINE.match(body) + if ( + m + and _secret_name(m.group(3)) + and not m.group(5).startswith(("|", ">", "{", "[", "&", "*", "#")) + and not _ONLY_A_VARIABLE.match(m.group(5)) + ): + body = f"{m.group(1)}{m.group(2)}{m.group(3)}{m.group(2)}{m.group(4)}{MASK}" + out.append(body + ending) + return "".join(out) + + def write_compose(stack_id: str, content: str, override: Optional[str] = None) -> None: directory = stack_dir(stack_id, override) os.makedirs(directory, exist_ok=True) @@ -106,10 +194,71 @@ def write_env(stack_id: str, content: str, override: Optional[str] = None) -> No fh.write(content) -def delete_stack_files(stack_id: str, override: Optional[str] = None) -> None: +def _mount_points() -> set[str]: + """Every mount point this process sees (from /proc/self/mountinfo).""" + points: set[str] = set() + try: + with open("/proc/self/mountinfo", "r", encoding="utf-8") as fh: + for line in fh: + fields = line.split() + if len(fields) >= 5: + # Spaces etc. are octal-escaped in mountinfo. + points.add(fields[4].encode().decode("unicode_escape")) + except OSError: + pass + return points + + +def delete_stack_files(stack_id: str, override: Optional[str] = None) -> list[str]: + """Remove the stack folder, but never what is mounted inside it. + + A stack folder can have a NAS share or a second disk mounted in it (a media + library at ``./media``). ``rmtree`` would have walked straight into it and + deleted the share's contents along with the compose file. Directories on + another filesystem, or that are mount points of their own (a bind mount of + the same disk), are left exactly as they are. Returns those paths. + """ directory = stack_dir(stack_id, override) - if os.path.isdir(directory): - shutil.rmtree(directory) + if not os.path.isdir(directory): + return [] + mounts = _mount_points() + root_dev = os.lstat(directory).st_dev + kept: list[str] = [] + + def remove(path: str) -> None: + with os.scandir(path) as it: + entries = list(it) + for entry in entries: + if entry.is_dir(follow_symlinks=False): + if ( + entry.stat(follow_symlinks=False).st_dev != root_dev + or entry.path in mounts + ): + kept.append(entry.path) + continue + remove(entry.path) + else: + os.unlink(entry.path) + try: + os.rmdir(path) + except OSError: + pass # still holds something that was kept + + remove(directory) + return kept + + +def _skip_special_files(directory: str, names: list[str]) -> list[str]: + """copytree ignore hook: sockets, FIFOs and devices cannot be copied.""" + skipped = [] + for name in names: + try: + mode = os.lstat(os.path.join(directory, name)).st_mode + except OSError: + continue + if not (stat.S_ISDIR(mode) or stat.S_ISREG(mode) or stat.S_ISLNK(mode)): + skipped.append(name) + return skipped def clone_stack_files(src_id: str, dst_id: str, override: Optional[str] = None) -> None: @@ -117,7 +266,42 @@ def clone_stack_files(src_id: str, dst_id: str, override: Optional[str] = None) dst = stack_dir(dst_id, override) if os.path.isdir(dst): raise StackFileError(f"Target stack '{dst_id}' already exists") - shutil.copytree(src, dst) + try: + # symlinks=True copies links as links: following them would duplicate + # whatever they point at (or fail on a dangling one). A container's + # runtime socket in a data folder would fail the whole copy. + shutil.copytree(src, dst, symlinks=True, ignore=_skip_special_files) + except (OSError, shutil.Error) as exc: + shutil.rmtree(dst, ignore_errors=True) + raise StackFileError(f"Could not copy the stack folder: {exc}") from exc + + +def force_remove_project(stack_id: str) -> int: + """Remove a project's containers and networks through the Docker API. + + The fallback for deleting a stack whose ``compose down`` failed — typically + because the compose file no longer parses. Without it the containers kept + running with nothing left in StackPilot to stop them. Returns how many + containers went. + """ + client = get_client() + label = f"{COMPOSE_LABEL}={project_name(stack_id)}" + removed = 0 + for container in safe_call(client.containers.list, all=True, filters={"label": label}): + try: + container.remove(force=True) + removed += 1 + except Exception: # noqa: BLE001 - best effort, report what went + pass + try: + for network in safe_call(client.networks.list, filters={"label": label}): + try: + network.remove() + except Exception: # noqa: BLE001 + pass + except DockerError: + pass + return removed def discover_stacks(override: Optional[str] = None) -> list[str]: @@ -178,7 +362,7 @@ def containers_for_stack(stack_id: str) -> list[ContainerInfo]: raw = safe_call( client.containers.list, all=True, - filters={"label": f"{COMPOSE_LABEL}={stack_id}"}, + filters={"label": f"{COMPOSE_LABEL}={project_name(stack_id)}"}, ) result = [] for c in raw: @@ -272,7 +456,7 @@ def _compose_base_cmd(stack_id: str, override: Optional[str] = None) -> list[str "docker", "compose", "-p", - stack_id, + project_name(stack_id), "--project-directory", directory, "-f", @@ -298,6 +482,7 @@ async def run_compose( stdout_b, stderr_b = await asyncio.wait_for(proc.communicate(), timeout=timeout) except asyncio.TimeoutError as exc: proc.kill() + await proc.wait() raise StackFileError(f"compose command timed out: {' '.join(args)}") from exc return { "returncode": proc.returncode, @@ -307,42 +492,69 @@ async def run_compose( } -async def validate_yaml(content: str, env_content: str = "") -> dict: - """Validate compose YAML by running ``docker compose config -q`` against it - in a throwaway directory. Returns ``{"ok": bool, "errors": str}`` — the - errors string carries compose's own diagnostics (parse errors, unknown - keys, bad interpolation) so the editor can surface them before saving.""" +async def validate_yaml( + content: str, env_content: str = "", project_dir: Optional[str] = None +) -> dict: + """Validate compose YAML by running ``docker compose config -q`` against it. + Returns ``{"ok": bool, "errors": str}`` — the errors string carries + compose's own diagnostics (parse errors, unknown keys, bad interpolation) + so the editor can surface them before saving. + + With ``project_dir`` (an existing stack's folder) relative paths resolve + against that folder, as they will on deploy. Validated from a bare temp + directory, a stack using ``env_file: ./app.env`` — or an ``include``, a + ``secrets: file:`` — was reported broken although it deploys fine. The + edited ``.env`` is still the one used for interpolation. + """ with tempfile.TemporaryDirectory(prefix="sp-validate-") as tmp: compose_file = os.path.join(tmp, DEFAULT_COMPOSE_NAME) with open(compose_file, "w", encoding="utf-8") as fh: fh.write(content) - if env_content: - with open(os.path.join(tmp, ".env"), "w", encoding="utf-8") as fh: - fh.write(env_content) + env_file = os.path.join(tmp, ".env") + with open(env_file, "w", encoding="utf-8") as fh: + fh.write(env_content or "") + directory = project_dir if project_dir and os.path.isdir(project_dir) else tmp cmd = [ "docker", "compose", - "--project-directory", tmp, + "--project-directory", directory, "-f", compose_file, + "--env-file", env_file, "config", "-q", ] + if directory != tmp: + # The folder's name may not be a valid project name; config does + # not care what the project is called. + cmd[2:2] = ["-p", "sp-validate"] + proc = await asyncio.create_subprocess_exec( + *cmd, + env=registry_service.cli_env(), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) try: - proc = await asyncio.create_subprocess_exec( - *cmd, - env=registry_service.cli_env(), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) _out, err = await asyncio.wait_for(proc.communicate(), timeout=30.0) except asyncio.TimeoutError as exc: + proc.kill() + await proc.wait() raise StackFileError("compose config validation timed out") from exc ok = proc.returncode == 0 - return {"ok": ok, "errors": "" if ok else err.decode("utf-8", "replace").strip()} + errors = "" if ok else err.decode("utf-8", "replace").strip() + # Name the file the user is editing, not our scratch copy of it. + errors = errors.replace(compose_file, DEFAULT_COMPOSE_NAME).replace(env_file, ".env") + errors = errors.replace(tmp + os.sep, "./") + return {"ok": ok, "errors": errors} async def stream_compose( stack_id: str, args: list[str], override: Optional[str] = None ): - """Yield lines from a `docker compose` subcommand as they are produced.""" + """Yield lines from a `docker compose` subcommand as they are produced. + + The subprocess is killed when the consumer stops early (the generator is + closed or its task cancelled). For ``logs -f`` that is the only way it ever + ends: without this every log view that was closed left a ``compose logs`` + process behind for as long as the containers lived. + """ cmd = _compose_base_cmd(stack_id, override) + args proc = await asyncio.create_subprocess_exec( *cmd, @@ -351,9 +563,22 @@ async def stream_compose( stderr=asyncio.subprocess.STDOUT, ) assert proc.stdout is not None - async for raw in proc.stdout: - yield raw.decode("utf-8", "replace").rstrip("\n") - await proc.wait() + try: + async for raw in proc.stdout: + yield raw.decode("utf-8", "replace").rstrip("\n") + await proc.wait() + finally: + if proc.returncode is None: + try: + proc.kill() + except ProcessLookupError: + pass + # Reap it, so it does not linger as a zombie. Shielded: this runs + # while the consuming task is being cancelled. + try: + await asyncio.shield(proc.wait()) + except (asyncio.CancelledError, ProcessLookupError): + pass _json_progress: Optional[bool] = None @@ -468,8 +693,15 @@ async def pull(stack_id: str, override: Optional[str] = None) -> dict: async def update(stack_id: str, override: Optional[str] = None) -> dict: - """Pull then up -d.""" + """Pull then up -d. + + A failed pull stops here, like the streamed update does. Running ``up`` + anyway recreated nothing new — the old images were still there — and its + exit code 0 then reported the update as a success. + """ pull_res = await run_compose(stack_id, ["pull"], override) + if pull_res["returncode"] != 0: + return {**pull_res, "command": "pull + up -d"} up_res = await run_compose(stack_id, ["up", "-d", "--remove-orphans"], override) return { "returncode": up_res["returncode"], @@ -487,7 +719,8 @@ async def logs( ) -> dict: args = ["logs", "--no-color", "--tail", str(tail), "--timestamps"] if service: - args.append(service) + # "--" so a service name can never be read as a compose option. + args += ["--", service] return await run_compose(stack_id, args, override, timeout=60.0) diff --git a/backend/services/convert_service.py b/backend/services/convert_service.py index f4f3ba1..1065bc7 100644 --- a/backend/services/convert_service.py +++ b/backend/services/convert_service.py @@ -1,135 +1,357 @@ -"""Convert a `docker run ...` command string into a Compose YAML fragment.""" +"""Convert a `docker run ...` command string into a Compose YAML fragment. + +The result has to be a compose file that ``docker compose config`` accepts and +that means what the command meant. Three things got that wrong before: + +* every flag the converter did not know was assumed to take a value, so a + boolean like ``--read-only`` or ``-P`` swallowed the image name, and + ``-dit ubuntu bash`` came out as a service running the image ``bash``; +* ``--network host`` became ``networks: [host]`` and ``-v data:/data`` a + reference to a volume nobody declared — both of which compose rejects; +* flags with real effect (``--gpus``, ``--mount``, ``--entrypoint``, + ``--cap-drop``, ``--shm-size`` …) were dropped without a word. + +``docker run``'s boolean flags are a short, closed list; everything else takes a +value. That is what the parser goes by now. +""" from __future__ import annotations +import re import shlex import yaml +#: docker run flags that take no value. +_BOOL_LONG = { + "--detach", "--interactive", "--tty", "--rm", "--init", "--privileged", + "--read-only", "--publish-all", "--oom-kill-disable", "--no-healthcheck", + "--disable-content-trust", "--quiet", "--sig-proxy", "--use-api-socket", +} +#: Single-letter booleans, which may be bundled ("-dit"). +_BOOL_SHORT = set("ditPq") +#: Single-letter flags that take a value ("-p 80:80", "-p80:80"). +_VALUE_SHORT = { + "p": "--publish", "v": "--volume", "e": "--env", "l": "--label", + "u": "--user", "w": "--workdir", "h": "--hostname", "m": "--memory", + "a": "--attach", "c": "--cpu-shares", +} +_SHORT_TO_LONG = {"d": "--detach", "i": "--interactive", "t": "--tty", + "P": "--publish-all", "q": "--quiet"} + +#: network_mode values, as opposed to networks a service joins. +_NETWORK_MODES = {"host", "none", "bridge"} + +_SERVICE_NAME_RE = re.compile(r"[^a-zA-Z0-9._-]+") + def _service_name_from_image(image: str) -> str: - name = image.split("/")[-1].split(":")[0] - return name or "app" + name = image.split("@", 1)[0].split("/")[-1].split(":")[0] + return _SERVICE_NAME_RE.sub("-", name).strip("-.") or "app" + + +#: A line continuation whose newline is already gone. Pasted into a one-line +#: field, backslash-newline arrives as a backslash followed by the next line's +#: indentation, which shlex read as an escaped space: a lone " " token that +#: became the image. A backslash followed by an option, or by more than one +#: blank, is such a continuation; a single escaped space inside a word +#: (``My\ Files``) is left alone. +_FLATTENED_CONTINUATION = re.compile(r"\\[ \t]+(?=-)|\\[ \t]{2,}") + + +def _tokens(command: str) -> list[str]: + # Shell line continuations, as pasted from a README. + command = command.replace("\\\r\n", " ").replace("\\\n", " ") + tokens = shlex.split(_FLATTENED_CONTINUATION.sub(" ", command)) + # Drop leading "sudo docker run" / "docker container run". + while tokens and tokens[0] in ("docker", "run", "container", "sudo", "podman"): + tokens.pop(0) + return tokens + + +def _parse(tokens: list[str]) -> tuple[list[tuple[str, str | None]], str | None, list[str]]: + """Split into ([(long_flag, value)], image, command).""" + flags: list[tuple[str, str | None]] = [] + i, n = 0, len(tokens) + while i < n: + tok = tokens[i] + if tok == "--": + i += 1 + break + if not tok.startswith("-") or tok == "-": + break # the image + if tok.startswith("--"): + name, eq, value = tok.partition("=") + if eq: + flags.append((name, value)) + i += 1 + elif name in _BOOL_LONG: + flags.append((name, None)) + i += 1 + else: + flags.append((name, tokens[i + 1] if i + 1 < n else "")) + i += 2 + continue + # Short form: bundled booleans ("-dit"), or one value flag with its + # value attached ("-p8080:80") or following ("-p 8080:80"). + body = tok[1:] + j = 0 + while j < len(body) and body[j] in _BOOL_SHORT: + flags.append((_SHORT_TO_LONG[body[j]], None)) + j += 1 + if j == len(body): + i += 1 + continue + letter = body[j] + long_name = _VALUE_SHORT.get(letter, f"-{letter}") + attached = body[j + 1 :] + if attached.startswith("="): + attached = attached[1:] + if attached: + flags.append((long_name, attached)) + i += 1 + else: + flags.append((long_name, tokens[i + 1] if i + 1 < n else "")) + i += 2 + image = tokens[i] if i < n else None + command = tokens[i + 1 :] if image is not None else [] + return flags, image, command + + +def _is_named_volume(source: str) -> bool: + return bool(source) and not source.startswith(("/", ".", "~", "$")) + + +def _mount(value: str) -> dict: + """``--mount type=bind,source=/a,target=/b,readonly`` as compose long syntax.""" + spec: dict = {} + for part in value.split(","): + key, eq, val = part.partition("=") + key = key.strip() + if key in ("source", "src"): + spec["source"] = val + elif key in ("target", "destination", "dst"): + spec["target"] = val + elif key == "type": + spec["type"] = val + elif key in ("readonly", "ro"): + spec["read_only"] = (not eq) or val.lower() in ("1", "true") + elif key == "bind-propagation": + spec.setdefault("bind", {})["propagation"] = val + elif key == "volume-nocopy": + spec.setdefault("volume", {})["nocopy"] = (not eq) or val.lower() in ("1", "true") + elif key == "tmpfs-size": + spec.setdefault("tmpfs", {})["size"] = val + spec.setdefault("type", "volume") + return spec + + +def _gpus(value: str) -> dict: + """``--gpus all`` / ``--gpus 2`` / ``--gpus device=0,1`` as a device reservation.""" + reservation: dict = {"driver": "nvidia", "capabilities": ["gpu"]} + value = value.strip().strip('"').strip("'") + if value.startswith("device="): + reservation["device_ids"] = [d for d in value[len("device="):].split(",") if d] + elif value.isdigit(): + reservation["count"] = int(value) + else: + reservation["count"] = "all" + return reservation + + +def _ulimit(value: str, ulimits: dict) -> None: + name, _, limits = value.partition("=") + soft, _, hard = limits.partition(":") + try: + ulimits[name] = ( + {"soft": int(soft), "hard": int(hard)} if hard else int(soft) + ) + except ValueError: + pass def convert_docker_run(command: str) -> str: - tokens = shlex.split(command) - # Drop leading "docker run" / "docker container run". - while tokens and tokens[0] in ("docker", "run", "container", "sudo"): - tokens.pop(0) - - service: dict = {} - name = None - image = None - ports: list[str] = [] - volumes: list[str] = [] - environment: list[str] = [] - env_file: list[str] = [] - devices: list[str] = [] - cap_add: list[str] = [] - labels: list[str] = [] - networks: list[str] = [] - command_args: list[str] = [] - - i = 0 - n = len(tokens) - - def take_value(idx: int, tok: str): - if "=" in tok and tok.startswith("--") and not tok.endswith("="): - return tok.split("=", 1)[1], idx + 1 - return tokens[idx + 1], idx + 2 - - while i < n: - tok = tokens[i] - if image is not None: - # Everything after the image is the container command. - command_args = tokens[i:] - break - if not tok.startswith("-"): - image = tok - i += 1 - continue - - flag = tok.split("=", 1)[0] - if flag in ("-d", "--detach", "-i", "--interactive", "-t", "--tty", "--rm", "--init"): - i += 1 - elif flag in ("--name",): - name, i = take_value(i, tok) - elif flag in ("-p", "--publish"): - val, i = take_value(i, tok) - ports.append(val) - elif flag in ("-v", "--volume", "--mount"): - val, i = take_value(i, tok) - volumes.append(val) - elif flag in ("-e", "--env"): - val, i = take_value(i, tok) - environment.append(val) - elif flag in ("--env-file",): - val, i = take_value(i, tok) - env_file.append(val) - elif flag in ("--device",): - val, i = take_value(i, tok) - devices.append(val) - elif flag in ("--cap-add",): - val, i = take_value(i, tok) - cap_add.append(val) - elif flag in ("-l", "--label"): - val, i = take_value(i, tok) - labels.append(val) - elif flag in ("--network", "--net"): - val, i = take_value(i, tok) - networks.append(val) - elif flag in ("--restart",): - val, i = take_value(i, tok) - service["restart"] = val - elif flag in ("--privileged",): - service["privileged"] = True - i += 1 - elif flag in ("--hostname", "-h"): - val, i = take_value(i, tok) - service["hostname"] = val - elif flag in ("-u", "--user"): - val, i = take_value(i, tok) - service["user"] = val - elif flag in ("-w", "--workdir"): - val, i = take_value(i, tok) - service["working_dir"] = val - else: - # Unknown flag — try to consume a value if it looks like it takes one. - if "=" in tok: - i += 1 - elif i + 1 < n and not tokens[i + 1].startswith("-"): - i += 2 - else: - i += 1 - + tokens = _tokens(command) + flags, image, command_args = _parse(tokens) if not image: raise ValueError("Could not find an image in the docker run command") - service["image"] = image - if "restart" not in service: - service["restart"] = "unless-stopped" - if ports: - service["ports"] = ports - if volumes: - service["volumes"] = volumes - if environment: - service["environment"] = environment - if env_file: - service["env_file"] = env_file - if devices: - service["devices"] = devices - if cap_add: - service["cap_add"] = cap_add - if labels: - service["labels"] = labels - if networks: - service["networks"] = networks + service: dict = {"image": image} + top_volumes: dict = {} + top_networks: dict = {} + name = None + lists: dict[str, list] = {} + healthcheck: dict = {} + logging: dict = {} + ulimits: dict = {} + gpus: list[dict] = [] + + def add(key: str, value) -> None: + lists.setdefault(key, []).append(value) + + for flag, value in flags: + value = value if value is not None else "" + if flag == "--name": + name = value + elif flag == "--publish": + add("ports", value) + elif flag == "--volume": + add("volumes", value) + source = value.split(":", 1)[0] if ":" in value else "" + if _is_named_volume(source): + top_volumes[source] = None + elif flag == "--mount": + spec = _mount(value) + add("volumes", spec) + if spec.get("type") == "volume" and _is_named_volume(spec.get("source", "")): + top_volumes[spec["source"]] = None + elif flag == "--tmpfs": + add("tmpfs", value) + elif flag == "--env": + add("environment", value) + elif flag == "--env-file": + add("env_file", value) + elif flag == "--device": + add("devices", value) + elif flag == "--cap-add": + add("cap_add", value) + elif flag == "--cap-drop": + add("cap_drop", value) + elif flag == "--label": + add("labels", value) + elif flag in ("--network", "--net"): + if value in _NETWORK_MODES or value.startswith(("container:", "service:")): + service["network_mode"] = value + elif value: + add("networks", value) + # `docker run --network x` needs x to exist already. + top_networks[value] = {"external": True} + elif flag == "--network-alias": + add("_aliases", value) + elif flag == "--add-host": + add("extra_hosts", value) + elif flag == "--dns": + add("dns", value) + elif flag == "--dns-search": + add("dns_search", value) + elif flag == "--security-opt": + add("security_opt", value) + elif flag == "--sysctl": + add("sysctls", value) + elif flag == "--group-add": + add("group_add", value) + elif flag == "--expose": + add("expose", value) + elif flag == "--gpus": + gpus.append(_gpus(value)) + elif flag == "--ulimit": + _ulimit(value, ulimits) + elif flag == "--restart": + service["restart"] = value + elif flag == "--privileged": + service["privileged"] = True + elif flag == "--read-only": + service["read_only"] = True + elif flag == "--init": + service["init"] = True + elif flag == "--interactive": + service["stdin_open"] = True + elif flag == "--tty": + service["tty"] = True + elif flag == "--hostname": + service["hostname"] = value + elif flag == "--user": + service["user"] = value + elif flag == "--workdir": + service["working_dir"] = value + elif flag == "--entrypoint": + service["entrypoint"] = value + elif flag == "--shm-size": + service["shm_size"] = value + elif flag == "--memory": + service["mem_limit"] = value + elif flag == "--memory-reservation": + service["mem_reservation"] = value + elif flag == "--cpus": + try: + service["cpus"] = float(value) + except ValueError: + pass + elif flag == "--cpu-shares": + try: + service["cpu_shares"] = int(value) + except ValueError: + pass + elif flag == "--pids-limit": + try: + service["pids_limit"] = int(value) + except ValueError: + pass + elif flag == "--pid": + service["pid"] = value + elif flag == "--ipc": + service["ipc"] = value + elif flag == "--stop-signal": + service["stop_signal"] = value + elif flag in ("--stop-timeout", "--time"): + service["stop_grace_period"] = f"{value}s" if value.isdigit() else value + elif flag == "--platform": + service["platform"] = value + elif flag == "--pull": + service["pull_policy"] = value + elif flag == "--log-driver": + logging["driver"] = value + elif flag == "--log-opt": + key, _, val = value.partition("=") + logging.setdefault("options", {})[key] = val + elif flag == "--health-cmd": + healthcheck["test"] = ["CMD-SHELL", value] + elif flag == "--health-interval": + healthcheck["interval"] = value + elif flag == "--health-timeout": + healthcheck["timeout"] = value + elif flag == "--health-start-period": + healthcheck["start_period"] = value + elif flag == "--health-retries": + try: + healthcheck["retries"] = int(value) + except ValueError: + pass + elif flag == "--no-healthcheck": + healthcheck = {"disable": True} + # Everything else (--detach, --rm, --publish-all, --cidfile, …) has no + # compose equivalent worth writing down. + + aliases = lists.pop("_aliases", []) + if aliases: + nets = lists.get("networks") or [] + if nets: + # Aliases apply to the networks given on the command line. + lists["networks"] = {net: {"aliases": aliases} for net in nets} + service.setdefault("restart", "unless-stopped") + for key in ( + "ports", "expose", "volumes", "tmpfs", "environment", "env_file", "devices", + "cap_add", "cap_drop", "security_opt", "sysctls", "group_add", "extra_hosts", + "dns", "dns_search", "labels", "networks", + ): + if key in lists: + service[key] = lists[key] + if ulimits: + service["ulimits"] = ulimits + if logging: + service["logging"] = logging + if healthcheck: + service["healthcheck"] = healthcheck + if gpus: + service["deploy"] = {"resources": {"reservations": {"devices": gpus}}} if command_args: service["command"] = command_args - - svc_name = name or _service_name_from_image(image) if name: service["container_name"] = name - doc = {"services": {svc_name: service}} + svc_name = _SERVICE_NAME_RE.sub("-", name).strip("-.") if name else "" + doc: dict = {"services": {svc_name or _service_name_from_image(image): service}} + if top_volumes: + doc["volumes"] = top_volumes + if top_networks: + doc["networks"] = top_networks return yaml.safe_dump(doc, sort_keys=False, default_flow_style=False) diff --git a/backend/services/dashboard_service.py b/backend/services/dashboard_service.py index 404f505..debeaba 100644 --- a/backend/services/dashboard_service.py +++ b/backend/services/dashboard_service.py @@ -132,7 +132,7 @@ def _local_host() -> tuple[dict, list[dict]]: unhealthy: list[str] = [] updates = 0 for stack_id in discovered: - containers = by_project.get(stack_id, []) + containers = by_project.get(compose_service.project_name(stack_id), []) status = compose_service._status_from_states([c.get("State", "") for c in containers]) statuses.append(status) if status == "running" and not _is_healthy(containers): diff --git a/backend/services/device_service.py b/backend/services/device_service.py index ce1bb13..dc9b700 100644 --- a/backend/services/device_service.py +++ b/backend/services/device_service.py @@ -118,12 +118,61 @@ def _is_allowed(path: str) -> bool: return False +def _toward_roots(path: str) -> list[dict]: + """For a path *above* the allowed roots: the folders below it that lead to + one (``/`` -> ``mnt``, ``opt``; ``/srv`` -> ``data`` for ``/srv/data``). + + The browser opens at ``/``, which the default roots do not include, and the + breadcrumbs and "up" lead there too — so the page used to open on an error + with no way into any root at all. + """ + prefix = "/" if path == "/" else path + os.sep + names: set[str] = set() + for root in settings.ALLOWED_BROWSE_ROOTS: + root = os.path.normpath(root) + if not root.startswith(prefix): + continue + try: + if not os.path.isdir(_real_root(root)): + continue + except BrowseError: + continue + names.add(root[len(prefix):].split(os.sep, 1)[0]) + entries = [] + for name in sorted(names, key=str.lower): + try: + st = os.stat(_real_root(os.path.join(path, name))) + except (OSError, BrowseError): + continue + entries.append( + { + "name": name, + "type": "dir", + "size": None, + "permissions": oct(st.st_mode & 0o777), + "mtime": st.st_mtime, + "symlink": False, + } + ) + return entries + + def browse(path: str = "/", show_hidden: bool = False) -> dict: path = os.path.normpath(path or "/") if not path.startswith("/"): raise BrowseError("Path must be absolute") if not _is_allowed(path): - raise BrowseError("Path is outside the allowed browse roots") + entries = _toward_roots(path) + if not entries: + raise BrowseError("Path is outside the allowed browse roots") + return { + "path": path, + "parent": os.path.dirname(path) if path != "/" else None, + "roots": settings.ALLOWED_BROWSE_ROOTS, + "entries": entries, + # Only a way through: nothing can be created or changed here. + "virtual": True, + } real = _real_root(path) if not os.path.isdir(real): diff --git a/backend/services/exec_service.py b/backend/services/exec_service.py index e68efd4..c609639 100644 --- a/backend/services/exec_service.py +++ b/backend/services/exec_service.py @@ -8,9 +8,12 @@ stdout/stderr stream), suitable for piping straight to xterm.js. from __future__ import annotations import asyncio +import codecs import contextlib import json +import queue import socket as _socket +import threading from starlette.websockets import WebSocketDisconnect @@ -71,22 +74,64 @@ async def pump_exec(websocket, exec_id: str, holder, raw) -> None: accepted as keystrokes). Container -> browser: ``{"type":"data","data":...}`` then a final ``{"type":"exit","code":...}``. - The blocking ``recv`` runs in the default executor; on teardown we shut the - socket down so that orphaned recv thread unblocks and exits. + The blocking ``recv`` and ``sendall`` run on two threads of this session's + own. They used to run on the event loop's default executor, where an open + terminal held a worker for its whole lifetime — shared with every + ``asyncio.to_thread`` in the app. On teardown the socket is shut down, which + is what unblocks the reader. """ raw.setblocking(True) - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() closed = asyncio.Event() + incoming: asyncio.Queue = asyncio.Queue() + outgoing: queue.Queue = queue.Queue() - async def to_browser() -> None: + def post(item) -> None: + try: + loop.call_soon_threadsafe(incoming.put_nowait, item) + except RuntimeError: # loop closed + pass + + def reader() -> None: try: while True: - data = await loop.run_in_executor(None, raw.recv, 4096) + data = raw.recv(4096) if not data: break - await websocket.send_text( - json.dumps({"type": "data", "data": data.decode("utf-8", "replace")}) - ) + post(data) + except Exception: # noqa: BLE001 - shutdown on teardown lands here + pass + finally: + post(None) + + def writer() -> None: + while True: + data = outgoing.get() + if data is None: + return + try: + raw.sendall(data) + except Exception: # noqa: BLE001 + return + + name = exec_id[:12] + threading.Thread(target=reader, name=f"exec-read-{name}", daemon=True).start() + threading.Thread(target=writer, name=f"exec-write-{name}", daemon=True).start() + + async def to_browser() -> None: + # Incremental: a multi-byte character (an umlaut, a box-drawing glyph in + # htop) split across two reads would otherwise turn into two U+FFFD. + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + try: + while True: + data = await incoming.get() + if data is None: + break + text = decoder.decode(data) + if text: + await websocket.send_text(json.dumps({"type": "data", "data": text})) + if tail := decoder.decode(b"", final=True): + await websocket.send_text(json.dumps({"type": "data", "data": tail})) except Exception: # noqa: BLE001 pass finally: @@ -105,9 +150,9 @@ async def pump_exec(websocket, exec_id: str, holder, raw) -> None: with contextlib.suppress(Exception): resize_exec(exec_id, int(obj.get("rows", 24)), int(obj.get("cols", 80))) elif isinstance(obj, dict) and "data" in obj: - await loop.run_in_executor(None, raw.sendall, str(obj["data"]).encode()) + outgoing.put(str(obj["data"]).encode()) else: - await loop.run_in_executor(None, raw.sendall, msg.encode()) + outgoing.put(msg.encode()) except WebSocketDisconnect: pass except Exception: # noqa: BLE001 @@ -119,6 +164,7 @@ async def pump_exec(websocket, exec_id: str, holder, raw) -> None: in_task = asyncio.create_task(from_browser()) await closed.wait() + outgoing.put(None) with contextlib.suppress(Exception): raw.shutdown(_socket.SHUT_RDWR) with contextlib.suppress(Exception): diff --git a/backend/services/file_service.py b/backend/services/file_service.py index 27becfb..b15cb53 100644 --- a/backend/services/file_service.py +++ b/backend/services/file_service.py @@ -124,6 +124,8 @@ def create_file(path: str, name: str) -> dict: def rename(path: str, new_name: str) -> dict: real = _safe_real(path) + if _is_root(path): + raise BrowseError("Refusing to rename a browse root") if not os.path.lexists(real): raise BrowseError(f"No such path: {path}") parent = os.path.dirname(path) or "/" @@ -138,10 +140,14 @@ def rename(path: str, new_name: str) -> dict: return {"path": dest} +def _is_root(path: str) -> bool: + norm = os.path.normpath(path) + return norm == "/" or norm in {os.path.normpath(r) for r in _root_paths()} + + def delete(path: str, recursive: bool = False) -> dict: real = _safe_real(path) - norm = os.path.normpath(path) - if norm == "/" or norm in {os.path.normpath(r) for r in _root_paths()}: + if _is_root(path): raise BrowseError("Refusing to delete a browse root") if not os.path.lexists(real): raise BrowseError(f"No such path: {path}") @@ -324,6 +330,10 @@ def _transfer_dest(src: str, dest_dir: str, overwrite: bool) -> tuple[str, str, name = os.path.basename(src.rstrip("/")) dest_logical = _child(dest_dir, name) dest_real = _safe_real(dest_logical) + # Overwriting removes the target first — which, for nested roots + # (/mnt and /mnt/user), could be a root. Same rule as delete. + if overwrite and _is_root(dest_logical): + raise BrowseError("Refusing to replace a browse root") # Refuse to copy/move a directory into itself or its own subtree. src_norm = os.path.normpath(src_real) @@ -350,6 +360,10 @@ def copy(src: str, dest_dir: str, overwrite: bool = False) -> dict: def move(src: str, dest_dir: str, overwrite: bool = False) -> dict: + # Moving a root away takes the whole tree with it — a delete by another + # name, and delete refuses roots. + if _is_root(src): + raise BrowseError("Refusing to move a browse root") src_real, dest_real, dest_logical = _transfer_dest(src, dest_dir, overwrite) try: if os.path.exists(dest_real) and overwrite: diff --git a/backend/services/git_service.py b/backend/services/git_service.py index 1c42832..1063928 100644 --- a/backend/services/git_service.py +++ b/backend/services/git_service.py @@ -53,6 +53,29 @@ class GitError(Exception): """A repository that cannot be reached, or a sync that cannot be completed.""" +#: What a branch name may look like here. git itself allows more, but not a +#: leading "-": as the refspec of `git fetch origin ` that would be read +#: as an option (`--upload-pack=…` runs a command). +_BRANCH_RE = re.compile(r"^[A-Za-z0-9._/][A-Za-z0-9._/+-]{0,199}$") + + +def validate_branch(branch: str) -> str: + branch = (branch or "").strip() + if not _BRANCH_RE.match(branch) or ".." in branch or branch.endswith((".lock", "/")): + raise GitError(f"Invalid branch name '{branch}'") + return branch + + +def validate_url(url: str) -> str: + url = (url or "").strip() + if not url or url.startswith("-") or any(c.isspace() for c in url): + raise GitError(f"Invalid repository URL '{url}'") + # git's ext:: and fd:: transports run commands or read file descriptors. + if url.lower().startswith(("ext::", "fd::")): + raise GitError("ext:: and fd:: repository URLs are not supported") + return url + + # --------------------------------------------------------------------------- # # Paths # --------------------------------------------------------------------------- # @@ -179,7 +202,8 @@ def _cleanup_auth(source: GitSource) -> None: async def _fetch(source: GitSource, env: dict, secret: Optional[str]) -> str: """Bring the cached clone to the tip of the configured branch. Returns the commit.""" directory = repo_dir(source.stack_id) - branch = source.branch or "main" + branch = validate_branch(source.branch or "main") + validate_url(source.url) os.makedirs(cache_root(), mode=0o700, exist_ok=True) if os.path.isdir(os.path.join(directory, ".git")): @@ -193,8 +217,10 @@ async def _fetch(source: GitSource, env: dict, secret: Optional[str]) -> str: shutil.rmtree(directory, ignore_errors=True) if not os.path.isdir(os.path.join(directory, ".git")): + # "--" ends the options: a URL that starts with "-" is a URL. await _git( - ["clone", "--depth", "1", "--branch", branch, source.url, directory], env, secret + ["clone", "--depth", "1", "--branch", branch, "--", source.url, directory], + env, secret, ) else: await _git(["-C", directory, "fetch", "--depth", "1", "origin", branch], env, secret) @@ -294,6 +320,12 @@ def _prune_empty_dirs(stack_dir: str, removed: list[str]) -> None: async def sync(session: Session, source: GitSource, actor: str = "system") -> SyncResult: """Fetch, copy into the stack, and deploy when something changed.""" + from models.stack import Stack + + if session.get(Stack, source.stack_id) is None: + # A source left behind by a deleted stack: syncing it would recreate + # the folder and, with auto deploy, start the stack again. + raise GitError(f"Stack '{source.stack_id}' no longer exists") env, secret = _auth_env(source) try: commit = await _fetch(source, env, secret) diff --git a/backend/services/gpu_service.py b/backend/services/gpu_service.py index 060fe3d..06395fe 100644 --- a/backend/services/gpu_service.py +++ b/backend/services/gpu_service.py @@ -8,6 +8,7 @@ from __future__ import annotations import glob import os +import re import subprocess from dataclasses import asdict, dataclass from typing import Literal, Optional @@ -83,6 +84,49 @@ def _detect_nvidia() -> list[GPUInfo]: return gpus +def _detect_nvidia_procfs() -> list[GPUInfo]: + """NVIDIA GPUs from the kernel driver's /proc entries. + + StackPilot's image has no ``nvidia-smi`` (and the container runs without + the NVIDIA runtime), and the DRI scan below skips NVIDIA on purpose — so on + a normal deployment an NVIDIA card was never detected at all. The driver + publishes every GPU under ``/proc/driver/nvidia/gpus//information``, + which the host /proc mount (``HOST_PROC_PATH``) makes readable here. + """ + from config import settings + + for proc in (settings.HOST_PROC_PATH, "/proc"): + base = os.path.join(proc, "driver", "nvidia", "gpus") + if not os.path.isdir(base): + continue + version = _read(os.path.join(proc, "driver", "nvidia", "version")) + match = re.search(r"Kernel Module\s+([\d.]+)", version) + driver = match.group(1) if match else "" + gpus: list[GPUInfo] = [] + for position, bus in enumerate(sorted(os.listdir(base))): + info: dict[str, str] = {} + for line in _read(os.path.join(base, bus, "information")).splitlines(): + key, sep, value = line.partition(":") + if sep: + info[key.strip()] = value.strip() + try: + index = int(info.get("Device Minor", position)) + except ValueError: + index = position + gpus.append( + GPUInfo( + vendor="nvidia", + index=index, + name=info.get("Model") or "NVIDIA GPU", + uuid=info.get("GPU UUID") or None, + driver=driver, + ) + ) + if gpus: + return sorted(gpus, key=lambda g: g.index) + return [] + + def _read(path: str) -> str: try: with open(path, "r", encoding="utf-8") as fh: @@ -151,7 +195,7 @@ def _detect_dri() -> list[GPUInfo]: def detect_gpus() -> list[GPUInfo]: - return _detect_nvidia() + _detect_dri() + return (_detect_nvidia() or _detect_nvidia_procfs()) + _detect_dri() def dri_group_gids() -> set[int]: diff --git a/backend/services/image_service.py b/backend/services/image_service.py index 234303a..9396f9e 100644 --- a/backend/services/image_service.py +++ b/backend/services/image_service.py @@ -14,23 +14,33 @@ COMPOSE_PROJECT_LABEL = "com.docker.compose.project" def list_images() -> list[dict]: - """Return one row per image tag, annotated with using-stacks + update status.""" + """Return one row per image tag, annotated with using-stacks + update status. + + Usage is matched on the image *id* the containers run, not on the name: + a compose file saying ``image: nginx`` or ``docker.io/library/nginx:1`` + produces containers whose configured image never equals the tag Docker + lists (``nginx:latest``, ``nginx:1``), so those images showed up as used + by no stack — and were left out of "scan all". + """ try: client = get_client() images = safe_call(client.images.list) - containers = safe_call(client.containers.list, all=True) + containers = safe_call(client.api.containers, all=True) except DockerError: return [] - # Map image ref -> stacks using it. - usage: dict[str, set[str]] = {} + # image id -> stacks using it, and the references they were created from + # (which is what the update checker caches its results under). + stacks_by_id: dict[str, set[str]] = {} + refs_by_id: dict[str, set[str]] = {} for c in containers: - ref = c.attrs.get("Config", {}).get("Image") - stack = c.labels.get(COMPOSE_PROJECT_LABEL) - if ref: - usage.setdefault(ref, set()) - if stack: - usage[ref].add(stack) + image_id = c.get("ImageID") or "" + stack = (c.get("Labels") or {}).get(COMPOSE_PROJECT_LABEL) + stacks_by_id.setdefault(image_id, set()) + if stack: + stacks_by_id[image_id].add(stack) + if (ref := c.get("Image")) and not ref.startswith("sha256:"): + refs_by_id.setdefault(image_id, set()).add(ref) cache = update_service.get_cache() result = [] @@ -38,15 +48,20 @@ def list_images() -> list[dict]: tags = img.tags or [] if not tags: continue + refs = refs_by_id.get(img.id, set()) for tag in tags: + update = cache.get(tag) + if update is None: + # Cached under the name the compose file used for it. + update = next((cache[r] for r in sorted(refs) if r in cache), None) result.append( { "id": img.short_id, "tag": tag, "size": img.attrs.get("Size", 0), "created": img.attrs.get("Created"), - "stacks": sorted(usage.get(tag, set())), - "update": cache.get(tag), + "stacks": sorted(stacks_by_id.get(img.id, set())), + "update": update, } ) result.sort(key=lambda r: r["tag"]) diff --git a/backend/services/image_status_store.py b/backend/services/image_status_store.py index 3deebbf..2822894 100644 --- a/backend/services/image_status_store.py +++ b/backend/services/image_status_store.py @@ -31,6 +31,7 @@ def _to_status(row: ImageStatus) -> update_service.UpdateStatus: remote_digest=row.remote_digest, checked_at=row.checked_at, error=row.error, + local=row.local, ) @@ -46,6 +47,7 @@ def save(status: update_service.UpdateStatus, notified: bool) -> None: row.remote_digest = status.remote_digest row.checked_at = status.checked_at row.error = status.error + row.local = status.local row.notified = notified session.add(row) session.commit() diff --git a/backend/services/notify_service.py b/backend/services/notify_service.py index f70327c..34916c4 100644 --- a/backend/services/notify_service.py +++ b/backend/services/notify_service.py @@ -11,6 +11,7 @@ All delivery is best-effort: failures are logged, never raised to the caller. """ from __future__ import annotations +import base64 import logging from typing import Optional @@ -37,14 +38,14 @@ def _build_request(wtype: str, url: str, event: str, title: str, message: str): return { "url": url, "content": message.encode("utf-8"), - "headers": {"Title": title, "Tags": _ntfy_tag(event)}, + "headers": {"Title": _header_value(title), "Tags": _ntfy_tag(event)}, } if wtype == "discord": return {"url": url, "json": {"content": f"**{title}**\n{message}"}} if wtype == "slack": return {"url": url, "json": {"text": f"*{title}*\n{message}"}} if wtype == "gotify": - priority = 8 if event in ("stack_error", "pull_failed") else 5 + priority = 8 if event in _URGENT_EVENTS else 5 return { "url": url, "json": {"title": title, "message": message, "priority": priority}, @@ -56,6 +57,24 @@ def _build_request(wtype: str, url: str, event: str, title: str, message: str): } +_URGENT_EVENTS = {"stack_error", "pull_failed", "backup_failed"} + + +def _header_value(text: str) -> str: + """A header value httpx can send, keeping non-ASCII readable. + + HTTP headers are ASCII; httpx raises UnicodeEncodeError on anything else, + so a title with an umlaut or a dash ("Backup to NAS Büro failed") never + reached ntfy at all. RFC 2047 encoded-words are what ntfy decodes. + """ + try: + text.encode("ascii") + return text + except UnicodeEncodeError: + encoded = base64.b64encode(text.encode("utf-8")).decode("ascii") + return f"=?UTF-8?B?{encoded}?=" + + def _ntfy_tag(event: str) -> str: return { "update_available": "arrow_up", @@ -63,17 +82,21 @@ def _ntfy_tag(event: str) -> str: "stack_stop": "stop_button", "stack_error": "rotating_light", "pull_failed": "warning", + "backup_failed": "floppy_disk", + "stack_auto_updated": "arrows_counterclockwise", }.get(event, "bell") async def _deliver(client: httpx.AsyncClient, wtype: str, url: str, event: str, title: str, message: str) -> bool: - kwargs = _build_request(wtype, url, event, title, message) - target = kwargs.pop("url") try: + kwargs = _build_request(wtype, url, event, title, message) + target = kwargs.pop("url") resp = await client.post(target, timeout=_TIMEOUT, **kwargs) resp.raise_for_status() return True - except httpx.HTTPError as exc: + except Exception as exc: # noqa: BLE001 - one bad webhook must not stop the rest + # Not just httpx.HTTPError: an invalid URL or an unencodable header + # raised past this and aborted delivery to every webhook after it. logger.warning("Webhook delivery failed (%s): %s", wtype, exc) return False diff --git a/backend/services/port_service.py b/backend/services/port_service.py index fb205c3..99d157b 100644 --- a/backend/services/port_service.py +++ b/backend/services/port_service.py @@ -8,6 +8,7 @@ import yaml from config import settings from docker_client import DockerError, get_client, safe_call +from services.compose_service import project_name # --------------------------------------------------------------------------- # @@ -21,30 +22,47 @@ def parse_compose_ports(yaml_str: str) -> list[dict]: data = yaml.safe_load(yaml_str) or {} except yaml.YAMLError: return [] + if not isinstance(data, dict): + return [] services = data.get("services") or {} + if not isinstance(services, dict): + return [] out: list[dict] = [] for svc_name, svc in services.items(): if not isinstance(svc, dict): continue for entry in svc.get("ports", []) or []: - parsed = _parse_port_entry(entry) - if parsed: + for parsed in _parse_port_entry(entry): parsed["service"] = svc_name out.append(parsed) return out -def _parse_port_entry(entry) -> Optional[dict]: +#: A published range is checked port by port, up to this many. +_MAX_RANGE = 1024 + + +def _host_ports(spec: str) -> list[int]: + """"8080" -> [8080]; "8080-8082" -> [8080, 8081, 8082]; junk -> [].""" + try: + if "-" in spec: + low, high = (int(p) for p in spec.split("-", 1)) + if low > high or high - low >= _MAX_RANGE: + return [low] + return list(range(low, high + 1)) + return [int(spec)] + except ValueError: + return [] + + +def _parse_port_entry(entry) -> list[dict]: # Long form: {target, published, protocol} if isinstance(entry, dict): published = entry.get("published") if published is None: - return None - try: - host_port = int(str(published).split("-")[0]) - except ValueError: - return None - return {"host_port": host_port, "protocol": entry.get("protocol", "tcp")} + return [] + proto = str(entry.get("protocol") or "tcp") + return [{"host_port": p, "protocol": proto} for p in _host_ports(str(published))] # Short form string: "[ip:]host:container[/proto]" or "container" s = str(entry) @@ -54,14 +72,10 @@ def _parse_port_entry(entry) -> Optional[dict]: parts = s.split(":") # No host mapping (only container port) -> random host port, no conflict. if len(parts) == 1: - return None - # host:container or ip:host:container - host = parts[-2] - try: - host_port = int(host.split("-")[0]) - except ValueError: - return None - return {"host_port": host_port, "protocol": proto} + return [] + # host:container or ip:host:container (an IPv6 address has more colons; + # the host port is always the second to last part) + return [{"host_port": p, "protocol": proto} for p in _host_ports(parts[-2])] # --------------------------------------------------------------------------- # @@ -69,11 +83,27 @@ def _parse_port_entry(entry) -> Optional[dict]: # --------------------------------------------------------------------------- # +def _proc_net_path(name: str) -> str: + """The host's socket table, not this container's. + + ``/proc/net`` is a link to ``self/net`` — the network namespace of whoever + reads it — so ``${HOST_PROC_PATH}/net/tcp`` read from inside the StackPilot + container lists the container's own sockets (just its API port), and every + port a host process listens on went unnoticed. PID 1 of the host's /proc + lives in the host's namespace. + """ + for candidate in ( + os.path.join(settings.HOST_PROC_PATH, "1", "net", name), + os.path.join(settings.HOST_PROC_PATH, "net", name), + ): + if os.path.isfile(candidate): + return candidate + return os.path.join("/proc/net", name) + + def _read_proc_net(name: str) -> set[int]: ports: set[int] = set() - path = os.path.join(settings.HOST_PROC_PATH, "net", name) - if not os.path.isfile(path): - path = os.path.join("/proc/net", name) + path = _proc_net_path(name) try: with open(path, "r", encoding="utf-8") as fh: lines = fh.readlines()[1:] @@ -154,7 +184,7 @@ def detect_conflicts(yaml_str: str, ignore_stack: Optional[str] = None) -> list[ # Match on the compose project label, falling back to the legacy # name-prefix check for containers without the label. if ignore_stack and ( - owner_project == ignore_stack + owner_project in (ignore_stack, project_name(ignore_stack)) or owner_name.startswith(f"{ignore_stack}-") ): continue diff --git a/backend/services/scan_service.py b/backend/services/scan_service.py index 360f945..15accc3 100644 --- a/backend/services/scan_service.py +++ b/backend/services/scan_service.py @@ -43,7 +43,7 @@ SCAN_TIMEOUT = 900.0 #: One sweep at a time. Scanning is CPU- and IO-heavy, and two passes over the #: same images would just fight each other for the daemon. _sweep_lock = threading.Lock() -_sweeping = {"active": False, "done": 0, "total": 0} +_sweeping = {"active": False, "done": 0, "total": 0, "scanned": 0, "failed": 0, "run": 0} class ScanError(Exception): @@ -241,15 +241,16 @@ def sweep_status() -> dict: return dict(_sweeping) -async def sweep(session: Session, images: list[str]) -> dict: - """Scan a list of images one after another. - - Deliberately serial: each scan is already CPU-hungry, and running several at - once on a homelab box would starve the very containers being protected. - """ +def _begin(total: int) -> None: if not _sweep_lock.acquire(blocking=False): raise ScanError("A scan is already running") - _sweeping.update({"active": True, "done": 0, "total": len(images)}) + _sweeping.update( + {"active": True, "done": 0, "total": total, "scanned": 0, "failed": 0, + "run": _sweeping.get("run", 0) + 1} + ) + + +async def _run(session: Session, images: list[str]) -> dict: failed = 0 try: for image in images: @@ -262,6 +263,46 @@ async def sweep(session: Session, images: list[str]) -> dict: failed += 1 _sweeping["done"] += 1 finally: - _sweeping.update({"active": False}) + _sweeping.update({"active": False, "scanned": len(images), "failed": failed}) _sweep_lock.release() return {"scanned": len(images), "failed": failed} + + +async def sweep(session: Session, images: list[str]) -> dict: + """Scan a list of images one after another. + + Deliberately serial: each scan is already CPU-hungry, and running several at + once on a homelab box would starve the very containers being protected. + """ + _begin(len(images)) + return await _run(session, images) + + +#: Background sweeps, kept referenced until they finish. +_tasks: set[asyncio.Task] = set() + + +def start_sweep(images: list[str], on_done=None) -> dict: + """Start a sweep in the background and return its status at once. + + A sweep over a dozen images takes far longer than the proxy waits for a + response (600s), so the button used to report a 504 while the scans went + on. Marked active before this returns, so a poll right after sees it. + """ + from database import engine + + _begin(len(images)) + + async def work() -> None: + with Session(engine) as session: + result = await _run(session, images) + if on_done is not None: + try: + on_done(session, result) + except Exception: # noqa: BLE001 - the sweep itself is done + logger.exception("Post-sweep hook failed") + + task = asyncio.create_task(work(), name="image-scan-sweep") + _tasks.add(task) + task.add_done_callback(_tasks.discard) + return sweep_status() diff --git a/backend/services/schedule_service.py b/backend/services/schedule_service.py index a5f4fe4..135a135 100644 --- a/backend/services/schedule_service.py +++ b/backend/services/schedule_service.py @@ -159,7 +159,9 @@ async def _backup_stack( stack_id, stack.name, include_volumes=schedule.include_volumes, stop_first=schedule.stop_first, ) - filename = backup_service.backup_filename(stack_id, schedule.include_volumes) + filename = backup_service.backup_filename( + stack_id, schedule.include_volumes, tz=schedule.timezone + ) await asyncio.to_thread(dest_service.upload, dest, path, filename) problems = _problems(report) if problems: diff --git a/backend/services/secret_service.py b/backend/services/secret_service.py index 039bbb1..f71a480 100644 --- a/backend/services/secret_service.py +++ b/backend/services/secret_service.py @@ -19,6 +19,17 @@ from typing import Optional from services import compose_edit_service, compose_service KIND_DIR = {"secret": ".secrets", "config": ".configs"} + +#: Who can read a secret is decided by the directory, not the file. +#: +#: Outside swarm, compose hands a file-based secret to the container as a bind +#: mount of the file itself, with the file's own owner and mode. At 0600 root a +#: container running as anything but root (postgres, every PUID/PGID image) got +#: "permission denied" on /run/secrets/. The bind mount reaches the file +#: without passing through its directory, so a 0700 directory keeps other users +#: on the host out while the file stays readable inside the container. +DIR_MODE = 0o700 +FILE_MODE = 0o644 _NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$") MAX_SECRET_BYTES = 1 * 1024 * 1024 # 1 MiB @@ -83,11 +94,11 @@ def write_secret(stack_id: str, kind: str, name: str, content: str, override: Op raise SecretError("content exceeds 1 MiB limit") directory = kind_dir(stack_id, kind, override) os.makedirs(directory, exist_ok=True) - os.chmod(directory, 0o700) + os.chmod(directory, DIR_MODE) path = _path(stack_id, kind, name, override) with open(path, "wb") as fh: fh.write(data) - os.chmod(path, 0o600) + os.chmod(path, FILE_MODE) return {"name": name, "kind": kind, "size": len(data)} @@ -107,6 +118,9 @@ def attach(stack_id: str, kind: str, name: str, service: str, file (relative ``file:`` path) and persist it. Returns the new YAML.""" _check_kind(kind) _check_name(name) + stored = _path(stack_id, kind, name, override) + if os.path.isfile(stored): + os.chmod(stored, FILE_MODE) # one written by an older version yaml_str = compose_service.read_compose(stack_id, override) file_path = rel_path(kind, name) if kind == "config": @@ -126,3 +140,33 @@ def detach(stack_id: str, kind: str, name: str, service: str, override: Optional new_yaml = compose_edit_service.remove_secret(yaml_str, service, name) compose_service.write_compose(stack_id, new_yaml, override) return new_yaml + + +def fix_permissions(root: Optional[str] = None) -> int: + """Make secrets written by older versions (0600) readable by containers. + + Runs at startup over every stack's ``.secrets`` and ``.configs`` — folders + only StackPilot writes. Returns how many files were changed. + """ + base = compose_service.stacks_root(root) + changed = 0 + try: + stacks = os.listdir(base) + except OSError: + return 0 + for stack in stacks: + for folder in KIND_DIR.values(): + directory = os.path.join(base, stack, folder) + if not os.path.isdir(directory) or os.path.islink(directory): + continue + try: + os.chmod(directory, DIR_MODE) + for name in os.listdir(directory): + path = os.path.join(directory, name) + if os.path.isfile(path) and not os.path.islink(path): + if os.stat(path).st_mode & 0o777 == 0o600: + os.chmod(path, FILE_MODE) + changed += 1 + except OSError: + continue + return changed diff --git a/backend/services/stack_assets_service.py b/backend/services/stack_assets_service.py index fb0c400..40eaea2 100644 --- a/backend/services/stack_assets_service.py +++ b/backend/services/stack_assets_service.py @@ -371,24 +371,48 @@ def import_volume( # --------------------------------------------------------------------------- # +#: Found once and kept: a running container's mounts do not change. +_HOST_STACKS_DIR: Optional[str] = None + + def host_stacks_dir() -> Optional[str]: """Host path backing ``STACKS_DIR`` inside this container, if detectable. - Read from /proc/self/mountinfo (field 4 is the source subtree on the host - filesystem). Returns None when not running in a container / not bind-mounted. + Taken from Docker's record of this container's mounts. Not from + /proc/self/mountinfo, as it once was: its 4th field is the path *inside the + mounted filesystem*, which is the host path only when that filesystem is the + host's root. A stacks folder on its own disk, a ZFS dataset or a btrfs + subvolume read as ``/stacks`` or ``/@/opt/stacks``, and every backup dialog + warned about a path mismatch that was not there. Returns None when not + running in a container or when the mount cannot be found. """ - target = settings.STACKS_DIR.rstrip("/") or "/" + global _HOST_STACKS_DIR + if _HOST_STACKS_DIR is not None: + return _HOST_STACKS_DIR + from services.self_update_service import _own_container + + target = os.path.normpath(settings.STACKS_DIR) try: - with open("/proc/self/mountinfo", "r", encoding="utf-8") as fh: - for line in fh: - parts = line.split() - if len(parts) < 5: - continue - if parts[4].rstrip("/") == target: - return parts[3] - except OSError: + container = _own_container() + except Exception: # noqa: BLE001 - no Docker, no answer return None - return None + if container is None: + return None + # The mount whose destination is STACKS_DIR or the closest folder above it + # (a whole /opt mounted, STACKS_DIR=/opt/stacks). + best = None + for mount in container.attrs.get("Mounts") or []: + dest = os.path.normpath(mount.get("Destination") or "") + if mount.get("Type") != "bind" or not mount.get("Source"): + continue + if target == dest or target.startswith(dest.rstrip("/") + "/"): + if best is None or len(dest) > len(best[0]): + best = (dest, mount["Source"]) + if best is None: + return None + dest, source = best + _HOST_STACKS_DIR = os.path.normpath(source + target[len(dest):]) + return _HOST_STACKS_DIR def stacks_path_mismatch() -> Optional[dict]: @@ -483,7 +507,7 @@ def _bind_specs_from_containers(stack_id: str) -> list[dict]: containers = safe_call( client.containers.list, all=True, - filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"}, + filters={"label": _project_label(stack_id)}, ) except DockerError: return [] @@ -503,6 +527,10 @@ def _bind_specs_from_containers(stack_id: str) -> list[dict]: return out +def _project_label(stack_id: str) -> str: + return f"{COMPOSE_PROJECT_LABEL}={compose_service.project_name(stack_id)}" + + def is_system_path(path: str) -> bool: if path in SYSTEM_PATHS: return True @@ -555,7 +583,7 @@ def _external_volume_names(stack_id: str) -> set[str]: containers = safe_call( client.containers.list, all=True, - filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"}, + filters={"label": _project_label(stack_id)}, ) for c in containers: for mount in c.attrs.get("Mounts") or []: @@ -589,7 +617,7 @@ def compose_volumes(stack_id: str) -> list[dict]: client = get_client() vols = safe_call( client.volumes.list, - filters={"label": f"{COMPOSE_PROJECT_LABEL}={stack_id}"}, + filters={"label": _project_label(stack_id)}, ) except DockerError: return [] diff --git a/backend/services/stack_cleanup_service.py b/backend/services/stack_cleanup_service.py new file mode 100644 index 0000000..b3eda1c --- /dev/null +++ b/backend/services/stack_cleanup_service.py @@ -0,0 +1,86 @@ +"""What has to go with a deleted stack, besides its containers and files. + +Several features keep rows keyed by stack id, and deleting the stack used to +leave all of them behind: + +* a **Git source**: the poll loop and the webhook kept syncing it — which + recreated the deleted stack's folder from the repository and, with auto + deploy on, started it again; +* an **auto-update policy**, which a new stack of the same name then silently + inherited; +* **backup schedules** naming it, which from then on failed on every run with + "stack not found" and paged somebody each time. +""" +from __future__ import annotations + +import logging + +from sqlmodel import Session, select + +from models.auto_update import AutoUpdate +from models.backup_schedule import BackupSchedule, set_stacks, stacks_of +from models.git_source import GitSource +from services import git_service + +logger = logging.getLogger("stackpilot.stack_cleanup") + + +def forget_stack(session: Session, stack_id: str) -> dict: + """Drop everything that refers to ``stack_id``. The caller deletes the Stack. + + A schedule that covered only this stack is disabled rather than deleted: + it may carry a destination and retention somebody wants to point at + another stack. All-stacks schedules need nothing — they resolve the stack + list when they run. + """ + report = {"git": False, "auto_update": 0, "schedules_changed": [], "schedules_disabled": []} + + source = session.exec(select(GitSource).where(GitSource.stack_id == stack_id)).first() + if source is not None: + session.delete(source) + report["git"] = True + + for policy in session.exec(select(AutoUpdate).where(AutoUpdate.stack_id == stack_id)).all(): + session.delete(policy) + report["auto_update"] += 1 + + for schedule in session.exec(select(BackupSchedule)).all(): + if schedule.all_stacks: + continue + ids = stacks_of(schedule) + if stack_id not in ids: + continue + remaining = [sid for sid in ids if sid != stack_id] + set_stacks(schedule, remaining) + if remaining: + report["schedules_changed"].append(schedule.id) + else: + schedule.enabled = False + schedule.last_status = f"disabled: its stack '{stack_id}' was deleted" + report["schedules_disabled"].append(schedule.id) + session.add(schedule) + + session.commit() + if report["git"]: + git_service.forget(stack_id) + if any(report[k] for k in report): + logger.info("Cleaned up after deleted stack '%s': %s", stack_id, report) + return report + + +def prune_orphans(session: Session) -> list[str]: + """Forget every stack that has rows here but no Stack any more. + + Run at startup, after on-disk stacks have been registered: installs that + deleted stacks before this existed still carry their Git sources and + policies. Returns the stack ids cleaned up. + """ + from models.stack import Stack + + known = {s.id for s in session.exec(select(Stack)).all()} + referenced = {g.stack_id for g in session.exec(select(GitSource)).all()} + referenced |= {p.stack_id for p in session.exec(select(AutoUpdate)).all()} + orphans = sorted(referenced - known) + for stack_id in orphans: + forget_stack(session, stack_id) + return orphans diff --git a/backend/services/update_service.py b/backend/services/update_service.py index 34ac461..ece13c0 100644 --- a/backend/services/update_service.py +++ b/backend/services/update_service.py @@ -39,6 +39,11 @@ class UpdateStatus: remote_digest: Optional[str] checked_at: float error: Optional[str] = None + #: Built on this host (``docker build``, compose ``build:``) rather than + #: pulled: there is no registry copy to compare against, so it is never + #: "out of date" — and asking Docker Hub about it only got a 401 that read + #: as "needs credentials". + local: bool = False def to_dict(self) -> dict: return asdict(self) @@ -100,6 +105,14 @@ def _store(status: UpdateStatus, notified: bool) -> None: # --------------------------------------------------------------------------- # +#: Docker Hub under the names people write in compose files. Only +#: registry-1.docker.io serves the v2 API; https://docker.io/v2/ redirects to +#: the marketing site, so a `docker.io/library/postgres` image could never be +#: checked and sat at "could not reach registry" forever. +_DOCKER_HUB = "registry-1.docker.io" +_DOCKER_HUB_ALIASES = {"docker.io", "index.docker.io", "registry.docker.io", _DOCKER_HUB} + + def parse_ref(image: str) -> tuple[str, str, str]: """Return (registry_host, repository, tag/digest).""" ref = image @@ -107,37 +120,68 @@ def parse_ref(image: str) -> tuple[str, str, str]: # split tag (but not the registry port colon) if "@" in ref: ref, tag = ref.split("@", 1) - else: - # find last colon after last slash - slash = ref.rfind("/") - colon = ref.rfind(":") - if colon > slash: + # find last colon after last slash. With a digest present this is the tag + # in "name:tag@sha256:…", which the digest outranks — but it must not stay + # glued to the repository name. + slash = ref.rfind("/") + colon = ref.rfind(":") + if colon > slash: + if "@" not in image: tag = ref[colon + 1 :] - ref = ref[:colon] + ref = ref[:colon] parts = ref.split("/", 1) if len(parts) == 2 and ("." in parts[0] or ":" in parts[0] or parts[0] == "localhost"): registry = parts[0] repo = parts[1] + if registry.lower() in _DOCKER_HUB_ALIASES: + registry = _DOCKER_HUB else: - registry = "registry-1.docker.io" + registry = _DOCKER_HUB repo = ref - if "/" not in repo: - repo = "library/" + repo + if registry == _DOCKER_HUB and "/" not in repo: + repo = "library/" + repo return registry, repo, tag -def _local_digest(image: str) -> Optional[str]: +def _local_attrs(image: str) -> Optional[dict]: try: - client = get_client() - img = safe_call(client.images.get, image) + return safe_call(get_client().images.get, image).attrs except DockerError: return None - repo_digests = img.attrs.get("RepoDigests") or [] + + +def _built_locally(attrs: Optional[dict]) -> bool: + """Whether the image came from a build on this host, not from a registry. + + Docker 29+ records where an image came from (``Identity.Pull`` / + ``Identity.Build``). Older daemons do not; there a built image is the one + without RepoDigests. Unknown means "pulled" — skipping the check for a + pulled image would hide its updates for good. + """ + if attrs is None: + return False + identity = attrs.get("Identity") + if isinstance(identity, dict) and identity: + return bool(identity.get("Build")) and not identity.get("Pull") + return not (attrs.get("RepoDigests") or []) + + +def _local_digest(image: str, attrs: Optional[dict] = None) -> Optional[str]: + if attrs is None: + attrs = _local_attrs(image) + if attrs is None: + return None + repo_digests = [rd for rd in attrs.get("RepoDigests") or [] if "@" in rd] + # An image can carry digests for several repositories (the same image + # pulled as lscr.io/... and as docker.io/...); compare against the one + # this reference is checked against, not whichever comes first. + wanted = parse_ref(image)[:2] for rd in repo_digests: - if "@" in rd: - return rd.split("@", 1)[1] - return None + name, digest = rd.split("@", 1) + if parse_ref(name)[:2] == wanted: + return digest + return repo_digests[0].split("@", 1)[1] if repo_digests else None # --------------------------------------------------------------------------- # @@ -236,7 +280,20 @@ async def remote_digest(image: str) -> Optional[str]: async def check_image(image: str) -> UpdateStatus: - local = _local_digest(image) + attrs = _local_attrs(image) + if _built_locally(attrs): + status = UpdateStatus( + image=image, + update_available=False, + current_digest=None, + remote_digest=None, + checked_at=time.time(), + local=True, + ) + _NOTIFIED.discard(image) + _store(status, False) + return status + local = _local_digest(image, attrs) error = None try: remote = await remote_digest(image) @@ -294,12 +351,15 @@ def _all_running_images() -> set[str]: def stack_images(stack_id: str) -> set[str]: """Images used by the containers of one compose project (== stack id).""" + from services.compose_service import project_name + + project = project_name(stack_id) images: set[str] = set() try: client = get_client() for c in safe_call(client.containers.list, all=True): labels = c.labels or {} - if labels.get("com.docker.compose.project") != stack_id: + if labels.get("com.docker.compose.project") != project: continue cfg_image = c.attrs.get("Config", {}).get("Image") if cfg_image: @@ -380,6 +440,9 @@ def refresh_stack_local(stack_id: str) -> None: status.checked_at = time.time() if not status.update_available: _NOTIFIED.discard(image) + # Persisted too, or a restart brought the cleared badge back until the + # next background check. + _store(status, image in _NOTIFIED) async def check_all() -> dict[str, dict]: diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 9c178a2..4b462df 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -30,6 +30,14 @@ os.environ["STACKS_DIR"] = os.path.join(_TMP, "stacks") os.environ["SECRET_KEY"] = "test-secret-key-not-used-anywhere-real" os.environ["ALLOWED_BROWSE_ROOTS"] = "/mnt,/media,/srv,/opt,/home" os.environ["HOST_ROOT_PREFIX"] = "" +# No Docker, even where there is one. The suite is written for a host without a +# daemon, and where one *is* reachable — a dev box, or the CI runner, which gets +# the host's socket — tests that go "far enough in" really ran `compose up` and +# left containers running on that host after every run. docker-py reads +# DOCKER_SOCKET, the docker CLI (compose) reads DOCKER_HOST; both point nowhere. +_NO_DOCKER = os.path.join(_TMP, "no-docker.sock") +os.environ["DOCKER_SOCKET"] = _NO_DOCKER +os.environ["DOCKER_HOST"] = f"unix://{_NO_DOCKER}" # The backend runs from its own root at runtime (`uvicorn main:app` with # /app as the workdir), so make the same layout importable here. diff --git a/backend/tests/test_compose_runs.py b/backend/tests/test_compose_runs.py new file mode 100644 index 0000000..5d0e74c --- /dev/null +++ b/backend/tests/test_compose_runs.py @@ -0,0 +1,226 @@ +"""Streamed compose runs: the deploy console and the update bar. + +Until 0.67.1 the run *was* the WebSocket handler, so closing the console in the +middle of a deploy released the stack lock at once — while compose went on — +and logged the deploy as a success with ``rc=None``. The run is its own task +now; a socket only watches it. And the deploy socket is admin-only, like the +REST ``/start`` it mirrors: before, the read-only role could run ``compose up`` +through it. +""" +from __future__ import annotations + +import asyncio +import json + +import pytest +from sqlmodel import Session + + +@pytest.fixture(autouse=True) +def clean_locks(db): + from sqlmodel import delete + + from database import engine + from models.runtime_state import StackLock + + with Session(engine) as session: + session.exec(delete(StackLock)) + session.commit() + yield + + +def _locked(stack_id: str) -> bool: + from database import engine + from services import stack_lock_service + + with Session(engine) as session: + return stack_lock_service.is_busy(session, stack_id) + + +def test_the_lock_is_held_until_compose_finishes(db): + from services import compose_run_service as runs + + async def scenario(): + release = asyncio.Event() + finished: list = [] + + async def stream(): + yield ("log", "pulling") + await release.wait() + yield ("log", "started") + yield ("done", 0) + + async def on_finish(rc): + finished.append(rc) + + run, queue, attached = runs.start("rx", "start", "alice", stream, on_finish) + assert not attached + assert await queue.get() == ("log", "pulling") + # Nobody is watching any more — the console was closed. + run.unsubscribe(queue) + await asyncio.sleep(0.05) + assert _locked("rx"), "the lock must outlive the socket" + assert finished == [] + + release.set() + for _ in range(50): + if finished: + break + await asyncio.sleep(0.02) + assert finished == [0], "the real exit code, after the fact" + assert not _locked("rx") + + asyncio.run(scenario()) + + +def test_a_second_socket_attaches_to_the_run_in_flight(db): + from services import compose_run_service as runs + + async def scenario(): + release = asyncio.Event() + + async def stream(): + yield ("log", "one") + await release.wait() + yield ("done", 0) + + run, first, _ = runs.start("ry", "start", "alice", stream) + assert await first.get() == ("log", "one") + + again, second, attached = runs.start("ry", "start", "bob", stream) + assert attached and again is run + assert await second.get() == ("log", "one"), "backlog first" + + with pytest.raises(runs.RunBusy): + runs.start("ry", "update", "carol", stream) + + release.set() + items = [] + while (item := await second.get()) is not None: + items.append(item) + assert items == [("done", 0)] + + asyncio.run(scenario()) + + +def test_a_run_that_cannot_get_the_lock_is_busy(db): + from database import engine + from services import compose_run_service as runs + from services import stack_lock_service + + with Session(engine) as session: + stack_lock_service.acquire(session, "rz", "backup", "scheduled") + + async def stream(): + yield ("done", 0) + + async def scenario(): + with pytest.raises(runs.RunBusy) as excinfo: + runs.start("rz", "start", "alice", stream) + assert excinfo.value.action == "backup" + + asyncio.run(scenario()) + + +def test_a_failing_stream_still_releases_the_lock(db): + from services import compose_run_service as runs + + async def scenario(): + async def stream(): + yield ("log", "about to break") + raise RuntimeError("compose binary missing") + + finished = [] + + async def on_finish(rc): + finished.append(rc) + + _run, queue, _ = runs.start("rf", "update", "alice", stream, on_finish) + items = [] + while (item := await queue.get()) is not None: + items.append(item) + await asyncio.sleep(0.05) + assert items[-1][0] == "done" and items[-1][1] != 0 + assert finished and finished[0] != 0 + assert not _locked("rf") + + asyncio.run(scenario()) + + +def test_the_deploy_socket_is_admin_only(client, user_token): + from starlette.websockets import WebSocketDisconnect + + with pytest.raises(WebSocketDisconnect) as excinfo: + with client.websocket_connect(f"/ws/deploy/anything?token={user_token}") as socket: + socket.receive_text() + assert excinfo.value.code == 4403 + + +def test_deploying_an_unknown_stack_is_refused(client, admin_token): + from starlette.websockets import WebSocketDisconnect + + with pytest.raises(WebSocketDisconnect) as excinfo: + with client.websocket_connect(f"/ws/deploy/no-such-stack?token={admin_token}") as socket: + frame = json.loads(socket.receive_text()) + assert frame["type"] == "error" + socket.receive_text() + assert excinfo.value.code == 4404 + + +# --------------------------------------------------------------------------- # +# The REST paths +# --------------------------------------------------------------------------- # + + +def test_update_stops_at_a_failed_pull(monkeypatch): + """`up` after a failed pull recreated nothing and reported success.""" + from services import compose_service + + calls = [] + + async def fake(stack_id, args, override=None, timeout=600.0): + calls.append(args[0]) + rc = 1 if args[0] == "pull" else 0 + return {"returncode": rc, "stdout": "", "stderr": "denied", "command": args[0]} + + monkeypatch.setattr(compose_service, "run_compose", fake) + result = asyncio.run(compose_service.update("s")) + assert calls == ["pull"] + assert result["returncode"] == 1 + + +def test_auto_update_reports_a_failed_compose_run(db, monkeypatch): + """The exit codes used to be ignored: a failure was recorded as "updated".""" + from database import engine + from models.auto_update import AutoUpdate + from services import auto_update_service, compose_service, update_service + + monkeypatch.setattr(compose_service, "compute_status", lambda stack_id: "running") + + async def stale(stack_id, refresh=False): + return {"update_available": True, "stale_images": ["nginx:latest"]} + + async def failing_update(stack_id, override=None): + return {"returncode": 1, "stdout": "", "stderr": "Error: pull access denied"} + + notified = [] + + async def notify(event, title, message, session=None): + notified.append(title) + + monkeypatch.setattr(update_service, "stack_updates", stale) + monkeypatch.setattr(compose_service, "update", failing_update) + monkeypatch.setattr(auto_update_service.notify_service, "notify", notify) + + with Session(engine) as session: + policy = AutoUpdate(stack_id="au-fail", enabled=True, redeploy=True) + session.add(policy) + session.commit() + session.refresh(policy) + asyncio.run(auto_update_service.run_policy(session, policy)) + session.refresh(policy) + assert policy.last_status == "error" + assert "pull access denied" in (policy.last_result or "") + session.delete(policy) + session.commit() + assert notified and "failed" in notified[0] diff --git a/backend/tests/test_compose_service.py b/backend/tests/test_compose_service.py index d211c90..f0d942e 100644 --- a/backend/tests/test_compose_service.py +++ b/backend/tests/test_compose_service.py @@ -137,16 +137,38 @@ def test_secret_names_reject_traversal_and_hidden_files(name): secret_service._check_name(name) -def test_secret_files_are_written_owner_only(tmp_path): +def test_secret_files_are_guarded_by_their_directory(tmp_path): + """The 0700 directory keeps other host users out; the file itself has to be + world-readable, because compose bind-mounts it into the container as it is + and a container running as postgres or PUID 1000 could not open a 0600 + root file.""" from services import secret_service secret_service.write_secret("s", "secret", "db_password", "hunter2", override=str(tmp_path)) path = tmp_path / "s" / ".secrets" / "db_password" assert path.read_text() == "hunter2" - assert oct(path.stat().st_mode)[-3:] == "600" + assert oct(path.stat().st_mode)[-3:] == "644" assert oct(path.parent.stat().st_mode)[-3:] == "700" +def test_secrets_from_older_versions_are_made_readable(tmp_path): + import os + + from services import secret_service + + folder = tmp_path / "old" / ".secrets" + folder.mkdir(parents=True) + old = folder / "pw" + old.write_text("x") + os.chmod(old, 0o600) + os.chmod(folder, 0o755) + + assert secret_service.fix_permissions(str(tmp_path)) == 1 + assert oct(old.stat().st_mode)[-3:] == "644" + assert oct(folder.stat().st_mode)[-3:] == "700" + assert secret_service.fix_permissions(str(tmp_path)) == 0, "idempotent" + + def test_listing_secrets_never_returns_their_content(tmp_path): from services import secret_service diff --git a/backend/tests/test_convert_service.py b/backend/tests/test_convert_service.py new file mode 100644 index 0000000..c778293 --- /dev/null +++ b/backend/tests/test_convert_service.py @@ -0,0 +1,167 @@ +"""Converting `docker run` into compose. + +The output has to be a compose file compose accepts and that means what the +command meant. Before 0.67.1 a boolean flag the converter did not know swallowed +the image name (``-dit ubuntu bash`` became a service running ``bash``), +``--network host`` and ``-v data:/data`` produced files compose rejects, and +``--gpus`` or ``--mount`` were dropped without a word. +""" +from __future__ import annotations + +import pytest +import yaml + + +def _service(command: str) -> tuple[str, dict, dict]: + from services.convert_service import convert_docker_run + + doc = yaml.safe_load(convert_docker_run(command)) + ((name, service),) = doc["services"].items() + return name, service, doc + + +@pytest.mark.parametrize( + "command, image", + [ + ("docker run -dit ubuntu bash", "ubuntu"), + ("docker run -d --read-only nginx", "nginx"), + ("docker run -P nginx", "nginx"), + ("docker run --rm --init --privileged alpine", "alpine"), + ("docker run -d --oom-kill-disable --no-healthcheck redis:7", "redis:7"), + ], +) +def test_boolean_flags_never_swallow_the_image(command, image): + _name, service, _doc = _service(command) + assert service["image"] == image + + +def test_the_command_after_the_image_is_kept(): + name, service, _doc = _service("docker run -it --rm ubuntu bash -lc 'echo hi'") + assert name == "ubuntu" + assert service["command"] == ["bash", "-lc", "echo hi"] + assert service["stdin_open"] is True and service["tty"] is True + + +@pytest.mark.parametrize("mode", ["host", "none", "bridge", "container:vpn"]) +def test_special_networks_become_network_mode(mode): + _name, service, doc = _service(f"docker run -d --network {mode} alpine") + assert service["network_mode"] == mode + assert "networks" not in service and "networks" not in doc + + +def test_a_named_network_is_declared_external(): + _name, service, doc = _service("docker run -d --net=proxy --network-alias web alpine") + assert service["networks"] == {"proxy": {"aliases": ["web"]}} + assert doc["networks"] == {"proxy": {"external": True}} + + +def test_named_volumes_are_declared_and_paths_are_not(): + _name, service, doc = _service( + "docker run -d -v data:/data -v /srv/media:/media:ro -v ./conf:/conf alpine" + ) + assert service["volumes"] == ["data:/data", "/srv/media:/media:ro", "./conf:/conf"] + assert doc["volumes"] == {"data": None} + + +def test_mount_becomes_long_syntax(): + _name, service, doc = _service( + "docker run --mount type=bind,source=/srv,target=/srv,readonly " + "--mount type=volume,src=cache,dst=/cache alpine" + ) + assert service["volumes"][0] == { + "type": "bind", "source": "/srv", "target": "/srv", "read_only": True, + } + assert service["volumes"][1] == {"type": "volume", "source": "cache", "target": "/cache"} + assert doc["volumes"] == {"cache": None} + + +@pytest.mark.parametrize( + "value, expected", + [ + ("all", {"count": "all"}), + ("2", {"count": 2}), + ('"device=0,1"', {"device_ids": ["0", "1"]}), + ], +) +def test_gpus_become_a_device_reservation(value, expected): + _name, service, _doc = _service(f"docker run -d --gpus {value} ollama/ollama") + (device,) = service["deploy"]["resources"]["reservations"]["devices"] + assert device["driver"] == "nvidia" + assert device["capabilities"] == ["gpu"] + for key, val in expected.items(): + assert device[key] == val + + +def test_attached_short_values_and_equals_forms(): + _name, service, _doc = _service( + "docker run -p8080:80 -eTZ=Europe/Berlin --restart=always --name=web nginx" + ) + assert service["ports"] == ["8080:80"] + assert service["environment"] == ["TZ=Europe/Berlin"] + assert service["restart"] == "always" + assert service["container_name"] == "web" + + +def test_common_runtime_flags_are_kept(): + _name, service, _doc = _service( + "sudo docker run -d --name plex --entrypoint /init --shm-size 1g -m 2g --cpus 1.5 " + "--cap-add NET_ADMIN --cap-drop ALL --security-opt no-new-privileges " + "--sysctl net.ipv4.ip_forward=1 --add-host nas:10.0.0.2 --dns 1.1.1.1 " + "--ulimit nofile=1024:2048 --log-opt max-size=10m --tmpfs /run " + "--health-cmd 'curl -f http://localhost' --health-interval 30s --stop-timeout 30 " + "lscr.io/linuxserver/plex:latest" + ) + assert service["entrypoint"] == "/init" + assert service["shm_size"] == "1g" + assert service["mem_limit"] == "2g" + assert service["cpus"] == 1.5 + assert service["cap_add"] == ["NET_ADMIN"] and service["cap_drop"] == ["ALL"] + assert service["security_opt"] == ["no-new-privileges"] + assert service["sysctls"] == ["net.ipv4.ip_forward=1"] + assert service["extra_hosts"] == ["nas:10.0.0.2"] + assert service["dns"] == ["1.1.1.1"] + assert service["ulimits"] == {"nofile": {"soft": 1024, "hard": 2048}} + assert service["logging"] == {"options": {"max-size": "10m"}} + assert service["tmpfs"] == ["/run"] + assert service["healthcheck"] == { + "test": ["CMD-SHELL", "curl -f http://localhost"], "interval": "30s", + } + assert service["stop_grace_period"] == "30s" + + +def test_line_continuations_from_a_readme(): + name, service, _doc = _service("docker run -d \\\n --name x \\\r\n -p 1:1 \\\n alpine") + assert name == "x" + assert service["ports"] == ["1:1"] + + +def test_the_service_name_comes_from_the_image_without_registry_or_tag(): + name, _service_, _doc = _service("docker run -d ghcr.io/home-assistant/home-assistant:stable") + assert name == "home-assistant" + + +def test_no_image_is_an_error(): + from services.convert_service import convert_docker_run + + with pytest.raises(ValueError): + convert_docker_run("docker run -d --name x") + + +@pytest.mark.parametrize( + "command", + [ + # Pasted into a one-line field: the newlines are gone, the backslashes + # and indentation are not. + "docker run -d \\ --name web \\ -p 8081:80 \\ -v data:/data \\ nginx:alpine", + "docker run -d \\ --name web \\ -p 8081:80 \\\n nginx:alpine", + ], +) +def test_continuations_that_lost_their_newline(command): + name, web, _ = _service(command) + assert name == "web" and web["image"] == "nginx:alpine" + assert web["ports"] == ["8081:80"] + + +def test_an_escaped_space_in_a_path_is_still_one(): + _, service, _ = _service("docker run -v /mnt/My\\ Files:/data nginx") + assert service["volumes"] == ["/mnt/My Files:/data"] diff --git a/backend/tests/test_destinations.py b/backend/tests/test_destinations.py new file mode 100644 index 0000000..e3d8ab2 --- /dev/null +++ b/backend/tests/test_destinations.py @@ -0,0 +1,276 @@ +"""Backup destinations: errors that say what happened, and names that stay names. + +An SFTP host that refused the connection was a bare 500 (paramiko connects in +the Transport constructor, outside every handler), so the "Test" button showed +nothing useful. And a backup file name from the API went straight into +``posixpath.join`` — ``../`` let a delete reach outside the backup folder. +""" +from __future__ import annotations + +import socket + +import pytest + + +@pytest.mark.parametrize("name", ["../etc/passwd", "a/b", "..", "", "x\\y", "a\0b"]) +def test_backup_names_must_be_bare_file_names(name): + from services import backup_destination_service as dest + + with pytest.raises(dest.DestinationError): + dest._check_file_name(name) + + +def test_a_real_backup_name_passes(): + from services import backup_destination_service as dest + + dest._check_file_name("backup-immich-full-20260101-030000.tar.gz") + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _destination(kind: str, config: dict): + from models.backup_destination import BackupDestination + from services import backup_destination_service as dest + + return BackupDestination(id=99, name="t", type=kind, config=dest.dump_config(config)) + + +def test_an_unreachable_sftp_host_is_a_destination_error(): + from services import backup_destination_service as dest + + d = _destination( + "sftp", {"host": "127.0.0.1", "port": _free_port(), "username": "u", "password": "p"} + ) + with pytest.raises(dest.DestinationError, match="SFTP connection"): + dest.test(d) + + +def test_sftp_delete_refuses_a_path_before_connecting(): + from services import backup_destination_service as dest + + d = _destination("sftp", {"host": "198.51.100.1", "username": "u"}) + with pytest.raises(dest.DestinationError, match="Invalid backup file name"): + dest.delete(d, "../../important") + + +def test_a_malformed_s3_endpoint_is_a_destination_error(): + from services import backup_destination_service as dest + + d = _destination("s3", {"bucket": "b", "endpoint_url": "not a url"}) + with pytest.raises(dest.DestinationError, match="Invalid S3 settings"): + dest.test(d) + + +def test_the_test_endpoint_reports_instead_of_failing(as_admin, db): + from sqlmodel import Session + + from database import engine + from models.backup_destination import BackupDestination + from services import backup_destination_service as dest + + with Session(engine) as session: + row = BackupDestination( + name="refusing", + type="sftp", + config=dest.dump_config( + {"host": "127.0.0.1", "port": _free_port(), "username": "u", "password": "p"} + ), + ) + session.add(row) + session.commit() + session.refresh(row) + dest_id = row.id + try: + response = as_admin.post(f"/api/backups/destinations/{dest_id}/test") + assert response.status_code == 200 + assert response.json()["ok"] is False + assert "SFTP connection" in response.json()["error"] + finally: + with Session(engine) as session: + session.delete(session.get(BackupDestination, dest_id)) + session.commit() + + +def test_a_destination_in_use_by_a_schedule_cannot_be_deleted(as_admin, db): + """Deleted from under it, the schedule failed on every run from then on.""" + from sqlmodel import Session + + from database import engine + from models.backup_destination import BackupDestination + from models.backup_schedule import BackupSchedule + from services import backup_destination_service as dest + + with Session(engine) as session: + row = BackupDestination(name="nas", type="sftp", config=dest.dump_config({"host": "x"})) + session.add(row) + session.commit() + session.refresh(row) + schedule = BackupSchedule(destination_id=row.id, all_stacks=True) + session.add(schedule) + session.commit() + session.refresh(schedule) + dest_id, schedule_id = row.id, schedule.id + try: + response = as_admin.delete(f"/api/backups/destinations/{dest_id}") + assert response.status_code == 409 + assert "schedule" in response.json()["detail"] + finally: + with Session(engine) as session: + session.delete(session.get(BackupSchedule, schedule_id)) + session.commit() + assert as_admin.delete(f"/api/backups/destinations/{dest_id}").status_code == 200 + + +# --------------------------------------------------------------------------- # +# SFTP host keys: a real SSH handshake against an in-process paramiko server. +# --------------------------------------------------------------------------- # + + +class _SshServer: + """One-connection SSH server with an SFTP subsystem and a known host key.""" + + def __init__(self, host_key): + import threading + + import paramiko + + self.host_key = host_key + self.passwords: list[str] = [] + outer = self + + class Iface(paramiko.ServerInterface): + def get_allowed_auths(self, username): + return "password" + + def check_auth_password(self, username, password): + outer.passwords.append(password) + return paramiko.AUTH_SUCCESSFUL + + def check_channel_request(self, kind, chanid): + return paramiko.OPEN_SUCCEEDED + + self._iface = Iface + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(1) + self.port = self.sock.getsockname()[1] + threading.Thread(target=self._run, daemon=True).start() + + def _run(self): + import paramiko + + conn, _ = self.sock.accept() + transport = paramiko.Transport(conn) + transport.add_server_key(self.host_key) + transport.set_subsystem_handler("sftp", paramiko.SFTPServer, paramiko.SFTPServerInterface) + try: + transport.start_server(server=self._iface()) + # Kept: a channel that is garbage collected closes itself, and the + # client saw "Channel closed" in the middle of the SFTP handshake. + self.channel = transport.accept(10) + while transport.is_active(): + transport.join(0.1) + except Exception: # noqa: BLE001 - the client hung up on purpose + pass + finally: + self.sock.close() + + def cfg(self, **extra): + return {"host": "127.0.0.1", "port": self.port, "username": "u", "password": "pw", **extra} + + +@pytest.fixture +def host_key(): + import paramiko + + return paramiko.ECDSAKey.generate() + + +def _connect(cfg, pin=None): + from services import backup_destination_service as dest + + sftp, transport = dest._sftp_connect(cfg, pin) + sftp.close() + transport.close() + + +def test_the_first_host_key_is_pinned(host_key): + server = _SshServer(host_key) + pinned = [] + _connect(server.cfg(), pinned.append) + assert pinned == [f"{host_key.get_name()} {host_key.get_base64()}"] + + +@pytest.mark.parametrize("form", ["line", "known_hosts", "fingerprint"]) +def test_a_pinned_key_lets_its_server_in(host_key, form): + from services import backup_destination_service as dest + + line = f"{host_key.get_name()} {host_key.get_base64()}" + stored = { + "line": line, + "known_hosts": f"[nas]:2222 {line}", + "fingerprint": dest.host_key_fingerprint(host_key), + }[form] + server = _SshServer(host_key) + pinned = [] + _connect(server.cfg(host_key=stored), pinned.append) + assert pinned == [] # nothing new to pin + assert server.passwords == ["pw"] + + +def test_another_key_is_refused_before_the_password_is_sent(host_key): + import paramiko + + from services import backup_destination_service as dest + + impostor = _SshServer(host_key) + real = paramiko.ECDSAKey.generate() + with pytest.raises(dest.DestinationError, match="does not match"): + _connect(impostor.cfg(host_key=f"{real.get_name()} {real.get_base64()}")) + assert impostor.passwords == [] + + +def test_the_fingerprint_of_a_stored_key_is_the_one_ssh_keygen_prints(host_key): + from services import backup_destination_service as dest + + line = f"{host_key.get_name()} {host_key.get_base64()}" + assert dest.fingerprint_of(line) == dest.host_key_fingerprint(host_key) + assert dest.fingerprint_of("SHA256:abc") == "SHA256:abc" + + +def test_moving_a_destination_to_another_host_forgets_its_key(as_admin, db): + from sqlmodel import Session + + from database import engine + from models.backup_destination import BackupDestination + from services import backup_destination_service as dest + + with Session(engine) as session: + row = BackupDestination( + name="moving", + type="sftp", + config=dest.dump_config({"host": "old", "username": "u", "host_key": "ssh-ed25519 AAAA"}), + ) + session.add(row) + session.commit() + session.refresh(row) + dest_id = row.id + try: + # The form sends every field back, the shown host key included. + same = {"host": "old", "username": "u", "host_key": "ssh-ed25519 AAAA", "path": "/b"} + as_admin.put(f"/api/backups/destinations/{dest_id}", json={"config": same}) + moved = {**same, "host": "new"} + assert as_admin.put( + f"/api/backups/destinations/{dest_id}", json={"config": moved} + ).status_code == 200 + with Session(engine) as session: + cfg = dest.parse_config(session.get(BackupDestination, dest_id)) + assert cfg["host"] == "new" and "host_key" not in cfg + finally: + with Session(engine) as session: + session.delete(session.get(BackupDestination, dest_id)) + session.commit() diff --git a/backend/tests/test_image_refs_and_ports.py b/backend/tests/test_image_refs_and_ports.py new file mode 100644 index 0000000..da84982 --- /dev/null +++ b/backend/tests/test_image_refs_and_ports.py @@ -0,0 +1,123 @@ +"""Image references, compose project names and published ports. + +Small parsers with large consequences: + +* ``docker.io/library/postgres`` is how many compose files name an image, and + https://docker.io/v2/ is not a registry — so every such image sat at "could + not reach registry" forever and an update never showed up. +* compose refuses ``-p`` with anything but ``[a-z0-9_-]``, so a stack folder + found on disk as ``Nextcloud`` could not be started, stopped or deleted. +* the host's listening ports live in the host's network namespace, which + ``/proc/net`` read from inside the container is not. +""" +from __future__ import annotations + +import pytest + + +@pytest.mark.parametrize( + "image, expected", + [ + ("nginx", ("registry-1.docker.io", "library/nginx", "latest")), + ("docker.io/nginx:1", ("registry-1.docker.io", "library/nginx", "1")), + ("docker.io/library/postgres:16", ("registry-1.docker.io", "library/postgres", "16")), + ("index.docker.io/foo/bar:2", ("registry-1.docker.io", "foo/bar", "2")), + ("ghcr.io/a/b:1@sha256:abc", ("ghcr.io", "a/b", "sha256:abc")), + ("localhost:5000/x", ("localhost:5000", "x", "latest")), + ("registry.example:5000/a/b:tag", ("registry.example:5000", "a/b", "tag")), + ], +) +def test_parse_ref(image, expected): + from services.update_service import parse_ref + + assert parse_ref(image) == expected + + +def test_the_local_digest_is_the_one_for_this_repository(monkeypatch): + """An image pulled under two names carries a digest per repository.""" + from services import update_service + + class Image: + attrs = {"RepoDigests": [ + "lscr.io/linuxserver/radarr@sha256:aaa", + "linuxserver/radarr@sha256:bbb", + ]} + + class Images: + def get(self, name): + return Image() + + class Client: + images = Images() + + monkeypatch.setattr(update_service, "get_client", lambda: Client()) + assert update_service._local_digest("linuxserver/radarr:latest") == "sha256:bbb" + assert update_service._local_digest("lscr.io/linuxserver/radarr:latest") == "sha256:aaa" + assert update_service._local_digest("docker.io/linuxserver/radarr") == "sha256:bbb" + + +@pytest.mark.parametrize( + "stack_id, project", + [ + ("immich", "immich"), + ("arr_stack-2", "arr_stack-2"), + ("Nextcloud", "nextcloud"), + ("my.app", "myapp"), + ("_hidden-ish", "hidden-ish"), + ("...", "stack"), + ], +) +def test_project_name_matches_compose(stack_id, project): + from services.compose_service import project_name + + assert project_name(stack_id) == project + + +def test_compose_is_called_with_the_normalised_project(): + from services import compose_service + + cmd = compose_service._compose_base_cmd("Nextcloud") + assert cmd[cmd.index("-p") + 1] == "nextcloud" + + +def test_published_ports_including_ranges_and_long_syntax(): + from services.port_service import parse_compose_ports + + ports = parse_compose_ports( + "services:\n a:\n ports:\n" + ' - "8080:80"\n' + ' - "127.0.0.1:9000:9000/udp"\n' + ' - "7000-7002:7000-7002"\n' + ' - "80"\n' + ' - {target: 80, published: "8443"}\n' + ' - "[::1]:5353:53/udp"\n' + ) + assert [(p["host_port"], p["protocol"]) for p in ports] == [ + (8080, "tcp"), (9000, "udp"), (7000, "tcp"), (7001, "tcp"), (7002, "tcp"), + (8443, "tcp"), (5353, "udp"), + ] + + +def test_garbage_compose_yields_no_ports(): + from services.port_service import parse_compose_ports + + assert parse_compose_ports("just a string") == [] + assert parse_compose_ports("services: [1, 2]") == [] + assert parse_compose_ports(": : :") == [] + + +def test_host_ports_are_read_from_the_hosts_namespace(tmp_path, monkeypatch): + """/proc/net is self/net; PID 1 of the host /proc is the host's namespace.""" + from config import settings + from services import port_service + + (tmp_path / "net").mkdir() + (tmp_path / "1" / "net").mkdir(parents=True) + header = " sl local_address rem_address st\n" + # The container's own table: just its API port. + (tmp_path / "net" / "tcp").write_text(header + " 0: 00000000:1390 00000000:0000 0A\n") + # The host's: ssh on 22 (0x16). + (tmp_path / "1" / "net" / "tcp").write_text(header + " 0: 00000000:0016 00000000:0000 0A\n") + monkeypatch.setattr(settings, "HOST_PROC_PATH", str(tmp_path)) + + assert port_service._read_proc_net("tcp") == {22} diff --git a/backend/tests/test_notifications.py b/backend/tests/test_notifications.py new file mode 100644 index 0000000..f32dd9a --- /dev/null +++ b/backend/tests/test_notifications.py @@ -0,0 +1,58 @@ +"""Webhook delivery. + +Two ways a notification silently went nowhere before 0.67.1: an ntfy title with +a non-ASCII character (a destination called "NAS Büro", an em dash) raised +UnicodeEncodeError inside httpx, and any error that was not an +``httpx.HTTPError`` escaped the per-webhook handler and aborted delivery to +every webhook after it. +""" +from __future__ import annotations + +import asyncio +import email.header + +import httpx + + +def test_a_non_ascii_ntfy_title_is_rfc2047_encoded(): + from services.notify_service import _build_request + + kwargs = _build_request("ntfy", "http://ntfy/t", "backup_failed", "Backup to NAS Büro — failed", "m") + request = httpx.Request("POST", kwargs["url"], headers=kwargs["headers"], content=kwargs["content"]) + (decoded, charset), = email.header.decode_header(request.headers["title"]) + assert decoded.decode(charset) == "Backup to NAS Büro — failed" + + +def test_an_ascii_title_is_sent_as_is(): + from services.notify_service import _header_value + + assert _header_value("Stack 'x' started") == "Stack 'x' started" + + +def test_one_broken_webhook_does_not_stop_the_others(monkeypatch): + from services import notify_service + + delivered = [] + + class Client: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def post(self, url, timeout=None, **kwargs): + if "broken" in url: + raise ValueError("not an httpx error") + delivered.append(url) + return httpx.Response(200, request=httpx.Request("POST", url)) + + monkeypatch.setattr(notify_service.httpx, "AsyncClient", lambda **kw: Client()) + monkeypatch.setattr( + notify_service, + "_targets_for_event", + lambda session, event: [("generic", "http://broken/"), ("generic", "http://fine/")], + ) + count = asyncio.run(notify_service.notify("stack_start", "t", "m", session=object())) + assert count == 1 + assert delivered == ["http://fine/"] diff --git a/backend/tests/test_registries.py b/backend/tests/test_registries.py index 29cace7..e30cdf5 100644 --- a/backend/tests/test_registries.py +++ b/backend/tests/test_registries.py @@ -331,7 +331,7 @@ def _refusing(monkeypatch): raise update_service.AuthRequired("ghcr.io") monkeypatch.setattr(update_service, "remote_digest", refuse) - monkeypatch.setattr(update_service, "_local_digest", lambda image: "sha256:local") + monkeypatch.setattr(update_service, "_local_digest", lambda image, attrs=None: "sha256:local") return update_service @@ -370,6 +370,6 @@ def test_a_registry_we_simply_cannot_reach_still_says_that(monkeypatch): return None monkeypatch.setattr(update_service, "remote_digest", unreachable) - monkeypatch.setattr(update_service, "_local_digest", lambda image: "sha256:local") + monkeypatch.setattr(update_service, "_local_digest", lambda image, attrs=None: "sha256:local") result = asyncio.run(update_service.check_image("ghcr.io/acme/app:1.0")) assert result.error == "could not reach registry" diff --git a/backend/tests/test_review_regressions.py b/backend/tests/test_review_regressions.py new file mode 100644 index 0000000..36d64fc --- /dev/null +++ b/backend/tests/test_review_regressions.py @@ -0,0 +1,137 @@ +"""Smaller regressions found in the 0.67 review, one test each. + +* NVIDIA cards were never detected: the image has no nvidia-smi and the DRI + scan skips NVIDIA on purpose. The driver's /proc entries are read instead. +* A container's inspect view handed its environment — credentials, by + convention — to the read-only role, which does not get the stack's .env. +* Git branch names and URLs go to git's command line; a leading "-" made them + options. +* "Scan all" ran for longer than the proxy waits; it now runs in the + background and reports through /scan-status. +""" +from __future__ import annotations + +import asyncio + +import pytest + + +def test_nvidia_gpus_are_read_from_the_driver_procfs(tmp_path, monkeypatch): + from config import settings + from services import gpu_service + + gpus = tmp_path / "driver" / "nvidia" / "gpus" + (gpus / "0000:01:00.0").mkdir(parents=True) + (gpus / "0000:01:00.0" / "information").write_text( + "Model: \t\t NVIDIA GeForce RTX 3060\n" + "IRQ: \t\t 150\n" + "GPU UUID: \t GPU-1234-abcd\n" + "Device Minor: \t 0\n" + ) + (tmp_path / "driver" / "nvidia" / "version").write_text( + "NVRM version: NVIDIA UNIX x86_64 Kernel Module 550.120 Fri Sep 13 2024\n" + ) + monkeypatch.setattr(settings, "HOST_PROC_PATH", str(tmp_path)) + monkeypatch.setattr(gpu_service, "_detect_nvidia", lambda: []) + monkeypatch.setattr(gpu_service, "_detect_dri", lambda: []) + + (gpu,) = gpu_service.detect_gpus() + assert gpu.vendor == "nvidia" + assert gpu.name == "NVIDIA GeForce RTX 3060" + assert gpu.uuid == "GPU-1234-abcd" + assert gpu.index == 0 + assert gpu.driver == "550.120" + + +@pytest.fixture +def fake_container(monkeypatch): + from services import container_service + + monkeypatch.setattr( + container_service, + "inspect_container", + lambda container_id: {"id": container_id, "env": ["DB_PASSWORD=hunter2", "NOVALUE"]}, + ) + + +def test_container_env_values_are_hidden_from_the_read_only_role(as_user, fake_container): + env = as_user.get("/api/containers/abc").json()["env"] + assert env == ["DB_PASSWORD=••••••", "NOVALUE"] + + +def test_admins_still_see_the_environment(as_admin, fake_container): + assert as_admin.get("/api/containers/abc").json()["env"][0] == "DB_PASSWORD=hunter2" + + +def test_a_missing_container_is_a_404(as_admin, monkeypatch): + from docker_client import DockerError + from services import container_service + + def missing(container_id): + raise DockerError("not_found", "No such container") + + monkeypatch.setattr(container_service, "inspect_container", missing) + assert as_admin.get("/api/containers/nope").status_code == 404 + + +@pytest.mark.parametrize("branch", ["-x", "--upload-pack=sh", "a..b", "x.lock", ""]) +def test_branch_names_that_git_would_read_as_options_are_refused(branch): + from services import git_service + + with pytest.raises(git_service.GitError): + git_service.validate_branch(branch) + + +@pytest.mark.parametrize("url", ["-oProxyCommand=x", "ext::sh -c id", "fd::3", ""]) +def test_repository_urls_that_run_commands_are_refused(url): + from services import git_service + + with pytest.raises(git_service.GitError): + git_service.validate_url(url) + + +def test_ordinary_branches_and_urls_pass(): + from services import git_service + + assert git_service.validate_branch("release/1.2") == "release/1.2" + assert git_service.validate_url("git@forge:me/stack.git") == "git@forge:me/stack.git" + + +def test_scan_all_in_the_background_is_active_before_the_request_returns(monkeypatch, db): + from services import scan_service + + release = asyncio.Event() + + async def slow_scan(session, image): + await release.wait() + + class Row: + error = None + + return Row() + + monkeypatch.setattr(scan_service, "scan", slow_scan) + + async def scenario(): + status = scan_service.start_sweep(["a", "b"]) + assert status["active"] is True and status["total"] == 2 + with pytest.raises(scan_service.ScanError): + scan_service.start_sweep(["c"]) + release.set() + for _ in range(100): + if not scan_service.sweep_status()["active"]: + break + await asyncio.sleep(0.01) + final = scan_service.sweep_status() + assert final["active"] is False + assert final["scanned"] == 2 and final["failed"] == 0 + + asyncio.run(scenario()) + + +def test_setup_refuses_an_empty_account(client, db, monkeypatch): + import auth as auth_mod + + monkeypatch.setattr(auth_mod, "users_exist", lambda session: False) + response = client.post("/api/auth/setup", json={"username": " ", "password": ""}) + assert response.status_code == 400 diff --git a/backend/tests/test_stack_delete.py b/backend/tests/test_stack_delete.py new file mode 100644 index 0000000..b7b9139 --- /dev/null +++ b/backend/tests/test_stack_delete.py @@ -0,0 +1,186 @@ +"""Deleting a stack: what goes with it, and what must not. + +Before 0.67.1 a deleted stack left its Git source behind — and the poll loop or +the next push webhook recreated the folder from the repository and started the +stack again. Its auto-update policy was inherited by the next stack of the same +name, and every backup schedule naming it failed from then on. Deleting the +folder also walked into whatever was mounted inside it (a NAS share at +``./media``) and deleted that too. +""" +from __future__ import annotations + +import asyncio +import json +import os + +import pytest +from sqlmodel import Session, delete, select + + +@pytest.fixture(autouse=True) +def clean(db): + from database import engine + from models.auto_update import AutoUpdate + from models.backup_schedule import BackupSchedule + from models.git_source import GitSource + from models.runtime_state import StackLock + + def wipe(): + with Session(engine) as session: + for model in (GitSource, AutoUpdate, BackupSchedule, StackLock): + session.exec(delete(model)) + session.commit() + + wipe() + yield + wipe() + + +def _schedule(session, stack_ids, all_stacks=False): + from models.backup_schedule import BackupSchedule, set_stacks + + schedule = BackupSchedule(destination_id=1, all_stacks=all_stacks) + set_stacks(schedule, stack_ids) + session.add(schedule) + session.commit() + session.refresh(schedule) + return schedule.id + + +def test_forget_stack_drops_what_refers_to_it(db): + from database import engine + from models.auto_update import AutoUpdate + from models.backup_schedule import BackupSchedule, stacks_of + from models.git_source import GitSource + from services import stack_cleanup_service + + with Session(engine) as session: + session.add(GitSource(stack_id="gone", url="https://example.invalid/x.git")) + session.add(AutoUpdate(stack_id="gone")) + session.add(AutoUpdate(stack_id="stays")) + both = _schedule(session, ["gone", "stays"]) + only = _schedule(session, ["gone"]) + everything = _schedule(session, [], all_stacks=True) + + report = stack_cleanup_service.forget_stack(session, "gone") + + assert report["git"] is True and report["auto_update"] == 1 + assert session.exec(select(GitSource)).all() == [] + assert [p.stack_id for p in session.exec(select(AutoUpdate)).all()] == ["stays"] + assert stacks_of(session.get(BackupSchedule, both)) == ["stays"] + lonely = session.get(BackupSchedule, only) + assert stacks_of(lonely) == [] and lonely.enabled is False + assert "was deleted" in lonely.last_status + assert session.get(BackupSchedule, everything).enabled is True + + +def test_orphans_from_older_versions_are_forgotten_at_startup(db): + from database import engine + from models.git_source import GitSource + from models.stack import Stack + from services import stack_cleanup_service + + with Session(engine) as session: + if not session.get(Stack, "alive"): + session.add(Stack(id="alive", name="alive")) + session.add(GitSource(stack_id="alive", url="https://example.invalid/a.git")) + session.add(GitSource(stack_id="long-deleted", url="https://example.invalid/b.git")) + session.commit() + + assert stack_cleanup_service.prune_orphans(session) == ["long-deleted"] + assert [g.stack_id for g in session.exec(select(GitSource)).all()] == ["alive"] + + +def test_git_never_syncs_a_stack_that_no_longer_exists(db): + from database import engine + from models.git_source import GitSource + from services import git_service + + with Session(engine) as session: + source = GitSource(stack_id="ghost", url="https://example.invalid/g.git") + session.add(source) + session.commit() + session.refresh(source) + with pytest.raises(git_service.GitError, match="no longer exists"): + asyncio.run(git_service.sync(session, source)) + + +def test_deleting_a_busy_stack_is_refused(as_admin, db): + from database import engine + from models.stack import Stack + from services import stack_lock_service + + with Session(engine) as session: + if not session.get(Stack, "busy-del"): + session.add(Stack(id="busy-del", name="busy")) + session.commit() + stack_lock_service.acquire(session, "busy-del", "backup", "scheduled") + + response = as_admin.delete("/api/stacks/busy-del") + assert response.status_code == 409 + assert "backup" in response.json()["detail"] + + +def test_folder_removal_stops_at_mount_points(tmp_path, monkeypatch): + from services import compose_service + + stack = tmp_path / "media-stack" + (stack / "config").mkdir(parents=True) + (stack / "config" / "app.db").write_text("x") + (stack / "compose.yaml").write_text("services: {}\n") + share = stack / "media" + share.mkdir() + (share / "movie.mkv").write_text("precious") + + monkeypatch.setattr(compose_service, "_mount_points", lambda: {str(share)}) + kept = compose_service.delete_stack_files("media-stack", override=str(tmp_path)) + + assert kept == [str(share)] + assert (share / "movie.mkv").read_text() == "precious" + assert not (stack / "config").exists() and not (stack / "compose.yaml").exists() + + +def test_cloning_copies_links_as_links_and_skips_sockets(tmp_path): + import socket + + from services import compose_service + + src = tmp_path / "orig" + src.mkdir() + (src / "compose.yaml").write_text("services: {}\n") + os.symlink("/definitely/not/there", src / "dangling") + sock = socket.socket(socket.AF_UNIX) + sock.bind(str(src / "app.sock")) + try: + compose_service.clone_stack_files("orig", "copy", override=str(tmp_path)) + finally: + sock.close() + + dst = tmp_path / "copy" + assert (dst / "compose.yaml").exists() + assert os.readlink(dst / "dangling") == "/definitely/not/there" + assert not os.path.lexists(dst / "app.sock") + + +def test_a_deleted_stack_name_does_not_inherit_its_schedules(as_admin, db, monkeypatch): + """End to end through the API, with Docker stubbed out.""" + from database import engine + from models.backup_schedule import BackupSchedule, stacks_of + from services import compose_service + + async def down(stack_id, override=None): + return {"returncode": 0, "stdout": "", "stderr": ""} + + monkeypatch.setattr(compose_service, "down", down) + assert as_admin.post("/api/stacks", json={"name": "del-me"}).status_code == 201 + with Session(engine) as session: + schedule_id = _schedule(session, ["del-me"]) + + response = as_admin.delete("/api/stacks/del-me") + assert response.status_code == 200, response.text + assert response.json()["kept"] == [] + with Session(engine) as session: + schedule = session.get(BackupSchedule, schedule_id) + assert stacks_of(schedule) == [] and not schedule.enabled + assert not os.path.isdir(compose_service.stack_dir("del-me")) + assert json.loads(as_admin.get("/api/stacks").text) is not None diff --git a/backend/tests/test_walkthrough_fixes.py b/backend/tests/test_walkthrough_fixes.py new file mode 100644 index 0000000..70aa546 --- /dev/null +++ b/backend/tests/test_walkthrough_fixes.py @@ -0,0 +1,242 @@ +"""Found clicking through the UI for 0.67.1. + +* The file browser opens at ``/``, which the default roots (/mnt, /opt, …) + do not include: the page opened on "outside the allowed browse roots" and + offered no way into any root. Paths above the roots now list the way to them. +* A root could not be deleted, but it could be moved — the whole tree with it. +* Locally built images (compose ``build:``) were checked against Docker Hub, + which answers 401 for a repository that does not exist: "needs credentials". +* Volumes, networks and images of a stack whose folder name compose had to + normalise ("Nextcloud" -> project "nextcloud") were grouped as leftovers of a + removed stack. +""" +from __future__ import annotations + +import asyncio + +import pytest + + +@pytest.fixture +def roots(tmp_path, monkeypatch): + from config import settings + + (tmp_path / "srv" / "data").mkdir(parents=True) + (tmp_path / "opt").mkdir() + monkeypatch.setattr(settings, "HOST_ROOT_PREFIX", str(tmp_path), raising=False) + monkeypatch.setattr( + settings, "ALLOWED_BROWSE_ROOTS", ["/opt", "/srv/data", "/media"], raising=False + ) + return tmp_path + + +def test_the_top_of_the_tree_lists_the_way_to_the_roots(roots): + from services import device_service + + top = device_service.browse("/") + assert top["virtual"] is True + # /media does not exist on this host, so it is not offered. + assert [e["name"] for e in top["entries"]] == ["opt", "srv"] + assert [e["name"] for e in device_service.browse("/srv")["entries"]] == ["data"] + inside = device_service.browse("/srv/data") + assert "virtual" not in inside and inside["parent"] == "/srv" + + +def test_a_path_leading_nowhere_is_still_refused(roots): + from services import device_service + + with pytest.raises(device_service.BrowseError): + device_service.browse("/etc") + + +def test_a_root_cannot_be_moved_or_renamed(roots): + from services import file_service + + with pytest.raises(file_service.BrowseError, match="browse root"): + file_service.move("/srv/data", "/opt") + with pytest.raises(file_service.BrowseError, match="browse root"): + file_service.rename("/opt", "opt2") + assert (roots / "srv" / "data").is_dir() + + +def test_moving_inside_a_root_still_works(roots): + from services import file_service + + (roots / "opt" / "a.txt").write_text("x") + file_service.move("/opt/a.txt", "/srv/data") + assert (roots / "srv" / "data" / "a.txt").read_text() == "x" + + +@pytest.mark.parametrize( + "attrs, local", + [ + ({"Identity": {"Build": [{"Ref": "x"}]}, "RepoDigests": ["app@sha256:1"]}, True), + ({"Identity": {"Pull": [{"Repository": "docker.io/library/nginx"}]}}, False), + ({"RepoDigests": []}, True), # older daemon, never pulled + ({"RepoDigests": ["nginx@sha256:1"]}, False), + ({"Identity": {}, "RepoDigests": ["nginx@sha256:1"]}, False), + (None, False), + ], +) +def test_where_an_image_came_from(attrs, local): + from services import update_service + + assert update_service._built_locally(attrs) is local + + +def test_a_local_build_is_not_asked_about_at_the_registry(monkeypatch): + from services import update_service + + monkeypatch.setattr( + update_service, "_local_attrs", lambda image: {"Identity": {"Build": [{"Ref": "x"}]}} + ) + + async def no_registry(image): + raise AssertionError("registry contacted for a local build") + + monkeypatch.setattr(update_service, "remote_digest", no_registry) + status = asyncio.run(update_service.check_image("myapp-web:latest")) + assert status.local is True and status.error is None and not status.update_available + + +def test_resources_of_a_normalised_stack_are_grouped_under_it(as_admin, db, monkeypatch): + from sqlmodel import Session + + from database import engine + from models.stack import Stack + from services import network_service, volume_service + + with Session(engine) as session: + if not session.get(Stack, "Nextcloud"): + session.add(Stack(id="Nextcloud", name="Nextcloud")) + session.commit() + monkeypatch.setattr( + volume_service, "list_volumes", lambda: [{"name": "nextcloud_db", "stack": "nextcloud"}] + ) + monkeypatch.setattr( + network_service, + "list_networks", + lambda: [{"name": "x", "stack": "elsewhere"}, {"name": "bridge", "stack": None}], + ) + assert as_admin.get("/api/volumes").json()[0]["stack"] == "Nextcloud" + assert [n["stack"] for n in as_admin.get("/api/networks").json()] == ["elsewhere", None] + + +class _Container: + def __init__(self, mounts): + self.attrs = {"Mounts": mounts} + + +@pytest.mark.parametrize( + "mounts, host", + [ + # The usual: same path on both sides — no warning. + ([{"Type": "bind", "Source": "/opt/stacks", "Destination": "/opt/stacks"}], None), + # A real mismatch. + ( + [{"Type": "bind", "Source": "/srv/compose", "Destination": "/opt/stacks"}], + "/srv/compose", + ), + # The folder above STACKS_DIR is what is mounted. + ([{"Type": "bind", "Source": "/srv", "Destination": "/opt"}], "/srv/stacks"), + # Not mounted at all. + ([{"Type": "volume", "Source": "/var/lib/docker/volumes/x", "Destination": "/data"}], None), + ], +) +def test_the_host_stacks_path_comes_from_docker_not_mountinfo(monkeypatch, mounts, host): + """mountinfo's root field is relative to the filesystem: a stacks folder on + its own disk or btrfs subvolume showed a mismatch that was not there.""" + from config import settings + from services import self_update_service, stack_assets_service + + monkeypatch.setattr(settings, "STACKS_DIR", "/opt/stacks") + monkeypatch.setattr(stack_assets_service, "_HOST_STACKS_DIR", None) + monkeypatch.setattr(self_update_service, "_own_container", lambda: _Container(mounts)) + mismatch = stack_assets_service.stacks_path_mismatch() + assert (mismatch["host"] if mismatch else None) == host + + +def test_backup_names_carry_the_time_on_the_users_clock(monkeypatch): + """Named in UTC, a backup taken at 12:21 in Berlin read as …-102154.""" + from datetime import datetime, timezone + + from services import backup_service, compose_service + + monkeypatch.setattr( + compose_service, "now", lambda: datetime(2026, 10, 15, 10, 21, 54, tzinfo=timezone.utc) + ) + berlin = backup_service.backup_filename("app", True, tz="Europe/Berlin") + assert berlin == "backup-app-full-20261015-122154.tar.gz" + # No zone, or one that does not exist: UTC, as before. + assert backup_service.backup_filename("app", False).endswith("-20261015-102154.tar.gz") + assert backup_service.backup_filename("app", False, tz="Mars/Olympus").endswith("-102154.tar.gz") + # Retention still recognises the files. + assert backup_service.backup_name_pattern("app").match(berlin) + + +COMPOSE_WITH_SECRETS = """services: + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: hunter2 + POSTGRES_USER: app + API_KEY: ${API_KEY:-changeme} + DB_PASS: ${DB_PASS} + KEYCLOAK_ADMIN: admin + app: + environment: + - "TS_AUTHKEY=tskey-123" + - TZ=Europe/Berlin +""" + + +def test_inline_credentials_are_masked_and_the_rest_left_alone(): + from services.compose_service import MASK, mask_secrets + + masked = mask_secrets(COMPOSE_WITH_SECRETS) + assert "hunter2" not in masked and "tskey-123" not in masked and "changeme" not in masked + assert f"POSTGRES_PASSWORD: {MASK}" in masked + assert f'- "TS_AUTHKEY={MASK}"' in masked + # A bare variable is no secret; names that merely contain "key" are not either. + for kept in ("DB_PASS: ${DB_PASS}", "KEYCLOAK_ADMIN: admin", "POSTGRES_USER: app", "TZ=Europe/Berlin"): + assert kept in masked + assert masked.count("\n") == COMPOSE_WITH_SECRETS.count("\n") + + +def test_the_read_only_role_gets_the_masked_compose_file(as_admin, as_user, db): + from services import compose_service + + assert as_admin.post("/api/stacks", json={"name": "masked", "yaml": COMPOSE_WITH_SECRETS}).status_code == 201 + try: + assert "hunter2" in as_admin.get("/api/stacks/masked").json()["yaml"] + assert "hunter2" not in as_user.get("/api/stacks/masked").json()["yaml"] + finally: + compose_service.delete_stack_files("masked") + + +def test_scan_all_covers_only_stackpilots_own_stacks(as_admin, db, monkeypatch): + from sqlmodel import Session + + from database import engine + from models.stack import Stack + from services import image_service, scan_service + + with Session(engine) as session: + if not session.get(Stack, "Mine"): + session.add(Stack(id="Mine", name="Mine")) + session.commit() + monkeypatch.setattr( + image_service, + "list_images", + lambda: [ + {"tag": "mine:1", "stacks": ["mine"]}, + {"tag": "theirs:1", "stacks": ["some-other-project"]}, + {"tag": "loose:1", "stacks": []}, + ], + ) + swept = [] + monkeypatch.setattr( + scan_service, "start_sweep", lambda images, on_done=None: swept.append(images) or {"active": True} + ) + assert as_admin.post("/api/images/scan-all?background=true").status_code == 202 + assert swept == [["mine:1"]] diff --git a/backend/version.py b/backend/version.py index 1f3b5c5..5cc1588 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1,3 +1,3 @@ """Single source of truth for the StackPilot release version.""" -APP_VERSION = "0.67.0" +APP_VERSION = "0.67.1" diff --git a/docker-compose.yml b/docker-compose.yml index 08f2ccc..ac8f847 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,10 @@ services: image: git.menzel.center/menzeljonas/stackpilot-frontend:latest build: ./frontend restart: unless-stopped + # Started once the API answers: before, nginx answered 502 for the first + # seconds after every (re)start. depends_on: - - backend + backend: + condition: service_healthy ports: - "5009:80" diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 05b1a88..9779e68 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,7 +1,10 @@ FROM node:20-alpine AS build WORKDIR /app -COPY package.json ./ -RUN npm install +# The lockfile, and `npm ci`: `npm install` against package.json alone +# resolved whatever the caret ranges allowed on the day of the build, so the +# image could ship dependency versions CI never typechecked or tested. +COPY package.json package-lock.json ./ +RUN npm ci COPY . . RUN npm run build diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 3e419b9..1680f05 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,3 +1,11 @@ +# Behind a TLS-terminating proxy (Traefik, NPM, Caddy) this nginx itself only +# sees plain http: passing $scheme on would tell the backend "http" for an +# https site. Keep what the outer proxy said, and fall back to our own scheme. +map $http_x_forwarded_proto $sp_forwarded_proto { + default $http_x_forwarded_proto; + "" $scheme; +} + server { listen 80; server_name _; @@ -12,17 +20,24 @@ server { # API proxy location /api/ { proxy_pass http://backend:5008; - proxy_set_header Host $host; + # $http_host, not $host: $host drops the port, so with the default + # "5009:80" mapping the OIDC redirect URI the backend derives from it + # pointed at port 80. + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # Without this uvicorn sees plain http even behind TLS, which would make # the OIDC redirect URI it derives wrong and the refresh cookie miss its # Secure flag. - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $sp_forwarded_proto; proxy_read_timeout 600s; # Stack backups (incl. volume data) can be large in both directions. client_max_body_size 0; proxy_request_buffering off; + # A backup download carries its report in a response header; past the + # default 4-8k header buffer nginx answers 502 for a backup that worked. + proxy_buffer_size 16k; + proxy_buffers 8 16k; } # WebSocket proxy @@ -31,7 +46,10 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $sp_forwarded_proto; proxy_read_timeout 86400s; } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index edf9d9a..b29f0ad 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "stackpilot-frontend", - "version": "0.49.0", + "version": "0.67.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "stackpilot-frontend", - "version": "0.49.0", + "version": "0.67.0", "dependencies": { "@fontsource-variable/schibsted-grotesk": "^5.2.8", "@monaco-editor/react": "^4.6.0", @@ -16,6 +16,7 @@ "axios": "^1.7.9", "clsx": "^2.1.1", "lucide-react": "^0.468.0", + "monaco-editor": "^0.57.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.28.0", @@ -618,9 +619,9 @@ } }, "node_modules/@remix-run/router": { - "version": "1.23.3", - "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.3.tgz", - "integrity": "sha512-4An71tdz9X8+3sI4Qqqd2LWd9vS39J7sqd9EU4Scw7TJE/qB10Flv/UuqbPVgfQV9XoK8Np6jNquZitnZq5i+Q==", + "version": "1.23.4", + "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.4.tgz", + "integrity": "sha512-q7j5geK7xs3UJSdm9/iytUNclBnLmYx1EnSeCFXHPeutdqgIMeFeHtUZgS3EhlKxdBEAu8OwtJCwmLrEzpSs7Q==", "license": "MIT", "engines": { "node": ">=14.0.0" @@ -1145,8 +1146,7 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true, - "peer": true + "optional": true }, "node_modules/@vitejs/plugin-react": { "version": "4.7.0", @@ -1170,14 +1170,14 @@ } }, "node_modules/@vitest/expect": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.8.tgz", - "integrity": "sha512-8ytZ/fFHq2g4PJVAtDX57mayemKgDR6X3Oa2Foro+EygiOJHUXhCqBAAKQYYajZpFoIfvBCF1j6R6IYRSIUFuw==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", + "integrity": "sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "2.1.8", - "@vitest/utils": "2.1.8", + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", "chai": "^5.1.2", "tinyrainbow": "^1.2.0" }, @@ -1186,13 +1186,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.8.tgz", - "integrity": "sha512-7guJ/47I6uqfttp33mgo6ga5Gr1VnL58rcqYKyShoRK9ebu8T5Rs6HN3s1NABiBeVTdWNrwUMcHH54uXZBN4zA==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.9.tgz", + "integrity": "sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "2.1.8", + "@vitest/spy": "2.1.9", "estree-walker": "^3.0.3", "magic-string": "^0.30.12" }, @@ -1226,13 +1226,13 @@ } }, "node_modules/@vitest/runner": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.8.tgz", - "integrity": "sha512-17ub8vQstRnRlIU5k50bG+QOMLHRhYPAna5tw8tYbj+jzjcspnwnwtPtiOlkuKC4+ixDPTuLZiqiWWQ2PSXHVg==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.9.tgz", + "integrity": "sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "2.1.8", + "@vitest/utils": "2.1.9", "pathe": "^1.1.2" }, "funding": { @@ -1240,13 +1240,13 @@ } }, "node_modules/@vitest/snapshot": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.8.tgz", - "integrity": "sha512-20T7xRFbmnkfcmgVEz+z3AU/3b0cEzZOt/zmnvZEctg64/QZbSDJEVm9fLnnlSi74KibmRsO9/Qabi+t0vCRPg==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.9.tgz", + "integrity": "sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.1.8", + "@vitest/pretty-format": "2.1.9", "magic-string": "^0.30.12", "pathe": "^1.1.2" }, @@ -1254,23 +1254,10 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/snapshot/node_modules/@vitest/pretty-format": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.8.tgz", - "integrity": "sha512-9HiSZ9zpqNLKlbIDRWOnAWqgcA7xu+8YxXSekhr0Ykab7PAYFkhkwoqVArPOtJhPmYeE2YHgKZlj3CP36z2AJQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^1.2.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/@vitest/spy": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.8.tgz", - "integrity": "sha512-5swjf2q95gXeYPevtW0BLk6H8+bPlMb4Vw/9Em4hFxDcaOxS+e0LOX4yqNxoHzMR2akEB2xfpnWUzkZokmgWDg==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.9.tgz", + "integrity": "sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1281,13 +1268,13 @@ } }, "node_modules/@vitest/utils": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.8.tgz", - "integrity": "sha512-dwSoui6djdwbfFmIgbIjX2ZhIoG7Ex/+xpxyiEgIGzjliY8xGkcpITKTlp6B4MgtGkF2ilvm97cPM96XZaAgcA==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.9.tgz", + "integrity": "sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.1.8", + "@vitest/pretty-format": "2.1.9", "loupe": "^3.1.2", "tinyrainbow": "^1.2.0" }, @@ -1295,19 +1282,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/utils/node_modules/@vitest/pretty-format": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.8.tgz", - "integrity": "sha512-9HiSZ9zpqNLKlbIDRWOnAWqgcA7xu+8YxXSekhr0Ykab7PAYFkhkwoqVArPOtJhPmYeE2YHgKZlj3CP36z2AJQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^1.2.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/@xterm/addon-fit": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.10.0.tgz", @@ -1453,21 +1427,21 @@ } }, "node_modules/axios": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.17.0.tgz", - "integrity": "sha512-J8SwNxprqqpbfenehxWYXE7CW+wM1BB4w3+N+g+/Wx40xM4rsLrfPmHHxSWIxJLYDgSY/HqlFPIYb2/S3rxafw==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", + "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.16.0", - "form-data": "^4.0.5", + "form-data": "^4.0.6", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, "node_modules/baseline-browser-mapping": { - "version": "2.10.34", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.34.tgz", - "integrity": "sha512-IMDedajPifLnHNY0X9n8hKxRTQ6/eTHwr5bDo04WnuqxyKw6LYtQywCuuqPZwhl3aBXMvQpJov42GLCwRRdQzw==", + "version": "2.11.28", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.28.tgz", + "integrity": "sha512-br0LOmjJqr6Rt4AEkadbcNf67Zpo7EPwKsWbjnRrJIa1QYFHQBqM4C7F2WTbV8ahTEcg1Eo+Y0aWjjORInYFyA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1504,9 +1478,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "version": "4.29.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", + "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", "dev": true, "funding": [ { @@ -1524,11 +1498,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.10.12", - "caniuse-lite": "^1.0.30001782", - "electron-to-chromium": "^1.5.328", - "node-releases": "^2.0.36", - "update-browserslist-db": "^1.2.3" + "baseline-browser-mapping": "^2.11.26", + "caniuse-lite": "^1.0.30001813", + "electron-to-chromium": "^1.5.439", + "node-releases": "^2.0.57", + "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" @@ -1571,9 +1545,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001797", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001797.tgz", - "integrity": "sha512-l8xKG+gwAIExZGl9FrF7KUwuOmk6wbEPC9Xoy/RtnWv1XG0Q4LFlagaLpUv3Kiza3W/wm27zy0yWJEieYKAP6w==", + "version": "1.0.30001816", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001816.tgz", + "integrity": "sha512-+8Yx7BXCoqk6IF8yTicjfUR6D/ldJzdlsodlEOiOSPmhK844WIgk67/zIQaGAOxNkZyyvBlrRFnLvqKIoqbAFw==", "dev": true, "funding": [ { @@ -1861,11 +1835,10 @@ "license": "MIT" }, "node_modules/dompurify": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.7.tgz", - "integrity": "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==", + "version": "3.4.15", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.15.tgz", + "integrity": "sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw==", "license": "(MPL-2.0 OR Apache-2.0)", - "peer": true, "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -1885,9 +1858,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.368", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.368.tgz", - "integrity": "sha512-7RckJJK4uESJF9PxvfMWd3TGqIiieUTG4HxnKaKuIpGbcr+r2ZEB3g2gAhCP3Fqm42vJSzLfgab9eva/C4/XVw==", + "version": "1.5.456", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.456.tgz", + "integrity": "sha512-8hboqI6xesozteFb8aa7OGknvxUtTeRJLz0FoePysUEh/oWbddykIGdgpEJRbPRg/nIMfBf2DFos+H2MtvJl2A==", "dev": true, "license": "ISC" }, @@ -2099,16 +2072,16 @@ } }, "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" @@ -2595,7 +2568,6 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -2658,13 +2630,12 @@ } }, "node_modules/monaco-editor": { - "version": "0.55.1", - "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.55.1.tgz", - "integrity": "sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.57.0.tgz", + "integrity": "sha512-5BkI9KGoqrNvBGUe15/QlZq3OooZ8WLg1AxTpaqHRCP3HNpzPPZKE2EDz8M7c+VRmCeUw1Brp4cx/PWm3kI/5A==", "license": "MIT", - "peer": true, "dependencies": { - "dompurify": "3.2.7", + "dompurify": "3.4.15", "marked": "14.0.0" } }, @@ -2687,9 +2658,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", "dev": true, "funding": [ { @@ -2706,9 +2677,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.47", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.47.tgz", - "integrity": "sha512-Uzmd6LXpouKo8EUK68IjH4+E01w/hXyV3R3g/geCJo+rXLNfh1xucB+LOzYEOQPSiUK3h/xZf0cQGcSsmyL2Og==", + "version": "2.0.59", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.59.tgz", + "integrity": "sha512-5qr2okC4ItAI0hG5SaV0HoYMpnM/Y6+Oqr7AmPYy/2uEoGvS+GbSKLeRxSS1uAdu59yz0ouzcuXievSgg///WA==", "dev": true, "license": "MIT", "engines": { @@ -2830,9 +2801,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.30", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.30.tgz", + "integrity": "sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA==", "dev": true, "funding": [ { @@ -2850,9 +2821,9 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.20", "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" + "source-map-js": "^1.2.2" }, "engines": { "node": "^10 || ^12 || >=14" @@ -2972,9 +2943,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3103,12 +3074,12 @@ } }, "node_modules/react-router": { - "version": "6.30.4", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.4.tgz", - "integrity": "sha512-SVUsDe+DybHM/WmYKIVYhZh1o5Dcuf16yM6WjG02Q9XVFMZIJyHYhwrr6bFBXZkVP6z69kNkMyBCujt8FaFLJA==", + "version": "6.30.6", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.6.tgz", + "integrity": "sha512-5HfK7k5im7LTOB0EqCQmfvy4C13G92Ssj1VTmouTK3AJvyjKTnFuCV0vcMAD/JS+JC4DvDIBRrlAeJIFjh5VWg==", "license": "MIT", "dependencies": { - "@remix-run/router": "1.23.3" + "@remix-run/router": "1.23.4" }, "engines": { "node": ">=14.0.0" @@ -3118,13 +3089,13 @@ } }, "node_modules/react-router-dom": { - "version": "6.30.4", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.4.tgz", - "integrity": "sha512-q4HvNl+mmDdkS0g+MqiBZNteQJCuimWoOyHMy4T/RQLAn9Z29+E91QXRaxOujeMl2HTzRSS0KFPd7lxX3PjV0Q==", + "version": "6.30.6", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.6.tgz", + "integrity": "sha512-0RHKZz7wwffvkU+2MFVT2NnjK44ssLEV+m0CAJaS2Ksmorrwj7WxH00jO0SOCW26/tINUnJHToXblDs33I38YQ==", "license": "MIT", "dependencies": { - "@remix-run/router": "1.23.3", - "react-router": "6.30.4" + "@remix-run/router": "1.23.4", + "react-router": "6.30.6" }, "engines": { "node": ">=14.0.0" @@ -3337,9 +3308,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -3673,9 +3644,9 @@ "license": "MIT" }, "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.4.tgz", + "integrity": "sha512-t8wonIOdQyOjtPd2VktlBu6QjTVTks6v8bSi7kd0r2Vll8TK1JYDZtRRdbmCHOAMphkZgrzK+L+vXK4pyZ8t4A==", "dev": true, "funding": [ { @@ -3771,9 +3742,9 @@ } }, "node_modules/vite-node": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.8.tgz", - "integrity": "sha512-uPAwSr57kYjAUux+8E2j0q0Fxpn8M9VoyfGiRI8Kfktz9NcYMCenwY5RnZxnF1WTu3TGiYipirIzacLL3VVGFg==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.9.tgz", + "integrity": "sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==", "dev": true, "license": "MIT", "dependencies": { @@ -3794,19 +3765,19 @@ } }, "node_modules/vitest": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.8.tgz", - "integrity": "sha512-1vBKTZskHw/aosXqQUlVWWlGUxSJR8YtiyZDJAFeW2kPAeX6S3Sool0mjspO+kXLuxVWlEDDowBAeqeAQefqLQ==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.9.tgz", + "integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "2.1.8", - "@vitest/mocker": "2.1.8", - "@vitest/pretty-format": "^2.1.8", - "@vitest/runner": "2.1.8", - "@vitest/snapshot": "2.1.8", - "@vitest/spy": "2.1.8", - "@vitest/utils": "2.1.8", + "@vitest/expect": "2.1.9", + "@vitest/mocker": "2.1.9", + "@vitest/pretty-format": "^2.1.9", + "@vitest/runner": "2.1.9", + "@vitest/snapshot": "2.1.9", + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", "chai": "^5.1.2", "debug": "^4.3.7", "expect-type": "^1.1.0", @@ -3818,7 +3789,7 @@ "tinypool": "^1.0.1", "tinyrainbow": "^1.2.0", "vite": "^5.0.0", - "vite-node": "2.1.8", + "vite-node": "2.1.9", "why-is-node-running": "^2.3.0" }, "bin": { @@ -3833,8 +3804,8 @@ "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "2.1.8", - "@vitest/ui": "2.1.8", + "@vitest/browser": "2.1.9", + "@vitest/ui": "2.1.9", "happy-dom": "*", "jsdom": "*" }, diff --git a/frontend/package.json b/frontend/package.json index deca6d3..c199f5e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "stackpilot-frontend", "private": true, - "version": "0.67.0", + "version": "0.67.1", "type": "module", "scripts": { "dev": "vite", @@ -18,6 +18,7 @@ "axios": "^1.7.9", "clsx": "^2.1.1", "lucide-react": "^0.468.0", + "monaco-editor": "^0.57.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.28.0", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b12b3c9..6d527c2 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,5 +1,12 @@ import { lazy, useEffect } from "react"; -import { BrowserRouter, Navigate, Outlet, Route, Routes } from "react-router-dom"; +import { + Navigate, + Outlet, + Route, + RouterProvider, + createBrowserRouter, + createRoutesFromElements, +} from "react-router-dom"; import { AppShell } from "@/components/layout/AppShell"; import { Login } from "@/pages/Login"; import { Dashboard } from "@/pages/Dashboard"; @@ -46,17 +53,52 @@ function RequireAuth() { } function BootSplash() { + // Set while the backend does not answer yet (starting, restarting after an + // update) — the session is kept and the restore keeps trying. + const waiting = useAuthStore((s) => s.waiting); return ( -
+
Restoring your session… ); } +// A data router rather than : only it supports useBlocker, which +// is what keeps the stack editor from dropping unsaved edits on a stray click. +const router = createBrowserRouter( + createRoutesFromElements( + <> + } /> + }> + }> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + } /> + + ) +); + export default function App() { const applyTheme = useThemeStore((s) => s.apply); const restore = useAuthStore((s) => s.restore); @@ -70,28 +112,5 @@ export default function App() { restore(); }, [restore]); - return ( - - - } /> - }> - }> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - - - } /> - - - ); + return ; } diff --git a/frontend/src/api/backups.ts b/frontend/src/api/backups.ts index bc6997d..b4756f7 100644 --- a/frontend/src/api/backups.ts +++ b/frontend/src/api/backups.ts @@ -1,4 +1,4 @@ -import api from "./client"; +import api, { withBlobError } from "./client"; export interface BackupDestination { id: number; @@ -83,7 +83,17 @@ export interface BackupOptions { } /** Query params for the backup endpoints. */ -export function backupParams(opts: BackupOptions) { +export /** The browser's time zone: the server names backup files in it, so the time + * in the name is the one on the user's clock rather than UTC. */ +function browserZone(): string | undefined { + try { + return Intl.DateTimeFormat().resolvedOptions().timeZone || undefined; + } catch { + return undefined; + } +} + +function backupParams(opts: BackupOptions) { return { include_volumes: opts.includeVolumes, include_binds: opts.includeBinds ?? true, @@ -128,10 +138,12 @@ export const backupsApi = { .then((r) => r.data), download: async (stackId: string, opts: BackupOptions) => { - const res = await api.get(`/api/stacks/${stackId}/backup`, { - params: backupParams(opts), - responseType: "blob", - }); + const res = await withBlobError( + api.get(`/api/stacks/${stackId}/backup`, { + params: { ...backupParams(opts), tz: browserZone() }, + responseType: "blob", + }) + ); const cd = res.headers["content-disposition"] as string | undefined; const match = cd?.match(/filename="?([^"]+)"?/); const name = match?.[1] ?? `backup-${stackId}.tar.gz`; @@ -175,7 +187,7 @@ export const backupsApi = { destination: string; name: string; report?: BackupReport; - }>(`/api/stacks/${stackId}/backup/push`, body) + }>(`/api/stacks/${stackId}/backup/push`, { ...body, tz: browserZone() }) .then((r) => r.data), restoreFrom: (body: { @@ -199,7 +211,8 @@ export const destinationsApi = { api.delete(`/api/backups/destinations/${id}`).then((r) => r.data), test: (id: number) => api - .post<{ ok: boolean; error?: string }>(`/api/backups/destinations/${id}/test`) + // host_key: the SFTP server's pinned key, as a SHA256 fingerprint. + .post<{ ok: boolean; error?: string; host_key?: string }>(`/api/backups/destinations/${id}/test`) .then((r) => r.data), backups: (id: number) => api.get(`/api/backups/destinations/${id}/backups`).then((r) => r.data), diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 8d7a859..36bde92 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -2,7 +2,14 @@ import axios, { AxiosError } from "axios"; import { useAuthStore } from "@/store/auth"; // withCredentials so the httpOnly refresh cookie rides along on /api/auth/refresh. -const api = axios.create({ baseURL: "/", withCredentials: true }); +const api = axios.create({ + baseURL: "/", + withCredentials: true, + // Arrays as `binds=a&binds=b`, which is what FastAPI reads as a list. Axios' + // default `binds[]=a` is a different parameter name to it, so the folders and + // volumes picked in the backup dialog were silently ignored on download. + paramsSerializer: { indexes: null }, +}); api.interceptors.request.use((config) => { const token = useAuthStore.getState().accessToken; @@ -40,6 +47,29 @@ api.interceptors.response.use( } ); +/** + * The server's JSON error body behind a `responseType: "blob"` request (a + * backup download), which axios hands back as a Blob. Without this such a + * failure read "Request failed with status code 409" instead of saying the + * stack was busy. + */ +export async function withBlobError(request: Promise): Promise { + try { + return await request; + } catch (err) { + const e = err as AxiosError; + const data = e?.response?.data; + if (e?.response && typeof Blob !== "undefined" && data instanceof Blob) { + try { + e.response.data = JSON.parse(await data.text()); + } catch { + /* not JSON — keep the generic message */ + } + } + throw err; + } +} + export function apiErrorMessage(err: unknown): string { const e = err as AxiosError; const detail = e?.response?.data?.detail; diff --git a/frontend/src/api/editor.ts b/frontend/src/api/editor.ts index 4fa6cb1..7a638b0 100644 --- a/frontend/src/api/editor.ts +++ b/frontend/src/api/editor.ts @@ -1,9 +1,15 @@ import api from "./client"; export const editorApi = { - validate: (yaml: string, env: string) => + /** With `stackId`, relative paths (env_file, include, secrets) resolve in + * that stack's folder, as they will when it is deployed. */ + validate: (yaml: string, env: string, stackId?: string) => api - .post<{ ok: boolean; errors: string }>("/api/editor/validate", { yaml, env }) + .post<{ ok: boolean; errors: string }>("/api/editor/validate", { + yaml, + env, + stack_id: stackId ?? null, + }) .then((r) => r.data), services: (yaml: string) => api diff --git a/frontend/src/api/files.ts b/frontend/src/api/files.ts index 1c1963d..bc16136 100644 --- a/frontend/src/api/files.ts +++ b/frontend/src/api/files.ts @@ -1,4 +1,4 @@ -import api from "./client"; +import api, { withBlobError } from "./client"; import type { HostPathResult } from "@/types"; export interface FileContent { @@ -61,13 +61,15 @@ export const filesApi = { filename: string, onProgress?: (loaded: number, total: number | undefined) => void, ) => { - const res = await api.get(`${base}/download`, { - params: { path }, - responseType: "blob", - onDownloadProgress: onProgress - ? (e) => onProgress(e.loaded, e.total || undefined) - : undefined, - }); + const res = await withBlobError( + api.get(`${base}/download`, { + params: { path }, + responseType: "blob", + onDownloadProgress: onProgress + ? (e) => onProgress(e.loaded, e.total || undefined) + : undefined, + }) + ); triggerDownload(res.data as Blob, filename); }, diff --git a/frontend/src/api/images.ts b/frontend/src/api/images.ts index 35c7628..28db1e5 100644 --- a/frontend/src/api/images.ts +++ b/frontend/src/api/images.ts @@ -8,6 +8,8 @@ export interface UpdateStatus { remote_digest: string | null; checked_at: number; error: string | null; + /** Built on this host: there is no registry copy to compare against. */ + local?: boolean; } export interface ImageRow { @@ -53,6 +55,17 @@ export interface ScanDetail extends ScanSummary { findings: Finding[]; } +export interface ScanSweepStatus { + active: boolean; + done: number; + total: number; + /** Set once a sweep has finished. */ + scanned: number; + failed: number; + /** Increments per sweep, to tell the one just started from the last. */ + run: number; +} + const base = "/api/images"; export const imagesApi = { @@ -64,12 +77,13 @@ export const imagesApi = { api.get(`${base}/scan`, { params: { image } }).then((r) => r.data), scan: (image: string) => api.post(`${base}/scan`, { image }).then((r) => r.data), - scanAll: () => - api.post<{ scanned: number; failed: number }>(`${base}/scan-all`).then((r) => r.data), - scanStatus: () => + /** Starts the sweep on the server and returns at once; poll scanStatus(). + * Waiting on the response outlived the proxy's 600s timeout. */ + startScanAll: () => api - .get<{ active: boolean; done: number; total: number }>(`${base}/scan-status`) + .post(`${base}/scan-all`, null, { params: { background: true } }) .then((r) => r.data), + scanStatus: () => api.get(`${base}/scan-status`).then((r) => r.data), // Runs in the background on the server; poll pruneStatus() for progress. startPrune: (allUnused: boolean) => api diff --git a/frontend/src/api/stacks.ts b/frontend/src/api/stacks.ts index 1b53599..7c87015 100644 --- a/frontend/src/api/stacks.ts +++ b/frontend/src/api/stacks.ts @@ -33,8 +33,12 @@ export const stacksApi = { env?: string; } ) => api.put(`/api/stacks/${id}`, body).then((r) => r.data), + /** `kept` lists folders left in place because another filesystem (a share, + * a second disk) is mounted there. */ remove: (id: string, deleteFiles = true) => - api.delete(`/api/stacks/${id}?delete_files=${deleteFiles}`).then((r) => r.data), + api + .delete<{ ok: boolean; kept?: string[] }>(`/api/stacks/${id}?delete_files=${deleteFiles}`) + .then((r) => r.data), clone: (id: string, name: string) => api.post(`/api/stacks/${id}/clone`, { name }).then((r) => r.data), diff --git a/frontend/src/components/stacks/AutoUpdatePanel.tsx b/frontend/src/components/stacks/AutoUpdatePanel.tsx index d30aaab..d628ee5 100644 --- a/frontend/src/components/stacks/AutoUpdatePanel.tsx +++ b/frontend/src/components/stacks/AutoUpdatePanel.tsx @@ -4,6 +4,7 @@ import { toast } from "sonner"; import { Badge, Button, Card } from "@/components/ui"; import { autoUpdateApi, type AutoUpdatePolicy } from "@/api/autoUpdate"; import { apiErrorMessage } from "@/api/client"; +import { parseServerTime } from "@/lib/utils"; const STATUS_TONE: Record = { updated: "text-green-500", @@ -110,7 +111,7 @@ export function AutoUpdatePanel({ {p.last_run && (

- Last run {new Date(p.last_run).toLocaleString()} —{" "} + Last run {parseServerTime(p.last_run).toLocaleString()} —{" "} {p.last_status} diff --git a/frontend/src/components/stacks/ContainerTerminal.tsx b/frontend/src/components/stacks/ContainerTerminal.tsx index 395a5a3..e03fbfe 100644 --- a/frontend/src/components/stacks/ContainerTerminal.tsx +++ b/frontend/src/components/stacks/ContainerTerminal.tsx @@ -25,16 +25,20 @@ export function ContainerTerminal({ service: string; onClose: () => void; }) { - const token = useAuthStore((s) => s.accessToken); const [shell, setShell] = useState(SHELLS[0]); const [status, setStatus] = useState("connecting"); const [detail, setDetail] = useState(null); const boxRef = useRef(null); + // Keyed on the container and shell only. With the access token as a + // dependency, every background token refresh tore the session down and + // opened a new shell — whatever was running in the terminal was gone. useEffect(() => { - if (!token || !boxRef.current) return; + if (!boxRef.current) return; setStatus("connecting"); setDetail(null); + let cancelled = false; + let ws: WebSocket | null = null; const term = new Terminal({ fontSize: 13, @@ -47,55 +51,68 @@ export function ContainerTerminal({ term.open(boxRef.current); fit.fit(); - const proto = window.location.protocol === "https:" ? "wss" : "ws"; - const url = - `${proto}://${window.location.host}/ws/exec/${containerId}` + - `?token=${encodeURIComponent(token)}&cmd=${encodeURIComponent(shell)}`; - const ws = new WebSocket(url); - const sendResize = () => { - if (ws.readyState === WebSocket.OPEN) { + if (ws?.readyState === WebSocket.OPEN) { ws.send(JSON.stringify({ type: "resize", rows: term.rows, cols: term.cols })); } }; - ws.onopen = () => { - setStatus("open"); - term.focus(); - sendResize(); - }; - ws.onmessage = (ev) => { - try { - const msg = JSON.parse(ev.data); - if (msg.type === "data") term.write(msg.data as string); - else if (msg.type === "exit") { - term.write(`\r\n\x1b[90m[process exited${msg.code != null ? ` with code ${msg.code}` : ""}]\x1b[0m\r\n`); - setStatus("closed"); - } else if (msg.type === "error") { - term.write(`\r\n\x1b[31m${msg.detail || "error"}\x1b[0m\r\n`); - setStatus("error"); - setDetail(msg.detail || "error"); - } - } catch { - /* non-JSON: write raw */ - term.write(typeof ev.data === "string" ? ev.data : ""); - } - }; - ws.onclose = (ev) => { - if (ev.code === 4403) { + const connect = async () => { + const token = await useAuthStore.getState().freshAccessToken(); + if (cancelled) return; + if (!token) { setStatus("error"); - setDetail("Terminal access is admin-only."); - term.write("\r\n\x1b[31mTerminal access is admin-only.\x1b[0m\r\n"); - } else if (ev.code === 4401) { - setStatus("error"); - setDetail("Authentication failed."); - } else { - setStatus((s) => (s === "open" ? "closed" : s)); + setDetail("Your session has ended — sign in again."); + return; } + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + ws = new WebSocket( + `${proto}://${window.location.host}/ws/exec/${encodeURIComponent(containerId)}` + + `?token=${encodeURIComponent(token)}&cmd=${encodeURIComponent(shell)}` + ); + wire(ws); }; + const wire = (ws: WebSocket) => { + ws.onopen = () => { + setStatus("open"); + term.focus(); + sendResize(); + }; + ws.onmessage = (ev) => { + try { + const msg = JSON.parse(ev.data); + if (msg.type === "data") term.write(msg.data as string); + else if (msg.type === "exit") { + term.write(`\r\n\x1b[90m[process exited${msg.code != null ? ` with code ${msg.code}` : ""}]\x1b[0m\r\n`); + setStatus("closed"); + } else if (msg.type === "error") { + term.write(`\r\n\x1b[31m${msg.detail || "error"}\x1b[0m\r\n`); + setStatus("error"); + setDetail(msg.detail || "error"); + } + } catch { + /* non-JSON: write raw */ + term.write(typeof ev.data === "string" ? ev.data : ""); + } + }; + ws.onclose = (ev) => { + if (ev.code === 4403) { + setStatus("error"); + setDetail("Terminal access is admin-only."); + term.write("\r\n\x1b[31mTerminal access is admin-only.\x1b[0m\r\n"); + } else if (ev.code === 4401) { + setStatus("error"); + setDetail("Authentication failed."); + } else { + setStatus((s) => (s === "open" ? "closed" : s)); + } + }; + }; + void connect(); + const onData = term.onData((d) => { - if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "data", data: d })); + if (ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "data", data: d })); }); const ro = new ResizeObserver(() => { @@ -109,12 +126,13 @@ export function ContainerTerminal({ ro.observe(boxRef.current); return () => { + cancelled = true; ro.disconnect(); onData.dispose(); - ws.close(); + ws?.close(); term.dispose(); }; - }, [containerId, shell, token]); + }, [containerId, shell]); return (

diff --git a/frontend/src/components/stacks/DeployConsole.tsx b/frontend/src/components/stacks/DeployConsole.tsx index e64405a..d7fa3d3 100644 --- a/frontend/src/components/stacks/DeployConsole.tsx +++ b/frontend/src/components/stacks/DeployConsole.tsx @@ -40,15 +40,19 @@ export function DeployConsole({ const [errorDetail, setErrorDetail] = useState(null); const [showLog, setShowLog] = useState(true); const boxRef = useRef(null); - const token = useAuthStore((s) => s.accessToken); const tracker = useMemo(() => new DeployTracker(), []); const bufferRef = useRef([]); const indexRef = useRef(new Map()); const dirtyRef = useRef(false); + // Connected once per stack — deliberately not keyed on the access token. + // Re-running this effect opens a *new* /ws/deploy, i.e. starts another + // `compose up`; with the token as a dependency, every background token + // refresh during (or after) a deploy did exactly that. useEffect(() => { - if (!token) return; + let ws: WebSocket | null = null; + let cancelled = false; // One line per layer/container, updated in place — the same way compose // renders on a TTY — so status changes replace their line instead of @@ -86,38 +90,63 @@ export function DeployConsole({ push(ev.id, `${ev.id} ${ev.text}${details}`, ev.status === "error" ? "error" : "plain"); }; - const proto = window.location.protocol === "https:" ? "wss" : "ws"; - const url = `${proto}://${window.location.host}/ws/deploy/${stackId}?token=${token}`; - const ws = new WebSocket(url); - ws.onmessage = (ev) => { - try { - const msg = JSON.parse(ev.data); - if (msg.type === "log") { - handleLine(msg.line as string); - } else if (msg.type === "done") { - const ok = msg.returncode === 0; - tracker.finish(ok); - dirtyRef.current = true; - setPhase(ok ? "success" : "failed"); - } else if (msg.type === "error") { - tracker.finish(false); - dirtyRef.current = true; - setPhase("error"); - setErrorDetail(msg.detail || "Deploy error"); - } - } catch { - /* ignore */ + const fail = (detail: string) => { + tracker.finish(false); + dirtyRef.current = true; + setPhase("error"); + setErrorDetail(detail); + }; + + // Set once a done/error frame arrived, so the close that follows it is + // not mistaken for a dropped connection. + let settled = false; + + const connect = async () => { + const token = await useAuthStore.getState().freshAccessToken(); + if (cancelled) return; + if (!token) { + settled = true; + fail("Your session has ended — sign in again."); + return; } + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + ws = new WebSocket( + `${proto}://${window.location.host}/ws/deploy/${encodeURIComponent(stackId)}` + + `?token=${encodeURIComponent(token)}` + ); + ws.onmessage = (ev) => { + try { + const msg = JSON.parse(ev.data); + if (msg.type === "log") { + handleLine(msg.line as string); + } else if (msg.type === "done") { + const ok = msg.returncode === 0; + settled = true; + tracker.finish(ok); + dirtyRef.current = true; + setPhase(ok ? "success" : "failed"); + } else if (msg.type === "error") { + settled = true; + fail(msg.detail || "Deploy error"); + } + } catch { + /* ignore */ + } + }; + ws.onclose = (ev) => { + // A socket that dropped before a done/error frame is surfaced. + if (settled || cancelled) return; + settled = true; + fail( + ev.code === 4403 + ? "Deploying is admin-only." + : ev.code === 4401 + ? "Your session has ended — sign in again." + : "Lost the connection to the server. The deploy carries on there." + ); + }; }; - ws.onclose = () => { - // If the socket dropped before a done/error frame, surface it. - setPhase((p) => { - if (p !== "running") return p; - tracker.finish(false); - dirtyRef.current = true; - return "error"; - }); - }; + void connect(); const timer = window.setInterval(() => { if (!dirtyRef.current) return; @@ -127,10 +156,11 @@ export function DeployConsole({ }, FLUSH_MS); return () => { + cancelled = true; window.clearInterval(timer); - ws.close(); + ws?.close(); }; - }, [stackId, token, tracker]); + }, [stackId, tracker]); useEffect(() => { if (boxRef.current) boxRef.current.scrollTop = boxRef.current.scrollHeight; diff --git a/frontend/src/components/stacks/EditorHelperPanel.tsx b/frontend/src/components/stacks/EditorHelperPanel.tsx index 80b84c9..e0e1f26 100644 --- a/frontend/src/components/stacks/EditorHelperPanel.tsx +++ b/frontend/src/components/stacks/EditorHelperPanel.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { RefreshCw, HardDrive, Cpu, Plug, Gauge } from "lucide-react"; import { VolumeWizard } from "@/components/volumes/VolumeWizard"; import { GPUSelector } from "@/components/gpu/GPUSelector"; @@ -22,9 +22,14 @@ export function EditorHelperPanel({ const [tab, setTab] = useState("volumes"); const [privileged, setPrivileged] = useState(false); + // Only the newest request may answer: an older one finishing late would put + // back the services of a YAML that has since changed. + const request = useRef(0); const loadServices = async () => { + const mine = ++request.current; try { const svc = await editorApi.services(yaml); + if (mine !== request.current) return; setServices(svc); setService((cur) => (cur && svc.includes(cur) ? cur : svc[0] ?? "")); } catch (e) { @@ -32,10 +37,15 @@ export function EditorHelperPanel({ } }; + // Follows the YAML (debounced, it is a server round trip). Read once on mount + // it listed the editor's starter service "app" for every existing stack — + // its file only arrives after the panel mounted — and the helpers then + // wrote into a service that did not exist. useEffect(() => { - loadServices(); + const t = setTimeout(loadServices, 500); + return () => clearTimeout(t); // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); + }, [yaml]); const guard = (): string | null => { if (!service) { diff --git a/frontend/src/components/stacks/LogViewer.tsx b/frontend/src/components/stacks/LogViewer.tsx index 805d99f..8f6bb58 100644 --- a/frontend/src/components/stacks/LogViewer.tsx +++ b/frontend/src/components/stacks/LogViewer.tsx @@ -95,45 +95,64 @@ export function LogViewer({ stackId }: { stackId: string }) { const [serviceFilter, setServiceFilter] = useState("__all__"); const [search, setSearch] = useState(""); const containerRef = useRef(null); - const token = useAuthStore((s) => s.accessToken); + // One stream per stack. Keyed on the access token, the stream reconnected on + // every background token refresh and the server replayed its 200-line tail + // into the view a second time. useEffect(() => { - if (!token) return; setError(null); + setLines([]); let gotError = false; - const proto = window.location.protocol === "https:" ? "wss" : "ws"; - const url = `${proto}://${window.location.host}/ws/logs/${stackId}?token=${token}`; - const ws = new WebSocket(url); - ws.onopen = () => setConnected(true); - ws.onclose = (ev) => { - setConnected(false); - // An expired or revoked session closes 4401. - if (!gotError && ev.code === 4401) { + let cancelled = false; + let ws: WebSocket | null = null; + + const connect = async () => { + const token = await useAuthStore.getState().freshAccessToken(); + if (cancelled) return; + if (!token) { setError("Not authorized to stream logs — sign in again."); + return; } - }; - ws.onmessage = (ev) => { - try { - const msg = JSON.parse(ev.data); - if (msg.type === "log") { - const { service, time, msg: body } = parseLine(msg.service, msg.line); - setLines((prev) => { - const next = [ - ...prev, - { service, time, msg: body, raw: msg.line, level: levelFor(body) }, - ]; - return next.length > MAX_LINES ? next.slice(-MAX_LINES) : next; - }); - } else if (msg.type === "error") { - gotError = true; - setError(msg.detail || "Log stream error"); + const proto = window.location.protocol === "https:" ? "wss" : "ws"; + ws = new WebSocket( + `${proto}://${window.location.host}/ws/logs/${encodeURIComponent(stackId)}` + + `?token=${encodeURIComponent(token)}` + ); + ws.onopen = () => setConnected(true); + ws.onclose = (ev) => { + setConnected(false); + // An expired or revoked session closes 4401. + if (!gotError && ev.code === 4401) { + setError("Not authorized to stream logs — sign in again."); } - } catch { - /* ignore */ - } + }; + ws.onmessage = (ev) => { + try { + const msg = JSON.parse(ev.data); + if (msg.type === "log") { + const { service, time, msg: body } = parseLine(msg.service, msg.line); + setLines((prev) => { + const next = [ + ...prev, + { service, time, msg: body, raw: msg.line, level: levelFor(body) }, + ]; + return next.length > MAX_LINES ? next.slice(-MAX_LINES) : next; + }); + } else if (msg.type === "error") { + gotError = true; + setError(msg.detail || "Log stream error"); + } + } catch { + /* ignore */ + } + }; }; - return () => ws.close(); - }, [stackId, token]); + void connect(); + return () => { + cancelled = true; + ws?.close(); + }; + }, [stackId]); // Unique services seen so far, for the container filter. const services = useMemo(() => { diff --git a/frontend/src/components/stacks/StacksTable.tsx b/frontend/src/components/stacks/StacksTable.tsx index 0872dab..f5e66fc 100644 --- a/frontend/src/components/stacks/StacksTable.tsx +++ b/frontend/src/components/stacks/StacksTable.tsx @@ -67,8 +67,10 @@ export function StacksTable({ Stack - CPU - Memory + {/* Left out on a phone: next to the name block there was no room, + and the meters were drawn over the service count. */} + CPU + Memory {isAdmin && } @@ -188,7 +190,7 @@ function StackRow({ {status && }
- + {running && stats ? ( — )} - + {running && stats ? ( + +

Admin only

+

{children}

+ + ); +} diff --git a/frontend/src/components/ui/StackGroupHeader.tsx b/frontend/src/components/ui/StackGroupHeader.tsx index eaaca41..7200bcc 100644 --- a/frontend/src/components/ui/StackGroupHeader.tsx +++ b/frontend/src/components/ui/StackGroupHeader.tsx @@ -70,14 +70,18 @@ function Label({ group }: { group: StackGroup }) { ); } - // Labelled with a compose project that is no longer a stack: exactly where - // things left behind by a deleted stack collect. + // Labelled with a compose project that is no StackPilot stack: left behind + // by a deleted stack, or started outside StackPilot (`docker compose up` in + // some other folder) — which of the two cannot be told from the label. return ( <> {group.stackId} - - stack removed + + not a stack ); diff --git a/frontend/src/hooks/useDockerEvents.ts b/frontend/src/hooks/useDockerEvents.ts index 782d7b9..bd25545 100644 --- a/frontend/src/hooks/useDockerEvents.ts +++ b/frontend/src/hooks/useDockerEvents.ts @@ -67,10 +67,14 @@ export function useDockerEvents() { } }; - const connect = () => { + const connect = async () => { + // The token this effect started with may have expired by the time a + // dropped socket reconnects; a refused one would just retry forever. + const fresh = await useAuthStore.getState().freshAccessToken(); + if (closed || !fresh) return; const proto = window.location.protocol === "https:" ? "wss" : "ws"; socket = new WebSocket( - `${proto}://${window.location.host}/ws/events?token=${encodeURIComponent(token)}` + `${proto}://${window.location.host}/ws/events?token=${encodeURIComponent(fresh)}` ); socket.onopen = () => { @@ -94,12 +98,12 @@ export function useDockerEvents() { // token is short-lived and the API client refreshes it out from under us. socket.onclose = () => { if (closed) return; - retry = window.setTimeout(connect, delay); + retry = window.setTimeout(() => void connect(), delay); delay = Math.min(delay * 2, RECONNECT_MAX_MS); }; }; - connect(); + void connect(); return () => { closed = true; diff --git a/frontend/src/hooks/useStackActions.ts b/frontend/src/hooks/useStackActions.ts index f29cd18..a341673 100644 --- a/frontend/src/hooks/useStackActions.ts +++ b/frontend/src/hooks/useStackActions.ts @@ -107,7 +107,7 @@ export function useStackActions() { begin(id, label); const t = toast.loading(`${label} ${id}…`); try { - await streamCompose(path, token!, (progress) => + await streamCompose(path, (progress) => setStatusMap((s) => (s[id] ? { ...s, [id]: { ...s[id], progress } } : s)) ); toast.success(`${label} ${id} ✓`, { id: t }); diff --git a/frontend/src/index.css b/frontend/src/index.css index a4bdfae..af45398 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -19,6 +19,10 @@ body { margin: 0; font-family: var(--sp-font); background: var(--sp-bg); + /* The same text colour AppShell sets: dialogs are portalled onto , + outside it, and inherited the browser's black there — unreadable on the + dark theme's dialog background. */ + @apply text-slate-900 dark:text-slate-100; } /* Custom scrollbar */ diff --git a/frontend/src/lib/composeStream.ts b/frontend/src/lib/composeStream.ts index 9159ced..a4ff812 100644 --- a/frontend/src/lib/composeStream.ts +++ b/frontend/src/lib/composeStream.ts @@ -7,6 +7,7 @@ * stacks list, where an update needs to be legible at a glance in one row. */ import { DeployTracker, parseDeployLine } from "./deployProgress"; +import { useAuthStore } from "@/store/auth"; /** Compose emits status changes far faster than the UI needs to repaint. */ const FLUSH_MS = 150; @@ -27,12 +28,16 @@ function snapshot(tracker: DeployTracker): ProgressSnapshot { * Opens `path`, reporting progress until compose finishes. Resolves on exit * code 0, rejects with the failure reason otherwise. The compose subprocess * keeps running server-side even if this socket is closed early. + * + * Fetches a fresh access token itself: an expired one is refused with 4401, + * which used to surface as "connection closed before the update finished". */ -export function streamCompose( +export async function streamCompose( path: string, - token: string, onProgress: (p: ProgressSnapshot) => void ): Promise { + const token = await useAuthStore.getState().freshAccessToken(); + if (!token) throw new Error("your session has ended — sign in again"); return new Promise((resolve, reject) => { const tracker = new DeployTracker(); const proto = window.location.protocol === "https:" ? "wss" : "ws"; @@ -82,6 +87,15 @@ export function streamCompose( ws.onerror = () => settle(new Error("connection to the server failed")); // Reached normally after settle() closes the socket; the guard makes this // a no-op then, so it only fires for a stream that dropped mid-update. - ws.onclose = () => settle(new Error("connection closed before the update finished")); + ws.onclose = (ev) => + settle( + new Error( + ev.code === 4403 + ? "admin rights required" + : ev.code === 4401 + ? "your session has ended — sign in again" + : "connection closed before the update finished" + ) + ); }); } diff --git a/frontend/src/lib/monaco.ts b/frontend/src/lib/monaco.ts new file mode 100644 index 0000000..cc832b6 --- /dev/null +++ b/frontend/src/lib/monaco.ts @@ -0,0 +1,41 @@ +/** + * Monaco from the bundle instead of a CDN. + * + * `@monaco-editor/react` fetches the editor from cdn.jsdelivr.net by default, + * at run time: on a host without internet access, or behind a DNS filter that + * blocks jsdelivr, the compose editor never got past "Loading…". Here it is + * built into the chunks of the two pages that edit text (both lazy-loaded, so + * nobody pays for it elsewhere): the API, the editor features (find, folding, + * the diff view …) and the languages the file browser picks by extension. + * Each language's tokenizer is itself a separate chunk, fetched on first use. + * + * Imported for its side effect, before any mounts. + */ +import { loader } from "@monaco-editor/react"; +import * as monaco from "monaco-editor/editor"; +import "monaco-editor/features/register.all"; +import "monaco-editor/languages/definitions/yaml/register"; +import "monaco-editor/languages/definitions/ini/register"; +import "monaco-editor/languages/definitions/shell/register"; +import "monaco-editor/languages/definitions/dockerfile/register"; +import "monaco-editor/languages/definitions/python/register"; +import "monaco-editor/languages/definitions/javascript/register"; +import "monaco-editor/languages/definitions/typescript/register"; +import "monaco-editor/languages/definitions/markdown/register"; +import "monaco-editor/languages/definitions/html/register"; +import "monaco-editor/languages/definitions/css/register"; +import "monaco-editor/languages/definitions/xml/register"; +import "monaco-editor/languages/definitions/sql/register"; +// JSON is the one with a language service (validation), and its own worker. +// Both JSON paths spell out ".js": without it Vite fails to resolve them +// through the package's "./*" export, the others resolve either way. +import "monaco-editor/languages/features/json/register.js"; +import EditorWorker from "monaco-editor/editor/editor.worker?worker"; +import JsonWorker from "monaco-editor/language/json/json.worker.js?worker"; + +self.MonacoEnvironment = { + getWorker: (_moduleId: string, label: string) => + label === "json" ? new JsonWorker() : new EditorWorker(), +}; + +loader.config({ monaco }); diff --git a/frontend/src/lib/utils.test.ts b/frontend/src/lib/utils.test.ts new file mode 100644 index 0000000..11a68f8 --- /dev/null +++ b/frontend/src/lib/utils.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { parseServerTime, relativeTime } from "./utils"; + +describe("parseServerTime", () => { + it("reads an offset-less API timestamp as UTC", () => { + // What SQLite hands back for a UTC datetime: no zone at all. + expect(parseServerTime("2026-10-15T09:56:04.321963").toISOString()).toBe( + "2026-10-15T09:56:04.321Z" + ); + }); + + it("leaves a timestamp that carries its zone alone", () => { + expect(parseServerTime("2026-10-15T11:56:04+02:00").toISOString()).toBe( + "2026-10-15T09:56:04.000Z" + ); + expect(parseServerTime("2026-10-15T09:56:04.123456789Z").toISOString()).toBe( + "2026-10-15T09:56:04.123Z" + ); + }); +}); + +describe("relativeTime", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-15T10:00:00Z")); + }); + afterEach(() => vi.useRealTimers()); + + it("is not hours off for a UTC timestamp without an offset", () => { + expect(relativeTime("2026-10-15T09:59:30")).toBe("just now"); + expect(relativeTime("2026-10-15T09:55:00")).toBe("5m ago"); + }); + + it("says when a time is still ahead", () => { + // A schedule's next run used to read "next just now", whenever it was. + expect(relativeTime("2026-10-15T13:00:00")).toBe("in 3h"); + expect(relativeTime("2026-10-25T10:00:00Z")).toBe("in 10d"); + }); + + it("does not print NaN for something unparseable", () => { + expect(relativeTime("not a date")).toBe("—"); + }); +}); diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 747f847..9e04022 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -21,15 +21,32 @@ export function formatUptime(seconds: number): string { return `${m}m`; } +/** + * A timestamp from the API as a Date. + * + * StackPilot's own times are UTC, but SQLite hands them back without an + * offset, and JavaScript reads an offset-less date-time as *local* time — which + * put every audit entry, sync and backup two hours in the past in German summer + * time. Docker's own timestamps carry a zone and pass through unchanged. + */ +export function parseServerTime(iso: string): Date { + const zoned = /(?:Z|[+-]\d{2}:?\d{2})$/i.test(iso); + return new Date(zoned || !iso.includes("T") ? iso : `${iso}Z`); +} + +/** "5m ago", or "in 5m" for a time still ahead (a schedule's next run). */ export function relativeTime(iso: string): string { - const then = new Date(iso).getTime(); + const then = parseServerTime(iso).getTime(); + if (Number.isNaN(then)) return "—"; const diff = Date.now() - then; - const s = Math.floor(diff / 1000); - if (s < 60) return "just now"; + const ahead = diff < 0; + const say = (v: string) => (ahead ? `in ${v}` : `${v} ago`); + const s = Math.floor(Math.abs(diff) / 1000); + if (s < 60) return ahead ? "in a moment" : "just now"; const m = Math.floor(s / 60); - if (m < 60) return `${m}m ago`; + if (m < 60) return say(`${m}m`); const h = Math.floor(m / 60); - if (h < 24) return `${h}h ago`; + if (h < 24) return say(`${h}h`); const d = Math.floor(h / 24); - return `${d}d ago`; + return say(`${d}d`); } diff --git a/frontend/src/pages/Audit.tsx b/frontend/src/pages/Audit.tsx index 186147a..f1d23be 100644 --- a/frontend/src/pages/Audit.tsx +++ b/frontend/src/pages/Audit.tsx @@ -5,10 +5,18 @@ import { Card, Input, Spinner } from "@/components/ui"; import api from "@/api/client"; import type { AuditEntry } from "@/types"; import { relativeTime } from "@/lib/utils"; +import { useAuthStore } from "@/store/auth"; +import { AdminOnly } from "@/components/ui/AdminOnly"; const PAGE = 100; export function Audit() { + const isAdmin = useAuthStore((s) => s.user?.role === "admin"); + if (!isAdmin) return The audit log is available to administrators only.; + return ; +} + +function AuditLog() { const [offset, setOffset] = useState(0); const [filter, setFilter] = useState(""); diff --git a/frontend/src/pages/Files.tsx b/frontend/src/pages/Files.tsx index 1554598..50ff26f 100644 --- a/frontend/src/pages/Files.tsx +++ b/frontend/src/pages/Files.tsx @@ -24,11 +24,14 @@ import { } from "lucide-react"; import { toast } from "sonner"; import Editor from "@monaco-editor/react"; +// Before any editor mounts: points the wrapper at the bundled Monaco. +import "@/lib/monaco"; import { Button, Card, Input, Spinner } from "@/components/ui"; import { ConfirmDialog } from "@/components/ui/ConfirmDialog"; import { filesApi } from "@/api/files"; import { apiErrorMessage } from "@/api/client"; import { useAuthStore } from "@/store/auth"; +import { AdminOnly } from "@/components/ui/AdminOnly"; import { useThemeStore } from "@/store/theme"; import { formatBytes, relativeTime } from "@/lib/utils"; import type { HostPathEntry } from "@/types"; @@ -71,6 +74,11 @@ function crumbs(path: string): { label: string; path: string }[] { export function Files() { const isAdmin = useAuthStore((s) => s.user?.role === "admin"); + if (!isAdmin) return The file browser is available to administrators only.; + return ; +} + +function FileBrowser({ isAdmin }: { isAdmin: boolean }) { const qc = useQueryClient(); const [path, setPath] = useState("/"); // Persist "Show hidden" across reloads — otherwise an uploaded dotfile (.env) @@ -293,7 +301,7 @@ export function Files() { - {isAdmin && ( + {isAdmin && !data?.virtual && ( <> - {isAdmin && ( - <> - - - - - - )} -
+ {/* A folder on the way to a root is only a way through. */} + {!data?.virtual && ( +
+ + {isAdmin && ( + <> + + + + + + )} +
+ )} ))} diff --git a/frontend/src/pages/Images.tsx b/frontend/src/pages/Images.tsx index 305a934..06709d3 100644 --- a/frontend/src/pages/Images.tsx +++ b/frontend/src/pages/Images.tsx @@ -18,6 +18,12 @@ import { useAuthStore } from "@/store/auth"; function UpdateBadge({ row }: { row: ImageRow }) { const u = row.update; if (!u) return not checked; + if (u.local) + return ( + + local build + + ); if (u.error) return ⚠ {u.error}; if (u.update_available) return ( @@ -83,10 +89,19 @@ function ImagesSection({ isAdmin }: { isAdmin: boolean }) { setScanning(true); const t = toast.loading("Scanning images… this runs the scanner per image"); try { - const r = await imagesApi.scanAll(); + const started = await imagesApi.startScanAll(); + // The sweep runs on the server; follow it until it is done. + let status = started; + while (status.active) { + await new Promise((r) => window.setTimeout(r, 2000)); + status = await imagesApi.scanStatus(); + if (status.run !== started.run) break; // a newer sweep took over + toast.loading(`Scanning images… ${status.done}/${status.total}`, { id: t }); + qc.invalidateQueries({ queryKey: ["image-scans"] }); + } await qc.invalidateQueries({ queryKey: ["image-scans"] }); toast.success( - `Scanned ${r.scanned} image(s)${r.failed ? `, ${r.failed} failed` : ""}`, + `Scanned ${status.scanned} image(s)${status.failed ? `, ${status.failed} failed` : ""}`, { id: t } ); } catch (e) { diff --git a/frontend/src/pages/Networks.tsx b/frontend/src/pages/Networks.tsx index bdb373f..12c47d3 100644 --- a/frontend/src/pages/Networks.tsx +++ b/frontend/src/pages/Networks.tsx @@ -250,15 +250,18 @@ function NetworkRows({ {isAdmin && ( {!n.is_default && ( + // Disabled while in use: Docker refuses to remove it anyway, and + // the red bin offered a delete that could only fail. )} diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index 812da11..2404b7d 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -7,7 +7,6 @@ import { Send, Trash2, Users as UsersIcon, - ShieldCheck, Power, RefreshCw, HardDrive, @@ -40,21 +39,14 @@ import { stacksApi } from "@/api/stacks"; import { apiErrorMessage } from "@/api/client"; import { relativeTime } from "@/lib/utils"; import { useAuthStore } from "@/store/auth"; +import { AdminOnly } from "@/components/ui/AdminOnly"; import type { User } from "@/types"; export function Settings() { const isAdmin = useAuthStore((s) => s.user?.role === "admin"); if (!isAdmin) { - return ( - - -

Admin only

-

- Settings are available to administrators only. -

- - ); + return Settings are available to administrators only.; } return ( @@ -514,6 +506,11 @@ const FIELDS: Record destinationsApi.test(dest.id), - onSuccess: (r) => - r.ok ? toast.success("Reachable") : toast.error(r.error || "Connection failed"), + onSuccess: (r) => { + if (!r.ok) return toast.error(r.error || "Connection failed"); + toast.success("Reachable", r.host_key ? { description: `Host key ${r.host_key}` } : undefined); + // A first SFTP connection pins the server's key; show it in the form. + onChange(); + }, onError: (e) => toast.error(apiErrorMessage(e)), }); const remove = useMutation({ diff --git a/frontend/src/pages/StackDetail.tsx b/frontend/src/pages/StackDetail.tsx index eb93ad0..3d60f19 100644 --- a/frontend/src/pages/StackDetail.tsx +++ b/frontend/src/pages/StackDetail.tsx @@ -107,15 +107,15 @@ export function StackDetail() { {/* Tabs */} -
+
{TABS.map((t) => (