Run compose like the terminal does, and follow changes made there (0.67.2)

StackPilot called `docker compose -p <folder> -f compose.yaml`, which is
not what `docker compose` typed in the stack's folder does. A
compose.override.yaml or a COMPOSE_FILE in .env applied in a terminal
and was dropped by the next Start in StackPilot, which recreated the
containers without it. A compose file's `name:` or a COMPOSE_PROJECT_NAME
in .env made the terminal's containers invisible: the stack showed as
stopped while running, and Start set up a second copy next to it.

Compose now runs in the stack's folder without -f, so it loads the same
files, and -p is the name compose itself picks there: COMPOSE_PROJECT_NAME
from .env, else the last `name:` among the compose files (${VAR} from
.env), else the folder name. A stack StackPilot already runs under its
folder name keeps that name while it has containers or volumes under it
and none under the declared one: switching would have brought it up
again on empty volumes. Backups record the project, and a restore takes
it from the files it has just put back.

Changes made outside StackPilot now reach every view:

- docker_watch_service reads Docker's events once for the backend. The
  dashboard rollup (cached 25s) and the stats cache are stale as soon as
  the event generation moves; the tiles followed a terminal `compose
  stop` after 15-30s, now within a second.
- Image events re-read the local digests, so a `docker compose pull` in
  a terminal clears the update badge instead of leaving it until the
  next registry check (an hour by default).
- The browser catches up when its event socket reconnects; what changed
  while it was down waited for the 60s poll.
- The stack detail carries a revision of the compose file and .env;
  saving after they changed on disk is refused with 409 and the editor
  offers to overwrite or to load the version on disk. A save used to
  replace a terminal edit without a word.
- A stack whose folder is gone is flagged "folder missing" instead of
  passing for an ordinary stopped stack.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Fx7gcuy8d6nPd95g43rGHu

backend/services/backup_service.py

@@ -507,6 +507,9 @@ async def _create(
         manifest = {
             "format_version": BACKUP_FORMAT_VERSION,
             "stack_id": stack_id,
+            # Volumes are prefixed with the project, which a compose file's
+            # name: or COMPOSE_PROJECT_NAME can make anything at all.
+            "project": compose_service.project_name(stack_id),
             "name": name,
             "created_at": compose_service.now().isoformat(),
             "include_volumes": include_volumes,
@@ -920,9 +923,11 @@ def restore_backup(
     old_id = compose_service.slugify(manifest.get("stack_id") or stack_id)
     renamed = stack_id != old_id
     # Volumes are named after the compose *project*, which differs from the id
-    # for a folder compose had to normalise ("my.app" -> "myapp_data").
-    old_project = compose_service.project_name(manifest.get("stack_id") or stack_id)
-    new_project = compose_service.project_name(stack_id)
+    # for a folder compose had to normalise ("my.app" -> "myapp_data"), and
+    # for one whose files name the project. Backups before 0.67.2 do not say.
+    old_project = manifest.get("project") or compose_service.project_name(
+        manifest.get("stack_id") or stack_id
+    )
     old_dir = manifest.get("stack_dir") or ""
 
     directory = compose_service.stack_dir(stack_id)
@@ -942,6 +947,8 @@ def restore_backup(
 
     with tarfile.open(tar_path, "r:gz") as tar:
         _restore_tree(tar, directory, excluded | held_back, held_back, warnings)
+        # Read from the files just put back: their name: / .env decide it.
+        new_project = compose_service.project_name(stack_id)
         skipped += [{"kind": "bind", "source": rel, "reason": "not requested"} for rel in held_back]
 
         for bind in manifest.get("binds", []):

backend/services/compose_service.py

@@ -37,17 +37,167 @@ class StackFileError(Exception):
 _PROJECT_INVALID_RE = re.compile(r"[^a-z0-9_-]+")
 
 
-def project_name(stack_id: str) -> str:
-    """The compose project name for a stack.
+def _normalize_project(name: str) -> str:
+    return _PROJECT_INVALID_RE.sub("", name.lower()).lstrip("_-")
 
-    The same normalisation compose applies to a directory name when it picks
-    the project name itself (lowercase, only ``a-z0-9_-``, no leading ``-``/``_``).
-    For every stack StackPilot creates that is the id unchanged. A folder found
-    on disk as ``Nextcloud`` or ``my.app`` used to be passed as ``-p Nextcloud``,
-    which compose refuses — so every action on such a stack failed, while
-    ``docker compose up`` in that folder worked.
+
+#: The override files compose loads next to the main file, in its order.
+COMPOSE_OVERRIDE_FILENAMES = (
+    "compose.override.yml",
+    "compose.override.yaml",
+    "docker-compose.override.yml",
+    "docker-compose.override.yaml",
+)
+_ENV_LINE = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$")
+_VAR_REF = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::?-([^}]*))?\}|\$([A-Za-z_][A-Za-z0-9_]*)")
+
+
+def _dotenv(directory: str) -> dict[str, str]:
+    """The folder's ``.env`` as compose reads it for its own settings."""
+    values: dict[str, str] = {}
+    try:
+        with open(os.path.join(directory, ".env"), encoding="utf-8", errors="replace") as fh:
+            for raw in fh:
+                if raw.lstrip().startswith("#"):
+                    continue
+                m = _ENV_LINE.match(raw)
+                if not m:
+                    continue
+                value = m.group(2)
+                if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+                    value = value[1:-1]
+                values[m.group(1)] = value
+    except OSError:
+        pass
+    return values
+
+
+def compose_files(directory: str, env: Optional[dict] = None) -> list[str]:
+    """The files ``docker compose`` loads in this folder, in order.
+
+    ``COMPOSE_FILE`` from the folder's ``.env`` if set, otherwise the first of
+    the default names plus an override file next to it — what running
+    ``docker compose`` in the folder picks.
     """
-    return _PROJECT_INVALID_RE.sub("", stack_id.lower()).lstrip("_-") or "stack"
+    env = _dotenv(directory) if env is None else env
+    if env.get("COMPOSE_FILE"):
+        sep = env.get("COMPOSE_PATH_SEPARATOR") or os.pathsep
+        return [
+            os.path.normpath(os.path.join(directory, part))
+            for part in env["COMPOSE_FILE"].split(sep)
+            if part.strip()
+        ]
+    files = []
+    main = find_compose_file(directory)
+    if main:
+        files.append(main)
+        for name in COMPOSE_OVERRIDE_FILENAMES:
+            candidate = os.path.join(directory, name)
+            if os.path.isfile(candidate):
+                files.append(candidate)
+                break
+    return files
+
+
+#: directory -> (fingerprint of the files that decide it, declared name)
+_DECLARED: dict[str, tuple[tuple, Optional[str]]] = {}
+
+
+def _fingerprint(paths) -> tuple:
+    out = []
+    for path in paths:
+        try:
+            st = os.stat(path)
+            out.append((path, st.st_mtime_ns, st.st_size))
+        except OSError:
+            out.append((path, None, None))
+    return tuple(out)
+
+
+def declared_project(directory: str) -> Optional[str]:
+    """The project name compose takes from the folder itself, if it sets one:
+    ``COMPOSE_PROJECT_NAME`` in ``.env``, else the last ``name:`` among its
+    compose files (``${VAR}`` resolved from ``.env``). Cached until one of
+    those files changes."""
+    candidates = [os.path.join(directory, ".env")]
+    candidates += [os.path.join(directory, n) for n in COMPOSE_FILENAMES + COMPOSE_OVERRIDE_FILENAMES]
+    env = _dotenv(directory)
+    files = compose_files(directory, env)
+    key = _fingerprint(candidates + [f for f in files if f not in candidates])
+    cached = _DECLARED.get(directory)
+    if cached and cached[0] == key:
+        return cached[1]
+    name: Optional[str] = None
+    if env.get("COMPOSE_PROJECT_NAME"):
+        name = env["COMPOSE_PROJECT_NAME"]
+    else:
+        import yaml
+
+        for path in files:
+            try:
+                with open(path, encoding="utf-8") as fh:
+                    doc = yaml.safe_load(fh)
+            except (OSError, yaml.YAMLError):
+                continue
+            if isinstance(doc, dict) and isinstance(doc.get("name"), str) and doc["name"].strip():
+                name = _VAR_REF.sub(
+                    lambda m: env.get(m.group(1) or m.group(3)) or (m.group(2) or ""),
+                    doc["name"].strip(),
+                )
+    declared = _normalize_project(name) if name else None
+    _DECLARED[directory] = (key, declared or None)
+    return declared or None
+
+
+#: project -> (monotonic time, Docker event generation, has containers or volumes)
+_RESOURCES: dict[str, tuple[float, int, bool]] = {}
+_RESOURCES_TTL = 15.0
+
+
+def _has_resources(project: str) -> bool:
+    import time
+
+    from services import docker_watch_service
+
+    gen = docker_watch_service.generation()
+    hit = _RESOURCES.get(project)
+    if hit and hit[1] == gen and time.monotonic() - hit[0] < _RESOURCES_TTL:
+        return hit[2]
+    label = {"label": f"{COMPOSE_LABEL}={project}"}
+    try:
+        client = get_client()
+        found = bool(safe_call(client.containers.list, all=True, filters=label)) or bool(
+            safe_call(client.volumes.list, filters=label)
+        )
+    except DockerError:
+        found = False
+    _RESOURCES[project] = (time.monotonic(), gen, found)
+    return found
+
+
+def project_name(stack_id: str) -> str:
+    """The compose project name for a stack — the one ``docker compose`` uses
+    when it is run in the stack's folder.
+
+    That is ``COMPOSE_PROJECT_NAME`` from the folder's ``.env``, else the
+    compose file's ``name:``, else the folder name, normalised the way compose
+    does it (lowercase, only ``a-z0-9_-``, no leading ``-``/``_``).
+
+    Before 0.67.2 StackPilot always used the folder name. A stack whose file
+    says ``name: immich`` in a folder called ``photos`` therefore ran as two
+    different projects: started in a terminal StackPilot did not see it, and
+    its Start button set up a second copy next to it. A stack StackPilot
+    started that old way keeps its folder-derived name for as long as it has
+    containers or volumes under it and none under the declared one — switching
+    would leave them running and bring up the stack again with empty volumes.
+    """
+    legacy = _normalize_project(stack_id) or "stack"
+    declared = declared_project(stack_dir(stack_id))
+    if not declared or declared == legacy:
+        return legacy
+    if _has_resources(legacy) and not _has_resources(declared):
+        return legacy
+    return declared
 
 
 def stack_ids_by_project(stack_ids) -> dict[str, str]:
@@ -114,6 +264,18 @@ def read_compose(stack_id: str, override: Optional[str] = None) -> str:
         return fh.read()
 
 
+def stack_revision(stack_id: str, override: Optional[str] = None) -> str:
+    """A short fingerprint of the compose file and ``.env`` as they are on
+    disk: what the editor loaded, compared again when it saves."""
+    import hashlib
+
+    digest = hashlib.sha256()
+    digest.update(read_compose(stack_id, override).encode())
+    digest.update(b"\0")
+    digest.update(read_env(stack_id, override).encode())
+    return digest.hexdigest()[:16]
+
+
 def read_env(stack_id: str, override: Optional[str] = None) -> str:
     path = env_path(stack_id, override)
     if not os.path.isfile(path):
@@ -450,8 +612,15 @@ def stack_status_summaries() -> dict[str, dict]:
 
 
 def _compose_base_cmd(stack_id: str, override: Optional[str] = None) -> list[str]:
+    """``docker compose`` as it runs when typed in the stack's folder.
+
+    No ``-f``: with it, compose loaded that one file only — a
+    ``compose.override.yaml`` or a ``COMPOSE_FILE`` in ``.env`` applied in a
+    terminal and was dropped again by the next Start in StackPilot, which
+    recreated the containers without it. The subprocess runs in the folder
+    (:func:`_compose_cwd`), which is where ``COMPOSE_FILE`` paths are read from.
+    """
     directory = stack_dir(stack_id, override)
-    compose_file = find_compose_file(directory) or os.path.join(directory, DEFAULT_COMPOSE_NAME)
     return [
         "docker",
         "compose",
@@ -459,11 +628,14 @@ def _compose_base_cmd(stack_id: str, override: Optional[str] = None) -> list[str
         project_name(stack_id),
         "--project-directory",
         directory,
-        "-f",
-        compose_file,
     ]
 
 
+def _compose_cwd(stack_id: str, override: Optional[str] = None) -> Optional[str]:
+    directory = stack_dir(stack_id, override)
+    return directory if os.path.isdir(directory) else None
+
+
 async def run_compose(
     stack_id: str,
     args: list[str],
@@ -475,6 +647,7 @@ async def run_compose(
     proc = await asyncio.create_subprocess_exec(
         *cmd,
         env=registry_service.cli_env(),
+        cwd=_compose_cwd(stack_id, override),
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.PIPE,
     )
@@ -559,6 +732,7 @@ async def stream_compose(
     proc = await asyncio.create_subprocess_exec(
         *cmd,
         env=registry_service.cli_env(),
+        cwd=_compose_cwd(stack_id, override),
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.STDOUT,
     )
@@ -621,6 +795,7 @@ async def _stream_phase(
     proc = await asyncio.create_subprocess_exec(
         *cmd,
         env=registry_service.cli_env(),
+        cwd=_compose_cwd(stack_id, override),
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.STDOUT,
     )

backend/services/dashboard_service.py

@@ -16,7 +16,7 @@ from sqlmodel import Session, select
 
 from docker_client import DockerError, get_client, safe_call
 from models.backup_schedule import BackupSchedule, describe_stacks, status_of
-from services import compose_service, update_service
+from services import compose_service, docker_watch_service, update_service
 
 COMPOSE_LABEL = compose_service.COMPOSE_LABEL
 
@@ -25,7 +25,7 @@ FLEET_TTL = 25.0
 DISK_PRESSURE = 0.85  # disk used fraction above which a host needs attention
 MEM_PRESSURE = 0.90   # memory used fraction above which a host needs attention
 
-_fleet_cache: dict = {"data": None, "ts": 0.0}
+_fleet_cache: dict = {"data": None, "ts": 0.0, "gen": None}
 
 
 # --------------------------------------------------------------------------- #
@@ -219,7 +219,17 @@ _SEVERITY_ORDER = {"error": 0, "warn": 1}
 
 async def compute_fleet(session: Session, refresh: bool = False) -> dict:
     now = time.time()
-    if not refresh and _fleet_cache["data"] and now - _fleet_cache["ts"] < FLEET_TTL:
+    # Stale as soon as Docker reported a change: the UI refetches on that very
+    # event, and got the rollup from before it — a stack stopped in a terminal
+    # stayed "running" in the tiles for up to FLEET_TTL. Read before the work,
+    # so a change during it is not mistaken for one already counted.
+    gen = docker_watch_service.generation()
+    if (
+        not refresh
+        and _fleet_cache["data"]
+        and now - _fleet_cache["ts"] < FLEET_TTL
+        and _fleet_cache["gen"] == gen
+    ):
         return _fleet_cache["data"]
 
     local_host, attention = await asyncio.to_thread(_local_host)
@@ -254,4 +264,5 @@ async def compute_fleet(session: Session, refresh: bool = False) -> dict:
     }
     _fleet_cache["data"] = data
     _fleet_cache["ts"] = now
+    _fleet_cache["gen"] = gen
     return data

backend/services/docker_watch_service.py

@@ -0,0 +1,119 @@
+"""One backend-wide reader of Docker's event stream, for server-side caches.
+
+The browser follows Docker on its own (``/ws/events``) and refetches whatever
+an event touched. Two of the answers it refetched were cached here, though,
+and did not know anything had happened:
+
+* the dashboard's fleet rollup, held for 25 seconds — a stack stopped in a
+  terminal showed as running in the dashboard tiles for up to half a minute,
+  and the event that should have refreshed them had come and gone;
+* the image update status, whose local digests were only re-read after a
+  pull *through StackPilot* — after ``docker compose pull && up -d`` in a
+  terminal the "update available" badge stayed until the next registry check,
+  an hour by default.
+
+So: :func:`generation` changes with every container or image event, and a
+cache that remembers the generation it was computed at knows when it is stale.
+Image events also re-read the local digests (no registry calls), debounced so
+a ``pull`` of ten layers is one pass.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+
+logger = logging.getLogger("stackpilot.docker_watch")
+
+_EVENT_TYPES = ["container", "image"]
+#: Container actions that change nothing a cache holds (see routers/ws.py).
+_IGNORED = {
+    "exec_create", "exec_start", "exec_die", "exec_detach",
+    "attach", "top", "resize", "archive-path", "extract-to-dir",
+}
+#: Wait this long after the last image event before re-reading digests.
+DIGEST_DEBOUNCE_S = 2.0
+RECONNECT_MAX_S = 30.0
+
+_generation = 0
+_lock = threading.Lock()
+_stop = threading.Event()
+_thread: threading.Thread | None = None
+_digest_timer: threading.Timer | None = None
+
+
+def generation() -> int:
+    """Changes whenever Docker reports a container or image change."""
+    return _generation
+
+
+def bump() -> None:
+    global _generation
+    with _lock:
+        _generation += 1
+
+
+def _refresh_digests() -> None:
+    from services import update_service
+
+    try:
+        changed = update_service.refresh_local_digests()
+        if changed:
+            logger.info("Re-read local digests after image events: %d changed", changed)
+    except Exception as exc:  # noqa: BLE001 - a cache refresh must never kill the watcher
+        logger.debug("Local digest refresh failed: %s", exc)
+
+
+def _schedule_digest_refresh() -> None:
+    global _digest_timer
+    with _lock:
+        if _digest_timer is not None:
+            _digest_timer.cancel()
+        _digest_timer = threading.Timer(DIGEST_DEBOUNCE_S, _refresh_digests)
+        _digest_timer.daemon = True
+        _digest_timer.start()
+
+
+def handle(event: dict) -> None:
+    """Apply one Docker event to the caches."""
+    kind = event.get("Type")
+    action = (event.get("Action") or "").split(":")[0]
+    if kind == "container" and action in _IGNORED:
+        return
+    bump()
+    if kind == "image":
+        _schedule_digest_refresh()
+
+
+def _run() -> None:
+    from docker_client import get_client
+
+    delay = 1.0
+    while not _stop.is_set():
+        try:
+            stream = get_client().events(decode=True, filters={"type": _EVENT_TYPES})
+            delay = 1.0
+            # Anything may have happened while we were not listening.
+            bump()
+            for event in stream:
+                if _stop.is_set():
+                    break
+                handle(event)
+        except Exception as exc:  # noqa: BLE001 - daemon restarts, socket hiccups
+            logger.debug("Docker event stream ended: %s", exc)
+        if _stop.wait(delay):
+            break
+        delay = min(delay * 2, RECONNECT_MAX_S)
+
+
+def start() -> None:
+    global _thread
+    if _thread is not None and _thread.is_alive():
+        return
+    _stop.clear()
+    _thread = threading.Thread(target=_run, name="docker-watch", daemon=True)
+    _thread.start()
+
+
+def stop() -> None:
+    _stop.set()
+

backend/services/stats_service.py

@@ -19,6 +19,7 @@ import time
 from concurrent.futures import ThreadPoolExecutor
 
 from docker_client import DockerError, get_client, safe_call
+from services import docker_watch_service
 
 COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
 
@@ -85,18 +86,27 @@ def stack_stats(refresh: bool = False) -> dict:
     when none of them have that limit set. Served from a short-lived cache
     unless ``refresh`` is set.
     """
-    if not refresh and _cache["data"] is not None:
-        if time.monotonic() - _cache["ts"] < CACHE_TTL:
-            return _cache["data"]
+    def fresh() -> bool:
+        # Also stale once Docker reported a change (a container stopped in a
+        # terminal), see services/docker_watch_service.py.
+        return (
+            _cache["data"] is not None
+            and time.monotonic() - _cache["ts"] < CACHE_TTL
+            and _cache.get("gen") == docker_watch_service.generation()
+        )
+
+    if not refresh and fresh():
+        return _cache["data"]
 
     with _lock:
         # Somebody may have refreshed it while we waited for the lock.
-        if not refresh and _cache["data"] is not None:
-            if time.monotonic() - _cache["ts"] < CACHE_TTL:
-                return _cache["data"]
+        if not refresh and fresh():
+            return _cache["data"]
+        gen = docker_watch_service.generation()
         data = _sample()
         _cache["data"] = data
         _cache["ts"] = time.monotonic()
+        _cache["gen"] = gen
         return data
 
 

backend/services/update_service.py

@@ -423,6 +423,24 @@ async def stack_updates(stack_id: str, refresh: bool = True) -> dict:
     }
 
 
+def _reconcile_local(image: str, status: UpdateStatus) -> bool:
+    """Re-read one image's local digest against the cached remote one (no
+    registry call). True when that changed what is stored."""
+    local = _local_digest(image)
+    available = bool(local and status.remote_digest and local != status.remote_digest)
+    if local == status.current_digest and available == status.update_available:
+        return False
+    status.current_digest = local
+    status.update_available = available
+    status.checked_at = time.time()
+    if not available:
+        _NOTIFIED.discard(image)
+    # Persisted too, or a restart brought the cleared badge back until the
+    # next background check.
+    _store(status, image in _NOTIFIED)
+    return True
+
+
 def refresh_stack_local(stack_id: str) -> None:
     """Re-read the local digests of one stack's images and reconcile them with
     the cached remote digests (no registry calls). Called right after a manual
@@ -430,19 +448,19 @@ def refresh_stack_local(stack_id: str) -> None:
     until the next background pass."""
     for image in stack_images(stack_id):
         status = _CACHE.get(image)
-        if status is None:
-            continue
-        local = _local_digest(image)
-        status.current_digest = local
-        status.update_available = bool(
-            local and status.remote_digest and local != status.remote_digest
-        )
-        status.checked_at = time.time()
-        if not status.update_available:
-            _NOTIFIED.discard(image)
-        # Persisted too, or a restart brought the cleared badge back until the
-        # next background check.
-        _store(status, image in _NOTIFIED)
+        if status is not None and not status.local:
+            _reconcile_local(image, status)
+
+
+def refresh_local_digests() -> int:
+    """:func:`refresh_stack_local` for every cached image — what an image
+    event triggers (services/docker_watch_service.py), so a pull made in a
+    terminal clears the badge too. Returns how many changed."""
+    return sum(
+        _reconcile_local(image, status)
+        for image, status in list(_CACHE.items())
+        if not status.local
+    )
 
 
 async def check_all() -> dict[str, dict]: