0.37.2: stream folder zip-downloads to fix 504 on large folders

A big folder hit a 504 Gateway Timeout: the zip was built into a temp
file *before* any response was sent, so for large folders the backend
stayed silent past nginx's proxy_read_timeout.

Now the zip is streamed as it's built, end to end:
- file_service.open_archive() returns (filename, byte iterator); _iter_zip
  walks the dir and yields zip bytes incrementally via a small drain
  buffer, writing each file in 1 MiB chunks (bounded memory, valid CRCs).
  Same hardening as before — only real regular files; FIFOs/sockets/
  devices/symlinks skipped without open(); per-file read errors skipped.
- /api/files/download and /agent/files/download return a StreamingResponse
  (no temp file). The agent proxy streams the agent response straight
  through (agent_service.stream_download), pulling the first chunk eagerly
  so an offline/bad-token agent still yields a clean status before 200.
- Files page: streamed downloads have no Content-Length, so the progress
  bar shows the running downloaded byte count ("Downloading … 12.3 MB")
  instead of a percentage, after the initial "Preparing …".

Verified end to end via TestClient (200, application/zip, valid zip,
2 MiB file intact, FIFO skipped, no hang).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/services/agent_service.py

@@ -137,6 +137,35 @@ async def download_to_file(
         raise AgentError(502, "agent_unreachable", str(exc)) from exc
 
 
+async def stream_download(
+    session: Session,
+    agent: Agent,
+    path: str,
+    *,
+    params: Optional[dict] = None,
+):
+    """Stream a GET from the agent straight through, yielding chunks.
+
+    Unlike :func:`download_to_file` this never buffers to disk, so a large
+    response (e.g. a folder zip the agent builds on the fly) starts flowing to
+    the browser immediately instead of being staged first.
+    """
+    url = agent.url.rstrip("/") + path
+    headers = {"Authorization": f"Bearer {agent.token}"}
+    try:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
+            async with client.stream("GET", url, headers=headers, params=params, timeout=None) as resp:
+                if resp.status_code >= 400:
+                    text = (await resp.aread()).decode("utf-8", "replace")
+                    _handle_status(session, agent, resp.status_code, text)
+                _handle_status(session, agent, resp.status_code)
+                async for chunk in resp.aiter_bytes(1024 * 256):
+                    yield chunk
+    except httpx.HTTPError as exc:
+        _mark(session, agent, "offline")
+        raise AgentError(502, "agent_unreachable", str(exc)) from exc
+
+
 async def upload_file(
     session: Session,
     agent: Agent,

backend/services/file_service.py

@@ -10,8 +10,8 @@ from __future__ import annotations
 
 import os
 import shutil
-import tempfile
 import zipfile
+from collections.abc import Iterator
 
 from services.device_service import BrowseError, _is_allowed, _real_root
 
@@ -182,54 +182,84 @@ def is_dir(path: str) -> bool:
     return os.path.isdir(_safe_real(path))
 
 
-def archive_dir(path: str) -> tuple[str, str]:
-    """Zip a directory (recursively) into a temp file.
+class _ZipBuffer:
+    """A writable sink that hands out and clears whatever was written to it.
 
-    Returns ``(tmp_zip_path, download_filename)``. The caller is responsible
-    for deleting the temp file once it has been streamed to the client.
+    Lets us drive ``zipfile`` while draining its output incrementally so the
+    archive can be streamed to the client instead of buffered to disk.
+    """
+
+    def __init__(self) -> None:
+        self._buf = bytearray()
+
+    def write(self, data: bytes) -> int:
+        self._buf += data
+        return len(data)
+
+    def flush(self) -> None:  # pragma: no cover - zipfile calls this
+        pass
+
+    def take(self) -> bytes:
+        data = bytes(self._buf)
+        self._buf.clear()
+        return data
+
+
+def open_archive(path: str) -> tuple[str, "Iterator[bytes]"]:
+    """Validate a directory and return ``(download_filename, byte_iterator)``.
+
+    The iterator zips the directory recursively **on the fly**, yielding bytes
+    as they are produced so the response starts immediately (no waiting for the
+    whole archive to build → no gateway timeout) and memory stays bounded.
 
     Only regular files and real subdirectories are archived. Symlinks are
     skipped (no sandbox escape / loops); special files (FIFOs, sockets,
-    devices) are skipped too — opening a FIFO would block forever and a
-    socket can't be read at all. Files that can't be read (permissions, or
-    that vanish mid-walk) are skipped individually rather than aborting the
-    whole archive.
+    devices) are skipped too — opening a FIFO would block forever and a socket
+    can't be read at all. Files that can't be read (permissions, or that vanish
+    mid-walk) are skipped individually rather than aborting the whole archive.
     """
     real = _safe_real(path)
     if not os.path.isdir(real):
         raise BrowseError(f"Not a directory: {path}")
     name = os.path.basename(path.rstrip("/")) or "root"
+    return f"{name}.zip", _iter_zip(real, name)
 
-    fd, tmp = tempfile.mkstemp(suffix=".zip")
-    os.close(fd)
-    try:
-        with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) as zf:
-            for root, dirs, files in os.walk(real):
-                # Don't follow symlinked directories (avoids loops / escapes).
-                dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d))]
-                rel_root = os.path.relpath(root, real)
-                if not files and not dirs and rel_root != ".":
-                    # Preserve otherwise-empty directories.
-                    zf.writestr(os.path.join(name, rel_root) + "/", "")
-                for f in files:
-                    full = os.path.join(root, f)
-                    # os.path.isfile follows symlinks; combined with the islink
-                    # check it admits only real regular files (skips FIFOs,
-                    # sockets, devices and symlinks without ever open()-ing them).
-                    if os.path.islink(full) or not os.path.isfile(full):
-                        continue
-                    arc = (os.path.join(name, rel_root, f) if rel_root != "."
-                           else os.path.join(name, f))
-                    try:
-                        zf.write(full, arc)
-                    except OSError:
-                        # Unreadable or vanished mid-walk — skip just this file.
-                        continue
-    except OSError:
-        if os.path.exists(tmp):
-            os.unlink(tmp)
-        raise
-    return tmp, f"{name}.zip"
+
+def _iter_zip(real: str, name: str):
+    sink = _ZipBuffer()
+    with zipfile.ZipFile(sink, "w", zipfile.ZIP_DEFLATED) as zf:
+        for root, dirs, files in os.walk(real):
+            # Don't follow symlinked directories (avoids loops / escapes).
+            dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d))]
+            rel_root = os.path.relpath(root, real)
+            if not files and not dirs and rel_root != ".":
+                # Preserve otherwise-empty directories.
+                zf.writestr(os.path.join(name, rel_root) + "/", "")
+                if chunk := sink.take():
+                    yield chunk
+            for f in files:
+                full = os.path.join(root, f)
+                # os.path.isfile follows symlinks; combined with the islink
+                # check it admits only real regular files (skips FIFOs, sockets,
+                # devices and symlinks without ever open()-ing them).
+                if os.path.islink(full) or not os.path.isfile(full):
+                    continue
+                arc = (os.path.join(name, rel_root, f) if rel_root != "."
+                       else os.path.join(name, f))
+                try:
+                    info = zipfile.ZipInfo.from_file(full, arc)
+                    info.compress_type = zipfile.ZIP_DEFLATED
+                    with open(full, "rb") as src, zf.open(info, "w") as dest:
+                        while buf := src.read(1024 * 1024):
+                            dest.write(buf)
+                            if chunk := sink.take():
+                                yield chunk
+                except OSError:
+                    # Unreadable or vanished mid-walk — skip just this file.
+                    continue
+                if chunk := sink.take():
+                    yield chunk
+    yield sink.take()
 
 
 def upload_target(