Stop long prunes failing with a bare 500 (0.60.1)
A prune that deletes a lot of data (image prune -a in particular) easily runs past docker-py's 60s read timeout. requests' ReadTimeout is not a DockerException, so it slipped past safe_call and the UI only showed "Request failed with status code 500". Prunes now use a client with a 540s timeout (below nginx's 600s proxy_read_timeout), and safe_call turns transport timeouts and connection errors into DockerError, so they surface as a 502 with a message. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8edea8f971
commit
f766ded528
7 files changed
+120
-11
No files matched your search
@@ -9,6 +9,7 @@ import functools
|
||||
from typing import Any
|
||||
|
||||
import docker
|
||||
import requests
|
||||
from docker.errors import APIError, DockerException, NotFound
|
||||
|
||||
from config import settings
|
||||
@@ -23,19 +24,37 @@ class DockerError(Exception):
|
||||
super().__init__(f"{error}: {detail}" if detail else error)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def get_client() -> docker.DockerClient:
|
||||
# Read timeout for get_long_client(). docker-py's default of 60s is fine for
|
||||
# interactive calls, but a prune that deletes gigabytes of volume data or image
|
||||
# layers routinely takes longer. Kept below nginx's proxy_read_timeout (600s)
|
||||
# so the caller still gets our error instead of a bare gateway timeout.
|
||||
LONG_TIMEOUT_SECONDS = 540
|
||||
|
||||
|
||||
def _connect(**kwargs) -> docker.DockerClient:
|
||||
try:
|
||||
base_url = f"unix://{settings.DOCKER_SOCKET}"
|
||||
client = docker.DockerClient(base_url=base_url)
|
||||
client = docker.DockerClient(base_url=base_url, **kwargs)
|
||||
client.ping()
|
||||
return client
|
||||
except DockerException as exc: # pragma: no cover - environment dependent
|
||||
raise DockerError("docker_unavailable", str(exc)) from exc
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def get_client() -> docker.DockerClient:
|
||||
return _connect()
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def get_long_client() -> docker.DockerClient:
|
||||
"""A client for calls that can legitimately run for minutes (prunes)."""
|
||||
return _connect(timeout=LONG_TIMEOUT_SECONDS)
|
||||
|
||||
|
||||
def reset_client() -> None:
|
||||
get_client.cache_clear()
|
||||
get_long_client.cache_clear()
|
||||
|
||||
|
||||
def safe_call(fn, *args, **kwargs) -> Any:
|
||||
@@ -48,3 +67,12 @@ def safe_call(fn, *args, **kwargs) -> Any:
|
||||
raise DockerError("docker_api_error", str(exc)) from exc
|
||||
except DockerException as exc:
|
||||
raise DockerError("docker_error", str(exc)) from exc
|
||||
# docker-py lets transport errors from requests through unwrapped; without
|
||||
# these a slow daemon surfaced as a bare 500 with no message.
|
||||
except requests.exceptions.Timeout as exc:
|
||||
raise DockerError(
|
||||
"docker_timeout",
|
||||
"Docker did not answer in time; the operation may still be running on the host",
|
||||
) from exc
|
||||
except requests.exceptions.RequestException as exc:
|
||||
raise DockerError("docker_unavailable", str(exc)) from exc
|
||||
Reference in new issue
Block a user