upravy runnera

evaluation/rag_runner.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 import json
 import os
 import re
+import socket
+import sys
 import time
 import urllib.error
 import urllib.request
@@ -33,15 +35,53 @@ DEFAULT_MODEL = (
 
 DEFAULT_TIMEOUT = 180
 
+DEFAULT_MAX_ATTEMPTS = 4
+
+DEFAULT_BACKOFF_BASE = 1.0
+
+DEFAULT_BACKOFF_MAX = 8.0
+
+
+RETRYABLE_HTTP_STATUS_CODES = {
+    408,
+    425,
+    429,
+}
+
 
 MARKDOWN_URL_RE = re.compile(
     r"\[[^\]]*\]\((https?://[^)]+)\)"
 )
 
 
+class RequestError(
+    RuntimeError
+):
+    def __init__(
+        self,
+        message: str,
+        *,
+        retryable: bool,
+        status_code: int | None = None,
+    ) -> None:
+        super().__init__(
+            message
+        )
+
+        self.retryable = (
+            retryable
+        )
+
+        self.status_code = (
+            status_code
+        )
+
+
 def load_env_value(
     key: str,
     env_path: Path | None = None,
+    *,
+    allow_env_file: bool = True,
 ) -> str:
     value = os.environ.get(
         key
@@ -50,6 +90,12 @@ def load_env_value(
     if value:
         return value
 
+    if not allow_env_file:
+        raise RuntimeError(
+            f"{key} musí byť nastavený "
+            "priamo v environment."
+        )
+
     if env_path is None:
         env_path = (
             PROJECT_ROOT
@@ -104,6 +150,32 @@ def load_env_value(
     )
 
 
+def retry_delay_seconds(
+    attempt: int,
+    *,
+    backoff_base: float,
+    backoff_max: float,
+) -> float:
+    if attempt <= 0:
+        return 0.0
+
+    delay = (
+        backoff_base
+        * (
+            2
+            ** (
+                attempt
+                - 1
+            )
+        )
+    )
+
+    return min(
+        delay,
+        backoff_max,
+    )
+
+
 def request_json(
     url: str,
     *,
@@ -111,7 +183,30 @@ def request_json(
     headers: dict[str, str] | None = None,
     payload: dict[str, Any] | None = None,
     timeout: int = DEFAULT_TIMEOUT,
+    max_attempts: int = 1,
+    backoff_base: float = DEFAULT_BACKOFF_BASE,
+    backoff_max: float = DEFAULT_BACKOFF_MAX,
 ) -> dict[str, Any]:
+    if timeout <= 0:
+        raise ValueError(
+            "timeout musí byť > 0"
+        )
+
+    if max_attempts <= 0:
+        raise ValueError(
+            "max_attempts musí byť > 0"
+        )
+
+    if backoff_base < 0:
+        raise ValueError(
+            "backoff_base nesmie byť záporné"
+        )
+
+    if backoff_max < 0:
+        raise ValueError(
+            "backoff_max nesmie byť záporné"
+        )
+
     data = None
 
     if payload is not None:
@@ -122,76 +217,167 @@ def request_json(
             "utf-8"
         )
 
-    request = urllib.request.Request(
+    last_error: RequestError | None = None
-        url,
-        data=data,
-        method=method,
-        headers=(
-            headers
-            or {}
-        ),
-    )
 
-    try:
+    for attempt in range(
-        with urllib.request.urlopen(
+        1,
-            request,
+        max_attempts + 1,
-            timeout=timeout,
+    ):
-        ) as response:
+        request = urllib.request.Request(
-            raw = (
+            url,
-                response
+            data=data,
-                .read()
+            method=method,
+            headers=(
+                headers
+                or {}
+            ),
+        )
+
+        try:
+            with urllib.request.urlopen(
+                request,
+                timeout=timeout,
+            ) as response:
+                raw = (
+                    response
+                    .read()
+                    .decode(
+                        "utf-8"
+                    )
+                )
+
+        except urllib.error.HTTPError as exc:
+            body = (
+                exc.read()
                 .decode(
-                    "utf-8"
+                    "utf-8",
+                    errors="replace",
                 )
             )
 
-    except urllib.error.HTTPError as exc:
+            status_code = int(
-        body = (
+                exc.code
-            exc.read()
-            .decode(
-                "utf-8",
-                errors="replace",
             )
+
+            retryable = (
+                status_code
+                in RETRYABLE_HTTP_STATUS_CODES
+                or (
+                    500
+                    <= status_code
+                    <= 599
+                )
+            )
+
+            error = RequestError(
+                (
+                    f"HTTP {status_code} "
+                    f"pre {url}: "
+                    f"{body[:1500]}"
+                ),
+                retryable=(
+                    retryable
+                ),
+                status_code=(
+                    status_code
+                ),
+            )
+
+        except (
+            urllib.error.URLError,
+            TimeoutError,
+            socket.timeout,
+        ) as exc:
+            error = RequestError(
+                (
+                    "Sieťová chyba "
+                    f"pre {url}: "
+                    f"{exc}"
+                ),
+                retryable=True,
+            )
+
+        else:
+            if not raw.strip():
+                raise RequestError(
+                    (
+                        "Prázdna odpoveď "
+                        f"z {url}."
+                    ),
+                    retryable=False,
+                )
+
+            try:
+                parsed = json.loads(
+                    raw
+                )
+
+            except json.JSONDecodeError as exc:
+                raise RequestError(
+                    (
+                        "Neplatný JSON "
+                        f"z {url}: "
+                        f"{raw[:1000]}"
+                    ),
+                    retryable=False,
+                ) from exc
+
+            if not isinstance(
+                parsed,
+                dict,
+            ):
+                raise RequestError(
+                    (
+                        "Očakávaný JSON objekt "
+                        f"z {url}, dostal som "
+                        f"{type(parsed).__name__}."
+                    ),
+                    retryable=False,
+                )
+
+            return parsed
+
+        last_error = error
+
+        if (
+            not error.retryable
+            or attempt >= max_attempts
+        ):
+            raise error
+
+        delay = retry_delay_seconds(
+            attempt,
+            backoff_base=(
+                backoff_base
+            ),
+            backoff_max=(
+                backoff_max
+            ),
         )
 
-        raise RuntimeError(
+        print(
-            f"HTTP {exc.code} pre {url}: "
+            (
-            f"{body[:1500]}"
+                "    retry HTTP request: "
-        ) from exc
+                f"pokus {attempt + 1}/"
-
+                f"{max_attempts} "
-    except urllib.error.URLError as exc:
+                f"za {delay:.1f}s "
-        raise RuntimeError(
+                f"({error})"
-            f"Sieťová chyba pre {url}: "
+            ),
-            f"{exc}"
+            file=sys.stderr,
-        ) from exc
-
-    if not raw.strip():
-        raise RuntimeError(
-            f"Prázdna odpoveď z {url}."
         )
 
-    try:
+        if delay > 0:
-        parsed = json.loads(
+            time.sleep(
-            raw
+                delay
-        )
+            )
 
-    except json.JSONDecodeError as exc:
+    if last_error is not None:
-        raise RuntimeError(
+        raise last_error
-            f"Neplatný JSON z {url}: "
-            f"{raw[:1000]}"
-        ) from exc
 
-    if not isinstance(
+    raise RuntimeError(
-        parsed,
+        "HTTP request skončil "
-        dict,
+        "v neočakávanom stave."
-    ):
+    )
-        raise RuntimeError(
-            "Očakávaný JSON objekt "
-            f"z {url}, dostal som "
-            f"{type(parsed).__name__}."
-        )
-
-    return parsed
 
 
 def resolve_refs(
@@ -589,6 +775,9 @@ def run_question(
     openwebui_api_key: str,
     search_api_key: str,
     timeout: int,
+    max_attempts: int = DEFAULT_MAX_ATTEMPTS,
+    backoff_base: float = DEFAULT_BACKOFF_BASE,
+    backoff_max: float = DEFAULT_BACKOFF_MAX,
 ) -> dict[str, Any]:
     question = question.strip()
 
@@ -650,6 +839,9 @@ def run_question(
             "stream": False,
         },
         timeout=timeout,
+        max_attempts=max_attempts,
+        backoff_base=backoff_base,
+        backoff_max=backoff_max,
     )
 
     first_latency = (
@@ -800,6 +992,9 @@ def run_question(
             },
             payload=arguments,
             timeout=timeout,
+            max_attempts=max_attempts,
+            backoff_base=backoff_base,
+            backoff_max=backoff_max,
         )
 
         tool_latency = (
@@ -889,6 +1084,9 @@ def run_question(
             "stream": False,
         },
         timeout=timeout,
+        max_attempts=max_attempts,
+        backoff_base=backoff_base,
+        backoff_max=backoff_max,
     )
 
     final_latency = (