upravy runnera

evaluation/rag_runner.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 import json
 import os
 import re
+import socket
+import sys
 import time
 import urllib.error
 import urllib.request
@@ -33,15 +35,53 @@ DEFAULT_MODEL = (
 
 DEFAULT_TIMEOUT = 180
 
+DEFAULT_MAX_ATTEMPTS = 4
+
+DEFAULT_BACKOFF_BASE = 1.0
+
+DEFAULT_BACKOFF_MAX = 8.0
+
+
+RETRYABLE_HTTP_STATUS_CODES = {
+    408,
+    425,
+    429,
+}
+
 
 MARKDOWN_URL_RE = re.compile(
     r"\[[^\]]*\]\((https?://[^)]+)\)"
 )
 
 
+class RequestError(
+    RuntimeError
+):
+    def __init__(
+        self,
+        message: str,
+        *,
+        retryable: bool,
+        status_code: int | None = None,
+    ) -> None:
+        super().__init__(
+            message
+        )
+
+        self.retryable = (
+            retryable
+        )
+
+        self.status_code = (
+            status_code
+        )
+
+
 def load_env_value(
     key: str,
     env_path: Path | None = None,
+    *,
+    allow_env_file: bool = True,
 ) -> str:
     value = os.environ.get(
         key
@@ -50,6 +90,12 @@ def load_env_value(
     if value:
         return value
 
+    if not allow_env_file:
+        raise RuntimeError(
+            f"{key} musí byť nastavený "
+            "priamo v environment."
+        )
+
     if env_path is None:
         env_path = (
             PROJECT_ROOT
@@ -104,6 +150,32 @@ def load_env_value(
     )
 
 
+def retry_delay_seconds(
+    attempt: int,
+    *,
+    backoff_base: float,
+    backoff_max: float,
+) -> float:
+    if attempt <= 0:
+        return 0.0
+
+    delay = (
+        backoff_base
+        * (
+            2
+            ** (
+                attempt
+                - 1
+            )
+        )
+    )
+
+    return min(
+        delay,
+        backoff_max,
+    )
+
+
 def request_json(
     url: str,
     *,
@@ -111,7 +183,30 @@ def request_json(
     headers: dict[str, str] | None = None,
     payload: dict[str, Any] | None = None,
     timeout: int = DEFAULT_TIMEOUT,
+    max_attempts: int = 1,
+    backoff_base: float = DEFAULT_BACKOFF_BASE,
+    backoff_max: float = DEFAULT_BACKOFF_MAX,
 ) -> dict[str, Any]:
+    if timeout <= 0:
+        raise ValueError(
+            "timeout musí byť > 0"
+        )
+
+    if max_attempts <= 0:
+        raise ValueError(
+            "max_attempts musí byť > 0"
+        )
+
+    if backoff_base < 0:
+        raise ValueError(
+            "backoff_base nesmie byť záporné"
+        )
+
+    if backoff_max < 0:
+        raise ValueError(
+            "backoff_max nesmie byť záporné"
+        )
+
     data = None
 
     if payload is not None:
@@ -122,6 +217,12 @@ def request_json(
             "utf-8"
         )
 
+    last_error: RequestError | None = None
+
+    for attempt in range(
+        1,
+        max_attempts + 1,
+    ):
         request = urllib.request.Request(
             url,
             data=data,
@@ -154,20 +255,56 @@ def request_json(
                 )
             )
 
-        raise RuntimeError(
+            status_code = int(
-            f"HTTP {exc.code} pre {url}: "
+                exc.code
+            )
+
+            retryable = (
+                status_code
+                in RETRYABLE_HTTP_STATUS_CODES
+                or (
+                    500
+                    <= status_code
+                    <= 599
+                )
+            )
+
+            error = RequestError(
+                (
+                    f"HTTP {status_code} "
+                    f"pre {url}: "
                     f"{body[:1500]}"
-        ) from exc
+                ),
+                retryable=(
+                    retryable
+                ),
+                status_code=(
+                    status_code
+                ),
+            )
 
-    except urllib.error.URLError as exc:
+        except (
-        raise RuntimeError(
+            urllib.error.URLError,
-            f"Sieťová chyba pre {url}: "
+            TimeoutError,
+            socket.timeout,
+        ) as exc:
+            error = RequestError(
+                (
+                    "Sieťová chyba "
+                    f"pre {url}: "
                     f"{exc}"
-        ) from exc
+                ),
+                retryable=True,
+            )
 
+        else:
             if not raw.strip():
-        raise RuntimeError(
+                raise RequestError(
-            f"Prázdna odpoveď z {url}."
+                    (
+                        "Prázdna odpoveď "
+                        f"z {url}."
+                    ),
+                    retryable=False,
                 )
 
             try:
@@ -176,23 +313,72 @@ def request_json(
                 )
 
             except json.JSONDecodeError as exc:
-        raise RuntimeError(
+                raise RequestError(
-            f"Neplatný JSON z {url}: "
+                    (
+                        "Neplatný JSON "
+                        f"z {url}: "
                         f"{raw[:1000]}"
+                    ),
+                    retryable=False,
                 ) from exc
 
             if not isinstance(
                 parsed,
                 dict,
             ):
-        raise RuntimeError(
+                raise RequestError(
+                    (
                         "Očakávaný JSON objekt "
                         f"z {url}, dostal som "
                         f"{type(parsed).__name__}."
+                    ),
+                    retryable=False,
                 )
 
             return parsed
 
+        last_error = error
+
+        if (
+            not error.retryable
+            or attempt >= max_attempts
+        ):
+            raise error
+
+        delay = retry_delay_seconds(
+            attempt,
+            backoff_base=(
+                backoff_base
+            ),
+            backoff_max=(
+                backoff_max
+            ),
+        )
+
+        print(
+            (
+                "    retry HTTP request: "
+                f"pokus {attempt + 1}/"
+                f"{max_attempts} "
+                f"za {delay:.1f}s "
+                f"({error})"
+            ),
+            file=sys.stderr,
+        )
+
+        if delay > 0:
+            time.sleep(
+                delay
+            )
+
+    if last_error is not None:
+        raise last_error
+
+    raise RuntimeError(
+        "HTTP request skončil "
+        "v neočakávanom stave."
+    )
+
 
 def resolve_refs(
     value: Any,
@@ -589,6 +775,9 @@ def run_question(
     openwebui_api_key: str,
     search_api_key: str,
     timeout: int,
+    max_attempts: int = DEFAULT_MAX_ATTEMPTS,
+    backoff_base: float = DEFAULT_BACKOFF_BASE,
+    backoff_max: float = DEFAULT_BACKOFF_MAX,
 ) -> dict[str, Any]:
     question = question.strip()
 
@@ -650,6 +839,9 @@ def run_question(
             "stream": False,
         },
         timeout=timeout,
+        max_attempts=max_attempts,
+        backoff_base=backoff_base,
+        backoff_max=backoff_max,
     )
 
     first_latency = (
@@ -800,6 +992,9 @@ def run_question(
             },
             payload=arguments,
             timeout=timeout,
+            max_attempts=max_attempts,
+            backoff_base=backoff_base,
+            backoff_max=backoff_max,
         )
 
         tool_latency = (
@@ -889,6 +1084,9 @@ def run_question(
             "stream": False,
         },
         timeout=timeout,
+        max_attempts=max_attempts,
+        backoff_base=backoff_base,
+        backoff_max=backoff_max,
     )
 
     final_latency = (