diff --git a/evaluation/rag_runner.py b/evaluation/rag_runner.py index 29c389d..ce92281 100644 --- a/evaluation/rag_runner.py +++ b/evaluation/rag_runner.py @@ -3,6 +3,8 @@ from __future__ import annotations import json import os import re +import socket +import sys import time import urllib.error import urllib.request @@ -33,15 +35,53 @@ DEFAULT_MODEL = ( DEFAULT_TIMEOUT = 180 +DEFAULT_MAX_ATTEMPTS = 4 + +DEFAULT_BACKOFF_BASE = 1.0 + +DEFAULT_BACKOFF_MAX = 8.0 + + +RETRYABLE_HTTP_STATUS_CODES = { + 408, + 425, + 429, +} + MARKDOWN_URL_RE = re.compile( r"\[[^\]]*\]\((https?://[^)]+)\)" ) +class RequestError( + RuntimeError +): + def __init__( + self, + message: str, + *, + retryable: bool, + status_code: int | None = None, + ) -> None: + super().__init__( + message + ) + + self.retryable = ( + retryable + ) + + self.status_code = ( + status_code + ) + + def load_env_value( key: str, env_path: Path | None = None, + *, + allow_env_file: bool = True, ) -> str: value = os.environ.get( key @@ -50,6 +90,12 @@ def load_env_value( if value: return value + if not allow_env_file: + raise RuntimeError( + f"{key} musí byť nastavený " + "priamo v environment." + ) + if env_path is None: env_path = ( PROJECT_ROOT @@ -104,6 +150,32 @@ def load_env_value( ) +def retry_delay_seconds( + attempt: int, + *, + backoff_base: float, + backoff_max: float, +) -> float: + if attempt <= 0: + return 0.0 + + delay = ( + backoff_base + * ( + 2 + ** ( + attempt + - 1 + ) + ) + ) + + return min( + delay, + backoff_max, + ) + + def request_json( url: str, *, @@ -111,7 +183,30 @@ def request_json( headers: dict[str, str] | None = None, payload: dict[str, Any] | None = None, timeout: int = DEFAULT_TIMEOUT, + max_attempts: int = 1, + backoff_base: float = DEFAULT_BACKOFF_BASE, + backoff_max: float = DEFAULT_BACKOFF_MAX, ) -> dict[str, Any]: + if timeout <= 0: + raise ValueError( + "timeout musí byť > 0" + ) + + if max_attempts <= 0: + raise ValueError( + "max_attempts musí byť > 0" + ) + + if backoff_base < 0: + raise ValueError( + "backoff_base nesmie byť záporné" + ) + + if backoff_max < 0: + raise ValueError( + "backoff_max nesmie byť záporné" + ) + data = None if payload is not None: @@ -122,76 +217,167 @@ def request_json( "utf-8" ) - request = urllib.request.Request( - url, - data=data, - method=method, - headers=( - headers - or {} - ), - ) + last_error: RequestError | None = None - try: - with urllib.request.urlopen( - request, - timeout=timeout, - ) as response: - raw = ( - response - .read() + for attempt in range( + 1, + max_attempts + 1, + ): + request = urllib.request.Request( + url, + data=data, + method=method, + headers=( + headers + or {} + ), + ) + + try: + with urllib.request.urlopen( + request, + timeout=timeout, + ) as response: + raw = ( + response + .read() + .decode( + "utf-8" + ) + ) + + except urllib.error.HTTPError as exc: + body = ( + exc.read() .decode( - "utf-8" + "utf-8", + errors="replace", ) ) - except urllib.error.HTTPError as exc: - body = ( - exc.read() - .decode( - "utf-8", - errors="replace", + status_code = int( + exc.code ) + + retryable = ( + status_code + in RETRYABLE_HTTP_STATUS_CODES + or ( + 500 + <= status_code + <= 599 + ) + ) + + error = RequestError( + ( + f"HTTP {status_code} " + f"pre {url}: " + f"{body[:1500]}" + ), + retryable=( + retryable + ), + status_code=( + status_code + ), + ) + + except ( + urllib.error.URLError, + TimeoutError, + socket.timeout, + ) as exc: + error = RequestError( + ( + "Sieťová chyba " + f"pre {url}: " + f"{exc}" + ), + retryable=True, + ) + + else: + if not raw.strip(): + raise RequestError( + ( + "Prázdna odpoveď " + f"z {url}." + ), + retryable=False, + ) + + try: + parsed = json.loads( + raw + ) + + except json.JSONDecodeError as exc: + raise RequestError( + ( + "Neplatný JSON " + f"z {url}: " + f"{raw[:1000]}" + ), + retryable=False, + ) from exc + + if not isinstance( + parsed, + dict, + ): + raise RequestError( + ( + "Očakávaný JSON objekt " + f"z {url}, dostal som " + f"{type(parsed).__name__}." + ), + retryable=False, + ) + + return parsed + + last_error = error + + if ( + not error.retryable + or attempt >= max_attempts + ): + raise error + + delay = retry_delay_seconds( + attempt, + backoff_base=( + backoff_base + ), + backoff_max=( + backoff_max + ), ) - raise RuntimeError( - f"HTTP {exc.code} pre {url}: " - f"{body[:1500]}" - ) from exc - - except urllib.error.URLError as exc: - raise RuntimeError( - f"Sieťová chyba pre {url}: " - f"{exc}" - ) from exc - - if not raw.strip(): - raise RuntimeError( - f"Prázdna odpoveď z {url}." + print( + ( + " retry HTTP request: " + f"pokus {attempt + 1}/" + f"{max_attempts} " + f"za {delay:.1f}s " + f"({error})" + ), + file=sys.stderr, ) - try: - parsed = json.loads( - raw - ) + if delay > 0: + time.sleep( + delay + ) - except json.JSONDecodeError as exc: - raise RuntimeError( - f"Neplatný JSON z {url}: " - f"{raw[:1000]}" - ) from exc + if last_error is not None: + raise last_error - if not isinstance( - parsed, - dict, - ): - raise RuntimeError( - "Očakávaný JSON objekt " - f"z {url}, dostal som " - f"{type(parsed).__name__}." - ) - - return parsed + raise RuntimeError( + "HTTP request skončil " + "v neočakávanom stave." + ) def resolve_refs( @@ -589,6 +775,9 @@ def run_question( openwebui_api_key: str, search_api_key: str, timeout: int, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + backoff_base: float = DEFAULT_BACKOFF_BASE, + backoff_max: float = DEFAULT_BACKOFF_MAX, ) -> dict[str, Any]: question = question.strip() @@ -650,6 +839,9 @@ def run_question( "stream": False, }, timeout=timeout, + max_attempts=max_attempts, + backoff_base=backoff_base, + backoff_max=backoff_max, ) first_latency = ( @@ -800,6 +992,9 @@ def run_question( }, payload=arguments, timeout=timeout, + max_attempts=max_attempts, + backoff_base=backoff_base, + backoff_max=backoff_max, ) tool_latency = ( @@ -889,6 +1084,9 @@ def run_question( "stream": False, }, timeout=timeout, + max_attempts=max_attempts, + backoff_base=backoff_base, + backoff_max=backoff_max, ) final_latency = (