518 lines
9.6 KiB
Python
518 lines
9.6 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import urllib.error
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
import tools.rag.evaluation.rag_runner as rag_runner
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(
|
|
self,
|
|
body: str,
|
|
) -> None:
|
|
self.body = body
|
|
|
|
def __enter__(
|
|
self,
|
|
) -> FakeResponse:
|
|
return self
|
|
|
|
def __exit__(
|
|
self,
|
|
exc_type: Any,
|
|
exc_value: Any,
|
|
traceback: Any,
|
|
) -> bool:
|
|
return False
|
|
|
|
def read(
|
|
self,
|
|
) -> bytes:
|
|
return self.body.encode("utf-8")
|
|
|
|
|
|
def make_http_error(
|
|
code: int,
|
|
*,
|
|
body: str = "temporary error",
|
|
) -> urllib.error.HTTPError:
|
|
return urllib.error.HTTPError(
|
|
url="https://example.test/api",
|
|
code=code,
|
|
msg="test error",
|
|
hdrs=None,
|
|
fp=io.BytesIO(body.encode("utf-8")),
|
|
)
|
|
|
|
|
|
def test_retry_delay_seconds_uses_exponential_backoff() -> None:
|
|
assert (
|
|
rag_runner.retry_delay_seconds(
|
|
1,
|
|
backoff_base=1.0,
|
|
backoff_max=8.0,
|
|
)
|
|
== 1.0
|
|
)
|
|
|
|
assert (
|
|
rag_runner.retry_delay_seconds(
|
|
2,
|
|
backoff_base=1.0,
|
|
backoff_max=8.0,
|
|
)
|
|
== 2.0
|
|
)
|
|
|
|
assert (
|
|
rag_runner.retry_delay_seconds(
|
|
3,
|
|
backoff_base=1.0,
|
|
backoff_max=8.0,
|
|
)
|
|
== 4.0
|
|
)
|
|
|
|
assert (
|
|
rag_runner.retry_delay_seconds(
|
|
4,
|
|
backoff_base=1.0,
|
|
backoff_max=8.0,
|
|
)
|
|
== 8.0
|
|
)
|
|
|
|
assert (
|
|
rag_runner.retry_delay_seconds(
|
|
5,
|
|
backoff_base=1.0,
|
|
backoff_max=8.0,
|
|
)
|
|
== 8.0
|
|
)
|
|
|
|
|
|
def test_request_json_retries_http_429_then_succeeds(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
|
|
sleeps: list[float] = []
|
|
|
|
responses: list[Any] = [
|
|
make_http_error(429),
|
|
FakeResponse('{"ok": true}'),
|
|
]
|
|
|
|
def fake_urlopen(
|
|
request: Any,
|
|
timeout: int,
|
|
) -> FakeResponse:
|
|
nonlocal calls
|
|
|
|
calls += 1
|
|
|
|
response = responses[calls - 1]
|
|
|
|
if isinstance(
|
|
response,
|
|
Exception,
|
|
):
|
|
raise response
|
|
|
|
return response
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.urllib.request,
|
|
"urlopen",
|
|
fake_urlopen,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.time,
|
|
"sleep",
|
|
lambda delay: sleeps.append(delay),
|
|
)
|
|
|
|
result = rag_runner.request_json(
|
|
"https://example.test/api",
|
|
max_attempts=3,
|
|
backoff_base=1.0,
|
|
backoff_max=8.0,
|
|
)
|
|
|
|
assert result == {
|
|
"ok": True,
|
|
}
|
|
|
|
assert calls == 2
|
|
|
|
assert sleeps == [
|
|
1.0,
|
|
]
|
|
|
|
|
|
def test_request_json_retries_http_503_then_succeeds(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
|
|
sleeps: list[float] = []
|
|
|
|
responses: list[Any] = [
|
|
make_http_error(503),
|
|
FakeResponse('{"status": "ready"}'),
|
|
]
|
|
|
|
def fake_urlopen(
|
|
request: Any,
|
|
timeout: int,
|
|
) -> FakeResponse:
|
|
nonlocal calls
|
|
|
|
calls += 1
|
|
|
|
response = responses[calls - 1]
|
|
|
|
if isinstance(
|
|
response,
|
|
Exception,
|
|
):
|
|
raise response
|
|
|
|
return response
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.urllib.request,
|
|
"urlopen",
|
|
fake_urlopen,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.time,
|
|
"sleep",
|
|
lambda delay: sleeps.append(delay),
|
|
)
|
|
|
|
result = rag_runner.request_json(
|
|
"https://example.test/api",
|
|
max_attempts=3,
|
|
backoff_base=1.0,
|
|
backoff_max=8.0,
|
|
)
|
|
|
|
assert result == {
|
|
"status": "ready",
|
|
}
|
|
|
|
assert calls == 2
|
|
|
|
assert sleeps == [
|
|
1.0,
|
|
]
|
|
|
|
|
|
def test_request_json_does_not_retry_http_400(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
|
|
sleeps: list[float] = []
|
|
|
|
def fake_urlopen(
|
|
request: Any,
|
|
timeout: int,
|
|
) -> FakeResponse:
|
|
nonlocal calls
|
|
|
|
calls += 1
|
|
|
|
raise make_http_error(
|
|
400,
|
|
body="bad request",
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.urllib.request,
|
|
"urlopen",
|
|
fake_urlopen,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.time,
|
|
"sleep",
|
|
lambda delay: sleeps.append(delay),
|
|
)
|
|
|
|
with pytest.raises(rag_runner.RequestError) as exc_info:
|
|
rag_runner.request_json(
|
|
"https://example.test/api",
|
|
max_attempts=4,
|
|
backoff_base=1.0,
|
|
backoff_max=8.0,
|
|
)
|
|
|
|
error = exc_info.value
|
|
|
|
assert error.retryable is False
|
|
|
|
assert error.status_code == 400
|
|
|
|
assert calls == 1
|
|
|
|
assert sleeps == []
|
|
|
|
|
|
def test_request_json_retries_network_error_then_succeeds(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
|
|
sleeps: list[float] = []
|
|
|
|
def fake_urlopen(
|
|
request: Any,
|
|
timeout: int,
|
|
) -> FakeResponse:
|
|
nonlocal calls
|
|
|
|
calls += 1
|
|
|
|
if calls == 1:
|
|
raise urllib.error.URLError("temporary network problem")
|
|
|
|
return FakeResponse('{"ok": true}')
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.urllib.request,
|
|
"urlopen",
|
|
fake_urlopen,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.time,
|
|
"sleep",
|
|
lambda delay: sleeps.append(delay),
|
|
)
|
|
|
|
result = rag_runner.request_json(
|
|
"https://example.test/api",
|
|
max_attempts=3,
|
|
backoff_base=0.5,
|
|
backoff_max=8.0,
|
|
)
|
|
|
|
assert result == {
|
|
"ok": True,
|
|
}
|
|
|
|
assert calls == 2
|
|
|
|
assert sleeps == [
|
|
0.5,
|
|
]
|
|
|
|
|
|
def test_request_json_does_not_retry_invalid_json(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
|
|
sleeps: list[float] = []
|
|
|
|
def fake_urlopen(
|
|
request: Any,
|
|
timeout: int,
|
|
) -> FakeResponse:
|
|
nonlocal calls
|
|
|
|
calls += 1
|
|
|
|
return FakeResponse("this is not json")
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.urllib.request,
|
|
"urlopen",
|
|
fake_urlopen,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.time,
|
|
"sleep",
|
|
lambda delay: sleeps.append(delay),
|
|
)
|
|
|
|
with pytest.raises(rag_runner.RequestError) as exc_info:
|
|
rag_runner.request_json(
|
|
"https://example.test/api",
|
|
max_attempts=4,
|
|
)
|
|
|
|
error = exc_info.value
|
|
|
|
assert error.retryable is False
|
|
|
|
assert error.status_code is None
|
|
|
|
assert calls == 1
|
|
|
|
assert sleeps == []
|
|
|
|
|
|
def test_request_json_stops_after_max_attempts(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
|
|
sleeps: list[float] = []
|
|
|
|
def fake_urlopen(
|
|
request: Any,
|
|
timeout: int,
|
|
) -> FakeResponse:
|
|
nonlocal calls
|
|
|
|
calls += 1
|
|
|
|
raise make_http_error(503)
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.urllib.request,
|
|
"urlopen",
|
|
fake_urlopen,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
rag_runner.time,
|
|
"sleep",
|
|
lambda delay: sleeps.append(delay),
|
|
)
|
|
|
|
with pytest.raises(rag_runner.RequestError) as exc_info:
|
|
rag_runner.request_json(
|
|
"https://example.test/api",
|
|
max_attempts=3,
|
|
backoff_base=0.5,
|
|
backoff_max=8.0,
|
|
)
|
|
|
|
error = exc_info.value
|
|
|
|
assert error.retryable is True
|
|
|
|
assert error.status_code == 503
|
|
|
|
assert calls == 3
|
|
|
|
assert sleeps == [
|
|
0.5,
|
|
1.0,
|
|
]
|
|
|
|
|
|
def test_request_json_validates_retry_configuration() -> None:
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="timeout",
|
|
):
|
|
rag_runner.request_json(
|
|
"https://example.test/api",
|
|
timeout=0,
|
|
)
|
|
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="max_attempts",
|
|
):
|
|
rag_runner.request_json(
|
|
"https://example.test/api",
|
|
max_attempts=0,
|
|
)
|
|
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="backoff_base",
|
|
):
|
|
rag_runner.request_json(
|
|
"https://example.test/api",
|
|
backoff_base=-1.0,
|
|
)
|
|
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="backoff_max",
|
|
):
|
|
rag_runner.request_json(
|
|
"https://example.test/api",
|
|
backoff_max=-1.0,
|
|
)
|
|
|
|
|
|
def test_load_env_value_can_require_environment_only(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
env_path = tmp_path / ".env"
|
|
|
|
env_path.write_text(
|
|
("OPENWEBUI_API_KEY=" "secret-from-file\n"),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
monkeypatch.delenv(
|
|
"OPENWEBUI_API_KEY",
|
|
raising=False,
|
|
)
|
|
|
|
with pytest.raises(
|
|
RuntimeError,
|
|
match=("priamo v environment"),
|
|
):
|
|
rag_runner.load_env_value(
|
|
"OPENWEBUI_API_KEY",
|
|
env_path=env_path,
|
|
allow_env_file=False,
|
|
)
|
|
|
|
monkeypatch.setenv(
|
|
"OPENWEBUI_API_KEY",
|
|
"secret-from-environment",
|
|
)
|
|
|
|
value = rag_runner.load_env_value(
|
|
"OPENWEBUI_API_KEY",
|
|
env_path=env_path,
|
|
allow_env_file=False,
|
|
)
|
|
|
|
assert value == "secret-from-environment"
|
|
|
|
|
|
def test_load_env_value_preserves_env_file_fallback(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
env_path = tmp_path / ".env"
|
|
|
|
env_path.write_text(
|
|
("SEARCH_API_KEY=" "search-secret-from-file\n"),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
monkeypatch.delenv(
|
|
"SEARCH_API_KEY",
|
|
raising=False,
|
|
)
|
|
|
|
value = rag_runner.load_env_value(
|
|
"SEARCH_API_KEY",
|
|
env_path=env_path,
|
|
)
|
|
|
|
assert value == "search-secret-from-file"
|