jp170na/dp-zp-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

uprava rag runnera

Browse Source
This commit is contained in:
Ján Pták 2026-09-27 14:56:22 +02:00
parent 312cb52843
commit 98da217151
1 changed files with 169 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

565
evaluation/rag_runner.py
View File

@ -12,94 +12,49 @@ from pathlib import Path
from typing import Any from typing import Any
PROJECT_ROOT = Path( PROJECT_ROOT = Path(__file__).resolve().parents[1]
__file__
).resolve().parents[1]
OPENWEBUI_URL = "https://ui.tukekemt.xyz/api/chat/completions"
LOCAL_OPENAPI_URL = "http://localhost:8000/openapi.json"
LOCAL_RAG_URL = "http://localhost:8000/rag"
OPENWEBUI_URL = ( DEFAULT_MODEL = "model120-fast"
"https://ui.tukekemt.xyz/api/chat/completions"
)
LOCAL_OPENAPI_URL = (
"http://localhost:8000/openapi.json"
)
LOCAL_RAG_URL = (
"http://localhost:8000/rag"
)
DEFAULT_MODEL = (
"model120-fast"
)
DEFAULT_TIMEOUT = 180 DEFAULT_TIMEOUT = 180
DEFAULT_MAX_ATTEMPTS = 4 DEFAULT_MAX_ATTEMPTS = 4
DEFAULT_BACKOFF_BASE = 1.0 DEFAULT_BACKOFF_BASE = 1.0
DEFAULT_BACKOFF_MAX = 8.0 DEFAULT_BACKOFF_MAX = 8.0
RETRYABLE_HTTP_STATUS_CODES = { RETRYABLE_HTTP_STATUS_CODES = {
408, 408,
425, 425,
429, 429,
} }
MARKDOWN_URL_RE = re.compile( MARKDOWN_URL_RE = re.compile(
r"\[[^\]]*\]\((https?://[^)]+)\)" r"\[[^\]]*\]\((https?://[^)]+)\)"
) )
TOOL_ROUTING_SYSTEM_PROMPT = """ TOOL_ROUTING_SYSTEM_PROMPT = (
Si prvá fáza RAG agenta pre ZP Wiki. "Si evaluačný klient pre ZP Agent. Pri každej otázke najprv použi "
"nástroj retrieve_zpwiki_context na vyhľadanie dôkazov v ZP Wiki. "
Tvojou úlohou je pri otázkach, ktoré vyžadujú informácie "Neodpovedaj z vlastnej pamäte a pred prvým vyhľadaním nežiadaj "
zo ZP Wiki, zavolať dostupný retrieval tool. "používateľa o spresnenie. Pri tvorbe argumentu query zachovaj všetky významovo "
"dôležité obmedzenia pôvodnej otázky a zachovaj celý význam pôvodnej otázky: "
Pri vytváraní argumentu `query` zachovaj všetky významovo "meno osoby alebo názov dokumentu, typ práce "
rozlišujúce informácie z používateľovej otázky. "(bakalárska, diplomová, diplomový projekt, stáž), požadovaný atribút "
"(názov, rok, téma, hardvér, databáza, backend, počet, nástroj), rok "
Zachovaj najmä: "a dôležité odborné kľúčové slová. Nezredukuj otázku iba na meno osoby. "
- meno alebo názov osoby, dokumentu alebo témy, "Ak je otázka krátka alebo kontextová, napríklad sa pýta na chatbot, "
- typ práce alebo projektu, napríklad bakalárska práca, "Question Answering, anotáciu alebo projekt bez mena osoby, aj tak najprv "
diplomová práca, dizertačná práca, diplomový projekt "vyhľadaj pôvodnú otázku alebo jej hlavné pojmy v ZP Wiki. Pri otázke na "
alebo tímový projekt, "viacero dokumentov zachovaj v query všetky tematické pojmy. Po získaní "
- požadovaný atribút, napríklad názov, téma, rok, autor, "výsledku odpovedz iba podľa vráteného kontextu a cituj iba source_url, "
cieľ, metóda alebo zadanie, "ktoré tvoju odpoveď podporujú."
- explicitne uvedený rok, )
- ďalšie odborné alebo rozlišujúce kľúčové slová.
Query môže byť stručnejší než pôvodná otázka, ale nesmie
stratiť významové obmedzenia potrebné na správne vyhľadanie.
Napríklad otázku:
"Aká téma diplomovej práce je uvedená pri osobe Ján Novák?"
je vhodné previesť na query podobný:
"Ján Novák diplomová práca téma"
Nevhodné je zredukovať ju iba na:
"Ján Novák"
Podobne pri otázke na rok diplomovej práce musí query
zachovať meno osoby, pojem diplomová práca a informáciu,
že sa hľadá rok.
Nevymýšľaj hodnoty, ktoré používateľ v otázke neuviedol.
Nevkladaj do query predpokladanú odpoveď.
Ak je potrebné použiť ZP Wiki, najprv zavolaj retrieval tool.
Po získaní výsledku odpovedaj iba podľa informácií,
inštrukcií, kontextu a zdrojov vrátených toolom.
""".strip()
class RequestError( class RequestError(RuntimeError):
RuntimeError
):
def __init__( def __init__(
self, self,
message: str, message: str,
@ -107,17 +62,9 @@ class RequestError(
retryable: bool, retryable: bool,
status_code: int | None = None, status_code: int | None = None,
) -> None: ) -> None:
super().__init__( super().__init__(message)
message self.retryable = retryable
) self.status_code = status_code
self.retryable = (
retryable
)
self.status_code = (
status_code
)
def load_env_value( def load_env_value(
@ -126,34 +73,25 @@ def load_env_value(
*, *,
allow_env_file: bool = True, allow_env_file: bool = True,
) -> str: ) -> str:
value = os.environ.get( value = os.environ.get(key)
key
)
if value: if value:
return value return value
if not allow_env_file: if not allow_env_file:
raise RuntimeError( raise RuntimeError(
f"{key} musí byť nastavený " f"{key} musí byť nastavený priamo v environment."
"priamo v environment."
) )
if env_path is None: if env_path is None:
env_path = ( env_path = PROJECT_ROOT / ".env"
PROJECT_ROOT
/ ".env"
)
if not env_path.exists(): if not env_path.exists():
raise RuntimeError( raise RuntimeError(
f"{key} nie je v environment " f"{key} nie je v environment a {env_path} neexistuje."
f"a {env_path} neexistuje."
) )
for raw_line in env_path.read_text( for raw_line in env_path.read_text(encoding="utf-8").splitlines():
encoding="utf-8"
).splitlines():
line = raw_line.strip() line = raw_line.strip()
if ( if (
@ -163,10 +101,7 @@ def load_env_value(
): ):
continue continue
name, value = line.split( name, value = line.split("=", 1)
"=",
1,
)
if name.strip() != key: if name.strip() != key:
continue continue
@ -176,14 +111,9 @@ def load_env_value(
if ( if (
len(value) >= 2 len(value) >= 2
and value[0] == value[-1] and value[0] == value[-1]
and value[0] in { and value[0] in {"'", '"'}
"'",
'"',
}
): ):
value = value[ value = value[1:-1]
1:-1
]
if value: if value:
return value return value
@ -202,21 +132,8 @@ def retry_delay_seconds(
if attempt <= 0: if attempt <= 0:
return 0.0 return 0.0
delay = ( delay = backoff_base * (2 ** (attempt - 1))
backoff_base return min(delay, backoff_max)
* (
2
** (
attempt
- 1
)
)
)
return min(
delay,
backoff_max,
)
def request_json( def request_json(
@ -231,24 +148,16 @@ def request_json(
backoff_max: float = DEFAULT_BACKOFF_MAX, backoff_max: float = DEFAULT_BACKOFF_MAX,
) -> dict[str, Any]: ) -> dict[str, Any]:
if timeout <= 0: if timeout <= 0:
raise ValueError( raise ValueError("timeout musí byť > 0")
"timeout musí byť > 0"
)
if max_attempts <= 0: if max_attempts <= 0:
raise ValueError( raise ValueError("max_attempts musí byť > 0")
"max_attempts musí byť > 0"
)
if backoff_base < 0: if backoff_base < 0:
raise ValueError( raise ValueError("backoff_base nesmie byť záporné")
"backoff_base nesmie byť záporné"
)
if backoff_max < 0: if backoff_max < 0:
raise ValueError( raise ValueError("backoff_max nesmie byť záporné")
"backoff_max nesmie byť záporné"
)
data = None data = None
@ -256,24 +165,16 @@ def request_json(
data = json.dumps( data = json.dumps(
payload, payload,
ensure_ascii=False, ensure_ascii=False,
).encode( ).encode("utf-8")
"utf-8"
)
last_error: RequestError | None = None last_error: RequestError | None = None
for attempt in range( for attempt in range(1, max_attempts + 1):
1,
max_attempts + 1,
):
request = urllib.request.Request( request = urllib.request.Request(
url, url,
data=data, data=data,
method=method, method=method,
headers=( headers=headers or {},
headers
or {}
),
) )
try: try:
@ -281,49 +182,25 @@ def request_json(
request, request,
timeout=timeout, timeout=timeout,
) as response: ) as response:
raw = ( raw = response.read().decode("utf-8")
response
.read()
.decode(
"utf-8"
)
)
except urllib.error.HTTPError as exc: except urllib.error.HTTPError as exc:
body = ( body = exc.read().decode(
exc.read() "utf-8",
.decode( errors="replace",
"utf-8",
errors="replace",
)
) )
status_code = int( status_code = int(exc.code)
exc.code
)
retryable = ( retryable = (
status_code status_code in RETRYABLE_HTTP_STATUS_CODES
in RETRYABLE_HTTP_STATUS_CODES or 500 <= status_code <= 599
or (
500
<= status_code
<= 599
)
) )
error = RequestError( error = RequestError(
( f"HTTP {status_code} pre {url}: {body[:1500]}",
f"HTTP {status_code} " retryable=retryable,
f"pre {url}: " status_code=status_code,
f"{body[:1500]}"
),
retryable=(
retryable
),
status_code=(
status_code
),
) )
except ( except (
@ -332,49 +209,30 @@ def request_json(
socket.timeout, socket.timeout,
) as exc: ) as exc:
error = RequestError( error = RequestError(
( f"Sieťová chyba pre {url}: {exc}",
"Sieťová chyba "
f"pre {url}: "
f"{exc}"
),
retryable=True, retryable=True,
) )
else: else:
if not raw.strip(): if not raw.strip():
raise RequestError( raise RequestError(
( f"Prázdna odpoveď z {url}.",
"Prázdna odpoveď "
f"z {url}."
),
retryable=False, retryable=False,
) )
try: try:
parsed = json.loads( parsed = json.loads(raw)
raw
)
except json.JSONDecodeError as exc: except json.JSONDecodeError as exc:
raise RequestError( raise RequestError(
( f"Neplatný JSON z {url}: {raw[:1000]}",
"Neplatný JSON "
f"z {url}: "
f"{raw[:1000]}"
),
retryable=False, retryable=False,
) from exc ) from exc
if not isinstance( if not isinstance(parsed, dict):
parsed,
dict,
):
raise RequestError( raise RequestError(
( "Očakávaný JSON objekt "
"Očakávaný JSON objekt " f"z {url}, dostal som {type(parsed).__name__}.",
f"z {url}, dostal som "
f"{type(parsed).__name__}."
),
retryable=False, retryable=False,
) )
@ -390,36 +248,25 @@ def request_json(
delay = retry_delay_seconds( delay = retry_delay_seconds(
attempt, attempt,
backoff_base=( backoff_base=backoff_base,
backoff_base backoff_max=backoff_max,
),
backoff_max=(
backoff_max
),
) )
print( print(
( " retry HTTP request: "
" retry HTTP request: " f"pokus {attempt + 1}/{max_attempts} "
f"pokus {attempt + 1}/" f"za {delay:.1f}s ({error})",
f"{max_attempts} "
f"za {delay:.1f}s "
f"({error})"
),
file=sys.stderr, file=sys.stderr,
) )
if delay > 0: if delay > 0:
time.sleep( time.sleep(delay)
delay
)
if last_error is not None: if last_error is not None:
raise last_error raise last_error
raise RuntimeError( raise RuntimeError(
"HTTP request skončil " "HTTP request skončil v neočakávanom stave."
"v neočakávanom stave."
) )
@ -427,45 +274,27 @@ def resolve_refs(
value: Any, value: Any,
document: dict[str, Any], document: dict[str, Any],
) -> Any: ) -> Any:
if isinstance( if isinstance(value, list):
value,
list,
):
return [ return [
resolve_refs( resolve_refs(item, document)
item,
document,
)
for item in value for item in value
] ]
if not isinstance( if not isinstance(value, dict):
value,
dict,
):
return value return value
ref = value.get( ref = value.get("$ref")
"$ref"
)
if ref: if ref:
if not ref.startswith( if not ref.startswith("#/"):
"#/"
):
raise RuntimeError( raise RuntimeError(
"Nepodporovaný " f"Nepodporovaný OpenAPI $ref: {ref}"
f"OpenAPI $ref: {ref}"
) )
current: Any = document current: Any = document
for part in ref[ for part in ref[2:].split("/"):
2: current = current[part]
].split("/"):
current = current[
part
]
resolved = resolve_refs( resolved = resolve_refs(
current, current,
@ -474,18 +303,11 @@ def resolve_refs(
extra = { extra = {
key: item key: item
for key, item for key, item in value.items()
in value.items()
if key != "$ref" if key != "$ref"
} }
if ( if extra and isinstance(resolved, dict):
extra
and isinstance(
resolved,
dict,
)
):
resolved = { resolved = {
**resolved, **resolved,
**resolve_refs( **resolve_refs(
@ -501,32 +323,25 @@ def resolve_refs(
item, item,
document, document,
) )
for key, item for key, item in value.items()
in value.items()
} }
def build_rag_tool( def build_rag_tool(
openapi: dict[str, Any], openapi: dict[str, Any],
) -> tuple[ ) -> tuple[str, dict[str, Any]]:
str,
dict[str, Any],
]:
try: try:
operation = ( operation = openapi[
openapi[ "paths"
"paths" ][
][ "/rag"
"/rag" ][
][ "post"
"post" ]
]
)
except KeyError as exc: except KeyError as exc:
raise RuntimeError( raise RuntimeError(
"OpenAPI schéma neobsahuje " "OpenAPI schéma neobsahuje POST /rag."
"POST /rag."
) from exc ) from exc
operation_id = operation.get( operation_id = operation.get(
@ -539,22 +354,19 @@ def build_rag_tool(
) )
try: try:
schema = ( schema = operation[
operation[ "requestBody"
"requestBody" ][
][ "content"
"content" ][
][ "application/json"
"application/json" ][
][ "schema"
"schema" ]
]
)
except KeyError as exc: except KeyError as exc:
raise RuntimeError( raise RuntimeError(
"POST /rag nemá " "POST /rag nemá request JSON schema."
"request JSON schema."
) from exc ) from exc
parameters = resolve_refs( parameters = resolve_refs(
@ -579,12 +391,8 @@ def build_rag_tool(
"type": "function", "type": "function",
"function": { "function": {
"name": operation_id, "name": operation_id,
"description": ( "description": description,
description "parameters": parameters,
),
"parameters": (
parameters
),
}, },
} }
@ -599,10 +407,8 @@ def normalize_url(
) -> str: ) -> str:
value = value.strip() value = value.strip()
match = ( match = MARKDOWN_URL_RE.search(
MARKDOWN_URL_RE.search( value
value
)
) )
if match: if match:
@ -618,14 +424,9 @@ def normalize_url(
def extract_urls_from_object( def extract_urls_from_object(
value: Any, value: Any,
) -> list[str]: ) -> list[str]:
result: list[ result: list[str] = []
str
] = []
if isinstance( if isinstance(value, dict):
value,
dict,
):
for key, item in value.items(): for key, item in value.items():
if ( if (
key == "source_url" key == "source_url"
@ -646,10 +447,7 @@ def extract_urls_from_object(
) )
) )
elif isinstance( elif isinstance(value, list):
value,
list,
):
for item in value: for item in value:
result.extend( result.extend(
extract_urls_from_object( extract_urls_from_object(
@ -781,8 +579,7 @@ def parse_tool_arguments(
str, str,
): ):
raise RuntimeError( raise RuntimeError(
"Neplatný formát " "Neplatný formát tool arguments."
"tool arguments."
) )
try: try:
@ -802,8 +599,7 @@ def parse_tool_arguments(
dict, dict,
): ):
raise RuntimeError( raise RuntimeError(
"Tool arguments nie sú " "Tool arguments nie sú JSON objekt."
"JSON objekt."
) )
return parsed return parsed
@ -812,9 +608,7 @@ def parse_tool_arguments(
def build_initial_messages( def build_initial_messages(
question: str, question: str,
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
question = ( question = question.strip()
question.strip()
)
if not question: if not question:
raise RuntimeError( raise RuntimeError(
@ -835,6 +629,17 @@ def build_initial_messages(
] ]
def forced_tool_choice(
operation_id: str,
) -> dict[str, Any]:
return {
"type": "function",
"function": {
"name": operation_id,
},
}
def run_question( def run_question(
question: str, question: str,
*, *,
@ -855,9 +660,7 @@ def run_question(
"Otázka je prázdna." "Otázka je prázdna."
) )
started = ( started = time.perf_counter()
time.perf_counter()
)
usage_total = { usage_total = {
"prompt_tokens": 0, "prompt_tokens": 0,
@ -873,23 +676,18 @@ def run_question(
str str
] = [] ] = []
messages = ( messages = build_initial_messages(
build_initial_messages( question
question
)
) )
first_started = ( first_started = time.perf_counter()
time.perf_counter()
)
first_response = request_json( first_response = request_json(
OPENWEBUI_URL, OPENWEBUI_URL,
method="POST", method="POST",
headers={ headers={
"Authorization": ( "Authorization": (
"Bearer " f"Bearer {openwebui_api_key}"
f"{openwebui_api_key}"
), ),
"Content-Type": ( "Content-Type": (
"application/json" "application/json"
@ -942,47 +740,25 @@ def run_question(
tool_calls = [] tool_calls = []
if not tool_calls: if not tool_calls:
answer = str( tool_calls = [
first_message.get( {
"content" "id": (
) "fallback_zpwiki_search"
or "" ),
) "type": "function",
"function": {
total_latency = ( "name": (
time.perf_counter() operation_id
- started ),
) "arguments": json.dumps(
{
return { "query": question,
"answer": answer, },
"tool_called": False, ensure_ascii=False,
"tool_call_count": 0, ),
"tool_calls": [], },
"rag_source_urls": [], }
"first_model_latency_seconds": ( ]
round(
first_latency,
6,
)
),
"tool_latency_seconds": 0.0,
"final_model_latency_seconds": 0.0,
"total_latency_seconds": (
round(
total_latency,
6,
)
),
"usage": (
usage_total
),
"response_model": (
first_response.get(
"model"
)
),
}
messages.append( messages.append(
{ {
@ -1041,6 +817,16 @@ def run_question(
) )
) )
# Query vždy prepíšeme celou
# pôvodnou používateľskou otázkou.
# Model si môže ponechať iba
# neškodné voliteľné parametre
# ako limit/published_only/
# max_per_document.
arguments[
"query"
] = question
tool_started = ( tool_started = (
time.perf_counter() time.perf_counter()
) )
@ -1091,11 +877,9 @@ def run_question(
{ {
"name": name, "name": name,
"arguments": arguments, "arguments": arguments,
"latency_seconds": ( "latency_seconds": round(
round( tool_latency,
tool_latency, 6,
6,
)
), ),
"source_urls": ( "source_urls": (
current_urls current_urls
@ -1137,8 +921,7 @@ def run_question(
method="POST", method="POST",
headers={ headers={
"Authorization": ( "Authorization": (
"Bearer " f"Bearer {openwebui_api_key}"
f"{openwebui_api_key}"
), ),
"Content-Type": ( "Content-Type": (
"application/json" "application/json"
@ -1197,33 +980,23 @@ def run_question(
"rag_source_urls": ( "rag_source_urls": (
rag_source_urls rag_source_urls
), ),
"first_model_latency_seconds": ( "first_model_latency_seconds": round(
round( first_latency,
first_latency, 6,
6,
)
), ),
"tool_latency_seconds": ( "tool_latency_seconds": round(
round( tool_latency_total,
tool_latency_total, 6,
6,
)
), ),
"final_model_latency_seconds": ( "final_model_latency_seconds": round(
round( final_latency,
final_latency, 6,
6,
)
), ),
"total_latency_seconds": ( "total_latency_seconds": round(
round( total_latency,
total_latency, 6,
6,
)
),
"usage": (
usage_total
), ),
"usage": usage_total,
"response_model": ( "response_model": (
final_response.get( final_response.get(
"model" "model"