546 lines
9.4 KiB
Python
546 lines
9.4 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
import statistics
|
|
import unicodedata
|
|
from collections import defaultdict
|
|
from typing import Any
|
|
|
|
|
|
NO_ANSWER_TEXT = (
|
|
"V dostupných dokumentoch ZP Wiki sa túto "
|
|
"informáciu nepodarilo spoľahlivo nájsť."
|
|
)
|
|
|
|
|
|
MARKDOWN_URL_RE = re.compile(
|
|
r"\[[^\]]*\]\((https?://[^)]+)\)"
|
|
)
|
|
|
|
|
|
def normalize_text(
|
|
value: str,
|
|
) -> str:
|
|
value = unicodedata.normalize(
|
|
"NFKC",
|
|
value,
|
|
)
|
|
|
|
value = value.casefold()
|
|
|
|
return " ".join(
|
|
value.split()
|
|
)
|
|
|
|
|
|
def normalize_url(
|
|
value: str,
|
|
) -> str:
|
|
value = value.strip()
|
|
|
|
match = (
|
|
MARKDOWN_URL_RE.search(
|
|
value
|
|
)
|
|
)
|
|
|
|
if match:
|
|
value = match.group(
|
|
1
|
|
)
|
|
|
|
return value.rstrip(
|
|
"/"
|
|
)
|
|
|
|
|
|
def evaluate_answer(
|
|
question: dict[str, Any],
|
|
answer: str,
|
|
*,
|
|
tool_called: bool,
|
|
) -> dict[str, Any]:
|
|
normalized_answer = (
|
|
normalize_text(
|
|
answer
|
|
)
|
|
)
|
|
|
|
expected_contains = (
|
|
question.get(
|
|
"expected_answer_contains",
|
|
[],
|
|
)
|
|
)
|
|
|
|
if not isinstance(
|
|
expected_contains,
|
|
list,
|
|
):
|
|
expected_contains = []
|
|
|
|
answer_matches: list[
|
|
bool
|
|
] = []
|
|
|
|
for expected in expected_contains:
|
|
expected_text = (
|
|
normalize_text(
|
|
str(
|
|
expected
|
|
)
|
|
)
|
|
)
|
|
|
|
answer_matches.append(
|
|
expected_text
|
|
in normalized_answer
|
|
)
|
|
|
|
if answer_matches:
|
|
answer_contains_score = (
|
|
sum(
|
|
answer_matches
|
|
)
|
|
/ len(
|
|
answer_matches
|
|
)
|
|
)
|
|
|
|
else:
|
|
answer_contains_score = 1.0
|
|
|
|
expected_urls = (
|
|
question.get(
|
|
"expected_source_urls",
|
|
[],
|
|
)
|
|
)
|
|
|
|
if not isinstance(
|
|
expected_urls,
|
|
list,
|
|
):
|
|
expected_urls = []
|
|
|
|
normalized_expected_urls = [
|
|
normalize_url(
|
|
str(
|
|
url
|
|
)
|
|
)
|
|
for url in expected_urls
|
|
]
|
|
|
|
source_matches: list[
|
|
bool
|
|
] = []
|
|
|
|
for expected_url in (
|
|
normalized_expected_urls
|
|
):
|
|
source_matches.append(
|
|
expected_url
|
|
in answer
|
|
)
|
|
|
|
if source_matches:
|
|
source_url_score = (
|
|
sum(
|
|
source_matches
|
|
)
|
|
/ len(
|
|
source_matches
|
|
)
|
|
)
|
|
|
|
else:
|
|
source_url_score = 1.0
|
|
|
|
should_answer = bool(
|
|
question.get(
|
|
"should_answer",
|
|
True,
|
|
)
|
|
)
|
|
|
|
normalized_no_answer = (
|
|
normalize_text(
|
|
NO_ANSWER_TEXT
|
|
)
|
|
)
|
|
|
|
returned_no_answer = (
|
|
normalized_no_answer
|
|
in normalized_answer
|
|
)
|
|
|
|
if should_answer:
|
|
should_answer_ok = (
|
|
bool(
|
|
answer.strip()
|
|
)
|
|
and not returned_no_answer
|
|
)
|
|
|
|
else:
|
|
should_answer_ok = (
|
|
returned_no_answer
|
|
)
|
|
|
|
tool_score = (
|
|
1.0
|
|
if tool_called
|
|
else 0.0
|
|
)
|
|
|
|
should_answer_score = (
|
|
1.0
|
|
if should_answer_ok
|
|
else 0.0
|
|
)
|
|
|
|
overall_score = (
|
|
statistics.mean(
|
|
[
|
|
answer_contains_score,
|
|
source_url_score,
|
|
should_answer_score,
|
|
tool_score,
|
|
]
|
|
)
|
|
)
|
|
|
|
strict_pass = (
|
|
bool(
|
|
answer.strip()
|
|
)
|
|
and all(
|
|
answer_matches
|
|
)
|
|
and all(
|
|
source_matches
|
|
)
|
|
and should_answer_ok
|
|
and tool_called
|
|
)
|
|
|
|
return {
|
|
"answer_matches": (
|
|
answer_matches
|
|
),
|
|
"answer_contains_score": (
|
|
answer_contains_score
|
|
),
|
|
"source_matches": (
|
|
source_matches
|
|
),
|
|
"source_url_score": (
|
|
source_url_score
|
|
),
|
|
"should_answer_ok": (
|
|
should_answer_ok
|
|
),
|
|
"should_answer_score": (
|
|
should_answer_score
|
|
),
|
|
"returned_no_answer": (
|
|
returned_no_answer
|
|
),
|
|
"tool_score": (
|
|
tool_score
|
|
),
|
|
"overall_score": (
|
|
overall_score
|
|
),
|
|
"strict_pass": (
|
|
strict_pass
|
|
),
|
|
}
|
|
|
|
|
|
def safe_mean(
|
|
values: list[float],
|
|
) -> float:
|
|
if not values:
|
|
return 0.0
|
|
|
|
return float(
|
|
statistics.mean(
|
|
values
|
|
)
|
|
)
|
|
|
|
|
|
def summarize_results(
|
|
results: list[
|
|
dict[str, Any]
|
|
],
|
|
*,
|
|
include_groups: bool = True,
|
|
) -> dict[str, Any]:
|
|
total = len(
|
|
results
|
|
)
|
|
|
|
errors = [
|
|
item
|
|
for item in results
|
|
if item.get(
|
|
"error"
|
|
)
|
|
]
|
|
|
|
completed = (
|
|
total
|
|
- len(
|
|
errors
|
|
)
|
|
)
|
|
|
|
tool_called_values = [
|
|
(
|
|
1.0
|
|
if item.get(
|
|
"tool_called"
|
|
)
|
|
else 0.0
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
answer_scores = [
|
|
float(
|
|
item.get(
|
|
"answer_contains_score",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
source_scores = [
|
|
float(
|
|
item.get(
|
|
"source_url_score",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
should_answer_scores = [
|
|
float(
|
|
item.get(
|
|
"should_answer_score",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
overall_scores = [
|
|
float(
|
|
item.get(
|
|
"overall_score",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
latencies = [
|
|
float(
|
|
item.get(
|
|
"total_latency_seconds",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
if not item.get(
|
|
"error"
|
|
)
|
|
]
|
|
|
|
strict_passes = sum(
|
|
1
|
|
for item in results
|
|
if item.get(
|
|
"strict_pass"
|
|
)
|
|
)
|
|
|
|
prompt_tokens = sum(
|
|
int(
|
|
(
|
|
item.get(
|
|
"usage"
|
|
)
|
|
or {}
|
|
).get(
|
|
"prompt_tokens",
|
|
0,
|
|
)
|
|
)
|
|
for item in results
|
|
)
|
|
|
|
completion_tokens = sum(
|
|
int(
|
|
(
|
|
item.get(
|
|
"usage"
|
|
)
|
|
or {}
|
|
).get(
|
|
"completion_tokens",
|
|
0,
|
|
)
|
|
)
|
|
for item in results
|
|
)
|
|
|
|
total_tokens = sum(
|
|
int(
|
|
(
|
|
item.get(
|
|
"usage"
|
|
)
|
|
or {}
|
|
).get(
|
|
"total_tokens",
|
|
0,
|
|
)
|
|
)
|
|
for item in results
|
|
)
|
|
|
|
summary: dict[
|
|
str,
|
|
Any,
|
|
] = {
|
|
"total": total,
|
|
"completed": completed,
|
|
"errors": len(
|
|
errors
|
|
),
|
|
"tool_call_rate": round(
|
|
safe_mean(
|
|
tool_called_values
|
|
),
|
|
6,
|
|
),
|
|
"answer_contains_score": round(
|
|
safe_mean(
|
|
answer_scores
|
|
),
|
|
6,
|
|
),
|
|
"source_url_score": round(
|
|
safe_mean(
|
|
source_scores
|
|
),
|
|
6,
|
|
),
|
|
"should_answer_score": round(
|
|
safe_mean(
|
|
should_answer_scores
|
|
),
|
|
6,
|
|
),
|
|
"overall_score": round(
|
|
safe_mean(
|
|
overall_scores
|
|
),
|
|
6,
|
|
),
|
|
"strict_pass_count": (
|
|
strict_passes
|
|
),
|
|
"strict_pass_rate": round(
|
|
(
|
|
strict_passes
|
|
/ total
|
|
if total
|
|
else 0.0
|
|
),
|
|
6,
|
|
),
|
|
"mean_latency_seconds": round(
|
|
safe_mean(
|
|
latencies
|
|
),
|
|
6,
|
|
),
|
|
"prompt_tokens": (
|
|
prompt_tokens
|
|
),
|
|
"completion_tokens": (
|
|
completion_tokens
|
|
),
|
|
"total_tokens": (
|
|
total_tokens
|
|
),
|
|
}
|
|
|
|
if include_groups:
|
|
summary[
|
|
"by_category"
|
|
] = aggregate_by_field(
|
|
results,
|
|
"category",
|
|
)
|
|
|
|
summary[
|
|
"by_difficulty"
|
|
] = aggregate_by_field(
|
|
results,
|
|
"difficulty",
|
|
)
|
|
|
|
return summary
|
|
|
|
|
|
def aggregate_by_field(
|
|
results: list[
|
|
dict[str, Any]
|
|
],
|
|
field: str,
|
|
) -> dict[
|
|
str,
|
|
dict[str, Any],
|
|
]:
|
|
grouped: dict[
|
|
str,
|
|
list[dict[str, Any]],
|
|
] = defaultdict(
|
|
list
|
|
)
|
|
|
|
for result in results:
|
|
group_name = str(
|
|
result.get(
|
|
field,
|
|
"unknown",
|
|
)
|
|
)
|
|
|
|
grouped[
|
|
group_name
|
|
].append(
|
|
result
|
|
)
|
|
|
|
return {
|
|
group_name: summarize_results(
|
|
group_rows,
|
|
include_groups=False,
|
|
)
|
|
for (
|
|
group_name,
|
|
group_rows,
|
|
) in sorted(
|
|
grouped.items()
|
|
)
|
|
}
|