688 lines
12 KiB
Python
688 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
import statistics
|
|
import unicodedata
|
|
from collections import defaultdict
|
|
from typing import Any
|
|
|
|
|
|
NO_ANSWER_TEXT = (
|
|
"V dostupných dokumentoch ZP Wiki sa túto "
|
|
"informáciu nepodarilo spoľahlivo nájsť."
|
|
)
|
|
|
|
|
|
MARKDOWN_URL_RE = re.compile(
|
|
r"\[[^\]]*\]\((https?://[^)]+)\)"
|
|
)
|
|
|
|
WORD_RE = re.compile(
|
|
r"[^\W_]+",
|
|
re.UNICODE,
|
|
)
|
|
|
|
|
|
def normalize_text(
|
|
value: str,
|
|
) -> str:
|
|
value = unicodedata.normalize(
|
|
"NFKC",
|
|
value,
|
|
).casefold()
|
|
|
|
return " ".join(
|
|
value.split()
|
|
)
|
|
|
|
|
|
def normalize_match_token(
|
|
value: str,
|
|
) -> str:
|
|
value = unicodedata.normalize(
|
|
"NFKD",
|
|
value,
|
|
)
|
|
|
|
value = "".join(
|
|
character
|
|
for character in value
|
|
if not unicodedata.combining(
|
|
character
|
|
)
|
|
)
|
|
|
|
return value.casefold()
|
|
|
|
|
|
def match_tokens(
|
|
value: str,
|
|
) -> list[str]:
|
|
normalized = (
|
|
unicodedata.normalize(
|
|
"NFKC",
|
|
value,
|
|
)
|
|
)
|
|
|
|
return [
|
|
normalize_match_token(
|
|
token
|
|
)
|
|
for token
|
|
in WORD_RE.findall(
|
|
normalized
|
|
)
|
|
]
|
|
|
|
|
|
def morphology_token_matches(
|
|
expected: str,
|
|
actual: str,
|
|
) -> bool:
|
|
if expected == actual:
|
|
return True
|
|
|
|
if (
|
|
expected.isdigit()
|
|
or actual.isdigit()
|
|
):
|
|
return False
|
|
|
|
shorter = min(
|
|
len(expected),
|
|
len(actual),
|
|
)
|
|
|
|
if shorter <= 2:
|
|
return False
|
|
|
|
required_prefix = (
|
|
3
|
|
if shorter <= 4
|
|
else 4
|
|
if shorter <= 6
|
|
else 5
|
|
)
|
|
|
|
return (
|
|
expected[
|
|
:required_prefix
|
|
]
|
|
== actual[
|
|
:required_prefix
|
|
]
|
|
)
|
|
|
|
|
|
def expected_phrase_matches(
|
|
expected: str,
|
|
answer: str,
|
|
) -> bool:
|
|
normalized_expected = (
|
|
normalize_text(
|
|
expected
|
|
)
|
|
)
|
|
|
|
normalized_answer = (
|
|
normalize_text(
|
|
answer
|
|
)
|
|
)
|
|
|
|
if not normalized_expected:
|
|
return True
|
|
|
|
expected_tokens = (
|
|
match_tokens(
|
|
expected
|
|
)
|
|
)
|
|
|
|
answer_tokens = (
|
|
match_tokens(
|
|
answer
|
|
)
|
|
)
|
|
|
|
has_numeric_token = any(
|
|
token.isdigit()
|
|
for token in expected_tokens
|
|
)
|
|
|
|
if (
|
|
not has_numeric_token
|
|
and normalized_expected
|
|
in normalized_answer
|
|
):
|
|
return True
|
|
|
|
if (
|
|
not expected_tokens
|
|
or len(answer_tokens)
|
|
< len(expected_tokens)
|
|
):
|
|
return False
|
|
|
|
window_size = len(
|
|
expected_tokens
|
|
)
|
|
|
|
for start in range(
|
|
len(answer_tokens)
|
|
- window_size
|
|
+ 1
|
|
):
|
|
window = answer_tokens[
|
|
start:
|
|
start + window_size
|
|
]
|
|
|
|
if all(
|
|
morphology_token_matches(
|
|
expected_token,
|
|
actual_token,
|
|
)
|
|
for (
|
|
expected_token,
|
|
actual_token,
|
|
)
|
|
in zip(
|
|
expected_tokens,
|
|
window,
|
|
)
|
|
):
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def normalize_url(
|
|
value: str,
|
|
) -> str:
|
|
value = value.strip()
|
|
|
|
match = (
|
|
MARKDOWN_URL_RE.search(
|
|
value
|
|
)
|
|
)
|
|
|
|
if match:
|
|
value = match.group(
|
|
1
|
|
)
|
|
|
|
return value.rstrip(
|
|
"/"
|
|
)
|
|
|
|
|
|
def evaluate_answer(
|
|
question: dict[str, Any],
|
|
answer: str,
|
|
*,
|
|
tool_called: bool,
|
|
) -> dict[str, Any]:
|
|
normalized_answer = (
|
|
normalize_text(
|
|
answer
|
|
)
|
|
)
|
|
|
|
expected_contains = (
|
|
question.get(
|
|
"expected_answer_contains",
|
|
[],
|
|
)
|
|
)
|
|
|
|
if not isinstance(
|
|
expected_contains,
|
|
list,
|
|
):
|
|
expected_contains = []
|
|
|
|
answer_matches = [
|
|
expected_phrase_matches(
|
|
str(expected),
|
|
answer,
|
|
)
|
|
for expected
|
|
in expected_contains
|
|
]
|
|
|
|
answer_contains_score = (
|
|
sum(
|
|
answer_matches
|
|
)
|
|
/ len(
|
|
answer_matches
|
|
)
|
|
if answer_matches
|
|
else 1.0
|
|
)
|
|
|
|
expected_urls = (
|
|
question.get(
|
|
"expected_source_urls",
|
|
[],
|
|
)
|
|
)
|
|
|
|
if not isinstance(
|
|
expected_urls,
|
|
list,
|
|
):
|
|
expected_urls = []
|
|
|
|
normalized_expected_urls = [
|
|
normalize_url(
|
|
str(url)
|
|
)
|
|
for url in expected_urls
|
|
]
|
|
|
|
source_matches = [
|
|
expected_url
|
|
in answer
|
|
for expected_url
|
|
in normalized_expected_urls
|
|
]
|
|
|
|
source_url_score = (
|
|
sum(
|
|
source_matches
|
|
)
|
|
/ len(
|
|
source_matches
|
|
)
|
|
if source_matches
|
|
else 1.0
|
|
)
|
|
|
|
should_answer = bool(
|
|
question.get(
|
|
"should_answer",
|
|
True,
|
|
)
|
|
)
|
|
|
|
returned_no_answer = (
|
|
normalize_text(
|
|
NO_ANSWER_TEXT
|
|
)
|
|
in normalized_answer
|
|
)
|
|
|
|
if should_answer:
|
|
should_answer_ok = (
|
|
bool(
|
|
answer.strip()
|
|
)
|
|
and not returned_no_answer
|
|
)
|
|
|
|
else:
|
|
should_answer_ok = (
|
|
returned_no_answer
|
|
)
|
|
|
|
tool_score = (
|
|
1.0
|
|
if tool_called
|
|
else 0.0
|
|
)
|
|
|
|
should_answer_score = (
|
|
1.0
|
|
if should_answer_ok
|
|
else 0.0
|
|
)
|
|
|
|
overall_score = (
|
|
statistics.mean(
|
|
[
|
|
answer_contains_score,
|
|
source_url_score,
|
|
should_answer_score,
|
|
tool_score,
|
|
]
|
|
)
|
|
)
|
|
|
|
strict_pass = (
|
|
bool(
|
|
answer.strip()
|
|
)
|
|
and all(
|
|
answer_matches
|
|
)
|
|
and all(
|
|
source_matches
|
|
)
|
|
and should_answer_ok
|
|
and tool_called
|
|
)
|
|
|
|
return {
|
|
"answer_matches": (
|
|
answer_matches
|
|
),
|
|
"answer_contains_score": (
|
|
answer_contains_score
|
|
),
|
|
"source_matches": (
|
|
source_matches
|
|
),
|
|
"source_url_score": (
|
|
source_url_score
|
|
),
|
|
"should_answer_ok": (
|
|
should_answer_ok
|
|
),
|
|
"should_answer_score": (
|
|
should_answer_score
|
|
),
|
|
"returned_no_answer": (
|
|
returned_no_answer
|
|
),
|
|
"tool_score": (
|
|
tool_score
|
|
),
|
|
"overall_score": (
|
|
overall_score
|
|
),
|
|
"strict_pass": (
|
|
strict_pass
|
|
),
|
|
}
|
|
|
|
|
|
def safe_mean(
|
|
values: list[float],
|
|
) -> float:
|
|
if not values:
|
|
return 0.0
|
|
|
|
return float(
|
|
statistics.mean(
|
|
values
|
|
)
|
|
)
|
|
|
|
|
|
def summarize_results(
|
|
results: list[
|
|
dict[str, Any]
|
|
],
|
|
*,
|
|
include_groups: bool = True,
|
|
) -> dict[str, Any]:
|
|
total = len(
|
|
results
|
|
)
|
|
|
|
errors = [
|
|
item
|
|
for item in results
|
|
if item.get(
|
|
"error"
|
|
)
|
|
]
|
|
|
|
completed = (
|
|
total
|
|
- len(
|
|
errors
|
|
)
|
|
)
|
|
|
|
tool_called_values = [
|
|
(
|
|
1.0
|
|
if item.get(
|
|
"tool_called"
|
|
)
|
|
else 0.0
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
answer_scores = [
|
|
float(
|
|
item.get(
|
|
"answer_contains_score",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
source_scores = [
|
|
float(
|
|
item.get(
|
|
"source_url_score",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
should_answer_scores = [
|
|
float(
|
|
item.get(
|
|
"should_answer_score",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
overall_scores = [
|
|
float(
|
|
item.get(
|
|
"overall_score",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
]
|
|
|
|
latencies = [
|
|
float(
|
|
item.get(
|
|
"total_latency_seconds",
|
|
0.0,
|
|
)
|
|
)
|
|
for item in results
|
|
if not item.get(
|
|
"error"
|
|
)
|
|
]
|
|
|
|
strict_passes = sum(
|
|
1
|
|
for item in results
|
|
if item.get(
|
|
"strict_pass"
|
|
)
|
|
)
|
|
|
|
prompt_tokens = sum(
|
|
int(
|
|
(
|
|
item.get(
|
|
"usage"
|
|
)
|
|
or {}
|
|
).get(
|
|
"prompt_tokens",
|
|
0,
|
|
)
|
|
)
|
|
for item in results
|
|
)
|
|
|
|
completion_tokens = sum(
|
|
int(
|
|
(
|
|
item.get(
|
|
"usage"
|
|
)
|
|
or {}
|
|
).get(
|
|
"completion_tokens",
|
|
0,
|
|
)
|
|
)
|
|
for item in results
|
|
)
|
|
|
|
total_tokens = sum(
|
|
int(
|
|
(
|
|
item.get(
|
|
"usage"
|
|
)
|
|
or {}
|
|
).get(
|
|
"total_tokens",
|
|
0,
|
|
)
|
|
)
|
|
for item in results
|
|
)
|
|
|
|
summary: dict[
|
|
str,
|
|
Any,
|
|
] = {
|
|
"total": total,
|
|
"completed": completed,
|
|
"errors": len(
|
|
errors
|
|
),
|
|
"tool_call_rate": round(
|
|
safe_mean(
|
|
tool_called_values
|
|
),
|
|
6,
|
|
),
|
|
"answer_contains_score": round(
|
|
safe_mean(
|
|
answer_scores
|
|
),
|
|
6,
|
|
),
|
|
"source_url_score": round(
|
|
safe_mean(
|
|
source_scores
|
|
),
|
|
6,
|
|
),
|
|
"should_answer_score": round(
|
|
safe_mean(
|
|
should_answer_scores
|
|
),
|
|
6,
|
|
),
|
|
"overall_score": round(
|
|
safe_mean(
|
|
overall_scores
|
|
),
|
|
6,
|
|
),
|
|
"strict_pass_count": (
|
|
strict_passes
|
|
),
|
|
"strict_pass_rate": round(
|
|
(
|
|
strict_passes
|
|
/ total
|
|
if total
|
|
else 0.0
|
|
),
|
|
6,
|
|
),
|
|
"mean_latency_seconds": round(
|
|
safe_mean(
|
|
latencies
|
|
),
|
|
6,
|
|
),
|
|
"prompt_tokens": (
|
|
prompt_tokens
|
|
),
|
|
"completion_tokens": (
|
|
completion_tokens
|
|
),
|
|
"total_tokens": (
|
|
total_tokens
|
|
),
|
|
}
|
|
|
|
if include_groups:
|
|
summary[
|
|
"by_category"
|
|
] = aggregate_by_field(
|
|
results,
|
|
"category",
|
|
)
|
|
|
|
summary[
|
|
"by_difficulty"
|
|
] = aggregate_by_field(
|
|
results,
|
|
"difficulty",
|
|
)
|
|
|
|
return summary
|
|
|
|
|
|
def aggregate_by_field(
|
|
results: list[
|
|
dict[str, Any]
|
|
],
|
|
field: str,
|
|
) -> dict[
|
|
str,
|
|
dict[str, Any],
|
|
]:
|
|
grouped: dict[
|
|
str,
|
|
list[dict[str, Any]],
|
|
] = defaultdict(
|
|
list
|
|
)
|
|
|
|
for result in results:
|
|
group_name = str(
|
|
result.get(
|
|
field,
|
|
"unknown",
|
|
)
|
|
)
|
|
|
|
grouped[
|
|
group_name
|
|
].append(
|
|
result
|
|
)
|
|
|
|
return {
|
|
group_name: summarize_results(
|
|
group_rows,
|
|
include_groups=False,
|
|
)
|
|
for (
|
|
group_name,
|
|
group_rows,
|
|
)
|
|
in sorted(
|
|
grouped.items()
|
|
)
|
|
}
|