115 lines
2.4 KiB
Python
115 lines
2.4 KiB
Python
from __future__ import annotations
|
|
|
|
from evaluation.rag_metrics import (
|
|
evaluate_answer,
|
|
expected_phrase_matches,
|
|
)
|
|
|
|
|
|
def test_exact_phrase_still_matches() -> None:
|
|
assert expected_phrase_matches(
|
|
"slovenský internet",
|
|
"Téma je slovenský internet.",
|
|
)
|
|
|
|
|
|
def test_slovak_adjective_and_noun_inflection_matches() -> None:
|
|
assert expected_phrase_matches(
|
|
"slovenský internet",
|
|
"Vyhľadávač na slovenskom internete.",
|
|
)
|
|
|
|
|
|
def test_graph_neural_network_inflection_matches() -> None:
|
|
assert expected_phrase_matches(
|
|
"grafové neurónové siete",
|
|
(
|
|
"Prehľad metód grafových "
|
|
"neurónových sietí."
|
|
),
|
|
)
|
|
|
|
|
|
def test_person_name_inflection_matches() -> None:
|
|
assert expected_phrase_matches(
|
|
"Ján Holp",
|
|
"Dokument patrí Jánovi Holpovi.",
|
|
)
|
|
|
|
|
|
def test_numeric_expected_value_requires_exact_numeric_token() -> None:
|
|
assert expected_phrase_matches(
|
|
"2021",
|
|
"Diplomová práca bola v roku 2021.",
|
|
)
|
|
|
|
assert not expected_phrase_matches(
|
|
"2021",
|
|
"Identifikátor je 20210.",
|
|
)
|
|
|
|
|
|
def test_unrelated_methods_do_not_match_gnn() -> None:
|
|
assert not expected_phrase_matches(
|
|
"grafové neurónové siete",
|
|
(
|
|
"Použili sa transformery, "
|
|
"autoenkódery a SVM."
|
|
),
|
|
)
|
|
|
|
|
|
def test_word_order_is_not_ignored() -> None:
|
|
assert not expected_phrase_matches(
|
|
"slovenský internet",
|
|
(
|
|
"Internetový projekt analyzuje "
|
|
"slovenský text."
|
|
),
|
|
)
|
|
|
|
|
|
def test_evaluate_answer_uses_morphology_aware_matcher() -> None:
|
|
question = {
|
|
"expected_answer_contains": [
|
|
"grafové neurónové siete",
|
|
],
|
|
"expected_source_urls": [
|
|
(
|
|
"https://zp.kemt.fei.tuke.sk/"
|
|
"students/2016/maros_harahus"
|
|
),
|
|
],
|
|
"should_answer": True,
|
|
}
|
|
|
|
answer = (
|
|
"Maroš mal pripraviť prehľad "
|
|
"grafových neurónových sietí.\n\n"
|
|
"Zdroj: "
|
|
"https://zp.kemt.fei.tuke.sk/"
|
|
"students/2016/maros_harahus"
|
|
)
|
|
|
|
metrics = evaluate_answer(
|
|
question,
|
|
answer,
|
|
tool_called=True,
|
|
)
|
|
|
|
assert (
|
|
metrics[
|
|
"answer_matches"
|
|
]
|
|
== [
|
|
True
|
|
]
|
|
)
|
|
|
|
assert (
|
|
metrics[
|
|
"strict_pass"
|
|
]
|
|
is True
|
|
)
|