91 lines
2.2 KiB
Python
91 lines
2.2 KiB
Python
from __future__ import annotations
|
|
|
|
from tools.rag.evaluation.rag_metrics import (
|
|
evaluate_answer,
|
|
expected_phrase_matches,
|
|
)
|
|
|
|
|
|
def test_exact_phrase_still_matches() -> None:
|
|
assert expected_phrase_matches(
|
|
"slovenský internet",
|
|
"Téma je slovenský internet.",
|
|
)
|
|
|
|
|
|
def test_slovak_adjective_and_noun_inflection_matches() -> None:
|
|
assert expected_phrase_matches(
|
|
"slovenský internet",
|
|
"Vyhľadávač na slovenskom internete.",
|
|
)
|
|
|
|
|
|
def test_graph_neural_network_inflection_matches() -> None:
|
|
assert expected_phrase_matches(
|
|
"grafové neurónové siete",
|
|
("Prehľad metód grafových " "neurónových sietí."),
|
|
)
|
|
|
|
|
|
def test_person_name_inflection_matches() -> None:
|
|
assert expected_phrase_matches(
|
|
"Ján Holp",
|
|
"Dokument patrí Jánovi Holpovi.",
|
|
)
|
|
|
|
|
|
def test_numeric_expected_value_requires_exact_numeric_token() -> None:
|
|
assert expected_phrase_matches(
|
|
"2021",
|
|
"Diplomová práca bola v roku 2021.",
|
|
)
|
|
|
|
assert not expected_phrase_matches(
|
|
"2021",
|
|
"Identifikátor je 20210.",
|
|
)
|
|
|
|
|
|
def test_unrelated_methods_do_not_match_gnn() -> None:
|
|
assert not expected_phrase_matches(
|
|
"grafové neurónové siete",
|
|
("Použili sa transformery, " "autoenkódery a SVM."),
|
|
)
|
|
|
|
|
|
def test_word_order_is_not_ignored() -> None:
|
|
assert not expected_phrase_matches(
|
|
"slovenský internet",
|
|
("Internetový projekt analyzuje " "slovenský text."),
|
|
)
|
|
|
|
|
|
def test_evaluate_answer_uses_morphology_aware_matcher() -> None:
|
|
question = {
|
|
"expected_answer_contains": [
|
|
"grafové neurónové siete",
|
|
],
|
|
"expected_source_urls": [
|
|
("https://zp.kemt.fei.tuke.sk/" "students/2016/maros_harahus"),
|
|
],
|
|
"should_answer": True,
|
|
}
|
|
|
|
answer = (
|
|
"Maroš mal pripraviť prehľad "
|
|
"grafových neurónových sietí.\n\n"
|
|
"Zdroj: "
|
|
"https://zp.kemt.fei.tuke.sk/"
|
|
"students/2016/maros_harahus"
|
|
)
|
|
|
|
metrics = evaluate_answer(
|
|
question,
|
|
answer,
|
|
tool_called=True,
|
|
)
|
|
|
|
assert metrics["answer_matches"] == [True]
|
|
|
|
assert metrics["strict_pass"] is True
|