85 lines
2.8 KiB
Python
85 lines
2.8 KiB
Python
from tools.evaluation.evaluate_graph_answers import grade_answer, graph_context
|
|
|
|
|
|
def test_context_does_not_present_profile_title_or_kind_as_an_academic_fact():
|
|
result = {
|
|
"status": "ok",
|
|
"reason": None,
|
|
"aggregation": None,
|
|
"constraints": {
|
|
"mentions_title": False,
|
|
"mentions_academic_year": False,
|
|
"years": [],
|
|
"mentions_start_year": False,
|
|
},
|
|
"facts": [
|
|
{
|
|
"fact_type": "person_category",
|
|
"person": "Eva Nová",
|
|
"category": "dp2026",
|
|
"category_year": 2026,
|
|
"category_type": "diploma_thesis",
|
|
}
|
|
],
|
|
"documents": [
|
|
{
|
|
"document_path": "p.md",
|
|
"source_url": "https://example.org/p",
|
|
"title": "Eva Nová",
|
|
"person_kind": "student",
|
|
}
|
|
],
|
|
}
|
|
context = graph_context(result)
|
|
assert context["facts"] == [{"person": "Eva Nová", "category": "dp2026"}]
|
|
assert context["sources"] == [{"document_path": "p.md", "source_url": "https://example.org/p"}]
|
|
|
|
|
|
def test_null_is_not_a_false_positive_assertion_but_wrong_year_is():
|
|
oracle = {
|
|
"people": {
|
|
"Eva Nová": {
|
|
"start_year": 2020,
|
|
"academics": [{"year": 2026, "title": "Vyhľadávanie informácií"}],
|
|
}
|
|
}
|
|
}
|
|
case = {
|
|
"expected_answer_contains": ["Eva Nová", "2026"],
|
|
"graph": {
|
|
"expected_bindings": [{"person": "Eva Nová", "year": 2026}],
|
|
"expected_evidence_documents": [],
|
|
},
|
|
}
|
|
answer = {
|
|
"answer": "Eva Nová má prácu z roku 2026.",
|
|
"status": "ok",
|
|
"source_urls": [],
|
|
"claims": [{"person": "Eva Nová", "year": 2026, "start_year": None, "title": None}],
|
|
}
|
|
assert grade_answer(case, answer, oracle)["strict"]
|
|
answer["claims"][0]["year"] = 2025
|
|
assert not grade_answer(case, answer, oracle)["strict"]
|
|
|
|
|
|
def test_correct_structured_category_does_not_excuse_a_wrong_prose_explanation():
|
|
oracle = {
|
|
"people": {"Eva Nová": {"documents": ["p.md"]}},
|
|
"documents": {"p.md": {"tags": [], "categories": ["dp2026"], "author": None}},
|
|
}
|
|
case = {
|
|
"expected_answer_contains": ["Eva Nová", "dp2026"],
|
|
"graph": {
|
|
"expected_bindings": [{"person": "Eva Nová", "category": "dp2026"}],
|
|
"expected_evidence_documents": [],
|
|
},
|
|
}
|
|
answer = {
|
|
"answer": "Eva Nová má dp2026, čo je dizertačná práca.",
|
|
"status": "ok",
|
|
"source_urls": [],
|
|
"claims": [{"person": "Eva Nová", "category": "dp2026"}],
|
|
}
|
|
grade = grade_answer(case, answer, oracle)
|
|
assert not grade["strict"] and grade["contradictory_thesis_label"]
|