35 lines
2.3 KiB
Python
35 lines
2.3 KiB
Python
from tools.answer_evaluation.evaluate_graph_answers import grade_answer, graph_context
|
|
|
|
|
|
def test_context_does_not_present_profile_title_or_kind_as_an_academic_fact():
|
|
result = {'status': 'ok', 'reason': None, 'aggregation': None,
|
|
'constraints': {'mentions_title': False, 'mentions_academic_year': False, 'years': [], 'mentions_start_year': False},
|
|
'facts': [{'fact_type': 'person_category', 'person': 'Eva Nová', 'category': 'dp2026',
|
|
'category_year': 2026, 'category_type': 'diploma_thesis'}],
|
|
'documents': [{'document_path': 'p.md', 'source_url': 'https://example.org/p', 'title': 'Eva Nová', 'person_kind': 'student'}]}
|
|
context = graph_context(result)
|
|
assert context['facts'] == [{'person': 'Eva Nová', 'category': 'dp2026'}]
|
|
assert context['sources'] == [{'document_path': 'p.md', 'source_url': 'https://example.org/p'}]
|
|
|
|
|
|
def test_null_is_not_a_false_positive_assertion_but_wrong_year_is():
|
|
oracle = {'people': {'Eva Nová': {'start_year': 2020, 'academics': [{'year': 2026, 'title': 'Vyhľadávanie informácií'}]}}}
|
|
case = {'expected_answer_contains': ['Eva Nová', '2026'], 'graph': {
|
|
'expected_bindings': [{'person': 'Eva Nová', 'year': 2026}], 'expected_evidence_documents': []}}
|
|
answer = {'answer': 'Eva Nová má prácu z roku 2026.', 'status': 'ok', 'source_urls': [],
|
|
'claims': [{'person': 'Eva Nová', 'year': 2026, 'start_year': None, 'title': None}]}
|
|
assert grade_answer(case, answer, oracle)['strict']
|
|
answer['claims'][0]['year'] = 2025
|
|
assert not grade_answer(case, answer, oracle)['strict']
|
|
|
|
|
|
def test_correct_structured_category_does_not_excuse_a_wrong_prose_explanation():
|
|
oracle = {'people': {'Eva Nová': {'documents': ['p.md']}},
|
|
'documents': {'p.md': {'tags': [], 'categories': ['dp2026'], 'author': None}}}
|
|
case = {'expected_answer_contains': ['Eva Nová', 'dp2026'], 'graph': {
|
|
'expected_bindings': [{'person': 'Eva Nová', 'category': 'dp2026'}], 'expected_evidence_documents': []}}
|
|
answer = {'answer': 'Eva Nová má dp2026, čo je dizertačná práca.', 'status': 'ok', 'source_urls': [],
|
|
'claims': [{'person': 'Eva Nová', 'category': 'dp2026'}]}
|
|
grade = grade_answer(case, answer, oracle)
|
|
assert not grade['strict'] and grade['contradictory_thesis_label']
|