112 lines
5.8 KiB
Python
112 lines
5.8 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.evaluation.audit_benchmark import audit_questions
|
|
from tools.graphrag.evaluation.graphrag_evaluator import evaluate_result, property_present, value_is_present
|
|
from .test_graph_search import FakeSession, sample_catalog
|
|
|
|
|
|
@pytest.mark.parametrize('word', ['názov', 'názvom', 'názvu'])
|
|
def test_title_inflections(word):
|
|
assert gs.extract_query_constraints(f'Prepoj osobu s {word} jej práce.')['mentions_title']
|
|
|
|
|
|
@pytest.mark.parametrize('question', [
|
|
'V ktorom rovnakom roku je uvedená práca pri osobách?',
|
|
'Aký rok práce spája tieto osoby?',
|
|
'Prepoj osobu s názvom jej práce a rokom.',
|
|
])
|
|
def test_academic_year_intent_does_not_require_a_numeric_year(question):
|
|
assert gs.extract_query_constraints(question)['mentions_academic_year']
|
|
|
|
|
|
def test_tag_is_extracted_without_following_clause():
|
|
assert gs.explicit_keyword_phrase('Kľúčové slovo RAG a jej práca v roku 2024?') == 'rag'
|
|
|
|
|
|
def test_incidental_zp_tag_does_not_filter_title_lookup():
|
|
catalog = sample_catalog() + [{'element_id': 'zp', 'kind': 'Keyword', 'id': 'zp', 'name': 'zp'}]
|
|
fact = {'person_element_id': 'p2', 'person': 'Jakub Maruniak', 'thesis_element_id': 'th1',
|
|
'title': 'Ukážková práca', 'year': 2021, 'thesis_type': 'diploma_thesis'}
|
|
session = FakeSession({gs.CATALOG_QUERY: catalog, gs.THESIS_FACTS_QUERY: [fact]})
|
|
result = gs.search_graph_with_session(session, 'Aký názov práce je v ZP Wiki pri osobe Jakub Maruniak?')
|
|
assert any(row['fact_type'] == 'person_thesis' for row in result['facts'])
|
|
assert gs.PERSON_KEYWORD_FACTS_QUERY not in {query for query, _ in session.calls}
|
|
|
|
|
|
def test_keyword_and_year_cannot_be_satisfied_by_different_people():
|
|
session = FakeSession({
|
|
gs.CATALOG_QUERY: sample_catalog(),
|
|
gs.THESIS_FACTS_QUERY: [{'person_element_id': 'p2', 'person': 'Jakub Maruniak',
|
|
'year': 2024, 'thesis_element_id': 'th1'}],
|
|
gs.PERSON_KEYWORD_FACTS_QUERY: [{'person_element_id': 'p1', 'person': 'Maroš Harahus',
|
|
'keyword_element_id': 'k2', 'keyword': 'RAG'}],
|
|
gs.THESIS_PATHS_QUERY: [{'nodes': [{'kind': 'Thesis', 'element_id': 'th1', 'year': 2024}],
|
|
'relationships': ['AUTHORED']}],
|
|
})
|
|
result = gs.search_graph_with_session(session,
|
|
'Ktorá z osôb Maroš Harahus a Jakub Maruniak má kľúčové slovo RAG a jej práca je z roku 2024?')
|
|
assert result['facts'] == []
|
|
assert result['paths'] == []
|
|
|
|
|
|
def test_unknown_explicit_tag_does_not_return_unfiltered_titles():
|
|
session = FakeSession({gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [{'title': 'Nepodložené'}]})
|
|
result = gs.search_graph_with_session(session,
|
|
'Aký názov práce má Jakub Maruniak, ak má kľúčové slovo neexistujucitag?')
|
|
assert result['facts'] == []
|
|
assert result['constraints']['unresolved_keyword']
|
|
|
|
|
|
def test_unknown_quoted_title_does_not_list_every_thesis():
|
|
session = FakeSession({gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [{'title': 'Iná práca'}]})
|
|
result = gs.search_graph_with_session(session, 'Kto riešil diplomovú prácu „Úplne vymyslený názov“?')
|
|
assert result['facts'] == []
|
|
|
|
|
|
def test_recognized_query_entities_are_not_answer_evidence():
|
|
result = {'resolved_entities': [{'kind': 'Thesis', 'title': 'Vymyslený názov', 'year': 2024}],
|
|
'facts': [], 'paths': [], 'documents': [{'document_path': 'Vymyslený názov'}]}
|
|
assert not value_is_present('Vymyslený názov', result)
|
|
assert not property_present('Thesis.title', result)
|
|
|
|
|
|
def test_value_matching_has_word_boundaries():
|
|
assert not value_is_present('202', {'facts': [{'year': 2024}]})
|
|
|
|
|
|
def test_wrong_entity_type_cannot_satisfy_expected_keyword():
|
|
question = {'graph': {'expected_entities': [{'type': 'keyword', 'value': 'RAG'}]}}
|
|
row = evaluate_result(question, {'facts': [{'title': 'RAG', 'thesis_type': 'diploma_thesis'}]}, latency_seconds=0)
|
|
assert row['entity_score'] == 0
|
|
|
|
|
|
def test_correct_set_with_wrong_count_fails_strict():
|
|
question = {'category': 'candidate_set_aggregation', 'graph': {
|
|
'expected_count': 1, 'expected_entity_set': ['Eva Nová'], 'entity_set_mode': 'exact'}}
|
|
result = {'facts': [{'fact_type': 'person_thesis', 'person': 'Eva Nová'}],
|
|
'aggregation': {'count': 2, 'complete': True}}
|
|
row = evaluate_result(question, result, latency_seconds=0)
|
|
assert not row['strict'] and not row['count_ok']
|
|
|
|
|
|
def test_truncated_result_cannot_pass_strict():
|
|
assert not evaluate_result({'graph': {}}, {'truncated': True}, latency_seconds=0)['strict']
|
|
|
|
|
|
def test_source_audit_detects_nonexistent_tag_and_omitted_child_category():
|
|
payload = {'people': [{'id': 'p', 'name': 'Eva Nová', 'source_document': 'p.md'}],
|
|
'documents': [{'path': 'p.md', 'categories': ['bp2024']},
|
|
{'path': 'child.md', 'categories': ['tp2023']}],
|
|
'person_documents': [{'person_id': 'p', 'document_path': path} for path in ['p.md', 'child.md']],
|
|
'keywords': [{'id': 'nlp', 'name': 'NLP'}], 'document_keywords': [], 'theses': [], 'works': []}
|
|
question = {'id': 'example', 'split': 'dev', 'category': 'person_multiple_categories', 'question': '',
|
|
'graph': {'start_entities': [{'type': 'person', 'value': 'Eva Nová'}],
|
|
'expected_entities': [{'type': 'keyword', 'value': 'Chýbajúci tag'}],
|
|
'expected_entity_set': ['bp2024'], 'entity_set_mode': 'exact'}}
|
|
issues = audit_questions([question], payload)['questions'][0]['issues']
|
|
assert 'keyword_not_in_frontmatter_catalog' in issues
|
|
assert 'exact_categories_omit_or_add_source_categories' in issues
|