from __future__ import annotations import json import pytest from tools.graphrag.evaluation.graphrag_evaluator import evaluate_result, load_questions def academic(person='Alena Nová', year=2024, title='Práca A', identifier='thesis-a', document='a.md', **extra): return {'fact_type': 'person_thesis', 'person': person, 'thesis_id': identifier, 'thesis_type': 'bachelor_thesis', 'title': title, 'year': year, 'document_path': document, 'source_url': 'https://example.test/' + document, **extra} def keyword(person='Alena Nová', value='RAG', document='a.md'): return {'fact_type': 'person_keyword', 'person': person, 'keyword': value, 'document_path': document, 'source_url': 'https://example.test/' + document} def question(bindings, **graph): return {'id': 'synthetic-connected', 'split': 'dev', 'question': 'Overenie dôkazu.', 'graph': {'schema_version': 3, 'expected_bindings': bindings, **graph}} def score(q, facts=(), **result): result.setdefault('status', 'ok') return evaluate_result(q, {'facts': list(facts), **result}, latency_seconds=0) def test_same_person_keyword_and_academic_year_have_connected_proofs(): q = question([{'person': 'Alena Nová', 'academic_kind': 'Thesis', 'year': 2024, 'keyword': 'RAG'}]) row = score(q, [academic(), keyword()], status='ok') assert row['strict'] and row['bindings_ok'] and row['paths_ok'] assert len(row['path_results']) == 2 assert all(path['witness'] for path in row['path_results']) def test_disconnected_persons_cannot_satisfy_conjunction(): q = question([{'person': 'Alena Nová', 'year': 2024, 'keyword': 'RAG'}]) row = score(q, [academic(), keyword(person='Boris Iný')]) assert not row['strict'] and not row['bindings_ok'] def test_legacy_annotations_also_reject_disconnected_persons(): q = {'graph': {'schema_version': 2, 'expected_entities': [ {'type': 'person', 'value': 'Alena Nová'}, {'type': 'keyword', 'value': 'RAG'}, {'type': 'year', 'value': 2024}], 'expected_properties': ['Thesis.year'], 'expected_relations': ['AUTHORED', 'HAS_DOCUMENT', 'HAS_KEYWORD']}} assert not score(q, [academic(), keyword(person='Boris Iný')])['strict'] def test_start_year_does_not_satisfy_academic_year(): q = question([{'person': 'Alena Nová', 'year': 2024}]) row = score(q, [academic(year=2025, start_year=2024)]) assert not row['strict'] def test_legacy_academic_year_entity_is_not_start_year(): q = {'expected_answer_contains': ['2024'], 'graph': {'expected_entities': [ {'type': 'year', 'value': '2024'}], 'expected_properties': ['Thesis.year']}} row = score(q, [academic(year=2025, start_year=2024)]) assert not row['strict'] and row['entity_score'] == 0 def test_title_and_year_must_belong_to_the_same_work(): q = question([{'person': 'Alena Nová', 'title': 'Práca A', 'year': 2025}]) assert not score(q, [academic(), academic(year=2025, title='Práca B', identifier='thesis-b')])['strict'] def test_conflicting_same_node_properties_do_not_create_synthetic_work(): q = question([{'person': 'Alena Nová', 'title': 'Práca A', 'year': 2025}]) conflicting = academic(year=2025) conflicting.pop('title') assert not score(q, [academic(), conflicting])['strict'] def test_thesis_and_project_types_are_not_interchangeable(): q = question([{'person': 'Alena Nová', 'academic_kind': 'Work', 'year': 2024}]) assert not score(q, [academic()])['strict'] def test_project_proof_uses_has_work_not_authored(): q = question([{'person': 'Alena Nová', 'academic_kind': 'Work', 'academic_type': 'diploma_project', 'year': 2024}]) fact = academic() fact.update(fact_type='person_work', work_id='project-a', work_type='diploma_project') fact.pop('thesis_id'); fact.pop('thesis_type') assert score(q, [fact])['strict'] def test_implicit_academic_proof_accepts_either_real_kind(): assert score(question([{'person': 'Alena Nová', 'year': 2024}]), [academic()])['strict'] def test_metadata_can_come_from_same_person_family_when_not_same_document_required(): q = question([{'person': 'Alena Nová', 'year': 2024, 'keyword': 'RAG'}]) assert score(q, [academic(), keyword(document='child.md')])['strict'] def test_same_document_constraint_rejects_family_only_join(): q = question([{'person': 'Alena Nová', 'year': 2024, 'keyword': 'RAG', 'same_document': True}]) assert not score(q, [academic(), keyword(document='child.md')])['strict'] def test_author_and_category_must_share_actual_document(): q = question([{'person': 'Alena Nová', 'category': 'bp2024', 'document_author': 'Autor Správny'}]) category = {'fact_type': 'person_category', 'person': 'Alena Nová', 'category': 'bp2024', 'document_path': 'a.md'} author = {'fact_type': 'person_document_author', 'person': 'Alena Nová', 'document_author': 'Autor Správny', 'document_path': 'other.md'} assert not score(q, [category, author])['strict'] author['document_path'] = 'a.md' assert score(q, [category, author])['strict'] def test_author_category_query_fact_can_carry_same_document_category_proof(): q = question([{'person': 'Alena Nová', 'category': 'bp2024', 'document_author': 'Autor Správny'}]) author = {'fact_type': 'person_document_author', 'person': 'Alena Nová', 'category': 'bp2024', 'document_author': 'Autor Správny', 'document_path': 'a.md'} assert score(q, [author])['strict'] def test_start_year_needs_direct_person_source_proof(): q = question([{'person': 'Alena Nová', 'start_year': 2020}]) assert not score(q, [academic(start_year=2020)])['strict'] direct = {'fact_type': 'person_start_year', 'person': 'Alena Nová', 'start_year': 2020, 'document_path': 'a.md'} assert score(q, [direct])['strict'] direct.pop('document_path') assert not score(q, [direct])['strict'] def test_conflicting_person_start_year_fails_closed(): q = question([{'person': 'Alena Nová', 'start_year': 2020}]) first = {'fact_type': 'person_start_year', 'person': 'Alena Nová', 'start_year': 2020, 'document_path': 'a.md'} assert not score(q, [first, {**first, 'start_year': 2021}])['strict'] def test_explicit_path_pattern_is_bound_to_expected_subject(): q = question([{'person': 'Alena Nová', 'keyword': 'RAG'}], expected_path_patterns=[{ 'binding_index': 0, 'node_kinds': ['Person', 'Document', 'Keyword'], 'relationships': ['HAS_DOCUMENT', 'HAS_KEYWORD']}]) assert not score(q, [keyword(person='Boris Iný')])['strict'] assert score(q, [keyword()])['strict'] def test_incompatible_ordered_path_does_not_forge_relationship(): q = question([{'person': 'Alena Nová', 'academic_kind': 'Work', 'year': 2024}]) path = {'nodes': [{'kind': 'Person', 'name': 'Alena Nová'}, {'kind': 'Work', 'id': 'work', 'year': 2024}, {'kind': 'Document', 'path': 'a.md'}], 'relationships': ['AUTHORED', 'EVIDENCED_BY']} assert not score(q, paths=[path])['strict'] path['relationships'][0] = 'HAS_WORK' assert score(q, paths=[path])['strict'] def test_unrecognized_query_entities_are_never_proof(): q = question([{'person': 'Alena Nová', 'year': 2024}]) assert not score(q, resolved_entities=[{'kind': 'Person', 'name': 'Alena Nová', 'year': 2024}])['strict'] def test_no_answer_is_a_success_only_when_no_positive_evidence_returned(): q = question([], expected_no_answer=True) q['should_answer'] = False assert score(q, status='no_answer', reason='unknown_person')['strict'] assert not score(q, [academic()], status='no_answer', reason='unknown_person')['strict'] assert not score(q, status='no_answer')['strict'] assert not score(q, status='ok', reason='unknown_person')['strict'] def test_no_answer_requires_expected_ambiguity_when_specified(): q = question([], expected_no_answer=True, expected_status='ambiguous') assert score(q, status='ambiguous', reason='multiple_people')['strict'] assert not score(q, status='no_answer', reason='unknown_person')['strict'] def test_false_comparison_can_be_no_answer_evidence_without_positive_claim(): q = question([], expected_no_answer=True) fact = {'fact_type': 'academic_year_comparison', 'shared_years': [], 'same_academic_year': False} assert score(q, [fact], status='no_answer', reason='no_shared_year')['strict'] def test_empty_positive_contract_does_not_pass_vacuously(): assert not score(question([]), status='ok')['strict'] assert not score(question([{}]), status='ok')['strict'] def test_zero_count_is_a_grounded_answer_not_a_no_answer(): q = question([], expected_count=0, expected_entity_set=[], entity_set_mode='exact', entity_set_field='person') result = {'aggregation': {'count': 0, 'people': [], 'complete': True}} assert score(q, status='ok', **result)['strict'] assert not score(q, status='no_answer', reason='no_rows', **result)['strict'] def test_schema3_loader_rejects_unbound_path_pattern(tmp_path): q = question([{'person': 'Alena Nová', 'keyword': 'RAG'}], expected_path_patterns=[{ 'binding_index': 2, 'node_kinds': ['Person', 'Document'], 'relationships': ['HAS_DOCUMENT']}]) path = tmp_path / 'questions.json'; path.write_text(json.dumps([q])) with pytest.raises(ValueError, match='invalid connected path pattern'): load_questions(path) def test_declared_count_and_people_must_match_actual_answer_evidence(): q = question([{'person': 'Alena Nová', 'year': 2024}], expected_count=1, expected_entity_set=['Alena Nová'], entity_set_mode='exact', entity_set_field='person') assert score(q, [academic()], aggregation={'count': 1, 'people': ['Alena Nová'], 'complete': True})['strict'] assert not score(q, [academic()], aggregation={'count': 1, 'people': ['Boris Iný'], 'complete': True})['strict'] assert not score(q, [academic()], aggregation={'count': True, 'people': ['Alena Nová'], 'complete': True})['strict'] def test_no_answer_cannot_hide_a_positive_comparison_claim(): q = question([], expected_no_answer=True) fact = {'fact_type': 'academic_year_comparison', 'shared_years': [], 'same_academic_year': True} assert not score(q, [fact], status='no_answer', reason='no_shared_year')['strict'] def test_evidence_accepts_one_of_valid_source_alternatives_not_every_duplicate(): q = question([{'person': 'Alena Nová', 'keyword': 'RAG'}], expected_evidence_documents=[{ 'binding_index': 0, 'role': 'keyword_document', 'any_of': [ {'document_path': 'a.md', 'source_url': 'https://example.test/a.md'}, {'document_path': 'child.md', 'source_url': 'https://example.test/child.md'}]}]) assert score(q, [keyword(document='a.md')])['strict'] assert score(q, [keyword(document='child.md')])['strict'] assert not score(q, [keyword(document='unrelated.md')])['strict'] fact = keyword(document='a.md'); fact['source_url'] = 'https://wrong.test/' assert not score(q, [fact])['strict']