from __future__ import annotations import json import pytest from tools.graphrag.evaluation.graphrag_evaluator import evaluate_result, load_questions def academic( person="Alena Nová", year=2024, title="Práca A", identifier="thesis-a", document="a.md", **extra ): return { "fact_type": "person_thesis", "person": person, "thesis_id": identifier, "thesis_type": "bachelor_thesis", "title": title, "year": year, "document_path": document, "source_url": "https://example.test/" + document, **extra, } def keyword(person="Alena Nová", value="RAG", document="a.md"): return { "fact_type": "person_keyword", "person": person, "keyword": value, "document_path": document, "source_url": "https://example.test/" + document, } def question(bindings, **graph): return { "id": "synthetic-connected", "split": "dev", "question": "Overenie dôkazu.", "graph": {"schema_version": 3, "expected_bindings": bindings, **graph}, } def score(q, facts=(), **result): result.setdefault("status", "ok") return evaluate_result(q, {"facts": list(facts), **result}, latency_seconds=0) def test_same_person_keyword_and_academic_year_have_connected_proofs(): q = question( [{"person": "Alena Nová", "academic_kind": "Thesis", "year": 2024, "keyword": "RAG"}] ) row = score(q, [academic(), keyword()], status="ok") assert row["strict"] and row["bindings_ok"] and row["paths_ok"] assert len(row["path_results"]) == 2 assert all(path["witness"] for path in row["path_results"]) def test_disconnected_persons_cannot_satisfy_conjunction(): q = question([{"person": "Alena Nová", "year": 2024, "keyword": "RAG"}]) row = score(q, [academic(), keyword(person="Boris Iný")]) assert not row["strict"] and not row["bindings_ok"] def test_legacy_annotations_also_reject_disconnected_persons(): q = { "graph": { "schema_version": 2, "expected_entities": [ {"type": "person", "value": "Alena Nová"}, {"type": "keyword", "value": "RAG"}, {"type": "year", "value": 2024}, ], "expected_properties": ["Thesis.year"], "expected_relations": ["AUTHORED", "HAS_DOCUMENT", "HAS_KEYWORD"], } } assert not score(q, [academic(), keyword(person="Boris Iný")])["strict"] def test_start_year_does_not_satisfy_academic_year(): q = question([{"person": "Alena Nová", "year": 2024}]) row = score(q, [academic(year=2025, start_year=2024)]) assert not row["strict"] def test_legacy_academic_year_entity_is_not_start_year(): q = { "expected_answer_contains": ["2024"], "graph": { "expected_entities": [{"type": "year", "value": "2024"}], "expected_properties": ["Thesis.year"], }, } row = score(q, [academic(year=2025, start_year=2024)]) assert not row["strict"] and row["entity_score"] == 0 def test_title_and_year_must_belong_to_the_same_work(): q = question([{"person": "Alena Nová", "title": "Práca A", "year": 2025}]) assert not score(q, [academic(), academic(year=2025, title="Práca B", identifier="thesis-b")])[ "strict" ] def test_conflicting_same_node_properties_do_not_create_synthetic_work(): q = question([{"person": "Alena Nová", "title": "Práca A", "year": 2025}]) conflicting = academic(year=2025) conflicting.pop("title") assert not score(q, [academic(), conflicting])["strict"] def test_thesis_and_project_types_are_not_interchangeable(): q = question([{"person": "Alena Nová", "academic_kind": "Work", "year": 2024}]) assert not score(q, [academic()])["strict"] def test_project_proof_uses_has_work_not_authored(): q = question( [ { "person": "Alena Nová", "academic_kind": "Work", "academic_type": "diploma_project", "year": 2024, } ] ) fact = academic() fact.update(fact_type="person_work", work_id="project-a", work_type="diploma_project") fact.pop("thesis_id") fact.pop("thesis_type") assert score(q, [fact])["strict"] def test_implicit_academic_proof_accepts_either_real_kind(): assert score(question([{"person": "Alena Nová", "year": 2024}]), [academic()])["strict"] def test_metadata_can_come_from_same_person_family_when_not_same_document_required(): q = question([{"person": "Alena Nová", "year": 2024, "keyword": "RAG"}]) assert score(q, [academic(), keyword(document="child.md")])["strict"] def test_same_document_constraint_rejects_family_only_join(): q = question([{"person": "Alena Nová", "year": 2024, "keyword": "RAG", "same_document": True}]) assert not score(q, [academic(), keyword(document="child.md")])["strict"] def test_author_and_category_must_share_actual_document(): q = question( [{"person": "Alena Nová", "category": "bp2024", "document_author": "Autor Správny"}] ) category = { "fact_type": "person_category", "person": "Alena Nová", "category": "bp2024", "document_path": "a.md", } author = { "fact_type": "person_document_author", "person": "Alena Nová", "document_author": "Autor Správny", "document_path": "other.md", } assert not score(q, [category, author])["strict"] author["document_path"] = "a.md" assert score(q, [category, author])["strict"] def test_author_category_query_fact_can_carry_same_document_category_proof(): q = question( [{"person": "Alena Nová", "category": "bp2024", "document_author": "Autor Správny"}] ) author = { "fact_type": "person_document_author", "person": "Alena Nová", "category": "bp2024", "document_author": "Autor Správny", "document_path": "a.md", } assert score(q, [author])["strict"] def test_start_year_needs_direct_person_source_proof(): q = question([{"person": "Alena Nová", "start_year": 2020}]) assert not score(q, [academic(start_year=2020)])["strict"] direct = { "fact_type": "person_start_year", "person": "Alena Nová", "start_year": 2020, "document_path": "a.md", } assert score(q, [direct])["strict"] direct.pop("document_path") assert not score(q, [direct])["strict"] def test_conflicting_person_start_year_fails_closed(): q = question([{"person": "Alena Nová", "start_year": 2020}]) first = { "fact_type": "person_start_year", "person": "Alena Nová", "start_year": 2020, "document_path": "a.md", } assert not score(q, [first, {**first, "start_year": 2021}])["strict"] def test_explicit_path_pattern_is_bound_to_expected_subject(): q = question( [{"person": "Alena Nová", "keyword": "RAG"}], expected_path_patterns=[ { "binding_index": 0, "node_kinds": ["Person", "Document", "Keyword"], "relationships": ["HAS_DOCUMENT", "HAS_KEYWORD"], } ], ) assert not score(q, [keyword(person="Boris Iný")])["strict"] assert score(q, [keyword()])["strict"] def test_incompatible_ordered_path_does_not_forge_relationship(): q = question([{"person": "Alena Nová", "academic_kind": "Work", "year": 2024}]) path = { "nodes": [ {"kind": "Person", "name": "Alena Nová"}, {"kind": "Work", "id": "work", "year": 2024}, {"kind": "Document", "path": "a.md"}, ], "relationships": ["AUTHORED", "EVIDENCED_BY"], } assert not score(q, paths=[path])["strict"] path["relationships"][0] = "HAS_WORK" assert score(q, paths=[path])["strict"] def test_unrecognized_query_entities_are_never_proof(): q = question([{"person": "Alena Nová", "year": 2024}]) assert not score(q, resolved_entities=[{"kind": "Person", "name": "Alena Nová", "year": 2024}])[ "strict" ] def test_no_answer_is_a_success_only_when_no_positive_evidence_returned(): q = question([], expected_no_answer=True) q["should_answer"] = False assert score(q, status="no_answer", reason="unknown_person")["strict"] assert not score(q, [academic()], status="no_answer", reason="unknown_person")["strict"] assert not score(q, status="no_answer")["strict"] assert not score(q, status="ok", reason="unknown_person")["strict"] def test_no_answer_requires_expected_ambiguity_when_specified(): q = question([], expected_no_answer=True, expected_status="ambiguous") assert score(q, status="ambiguous", reason="multiple_people")["strict"] assert not score(q, status="no_answer", reason="unknown_person")["strict"] def test_false_comparison_can_be_no_answer_evidence_without_positive_claim(): q = question([], expected_no_answer=True) fact = { "fact_type": "academic_year_comparison", "shared_years": [], "same_academic_year": False, } assert score(q, [fact], status="no_answer", reason="no_shared_year")["strict"] def test_empty_positive_contract_does_not_pass_vacuously(): assert not score(question([]), status="ok")["strict"] assert not score(question([{}]), status="ok")["strict"] def test_zero_count_is_a_grounded_answer_not_a_no_answer(): q = question( [], expected_count=0, expected_entity_set=[], entity_set_mode="exact", entity_set_field="person", ) result = {"aggregation": {"count": 0, "people": [], "complete": True}} assert score(q, status="ok", **result)["strict"] assert not score(q, status="no_answer", reason="no_rows", **result)["strict"] def test_schema3_loader_rejects_unbound_path_pattern(tmp_path): q = question( [{"person": "Alena Nová", "keyword": "RAG"}], expected_path_patterns=[ { "binding_index": 2, "node_kinds": ["Person", "Document"], "relationships": ["HAS_DOCUMENT"], } ], ) path = tmp_path / "questions.json" path.write_text(json.dumps([q])) with pytest.raises(ValueError, match="invalid connected path pattern"): load_questions(path) def test_declared_count_and_people_must_match_actual_answer_evidence(): q = question( [{"person": "Alena Nová", "year": 2024}], expected_count=1, expected_entity_set=["Alena Nová"], entity_set_mode="exact", entity_set_field="person", ) assert score( q, [academic()], aggregation={"count": 1, "people": ["Alena Nová"], "complete": True} )["strict"] assert not score( q, [academic()], aggregation={"count": 1, "people": ["Boris Iný"], "complete": True} )["strict"] assert not score( q, [academic()], aggregation={"count": True, "people": ["Alena Nová"], "complete": True} )["strict"] def test_no_answer_cannot_hide_a_positive_comparison_claim(): q = question([], expected_no_answer=True) fact = {"fact_type": "academic_year_comparison", "shared_years": [], "same_academic_year": True} assert not score(q, [fact], status="no_answer", reason="no_shared_year")["strict"] def test_evidence_accepts_one_of_valid_source_alternatives_not_every_duplicate(): q = question( [{"person": "Alena Nová", "keyword": "RAG"}], expected_evidence_documents=[ { "binding_index": 0, "role": "keyword_document", "any_of": [ {"document_path": "a.md", "source_url": "https://example.test/a.md"}, {"document_path": "child.md", "source_url": "https://example.test/child.md"}, ], } ], ) assert score(q, [keyword(document="a.md")])["strict"] assert score(q, [keyword(document="child.md")])["strict"] assert not score(q, [keyword(document="unrelated.md")])["strict"] fact = keyword(document="a.md") fact["source_url"] = "https://wrong.test/" assert not score(q, [fact])["strict"]