from __future__ import annotations import pytest from tools.graphrag import graph_search as gs from tools.graphrag.evaluation.audit_benchmark import audit_questions from tools.graphrag.evaluation.graphrag_evaluator import ( evaluate_result, property_present, value_is_present, ) from .test_graph_search import FakeSession, sample_catalog @pytest.mark.parametrize("word", ["názov", "názvom", "názvu"]) def test_title_inflections(word): assert gs.extract_query_constraints(f"Prepoj osobu s {word} jej práce.")["mentions_title"] @pytest.mark.parametrize( "question", [ "V ktorom rovnakom roku je uvedená práca pri osobách?", "Aký rok práce spája tieto osoby?", "Prepoj osobu s názvom jej práce a rokom.", ], ) def test_academic_year_intent_does_not_require_a_numeric_year(question): assert gs.extract_query_constraints(question)["mentions_academic_year"] def test_tag_is_extracted_without_following_clause(): assert gs.explicit_keyword_phrase("Kľúčové slovo RAG a jej práca v roku 2024?") == "rag" def test_incidental_zp_tag_does_not_filter_title_lookup(): catalog = sample_catalog() + [{"element_id": "zp", "kind": "Keyword", "id": "zp", "name": "zp"}] fact = { "person_element_id": "p2", "person": "Jakub Maruniak", "thesis_element_id": "th1", "title": "Ukážková práca", "year": 2021, "thesis_type": "diploma_thesis", } session = FakeSession({gs.CATALOG_QUERY: catalog, gs.THESIS_FACTS_QUERY: [fact]}) result = gs.search_graph_with_session( session, "Aký názov práce je v ZP Wiki pri osobe Jakub Maruniak?" ) assert any(row["fact_type"] == "person_thesis" for row in result["facts"]) assert gs.PERSON_KEYWORD_FACTS_QUERY not in {query for query, _ in session.calls} def test_keyword_and_year_cannot_be_satisfied_by_different_people(): session = FakeSession( { gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [ { "person_element_id": "p2", "person": "Jakub Maruniak", "year": 2024, "thesis_element_id": "th1", } ], gs.PERSON_KEYWORD_FACTS_QUERY: [ { "person_element_id": "p1", "person": "Maroš Harahus", "keyword_element_id": "k2", "keyword": "RAG", } ], gs.THESIS_PATHS_QUERY: [ { "nodes": [{"kind": "Thesis", "element_id": "th1", "year": 2024}], "relationships": ["AUTHORED"], } ], } ) result = gs.search_graph_with_session( session, "Ktorá z osôb Maroš Harahus a Jakub Maruniak má kľúčové slovo RAG a jej práca je z roku 2024?", ) assert result["facts"] == [] assert result["paths"] == [] def test_unknown_explicit_tag_does_not_return_unfiltered_titles(): session = FakeSession( {gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [{"title": "Nepodložené"}]} ) result = gs.search_graph_with_session( session, "Aký názov práce má Jakub Maruniak, ak má kľúčové slovo neexistujucitag?" ) assert result["facts"] == [] assert result["constraints"]["unresolved_keyword"] def test_unknown_quoted_title_does_not_list_every_thesis(): session = FakeSession( {gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [{"title": "Iná práca"}]} ) result = gs.search_graph_with_session( session, "Kto riešil diplomovú prácu „Úplne vymyslený názov“?" ) assert result["facts"] == [] def test_recognized_query_entities_are_not_answer_evidence(): result = { "resolved_entities": [{"kind": "Thesis", "title": "Vymyslený názov", "year": 2024}], "facts": [], "paths": [], "documents": [{"document_path": "Vymyslený názov"}], } assert not value_is_present("Vymyslený názov", result) assert not property_present("Thesis.title", result) def test_value_matching_has_word_boundaries(): assert not value_is_present("202", {"facts": [{"year": 2024}]}) def test_wrong_entity_type_cannot_satisfy_expected_keyword(): question = {"graph": {"expected_entities": [{"type": "keyword", "value": "RAG"}]}} row = evaluate_result( question, {"facts": [{"title": "RAG", "thesis_type": "diploma_thesis"}]}, latency_seconds=0 ) assert row["entity_score"] == 0 def test_correct_set_with_wrong_count_fails_strict(): question = { "category": "candidate_set_aggregation", "graph": { "expected_count": 1, "expected_entity_set": ["Eva Nová"], "entity_set_mode": "exact", }, } result = { "facts": [{"fact_type": "person_thesis", "person": "Eva Nová"}], "aggregation": {"count": 2, "complete": True}, } row = evaluate_result(question, result, latency_seconds=0) assert not row["strict"] and not row["count_ok"] def test_truncated_result_cannot_pass_strict(): assert not evaluate_result({"graph": {}}, {"truncated": True}, latency_seconds=0)["strict"] def test_source_audit_detects_nonexistent_tag_and_omitted_child_category(): payload = { "people": [{"id": "p", "name": "Eva Nová", "source_document": "p.md"}], "documents": [ {"path": "p.md", "categories": ["bp2024"]}, {"path": "child.md", "categories": ["tp2023"]}, ], "person_documents": [ {"person_id": "p", "document_path": path} for path in ["p.md", "child.md"] ], "keywords": [{"id": "nlp", "name": "NLP"}], "document_keywords": [], "theses": [], "works": [], } question = { "id": "example", "split": "dev", "category": "person_multiple_categories", "question": "", "graph": { "start_entities": [{"type": "person", "value": "Eva Nová"}], "expected_entities": [{"type": "keyword", "value": "Chýbajúci tag"}], "expected_entity_set": ["bp2024"], "entity_set_mode": "exact", }, } issues = audit_questions([question], payload)["questions"][0]["issues"] assert "keyword_not_in_frontmatter_catalog" in issues assert "exact_categories_omit_or_add_source_categories" in issues