"""Prevent query leakage and regression of raw-source title/year semantics.""" from __future__ import annotations import copy import pytest from tools.graphrag import graph_search as gs from tools.graphrag.graphrag import WorkHeading, extract_work_title, resolve_work_year from tools.graphrag.evaluation.benchmark_v4 import logical_group from tools.graphrag.evaluation.freeze_graphrag import make_freeze from tools.graphrag.evaluation.source_oracle import extract_academics from .test_evaluation_pipeline_v2 import freeze_inputs from .test_graph_search import FakeSession, sample_catalog def query_case(text, year=2024): return { "question": text, "category": "person_title_year", "graph": { "schema_version": 3, "source_profiles": ["profile.md"], "start_entities": [{"type": "person", "value": "Eva Nová"}], "expected_bindings": [ {"person": "Eva Nová", "year": year, "title": "Vyhľadávanie informácií"} ], }, } def test_paraphrases_and_evidence_path_variations_cannot_be_new_holdout(): first = query_case("Aký názov práce má Eva Nová v roku 2024?") second = query_case("Uveď názov práce osoby Eva Nová pre rok 2024.") second["graph"]["expected_bindings"][0]["document_path"] = "alternative.md" assert logical_group(first) == logical_group(second) assert logical_group(first) != logical_group( query_case("Aký názov práce má Eva Nová v roku 2025?", 2025) ) def group_freeze_inputs(): dev, _, fingerprints, audit = freeze_inputs() questions = [ query_case("Aký názov práce má Eva Nová v roku 2024?"), query_case("Aký názov práce má Eva Nová v roku 2025?", 2025), ] for index, q in enumerate(questions): q.update(id=["dev-a", "test-b"][index], split=["dev", "test"][index]) q["graph"]["partition_group"] = logical_group(q) audit["partition_policy"] = { "method": "disjoint logical query groups", "previously_exposed_groups": [logical_group(questions[0])], "limitations": ["Known profiles."], } return dev, questions, fingerprints, audit def test_group_freeze_explicitly_reports_known_profiles(): manifest = make_freeze(*group_freeze_inputs()) assert manifest["partition"]["source_profile_overlap"] == 1 assert manifest["partition"]["logical_group_overlap"] == 0 assert manifest["partition"]["method"] == "disjoint logical query groups" @pytest.mark.parametrize("mutation", ["overlap", "exposed", "forged_group"]) def test_group_freeze_rejects_leakage(mutation): dev, questions, fingerprints, audit = group_freeze_inputs() if mutation == "overlap": questions[1] = {**copy.deepcopy(questions[0]), "id": "test-b", "split": "test"} elif mutation == "exposed": audit["partition_policy"]["previously_exposed_groups"].append(logical_group(questions[1])) else: questions[1]["graph"]["partition_group"] = "invented" with pytest.raises(RuntimeError): make_freeze(dev, questions, fingerprints, audit) def test_explicit_academic_range_is_not_overridden_by_unrelated_category(): assert ( resolve_work_year( WorkHeading("research_project", "Vedecký projekt 2022/23", "vp"), ["vp2021"] ) == 2023 ) @pytest.mark.parametrize( "prefix", ["Pozrieť si", "Projekt odložený.", "Konzultácie sa budú konať osobne."] ) def test_instructions_are_not_academic_titles(prefix): assert extract_work_title([prefix + "\n\n- Oboznámte sa s nástrojmi."]) is None doc = { "path": "p.md", "body_offset": 0, "categories": [], "body_lines": ["# Bakalársky projekt 2024", prefix, "", "- Oboznámte sa s nástrojmi."], } assert extract_academics(doc)[0]["title"] is None def test_last_leading_title_wins_over_introductory_description(): doc = { "path": "p.md", "body_offset": 0, "categories": [], "body_lines": [ "# Bakalárska práca 2025", "Chceme vytvoriť asistenta pre vyhľadávanie.", "", "Generovanie jazyka s podporou vyhľadávania", "", "1. Vypracujte prehľad.", ], } assert extract_academics(doc)[0]["title"] == "Generovanie jazyka s podporou vyhľadávania" def test_title_lookup_preserves_known_record_without_claiming_a_title(): fact = { "person_element_id": "p2", "person": "Jakub Maruniak", "thesis_element_id": "th1", "year": 2024, "title": None, "thesis_type": "diploma_thesis", } session = FakeSession({gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [fact]}) result = gs.search_graph_with_session( session, "Aký názov práce má osoba Jakub Maruniak v roku 2024?" ) # A missing title must not erase the independently known work/year. The # delivery regression checks the explicit "title missing" presentation. assert result["status"] == "ok" assert len(result["facts"]) == 1 assert result["facts"][0]["title"] is None assert result["facts"][0]["year"] == 2024 def test_explicit_name_precedes_a_different_topic_and_link_title_is_preserved(): assert ( extract_work_title(["Téma: Všeobecná téma projektu\nNávrh na názov: Konkrétny názov práce"]) == "Konkrétny názov práce" ) assert ( extract_work_title(["*Písomná práca*: [Rešerš na tému prekladu](./report)"]) == "Rešerš na tému prekladu" ) def test_raw_oracle_skips_repository_and_related_work_lines(): doc = { "path": "p.md", "body_offset": 0, "categories": [], "body_lines": [ "# Bakalárska práca 2025", "Systém pre monitoring médií", "", "Práca súvisí s DP iného študenta", "Repo https://example.org/repo", "", "Zadanie:", "- Vypracujte prehľad.", ], } assert extract_academics(doc)[0]["title"] == "Systém pre monitoring médií" def test_generated_claim_year_is_not_satisfied_by_start_year(): from tools.graphrag.evaluation.check_model_sample import grade_claims oracle = { "people": { "Eva Nová": { "start_year": 2024, "academics": [{"year": 2025, "title": "Vyhľadávanie informácií"}], } } } case = query_case("Aká práca v roku 2024?") answer = { "status": "ok", "claims": [{"person": "Eva Nová", "year": 2024, "title": "Vyhľadávanie informácií"}], "source_urls": [], } grade = grade_claims(case, answer, oracle) assert not grade["strict"] and grade["unsupported_claims"] def test_generated_claims_cannot_distribute_conditions_across_people(): from tools.graphrag.evaluation.check_model_sample import grade_claims case = query_case("Kto má tag aj rok?") case["graph"]["expected_bindings"] = [{"person": "Eva Nová", "year": 2024, "keyword": "nlp"}] oracle = { "people": { "Eva Nová": { "start_year": 2020, "academics": [{"year": 2024, "title": None}], "documents": [], }, "Boris Malý": {"start_year": 2020, "academics": [], "documents": ["b.md"]}, }, "documents": {"b.md": {"tags": ["nlp"], "categories": [], "author": None}}, } answer = { "status": "ok", "claims": [ {"person": "Eva Nová", "year": 2024}, {"person": "Boris Malý", "keyword": "nlp"}, ], "source_urls": [], } assert not grade_claims(case, answer, oracle)["bindings_ok"]