"""Prevent query leakage and regression of raw-source title/year semantics.""" from __future__ import annotations import copy import pytest from tools.graphrag import graph_search as gs from tools.graphrag.graphrag import WorkHeading, extract_work_title, resolve_work_year from tools.graphrag.evaluation.benchmark_v4 import logical_group from tools.graphrag.evaluation.freeze_graphrag import make_freeze from tools.graphrag.evaluation.source_oracle import extract_academics from .test_evaluation_pipeline_v2 import freeze_inputs from .test_graph_search import FakeSession, sample_catalog def query_case(text, year=2024): return {'question': text, 'category': 'person_title_year', 'graph': { 'schema_version': 3, 'source_profiles': ['profile.md'], 'start_entities': [{'type': 'person', 'value': 'Eva Nová'}], 'expected_bindings': [{'person': 'Eva Nová', 'year': year, 'title': 'Vyhľadávanie informácií'}]}} def test_paraphrases_and_evidence_path_variations_cannot_be_new_holdout(): first = query_case('Aký názov práce má Eva Nová v roku 2024?') second = query_case('Uveď názov práce osoby Eva Nová pre rok 2024.') second['graph']['expected_bindings'][0]['document_path'] = 'alternative.md' assert logical_group(first) == logical_group(second) assert logical_group(first) != logical_group(query_case('Aký názov práce má Eva Nová v roku 2025?', 2025)) def group_freeze_inputs(): dev, _, fingerprints, audit = freeze_inputs() questions = [query_case('Aký názov práce má Eva Nová v roku 2024?'), query_case('Aký názov práce má Eva Nová v roku 2025?', 2025)] for index, q in enumerate(questions): q.update(id=['dev-a', 'test-b'][index], split=['dev', 'test'][index]) q['graph']['partition_group'] = logical_group(q) audit['partition_policy'] = {'method': 'disjoint logical query groups', 'previously_exposed_groups': [logical_group(questions[0])], 'limitations': ['Known profiles.']} return dev, questions, fingerprints, audit def test_group_freeze_explicitly_reports_known_profiles(): manifest = make_freeze(*group_freeze_inputs()) assert manifest['partition']['source_profile_overlap'] == 1 assert manifest['partition']['logical_group_overlap'] == 0 assert manifest['partition']['method'] == 'disjoint logical query groups' @pytest.mark.parametrize('mutation', ['overlap', 'exposed', 'forged_group']) def test_group_freeze_rejects_leakage(mutation): dev, questions, fingerprints, audit = group_freeze_inputs() if mutation == 'overlap': questions[1] = {**copy.deepcopy(questions[0]), 'id': 'test-b', 'split': 'test'} elif mutation == 'exposed': audit['partition_policy']['previously_exposed_groups'].append(logical_group(questions[1])) else: questions[1]['graph']['partition_group'] = 'invented' with pytest.raises(RuntimeError): make_freeze(dev, questions, fingerprints, audit) def test_explicit_academic_range_is_not_overridden_by_unrelated_category(): assert resolve_work_year(WorkHeading('research_project', 'Vedecký projekt 2022/23', 'vp'), ['vp2021']) == 2023 @pytest.mark.parametrize('prefix', ['Pozrieť si', 'Projekt odložený.', 'Konzultácie sa budú konať osobne.']) def test_instructions_are_not_academic_titles(prefix): assert extract_work_title([prefix + '\n\n- Oboznámte sa s nástrojmi.']) is None doc = {'path': 'p.md', 'body_offset': 0, 'categories': [], 'body_lines': ['# Bakalársky projekt 2024', prefix, '', '- Oboznámte sa s nástrojmi.']} assert extract_academics(doc)[0]['title'] is None def test_last_leading_title_wins_over_introductory_description(): doc = {'path': 'p.md', 'body_offset': 0, 'categories': [], 'body_lines': [ '# Bakalárska práca 2025', 'Chceme vytvoriť asistenta pre vyhľadávanie.', '', 'Generovanie jazyka s podporou vyhľadávania', '', '1. Vypracujte prehľad.']} assert extract_academics(doc)[0]['title'] == 'Generovanie jazyka s podporou vyhľadávania' def test_title_lookup_does_not_claim_untitled_work_as_an_answer(): fact = {'person_element_id': 'p2', 'person': 'Jakub Maruniak', 'thesis_element_id': 'th1', 'year': 2024, 'title': None, 'thesis_type': 'diploma_thesis'} session = FakeSession({gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [fact]}) result = gs.search_graph_with_session(session, 'Aký názov práce má osoba Jakub Maruniak v roku 2024?') assert result['status'] == 'no_answer' assert result['facts'] == [] def test_explicit_name_precedes_a_different_topic_and_link_title_is_preserved(): assert extract_work_title(['Téma: Všeobecná téma projektu\nNávrh na názov: Konkrétny názov práce']) == 'Konkrétny názov práce' assert extract_work_title(['*Písomná práca*: [Rešerš na tému prekladu](./report)']) == 'Rešerš na tému prekladu' def test_raw_oracle_skips_repository_and_related_work_lines(): doc = {'path': 'p.md', 'body_offset': 0, 'categories': [], 'body_lines': [ '# Bakalárska práca 2025', 'Systém pre monitoring médií', '', 'Práca súvisí s DP iného študenta', 'Repo https://example.org/repo', '', 'Zadanie:', '- Vypracujte prehľad.']} assert extract_academics(doc)[0]['title'] == 'Systém pre monitoring médií' def test_generated_claim_year_is_not_satisfied_by_start_year(): from tools.graphrag.evaluation.check_openwebui import grade_claims oracle = {'people': {'Eva Nová': {'start_year': 2024, 'academics': [ {'year': 2025, 'title': 'Vyhľadávanie informácií'}]}}} case = query_case('Aká práca v roku 2024?') answer = {'status': 'ok', 'claims': [{'person': 'Eva Nová', 'year': 2024, 'title': 'Vyhľadávanie informácií'}], 'source_urls': []} grade = grade_claims(case, answer, oracle) assert not grade['strict'] and grade['unsupported_claims'] def test_generated_claims_cannot_distribute_conditions_across_people(): from tools.graphrag.evaluation.check_openwebui import grade_claims case = query_case('Kto má tag aj rok?') case['graph']['expected_bindings'] = [{'person': 'Eva Nová', 'year': 2024, 'keyword': 'nlp'}] oracle = {'people': {'Eva Nová': {'start_year': 2020, 'academics': [{'year': 2024, 'title': None}], 'documents': []}, 'Boris Malý': {'start_year': 2020, 'academics': [], 'documents': ['b.md']}}, 'documents': {'b.md': {'tags': ['nlp'], 'categories': [], 'author': None}}} answer = {'status': 'ok', 'claims': [{'person': 'Eva Nová', 'year': 2024}, {'person': 'Boris Malý', 'keyword': 'nlp'}], 'source_urls': []} assert not grade_claims(case, answer, oracle)['bindings_ok']