124 lines
6.7 KiB
Python
124 lines
6.7 KiB
Python
"""Prevent query leakage and regression of raw-source title/year semantics."""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
|
|
import pytest
|
|
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.graphrag import WorkHeading, extract_work_title, resolve_work_year
|
|
from tools.graphrag.evaluation.benchmark_v4 import logical_group
|
|
from tools.graphrag.evaluation.freeze_graphrag import make_freeze
|
|
from tools.graphrag.evaluation.source_oracle import extract_academics
|
|
from .test_evaluation_pipeline_v2 import freeze_inputs
|
|
from .test_graph_search import FakeSession, sample_catalog
|
|
|
|
|
|
def query_case(text, year=2024):
|
|
return {'question': text, 'category': 'person_title_year', 'graph': {
|
|
'schema_version': 3, 'source_profiles': ['profile.md'],
|
|
'start_entities': [{'type': 'person', 'value': 'Eva Nová'}],
|
|
'expected_bindings': [{'person': 'Eva Nová', 'year': year, 'title': 'Vyhľadávanie informácií'}]}}
|
|
|
|
|
|
def test_paraphrases_and_evidence_path_variations_cannot_be_new_holdout():
|
|
first = query_case('Aký názov práce má Eva Nová v roku 2024?')
|
|
second = query_case('Uveď názov práce osoby Eva Nová pre rok 2024.')
|
|
second['graph']['expected_bindings'][0]['document_path'] = 'alternative.md'
|
|
assert logical_group(first) == logical_group(second)
|
|
assert logical_group(first) != logical_group(query_case('Aký názov práce má Eva Nová v roku 2025?', 2025))
|
|
|
|
|
|
def group_freeze_inputs():
|
|
dev, _, fingerprints, audit = freeze_inputs()
|
|
questions = [query_case('Aký názov práce má Eva Nová v roku 2024?'),
|
|
query_case('Aký názov práce má Eva Nová v roku 2025?', 2025)]
|
|
for index, q in enumerate(questions):
|
|
q.update(id=['dev-a', 'test-b'][index], split=['dev', 'test'][index])
|
|
q['graph']['partition_group'] = logical_group(q)
|
|
audit['partition_policy'] = {'method': 'disjoint logical query groups',
|
|
'previously_exposed_groups': [logical_group(questions[0])], 'limitations': ['Known profiles.']}
|
|
return dev, questions, fingerprints, audit
|
|
|
|
|
|
def test_group_freeze_explicitly_reports_known_profiles():
|
|
manifest = make_freeze(*group_freeze_inputs())
|
|
assert manifest['partition']['source_profile_overlap'] == 1
|
|
assert manifest['partition']['logical_group_overlap'] == 0
|
|
assert manifest['partition']['method'] == 'disjoint logical query groups'
|
|
|
|
|
|
@pytest.mark.parametrize('mutation', ['overlap', 'exposed', 'forged_group'])
|
|
def test_group_freeze_rejects_leakage(mutation):
|
|
dev, questions, fingerprints, audit = group_freeze_inputs()
|
|
if mutation == 'overlap':
|
|
questions[1] = {**copy.deepcopy(questions[0]), 'id': 'test-b', 'split': 'test'}
|
|
elif mutation == 'exposed':
|
|
audit['partition_policy']['previously_exposed_groups'].append(logical_group(questions[1]))
|
|
else:
|
|
questions[1]['graph']['partition_group'] = 'invented'
|
|
with pytest.raises(RuntimeError):
|
|
make_freeze(dev, questions, fingerprints, audit)
|
|
|
|
|
|
def test_explicit_academic_range_is_not_overridden_by_unrelated_category():
|
|
assert resolve_work_year(WorkHeading('research_project', 'Vedecký projekt 2022/23', 'vp'), ['vp2021']) == 2023
|
|
|
|
|
|
@pytest.mark.parametrize('prefix', ['Pozrieť si', 'Projekt odložený.', 'Konzultácie sa budú konať osobne.'])
|
|
def test_instructions_are_not_academic_titles(prefix):
|
|
assert extract_work_title([prefix + '\n\n- Oboznámte sa s nástrojmi.']) is None
|
|
doc = {'path': 'p.md', 'body_offset': 0, 'categories': [],
|
|
'body_lines': ['# Bakalársky projekt 2024', prefix, '', '- Oboznámte sa s nástrojmi.']}
|
|
assert extract_academics(doc)[0]['title'] is None
|
|
|
|
|
|
def test_last_leading_title_wins_over_introductory_description():
|
|
doc = {'path': 'p.md', 'body_offset': 0, 'categories': [], 'body_lines': [
|
|
'# Bakalárska práca 2025', 'Chceme vytvoriť asistenta pre vyhľadávanie.', '',
|
|
'Generovanie jazyka s podporou vyhľadávania', '', '1. Vypracujte prehľad.']}
|
|
assert extract_academics(doc)[0]['title'] == 'Generovanie jazyka s podporou vyhľadávania'
|
|
|
|
|
|
def test_title_lookup_does_not_claim_untitled_work_as_an_answer():
|
|
fact = {'person_element_id': 'p2', 'person': 'Jakub Maruniak', 'thesis_element_id': 'th1',
|
|
'year': 2024, 'title': None, 'thesis_type': 'diploma_thesis'}
|
|
session = FakeSession({gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [fact]})
|
|
result = gs.search_graph_with_session(session, 'Aký názov práce má osoba Jakub Maruniak v roku 2024?')
|
|
assert result['status'] == 'no_answer'
|
|
assert result['facts'] == []
|
|
|
|
|
|
def test_explicit_name_precedes_a_different_topic_and_link_title_is_preserved():
|
|
assert extract_work_title(['Téma: Všeobecná téma projektu\nNávrh na názov: Konkrétny názov práce']) == 'Konkrétny názov práce'
|
|
assert extract_work_title(['*Písomná práca*: [Rešerš na tému prekladu](./report)']) == 'Rešerš na tému prekladu'
|
|
|
|
|
|
def test_raw_oracle_skips_repository_and_related_work_lines():
|
|
doc = {'path': 'p.md', 'body_offset': 0, 'categories': [], 'body_lines': [
|
|
'# Bakalárska práca 2025', 'Systém pre monitoring médií', '',
|
|
'Práca súvisí s DP iného študenta', 'Repo https://example.org/repo', '', 'Zadanie:', '- Vypracujte prehľad.']}
|
|
assert extract_academics(doc)[0]['title'] == 'Systém pre monitoring médií'
|
|
|
|
|
|
def test_generated_claim_year_is_not_satisfied_by_start_year():
|
|
from tools.graphrag.evaluation.check_openwebui import grade_claims
|
|
oracle = {'people': {'Eva Nová': {'start_year': 2024, 'academics': [
|
|
{'year': 2025, 'title': 'Vyhľadávanie informácií'}]}}}
|
|
case = query_case('Aká práca v roku 2024?')
|
|
answer = {'status': 'ok', 'claims': [{'person': 'Eva Nová', 'year': 2024, 'title': 'Vyhľadávanie informácií'}], 'source_urls': []}
|
|
grade = grade_claims(case, answer, oracle)
|
|
assert not grade['strict'] and grade['unsupported_claims']
|
|
|
|
|
|
def test_generated_claims_cannot_distribute_conditions_across_people():
|
|
from tools.graphrag.evaluation.check_openwebui import grade_claims
|
|
case = query_case('Kto má tag aj rok?')
|
|
case['graph']['expected_bindings'] = [{'person': 'Eva Nová', 'year': 2024, 'keyword': 'nlp'}]
|
|
oracle = {'people': {'Eva Nová': {'start_year': 2020, 'academics': [{'year': 2024, 'title': None}], 'documents': []},
|
|
'Boris Malý': {'start_year': 2020, 'academics': [], 'documents': ['b.md']}},
|
|
'documents': {'b.md': {'tags': ['nlp'], 'categories': [], 'author': None}}}
|
|
answer = {'status': 'ok', 'claims': [{'person': 'Eva Nová', 'year': 2024},
|
|
{'person': 'Boris Malý', 'keyword': 'nlp'}], 'source_urls': []}
|
|
assert not grade_claims(case, answer, oracle)['bindings_ok']
|