217 lines
7.7 KiB
Python
217 lines
7.7 KiB
Python
"""Prevent query leakage and regression of raw-source title/year semantics."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
|
|
import pytest
|
|
|
|
from tools.graphrag import graph_search as gs
|
|
from tools.graphrag.graphrag import WorkHeading, extract_work_title, resolve_work_year
|
|
from tools.graphrag.evaluation.benchmark_v4 import logical_group
|
|
from tools.graphrag.evaluation.freeze_graphrag import make_freeze
|
|
from tools.graphrag.evaluation.source_oracle import extract_academics
|
|
from .test_evaluation_pipeline_v2 import freeze_inputs
|
|
from .test_graph_search import FakeSession, sample_catalog
|
|
|
|
|
|
def query_case(text, year=2024):
|
|
return {
|
|
"question": text,
|
|
"category": "person_title_year",
|
|
"graph": {
|
|
"schema_version": 3,
|
|
"source_profiles": ["profile.md"],
|
|
"start_entities": [{"type": "person", "value": "Eva Nová"}],
|
|
"expected_bindings": [
|
|
{"person": "Eva Nová", "year": year, "title": "Vyhľadávanie informácií"}
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
def test_paraphrases_and_evidence_path_variations_cannot_be_new_holdout():
|
|
first = query_case("Aký názov práce má Eva Nová v roku 2024?")
|
|
second = query_case("Uveď názov práce osoby Eva Nová pre rok 2024.")
|
|
second["graph"]["expected_bindings"][0]["document_path"] = "alternative.md"
|
|
assert logical_group(first) == logical_group(second)
|
|
assert logical_group(first) != logical_group(
|
|
query_case("Aký názov práce má Eva Nová v roku 2025?", 2025)
|
|
)
|
|
|
|
|
|
def group_freeze_inputs():
|
|
dev, _, fingerprints, audit = freeze_inputs()
|
|
questions = [
|
|
query_case("Aký názov práce má Eva Nová v roku 2024?"),
|
|
query_case("Aký názov práce má Eva Nová v roku 2025?", 2025),
|
|
]
|
|
for index, q in enumerate(questions):
|
|
q.update(id=["dev-a", "test-b"][index], split=["dev", "test"][index])
|
|
q["graph"]["partition_group"] = logical_group(q)
|
|
audit["partition_policy"] = {
|
|
"method": "disjoint logical query groups",
|
|
"previously_exposed_groups": [logical_group(questions[0])],
|
|
"limitations": ["Known profiles."],
|
|
}
|
|
return dev, questions, fingerprints, audit
|
|
|
|
|
|
def test_group_freeze_explicitly_reports_known_profiles():
|
|
manifest = make_freeze(*group_freeze_inputs())
|
|
assert manifest["partition"]["source_profile_overlap"] == 1
|
|
assert manifest["partition"]["logical_group_overlap"] == 0
|
|
assert manifest["partition"]["method"] == "disjoint logical query groups"
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["overlap", "exposed", "forged_group"])
|
|
def test_group_freeze_rejects_leakage(mutation):
|
|
dev, questions, fingerprints, audit = group_freeze_inputs()
|
|
if mutation == "overlap":
|
|
questions[1] = {**copy.deepcopy(questions[0]), "id": "test-b", "split": "test"}
|
|
elif mutation == "exposed":
|
|
audit["partition_policy"]["previously_exposed_groups"].append(logical_group(questions[1]))
|
|
else:
|
|
questions[1]["graph"]["partition_group"] = "invented"
|
|
with pytest.raises(RuntimeError):
|
|
make_freeze(dev, questions, fingerprints, audit)
|
|
|
|
|
|
def test_explicit_academic_range_is_not_overridden_by_unrelated_category():
|
|
assert (
|
|
resolve_work_year(
|
|
WorkHeading("research_project", "Vedecký projekt 2022/23", "vp"), ["vp2021"]
|
|
)
|
|
== 2023
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"prefix", ["Pozrieť si", "Projekt odložený.", "Konzultácie sa budú konať osobne."]
|
|
)
|
|
def test_instructions_are_not_academic_titles(prefix):
|
|
assert extract_work_title([prefix + "\n\n- Oboznámte sa s nástrojmi."]) is None
|
|
doc = {
|
|
"path": "p.md",
|
|
"body_offset": 0,
|
|
"categories": [],
|
|
"body_lines": ["# Bakalársky projekt 2024", prefix, "", "- Oboznámte sa s nástrojmi."],
|
|
}
|
|
assert extract_academics(doc)[0]["title"] is None
|
|
|
|
|
|
def test_last_leading_title_wins_over_introductory_description():
|
|
doc = {
|
|
"path": "p.md",
|
|
"body_offset": 0,
|
|
"categories": [],
|
|
"body_lines": [
|
|
"# Bakalárska práca 2025",
|
|
"Chceme vytvoriť asistenta pre vyhľadávanie.",
|
|
"",
|
|
"Generovanie jazyka s podporou vyhľadávania",
|
|
"",
|
|
"1. Vypracujte prehľad.",
|
|
],
|
|
}
|
|
assert extract_academics(doc)[0]["title"] == "Generovanie jazyka s podporou vyhľadávania"
|
|
|
|
|
|
def test_title_lookup_preserves_known_record_without_claiming_a_title():
|
|
fact = {
|
|
"person_element_id": "p2",
|
|
"person": "Jakub Maruniak",
|
|
"thesis_element_id": "th1",
|
|
"year": 2024,
|
|
"title": None,
|
|
"thesis_type": "diploma_thesis",
|
|
}
|
|
session = FakeSession({gs.CATALOG_QUERY: sample_catalog(), gs.THESIS_FACTS_QUERY: [fact]})
|
|
result = gs.search_graph_with_session(
|
|
session, "Aký názov práce má osoba Jakub Maruniak v roku 2024?"
|
|
)
|
|
# A missing title must not erase the independently known work/year. The
|
|
# delivery regression checks the explicit "title missing" presentation.
|
|
assert result["status"] == "ok"
|
|
assert len(result["facts"]) == 1
|
|
assert result["facts"][0]["title"] is None
|
|
assert result["facts"][0]["year"] == 2024
|
|
|
|
|
|
def test_explicit_name_precedes_a_different_topic_and_link_title_is_preserved():
|
|
assert (
|
|
extract_work_title(["Téma: Všeobecná téma projektu\nNávrh na názov: Konkrétny názov práce"])
|
|
== "Konkrétny názov práce"
|
|
)
|
|
assert (
|
|
extract_work_title(["*Písomná práca*: [Rešerš na tému prekladu](./report)"])
|
|
== "Rešerš na tému prekladu"
|
|
)
|
|
|
|
|
|
def test_raw_oracle_skips_repository_and_related_work_lines():
|
|
doc = {
|
|
"path": "p.md",
|
|
"body_offset": 0,
|
|
"categories": [],
|
|
"body_lines": [
|
|
"# Bakalárska práca 2025",
|
|
"Systém pre monitoring médií",
|
|
"",
|
|
"Práca súvisí s DP iného študenta",
|
|
"Repo https://example.org/repo",
|
|
"",
|
|
"Zadanie:",
|
|
"- Vypracujte prehľad.",
|
|
],
|
|
}
|
|
assert extract_academics(doc)[0]["title"] == "Systém pre monitoring médií"
|
|
|
|
|
|
def test_generated_claim_year_is_not_satisfied_by_start_year():
|
|
from tools.graphrag.evaluation.check_model_sample import grade_claims
|
|
|
|
oracle = {
|
|
"people": {
|
|
"Eva Nová": {
|
|
"start_year": 2024,
|
|
"academics": [{"year": 2025, "title": "Vyhľadávanie informácií"}],
|
|
}
|
|
}
|
|
}
|
|
case = query_case("Aká práca v roku 2024?")
|
|
answer = {
|
|
"status": "ok",
|
|
"claims": [{"person": "Eva Nová", "year": 2024, "title": "Vyhľadávanie informácií"}],
|
|
"source_urls": [],
|
|
}
|
|
grade = grade_claims(case, answer, oracle)
|
|
assert not grade["strict"] and grade["unsupported_claims"]
|
|
|
|
|
|
def test_generated_claims_cannot_distribute_conditions_across_people():
|
|
from tools.graphrag.evaluation.check_model_sample import grade_claims
|
|
|
|
case = query_case("Kto má tag aj rok?")
|
|
case["graph"]["expected_bindings"] = [{"person": "Eva Nová", "year": 2024, "keyword": "nlp"}]
|
|
oracle = {
|
|
"people": {
|
|
"Eva Nová": {
|
|
"start_year": 2020,
|
|
"academics": [{"year": 2024, "title": None}],
|
|
"documents": [],
|
|
},
|
|
"Boris Malý": {"start_year": 2020, "academics": [], "documents": ["b.md"]},
|
|
},
|
|
"documents": {"b.md": {"tags": ["nlp"], "categories": [], "author": None}},
|
|
}
|
|
answer = {
|
|
"status": "ok",
|
|
"claims": [
|
|
{"person": "Eva Nová", "year": 2024},
|
|
{"person": "Boris Malý", "keyword": "nlp"},
|
|
],
|
|
"source_urls": [],
|
|
}
|
|
assert not grade_claims(case, answer, oracle)["bindings_ok"]
|