184 lines
6.7 KiB
Python
184 lines
6.7 KiB
Python
"""Declare a fresh query-group holdout using raw sources only, before retrieval.
|
|
|
|
All 1,042 previously exposed questions become DEV. No profile is new; this is
|
|
a new-combination holdout over a known corpus, not an unseen-person experiment.
|
|
"""
|
|
|
|
import copy
|
|
import json
|
|
import random
|
|
from collections import Counter
|
|
|
|
from .benchmark_v2 import (
|
|
BASE,
|
|
audit_dataset,
|
|
make_case,
|
|
validate_case,
|
|
write_json,
|
|
binding_source_evidence,
|
|
)
|
|
from .benchmark_v4 import build_pool, logical_group, refresh_titles
|
|
from .source_oracle import load_source_oracle, norm
|
|
|
|
QUOTAS = {
|
|
"shared_keyword_pair": 10,
|
|
"same_academic_year_pair": 10,
|
|
"same_start_year_pair": 10,
|
|
"person_title_year": 10,
|
|
"person_keyword_title": 10,
|
|
"document_author_category_multi_hop": 2,
|
|
"person_multiple_categories": 7,
|
|
"candidate_set_aggregation": 7,
|
|
"conjunctive_disambiguation": 6,
|
|
"negative_lookup": 8,
|
|
}
|
|
SEED = 20261026
|
|
|
|
|
|
def main():
|
|
output = BASE / "graphrag_questions_v7.json"
|
|
artifacts = BASE / "results/v7_preparation"
|
|
if output.exists():
|
|
raise RuntimeError("Refusing to overwrite declared questions.")
|
|
oracle = load_source_oracle()
|
|
old = json.loads((BASE / "graphrag_questions_v6.json").read_text())
|
|
dev = []
|
|
changes = []
|
|
exposed = {logical_group(q) for q in old}
|
|
for original in old:
|
|
case = copy.deepcopy(original)
|
|
case["split"] = "dev"
|
|
names = [r["value"] for r in case["graph"]["start_entities"] if r["type"] == "person"]
|
|
if any(
|
|
b.get("start_year") is not None and oracle["people"][b["person"]]["start_year"] is None
|
|
for b in case["graph"]["expected_bindings"]
|
|
):
|
|
case = make_case(
|
|
case["id"],
|
|
"dev",
|
|
case["category"],
|
|
case["question"],
|
|
names,
|
|
oracle,
|
|
[],
|
|
no_answer=True,
|
|
reason="The original Markdown does not explicitly document every requested study-start year.",
|
|
negative_constraints={"missing_start_year": True},
|
|
)
|
|
changes.append(
|
|
{
|
|
"id": case["id"],
|
|
"old_bindings": original["graph"]["expected_bindings"],
|
|
"new_status": "no_answer",
|
|
"reason": "Directory cohort is not a documented study-start year.",
|
|
}
|
|
)
|
|
elif case["should_answer"]:
|
|
case["source_evidence"] = [
|
|
e
|
|
for b in case["graph"]["expected_bindings"]
|
|
for e in binding_source_evidence(b, oracle)
|
|
]
|
|
case["graph"]["partition_group"] = logical_group(case)
|
|
dev.append(case)
|
|
# Fixed audit regressions; these are explicitly DEV, not extra easy TEST negatives.
|
|
for year in [1999, 1899, 2100, 9999]:
|
|
case = make_case(
|
|
f"v7-dev-year-{year}",
|
|
"dev",
|
|
"negative_lookup",
|
|
f"Aký je názov práce osoby Ján Holp v roku {year}?",
|
|
["Ján Holp"],
|
|
oracle,
|
|
[],
|
|
no_answer=True,
|
|
negative_constraints={"year": year},
|
|
)
|
|
case["graph"]["partition_group"] = logical_group(case)
|
|
dev.append(case)
|
|
exposed.update(logical_group(q) for q in dev)
|
|
texts = {norm(q["question"]) for q in dev}
|
|
rng = random.Random(SEED)
|
|
holdout = []
|
|
capacity = {}
|
|
for category, pool in build_pool(oracle).items():
|
|
unique = {}
|
|
for candidate in pool:
|
|
case = refresh_titles(candidate, oracle)
|
|
group = logical_group(case)
|
|
if group not in exposed and norm(case["question"]) not in texts:
|
|
unique[group] = case
|
|
capacity[category] = len(unique)
|
|
candidates = [unique[key] for key in sorted(unique)]
|
|
rng.shuffle(candidates)
|
|
quota = QUOTAS[category]
|
|
if len(candidates) < quota:
|
|
raise RuntimeError(f"Insufficient source capacity: {category}")
|
|
for case in candidates[:quota]:
|
|
case["id"] = f"fresh7-{len(holdout)+1:03d}"
|
|
case["split"] = "test"
|
|
case["graph"]["partition_group"] = logical_group(case)
|
|
holdout.append(case)
|
|
questions = dev + holdout
|
|
for case in questions:
|
|
issues = validate_case(case, oracle)
|
|
if issues:
|
|
raise RuntimeError(f'{case["id"]}: {issues}')
|
|
write_json(output, questions)
|
|
audit = audit_dataset(questions, oracle, output)
|
|
audit["partition_policy"] = {
|
|
"method": "disjoint logical query groups",
|
|
"shared_source_profiles": True,
|
|
"seed": SEED,
|
|
"previously_exposed_groups": sorted(exposed),
|
|
"limitations": [
|
|
"Known corpus, persons and template families; only the query/fact combinations are fresh.",
|
|
"Two remaining document-author/category groups and seven category-list groups constrain category coverage.",
|
|
"AI source annotation, not human annotation. No tuning on this TEST.",
|
|
],
|
|
}
|
|
write_json(artifacts / "source_audit.json", audit)
|
|
write_json(
|
|
artifacts / "source_corrections.json",
|
|
{
|
|
"questions": changes,
|
|
"policy": "Only explicit study-start text is a fact. Cohort remains separate metadata; missing means unknown.",
|
|
},
|
|
)
|
|
write_json(artifacts / "regression_ids.json", [q["id"] for q in old if q["split"] == "test"])
|
|
ids = set(json.loads((BASE / "results/v6_preparation/generation_dev_ids.json").read_text()))
|
|
ids.update(f"v7-dev-year-{year}" for year in [1999, 1899, 2100, 9999])
|
|
write_json(artifacts / "generation_dev_ids.json", sorted(ids))
|
|
write_json(
|
|
artifacts / "protocol.json",
|
|
{
|
|
"seed": SEED,
|
|
"dev_count": len(dev),
|
|
"fresh_test_count": len(holdout),
|
|
"test_categories": dict(Counter(q["category"] for q in holdout)),
|
|
"test_positives": sum(q["should_answer"] for q in holdout),
|
|
"test_negatives": sum(not q["should_answer"] for q in holdout),
|
|
"available_groups": capacity,
|
|
"old_gold_corrections": len(changes),
|
|
"regression_count": 320,
|
|
"llm_dev_count": len(ids),
|
|
"target": 0.92,
|
|
"policy": "Freeze after complete DEV; run fresh TEST once, retain every failure. Regression remains separate.",
|
|
"annotation": "Independent raw-source oracle plus AI source review; not human review.",
|
|
},
|
|
)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"dev": len(dev),
|
|
"test": len(holdout),
|
|
"source_corrections": len(changes),
|
|
"quotas": QUOTAS,
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|