"""Build a source-grounded schema-v3 regression set and profile-disjoint holdout. This module never imports graph construction/search code and never reads any retrieval output. Legacy cases remain development-only. New TEST subjects are student profiles absent from every legacy start entity. TEST is generated once from raw source facts, before the corrected implementation is evaluated. """ from __future__ import annotations import argparse import hashlib import itertools import json import random import re from collections import Counter from pathlib import Path from typing import Any from .source_oracle import ROOT, DEFAULT_WIKI, load_source_oracle, norm, supporting_documents BASE = ROOT / "experiments/graphrag" SEED = 20261024 def fingerprint(value: Any) -> str: return hashlib.sha256( json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() ).hexdigest() def people_for(question: dict, oracle: dict) -> list[str]: names = {norm(name): name for name in oracle["people"]} return [ names[norm(item["value"])] for item in question["graph"].get("start_entities", []) if item["type"] == "person" and norm(item["value"]) in names ] def academics( oracle: dict, person: str, *, title: str | None = None, year: int | None = None, kind: str | None = None, ) -> list[dict]: rows = oracle["people"][person]["academics"] return [ row for row in rows if (title is None or norm(row["title"]) == norm(title)) and (year is None or row["year"] == year) and (kind is None or row["academic_kind"] == kind) ] def has_tag(oracle: dict, person: str, keyword: str) -> bool: return norm(keyword) in {norm(tag) for tag in oracle["people"][person]["tags"]} def academic_binding( person: str, record: dict, *, title: bool = False, keyword: str | None = None ) -> dict: binding = { "person": person, "academic_kind": record["academic_kind"], "academic_type": record["academic_type"], "year": record["year"], } if title: binding["title"] = record["title"] if keyword is not None: binding["keyword"] = keyword return {key: value for key, value in binding.items() if value is not None} def path_patterns(bindings: list[dict]) -> list[dict]: patterns = [] for index, binding in enumerate(bindings): if any(key in binding for key in ["year", "title", "academic_kind", "academic_type"]): kind = binding.get("academic_kind", "Academic") relation = ( "AUTHORED" if kind == "Thesis" else "HAS_WORK" if kind == "Work" else "AUTHORED|HAS_WORK" ) patterns.append( { "binding_index": index, "node_kinds": ["Person", kind, "Document"], "relationships": [relation, "EVIDENCED_BY"], } ) for field, kind, relation in [ ("keyword", "Keyword", "HAS_KEYWORD"), ("category", "Category", "IN_CATEGORY"), ("document_author", "DocumentAuthor", "AUTHORED_DOCUMENT"), ]: if field in binding: patterns.append( { "binding_index": index, "node_kinds": ["Person", "Document", kind], "relationships": ["HAS_DOCUMENT", relation], } ) return patterns def binding_source_evidence(binding: dict, oracle: dict) -> list[dict]: person = oracle["people"][binding["person"]] evidence = [] for field in ["keyword", "category", "document_author"]: if field not in binding: continue docs = supporting_documents(oracle, binding["person"], **{field: binding[field]}) if binding.get("document_path"): docs = [p for p in docs if p == binding["document_path"]] for path in docs: evidence.append( { "document_path": path, "sha256": oracle["documents"][path]["sha256"], "field": { "keyword": "taxonomy.tag", "category": "taxonomy.category", "document_author": "taxonomy.author", }[field], "value": binding[field], } ) if any(key in binding for key in ["year", "title", "academic_kind", "academic_type"]): rows = academics( oracle, binding["person"], title=binding.get("title"), year=binding.get("year"), kind=binding.get("academic_kind"), ) for row in rows: if binding.get("academic_type") and row["academic_type"] != binding["academic_type"]: continue evidence.append( { "document_path": row["document_path"], "sha256": oracle["documents"][row["document_path"]]["sha256"], "field": "academic_record", "academic_kind": row["academic_kind"], "academic_type": row["academic_type"], "title": row["title"], "year": row["year"], "heading_line": row["heading_line"], "title_line": row["title_line"], "year_basis": row["year_basis"], } ) if "start_year" in binding: evidence.append( { "document_path": person["profile"], "sha256": oracle["documents"][person["profile"]]["sha256"], "field": "start_year", "value": binding["start_year"], "basis": person.get("start_year_basis", "explicit_text"), "confidence": "documented", } ) return evidence def make_case( case_id: str, split: str, category: str, question: str, names: list[str], oracle: dict, bindings: list[dict], *, no_answer: bool = False, reason: str | None = None, expected_set: list[str] | None = None, set_field: str | None = None, count: int | None = None, context_documents: list[str] | None = None, negative_constraints: dict | None = None, ) -> dict: known = [name for name in names if name in oracle["people"]] profiles = sorted({oracle["people"][name]["profile"] for name in known}) paths = set(context_documents or profiles) evidence = [item for binding in bindings for item in binding_source_evidence(binding, oracle)] # Multiple documents can independently prove the same tag/category. Gold # sources are alternatives for each bound proof, not an obligation to cite # every duplicate occurrence. Context profiles still cover all candidates. evidence_choices = [] roles = { "taxonomy.tag": "keyword_document", "taxonomy.category": "category_document", "taxonomy.author": "author_document", "academic_record": "academic_document", "start_year": "person_document", } if not no_answer: for index, binding in enumerate(bindings): per_role: dict[str, set[str]] = {} for item in binding_source_evidence(binding, oracle): per_role.setdefault(roles[item["field"]], set()).add(item["document_path"]) for role, alternatives in per_role.items(): evidence_choices.append( { "binding_index": index, "role": role, "any_of": [ { "document_path": path, "source_url": oracle["documents"][path]["source_url"], } for path in sorted(alternatives) ], } ) expected_values = [] expected_entities = [] if not no_answer: for binding in bindings: for field in [ "person", "keyword", "category", "document_author", "title", "year", "start_year", ]: if field in binding: value = str(binding[field]) expected_values.append(value) kind = ( ( "thesis_title" if binding.get("academic_kind") == "Thesis" else "work_title" ) if field == "title" else field ) expected_entities.append({"type": kind, "value": value}) if count is not None: expected_values.append(str(count)) graph = { "schema_version": 3, "task": category, "start_entities": [{"type": "person", "value": name} for name in names], "expected_bindings": bindings if not no_answer else [], "expected_evidence_documents": evidence_choices, "expected_path_patterns": path_patterns(bindings) if not no_answer else [], "expected_no_answer": no_answer, "expected_status": "no_answer" if no_answer else "ok", "expected_entities": expected_entities, "expected_relations": [], "expected_properties": [], "source_profiles": profiles, } if no_answer: graph["no_answer_basis"] = reason or "No source record satisfies every requested condition." graph["negative_constraints"] = negative_constraints or {} # Closed-world negative: retain all original metadata considered, not a fabricated positive fact. evidence = [ { "document_path": path, "sha256": oracle["documents"][path]["sha256"], "field": "closed_world_metadata", "tags": oracle["documents"][path]["tags"], "categories": oracle["documents"][path]["categories"], } for name in known for path in oracle["people"][name]["documents"] ] if expected_set is not None: graph.update( expected_entity_set=sorted(expected_set, key=norm), entity_set_mode="exact", entity_set_field=set_field or "person", ) if count is not None: graph["expected_count"] = count graph["partition_group"] = fingerprint( { "category": category, "people": sorted(names, key=norm), "bindings": bindings, "negative": no_answer, "requested": question if no_answer else None, } )[:20] return { "id": case_id, "split": split, "category": category, "difficulty": "hard", "question": question, "should_answer": not no_answer, "expected_documents": sorted(paths), "expected_source_urls": sorted(oracle["documents"][path]["source_url"] for path in paths), "expected_answer_contains": list(dict.fromkeys(expected_values)), "graph": graph, "source_evidence": evidence, "evidence_basis": ["independent_raw_markdown_oracle_v2"], } def correct_legacy(question: dict, oracle: dict) -> dict: names = people_for(question, oracle) cat = question["category"] old = question["graph"] entities = old.get("expected_entities", []) keyword = next((item["value"] for item in entities if item["type"] == "keyword"), None) years = [int(item["value"]) for item in entities if item["type"] == "year"] bindings = [] expected_set = None set_field = None count = None reason = None if cat == "shared_keyword_pair": if keyword and all(has_tag(oracle, name, keyword) for name in names): bindings = [{"person": name, "keyword": keyword} for name in names] else: reason = "Requested shared keyword is absent from the raw frontmatter of at least one named person." elif cat == "same_start_year_pair": bindings = [ {"person": name, "start_year": oracle["people"][name]["start_year"]} for name in names ] elif cat == "same_academic_year_pair": common = set.intersection( *[ {row["year"] for row in academics(oracle, name) if row["year"] is not None} for name in names ] ) bindings = [{"person": name, "year": year} for year in sorted(common) for name in names] if not common: reason = "No shared academic year is stated in the original sources." elif cat in {"person_title_year", "person_keyword_title"}: titles = [item for item in entities if item["type"] in {"thesis_title", "work_title"}] for item in titles: kind = "Thesis" if item["type"] == "thesis_title" else "Work" rows = academics(oracle, names[0], title=item["value"], kind=kind) if not rows and norm(item["value"]) in norm( oracle["documents"][oracle["people"][names[0]]["profile"]]["raw_text"] ): raise RuntimeError( f"Oracle extraction needs review for supported raw title: {question['id']}" ) if years: rows = [row for row in rows if row["year"] in years] if cat == "person_keyword_title" and not has_tag(oracle, names[0], keyword or ""): reason = "Requested keyword is not a raw frontmatter tag of this person." break for row in rows: binding = academic_binding( names[0], row, title=True, keyword=keyword if cat == "person_keyword_title" else None, ) if cat == "person_keyword_title" and not years: binding.pop("year", None) bindings.append(binding) if not bindings and not reason: reason = "Requested title/type/year combination is not independently supported by the source." elif cat == "document_author_category_multi_hop": category = next(item["value"] for item in entities if item["type"] == "category") for path in supporting_documents(oracle, names[0], category=category): author = oracle["documents"][path]["author"] if author: bindings.append( { "person": names[0], "category": category, "document_author": author, "document_path": path, "same_document": True, } ) if not bindings: reason = "No same document supports both the category and metadata author." elif cat == "person_multiple_categories": expected_set = oracle["people"][names[0]]["categories"] set_field = "category" bindings = [{"person": names[0], "category": category} for category in expected_set] elif cat == "candidate_set_aggregation": query_years = [int(x) for x in re.findall(r"\b20\d{2}\b", question["question"])] year = query_years[-1] matches = [name for name in names if academics(oracle, name, year=year)] bindings = [{"person": name, "year": year} for name in matches] expected_set = matches set_field = "person" count = len(matches) elif cat == "conjunctive_disambiguation": year = years[0] matches = [ name for name in names if has_tag(oracle, name, keyword) and academics(oracle, name, year=year) ] bindings = [{"person": name, "keyword": keyword, "year": year} for name in matches] if not matches: reason = "No named person satisfies both raw frontmatter keyword and academic year." else: raise ValueError(f"Unsupported legacy category: {cat}") case = make_case( question["id"], "dev", cat, question["question"], names, oracle, bindings, no_answer=reason is not None, reason=reason, expected_set=expected_set, set_field=set_field, count=count, negative_constraints=( {"keyword": keyword, "require_all_people": True} if cat == "shared_keyword_pair" else ( {"keyword": keyword} if cat == "person_keyword_title" else ( {"keyword": keyword, "year": years[0]} if cat == "conjunctive_disambiguation" else {"shared_academic_year": True} if cat == "same_academic_year_pair" else {} ) ) ), ) case["legacy_split"] = question["split"] case["legacy_role"] = "previously_seen_regression_only" return case def independent_pool(oracle: dict, names: list[str]) -> dict[str, list[dict]]: pool: dict[str, list[dict]] = { category: [] for category in [ "shared_keyword_pair", "same_academic_year_pair", "same_start_year_pair", "person_title_year", "person_keyword_title", "document_author_category_multi_hop", "person_multiple_categories", "candidate_set_aggregation", "conjunctive_disambiguation", ] } def add(category: str, text: str, people: list[str], bindings: list[dict], **kwargs): pool[category].append( make_case("", "test", category, text, people, oracle, bindings, **kwargs) ) for left, right in itertools.combinations(names, 2): common_tags = sorted( { norm(tag): tag for tag in oracle["people"][left]["tags"] if has_tag(oracle, right, tag) }.values(), key=norm, ) for tag in common_tags: add( "shared_keyword_pair", f"Over spoločné kľúčové slovo {tag} pri osobách {left} a {right}.", [left, right], [{"person": name, "keyword": tag} for name in [left, right]], ) shared_years = {row["year"] for row in academics(oracle, left)} & { row["year"] for row in academics(oracle, right) } - {None} # A query for an unspecified shared year expects all supported intersections. if shared_years: add( "same_academic_year_pair", f"Aké spoločné roky práce spájajú osoby {left} a {right}?", [left, right], [ {"person": name, "year": year} for year in sorted(shared_years) for name in [left, right] ], ) start_l, start_r = ( oracle["people"][left]["start_year"], oracle["people"][right]["start_year"], ) if start_l is not None and start_l == start_r: add( "same_start_year_pair", f"Porovnaj rovnaký rok začiatku štúdia osôb {left} a {right}.", [left, right], [{"person": name, "start_year": start_l} for name in [left, right]], ) for name in names: person = oracle["people"][name] if person["categories"]: add( "person_multiple_categories", f"Vypíš všetky kategórie dokumentov osoby {name}.", [name], [{"person": name, "category": cat} for cat in person["categories"]], expected_set=person["categories"], set_field="category", ) for category in person["categories"]: for path in supporting_documents(oracle, name, category=category): author = oracle["documents"][path]["author"] if author: add( "document_author_category_multi_hop", f"Kto je autor dokumentu osoby {name}, ktorý patrí do kategórie {category}?", [name], [ { "person": name, "category": category, "document_author": author, "document_path": path, "same_document": True, } ], ) # Title-year cases use only explicit, nonconflicting title records. for record in person["academics"]: if not record["title"] or record["year"] is None or record in person["year_conflicts"]: continue year = record["year"] kind_label = ( "diplomového projektu" if record["academic_type"] == "diploma_project" else "práce" ) add( "person_title_year", f"Aký názov {kind_label} a rok práce sú pri osobe {name} uvedené pre rok {year}?", [name], [academic_binding(name, record, title=True)], ) for tag in person["tags"]: binding = academic_binding(name, record, title=True, keyword=tag) # Pin the academic year in the question as the independent source can contain several titles. add( "person_keyword_title", f"Aký názov práce osoby {name} v roku {year} súvisí s kľúčovým slovom {tag}?", [name], [binding], ) # Candidate groups are source-derived; the query mentions every candidate. for year in sorted( { row["year"] for name in names for row in academics(oracle, name) if row["year"] is not None } ): matching = [name for name in names if academics(oracle, name, year=year)] other = [name for name in names if name not in matching] for selected in itertools.islice(itertools.combinations(matching, 2), 12): if other: candidates = list(selected) + [other[0]] add( "candidate_set_aggregation", f'Z kandidátov {", ".join(candidates)} vyber osoby s prácou v roku {year}. Koľko ich je?', candidates, [{"person": name, "year": year} for name in selected], expected_set=list(selected), set_field="person", count=len(selected), ) for name in names: person = oracle["people"][name] for year in sorted( {record["year"] for record in person["academics"] if record["year"] is not None} ): for tag in person["tags"]: distractors = [ other for other in names if other != name and (has_tag(oracle, other, tag) != bool(academics(oracle, other, year=year))) ] for other in distractors[:2]: candidates = [name, other] add( "conjunctive_disambiguation", f"Ktorá z osôb {name} a {other} má kľúčové slovo {tag} a jej práca je z roku {year}?", candidates, [{"person": name, "keyword": tag, "year": year}], ) return pool def necessary_negatives(oracle: dict, names: list[str], split: str) -> list[dict]: cases = [] prefix = "dev" if split == "dev" else "holdout" for i, name in enumerate(names[:4]): tag = next(iter(oracle["people"][name]["tags"]), "nlp") cases.append( make_case( f"{prefix}-negative-year-{i}", split, "negative_lookup", f"Aký názov práce má osoba {name}, ak jej práca je z roku 2099 a jej dokument obsahuje kľúčové slovo {tag}?", [name], oracle, [], no_answer=True, reason="No source academic record has year 2099.", negative_constraints={"year": 2099, "keyword": tag}, ) ) for i, name in enumerate(names[4:6]): cases.append( make_case( f"{prefix}-negative-tag-{i}", split, "negative_lookup", f"Aký názov práce má osoba {name}, ak jej dokument obsahuje kľúčové slovo neexistujucitagvzor?", [name], oracle, [], no_answer=True, reason="Requested tag absent from complete raw frontmatter catalog.", negative_constraints={"keyword": "neexistujucitagvzor"}, ) ) for i, unknown in enumerate( ["Zuzana Neexistujúca Vzorová", "Roman Neskutočný Testovací"] if split == "dev" else ["Tibor Neexistujúci Skúšobný", "Lenka Neskutočná Vzorová"] ): cases.append( make_case( f"{prefix}-negative-person-{i}", split, "negative_lookup", f"Aký názov diplomovej práce má osoba {unknown}?", [unknown], oracle, [], no_answer=True, reason="Explicit full name absent from all raw profile frontmatter titles.", negative_constraints={"unknown_person": unknown}, ) ) candidates = names[:3] cases.append( make_case( f"{prefix}-zero-count", split, "candidate_set_aggregation", f'Z kandidátov {", ".join(candidates)} vyber osoby s prácou v roku 2099. Koľko ich je?', candidates, oracle, [], expected_set=[], set_field="person", count=0, ) ) return cases def validate_case(case: dict, oracle: dict) -> list[str]: issues = [] graph = case["graph"] names = people_for(case, oracle) if set(graph.get("source_profiles", [])) != { oracle["people"][name]["profile"] for name in names }: issues.append("incorrect_source_profile_partition") for binding in graph.get("expected_bindings", []): person = binding["person"] if person not in oracle["people"]: issues.append("unknown_bound_person") continue if ( "start_year" in binding and oracle["people"][person]["start_year"] != binding["start_year"] ): issues.append("wrong_start_year") metadata = { key: binding[key] for key in ["keyword", "category", "document_author"] if key in binding } if binding.get("same_document") and not supporting_documents(oracle, person, **metadata): issues.append("metadata_not_connected_in_same_document") for key, value in metadata.items(): if not supporting_documents(oracle, person, **{key: value}): issues.append("unsupported_" + key) if any(key in binding for key in ["title", "year", "academic_type", "academic_kind"]): records = academics( oracle, person, title=binding.get("title"), year=binding.get("year"), kind=binding.get("academic_kind"), ) if binding.get("academic_type"): records = [r for r in records if r["academic_type"] == binding["academic_type"]] if not records: issues.append("unsupported_academic_binding") if graph.get("expected_no_answer"): if ( graph.get("expected_bindings") or case.get("expected_answer_contains") or graph.get("expected_entities") ): issues.append("negative_contains_positive_expectations") constraints = graph.get("negative_constraints", {}) names = people_for(case, oracle) if constraints.get("unknown_person"): if norm(constraints["unknown_person"]) in {norm(name) for name in oracle["people"]}: issues.append("negative_person_actually_exists") elif constraints.get("missing_start_year"): if names and all(oracle["people"][name]["start_year"] is not None for name in names): issues.append("negative_start_year_actually_documented") elif constraints.get("shared_academic_year"): year_sets = [ {r["year"] for r in academics(oracle, name) if r["year"] is not None} for name in names ] if year_sets and set.intersection(*year_sets): issues.append("negative_shared_year_actually_exists") elif constraints.get("require_all_people"): if not names or all( has_tag(oracle, name, constraints.get("keyword", "")) for name in names ): issues.append("negative_shared_keyword_actually_exists") elif constraints: matching = [] for name in names: valid = not constraints.get("keyword") or has_tag( oracle, name, constraints["keyword"] ) if any( key in constraints for key in ["year", "title", "academic_kind", "title_required"] ): records = academics( oracle, name, year=constraints.get("year"), title=constraints.get("title"), kind=constraints.get("academic_kind"), ) if constraints.get("academic_type"): records = [ r for r in records if r["academic_type"] == constraints["academic_type"] ] if constraints.get("title_required"): records = [r for r in records if r.get("title")] valid = valid and bool(records) if valid: matching.append(name) if matching: issues.append("negative_conjunction_actually_satisfied") else: issues.append("negative_without_machine_verifiable_false_premise") if graph.get("expected_entity_set") is not None and graph.get("entity_set_field") == "category": names = people_for(case, oracle) if names and set(graph["expected_entity_set"]) != set( oracle["people"][names[0]]["categories"] ): issues.append("incomplete_category_set") if graph.get("expected_count") is not None: years = re.findall(r"\b\d{4}\b", case["question"]) if not years: issues.append("count_without_supported_year_constraint") else: matches = {name for name in names if academics(oracle, name, year=int(years[-1]))} if ( matches != set(graph.get("expected_entity_set", [])) or len(matches) != graph["expected_count"] ): issues.append("incorrect_candidate_set_or_count") return sorted(set(issues)) def audit_dataset(questions: list[dict], oracle: dict, dataset_path: Path) -> dict: rows = [ { "id": question["id"], "split": question["split"], "source_consistent": not (issues := validate_case(question, oracle)), "issues": issues, "category": question["category"], } for question in questions ] inputs = { name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in ["data/documents.json", "data/chunks.json"] } return { "method": "Independent original Markdown frontmatter/heading/title oracle; no graph parser, chunks or retrieval results used for labels.", "dataset_sha256": hashlib.sha256(dataset_path.read_bytes()).hexdigest(), "inputs": inputs, "source_corpus": oracle["source_corpus"], "questions": rows, "by_split": { split: { "questions": len(selected), "source_consistent": sum(row["source_consistent"] for row in selected), "issues": dict(Counter(issue for row in selected for issue in row["issues"])), } for split in ["dev", "test"] for selected in [[row for row in rows if row["split"] == split]] }, } def write_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n") def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--wiki-root", type=Path, default=DEFAULT_WIKI) parser.add_argument("--legacy", type=Path, default=BASE / "graphrag_questions.json") parser.add_argument("--output", type=Path, default=BASE / "graphrag_questions_v2.json") parser.add_argument("--artifacts", type=Path, default=BASE / "results/v2_preparation") args = parser.parse_args() oracle = load_source_oracle(args.wiki_root) legacy = json.loads(args.legacy.read_text()) corrected = [correct_legacy(question, oracle) for question in legacy] legacy_names = {name for question in legacy for name in people_for(question, oracle)} exposed_path = BASE / "results/final_2026-10-24/holdout_failures.json" previously_exposed = set() if exposed_path.exists(): for row in json.loads(exposed_path.read_text()).get("questions", []): previously_exposed.update( set(row.get("actual_documents", [])) - set(row.get("expected_documents", [])) ) reserved_names = sorted( [ name for name, person in oracle["people"].items() if name not in legacy_names and person["profile"] not in previously_exposed and person["profile"].startswith("pages/students/") ], key=norm, ) dev_negatives = necessary_negatives(oracle, sorted(legacy_names, key=norm), "dev") test_pool = independent_pool(oracle, reserved_names) rng = random.Random(SEED) holdout = [] for category, cases in test_pool.items(): unique = {case["graph"]["partition_group"]: case for case in cases} candidates = list(unique.values()) rng.shuffle(candidates) if len(candidates) < 8: raise RuntimeError( f"Insufficient independent source cases for {category}: {len(candidates)}" ) for case in candidates[:8]: case["id"] = f"h{len(holdout)+1:04d}" holdout.append(case) negative_cases = necessary_negatives(oracle, reserved_names, "test") # A zero-result count is an answerable numeric result, not a no-answer. zero_count = negative_cases.pop() count_index = next( i for i, case in enumerate(holdout) if case["category"] == "candidate_set_aggregation" ) zero_count["id"] = holdout[count_index]["id"] holdout[count_index] = zero_count holdout.extend(negative_cases) dev = corrected + dev_negatives questions = dev + holdout all_issues = { case["id"]: validate_case(case, oracle) for case in questions if validate_case(case, oracle) } if all_issues: raise RuntimeError(f"Source consistency issues: {all_issues}") dev_profiles = {p for case in dev for p in case["graph"]["source_profiles"]} test_profiles = {p for case in holdout for p in case["graph"]["source_profiles"]} assert not dev_profiles & test_profiles assert not ( {case["graph"]["partition_group"] for case in dev} & {case["graph"]["partition_group"] for case in holdout} ) write_json(args.output, questions) corrections = [] old_audit_path = BASE / "results/final_2026-10-24/source_audit.json" old_issues = { row["id"]: row["issues"] for row in json.loads(old_audit_path.read_text())["questions"] if row["issues"] } for old, new in zip(legacy, corrected): corrections.append( { "id": old["id"], "legacy_split": old["split"], "new_split": "dev", "v1_source_audit_issues": old_issues.get(old["id"], []), "old_should_answer": old.get("should_answer", True), "new_should_answer": new["should_answer"], "old_expected_values": old["expected_answer_contains"], "new_expected_values": new["expected_answer_contains"], "new_expected_bindings": new["graph"]["expected_bindings"], "no_answer_basis": new["graph"].get("no_answer_basis"), "source_evidence": new["source_evidence"], } ) write_json( args.artifacts / "legacy_corrections.json", { "legacy_count": len(legacy), "v1_flagged_count": len(old_issues), "resolved_v1_flagged_count": sum( not validate_case(new, oracle) for new in corrected if new["id"] in old_issues ), "negative_count": sum(not case["should_answer"] for case in corrected), "questions": corrections, }, ) write_json(args.artifacts / "source_audit.json", audit_dataset(questions, oracle, args.output)) partition = { "seed": SEED, "strategy": "All previously seen 500 cases are DEV regression. TEST profiles never appeared in legacy start_entities; all real candidate profiles are disjoint.", "dev_count": len(dev), "test_count": len(holdout), "previously_exposed_profiles_excluded_from_test": sorted(previously_exposed), "dev_profiles": sorted(dev_profiles), "test_profiles": sorted(test_profiles), "dev_groups": sorted({case["graph"]["partition_group"] for case in dev}), "test_groups": sorted({case["graph"]["partition_group"] for case in holdout}), "dev_categories": dict(Counter(case["category"] for case in dev)), "test_categories": dict(Counter(case["category"] for case in holdout)), "source_corpus": oracle["source_corpus"], "dataset_sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(), "limitations": [ "Same source corpus and shared tag/category/author vocabulary.", "Template families are known; TEST persons and their fact groups are withheld.", "Oracle is independent of graph extraction but still rule-based; provenance permits human review.", ], } write_json(args.artifacts / "partition.json", partition) print( json.dumps( { "dataset": str(args.output), "dev": len(dev), "test": len(holdout), "reserved_profile_count": len(test_profiles), "source_issues": 0, "corrected_legacy_negatives": sum(not q["should_answer"] for q in corrected), "resolved_original_issues": len(old_issues), "test_categories": partition["test_categories"], }, ensure_ascii=False, indent=2, ) ) if __name__ == "__main__": main()