"""Version known regression questions after independent source adjudication. No retrieval output is used to choose, remove or relabel questions. Historical v4 remains immutable; all IDs, wording, categories and split membership remain. """ import copy import json from tools.graphrag.evaluation.benchmark_v2 import ( BASE, audit_dataset, binding_source_evidence, validate_case, write_json, ) from tools.graphrag.evaluation.benchmark_v4 import refresh_titles, logical_group from tools.graphrag.evaluation.source_oracle import load_source_oracle def main(): output = BASE / "graphrag_questions_v6.json" artifacts = BASE / "results/v6_preparation" if output.exists(): raise RuntimeError("Refusing to overwrite versioned questions.") original = json.loads((BASE / "graphrag_questions_v4.json").read_text()) oracle = load_source_oracle() cases = [] changes = [] for before in original: after = refresh_titles(before, oracle) after["graph"]["partition_group"] = logical_group(after) if after["should_answer"]: after["source_evidence"] = [ proof for binding in after["graph"]["expected_bindings"] for proof in binding_source_evidence(binding, oracle) ] if (before["should_answer"], before["graph"]["expected_bindings"]) != ( after["should_answer"], after["graph"]["expected_bindings"], ): changes.append( { "id": before["id"], "split": before["split"], "before": copy.deepcopy(before), "after": copy.deepcopy(after), "reason": "Generic written-deliverable link label is not a supported academic title.", } ) assert before["question"] == after["question"] and before["category"] == after["category"] issues = validate_case(after, oracle) if issues: raise RuntimeError(f'{after["id"]}: {issues}') cases.append(after) write_json(output, cases) write_json( artifacts / "source_adjudications.json", { "reviewer": "AI direct Markdown review; not a human adjudication", "policy": "A bare artifact label (rešerš, tutoriál, projekt, dokument, repozitár) is not an academic title. " "Do not inherit a child-document title or category-year into its parent academic record.", "measurement": "All questions are known historical regression; corrected gold prevents direct accuracy comparison.", "questions": changes, }, ) write_json(artifacts / "source_audit.json", audit_dataset(cases, oracle, output)) print( json.dumps( { "questions": len(cases), "source_issues": 0, "adjudicated_ids": [c["id"] for c in changes], } ) ) if __name__ == "__main__": main()