82 lines
3.0 KiB
Python
82 lines
3.0 KiB
Python
"""Version known regression questions after independent source adjudication.
|
|
|
|
No retrieval output is used to choose, remove or relabel questions. Historical
|
|
v4 remains immutable; all IDs, wording, categories and split membership remain.
|
|
"""
|
|
|
|
import copy
|
|
import json
|
|
|
|
from tools.graphrag.evaluation.benchmark_v2 import (
|
|
BASE,
|
|
audit_dataset,
|
|
binding_source_evidence,
|
|
validate_case,
|
|
write_json,
|
|
)
|
|
from tools.graphrag.evaluation.benchmark_v4 import refresh_titles, logical_group
|
|
from tools.graphrag.evaluation.source_oracle import load_source_oracle
|
|
|
|
|
|
def main():
|
|
output = BASE / "graphrag_questions_v6.json"
|
|
artifacts = BASE / "results/v6_preparation"
|
|
if output.exists():
|
|
raise RuntimeError("Refusing to overwrite versioned questions.")
|
|
original = json.loads((BASE / "graphrag_questions_v4.json").read_text())
|
|
oracle = load_source_oracle()
|
|
cases = []
|
|
changes = []
|
|
for before in original:
|
|
after = refresh_titles(before, oracle)
|
|
after["graph"]["partition_group"] = logical_group(after)
|
|
if after["should_answer"]:
|
|
after["source_evidence"] = [
|
|
proof
|
|
for binding in after["graph"]["expected_bindings"]
|
|
for proof in binding_source_evidence(binding, oracle)
|
|
]
|
|
if (before["should_answer"], before["graph"]["expected_bindings"]) != (
|
|
after["should_answer"],
|
|
after["graph"]["expected_bindings"],
|
|
):
|
|
changes.append(
|
|
{
|
|
"id": before["id"],
|
|
"split": before["split"],
|
|
"before": copy.deepcopy(before),
|
|
"after": copy.deepcopy(after),
|
|
"reason": "Generic written-deliverable link label is not a supported academic title.",
|
|
}
|
|
)
|
|
assert before["question"] == after["question"] and before["category"] == after["category"]
|
|
issues = validate_case(after, oracle)
|
|
if issues:
|
|
raise RuntimeError(f'{after["id"]}: {issues}')
|
|
cases.append(after)
|
|
write_json(output, cases)
|
|
write_json(
|
|
artifacts / "source_adjudications.json",
|
|
{
|
|
"reviewer": "AI direct Markdown review; not a human adjudication",
|
|
"policy": "A bare artifact label (rešerš, tutoriál, projekt, dokument, repozitár) is not an academic title. "
|
|
"Do not inherit a child-document title or category-year into its parent academic record.",
|
|
"measurement": "All questions are known historical regression; corrected gold prevents direct accuracy comparison.",
|
|
"questions": changes,
|
|
},
|
|
)
|
|
write_json(artifacts / "source_audit.json", audit_dataset(cases, oracle, output))
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"questions": len(cases),
|
|
"source_issues": 0,
|
|
"adjudicated_ids": [c["id"] for c in changes],
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|