53 lines
2.6 KiB
Python
53 lines
2.6 KiB
Python
"""Version known regression questions after independent source adjudication.
|
|
|
|
No retrieval output is used to choose, remove or relabel questions. Historical
|
|
v4 remains immutable; all IDs, wording, categories and split membership remain.
|
|
"""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from .benchmark_v2 import BASE, audit_dataset, binding_source_evidence, validate_case, write_json
|
|
from .benchmark_v4 import refresh_titles, logical_group
|
|
from .source_oracle import load_source_oracle
|
|
|
|
|
|
def main():
|
|
output = BASE / 'graphrag_questions_v6.json'
|
|
artifacts = BASE / 'results/v6_preparation'
|
|
if output.exists():
|
|
raise RuntimeError('Refusing to overwrite versioned questions.')
|
|
original = json.loads((BASE / 'graphrag_questions_v4.json').read_text())
|
|
oracle = load_source_oracle()
|
|
cases = []; changes = []
|
|
for before in original:
|
|
after = refresh_titles(before, oracle)
|
|
after['graph']['partition_group'] = logical_group(after)
|
|
if after['should_answer']:
|
|
after['source_evidence'] = [proof for binding in after['graph']['expected_bindings']
|
|
for proof in binding_source_evidence(binding, oracle)]
|
|
if (before['should_answer'], before['graph']['expected_bindings']) != (
|
|
after['should_answer'], after['graph']['expected_bindings']):
|
|
changes.append({'id': before['id'], 'split': before['split'],
|
|
'before': copy.deepcopy(before), 'after': copy.deepcopy(after),
|
|
'reason': 'Generic written-deliverable link label is not a supported academic title.'})
|
|
assert before['question'] == after['question'] and before['category'] == after['category']
|
|
issues = validate_case(after, oracle)
|
|
if issues:
|
|
raise RuntimeError(f'{after["id"]}: {issues}')
|
|
cases.append(after)
|
|
write_json(output, cases)
|
|
write_json(artifacts / 'source_adjudications.json', {
|
|
'reviewer': 'AI direct Markdown review; not a human adjudication',
|
|
'policy': 'A bare artifact label (rešerš, tutoriál, projekt, dokument, repozitár) is not an academic title. '
|
|
'Do not inherit a child-document title or category-year into its parent academic record.',
|
|
'measurement': 'All questions are known historical regression; corrected gold prevents direct accuracy comparison.',
|
|
'questions': changes})
|
|
write_json(artifacts / 'source_audit.json', audit_dataset(cases, oracle, output))
|
|
print(json.dumps({'questions': len(cases), 'source_issues': 0,
|
|
'adjudicated_ids': [c['id'] for c in changes]}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|