92 lines
5.8 KiB
Python
92 lines
5.8 KiB
Python
"""Declare a fresh query-group holdout using raw sources only, before retrieval.
|
|
|
|
All 1,042 previously exposed questions become DEV. No profile is new; this is
|
|
a new-combination holdout over a known corpus, not an unseen-person experiment.
|
|
"""
|
|
import copy
|
|
import json
|
|
import random
|
|
from collections import Counter
|
|
|
|
from .benchmark_v2 import BASE, audit_dataset, make_case, validate_case, write_json, binding_source_evidence
|
|
from .benchmark_v4 import build_pool, logical_group, refresh_titles
|
|
from .source_oracle import load_source_oracle, norm
|
|
|
|
QUOTAS = {'shared_keyword_pair': 10, 'same_academic_year_pair': 10,
|
|
'same_start_year_pair': 10, 'person_title_year': 10, 'person_keyword_title': 10,
|
|
'document_author_category_multi_hop': 2, 'person_multiple_categories': 7,
|
|
'candidate_set_aggregation': 7, 'conjunctive_disambiguation': 6, 'negative_lookup': 8}
|
|
SEED = 20261026
|
|
|
|
|
|
def main():
|
|
output = BASE/'graphrag_questions_v7.json'; artifacts = BASE/'results/v7_preparation'
|
|
if output.exists(): raise RuntimeError('Refusing to overwrite declared questions.')
|
|
oracle = load_source_oracle()
|
|
old = json.loads((BASE/'graphrag_questions_v6.json').read_text())
|
|
dev = []; changes = []
|
|
exposed = {logical_group(q) for q in old}
|
|
for original in old:
|
|
case = copy.deepcopy(original); case['split'] = 'dev'
|
|
names = [r['value'] for r in case['graph']['start_entities'] if r['type'] == 'person']
|
|
if any(b.get('start_year') is not None and oracle['people'][b['person']]['start_year'] is None
|
|
for b in case['graph']['expected_bindings']):
|
|
case = make_case(case['id'], 'dev', case['category'], case['question'], names, oracle, [],
|
|
no_answer=True, reason='The original Markdown does not explicitly document every requested study-start year.',
|
|
negative_constraints={'missing_start_year': True})
|
|
changes.append({'id': case['id'], 'old_bindings': original['graph']['expected_bindings'],
|
|
'new_status': 'no_answer', 'reason': 'Directory cohort is not a documented study-start year.'})
|
|
elif case['should_answer']:
|
|
case['source_evidence'] = [e for b in case['graph']['expected_bindings'] for e in binding_source_evidence(b, oracle)]
|
|
case['graph']['partition_group'] = logical_group(case); dev.append(case)
|
|
# Fixed audit regressions; these are explicitly DEV, not extra easy TEST negatives.
|
|
for year in [1999, 1899, 2100, 9999]:
|
|
case = make_case(f'v7-dev-year-{year}', 'dev', 'negative_lookup',
|
|
f'Aký je názov práce osoby Ján Holp v roku {year}?', ['Ján Holp'], oracle, [],
|
|
no_answer=True, negative_constraints={'year': year})
|
|
case['graph']['partition_group'] = logical_group(case); dev.append(case)
|
|
exposed.update(logical_group(q) for q in dev)
|
|
texts = {norm(q['question']) for q in dev}; rng = random.Random(SEED)
|
|
holdout = []; capacity = {}
|
|
for category, pool in build_pool(oracle).items():
|
|
unique = {}
|
|
for candidate in pool:
|
|
case = refresh_titles(candidate, oracle); group = logical_group(case)
|
|
if group not in exposed and norm(case['question']) not in texts:
|
|
unique[group] = case
|
|
capacity[category] = len(unique)
|
|
candidates = [unique[key] for key in sorted(unique)]; rng.shuffle(candidates)
|
|
quota = QUOTAS[category]
|
|
if len(candidates) < quota: raise RuntimeError(f'Insufficient source capacity: {category}')
|
|
for case in candidates[:quota]:
|
|
case['id'] = f'fresh7-{len(holdout)+1:03d}'; case['split'] = 'test'
|
|
case['graph']['partition_group'] = logical_group(case); holdout.append(case)
|
|
questions = dev + holdout
|
|
for case in questions:
|
|
issues = validate_case(case, oracle)
|
|
if issues: raise RuntimeError(f'{case["id"]}: {issues}')
|
|
write_json(output, questions)
|
|
audit = audit_dataset(questions, oracle, output)
|
|
audit['partition_policy'] = {'method': 'disjoint logical query groups',
|
|
'shared_source_profiles': True, 'seed': SEED, 'previously_exposed_groups': sorted(exposed),
|
|
'limitations': ['Known corpus, persons and template families; only the query/fact combinations are fresh.',
|
|
'Two remaining document-author/category groups and seven category-list groups constrain category coverage.',
|
|
'AI source annotation, not human annotation. No tuning on this TEST.']}
|
|
write_json(artifacts/'source_audit.json', audit)
|
|
write_json(artifacts/'source_corrections.json', {'questions': changes,
|
|
'policy': 'Only explicit study-start text is a fact. Cohort remains separate metadata; missing means unknown.'})
|
|
write_json(artifacts/'regression_ids.json', [q['id'] for q in old if q['split'] == 'test'])
|
|
ids = set(json.loads((BASE/'results/v6_preparation/generation_dev_ids.json').read_text()))
|
|
ids.update(f'v7-dev-year-{year}' for year in [1999, 1899, 2100, 9999])
|
|
write_json(artifacts/'generation_dev_ids.json', sorted(ids))
|
|
write_json(artifacts/'protocol.json', {'seed': SEED, 'dev_count': len(dev), 'fresh_test_count': len(holdout),
|
|
'test_categories': dict(Counter(q['category'] for q in holdout)), 'test_positives': sum(q['should_answer'] for q in holdout),
|
|
'test_negatives': sum(not q['should_answer'] for q in holdout), 'available_groups': capacity,
|
|
'old_gold_corrections': len(changes), 'regression_count': 320, 'llm_dev_count': len(ids),
|
|
'target': .92, 'policy': 'Freeze after complete DEV; run fresh TEST once, retain every failure. Regression remains separate.',
|
|
'annotation': 'Independent raw-source oracle plus AI source review; not human review.'})
|
|
print(json.dumps({'dev': len(dev), 'test': len(holdout), 'source_corrections': len(changes), 'quotas': QUOTAS}))
|
|
|
|
|
|
if __name__ == '__main__': main()
|