"""Continue after exposed v2: all old cases become DEV; reserve the last unseen profiles. The small supplementary holdout cannot replace the broader independent v2 measurement. It is kept profile-disjoint and must be reported with its size. """ from __future__ import annotations import argparse import copy import hashlib import json import re from collections import Counter from pathlib import Path from .benchmark_v2 import (BASE, make_case, correct_legacy, people_for, academics, has_tag, academic_binding, validate_case, audit_dataset, write_json) from .source_oracle import load_source_oracle, supporting_documents, norm def reannotate(case, oracle, original): if case['id'] in original: return correct_legacy(original[case['id']], oracle) category = case['category']; graph = case['graph']; names = people_for(case, oracle) bindings = []; negative = None; expected_set = None; field = None; count = None; constraints = {} old = graph.get('expected_bindings', []) years = [int(y) for y in re.findall(r'\b20\d{2}\b', case['question'])] keyword = next((row.get('keyword') for row in old if row.get('keyword')), None) if category == 'negative_lookup': corrected = copy.deepcopy(case); corrected['split'] = 'dev' return corrected if category == 'shared_keyword_pair': if all(has_tag(oracle, name, keyword) for name in names): bindings = [{'person': name, 'keyword': keyword} for name in names] else: negative = 'No common literal frontmatter tag.'; constraints = {'keyword': keyword, 'require_all_people': True} elif category == 'same_academic_year_pair': common = set.intersection(*[{r['year'] for r in academics(oracle, name) if r['year']} for name in names]) bindings = [{'person': name, 'year': year} for year in sorted(common) for name in names] if not common: negative = 'No shared academic year in original sources.'; constraints = {'shared_academic_year': True} elif category == 'same_start_year_pair': bindings = [{'person': name, 'start_year': oracle['people'][name]['start_year']} for name in names] elif category in {'person_title_year', 'person_keyword_title'}: records = [r for r in academics(oracle, names[0], year=years[-1]) if r.get('title')] if 'diplomoveho projektu' in norm(case['question']): records = [r for r in records if r['academic_type'] == 'diploma_project'] if category == 'person_keyword_title' and not has_tag(oracle, names[0], keyword): records = [] bindings = [academic_binding(names[0], row, title=True, keyword=keyword) for row in records] if not bindings: negative = 'Requested year/type/title is absent; a project title is not a future thesis title.' constraints = {'year': years[-1], **({'keyword': keyword} if keyword else {})} elif category == 'document_author_category_multi_hop': for row in old: for path in supporting_documents(oracle, names[0], category=row['category']): author = oracle['documents'][path]['author'] if author: bindings.append({'person': names[0], 'category': row['category'], 'document_author': author, 'document_path': path, 'same_document': True}) elif category == 'person_multiple_categories': expected_set = oracle['people'][names[0]]['categories']; field = 'category' bindings = [{'person': names[0], 'category': value} for value in expected_set] elif category == 'candidate_set_aggregation': expected_set = [name for name in names if academics(oracle, name, year=years[-1])] count = len(expected_set); field = 'person' bindings = [{'person': name, 'year': years[-1]} for name in expected_set] elif category == 'conjunctive_disambiguation': matches = [name for name in names if has_tag(oracle, name, keyword) and academics(oracle, name, year=years[-1])] bindings = [{'person': name, 'keyword': keyword, 'year': years[-1]} for name in matches] if not matches: negative = 'No candidate satisfies both source conditions.' constraints = {'keyword': keyword, 'year': years[-1]} else: raise ValueError(category) result = make_case(case['id'], 'dev', category, case['question'], names, oracle, bindings, no_answer=negative is not None, reason=negative, negative_constraints=constraints, expected_set=expected_set, set_field=field, count=count) result['legacy_role'] = 'exposed_v2_case_development_only' return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, default=BASE / 'graphrag_questions_v3.json') parser.add_argument('--artifacts', type=Path, default=BASE / 'results/v3_preparation') args = parser.parse_args() oracle = load_source_oracle() previous = json.loads((BASE / 'graphrag_questions_v2.json').read_text()) original = {q['id']: q for q in json.loads((BASE / 'graphrag_questions.json').read_text())} dev = [reannotate(case, oracle, original) for case in previous] used = {p for case in previous for p in case['graph']['source_profiles']} used.update(json.loads((BASE / 'results/v2_preparation/partition.json').read_text())[ 'previously_exposed_profiles_excluded_from_test']) reserved = sorted([name for name, person in oracle['people'].items() if person['profile'].startswith('pages/students/') and person['profile'] not in used], key=norm) if len(reserved) != 2: raise RuntimeError('Unexpected remaining unseen profile population; review partition policy.') holdout = [] def add(category, text, names, bindings, **kwargs): holdout.append(make_case(f'fresh-{len(holdout)+1:03d}', 'test', category, text, names, oracle, bindings, **kwargs)) for name in reserved: person = oracle['people'][name] add('person_multiple_categories', f'Vypíš všetky kategórie dokumentov osoby {name}.', [name], [{'person': name, 'category': category} for category in person['categories']], expected_set=person['categories'], set_field='category') category = person['categories'][0] for path in supporting_documents(oracle, name, category=category)[:1]: author = oracle['documents'][path]['author'] if author: add('document_author_category_multi_hop', f'Ktorý autor dokumentu je pri osobe {name} spojený s kategóriou {category}?', [name], [{'person': name, 'category': category, 'document_author': author, 'document_path': path, 'same_document': True}]) add('person_start_year', f'Aký rok začiatku štúdia je uvedený pri osobe {name}?', [name], [{'person': name, 'start_year': person['start_year']}]) add('negative_lookup', f'Aký názov práce má osoba {name}, ak jej dokument obsahuje kľúčové slovo nenajdenyoverovacitag?', [name], [], no_answer=True, reason='Tag absent from raw frontmatter.', negative_constraints={'keyword': 'nenajdenyoverovacitag'}) add('negative_lookup', f'Aký názov práce má osoba {name} v roku 2098?', [name], [], no_answer=True, reason='No academic record for requested year.', negative_constraints={'year': 2098}) for name in ['Diana Neznáma Overovacia', 'Milan Neprítomný Vzorový']: add('negative_lookup', f'Aký názov diplomovej práce má osoba {name}?', [name], [], no_answer=True, reason='No profile with requested full name.', negative_constraints={'unknown_person': name}) add('candidate_set_aggregation', f'Z kandidátov {", ".join(reserved)} vyber osoby s prácou v roku 2098. Koľko ich je?', reserved, [], expected_set=[], set_field='person', count=0) questions = dev + holdout issues = {q['id']: validate_case(q, oracle) for q in questions if validate_case(q, oracle)} if issues: raise RuntimeError(f'Independent source audit failed: {issues}') dev_profiles = {p for q in dev for p in q['graph']['source_profiles']} test_profiles = {p for q in holdout for p in q['graph']['source_profiles']} assert not dev_profiles & test_profiles write_json(args.output, questions) write_json(args.artifacts / 'source_audit.json', audit_dataset(questions, oracle, args.output)) changes = [{'id': old['id'], 'old_bindings': old['graph']['expected_bindings'], 'new_bindings': new['graph']['expected_bindings'], 'old_should_answer': old['should_answer'], 'new_should_answer': new['should_answer'], 'source_evidence': new['source_evidence']} for old, new in zip(previous, dev) if old['graph']['expected_bindings'] != new['graph']['expected_bindings'] or old['should_answer'] != new['should_answer']] write_json(args.artifacts / 'source_adjudications.json', {'method': 'Raw heading project/thesis distinction; no answer-output based relabeling.', 'changes': changes}) partition = {'dev_count': len(dev), 'test_count': len(holdout), 'dev_profiles': sorted(dev_profiles), 'test_profiles': sorted(test_profiles), 'source_profile_overlap': 0, 'limitations': ['Only two entirely unseen profiles remain; supplementary holdout has limited category coverage.', 'The broader v2 independent TEST remains 67/80; v2 cases are now DEV, never fresh TEST.']} write_json(args.artifacts / 'partition.json', partition) print(json.dumps({'dev': len(dev), 'test': len(holdout), 'unseen_profiles': len(reserved), 'source_adjudications': len(changes), 'source_issues': 0}, indent=2)) if __name__ == '__main__': main()