523 lines
34 KiB
Python
523 lines
34 KiB
Python
"""Build a source-grounded schema-v3 regression set and profile-disjoint holdout.
|
|
|
|
This module never imports graph construction/search code and never reads any
|
|
retrieval output. Legacy cases remain development-only. New TEST subjects are
|
|
student profiles absent from every legacy start entity. TEST is generated once
|
|
from raw source facts, before the corrected implementation is evaluated.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import copy
|
|
import hashlib
|
|
import itertools
|
|
import json
|
|
import random
|
|
import re
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .source_oracle import ROOT, DEFAULT_WIKI, load_source_oracle, norm, supporting_documents
|
|
|
|
BASE = Path(__file__).resolve().parent
|
|
SEED = 20261024
|
|
|
|
|
|
def fingerprint(value: Any) -> str:
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(',', ':')).encode()).hexdigest()
|
|
|
|
|
|
def people_for(question: dict, oracle: dict) -> list[str]:
|
|
names = {norm(name): name for name in oracle['people']}
|
|
return [names[norm(item['value'])] for item in question['graph'].get('start_entities', [])
|
|
if item['type'] == 'person' and norm(item['value']) in names]
|
|
|
|
|
|
def academics(oracle: dict, person: str, *, title: str | None = None, year: int | None = None,
|
|
kind: str | None = None) -> list[dict]:
|
|
rows = oracle['people'][person]['academics']
|
|
return [row for row in rows if (title is None or norm(row['title']) == norm(title))
|
|
and (year is None or row['year'] == year) and (kind is None or row['academic_kind'] == kind)]
|
|
|
|
|
|
def has_tag(oracle: dict, person: str, keyword: str) -> bool:
|
|
return norm(keyword) in {norm(tag) for tag in oracle['people'][person]['tags']}
|
|
|
|
|
|
def academic_binding(person: str, record: dict, *, title: bool = False, keyword: str | None = None) -> dict:
|
|
binding = {'person': person, 'academic_kind': record['academic_kind'],
|
|
'academic_type': record['academic_type'], 'year': record['year']}
|
|
if title:
|
|
binding['title'] = record['title']
|
|
if keyword is not None:
|
|
binding['keyword'] = keyword
|
|
return {key: value for key, value in binding.items() if value is not None}
|
|
|
|
|
|
def path_patterns(bindings: list[dict]) -> list[dict]:
|
|
patterns = []
|
|
for index, binding in enumerate(bindings):
|
|
if any(key in binding for key in ['year', 'title', 'academic_kind', 'academic_type']):
|
|
kind = binding.get('academic_kind', 'Academic')
|
|
relation = 'AUTHORED' if kind == 'Thesis' else 'HAS_WORK' if kind == 'Work' else 'AUTHORED|HAS_WORK'
|
|
patterns.append({'binding_index': index, 'node_kinds': ['Person', kind, 'Document'],
|
|
'relationships': [relation, 'EVIDENCED_BY']})
|
|
for field, kind, relation in [('keyword', 'Keyword', 'HAS_KEYWORD'),
|
|
('category', 'Category', 'IN_CATEGORY'),
|
|
('document_author', 'DocumentAuthor', 'AUTHORED_DOCUMENT')]:
|
|
if field in binding:
|
|
patterns.append({'binding_index': index, 'node_kinds': ['Person', 'Document', kind],
|
|
'relationships': ['HAS_DOCUMENT', relation]})
|
|
return patterns
|
|
|
|
|
|
def binding_source_evidence(binding: dict, oracle: dict) -> list[dict]:
|
|
person = oracle['people'][binding['person']]
|
|
evidence = []
|
|
for field in ['keyword', 'category', 'document_author']:
|
|
if field not in binding:
|
|
continue
|
|
docs = supporting_documents(oracle, binding['person'], **{field: binding[field]})
|
|
if binding.get('document_path'):
|
|
docs = [p for p in docs if p == binding['document_path']]
|
|
for path in docs:
|
|
evidence.append({'document_path': path, 'sha256': oracle['documents'][path]['sha256'],
|
|
'field': {'keyword': 'taxonomy.tag', 'category': 'taxonomy.category',
|
|
'document_author': 'taxonomy.author'}[field], 'value': binding[field]})
|
|
if any(key in binding for key in ['year', 'title', 'academic_kind', 'academic_type']):
|
|
rows = academics(oracle, binding['person'], title=binding.get('title'), year=binding.get('year'),
|
|
kind=binding.get('academic_kind'))
|
|
for row in rows:
|
|
if binding.get('academic_type') and row['academic_type'] != binding['academic_type']:
|
|
continue
|
|
evidence.append({'document_path': row['document_path'],
|
|
'sha256': oracle['documents'][row['document_path']]['sha256'],
|
|
'field': 'academic_record', 'academic_kind': row['academic_kind'],
|
|
'academic_type': row['academic_type'], 'title': row['title'], 'year': row['year'],
|
|
'heading_line': row['heading_line'], 'title_line': row['title_line'],
|
|
'year_basis': row['year_basis']})
|
|
if 'start_year' in binding:
|
|
evidence.append({'document_path': person['profile'], 'sha256': oracle['documents'][person['profile']]['sha256'],
|
|
'field': 'start_year', 'value': binding['start_year']})
|
|
return evidence
|
|
|
|
|
|
def make_case(case_id: str, split: str, category: str, question: str, names: list[str], oracle: dict,
|
|
bindings: list[dict], *, no_answer: bool = False, reason: str | None = None,
|
|
expected_set: list[str] | None = None, set_field: str | None = None,
|
|
count: int | None = None, context_documents: list[str] | None = None,
|
|
negative_constraints: dict | None = None) -> dict:
|
|
known = [name for name in names if name in oracle['people']]
|
|
profiles = sorted({oracle['people'][name]['profile'] for name in known})
|
|
paths = set(context_documents or profiles)
|
|
evidence = [item for binding in bindings for item in binding_source_evidence(binding, oracle)]
|
|
# Multiple documents can independently prove the same tag/category. Gold
|
|
# sources are alternatives for each bound proof, not an obligation to cite
|
|
# every duplicate occurrence. Context profiles still cover all candidates.
|
|
evidence_choices = []
|
|
roles = {'taxonomy.tag': 'keyword_document', 'taxonomy.category': 'category_document',
|
|
'taxonomy.author': 'author_document', 'academic_record': 'academic_document',
|
|
'start_year': 'person_document'}
|
|
if not no_answer:
|
|
for index, binding in enumerate(bindings):
|
|
per_role: dict[str, set[str]] = {}
|
|
for item in binding_source_evidence(binding, oracle):
|
|
per_role.setdefault(roles[item['field']], set()).add(item['document_path'])
|
|
for role, alternatives in per_role.items():
|
|
evidence_choices.append({'binding_index': index, 'role': role, 'any_of': [
|
|
{'document_path': path, 'source_url': oracle['documents'][path]['source_url']}
|
|
for path in sorted(alternatives)]})
|
|
expected_values = []
|
|
expected_entities = []
|
|
if not no_answer:
|
|
for binding in bindings:
|
|
for field in ['person', 'keyword', 'category', 'document_author', 'title', 'year', 'start_year']:
|
|
if field in binding:
|
|
value = str(binding[field])
|
|
expected_values.append(value)
|
|
kind = ('thesis_title' if binding.get('academic_kind') == 'Thesis' else 'work_title') if field == 'title' else field
|
|
expected_entities.append({'type': kind, 'value': value})
|
|
if count is not None:
|
|
expected_values.append(str(count))
|
|
graph = {'schema_version': 3, 'task': category, 'start_entities': [{'type': 'person', 'value': name} for name in names],
|
|
'expected_bindings': bindings if not no_answer else [],
|
|
'expected_evidence_documents': evidence_choices,
|
|
'expected_path_patterns': path_patterns(bindings) if not no_answer else [],
|
|
'expected_no_answer': no_answer, 'expected_status': 'no_answer' if no_answer else 'ok',
|
|
'expected_entities': expected_entities, 'expected_relations': [], 'expected_properties': [],
|
|
'source_profiles': profiles}
|
|
if no_answer:
|
|
graph['no_answer_basis'] = reason or 'No source record satisfies every requested condition.'
|
|
graph['negative_constraints'] = negative_constraints or {}
|
|
# Closed-world negative: retain all original metadata considered, not a fabricated positive fact.
|
|
evidence = [{'document_path': path, 'sha256': oracle['documents'][path]['sha256'],
|
|
'field': 'closed_world_metadata', 'tags': oracle['documents'][path]['tags'],
|
|
'categories': oracle['documents'][path]['categories']}
|
|
for name in known for path in oracle['people'][name]['documents']]
|
|
if expected_set is not None:
|
|
graph.update(expected_entity_set=sorted(expected_set, key=norm), entity_set_mode='exact', entity_set_field=set_field or 'person')
|
|
if count is not None:
|
|
graph['expected_count'] = count
|
|
graph['partition_group'] = fingerprint({'category': category, 'people': sorted(names, key=norm),
|
|
'bindings': bindings, 'negative': no_answer,
|
|
'requested': question if no_answer else None})[:20]
|
|
return {'id': case_id, 'split': split, 'category': category, 'difficulty': 'hard', 'question': question,
|
|
'should_answer': not no_answer, 'expected_documents': sorted(paths),
|
|
'expected_source_urls': sorted(oracle['documents'][path]['source_url'] for path in paths),
|
|
'expected_answer_contains': list(dict.fromkeys(expected_values)), 'graph': graph,
|
|
'source_evidence': evidence, 'evidence_basis': ['independent_raw_markdown_oracle_v2']}
|
|
|
|
|
|
def correct_legacy(question: dict, oracle: dict) -> dict:
|
|
names = people_for(question, oracle); cat = question['category']; old = question['graph']
|
|
entities = old.get('expected_entities', [])
|
|
keyword = next((item['value'] for item in entities if item['type'] == 'keyword'), None)
|
|
years = [int(item['value']) for item in entities if item['type'] == 'year']
|
|
bindings = []; expected_set = None; set_field = None; count = None; reason = None
|
|
if cat == 'shared_keyword_pair':
|
|
if keyword and all(has_tag(oracle, name, keyword) for name in names):
|
|
bindings = [{'person': name, 'keyword': keyword} for name in names]
|
|
else:
|
|
reason = 'Requested shared keyword is absent from the raw frontmatter of at least one named person.'
|
|
elif cat == 'same_start_year_pair':
|
|
bindings = [{'person': name, 'start_year': oracle['people'][name]['start_year']} for name in names]
|
|
elif cat == 'same_academic_year_pair':
|
|
common = set.intersection(*[{row['year'] for row in academics(oracle, name) if row['year'] is not None} for name in names])
|
|
bindings = [{'person': name, 'year': year} for year in sorted(common) for name in names]
|
|
if not common:
|
|
reason = 'No shared academic year is stated in the original sources.'
|
|
elif cat in {'person_title_year', 'person_keyword_title'}:
|
|
titles = [item for item in entities if item['type'] in {'thesis_title', 'work_title'}]
|
|
for item in titles:
|
|
kind = 'Thesis' if item['type'] == 'thesis_title' else 'Work'
|
|
rows = academics(oracle, names[0], title=item['value'], kind=kind)
|
|
if not rows and norm(item['value']) in norm(oracle['documents'][oracle['people'][names[0]]['profile']]['raw_text']):
|
|
raise RuntimeError(f"Oracle extraction needs review for supported raw title: {question['id']}")
|
|
if years:
|
|
rows = [row for row in rows if row['year'] in years]
|
|
if cat == 'person_keyword_title' and not has_tag(oracle, names[0], keyword or ''):
|
|
reason = 'Requested keyword is not a raw frontmatter tag of this person.'
|
|
break
|
|
for row in rows:
|
|
binding = academic_binding(names[0], row, title=True, keyword=keyword if cat == 'person_keyword_title' else None)
|
|
if cat == 'person_keyword_title' and not years:
|
|
binding.pop('year', None)
|
|
bindings.append(binding)
|
|
if not bindings and not reason:
|
|
reason = 'Requested title/type/year combination is not independently supported by the source.'
|
|
elif cat == 'document_author_category_multi_hop':
|
|
category = next(item['value'] for item in entities if item['type'] == 'category')
|
|
for path in supporting_documents(oracle, names[0], category=category):
|
|
author = oracle['documents'][path]['author']
|
|
if author:
|
|
bindings.append({'person': names[0], 'category': category, 'document_author': author,
|
|
'document_path': path, 'same_document': True})
|
|
if not bindings:
|
|
reason = 'No same document supports both the category and metadata author.'
|
|
elif cat == 'person_multiple_categories':
|
|
expected_set = oracle['people'][names[0]]['categories']; set_field = 'category'
|
|
bindings = [{'person': names[0], 'category': category} for category in expected_set]
|
|
elif cat == 'candidate_set_aggregation':
|
|
query_years = [int(x) for x in re.findall(r'\b20\d{2}\b', question['question'])]
|
|
year = query_years[-1]
|
|
matches = [name for name in names if academics(oracle, name, year=year)]
|
|
bindings = [{'person': name, 'year': year} for name in matches]
|
|
expected_set = matches; set_field = 'person'; count = len(matches)
|
|
elif cat == 'conjunctive_disambiguation':
|
|
year = years[0]
|
|
matches = [name for name in names if has_tag(oracle, name, keyword) and academics(oracle, name, year=year)]
|
|
bindings = [{'person': name, 'keyword': keyword, 'year': year} for name in matches]
|
|
if not matches:
|
|
reason = 'No named person satisfies both raw frontmatter keyword and academic year.'
|
|
else:
|
|
raise ValueError(f'Unsupported legacy category: {cat}')
|
|
case = make_case(question['id'], 'dev', cat, question['question'], names, oracle, bindings,
|
|
no_answer=reason is not None, reason=reason, expected_set=expected_set,
|
|
set_field=set_field, count=count,
|
|
negative_constraints=({'keyword': keyword, 'require_all_people': True} if cat == 'shared_keyword_pair'
|
|
else {'keyword': keyword} if cat == 'person_keyword_title'
|
|
else {'keyword': keyword, 'year': years[0]} if cat == 'conjunctive_disambiguation'
|
|
else {'shared_academic_year': True} if cat == 'same_academic_year_pair'
|
|
else {}))
|
|
case['legacy_split'] = question['split']
|
|
case['legacy_role'] = 'previously_seen_regression_only'
|
|
return case
|
|
|
|
|
|
def independent_pool(oracle: dict, names: list[str]) -> dict[str, list[dict]]:
|
|
pool: dict[str, list[dict]] = {category: [] for category in [
|
|
'shared_keyword_pair', 'same_academic_year_pair', 'same_start_year_pair', 'person_title_year',
|
|
'person_keyword_title', 'document_author_category_multi_hop', 'person_multiple_categories',
|
|
'candidate_set_aggregation', 'conjunctive_disambiguation']}
|
|
def add(category: str, text: str, people: list[str], bindings: list[dict], **kwargs):
|
|
pool[category].append(make_case('', 'test', category, text, people, oracle, bindings, **kwargs))
|
|
for left, right in itertools.combinations(names, 2):
|
|
common_tags = sorted({norm(tag): tag for tag in oracle['people'][left]['tags']
|
|
if has_tag(oracle, right, tag)}.values(), key=norm)
|
|
for tag in common_tags:
|
|
add('shared_keyword_pair', f'Over spoločné kľúčové slovo {tag} pri osobách {left} a {right}.',
|
|
[left, right], [{'person': name, 'keyword': tag} for name in [left, right]])
|
|
shared_years = {row['year'] for row in academics(oracle, left)} & {row['year'] for row in academics(oracle, right)} - {None}
|
|
# A query for an unspecified shared year expects all supported intersections.
|
|
if shared_years:
|
|
add('same_academic_year_pair', f'Aké spoločné roky práce spájajú osoby {left} a {right}?',
|
|
[left, right], [{'person': name, 'year': year} for year in sorted(shared_years) for name in [left, right]])
|
|
start_l, start_r = oracle['people'][left]['start_year'], oracle['people'][right]['start_year']
|
|
if start_l is not None and start_l == start_r:
|
|
add('same_start_year_pair', f'Porovnaj rovnaký rok začiatku štúdia osôb {left} a {right}.',
|
|
[left, right], [{'person': name, 'start_year': start_l} for name in [left, right]])
|
|
for name in names:
|
|
person = oracle['people'][name]
|
|
if person['categories']:
|
|
add('person_multiple_categories', f'Vypíš všetky kategórie dokumentov osoby {name}.', [name],
|
|
[{'person': name, 'category': cat} for cat in person['categories']],
|
|
expected_set=person['categories'], set_field='category')
|
|
for category in person['categories']:
|
|
for path in supporting_documents(oracle, name, category=category):
|
|
author = oracle['documents'][path]['author']
|
|
if author:
|
|
add('document_author_category_multi_hop',
|
|
f'Kto je autor dokumentu osoby {name}, ktorý patrí do kategórie {category}?', [name],
|
|
[{'person': name, 'category': category, 'document_author': author,
|
|
'document_path': path, 'same_document': True}])
|
|
# Title-year cases use only explicit, nonconflicting title records.
|
|
for record in person['academics']:
|
|
if not record['title'] or record['year'] is None or record in person['year_conflicts']:
|
|
continue
|
|
year = record['year']
|
|
kind_label = 'diplomového projektu' if record['academic_type'] == 'diploma_project' else 'práce'
|
|
add('person_title_year', f'Aký názov {kind_label} a rok práce sú pri osobe {name} uvedené pre rok {year}?',
|
|
[name], [academic_binding(name, record, title=True)])
|
|
for tag in person['tags']:
|
|
binding = academic_binding(name, record, title=True, keyword=tag)
|
|
# Pin the academic year in the question as the independent source can contain several titles.
|
|
add('person_keyword_title', f'Aký názov práce osoby {name} v roku {year} súvisí s kľúčovým slovom {tag}?',
|
|
[name], [binding])
|
|
# Candidate groups are source-derived; the query mentions every candidate.
|
|
for year in sorted({row['year'] for name in names for row in academics(oracle, name) if row['year'] is not None}):
|
|
matching = [name for name in names if academics(oracle, name, year=year)]
|
|
other = [name for name in names if name not in matching]
|
|
for selected in itertools.islice(itertools.combinations(matching, 2), 12):
|
|
if other:
|
|
candidates = list(selected) + [other[0]]
|
|
add('candidate_set_aggregation',
|
|
f'Z kandidátov {", ".join(candidates)} vyber osoby s prácou v roku {year}. Koľko ich je?',
|
|
candidates, [{'person': name, 'year': year} for name in selected],
|
|
expected_set=list(selected), set_field='person', count=len(selected))
|
|
for name in names:
|
|
person = oracle['people'][name]
|
|
for year in sorted({record['year'] for record in person['academics'] if record['year'] is not None}):
|
|
for tag in person['tags']:
|
|
distractors = [other for other in names if other != name
|
|
and (has_tag(oracle, other, tag) != bool(academics(oracle, other, year=year)))]
|
|
for other in distractors[:2]:
|
|
candidates = [name, other]
|
|
add('conjunctive_disambiguation',
|
|
f'Ktorá z osôb {name} a {other} má kľúčové slovo {tag} a jej práca je z roku {year}?',
|
|
candidates, [{'person': name, 'keyword': tag, 'year': year}])
|
|
return pool
|
|
|
|
|
|
def necessary_negatives(oracle: dict, names: list[str], split: str) -> list[dict]:
|
|
cases = []
|
|
prefix = 'dev' if split == 'dev' else 'holdout'
|
|
for i, name in enumerate(names[:4]):
|
|
tag = next(iter(oracle['people'][name]['tags']), 'nlp')
|
|
cases.append(make_case(f'{prefix}-negative-year-{i}', split, 'negative_lookup',
|
|
f'Aký názov práce má osoba {name}, ak jej práca je z roku 2099 a jej dokument obsahuje kľúčové slovo {tag}?',
|
|
[name], oracle, [], no_answer=True, reason='No source academic record has year 2099.',
|
|
negative_constraints={'year': 2099, 'keyword': tag}))
|
|
for i, name in enumerate(names[4:6]):
|
|
cases.append(make_case(f'{prefix}-negative-tag-{i}', split, 'negative_lookup',
|
|
f'Aký názov práce má osoba {name}, ak jej dokument obsahuje kľúčové slovo neexistujucitagvzor?',
|
|
[name], oracle, [], no_answer=True, reason='Requested tag absent from complete raw frontmatter catalog.',
|
|
negative_constraints={'keyword': 'neexistujucitagvzor'}))
|
|
for i, unknown in enumerate(['Zuzana Neexistujúca Vzorová', 'Roman Neskutočný Testovací'] if split == 'dev'
|
|
else ['Tibor Neexistujúci Skúšobný', 'Lenka Neskutočná Vzorová']):
|
|
cases.append(make_case(f'{prefix}-negative-person-{i}', split, 'negative_lookup',
|
|
f'Aký názov diplomovej práce má osoba {unknown}?', [unknown], oracle, [],
|
|
no_answer=True, reason='Explicit full name absent from all raw profile frontmatter titles.',
|
|
negative_constraints={'unknown_person': unknown}))
|
|
candidates = names[:3]
|
|
cases.append(make_case(f'{prefix}-zero-count', split, 'candidate_set_aggregation',
|
|
f'Z kandidátov {", ".join(candidates)} vyber osoby s prácou v roku 2099. Koľko ich je?',
|
|
candidates, oracle, [], expected_set=[], set_field='person', count=0))
|
|
return cases
|
|
|
|
|
|
def validate_case(case: dict, oracle: dict) -> list[str]:
|
|
issues = []
|
|
graph = case['graph']
|
|
names = people_for(case, oracle)
|
|
if set(graph.get('source_profiles', [])) != {oracle['people'][name]['profile'] for name in names}:
|
|
issues.append('incorrect_source_profile_partition')
|
|
for binding in graph.get('expected_bindings', []):
|
|
person = binding['person']
|
|
if person not in oracle['people']:
|
|
issues.append('unknown_bound_person'); continue
|
|
if 'start_year' in binding and oracle['people'][person]['start_year'] != binding['start_year']:
|
|
issues.append('wrong_start_year')
|
|
metadata = {key: binding[key] for key in ['keyword', 'category', 'document_author'] if key in binding}
|
|
if binding.get('same_document') and not supporting_documents(oracle, person, **metadata):
|
|
issues.append('metadata_not_connected_in_same_document')
|
|
for key, value in metadata.items():
|
|
if not supporting_documents(oracle, person, **{key: value}):
|
|
issues.append('unsupported_' + key)
|
|
if any(key in binding for key in ['title', 'year', 'academic_type', 'academic_kind']):
|
|
records = academics(oracle, person, title=binding.get('title'), year=binding.get('year'), kind=binding.get('academic_kind'))
|
|
if binding.get('academic_type'):
|
|
records = [r for r in records if r['academic_type'] == binding['academic_type']]
|
|
if not records:
|
|
issues.append('unsupported_academic_binding')
|
|
if graph.get('expected_no_answer'):
|
|
if graph.get('expected_bindings') or case.get('expected_answer_contains') or graph.get('expected_entities'):
|
|
issues.append('negative_contains_positive_expectations')
|
|
constraints = graph.get('negative_constraints', {})
|
|
names = people_for(case, oracle)
|
|
if constraints.get('unknown_person'):
|
|
if norm(constraints['unknown_person']) in {norm(name) for name in oracle['people']}:
|
|
issues.append('negative_person_actually_exists')
|
|
elif constraints.get('shared_academic_year'):
|
|
year_sets = [{r['year'] for r in academics(oracle, name) if r['year'] is not None} for name in names]
|
|
if year_sets and set.intersection(*year_sets):
|
|
issues.append('negative_shared_year_actually_exists')
|
|
elif constraints.get('require_all_people'):
|
|
if not names or all(has_tag(oracle, name, constraints.get('keyword', '')) for name in names):
|
|
issues.append('negative_shared_keyword_actually_exists')
|
|
elif constraints:
|
|
matching = []
|
|
for name in names:
|
|
valid = not constraints.get('keyword') or has_tag(oracle, name, constraints['keyword'])
|
|
if any(key in constraints for key in ['year', 'title', 'academic_kind', 'title_required']):
|
|
records = academics(oracle, name, year=constraints.get('year'),
|
|
title=constraints.get('title'), kind=constraints.get('academic_kind'))
|
|
if constraints.get('academic_type'):
|
|
records = [r for r in records if r['academic_type'] == constraints['academic_type']]
|
|
if constraints.get('title_required'):
|
|
records = [r for r in records if r.get('title')]
|
|
valid = valid and bool(records)
|
|
if valid:
|
|
matching.append(name)
|
|
if matching:
|
|
issues.append('negative_conjunction_actually_satisfied')
|
|
else:
|
|
issues.append('negative_without_machine_verifiable_false_premise')
|
|
if graph.get('expected_entity_set') is not None and graph.get('entity_set_field') == 'category':
|
|
names = people_for(case, oracle)
|
|
if names and set(graph['expected_entity_set']) != set(oracle['people'][names[0]]['categories']):
|
|
issues.append('incomplete_category_set')
|
|
if graph.get('expected_count') is not None:
|
|
years = re.findall(r'\b20\d{2}\b', case['question'])
|
|
if not years:
|
|
issues.append('count_without_supported_year_constraint')
|
|
else:
|
|
matches = {name for name in names if academics(oracle, name, year=int(years[-1]))}
|
|
if matches != set(graph.get('expected_entity_set', [])) or len(matches) != graph['expected_count']:
|
|
issues.append('incorrect_candidate_set_or_count')
|
|
return sorted(set(issues))
|
|
|
|
|
|
def audit_dataset(questions: list[dict], oracle: dict, dataset_path: Path) -> dict:
|
|
rows = [{'id': question['id'], 'split': question['split'], 'source_consistent': not (issues := validate_case(question, oracle)),
|
|
'issues': issues, 'category': question['category']} for question in questions]
|
|
inputs = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in ['data/documents.json', 'data/chunks.json']}
|
|
return {'method': 'Independent original Markdown frontmatter/heading/title oracle; no graph parser, chunks or retrieval results used for labels.',
|
|
'dataset_sha256': hashlib.sha256(dataset_path.read_bytes()).hexdigest(),
|
|
'inputs': inputs, 'source_corpus': oracle['source_corpus'], 'questions': rows,
|
|
'by_split': {split: {'questions': len(selected), 'source_consistent': sum(row['source_consistent'] for row in selected),
|
|
'issues': dict(Counter(issue for row in selected for issue in row['issues']))}
|
|
for split in ['dev', 'test'] for selected in [[row for row in rows if row['split'] == split]]}}
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--wiki-root', type=Path, default=DEFAULT_WIKI)
|
|
parser.add_argument('--legacy', type=Path, default=BASE / 'graphrag_questions.json')
|
|
parser.add_argument('--output', type=Path, default=BASE / 'graphrag_questions_v2.json')
|
|
parser.add_argument('--artifacts', type=Path, default=BASE / 'results/v2_preparation')
|
|
args = parser.parse_args()
|
|
oracle = load_source_oracle(args.wiki_root)
|
|
legacy = json.loads(args.legacy.read_text())
|
|
corrected = [correct_legacy(question, oracle) for question in legacy]
|
|
legacy_names = {name for question in legacy for name in people_for(question, oracle)}
|
|
exposed_path = BASE / 'results/final_2026-10-24/holdout_failures.json'
|
|
previously_exposed = set()
|
|
if exposed_path.exists():
|
|
for row in json.loads(exposed_path.read_text()).get('questions', []):
|
|
previously_exposed.update(set(row.get('actual_documents', [])) - set(row.get('expected_documents', [])))
|
|
reserved_names = sorted([name for name, person in oracle['people'].items()
|
|
if name not in legacy_names and person['profile'] not in previously_exposed
|
|
and person['profile'].startswith('pages/students/')], key=norm)
|
|
dev_negatives = necessary_negatives(oracle, sorted(legacy_names, key=norm), 'dev')
|
|
test_pool = independent_pool(oracle, reserved_names)
|
|
rng = random.Random(SEED)
|
|
holdout = []
|
|
for category, cases in test_pool.items():
|
|
unique = {case['graph']['partition_group']: case for case in cases}
|
|
candidates = list(unique.values()); rng.shuffle(candidates)
|
|
if len(candidates) < 8:
|
|
raise RuntimeError(f'Insufficient independent source cases for {category}: {len(candidates)}')
|
|
for case in candidates[:8]:
|
|
case['id'] = f'h{len(holdout)+1:04d}'
|
|
holdout.append(case)
|
|
negative_cases = necessary_negatives(oracle, reserved_names, 'test')
|
|
# A zero-result count is an answerable numeric result, not a no-answer.
|
|
zero_count = negative_cases.pop()
|
|
count_index = next(i for i, case in enumerate(holdout) if case['category'] == 'candidate_set_aggregation')
|
|
zero_count['id'] = holdout[count_index]['id']
|
|
holdout[count_index] = zero_count
|
|
holdout.extend(negative_cases)
|
|
dev = corrected + dev_negatives
|
|
questions = dev + holdout
|
|
all_issues = {case['id']: validate_case(case, oracle) for case in questions if validate_case(case, oracle)}
|
|
if all_issues:
|
|
raise RuntimeError(f'Source consistency issues: {all_issues}')
|
|
dev_profiles = {p for case in dev for p in case['graph']['source_profiles']}
|
|
test_profiles = {p for case in holdout for p in case['graph']['source_profiles']}
|
|
assert not dev_profiles & test_profiles
|
|
assert not ({case['graph']['partition_group'] for case in dev} & {case['graph']['partition_group'] for case in holdout})
|
|
write_json(args.output, questions)
|
|
corrections = []
|
|
old_audit_path = BASE / 'results/final_2026-10-24/source_audit.json'
|
|
old_issues = {row['id']: row['issues'] for row in json.loads(old_audit_path.read_text())['questions'] if row['issues']}
|
|
for old, new in zip(legacy, corrected):
|
|
corrections.append({'id': old['id'], 'legacy_split': old['split'], 'new_split': 'dev',
|
|
'v1_source_audit_issues': old_issues.get(old['id'], []),
|
|
'old_should_answer': old.get('should_answer', True), 'new_should_answer': new['should_answer'],
|
|
'old_expected_values': old['expected_answer_contains'], 'new_expected_values': new['expected_answer_contains'],
|
|
'new_expected_bindings': new['graph']['expected_bindings'],
|
|
'no_answer_basis': new['graph'].get('no_answer_basis'),
|
|
'source_evidence': new['source_evidence']})
|
|
write_json(args.artifacts / 'legacy_corrections.json', {'legacy_count': len(legacy), 'v1_flagged_count': len(old_issues),
|
|
'resolved_v1_flagged_count': sum(not validate_case(new, oracle) for new in corrected if new['id'] in old_issues),
|
|
'negative_count': sum(not case['should_answer'] for case in corrected), 'questions': corrections})
|
|
write_json(args.artifacts / 'source_audit.json', audit_dataset(questions, oracle, args.output))
|
|
partition = {'seed': SEED, 'strategy': 'All previously seen 500 cases are DEV regression. TEST profiles never appeared in legacy start_entities; all real candidate profiles are disjoint.',
|
|
'dev_count': len(dev), 'test_count': len(holdout), 'previously_exposed_profiles_excluded_from_test': sorted(previously_exposed),
|
|
'dev_profiles': sorted(dev_profiles), 'test_profiles': sorted(test_profiles),
|
|
'dev_groups': sorted({case['graph']['partition_group'] for case in dev}),
|
|
'test_groups': sorted({case['graph']['partition_group'] for case in holdout}),
|
|
'dev_categories': dict(Counter(case['category'] for case in dev)),
|
|
'test_categories': dict(Counter(case['category'] for case in holdout)),
|
|
'source_corpus': oracle['source_corpus'],
|
|
'dataset_sha256': hashlib.sha256(args.output.read_bytes()).hexdigest(),
|
|
'limitations': ['Same source corpus and shared tag/category/author vocabulary.',
|
|
'Template families are known; TEST persons and their fact groups are withheld.',
|
|
'Oracle is independent of graph extraction but still rule-based; provenance permits human review.']}
|
|
write_json(args.artifacts / 'partition.json', partition)
|
|
print(json.dumps({'dataset': str(args.output), 'dev': len(dev), 'test': len(holdout),
|
|
'reserved_profile_count': len(test_profiles), 'source_issues': 0,
|
|
'corrected_legacy_negatives': sum(not q['should_answer'] for q in corrected),
|
|
'resolved_original_issues': len(old_issues),
|
|
'test_categories': partition['test_categories']}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|