124 lines
6.8 KiB
Python
124 lines
6.8 KiB
Python
"""Check source consistency without changing questions or consulting search output."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from ..graphrag import build_graph_payload
|
|
from .graphrag_evaluator import normalize_text
|
|
from .reproducibility import content_fingerprints
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
|
|
|
|
def audit_questions(questions: list[dict], payload: dict) -> dict[str, Any]:
|
|
people = {normalize_text(row['name']): row for row in payload['people']}
|
|
documents = {row['path']: row for row in payload['documents']}
|
|
person_docs: dict[str, set[str]] = defaultdict(set)
|
|
for row in payload['person_documents']:
|
|
person_docs[row['person_id']].add(row['document_path'])
|
|
doc_keywords: dict[str, set[str]] = defaultdict(set)
|
|
for row in payload['document_keywords']:
|
|
doc_keywords[row['document_path']].add(row['keyword_id'])
|
|
keywords = {normalize_text(row['name']): row['id'] for row in payload['keywords']}
|
|
keywords.update({normalize_text(row['id']): row['id'] for row in payload['keywords']})
|
|
academic: dict[str, list[dict]] = defaultdict(list)
|
|
for row in payload['theses'] + payload['works']:
|
|
academic[row['source_document']].append(row)
|
|
rows = []
|
|
for question in questions:
|
|
graph = question['graph']
|
|
issues = []
|
|
starts = [people.get(normalize_text(item['value']))
|
|
for item in graph.get('start_entities', []) if item['type'] == 'person']
|
|
if any(person is None for person in starts):
|
|
issues.append('unknown_start_person')
|
|
starts = [person for person in starts if person]
|
|
expected = graph.get('expected_entities', [])
|
|
targets = [people.get(normalize_text(item['value'])) for item in expected if item['type'] == 'person']
|
|
targets = [person for person in targets if person] or starts
|
|
required_people = starts if question['category'] == 'shared_keyword_pair' else targets
|
|
for entity in expected:
|
|
if entity['type'] == 'keyword':
|
|
key = keywords.get(normalize_text(entity['value']))
|
|
if key is None:
|
|
issues.append('keyword_not_in_frontmatter_catalog')
|
|
elif any(not any(key in doc_keywords[path] for path in person_docs[person['id']])
|
|
for person in required_people):
|
|
issues.append('keyword_not_linked_to_expected_person')
|
|
elif entity['type'] in {'thesis_title', 'work_title'}:
|
|
records = payload['theses'] if entity['type'] == 'thesis_title' else payload['works']
|
|
paths = {person['source_document'] for person in starts}
|
|
if not any(row['source_document'] in paths
|
|
and normalize_text(row.get('title')) == normalize_text(entity['value'])
|
|
for row in records):
|
|
issues.append('title_not_supported_by_source')
|
|
category = question['category']
|
|
if category == 'person_multiple_categories' and starts:
|
|
actual = {normalize_text(cat) for path in person_docs[starts[0]['id']]
|
|
for cat in documents[path]['categories']}
|
|
expected_set = {normalize_text(value) for value in graph.get('expected_entity_set', [])}
|
|
if graph.get('entity_set_mode') == 'exact' and actual != expected_set:
|
|
issues.append('exact_categories_omit_or_add_source_categories')
|
|
years = {int(value) for value in re.findall(r'\b20\d{2}\b', question['question'])}
|
|
if category == 'same_academic_year_pair':
|
|
expected_years = {int(item['value']) for item in expected if item['type'] == 'year'}
|
|
if any(not expected_years <= {r['year'] for r in academic[p['source_document']]}
|
|
for p in starts):
|
|
issues.append('shared_year_not_supported_by_source')
|
|
if category == 'person_title_year':
|
|
expected_years = {int(item['value']) for item in expected if item['type'] == 'year'}
|
|
titles = {normalize_text(item['value']) for item in expected
|
|
if item['type'] in {'thesis_title', 'work_title'}}
|
|
if starts and not any(normalize_text(r.get('title')) in titles and r['year'] in expected_years
|
|
for p in starts for r in academic[p['source_document']]):
|
|
issues.append('title_year_pair_not_supported_by_source')
|
|
if category == 'candidate_set_aggregation':
|
|
actual = {normalize_text(p['name']) for p in starts
|
|
if any(r['year'] in years for r in academic[p['source_document']])}
|
|
expected_set = {normalize_text(value) for value in graph.get('expected_entity_set', [])}
|
|
if actual != expected_set or len(actual) != graph.get('expected_count'):
|
|
issues.append('candidate_set_or_count_not_supported_by_source')
|
|
for path in question.get('expected_documents', []):
|
|
if path not in documents:
|
|
issues.append('unknown_expected_document')
|
|
rows.append({'id': question['id'], 'split': question['split'],
|
|
'category': category, 'source_consistent': not issues,
|
|
'issues': sorted(set(issues))})
|
|
return {
|
|
'method': 'Static source checks; no retrieval output, question rewriting or alias invention.',
|
|
'questions': rows,
|
|
'by_split': {
|
|
split: {'questions': len(selected),
|
|
'source_consistent': sum(row['source_consistent'] for row in selected),
|
|
'issues': dict(Counter(issue for row in selected for issue in row['issues']))}
|
|
for split in ['dev', 'test']
|
|
for selected in [[row for row in rows if row['split'] == split]]
|
|
},
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--questions', type=Path, default=Path(__file__).parent / 'graphrag_questions.json')
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
questions = json.loads(args.questions.read_text())
|
|
payload = build_graph_payload(json.loads((ROOT / 'data/documents.json').read_text()),
|
|
json.loads((ROOT / 'data/chunks.json').read_text()))
|
|
audit = audit_questions(questions, payload)
|
|
audit['dataset_sha256'] = hashlib.sha256(args.questions.read_bytes()).hexdigest()
|
|
audit['inputs'] = content_fingerprints(args.questions)['inputs']
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(audit, ensure_ascii=False, indent=2) + '\n')
|
|
print(json.dumps(audit['by_split'], ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|