"""Check source consistency without changing questions or consulting search output.""" from __future__ import annotations import argparse import hashlib import json import re from collections import Counter, defaultdict from pathlib import Path from typing import Any from ..graphrag import build_graph_payload from .graphrag_evaluator import normalize_text from .reproducibility import content_fingerprints ROOT = Path(__file__).resolve().parents[3] def audit_questions(questions: list[dict], payload: dict) -> dict[str, Any]: people = {normalize_text(row['name']): row for row in payload['people']} documents = {row['path']: row for row in payload['documents']} person_docs: dict[str, set[str]] = defaultdict(set) for row in payload['person_documents']: person_docs[row['person_id']].add(row['document_path']) doc_keywords: dict[str, set[str]] = defaultdict(set) for row in payload['document_keywords']: doc_keywords[row['document_path']].add(row['keyword_id']) keywords = {normalize_text(row['name']): row['id'] for row in payload['keywords']} keywords.update({normalize_text(row['id']): row['id'] for row in payload['keywords']}) academic: dict[str, list[dict]] = defaultdict(list) for row in payload['theses'] + payload['works']: academic[row['source_document']].append(row) rows = [] for question in questions: graph = question['graph'] issues = [] starts = [people.get(normalize_text(item['value'])) for item in graph.get('start_entities', []) if item['type'] == 'person'] if any(person is None for person in starts): issues.append('unknown_start_person') starts = [person for person in starts if person] expected = graph.get('expected_entities', []) targets = [people.get(normalize_text(item['value'])) for item in expected if item['type'] == 'person'] targets = [person for person in targets if person] or starts required_people = starts if question['category'] == 'shared_keyword_pair' else targets for entity in expected: if entity['type'] == 'keyword': key = keywords.get(normalize_text(entity['value'])) if key is None: issues.append('keyword_not_in_frontmatter_catalog') elif any(not any(key in doc_keywords[path] for path in person_docs[person['id']]) for person in required_people): issues.append('keyword_not_linked_to_expected_person') elif entity['type'] in {'thesis_title', 'work_title'}: records = payload['theses'] if entity['type'] == 'thesis_title' else payload['works'] paths = {person['source_document'] for person in starts} if not any(row['source_document'] in paths and normalize_text(row.get('title')) == normalize_text(entity['value']) for row in records): issues.append('title_not_supported_by_source') category = question['category'] if category == 'person_multiple_categories' and starts: actual = {normalize_text(cat) for path in person_docs[starts[0]['id']] for cat in documents[path]['categories']} expected_set = {normalize_text(value) for value in graph.get('expected_entity_set', [])} if graph.get('entity_set_mode') == 'exact' and actual != expected_set: issues.append('exact_categories_omit_or_add_source_categories') years = {int(value) for value in re.findall(r'\b20\d{2}\b', question['question'])} if category == 'same_academic_year_pair': expected_years = {int(item['value']) for item in expected if item['type'] == 'year'} if any(not expected_years <= {r['year'] for r in academic[p['source_document']]} for p in starts): issues.append('shared_year_not_supported_by_source') if category == 'person_title_year': expected_years = {int(item['value']) for item in expected if item['type'] == 'year'} titles = {normalize_text(item['value']) for item in expected if item['type'] in {'thesis_title', 'work_title'}} if starts and not any(normalize_text(r.get('title')) in titles and r['year'] in expected_years for p in starts for r in academic[p['source_document']]): issues.append('title_year_pair_not_supported_by_source') if category == 'candidate_set_aggregation': actual = {normalize_text(p['name']) for p in starts if any(r['year'] in years for r in academic[p['source_document']])} expected_set = {normalize_text(value) for value in graph.get('expected_entity_set', [])} if actual != expected_set or len(actual) != graph.get('expected_count'): issues.append('candidate_set_or_count_not_supported_by_source') for path in question.get('expected_documents', []): if path not in documents: issues.append('unknown_expected_document') rows.append({'id': question['id'], 'split': question['split'], 'category': category, 'source_consistent': not issues, 'issues': sorted(set(issues))}) return { 'method': 'Static source checks; no retrieval output, question rewriting or alias invention.', 'questions': rows, 'by_split': { split: {'questions': len(selected), 'source_consistent': sum(row['source_consistent'] for row in selected), 'issues': dict(Counter(issue for row in selected for issue in row['issues']))} for split in ['dev', 'test'] for selected in [[row for row in rows if row['split'] == split]] }, } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--questions', type=Path, default=Path(__file__).parent / 'graphrag_questions.json') parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() questions = json.loads(args.questions.read_text()) payload = build_graph_payload(json.loads((ROOT / 'data/documents.json').read_text()), json.loads((ROOT / 'data/chunks.json').read_text())) audit = audit_questions(questions, payload) audit['dataset_sha256'] = hashlib.sha256(args.questions.read_bytes()).hexdigest() audit['inputs'] = content_fingerprints(args.questions)['inputs'] args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(audit, ensure_ascii=False, indent=2) + '\n') print(json.dumps(audit['by_split'], ensure_ascii=False, indent=2)) if __name__ == '__main__': main()