47 lines
2.7 KiB
Python
47 lines
2.7 KiB
Python
"""Recheck archived citations against archived tool sources, without generation.
|
|
|
|
This changes only citation eligibility; it does not replace historical gold or
|
|
claim a new full semantic evaluation. Missing archived context is unverifiable.
|
|
"""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from tools.graphrag.evaluation.reproducibility import sha256_file
|
|
from .context_policy import citation_check, allowed_urls
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--input', type=Path, action='append', required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.output.exists(): parser.error('Refusing to overwrite regrading.')
|
|
reports = []
|
|
for path in args.input:
|
|
data = json.loads(path.read_text()); rows = []
|
|
for old in data.get('questions', data.get('results', [])):
|
|
answer = old.get('answer') or ''
|
|
if isinstance(answer, str): answer = {'answer': answer, 'source_urls': []}
|
|
context = old.get('graph_context')
|
|
legacy_sources = ({row['source_url'] for row in context.get('documents', []) if row.get('source_url')}
|
|
if isinstance(context, dict) and 'documents' in context else None)
|
|
urls = (allowed_urls(context) if isinstance(context, dict) and 'sources' in context else
|
|
legacy_sources if legacy_sources is not None else
|
|
set(old['rag_source_urls']) if isinstance(old.get('rag_source_urls'), list) else None)
|
|
check = citation_check(answer, urls) if urls is not None else {'citations_ok': None, 'unsupported_urls': []}
|
|
strict = bool(old.get('strict', old.get('strict_pass', False)))
|
|
rows.append({'id': old['id'], 'previous_strict': strict, **check,
|
|
'citation_adjusted_strict': (strict and check['citations_ok']) if urls is not None else None})
|
|
reports.append({'file': str(path), 'sha256': sha256_file(path), 'rows': rows,
|
|
'summary': {'questions': len(rows), 'checked': sum(r['citations_ok'] is not None for r in rows),
|
|
'invalid_citations': sum(r['citations_ok'] is False for r in rows),
|
|
'previous_strict': sum(r['previous_strict'] for r in rows),
|
|
'strict_after_citation_check': sum(r['citation_adjusted_strict'] is True for r in rows)}})
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps({'method': __doc__, 'files': reports}, ensure_ascii=False, indent=2)+'\n')
|
|
print(json.dumps([{'file': r['file'], **r['summary']} for r in reports], ensure_ascii=False))
|
|
|
|
|
|
if __name__ == '__main__': main()
|