"""Render a truthful coverage notice without altering the model's facts. No gold or source oracle is accepted. Unsupported citations and unqualified exhaustive statements remain rejected. Keep the exact original model answer. """ import copy def finalize_rag_answer(result: dict) -> dict: out = copy.deepcopy(result) if out.get('display_notice_added'): return out issues = out.get('context_issues', []) out['model_answer'] = out['answer'] out['first_pass_accepted_for_delivery'] = out.get('accepted_for_delivery') out['first_pass_context_issues'] = list(issues) out['display_notice_added'] = False # Only a missing notice can be rendered automatically. Adding a caveat to # a contradictory exhaustive statement would not repair that statement. if issues == ['missing_incompleteness_notice'] and out.get('citations_ok') is True: out['answer'] = 'Výpis je neúplný.\n\n' + out['model_answer'] out['context_issues'] = [] out['accepted_for_delivery'] = True out['display_notice_added'] = True return out def main(): import argparse import json from pathlib import Path from tools.graphrag.evaluation.reproducibility import sha256_file from tools.rag.evaluation.rag_metrics import evaluate_answer parser = argparse.ArgumentParser(description='Replay saved RAG outputs through the delivery layer; no model calls.') parser.add_argument('--input', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() if args.output.exists(): parser.error('Refusing to overwrite a replay.') original = json.loads(args.input.read_text()) source = Path(original['configuration']['source_file']) assert sha256_file(source) == original['configuration']['source_sha256'] gold = {case['id']: case for case in json.loads(source.read_text())['results']} files = [Path(__file__), Path('tools/rag/evaluation/evaluate_rag_answers.py'), Path('tools/rag/evaluation/rag_metrics.py'), Path('tools/rag/evaluation/rag_runner.py'), Path('tools/answer_evaluation/context_policy.py')] rows = [] for before in original['results']: after = finalize_rag_answer(before) score = evaluate_answer(gold[after['id']], after['answer'], tool_called=after['tool_called'], allowed_source_urls=after['rag_source_urls']) after.update(score) after['strict_pass'] = bool(score['strict_pass'] and after['accepted_for_delivery']) rows.append(after) result = {'configuration': {'method': 'Deterministic delivery replay on the same saved model responses, not new generation.', 'input': str(args.input), 'input_sha256': sha256_file(args.input), 'delivery_files': {str(p): sha256_file(p) for p in files}}, 'results': rows, 'summary': {'questions': len(rows), 'strict_pass': sum(r['strict_pass'] for r in rows), 'display_notice_added': sum(r['display_notice_added'] for r in rows)}} args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') print(json.dumps(result['summary'])) if __name__ == '__main__': main()