66 lines
3.3 KiB
Python
66 lines
3.3 KiB
Python
"""Render a truthful coverage notice without altering the model's facts.
|
|
|
|
No gold or source oracle is accepted. Unsupported citations and unqualified
|
|
exhaustive statements remain rejected. Keep the exact original model answer.
|
|
"""
|
|
import copy
|
|
|
|
|
|
def finalize_rag_answer(result: dict) -> dict:
|
|
out = copy.deepcopy(result)
|
|
if out.get('display_notice_added'):
|
|
return out
|
|
issues = out.get('context_issues', [])
|
|
out['model_answer'] = out['answer']
|
|
out['first_pass_accepted_for_delivery'] = out.get('accepted_for_delivery')
|
|
out['first_pass_context_issues'] = list(issues)
|
|
out['display_notice_added'] = False
|
|
# Only a missing notice can be rendered automatically. Adding a caveat to
|
|
# a contradictory exhaustive statement would not repair that statement.
|
|
if issues == ['missing_incompleteness_notice'] and out.get('citations_ok') is True:
|
|
out['answer'] = 'Výpis je neúplný.\n\n' + out['model_answer']
|
|
out['context_issues'] = []
|
|
out['accepted_for_delivery'] = True
|
|
out['display_notice_added'] = True
|
|
return out
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from tools.graphrag.evaluation.reproducibility import sha256_file
|
|
from tools.rag.evaluation.rag_metrics import evaluate_answer
|
|
|
|
parser = argparse.ArgumentParser(description='Replay saved RAG outputs through the delivery layer; no model calls.')
|
|
parser.add_argument('--input', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.output.exists(): parser.error('Refusing to overwrite a replay.')
|
|
original = json.loads(args.input.read_text())
|
|
source = Path(original['configuration']['source_file'])
|
|
assert sha256_file(source) == original['configuration']['source_sha256']
|
|
gold = {case['id']: case for case in json.loads(source.read_text())['results']}
|
|
files = [Path(__file__), Path('tools/rag/evaluation/evaluate_rag_answers.py'),
|
|
Path('tools/rag/evaluation/rag_metrics.py'), Path('tools/rag/evaluation/rag_runner.py'),
|
|
Path('tools/answer_evaluation/context_policy.py')]
|
|
rows = []
|
|
for before in original['results']:
|
|
after = finalize_rag_answer(before)
|
|
score = evaluate_answer(gold[after['id']], after['answer'], tool_called=after['tool_called'],
|
|
allowed_source_urls=after['rag_source_urls'])
|
|
after.update(score)
|
|
after['strict_pass'] = bool(score['strict_pass'] and after['accepted_for_delivery'])
|
|
rows.append(after)
|
|
result = {'configuration': {'method': 'Deterministic delivery replay on the same saved model responses, not new generation.',
|
|
'input': str(args.input), 'input_sha256': sha256_file(args.input),
|
|
'delivery_files': {str(p): sha256_file(p) for p in files}},
|
|
'results': rows, 'summary': {'questions': len(rows), 'strict_pass': sum(r['strict_pass'] for r in rows),
|
|
'display_notice_added': sum(r['display_notice_added'] for r in rows)}}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
|
print(json.dumps(result['summary']))
|
|
|
|
|
|
if __name__ == '__main__': main()
|