50 lines
2.9 KiB
Python
50 lines
2.9 KiB
Python
"""Five preselected historical RAG smoke cases; not a new accuracy estimate."""
|
|
import argparse
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from tools.rag.evaluation.rag_runner import build_rag_tool, request_json, run_question, LOCAL_OPENAPI_URL, DEFAULT_MODEL
|
|
from tools.rag.evaluation.rag_metrics import evaluate_answer
|
|
from tools.graphrag.evaluation.reproducibility import sha256_file
|
|
from .rag_delivery import finalize_rag_answer
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.output.exists(): parser.error('Refusing to overwrite smoke results.')
|
|
source = Path('tools/rag/evaluation/results/test_classic_rag_frozen/rag_answers_test.json')
|
|
historic = json.loads(source.read_text())['results']
|
|
selected = ['q0053', 'q0060', 'q0608', 'q1147', 'q0888']
|
|
cases = [next(row for row in historic if row['id'] == id) for id in selected]
|
|
operation, tool = build_rag_tool(request_json(LOCAL_OPENAPI_URL))
|
|
metadata = {'method': __doc__, 'selected_ids': selected, 'model': DEFAULT_MODEL,
|
|
'source_file': str(source), 'source_sha256': sha256_file(source),
|
|
'generation_files': {str(p): sha256_file(p) for p in [Path(__file__),
|
|
Path('tools/rag/evaluation/rag_runner.py'), Path('tools/rag/evaluation/rag_metrics.py'),
|
|
Path('tools/answer_evaluation/context_policy.py'), Path('tools/answer_evaluation/rag_delivery.py')]}}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.with_suffix('.declaration.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2)+'\n')
|
|
def run(case):
|
|
out = run_question(case['question'], model=DEFAULT_MODEL, operation_id=operation,
|
|
rag_tool=tool, openwebui_api_key=os.environ['OPENWEBUI_API_KEY'],
|
|
search_api_key=os.environ['SEARCH_API_KEY'], timeout=120, max_attempts=2)
|
|
out = finalize_rag_answer(out)
|
|
score = evaluate_answer(case, out['answer'], tool_called=out['tool_called'],
|
|
allowed_source_urls=out['rag_source_urls'])
|
|
return {'id': case['id'], 'question': case['question'], **out, **score,
|
|
'strict_pass': bool(score['strict_pass'] and out['accepted_for_delivery'])}
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
|
rows = list(executor.map(run, cases))
|
|
assert metadata['generation_files'] == {p: sha256_file(Path(p)) for p in metadata['generation_files']}
|
|
result = {'configuration': metadata, 'results': rows,
|
|
'summary': {'questions': len(rows), 'strict_pass': sum(row['strict_pass'] for row in rows)}}
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
|
print(json.dumps(result['summary']))
|
|
|
|
|
|
if __name__ == '__main__': main()
|