"""Five preselected historical RAG smoke cases; not a new accuracy estimate."""
import argparse
import concurrent.futures
import json
import os
from pathlib import Path

from tools.rag.evaluation.rag_runner import build_rag_tool, request_json, run_question, LOCAL_OPENAPI_URL, DEFAULT_MODEL
from tools.rag.evaluation.rag_metrics import evaluate_answer
from tools.graphrag.evaluation.reproducibility import sha256_file
from .rag_delivery import finalize_rag_answer


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output', type=Path, required=True)
    args = parser.parse_args()
    if args.output.exists(): parser.error('Refusing to overwrite smoke results.')
    source = Path('tools/rag/evaluation/results/test_classic_rag_frozen/rag_answers_test.json')
    historic = json.loads(source.read_text())['results']
    selected = ['q0053', 'q0060', 'q0608', 'q1147', 'q0888']
    cases = [next(row for row in historic if row['id'] == id) for id in selected]
    operation, tool = build_rag_tool(request_json(LOCAL_OPENAPI_URL))
    metadata = {'method': __doc__, 'selected_ids': selected, 'model': DEFAULT_MODEL,
                'source_file': str(source), 'source_sha256': sha256_file(source),
                'generation_files': {str(p): sha256_file(p) for p in [Path(__file__),
                    Path('tools/rag/evaluation/rag_runner.py'), Path('tools/rag/evaluation/rag_metrics.py'),
                    Path('tools/answer_evaluation/context_policy.py'), Path('tools/answer_evaluation/rag_delivery.py')]}}
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.with_suffix('.declaration.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2)+'\n')
    def run(case):
        out = run_question(case['question'], model=DEFAULT_MODEL, operation_id=operation,
            rag_tool=tool, openwebui_api_key=os.environ['OPENWEBUI_API_KEY'],
            search_api_key=os.environ['SEARCH_API_KEY'], timeout=120, max_attempts=2)
        out = finalize_rag_answer(out)
        score = evaluate_answer(case, out['answer'], tool_called=out['tool_called'],
                                allowed_source_urls=out['rag_source_urls'])
        return {'id': case['id'], 'question': case['question'], **out, **score,
                'strict_pass': bool(score['strict_pass'] and out['accepted_for_delivery'])}
    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
        rows = list(executor.map(run, cases))
    assert metadata['generation_files'] == {p: sha256_file(Path(p)) for p in metadata['generation_files']}
    result = {'configuration': metadata, 'results': rows,
              'summary': {'questions': len(rows), 'strict_pass': sum(row['strict_pass'] for row in rows)}}
    args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
    print(json.dumps(result['summary']))


if __name__ == '__main__': main()