"""Audit the real checked RAG path on a predeclared, known regression sample. Default mode only retrieves locally. `--mode deliver` explicitly invokes the configured backend model. Expected answers are used here, never sent to /answer. """ from __future__ import annotations import argparse from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed import hashlib import json import os from pathlib import Path import time import urllib.request from dotenv import load_dotenv from tools.answer_evaluation.context_policy import normalized, citation_check, allowed_urls from tools.answer_evaluation.rag_evidence import rag_evidence_context from tools.answer_evaluation.verified_rag import rag_answer_issues, render_rag_answer from tools.graphrag.evaluation.source_oracle import DEFAULT_WIKI, load_source_oracle from tools.rag.evaluation.rag_metrics import expected_phrase_matches from tools.runtime.report_files import report_output, resolve_report_input ROOT = Path(__file__).resolve().parents[2] DATASET = ROOT / 'tools/rag/evaluation/json_files/verified_rag_regression.json' FIELDS = ['person', 'work_type', 'title', 'year', 'start_year'] def digest(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def binding_matches(expected: dict, rows: list[dict]) -> bool: # Study start can be a separate fact about the same person; the academic # fields must all occur together in one actual academic row. academic = {k: expected[k] for k in FIELDS if k in expected and k != 'start_year'} academic_ok = any(all(normalized(row.get(k)) == normalized(v) for k, v in academic.items()) for row in rows) start_ok = 'start_year' not in expected or any( normalized(row.get('person')) == normalized(expected['person']) and row.get('start_year') == expected['start_year'] for row in rows) return academic_ok and start_ok def source_audit(case: dict, oracle: dict) -> list[str]: issues = [] for item in case['source_checks']: if digest(DEFAULT_WIKI / item['document_path']) != item['sha256']: issues.append('changed_source_document') for binding in case['expected_bindings']: person = oracle['people'].get(binding['person'], {}) rows = [{'person': binding['person'], 'work_type': a['academic_type'], **a} for a in person.get('academics', []) if a['document_path'] == binding['document_path']] rows.append({'person': binding['person'], 'start_year': person.get('start_year')}) if not binding_matches(binding, rows): issues.append('gold_binding_not_in_original_markdown') if 'start_year' in binding and person.get('start_year_basis') != 'explicit_text': issues.append('study_start_not_explicit') return issues def evaluate_case(case: dict, payload: dict, delivered: dict | None = None) -> dict: context = rag_evidence_context(case['question'], payload) source_urls = allowed_urls(payload) missing = [b for b in case['expected_bindings'] if not binding_matches(b, context['facts'])] retrieval = {'expected_sources_found': bool(source_urls & set(case['expected_source_urls'])), 'missing_bindings': missing, 'expected_values_in_context': [v for v in case['expected_answer_contains'] if expected_phrase_matches(v, payload.get('context', ''))]} row = {'id': case['id'], 'category': case['category'], 'question': case['question'], 'should_answer': case['should_answer'], 'retrieval': retrieval, 'typed_expected': bool(case['expected_bindings']), 'index_version': payload.get('index_version')} if delivered is None: return row issues = [] accepted = delivered.get('accepted_for_delivery') is True status = delivered.get('status') attempts = delivered.get('attempts', []) parsed = attempts[-1].get('parsed_answer') if attempts else None if accepted and parsed: issues.extend(rag_answer_issues(parsed, context)) if delivered.get('answer') != render_rag_answer(parsed, context): issues.append('delivered_text_differs_from_verified_renderer') if delivered.get('claims', []) != parsed.get('claims', []): issues.append('delivered_claims_differ_from_verified_selection') if accepted and not citation_check(delivered, source_urls)['citations_ok']: issues.append('delivered_unsupported_citation') claims = delivered.get('claims', []) if accepted: # Validate every delivered claim against the actual retrieved tuple. # Additional supported projects of the same person are not a wrong # thesis year. Required benchmark tuples are checked separately below. issues.extend(rag_answer_issues({'status':status, 'claims':claims, 'source_urls':delivered.get('source_urls', []), 'evidence_quotes':(parsed or {}).get('evidence_quotes', [])}, context)) if case['should_answer']: if not accepted: issues.append('delivery_blocked') elif status != 'ok': issues.append('false_no_answer' if status == 'no_answer' else 'nonpositive_answer') if not all(binding_matches(b, claims) for b in case['expected_bindings']): issues.append('missing_expected_binding') # Exclude sources from factual scoring: a name/year in a URL cannot # supply the requested fact. Quotes must cover the expected subject. text = delivered.get('answer', '').split('\nZdroje:', 1)[0] missing_values = [v for v in case['expected_answer_contains'] if not expected_phrase_matches(v, text)] if missing_values: issues.append('missing_expected_answer_value') quotes = (parsed or {}).get('evidence_quotes', []) if quotes: content = '\n'.join(q['quote'] for q in quotes) names = {s.get('title') for s in context['source_blocks']} # Names in source headers are attribution, not answers to a factual # question. Value coverage checks relevance beyond literal copying. relevant = [v for v in case['expected_answer_contains'] if v not in names] row['quote_expected_value_coverage'] = {v: expected_phrase_matches(v, content) for v in relevant} elif not accepted or status != 'no_answer': issues.append('incorrect_negative_answer') row.update(passed=not issues, issues=sorted(set(issues)), delivered=delivered, first_attempt_accepted=bool(attempts and not attempts[0].get('issues'))) return row def summarize(rows: list[dict]) -> dict: counted = [r for r in rows if 'passed' in r] total = len(counted) passed = sum(r['passed'] for r in counted) return {'total': len(rows), 'answers_evaluated': total, 'passed': passed, 'success_percent': round(100 * passed / total, 2) if total else None, 'false_no_answers': sum('false_no_answer' in r['issues'] for r in counted), 'blocked_deliveries': sum('delivery_blocked' in r['issues'] for r in counted), 'first_attempt_accepted': sum(r.get('first_attempt_accepted', False) for r in counted), 'failure_reasons': dict(Counter(i for r in counted for i in r['issues'])), 'source_coverage': sum(r['retrieval']['expected_sources_found'] for r in rows), 'answerable_source_coverage': {'passed': sum(r['retrieval']['expected_sources_found'] for r in rows if r['should_answer']), 'total': sum(r['should_answer'] for r in rows)}, 'typed_binding_coverage': {'passed': sum(not r['retrieval']['missing_bindings'] for r in rows if r.get('typed_expected')), 'total': sum(r.get('typed_expected', False) for r in rows)}, 'categories': {c: {'total': sum(r['category'] == c for r in rows), 'passed': sum(r.get('passed', False) for r in counted if r['category'] == c)} for c in sorted({r['category'] for r in rows})}} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--mode', choices=['retrieve', 'deliver', 'replay'], default='retrieve') parser.add_argument('--input', type=Path, help='Saved report to regrade without model calls') parser.add_argument('--dataset', type=Path, default=DATASET) parser.add_argument('--backend', default='http://127.0.0.1:8000') parser.add_argument('--output', type=Path) args = parser.parse_args() load_dotenv(ROOT / '.env', override=False) dataset = json.loads(args.dataset.read_text()) assert digest(ROOT / dataset['source_file']) == dataset['source_sha256'], 'Historical baseline changed' oracle = load_source_oracle() audit = {c['id']: source_audit(c, oracle) for c in dataset['cases']} assert not any(audit.values()), 'Expected answer/source audit failed' output = report_output(args.output, 'rag_verified') files = [Path(__file__), ROOT/'tools/answers/answer_delivery.py', ROOT/'app/routes.py', *[(ROOT/'tools/answer_evaluation'/name) for name in ['verified_rag.py','rag_evidence.py','context_policy.py']], ROOT/'tools/rag/rag_utils.py', ROOT/'tools/rag/llm_context.py'] metadata = {'method': dataset['method'], 'mode': args.mode, 'dataset_sha256': digest(args.dataset), 'limitations': dataset.get('limitations', []), 'source_audit': audit, 'source_corpus': oracle['source_corpus'], 'code_hashes': {str(p.relative_to(ROOT)): digest(p) for p in files}, 'model_calls': args.mode == 'deliver', 'backend': args.backend, 'model': os.getenv('ANSWER_MODEL', 'model120-fast') if args.mode == 'deliver' else None} output.with_suffix('.declaration.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2)+'\n') key = os.getenv('SEARCH_API_KEY', '') saved = None if args.mode == 'replay': if not args.input: parser.error('--input is required for replay') saved = {r['id']: r for r in json.loads(resolve_report_input(args.input).read_text())['results']} def run(case): started = time.monotonic() if saved is not None: old = saved[case['id']] return evaluate_case(case, old['retrieval_context'], old.get('delivered')) | {'retrieval_context': old['retrieval_context']} endpoint = '/answer' if args.mode == 'deliver' else '/rag' body = {'query': case['question'], 'limit': 5} if args.mode == 'deliver': body.update(engine='rag', include_diagnostics=True) req = urllib.request.Request(args.backend.rstrip('/')+endpoint, data=json.dumps(body).encode(), headers={'Content-Type': 'application/json', 'X-API-Key': key}) try: with urllib.request.urlopen(req, timeout=300) as response: data = json.load(response) payload = data.pop('retrieval_context') if args.mode == 'deliver' else data result = evaluate_case(case, payload, data if args.mode == 'deliver' else None) result.update(retrieval_context=payload, seconds=round(time.monotonic()-started, 2)) return result except Exception as error: # Count operational failures in the denominator; never retry only # selected failed questions or expose raw HTTP/token-bearing errors. return {'id': case['id'], 'category': case['category'], 'question': case['question'], 'should_answer': case['should_answer'], 'passed': False, 'issues': ['request_or_evaluation_error'], 'typed_expected': bool(case['expected_bindings']), 'error_type': type(error).__name__, 'retrieval': {'expected_sources_found': False, 'missing_bindings': case['expected_bindings']}} rows = [] with ThreadPoolExecutor(max_workers=2) as pool: futures = {pool.submit(run, case): case for case in dataset['cases']} for future in as_completed(futures): rows.append(future.result()) result = {'configuration': metadata, 'summary': summarize(rows), 'results': sorted(rows, key=lambda r: r['id'])} output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') print(f'{len(rows)}/{len(dataset["cases"])} {rows[-1]["id"]}: {rows[-1].get("issues", "retrieved")}', flush=True) assert metadata['code_hashes'] == {name: digest(ROOT/name) for name in metadata['code_hashes']}, 'Code changed during measurement' print(json.dumps(result['summary'], ensure_ascii=False)) print('Report: '+str(output)) if __name__ == '__main__': main()