230 lines
13 KiB
Python
230 lines
13 KiB
Python
"""Audit the real checked RAG path on a predeclared, known regression sample.
|
|
|
|
Default mode only retrieves locally. `--mode deliver` explicitly invokes the
|
|
configured backend model. Expected answers are used here, never sent to /answer.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.request
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
from tools.answer_evaluation.context_policy import normalized, citation_check, allowed_urls
|
|
from tools.answer_evaluation.rag_evidence import rag_evidence_context
|
|
from tools.answer_evaluation.verified_rag import rag_answer_issues, render_rag_answer, quoted_reference_urls
|
|
from tools.graphrag.evaluation.source_oracle import DEFAULT_WIKI, load_source_oracle
|
|
from tools.rag.evaluation.rag_metrics import expected_phrase_matches
|
|
from tools.runtime.report_files import report_output, resolve_report_input
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
DATASET = ROOT / 'tools/rag/evaluation/json_files/verified_rag_regression.json'
|
|
FIELDS = ['person', 'work_type', 'title', 'year', 'start_year']
|
|
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def binding_matches(expected: dict, rows: list[dict]) -> bool:
|
|
# Study start can be a separate fact about the same person; the academic
|
|
# fields must all occur together in one actual academic row.
|
|
academic = {k: expected[k] for k in FIELDS if k in expected and k != 'start_year'}
|
|
academic_ok = any(all(normalized(row.get(k)) == normalized(v) for k, v in academic.items()) for row in rows)
|
|
start_ok = 'start_year' not in expected or any(
|
|
normalized(row.get('person')) == normalized(expected['person']) and
|
|
row.get('start_year') == expected['start_year'] for row in rows)
|
|
return academic_ok and start_ok
|
|
|
|
|
|
def source_audit(case: dict, oracle: dict) -> list[str]:
|
|
issues = []
|
|
for item in case['source_checks']:
|
|
if digest(DEFAULT_WIKI / item['document_path']) != item['sha256']:
|
|
issues.append('changed_source_document')
|
|
for binding in case['expected_bindings']:
|
|
person = oracle['people'].get(binding['person'], {})
|
|
rows = [{'person': binding['person'], 'work_type': a['academic_type'], **a}
|
|
for a in person.get('academics', []) if a['document_path'] == binding['document_path']]
|
|
rows.append({'person': binding['person'], 'start_year': person.get('start_year')})
|
|
if not binding_matches(binding, rows):
|
|
issues.append('gold_binding_not_in_original_markdown')
|
|
if 'start_year' in binding and person.get('start_year_basis') != 'explicit_text':
|
|
issues.append('study_start_not_explicit')
|
|
return issues
|
|
|
|
|
|
def evaluate_case(case: dict, payload: dict, delivered: dict | None = None) -> dict:
|
|
context = rag_evidence_context(case['question'], payload)
|
|
source_urls = allowed_urls(payload)
|
|
missing = [b for b in case['expected_bindings'] if not binding_matches(b, context['facts'])]
|
|
retrieval = {'expected_sources_found': bool(source_urls & set(case['expected_source_urls'])),
|
|
'missing_bindings': missing,
|
|
'expected_values_in_context': [v for v in case['expected_answer_contains']
|
|
if expected_phrase_matches(v, payload.get('context', ''))]}
|
|
row = {'id': case['id'], 'category': case['category'], 'question': case['question'],
|
|
'should_answer': case['should_answer'], 'retrieval': retrieval,
|
|
'typed_expected': bool(case['expected_bindings']),
|
|
'index_version': payload.get('index_version')}
|
|
if delivered is None:
|
|
return row
|
|
issues = []
|
|
accepted = delivered.get('accepted_for_delivery') is True
|
|
status = delivered.get('status')
|
|
attempts = delivered.get('attempts', [])
|
|
parsed = attempts[-1].get('parsed_answer') if attempts else None
|
|
if accepted and parsed:
|
|
issues.extend(rag_answer_issues(parsed, context))
|
|
if delivered.get('answer') != render_rag_answer(parsed, context):
|
|
issues.append('delivered_text_differs_from_verified_renderer')
|
|
if delivered.get('claims', []) != parsed.get('claims', []):
|
|
issues.append('delivered_claims_differ_from_verified_selection')
|
|
references = quoted_reference_urls(parsed or {}, context)
|
|
if accepted and not citation_check(delivered, source_urls | references)['citations_ok']:
|
|
issues.append('delivered_unsupported_citation')
|
|
claims = delivered.get('claims', [])
|
|
if accepted:
|
|
# Validate every delivered claim against the actual retrieved tuple.
|
|
# Additional supported projects of the same person are not a wrong
|
|
# thesis year. Required benchmark tuples are checked separately below.
|
|
issues.extend(rag_answer_issues({'status':status, 'claims':claims,
|
|
'source_urls':delivered.get('source_urls', []),
|
|
'evidence_quotes':(parsed or {}).get('evidence_quotes', [])}, context))
|
|
if case['should_answer']:
|
|
if not accepted:
|
|
issues.append('delivery_blocked')
|
|
elif status != 'ok':
|
|
issues.append('false_no_answer' if status == 'no_answer' else 'nonpositive_answer')
|
|
if not all(binding_matches(b, claims) for b in case['expected_bindings']):
|
|
issues.append('missing_expected_binding')
|
|
# Exclude sources from factual scoring: a name/year in a URL cannot
|
|
# supply the requested fact. Quotes must cover the expected subject.
|
|
text = delivered.get('answer', '').split('\nZdroje:', 1)[0]
|
|
missing_values = [v for v in case['expected_answer_contains'] if not expected_phrase_matches(v, text)]
|
|
if missing_values:
|
|
issues.append('missing_expected_answer_value')
|
|
quotes = (parsed or {}).get('evidence_quotes', [])
|
|
if quotes:
|
|
content = '\n'.join(q['quote'] for q in quotes)
|
|
names = {s.get('title') for s in context['source_blocks']}
|
|
# Names in source headers are attribution, not answers to a factual
|
|
# question. Value coverage checks relevance beyond literal copying.
|
|
relevant = [v for v in case['expected_answer_contains'] if v not in names]
|
|
row['quote_expected_value_coverage'] = {v: expected_phrase_matches(v, content) for v in relevant}
|
|
elif not accepted or status != 'no_answer':
|
|
issues.append('incorrect_negative_answer')
|
|
row.update(passed=not issues, issues=sorted(set(issues)), delivered=delivered,
|
|
first_attempt_accepted=bool(attempts and not attempts[0].get('issues')))
|
|
return row
|
|
|
|
|
|
def summarize(rows: list[dict]) -> dict:
|
|
counted = [r for r in rows if 'passed' in r]
|
|
total = len(counted)
|
|
passed = sum(r['passed'] for r in counted)
|
|
return {'total': len(rows), 'answers_evaluated': total, 'passed': passed,
|
|
'success_percent': round(100 * passed / total, 2) if total else None,
|
|
'false_no_answers': sum('false_no_answer' in r['issues'] for r in counted),
|
|
'blocked_deliveries': sum('delivery_blocked' in r['issues'] for r in counted),
|
|
'first_attempt_accepted': sum(r.get('first_attempt_accepted', False) for r in counted),
|
|
'failure_reasons': dict(Counter(i for r in counted for i in r['issues'])),
|
|
'source_coverage': sum(r['retrieval']['expected_sources_found'] for r in rows),
|
|
'answerable_source_coverage': {'passed': sum(r['retrieval']['expected_sources_found'] for r in rows if r['should_answer']),
|
|
'total': sum(r['should_answer'] for r in rows)},
|
|
'typed_binding_coverage': {'passed': sum(not r['retrieval']['missing_bindings'] for r in rows if r.get('typed_expected')),
|
|
'total': sum(r.get('typed_expected', False) for r in rows)},
|
|
'categories': {c: {'total': sum(r['category'] == c for r in rows),
|
|
'passed': sum(r.get('passed', False) for r in counted if r['category'] == c)} for c in sorted({r['category'] for r in rows})}}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--mode', choices=['retrieve', 'deliver', 'replay'], default='retrieve')
|
|
parser.add_argument('--input', type=Path, help='Saved report to regrade without model calls')
|
|
parser.add_argument('--dataset', type=Path, default=DATASET)
|
|
parser.add_argument('--backend', default='http://127.0.0.1:8000')
|
|
parser.add_argument('--output', type=Path)
|
|
parser.add_argument('--no-save', action='store_true', help='Print results only; create no reports or logs')
|
|
args = parser.parse_args()
|
|
load_dotenv(ROOT / '.env', override=False)
|
|
dataset = json.loads(args.dataset.read_text())
|
|
assert digest(ROOT / dataset['source_file']) == dataset['source_sha256'], 'Historical baseline changed'
|
|
oracle = load_source_oracle()
|
|
audit = {c['id']: source_audit(c, oracle) for c in dataset['cases']}
|
|
assert not any(audit.values()), 'Expected answer/source audit failed'
|
|
if args.no_save and args.output:
|
|
parser.error('--no-save cannot be combined with --output')
|
|
output = None if args.no_save else report_output(args.output, 'rag_verified')
|
|
files = [Path(__file__), ROOT/'tools/answers/answer_delivery.py', ROOT/'app/routes.py',
|
|
*[(ROOT/'tools/answer_evaluation'/name) for name in ['verified_rag.py','rag_evidence.py','context_policy.py']],
|
|
ROOT/'tools/rag/rag_utils.py', ROOT/'tools/rag/llm_context.py']
|
|
metadata = {'method': dataset['method'], 'mode': args.mode, 'dataset_sha256': digest(args.dataset),
|
|
'limitations': dataset.get('limitations', []),
|
|
'source_audit': audit, 'source_corpus': oracle['source_corpus'],
|
|
'code_hashes': {str(p.relative_to(ROOT)): digest(p) for p in files},
|
|
'model_calls': args.mode == 'deliver', 'backend': args.backend,
|
|
'model': os.getenv('ANSWER_MODEL', 'model120-fast') if args.mode == 'deliver' else None}
|
|
if output:
|
|
output.with_suffix('.declaration.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2)+'\n')
|
|
key = os.getenv('SEARCH_API_KEY', '')
|
|
saved = None
|
|
if args.mode == 'replay':
|
|
if not args.input:
|
|
parser.error('--input is required for replay')
|
|
saved = {r['id']: r for r in json.loads(resolve_report_input(args.input).read_text())['results']}
|
|
|
|
def run(case):
|
|
started = time.monotonic()
|
|
if saved is not None:
|
|
old = saved[case['id']]
|
|
return evaluate_case(case, old['retrieval_context'], old.get('delivered')) | {'retrieval_context': old['retrieval_context']}
|
|
endpoint = '/answer' if args.mode == 'deliver' else '/rag'
|
|
body = {'query': case['question'], 'limit': 5}
|
|
if args.mode == 'deliver':
|
|
body.update(engine='rag', include_diagnostics=True)
|
|
req = urllib.request.Request(args.backend.rstrip('/')+endpoint, data=json.dumps(body).encode(),
|
|
headers={'Content-Type': 'application/json', 'X-API-Key': key})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=300) as response:
|
|
data = json.load(response)
|
|
payload = data.pop('retrieval_context') if args.mode == 'deliver' else data
|
|
result = evaluate_case(case, payload, data if args.mode == 'deliver' else None)
|
|
result.update(retrieval_context=payload, seconds=round(time.monotonic()-started, 2))
|
|
return result
|
|
except Exception as error:
|
|
# Count operational failures in the denominator; never retry only
|
|
# selected failed questions or expose raw HTTP/token-bearing errors.
|
|
return {'id': case['id'], 'category': case['category'], 'question': case['question'],
|
|
'should_answer': case['should_answer'], 'passed': False, 'issues': ['request_or_evaluation_error'],
|
|
'typed_expected': bool(case['expected_bindings']),
|
|
'error_type': type(error).__name__, 'retrieval': {'expected_sources_found': False, 'missing_bindings': case['expected_bindings']}}
|
|
|
|
rows = []
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = {pool.submit(run, case): case for case in dataset['cases']}
|
|
for future in as_completed(futures):
|
|
rows.append(future.result())
|
|
result = {'configuration': metadata, 'summary': summarize(rows), 'results': sorted(rows, key=lambda r: r['id'])}
|
|
if output:
|
|
output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
|
print(f'{len(rows)}/{len(dataset["cases"])} {rows[-1]["id"]}: {rows[-1].get("issues", "retrieved")}', flush=True)
|
|
assert metadata['code_hashes'] == {name: digest(ROOT/name) for name in metadata['code_hashes']}, 'Code changed during measurement'
|
|
print(json.dumps(result['summary'], ensure_ascii=False))
|
|
if output:
|
|
print('Report: '+str(output))
|
|
if args.no_save:
|
|
print(json.dumps({'failures':[{'id':r['id'],'issues':r['issues'],
|
|
'attempt_issues':[a.get('issues') for a in r.get('delivered',{}).get('attempts',[])]}
|
|
for r in result['results'] if not r['passed']]}, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|