"""Supplementary LLM check with graph context supplied by the local harness. This exercises the school's OpenWebUI chat API and generated claims, NOT the WebUI browser, installed tools, or autonomous routing. Gold never enters a request. The primary experiment remains the complete frozen retrieval TEST. """ from __future__ import annotations import argparse import json import os import time from collections import defaultdict from pathlib import Path from tools.rag.evaluation.rag_runner import DEFAULT_MODEL, OPENWEBUI_URL, request_json from ..graph_db import create_neo4j_driver, get_neo4j_settings from ..graph_search import search_graph_with_session from .benchmark_v2 import BASE, academics, supporting_documents from .reproducibility import graph_fingerprint, run_metadata, verify_freeze from .source_oracle import load_source_oracle, norm from .source_validation import claim_issues SYSTEM = '''Odpovedaj po slovensky výhradne podľa dodaného výsledku grafového vyhľadávania. Nedoplňuj vedomosti ani údaje z otázky bez potvrdenia v kontexte. Pri no_answer oznám, že údaj nebol nájdený; nevymýšľaj názov. Zachovaj presné znenie mien, názvov, tagov a kategórií. Rozlišuj rok práce (year) od začiatku štúdia (start_year), autora dokumentu od študenta. Vráť JSON objekt s poľami answer (stručný prirodzený text), status (ok/no_answer/ambiguous), claims (zoznam objektov s person a len relevantnými faktami: title, year, start_year, keyword, category, document_author), source_urls (zoznam citovaných URL), count (číslo alebo null). Každá claim musí patriť jednej osobe; nespoj podmienky z rôznych osôb. Pri porovnaní uveď samostatné claims pre každú osobu; pri počte iba zodpovedajúcich kandidátov. Pri otázke na názov uveď aj rok, ak je známy. Pri no_answer vráť prázdne claims. V odpovedi aj source_urls cituj skutočné relevantné zdroje z kontextu.''' def grade_claims(case: dict, answer: dict, oracle: dict) -> dict: graph = case['graph']; claims = answer.get('claims', []) if not isinstance(claims, list) or any(not isinstance(c, dict) for c in claims): return {'strict': False, 'reason': 'invalid_claim_schema'} fields = ['person', 'title', 'year', 'start_year', 'keyword', 'category', 'document_author'] def matches(binding, claim): return all(norm(claim.get(k, '')) == norm(binding[k]) for k in fields if k in binding) covered = all(any(matches(b, c) for c in claims) for b in graph['expected_bindings']) names = {norm(name): name for name in oracle['people']} unsupported = [] for claim in claims: person = names.get(norm(claim.get('person'))) valid = person is not None if valid and 'start_year' in claim: valid = norm(claim['start_year']) == norm(oracle['people'][person]['start_year']) metadata = {k: claim[k] for k in ['keyword', 'category', 'document_author'] if claim.get(k) is not None} if valid and metadata: valid = all(supporting_documents(oracle, person, **{k: v}) for k, v in metadata.items()) if 'category' in metadata and 'document_author' in metadata: valid = valid and bool(supporting_documents(oracle, person, **metadata)) if valid and any(claim.get(k) is not None for k in ['title', 'year']): records = academics(oracle, person) valid = any(all(norm(r.get(k)) == norm(claim[k]) for k in ['title', 'year'] if claim.get(k) is not None) for r in records) if not valid: unsupported.append(claim) elif claim_issues(claim, oracle): unsupported.append(claim) urls = answer.get('source_urls', []) sources_ok = isinstance(urls, list) and all(any(a['source_url'] in urls for a in proof['any_of']) for proof in graph.get('expected_evidence_documents', [])) negative = graph.get('expected_no_answer', False) status_ok = (answer.get('status') == 'no_answer' and not claims) if negative else answer.get('status') == 'ok' count_ok = 'expected_count' not in graph or answer.get('count') == graph['expected_count'] set_ok = True if graph.get('expected_entity_set') is not None: field = graph['entity_set_field'] actual = {norm(c[field]) for c in claims if c.get(field) is not None} set_ok = actual == {norm(v) for v in graph['expected_entity_set']} return {'strict': bool(covered and not unsupported and sources_ok and status_ok and count_ok and set_ok), 'bindings_ok': covered, 'unsupported_claims': unsupported, 'sources_ok': sources_ok, 'status_ok': status_ok, 'count_ok': count_ok, 'set_ok': set_ok} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--questions', type=Path, default=BASE / 'graphrag_questions_v4.json') parser.add_argument('--freeze-manifest', type=Path, required=True) parser.add_argument('--audit', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--per-category', type=int, default=2) parser.add_argument('--model', default=DEFAULT_MODEL) args = parser.parse_args() if args.output.exists(): parser.error('Refusing to overwrite a previous external measurement.') if not os.getenv('OPENWEBUI_API_KEY'): parser.error('OPENWEBUI_API_KEY is not configured.') questions = json.loads(args.questions.read_text()); selected = []; counts = defaultdict(int) for case in questions: if case['split'] == 'test' and counts[case['category']] < args.per_category: selected.append(case); counts[case['category']] += 1 oracle = load_source_oracle(); manifest = json.loads(args.freeze_manifest.read_text()) settings = get_neo4j_settings(); rows = [] payload = {'method': 'OpenWebUI API generation with locally supplied GraphRAG context; no autonomous tool routing.', 'model': args.model, 'selected_ids': [q['id'] for q in selected], 'system_prompt': SYSTEM, 'limitations': ['Small stratified supplementary sample; not the primary 320-case accuracy.', 'Structured claims, source citations and binding checks; free prose requires separate review.'], 'questions': rows} args.output.parent.mkdir(parents=True, exist_ok=True) with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: metadata = run_metadata(args.questions) verify_freeze(manifest, metadata, graph_fingerprint(session), audit=args.audit, **manifest['parameters']) payload['run_metadata'] = metadata for case in selected: started = time.perf_counter() result = search_graph_with_session(session, case['question'], **manifest['parameters']) context = {k: result[k] for k in ['status', 'reason', 'facts', 'documents', 'aggregation']} row = {'id': case['id'], 'category': case['category'], 'question': case['question'], 'graph_context': context, 'strict': False} try: response = request_json(OPENWEBUI_URL, method='POST', timeout=90, max_attempts=1, headers={'Authorization': f'Bearer {os.environ["OPENWEBUI_API_KEY"]}', 'Content-Type': 'application/json'}, payload={'model': args.model, 'stream': False, 'response_format': {'type': 'json_object'}, 'messages': [{'role': 'system', 'content': SYSTEM}, {'role': 'user', 'content': json.dumps({'question': case['question'], 'graph_context': context}, ensure_ascii=False)}]}) row['response'] = response text = response['choices'][0]['message']['content'].strip() if text.startswith('```'): text = text.split('\n', 1)[1].rsplit('```', 1)[0] answer = json.loads(text) row['answer'] = answer; row.update(grade_claims(case, answer, oracle)) except Exception as error: row['error'] = f'{type(error).__name__}: {str(error)[:500]}' row['latency_seconds'] = time.perf_counter() - started rows.append(row) payload['summary'] = {'questions': len(rows), 'correct': sum(r['strict'] for r in rows), 'accuracy': sum(r['strict'] for r in rows) / len(rows)} args.output.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + '\n') print(f'{case["id"]}: {"OK" if row["strict"] else "FAIL"} ({row["latency_seconds"]:.1f}s)', flush=True) verify_freeze(manifest, run_metadata(args.questions), graph_fingerprint(session), audit=args.audit, **manifest['parameters']) print(json.dumps(payload['summary'])) if __name__ == '__main__': main()