"""Evaluate 300 fresh LLM outputs after 20 exposed development outputs. The 320-query graph benchmark stays frozen. This separate experiment supplies the retriever's context to OpenWebUI; it does not test native tool routing. """ from __future__ import annotations import argparse import concurrent.futures import hashlib import json import os import time from collections import Counter from pathlib import Path from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings from tools.graphrag.graph_search import search_graph_with_session from tools.graphrag.evaluation.benchmark_v2 import BASE from tools.graphrag.evaluation.check_openwebui import grade_claims from tools.graphrag.evaluation.reproducibility import graph_fingerprint, run_metadata, sha256_file, verify_freeze from tools.graphrag.evaluation.source_oracle import load_source_oracle, norm from tools.rag.evaluation.rag_runner import DEFAULT_MODEL, OPENWEBUI_URL, request_json HERE = Path(__file__).resolve().parent def graph_context(result: dict) -> dict: """Remove storage/profile metadata that cannot answer the actual question. Uses only actual retrieval output and query intent, never gold annotations. Values/subjects/source paths are copied without inventing or rewriting facts. """ facts = []; intent = result['constraints'] for row in result['facts']: kind = row.get('fact_type', '') if kind.startswith('shared_'): field = {'shared_keyword': 'keyword', 'shared_category': 'category', 'shared_document_author': 'document_author'}.get(kind) if field: for side in ['left', 'right']: facts.append({'person': row[f'{side}_person'], field: row[field], 'document_path': row.get(f'{side}_document_path'), 'source_url': row.get(f'{side}_source_url')}) continue if not row.get('person'): continue allowed = {'person', 'document_path', 'source_url', 'keyword', 'category', 'document_author'} if intent['mentions_title']: allowed.add('title') if intent['mentions_academic_year'] or intent['years']: allowed.add('year') if intent['mentions_start_year']: allowed.add('start_year') fact = {k: v for k, v in row.items() if k in allowed and v is not None} if fact not in facts: facts.append(fact) return {'status': result['status'], 'reason': result['reason'], 'facts': facts, 'sources': [{k: d[k] for k in ['document_path', 'source_url'] if d.get(k)} for d in result['documents']], 'aggregation': result['aggregation']} def grade_answer(case: dict, answer: dict, oracle: dict) -> dict: # JSON null is absence, not a positive factual assertion. The same rule is # applicable to all rounds, including the archived initial sample. normalized = dict(answer) if isinstance(answer.get('claims'), list): normalized['claims'] = [{k: v for k, v in claim.items() if v is not None} if isinstance(claim, dict) else claim for claim in answer['claims']] grade = grade_claims(case, normalized, oracle) text = norm(answer.get('answer', '')) missing = [v for v in case.get('expected_answer_contains', []) if norm(v) not in text] # The initial sample exposed a real contradiction in otherwise correct # category claims. Check it explicitly rather than silently accepting prose. wrong_thesis_label = 'dizertac' in text and any(str(b.get('category', '')).startswith('dp') for b in case['graph']['expected_bindings']) and not any( str(b.get('category', '')).startswith('phd') for b in case['graph']['expected_bindings']) grade.update(text_values_ok=not missing, missing_text_values=missing, contradictory_thesis_label=wrong_thesis_label) grade['strict'] = bool(grade['strict'] and not missing and not wrong_thesis_label) return grade def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output-dir', type=Path, required=True) parser.add_argument('--sample', type=Path, default=BASE / 'results/final_v4_2026-10-25/openwebui_sample.json') parser.add_argument('--workers', type=int, default=3) parser.add_argument('--model', default=DEFAULT_MODEL) parser.add_argument('--split', choices=['dev', 'test'], default='test') args = parser.parse_args() if args.output_dir.exists(): parser.error('Refusing to overwrite an answer-level TEST.') if not os.getenv('OPENWEBUI_API_KEY'): parser.error('OPENWEBUI_API_KEY missing.') dataset = BASE / 'graphrag_questions_v4.json'; audit = BASE / 'results/v4_preparation/source_audit.json' frozen = BASE / 'frozen_v4.json'; manifest = json.loads(frozen.read_text()) questions = json.loads(dataset.read_text()); oracle = load_source_oracle() sample = json.loads(args.sample.read_text()); exposed = set(sample['selected_ids']) test = [q for q in questions if q['split'] == 'test' and q['id'] not in exposed] assert len(exposed) == 20 and len(test) == 300 selected = test if args.split == 'test' else [q for q in questions if q['id'] in exposed] prompt_path = HERE / 'graph_answer_prompt.txt'; prompt = prompt_path.read_text() args.output_dir.mkdir(parents=True) metadata = run_metadata(dataset); settings = get_neo4j_settings(); contexts = {} with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: verify_freeze(manifest, metadata, graph_fingerprint(session), audit=audit, **manifest['parameters']) # Generate every context before making any external request. for case in selected: contexts[case['id']] = graph_context(search_graph_with_session(session, case['question'], **manifest['parameters'])) code_files = [Path(__file__), prompt_path, Path('tools/rag/evaluation/rag_runner.py')] hashes = {str(p): sha256_file(p) for p in code_files} declaration = {'method': 'Fresh LLM-generation holdout over previously measured, frozen graph queries.', 'graph_freeze_sha256': sha256_file(frozen), 'audit_sha256': sha256_file(audit), 'generation_files': hashes, 'model': args.model, 'temperature': 0, 'workers': args.workers, 'dev_ids': sorted(exposed), 'test_ids': [q['id'] for q in test], 'dev_count': 20, 'test_count': 300, 'target_accuracy': 0.92, 'selected_split': args.split, 'selected_ids': [q['id'] for q in selected], 'policy': 'Evaluate all 300; retain every failure; do not tune on this TEST.', 'test_categories': dict(Counter(q['category'] for q in test)), 'limitations': ['Known corpus/profiles and templates; model outputs for these 300 questions were not previously observed.', 'Harness supplies graph context; native OpenWebUI tool choice and browser UI are outside this experiment.', 'Checks structured factual claims, their source support/citations and required prose values; not a general semantic judge of arbitrary prose.']} (args.output_dir / 'frozen_answers.json').write_text(json.dumps(declaration, ensure_ascii=False, indent=2) + '\n') results = {} def run(case): started = time.perf_counter(); context = contexts[case['id']] row = {'id': case['id'], 'category': case['category'], 'question': case['question'], 'graph_context': context, 'strict': False} try: response = request_json(OPENWEBUI_URL, method='POST', timeout=90, max_attempts=2, headers={'Authorization': f'Bearer {os.environ["OPENWEBUI_API_KEY"]}', 'Content-Type': 'application/json'}, payload={'model': args.model, 'temperature': 0, 'stream': False, 'response_format': {'type': 'json_object'}, 'messages': [ {'role': 'system', 'content': prompt}, {'role': 'user', 'content': json.dumps( {'question': case['question'], 'graph_context': context}, ensure_ascii=False)}]}) row['response'] = response text = response['choices'][0]['message']['content'].strip() if text.startswith('```'): text = text.split('\n', 1)[1].rsplit('```', 1)[0] answer = json.loads(text); row['answer'] = answer; row.update(grade_answer(case, answer, oracle)) except Exception as error: row['error'] = f'{type(error).__name__}: {str(error)[:500]}' row['latency_seconds'] = time.perf_counter() - started return row output = {'configuration': declaration, 'run_metadata': metadata, 'questions': []} with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor: futures = [executor.submit(run, case) for case in selected] for future in concurrent.futures.as_completed(futures): row = future.result(); results[row['id']] = row output['questions'] = [results[q['id']] for q in selected if q['id'] in results] output['summary'] = {'questions': len(results), 'correct': sum(r['strict'] for r in results.values()), 'accuracy': sum(r['strict'] for r in results.values()) / len(results), 'errors': sum(bool(r.get('error')) for r in results.values())} (args.output_dir / f'answers_{args.split}.json').write_text(json.dumps(output, ensure_ascii=False, indent=2) + '\n') print(f'{len(results)}/{len(selected)} {row["id"]}: {"OK" if row["strict"] else "FAIL"} {row["latency_seconds"]:.1f}s', flush=True) with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: verify_freeze(manifest, run_metadata(dataset), graph_fingerprint(session), audit=audit, **manifest['parameters']) assert hashes == {str(p): sha256_file(p) for p in code_files}, 'Generation code changed during TEST.' output['by_category'] = {category: {'questions': len(rows), 'correct': sum(r['strict'] for r in rows), 'accuracy': sum(r['strict'] for r in rows) / len(rows)} for category in sorted({q['category'] for q in selected}) for rows in [[r for r in output['questions'] if r['category'] == category]]} (args.output_dir / f'answers_{args.split}.json').write_text(json.dumps(output, ensure_ascii=False, indent=2) + '\n') print(json.dumps(output['summary'])) if __name__ == '__main__': main()