225 lines
14 KiB
Python
225 lines
14 KiB
Python
"""Evaluate 300 fresh LLM outputs after 20 exposed development outputs.
|
|
|
|
The 320-query graph benchmark stays frozen. This separate experiment supplies
|
|
the retriever's context to OpenWebUI; it does not test native tool routing.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import time
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings
|
|
from tools.graphrag.graph_search import search_graph_with_session
|
|
from tools.graphrag.evaluation.benchmark_v2 import BASE
|
|
from tools.graphrag.evaluation.check_openwebui import grade_claims
|
|
from tools.graphrag.evaluation.reproducibility import graph_fingerprint, run_metadata, sha256_file, verify_freeze
|
|
from tools.graphrag.evaluation.source_oracle import load_source_oracle, norm
|
|
from tools.rag.evaluation.rag_runner import DEFAULT_MODEL, OPENWEBUI_URL, request_json
|
|
from .answer_format import parse_answer, Answer
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
|
|
|
|
def graph_context(result: dict) -> dict:
|
|
"""Remove storage/profile metadata that cannot answer the actual question.
|
|
|
|
Uses only actual retrieval output and query intent, never gold annotations.
|
|
Values/subjects/source paths are copied without inventing or rewriting facts.
|
|
"""
|
|
facts = []; intent = result['constraints']
|
|
for row in result['facts']:
|
|
kind = row.get('fact_type', '')
|
|
if kind.startswith('shared_'):
|
|
field = {'shared_keyword': 'keyword', 'shared_category': 'category',
|
|
'shared_document_author': 'document_author'}.get(kind)
|
|
if field:
|
|
for side in ['left', 'right']:
|
|
facts.append({'person': row[f'{side}_person'], field: row[field],
|
|
'document_path': row.get(f'{side}_document_path'),
|
|
'source_url': row.get(f'{side}_source_url')})
|
|
continue
|
|
if not row.get('person'):
|
|
continue
|
|
allowed = {'person', 'document_path', 'source_url', 'keyword', 'category', 'document_author'}
|
|
if intent['mentions_title']:
|
|
allowed.add('title')
|
|
if intent['mentions_academic_year'] or intent['years']:
|
|
allowed.add('year')
|
|
if intent['mentions_start_year']:
|
|
allowed.add('start_year')
|
|
fact = {k: v for k, v in row.items() if k in allowed and v is not None}
|
|
if fact not in facts:
|
|
facts.append(fact)
|
|
return {'status': result['status'], 'reason': result['reason'], 'facts': facts,
|
|
'sources': [{k: d[k] for k in ['document_path', 'source_url'] if d.get(k)}
|
|
for d in result['documents']], 'aggregation': result['aggregation']}
|
|
|
|
|
|
def grade_answer(case: dict, answer: dict, oracle: dict) -> dict:
|
|
# JSON null is absence, not a positive factual assertion. The same rule is
|
|
# applicable to all rounds, including the archived initial sample.
|
|
normalized = dict(answer)
|
|
if isinstance(answer.get('claims'), list):
|
|
normalized['claims'] = [{k: v for k, v in claim.items() if v is not None}
|
|
if isinstance(claim, dict) else claim for claim in answer['claims']]
|
|
grade = grade_claims(case, normalized, oracle)
|
|
text = norm(answer.get('answer', ''))
|
|
missing = [v for v in case.get('expected_answer_contains', []) if norm(v) not in text]
|
|
# The initial sample exposed a real contradiction in otherwise correct
|
|
# category claims. Check it explicitly rather than silently accepting prose.
|
|
wrong_thesis_label = 'dizertac' in text and any(str(b.get('category', '')).startswith('dp')
|
|
for b in case['graph']['expected_bindings']) and not any(
|
|
str(b.get('category', '')).startswith('phd') for b in case['graph']['expected_bindings'])
|
|
grade.update(text_values_ok=not missing, missing_text_values=missing,
|
|
contradictory_thesis_label=wrong_thesis_label)
|
|
grade['strict'] = bool(grade['strict'] and not missing and not wrong_thesis_label)
|
|
return grade
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output-dir', type=Path, required=True)
|
|
parser.add_argument('--sample', type=Path, default=BASE / 'results/final_v4_2026-10-25/openwebui_sample.json')
|
|
parser.add_argument('--workers', type=int, default=3)
|
|
parser.add_argument('--model', default=DEFAULT_MODEL)
|
|
parser.add_argument('--split', choices=['dev', 'test', 'regression'], default='test')
|
|
parser.add_argument('--questions', type=Path, default=BASE/'graphrag_questions_v4.json')
|
|
parser.add_argument('--audit', type=Path, default=BASE/'results/v4_preparation/source_audit.json')
|
|
parser.add_argument('--freeze-manifest', type=Path, default=BASE/'frozen_v4.json')
|
|
parser.add_argument('--dev-ids', type=Path)
|
|
parser.add_argument('--dev-results', type=Path, help='Required completed DEV before final regression.')
|
|
args = parser.parse_args()
|
|
if args.output_dir.exists():
|
|
parser.error('Refusing to overwrite an answer-level TEST.')
|
|
if not os.getenv('OPENWEBUI_API_KEY'):
|
|
parser.error('OPENWEBUI_API_KEY missing.')
|
|
if args.workers < 1:
|
|
parser.error('--workers must be positive.')
|
|
if args.split=='regression' and args.dev_results is None:
|
|
parser.error('Final regression requires completed --dev-results.')
|
|
dataset=args.questions; audit=args.audit; frozen=args.freeze_manifest
|
|
manifest = json.loads(frozen.read_text())
|
|
questions = json.loads(dataset.read_text()); oracle = load_source_oracle()
|
|
sample = json.loads(args.sample.read_text()); exposed = set(sample['selected_ids'])
|
|
test = [q for q in questions if q['split'] == 'test' and q['id'] not in exposed]
|
|
assert len(exposed) == 20 and len(test) == 300
|
|
development=set(json.loads(args.dev_ids.read_text())) if args.dev_ids else exposed
|
|
selected = ([q for q in questions if q['split']=='test'] if args.split=='regression' else
|
|
test if args.split=='test' else [q for q in questions if q['id'] in development])
|
|
if args.split=='regression' and manifest.get('evaluation_kind')!='historical_regression':
|
|
parser.error('Regression requires an explicitly declared historical regression freeze.')
|
|
prompt_path = HERE / 'graph_answer_prompt.txt'; prompt = prompt_path.read_text()
|
|
args.output_dir.mkdir(parents=True)
|
|
metadata = run_metadata(dataset); settings = get_neo4j_settings(); contexts = {}
|
|
with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session:
|
|
verify_freeze(manifest, metadata, graph_fingerprint(session), audit=audit, **manifest['parameters'])
|
|
# Generate every context before making any external request.
|
|
for case in selected:
|
|
contexts[case['id']] = graph_context(search_graph_with_session(session, case['question'], **manifest['parameters']))
|
|
code_files = [Path(__file__), prompt_path, HERE/'answer_format.py', Path('tools/rag/evaluation/rag_runner.py')]
|
|
hashes = {str(p): sha256_file(p) for p in code_files}
|
|
if args.split=='regression':
|
|
dev=json.loads(args.dev_results.read_text())
|
|
if (dev['configuration']['selected_split']!='dev'
|
|
or set(dev['configuration']['selected_ids'])!=development
|
|
or dev['summary']['questions']!=len(development)
|
|
or dev['configuration']['generation_files']!=hashes
|
|
or dev['run_metadata']['fingerprints']!=metadata['fingerprints']):
|
|
raise RuntimeError('Generation code, prompt, data or complete DEV differs; rerun DEV before final freeze.')
|
|
declaration = {'method': ('Previously exposed LLM questions; regression, not independent TEST.'
|
|
if manifest.get('evaluation_kind')=='historical_regression' else
|
|
'Fresh LLM-generation holdout over previously measured, frozen graph queries.'),
|
|
'graph_freeze_sha256': sha256_file(frozen), 'audit_sha256': sha256_file(audit),
|
|
'generation_files': hashes, 'model': args.model, 'temperature': 0, 'workers': args.workers,
|
|
'dev_ids': sorted(development), 'test_ids': [q['id'] for q in test],
|
|
'dev_count': len(development), 'test_count': len(test), 'target_accuracy': 0.92,
|
|
'selected_split': args.split, 'selected_ids': [q['id'] for q in selected], 'selected_count':len(selected),
|
|
'selected_dev_overlap': len(development & {q['id'] for q in selected}),
|
|
'format_repair_limit':1, 'repair_policy':'Only JSON/schema failures; same question/context, no gold; retain all attempts.',
|
|
'dev_results_sha256':sha256_file(args.dev_results) if args.dev_results else None,
|
|
'policy': ('Evaluate every declared regression case, including DEV overlap; retain all failures and stop tuning after this freeze.'
|
|
if manifest.get('evaluation_kind')=='historical_regression' else
|
|
'Evaluate all 300; retain every failure; do not tune on this TEST.'),
|
|
'selected_categories': dict(Counter(q['category'] for q in selected)),
|
|
'limitations': ['Known corpus/profiles and templates; model outputs for these 300 questions were not previously observed.',
|
|
'Harness supplies graph context; native OpenWebUI tool choice and browser UI are outside this experiment.',
|
|
'Checks structured factual claims, their source support/citations and required prose values; not a general semantic judge of arbitrary prose.']}
|
|
(args.output_dir / 'frozen_answers.json').write_text(json.dumps(declaration, ensure_ascii=False, indent=2) + '\n')
|
|
results = {}
|
|
def run(case):
|
|
started = time.perf_counter(); context = contexts[case['id']]
|
|
row = {'id': case['id'], 'category': case['category'], 'question': case['question'],
|
|
'graph_context': context, 'strict': False}
|
|
messages=[{'role':'system','content':prompt},{'role':'user','content':json.dumps(
|
|
{'question':case['question'],'graph_context':context},ensure_ascii=False)}]
|
|
row['attempts']=[]
|
|
try:
|
|
for attempt in range(2):
|
|
response = request_json(OPENWEBUI_URL, method='POST', timeout=90, max_attempts=2,
|
|
headers={'Authorization': f'Bearer {os.environ["OPENWEBUI_API_KEY"]}', 'Content-Type': 'application/json'},
|
|
payload={'model': args.model, 'temperature': 0, 'stream': False,
|
|
'response_format': {'type': 'json_object'}, 'messages': messages})
|
|
row['response'] = response
|
|
text = response['choices'][0]['message']['content'].strip()
|
|
record={'response':response}
|
|
row['attempts'].append(record)
|
|
try:
|
|
answer,normalized=parse_answer(text)
|
|
except (ValueError, TypeError) as exc:
|
|
record['format_error']=type(exc).__name__
|
|
if attempt==1:
|
|
raise
|
|
messages += [{'role':'assistant','content':text}, {'role':'user','content':
|
|
'Predchádzajúci výstup nie je platný JSON podľa schémy. Vráť len opravený jediný JSON objekt '
|
|
'bez zmeny vecných údajov; používaj iba pôvodný kontext. Schéma: '+json.dumps(Answer.model_json_schema(),ensure_ascii=False)}]
|
|
continue
|
|
record['format_normalized']=normalized
|
|
grade=grade_answer(case,answer,oracle)
|
|
record['strict']=grade['strict']
|
|
row['answer']=answer; row.update(grade)
|
|
break
|
|
except Exception as error:
|
|
row['error'] = f'{type(error).__name__}: {str(error)[:500]}'
|
|
row['latency_seconds'] = time.perf_counter() - started
|
|
return row
|
|
output = {'configuration': declaration, 'run_metadata': metadata, 'questions': []}
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor:
|
|
futures = [executor.submit(run, case) for case in selected]
|
|
for future in concurrent.futures.as_completed(futures):
|
|
row = future.result(); results[row['id']] = row
|
|
output['questions'] = [results[q['id']] for q in selected if q['id'] in results]
|
|
output['summary'] = {'questions': len(results), 'correct': sum(r['strict'] for r in results.values()),
|
|
'accuracy': sum(r['strict'] for r in results.values()) / len(results),
|
|
'errors': sum(bool(r.get('error')) for r in results.values()),
|
|
'first_pass_correct':sum(bool(r.get('attempts') and r['attempts'][0].get('strict')) for r in results.values()),
|
|
'format_repair_attempted':sum(len(r.get('attempts',[]))>1 for r in results.values()),
|
|
'normalized_first_response':sum(bool(r.get('attempts') and r['attempts'][0].get('format_normalized')) for r in results.values())}
|
|
(args.output_dir / f'answers_{args.split}.json').write_text(json.dumps(output, ensure_ascii=False, indent=2) + '\n')
|
|
print(f'{len(results)}/{len(selected)} {row["id"]}: {"OK" if row["strict"] else "FAIL"} {row["latency_seconds"]:.1f}s', flush=True)
|
|
with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session:
|
|
verify_freeze(manifest, run_metadata(dataset), graph_fingerprint(session), audit=audit, **manifest['parameters'])
|
|
assert hashes == {str(p): sha256_file(p) for p in code_files}, 'Generation code changed during TEST.'
|
|
output['by_category'] = {category: {'questions': len(rows), 'correct': sum(r['strict'] for r in rows),
|
|
'accuracy': sum(r['strict'] for r in rows) / len(rows)} for category in sorted({q['category'] for q in selected})
|
|
for rows in [[r for r in output['questions'] if r['category'] == category]]}
|
|
(args.output_dir / f'answers_{args.split}.json').write_text(json.dumps(output, ensure_ascii=False, indent=2) + '\n')
|
|
fields=['id','category','strict','error','latency_seconds','attempts','first_pass_strict']
|
|
with (args.output_dir/f'answers_{args.split}.csv').open('w',newline='',encoding='utf-8') as handle:
|
|
writer=csv.DictWriter(handle,fieldnames=fields);writer.writeheader()
|
|
for row in output['questions']:
|
|
writer.writerow({**{k:row.get(k) for k in fields if k not in {'attempts','first_pass_strict'}},
|
|
'attempts':len(row.get('attempts',[])),
|
|
'first_pass_strict':bool(row.get('attempts') and row['attempts'][0].get('strict'))})
|
|
print(json.dumps(output['summary']))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|