diff --git a/evaluation/evaluate_rag_answers.py b/evaluation/evaluate_rag_answers.py index 48cb42f..081f6fc 100644 --- a/evaluation/evaluate_rag_answers.py +++ b/evaluation/evaluate_rag_answers.py @@ -1,7 +1,6 @@ from __future__ import annotations import argparse -import csv import json import sys import time @@ -19,6 +18,31 @@ if str(PROJECT_ROOT) not in sys.path: ) +from evaluation.rag_answer_data import ( + DEFAULT_DELAY, + apply_question_overrides, + questions_fingerprint, + result_prefix, + select_questions, + validate_answer_questions, + validate_string_list, +) +from evaluation.rag_answer_state import ( + PARTIAL_SCHEMA_VERSION, + build_base_result, + build_error_result, + build_partial_payload, + build_run_configuration, + determine_run_status, + extract_resume_state, + load_partial_payload, + ordered_results, + prepare_output_state, + print_summary, + save_csv_results, + save_json_results, + validate_resume_compatibility, +) from evaluation.rag_metrics import ( evaluate_answer, summarize_results, @@ -30,15 +54,12 @@ from evaluation.rag_runner import ( DEFAULT_MODEL, DEFAULT_TIMEOUT, LOCAL_OPENAPI_URL, - LOCAL_RAG_URL, - OPENWEBUI_URL, build_rag_tool, load_env_value, request_json, run_question, ) from evaluation.retrieval_runner import ( - filter_questions_by_split, load_questions, ) @@ -69,15 +90,6 @@ RESULTS_DIR = ( ) -ALLOWED_OVERRIDE_FIELDS = { - "question", - "expected_answer_contains", - "expected_source_urls", - "should_answer", - "note", -} - - def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( @@ -134,8 +146,8 @@ def parse_args() -> argparse.Namespace: type=Path, default=RAG_ANSWER_OVERRIDES_PATH, help=( - "Cesta k answer-level override " - "súboru." + "Cesta k answer-level " + "override súboru." ), ) @@ -165,7 +177,8 @@ def parse_args() -> argparse.Namespace: help=( "Maximálny počet HTTP pokusov " "pre retryable chyby. " - f"Predvolené: {DEFAULT_MAX_ATTEMPTS}." + f"Predvolené: " + f"{DEFAULT_MAX_ATTEMPTS}." ), ) @@ -174,9 +187,10 @@ def parse_args() -> argparse.Namespace: type=float, default=DEFAULT_BACKOFF_BASE, help=( - "Počiatočný exponential backoff " - "v sekundách. " - f"Predvolené: {DEFAULT_BACKOFF_BASE}." + "Počiatočný exponential " + "backoff v sekundách. " + f"Predvolené: " + f"{DEFAULT_BACKOFF_BASE}." ), ) @@ -185,19 +199,22 @@ def parse_args() -> argparse.Namespace: type=float, default=DEFAULT_BACKOFF_MAX, help=( - "Maximálny exponential backoff " - "v sekundách. " - f"Predvolené: {DEFAULT_BACKOFF_MAX}." + "Maximálny exponential " + "backoff v sekundách. " + f"Predvolené: " + f"{DEFAULT_BACKOFF_MAX}." ), ) parser.add_argument( "--delay", type=float, - default=0.0, + default=DEFAULT_DELAY, help=( "Pauza medzi otázkami " - "v sekundách." + "v sekundách. " + f"Predvolené: " + f"{DEFAULT_DELAY}." ), ) @@ -221,6 +238,31 @@ def parse_args() -> argparse.Namespace: ), ) + output_group = ( + parser.add_mutually_exclusive_group() + ) + + output_group.add_argument( + "--resume", + action="store_true", + help=( + "Pokračuj z kompatibilného " + ".partial.json. Úspešné otázky " + "sa preskočia a chybové sa " + "vyhodnotia znova." + ), + ) + + output_group.add_argument( + "--overwrite", + action="store_true", + help=( + "Začni nový run a povoľ " + "prepísanie existujúcich " + "výsledkov." + ), + ) + args = parser.parse_args() if ( @@ -243,12 +285,14 @@ def parse_args() -> argparse.Namespace: if args.backoff_base < 0: parser.error( - "--backoff-base nesmie byť záporné" + "--backoff-base " + "nesmie byť záporné" ) if args.backoff_max < 0: parser.error( - "--backoff-max nesmie byť záporné" + "--backoff-max " + "nesmie byť záporné" ) if args.delay < 0: @@ -259,533 +303,6 @@ def parse_args() -> argparse.Namespace: return args -def apply_question_overrides( - questions: list[ - dict[str, Any] - ], - path: Path, -) -> tuple[ - list[dict[str, Any]], - list[str], -]: - if not path.exists(): - raise FileNotFoundError( - "RAG answer override súbor " - f"neexistuje: {path}" - ) - - with path.open( - "r", - encoding="utf-8", - ) as file: - overrides = json.load( - file - ) - - if not isinstance( - overrides, - dict, - ): - raise ValueError( - "rag_answer_overrides.json " - "musí obsahovať JSON objekt." - ) - - known_ids = { - str( - item["id"] - ) - for item in questions - } - - unknown_ids = ( - set( - overrides.keys() - ) - - known_ids - ) - - if unknown_ids: - raise ValueError( - "Override súbor obsahuje " - "neznáme question ID: " - + ", ".join( - sorted( - unknown_ids - ) - ) - ) - - result: list[ - dict[str, Any] - ] = [] - - applied_ids: list[ - str - ] = [] - - for question in questions: - item = dict( - question - ) - - question_id = str( - item["id"] - ) - - override = overrides.get( - question_id - ) - - if override is None: - result.append( - item - ) - - continue - - if not isinstance( - override, - dict, - ): - raise ValueError( - f"{question_id}: " - "override musí byť " - "JSON objekt." - ) - - unsupported_fields = ( - set( - override.keys() - ) - - ALLOWED_OVERRIDE_FIELDS - ) - - if unsupported_fields: - raise ValueError( - f"{question_id}: " - "nepovolené override polia: " - + ", ".join( - sorted( - unsupported_fields - ) - ) - ) - - if "question" in override: - overridden_question = ( - override[ - "question" - ] - ) - - if ( - not isinstance( - overridden_question, - str, - ) - or not overridden_question.strip() - ): - raise ValueError( - f"{question_id}: " - "override question musí byť " - "neprázdny reťazec." - ) - - item.update( - override - ) - - result.append( - item - ) - - applied_ids.append( - question_id - ) - - return ( - result, - applied_ids, - ) - - -def select_questions( - questions: list[ - dict[str, Any] - ], - *, - split: str, - question_ids: list[str], - limit: int | None, -) -> list[ - dict[str, Any] -]: - selected = ( - filter_questions_by_split( - questions, - split, - ) - ) - - if question_ids: - wanted = set( - question_ids - ) - - selected = [ - item - for item in selected - if item.get( - "id" - ) in wanted - ] - - if limit is not None: - selected = selected[ - :limit - ] - - return selected - - -def result_prefix( - *, - split: str, - limit: int | None, - question_ids: list[str], -) -> str: - prefix = ( - f"rag_answers_{split}" - ) - - if question_ids: - return ( - prefix - + "_selected" - ) - - if limit is not None: - return ( - prefix - + f"_limit{limit}" - ) - - return prefix - - -def save_json_results( - path: Path, - payload: dict[str, Any], -) -> None: - path.parent.mkdir( - parents=True, - exist_ok=True, - ) - - with path.open( - "w", - encoding="utf-8", - ) as file: - json.dump( - payload, - file, - ensure_ascii=False, - indent=2, - ) - - file.write( - "\n" - ) - - -def save_csv_results( - path: Path, - results: list[ - dict[str, Any] - ], -) -> None: - path.parent.mkdir( - parents=True, - exist_ok=True, - ) - - fieldnames = [ - "id", - "split", - "category", - "difficulty", - "question", - "requested_model", - "response_model", - "tool_called", - "tool_call_count", - "answer_contains_score", - "source_url_score", - "should_answer_score", - "overall_score", - "strict_pass", - "expected_answer_contains", - "expected_source_urls", - "rag_source_urls", - "answer", - "first_model_latency_seconds", - "tool_latency_seconds", - "final_model_latency_seconds", - "total_latency_seconds", - "prompt_tokens", - "completion_tokens", - "total_tokens", - "error", - ] - - with path.open( - "w", - encoding="utf-8", - newline="", - ) as file: - writer = csv.DictWriter( - file, - fieldnames=fieldnames, - ) - - writer.writeheader() - - for result in results: - usage = ( - result.get( - "usage" - ) - or {} - ) - - row = { - "id": result.get( - "id" - ), - "split": result.get( - "split" - ), - "category": result.get( - "category" - ), - "difficulty": result.get( - "difficulty" - ), - "question": result.get( - "question" - ), - "requested_model": result.get( - "requested_model" - ), - "response_model": result.get( - "response_model" - ), - "tool_called": result.get( - "tool_called" - ), - "tool_call_count": result.get( - "tool_call_count" - ), - "answer_contains_score": result.get( - "answer_contains_score" - ), - "source_url_score": result.get( - "source_url_score" - ), - "should_answer_score": result.get( - "should_answer_score" - ), - "overall_score": result.get( - "overall_score" - ), - "strict_pass": result.get( - "strict_pass" - ), - "expected_answer_contains": ( - json.dumps( - result.get( - "expected_answer_contains", - [], - ), - ensure_ascii=False, - ) - ), - "expected_source_urls": ( - json.dumps( - result.get( - "expected_source_urls", - [], - ), - ensure_ascii=False, - ) - ), - "rag_source_urls": ( - json.dumps( - result.get( - "rag_source_urls", - [], - ), - ensure_ascii=False, - ) - ), - "answer": result.get( - "answer", - "", - ), - "first_model_latency_seconds": ( - result.get( - "first_model_latency_seconds" - ) - ), - "tool_latency_seconds": result.get( - "tool_latency_seconds" - ), - "final_model_latency_seconds": ( - result.get( - "final_model_latency_seconds" - ) - ), - "total_latency_seconds": result.get( - "total_latency_seconds" - ), - "prompt_tokens": usage.get( - "prompt_tokens", - 0, - ), - "completion_tokens": usage.get( - "completion_tokens", - 0, - ), - "total_tokens": usage.get( - "total_tokens", - 0, - ), - "error": result.get( - "error", - "", - ), - } - - writer.writerow( - row - ) - - -def build_error_result( - base_result: dict[str, Any], - exc: Exception, -) -> dict[str, Any]: - return { - **base_result, - "answer": "", - "tool_called": False, - "tool_call_count": 0, - "tool_calls": [], - "rag_source_urls": [], - "first_model_latency_seconds": 0.0, - "tool_latency_seconds": 0.0, - "final_model_latency_seconds": 0.0, - "total_latency_seconds": 0.0, - "usage": { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 0, - }, - "response_model": None, - "answer_matches": [], - "answer_contains_score": 0.0, - "source_matches": [], - "source_url_score": 0.0, - "should_answer_ok": False, - "should_answer_score": 0.0, - "returned_no_answer": False, - "tool_score": 0.0, - "overall_score": 0.0, - "strict_pass": False, - "error": ( - f"{type(exc).__name__}: " - f"{exc}" - ), - } - - -def print_summary( - summary: dict[str, Any], -) -> None: - print() - print( - "RAG answer evaluation" - ) - - print( - "=" * 78 - ) - - print( - f"Total: " - f"{summary['total']}" - ) - - print( - f"Completed: " - f"{summary['completed']}" - ) - - print( - f"Errors: " - f"{summary['errors']}" - ) - - print( - "Tool call rate: " - f"{summary['tool_call_rate']:.3f}" - ) - - print( - "Answer contains: " - f"{summary['answer_contains_score']:.3f}" - ) - - print( - "Source URL: " - f"{summary['source_url_score']:.3f}" - ) - - print( - "Should answer: " - f"{summary['should_answer_score']:.3f}" - ) - - print( - "Overall: " - f"{summary['overall_score']:.3f}" - ) - - print( - "Strict pass: " - f"{summary['strict_pass_count']}" - f"/{summary['total']} " - f"({summary['strict_pass_rate']:.3f})" - ) - - print( - "Mean latency: " - f"{summary['mean_latency_seconds']:.3f} s" - ) - - print( - "Total tokens: " - f"{summary['total_tokens']}" - ) - - print( - "=" * 78 - ) - - print() - - def main() -> int: args = parse_args() @@ -804,6 +321,25 @@ def main() -> int: args.overrides, ) + questions = select_questions( + all_questions, + split=args.split, + question_ids=( + args.question_id + ), + limit=args.limit, + ) + + if not questions: + raise ValueError( + "Žiadne otázky " + "nezodpovedajú filtru." + ) + + validate_answer_questions( + questions + ) + except ( FileNotFoundError, ValueError, @@ -816,24 +352,6 @@ def main() -> int: return 2 - questions = select_questions( - all_questions, - split=args.split, - question_ids=( - args.question_id - ), - limit=args.limit, - ) - - if not questions: - print( - "ERROR: Žiadne otázky " - "nezodpovedajú filtru.", - file=sys.stderr, - ) - - return 2 - selected_ids = { str( question["id"] @@ -843,10 +361,57 @@ def main() -> int: selected_override_ids = [ question_id - for question_id in applied_override_ids - if question_id in selected_ids + for question_id + in applied_override_ids + if question_id + in selected_ids ] + prefix = result_prefix( + split=args.split, + limit=args.limit, + question_ids=( + args.question_id + ), + ) + + json_path = ( + args.results_dir + / f"{prefix}.json" + ) + + csv_path = ( + args.results_dir + / f"{prefix}.csv" + ) + + partial_path = ( + args.results_dir + / f"{prefix}.partial.json" + ) + + try: + prepare_output_state( + json_path=json_path, + csv_path=csv_path, + partial_path=( + partial_path + ), + resume=args.resume, + overwrite=args.overwrite, + ) + + except ( + FileNotFoundError, + FileExistsError, + ) as exc: + print( + f"ERROR: {exc}", + file=sys.stderr, + ) + + return 2 + try: openwebui_api_key = ( load_env_value( @@ -884,10 +449,11 @@ def main() -> int: ), ) - operation_id, rag_tool = ( - build_rag_tool( - openapi - ) + ( + operation_id, + rag_tool, + ) = build_rag_tool( + openapi ) except RuntimeError as exc: @@ -898,48 +464,89 @@ def main() -> int: return 2 - prefix = result_prefix( - split=args.split, - limit=args.limit, - question_ids=( - args.question_id - ), + configuration = ( + build_run_configuration( + args=args, + questions=questions, + selected_override_ids=( + selected_override_ids + ), + operation_id=( + operation_id + ), + ) ) - json_path = ( - args.results_dir - / f"{prefix}.json" - ) + results_by_id: dict[ + str, + dict[str, Any], + ] = {} - csv_path = ( - args.results_dir - / f"{prefix}.csv" - ) + previous_failed_ids: list[ + str + ] = [] - partial_path = ( - args.results_dir - / f"{prefix}.partial.json" - ) + if args.resume: + try: + partial_payload = ( + load_partial_payload( + partial_path + ) + ) + + validate_resume_compatibility( + partial_payload[ + "configuration" + ], + configuration, + ) + + ( + results_by_id, + previous_failed_ids, + ) = extract_resume_state( + partial_payload, + selected_question_ids=( + selected_ids + ), + ) + + except ( + FileNotFoundError, + ValueError, + json.JSONDecodeError, + ) as exc: + print( + f"ERROR: {exc}", + file=sys.stderr, + ) + + return 2 + + pending_questions = [ + question + for question in questions + if str( + question["id"] + ) + not in results_by_id + ] print() print( "RAG answer evaluation" ) - print( "=" * 60 ) - print( f"Dataset: " f"{args.questions}" ) - print( f"Overrides: " f"{args.overrides}" ) - print( "Applied overrides: " f"{len(selected_override_ids)}" @@ -957,47 +564,71 @@ def main() -> int: f"Model: " f"{args.model}" ) - print( f"Tool: " f"{operation_id}" ) - print( f"Split: " f"{args.split}" ) - print( f"Questions: " f"{len(questions)}" ) - print( "Retry: " - f"{args.max_attempts} pokusy" + f"{args.max_attempts} " + "pokusy" ) - print( "Backoff: " f"{args.backoff_base}s " - f"→ max {args.backoff_max}s" + "→ max " + f"{args.backoff_max}s" ) + print( + f"Delay: " + f"{args.delay}s" + ) + + if args.resume: + print( + "Resume: áno" + ) + print( + " úspešne obnovené: " + f"{len(results_by_id)}" + ) + print( + " predchádzajúce chyby " + "na retry: " + f"{len(previous_failed_ids)}" + ) + print( + " zostáva spustiť: " + f"{len(pending_questions)}" + ) + + else: + print( + "Resume: nie" + ) print( "=" * 60 ) - print() - results: list[ - dict[str, Any] - ] = [] - total = len( questions ) + pending_total = len( + pending_questions + ) + + processed_pending = 0 stopped_early = False for index, question in enumerate( @@ -1005,92 +636,77 @@ def main() -> int: start=1, ): question_id = str( - question[ - "id" - ] + question["id"] ) question_text = str( - question[ - "question" - ] + question["question"] ) + if ( + question_id + in results_by_id + ): + print( + f"[{index:04d}/" + f"{total:04d}] " + f"{question_id}: " + "SKIP (resume)" + ) + + continue + print( - f"[{index:04d}/{total:04d}] " + f"[{index:04d}/" + f"{total:04d}] " f"{question_id}: " f"{question_text}" ) - base_result: dict[ - str, - Any, - ] = { - "id": question_id, - "split": question.get( - "split" - ), - "category": question.get( - "category", - "unknown", - ), - "difficulty": question.get( - "difficulty", - "unknown", - ), - "question": question_text, - "override_applied": ( - question_id - in selected_override_ids - ), - "requested_model": ( - args.model - ), - "expected_documents": question.get( - "expected_documents", - [], - ), - "expected_source_urls": question.get( - "expected_source_urls", - [], - ), - "expected_answer_contains": question.get( - "expected_answer_contains", - [], - ), - "should_answer": question.get( - "should_answer", - True, - ), - "note": question.get( - "note" - ), - } + base_result = ( + build_base_result( + question=question, + question_id=( + question_id + ), + question_text=( + question_text + ), + selected_override_ids=( + selected_override_ids + ), + requested_model=( + args.model + ), + ) + ) try: - run_result = run_question( - question_text, - model=args.model, - operation_id=( - operation_id - ), - rag_tool=rag_tool, - openwebui_api_key=( - openwebui_api_key - ), - search_api_key=( - search_api_key - ), - timeout=args.timeout, - max_attempts=( - args.max_attempts - ), - backoff_base=( - args.backoff_base - ), - backoff_max=( - args.backoff_max - ), + run_result = ( + run_question( + question_text, + model=args.model, + operation_id=( + operation_id + ), + rag_tool=rag_tool, + openwebui_api_key=( + openwebui_api_key + ), + search_api_key=( + search_api_key + ), + timeout=args.timeout, + max_attempts=( + args.max_attempts + ), + backoff_base=( + args.backoff_base + ), + backoff_max=( + args.backoff_max + ), + ) ) scores = evaluate_answer( @@ -1147,56 +763,33 @@ def main() -> int: if args.fail_fast: stopped_early = True - results.append( - result + results_by_id[ + question_id + ] = result + + processed_pending += 1 + + current_results = ( + ordered_results( + questions, + results_by_id, + ) ) - partial_payload = { - "generated_at": ( - datetime.now( - timezone.utc - ).isoformat() - ), - "status": "partial", - "split": ( - args.split - ), - "requested_model": ( - args.model - ), - "questions_file": str( - args.questions - ), - "overrides_file": str( - args.overrides - ), - "applied_override_ids": ( - selected_override_ids - ), - "max_attempts": ( - args.max_attempts - ), - "backoff_base": ( - args.backoff_base - ), - "backoff_max": ( - args.backoff_max - ), - "completed_so_far": len( - results - ), - "expected_total": ( - total - ), - "summary": ( - summarize_results( - results - ) - ), - "results": ( - results - ), - } + partial_payload = ( + build_partial_payload( + status="partial", + configuration=( + configuration + ), + results=( + current_results + ), + expected_total=( + total + ), + ) + ) save_json_results( partial_path, @@ -1208,81 +801,43 @@ def main() -> int: if ( args.delay > 0 - and index < total + and processed_pending + < pending_total ): time.sleep( args.delay ) + results = ordered_results( + questions, + results_by_id, + ) + + status = determine_run_status( + results, + expected_total=( + total + ), + ) + summary = ( summarize_results( results ) ) - completed_all = ( - len(results) - == total - ) - payload = { "generated_at": ( datetime.now( timezone.utc ).isoformat() ), - "status": ( - "complete" - if completed_all - else "partial" - ), - "configuration": { - "questions_file": str( - args.questions - ), - "overrides_file": str( - args.overrides - ), - "applied_override_ids": ( - selected_override_ids - ), - "split": ( - args.split - ), - "selected_question_count": ( - total - ), - "requested_model": ( - args.model - ), - "openwebui_url": ( - OPENWEBUI_URL - ), - "rag_url": ( - LOCAL_RAG_URL - ), - "timeout": ( - args.timeout - ), - "max_attempts": ( - args.max_attempts - ), - "backoff_base": ( - args.backoff_base - ), - "backoff_max": ( - args.backoff_max - ), - "delay": ( - args.delay - ), - }, - "summary": ( - summary - ), - "results": ( - results + "status": status, + "configuration": ( + configuration ), + "summary": summary, + "results": results, } save_json_results( @@ -1295,11 +850,24 @@ def main() -> int: results, ) - if ( - partial_path.exists() - and completed_all - ): - partial_path.unlink() + if status == "complete": + if partial_path.exists(): + partial_path.unlink() + + else: + save_json_results( + partial_path, + build_partial_payload( + status=status, + configuration=( + configuration + ), + results=results, + expected_total=( + total + ), + ), + ) print_summary( summary @@ -1308,17 +876,26 @@ def main() -> int: print( "Výsledky:" ) - print( f" JSON: " f"{json_path}" ) - print( f" CSV: " f"{csv_path}" ) + if partial_path.exists(): + print( + f" Partial: " + f"{partial_path}" + ) + + print( + f" Status: " + f"{status}" + ) + if summary[ "errors" ]: diff --git a/evaluation/rag_answer_data.py b/evaluation/rag_answer_data.py new file mode 100644 index 0000000..9c830b7 --- /dev/null +++ b/evaluation/rag_answer_data.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from evaluation.retrieval_runner import filter_questions_by_split + + +DEFAULT_DELAY = 0.5 + +ALLOWED_OVERRIDE_FIELDS = { + "question", + "expected_answer_contains", + "expected_source_urls", + "should_answer", + "note", +} + + +def apply_question_overrides( + questions: list[dict[str, Any]], + path: Path, +) -> tuple[list[dict[str, Any]], list[str]]: + if not path.exists(): + raise FileNotFoundError( + f"RAG answer override súbor neexistuje: {path}" + ) + + with path.open("r", encoding="utf-8") as file: + overrides = json.load(file) + + if not isinstance(overrides, dict): + raise ValueError( + "rag_answer_overrides.json musí obsahovať JSON objekt." + ) + + known_ids = {str(item["id"]) for item in questions} + unknown_ids = set(overrides.keys()) - known_ids + + if unknown_ids: + raise ValueError( + "Override súbor obsahuje neznáme question ID: " + + ", ".join(sorted(unknown_ids)) + ) + + result: list[dict[str, Any]] = [] + applied_ids: list[str] = [] + + for question in questions: + item = dict(question) + question_id = str(item["id"]) + override = overrides.get(question_id) + + if override is None: + result.append(item) + continue + + if not isinstance(override, dict): + raise ValueError( + f"{question_id}: override musí byť JSON objekt." + ) + + unsupported_fields = ( + set(override.keys()) + - ALLOWED_OVERRIDE_FIELDS + ) + + if unsupported_fields: + raise ValueError( + f"{question_id}: nepovolené override polia: " + + ", ".join( + sorted(unsupported_fields) + ) + ) + + if "question" in override: + overridden_question = override["question"] + + if ( + not isinstance( + overridden_question, + str, + ) + or not overridden_question.strip() + ): + raise ValueError( + f"{question_id}: " + "override question musí byť " + "neprázdny reťazec." + ) + + item.update(override) + result.append(item) + applied_ids.append(question_id) + + return result, applied_ids + + +def validate_string_list( + question_id: str, + field_name: str, + value: Any, +) -> None: + if not isinstance(value, list): + raise ValueError( + f"{question_id}: " + f"{field_name} musí byť JSON pole." + ) + + for index, item in enumerate(value): + if not isinstance(item, str): + raise ValueError( + f"{question_id}: " + f"{field_name}[{index}] " + "musí byť reťazec." + ) + + +def validate_answer_questions( + questions: list[dict[str, Any]], +) -> None: + seen_ids: set[str] = set() + + for index, question in enumerate( + questions, + start=1, + ): + if not isinstance(question, dict): + raise ValueError( + f"Otázka #{index} " + "musí byť JSON objekt." + ) + + question_id = question.get("id") + + if ( + not isinstance(question_id, str) + or not question_id.strip() + ): + raise ValueError( + f"Otázka #{index}: " + "id musí byť neprázdny reťazec." + ) + + if question_id in seen_ids: + raise ValueError( + f"Duplicitné question ID: {question_id}" + ) + + seen_ids.add(question_id) + + split = question.get("split") + + if split not in { + "dev", + "test", + }: + raise ValueError( + f"{question_id}: " + "split musí byť " + "'dev' alebo 'test'." + ) + + question_text = question.get( + "question" + ) + + if ( + not isinstance( + question_text, + str, + ) + or not question_text.strip() + ): + raise ValueError( + f"{question_id}: " + "question musí byť " + "neprázdny reťazec." + ) + + validate_string_list( + question_id, + "expected_documents", + question.get( + "expected_documents" + ), + ) + + validate_string_list( + question_id, + "expected_source_urls", + question.get( + "expected_source_urls" + ), + ) + + validate_string_list( + question_id, + "expected_answer_contains", + question.get( + "expected_answer_contains" + ), + ) + + if not isinstance( + question.get( + "should_answer" + ), + bool, + ): + raise ValueError( + f"{question_id}: " + "should_answer musí byť boolean." + ) + + for field_name in ( + "category", + "difficulty", + ): + value = question.get( + field_name + ) + + if ( + value is not None + and not isinstance( + value, + str, + ) + ): + raise ValueError( + f"{question_id}: " + f"{field_name} musí byť " + "reťazec." + ) + + note = question.get("note") + + if ( + note is not None + and not isinstance( + note, + str, + ) + ): + raise ValueError( + f"{question_id}: " + "note musí byť reťazec " + "alebo null." + ) + + +def select_questions( + questions: list[dict[str, Any]], + *, + split: str, + question_ids: list[str], + limit: int | None, +) -> list[dict[str, Any]]: + selected = filter_questions_by_split( + questions, + split, + ) + + if question_ids: + wanted = set(question_ids) + + selected = [ + item + for item in selected + if item.get("id") in wanted + ] + + if limit is not None: + selected = selected[:limit] + + return selected + + +def result_prefix( + *, + split: str, + limit: int | None, + question_ids: list[str], +) -> str: + prefix = f"rag_answers_{split}" + + if question_ids: + return prefix + "_selected" + + if limit is not None: + return prefix + f"_limit{limit}" + + return prefix + + +def questions_fingerprint( + questions: list[dict[str, Any]], +) -> str: + canonical = json.dumps( + questions, + ensure_ascii=False, + sort_keys=True, + separators=( + ",", + ":", + ), + allow_nan=False, + ).encode("utf-8") + + return hashlib.sha256( + canonical + ).hexdigest() diff --git a/evaluation/rag_answer_state.py b/evaluation/rag_answer_state.py new file mode 100644 index 0000000..65f8f3f --- /dev/null +++ b/evaluation/rag_answer_state.py @@ -0,0 +1,830 @@ +from __future__ import annotations + +import argparse +import csv +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from evaluation.rag_answer_data import ( + questions_fingerprint, +) +from evaluation.rag_metrics import ( + summarize_results, +) +from evaluation.rag_runner import ( + LOCAL_RAG_URL, + OPENWEBUI_URL, +) + + +PARTIAL_SCHEMA_VERSION = 2 + +RESUME_COMPATIBILITY_KEYS = ( + "schema_version", + "questions_file", + "overrides_file", + "questions_fingerprint", + "selected_question_ids", + "applied_override_ids", + "split", + "selected_question_count", + "requested_model", + "operation_id", +) + + +def _temp_path( + path: Path, +) -> Path: + return path.with_name( + f".{path.name}." + f"{os.getpid()}.tmp" + ) + + +def save_json_results( + path: Path, + payload: dict[str, Any], +) -> None: + path.parent.mkdir( + parents=True, + exist_ok=True, + ) + + temp_path = _temp_path( + path + ) + + try: + with temp_path.open( + "w", + encoding="utf-8", + ) as file: + json.dump( + payload, + file, + ensure_ascii=False, + indent=2, + ) + + file.write("\n") + file.flush() + os.fsync( + file.fileno() + ) + + os.replace( + temp_path, + path, + ) + + finally: + if temp_path.exists(): + temp_path.unlink() + + +def save_csv_results( + path: Path, + results: list[dict[str, Any]], +) -> None: + path.parent.mkdir( + parents=True, + exist_ok=True, + ) + + temp_path = _temp_path( + path + ) + + fieldnames = [ + "id", + "split", + "category", + "difficulty", + "question", + "requested_model", + "response_model", + "tool_called", + "tool_call_count", + "answer_contains_score", + "source_url_score", + "should_answer_score", + "overall_score", + "strict_pass", + "expected_answer_contains", + "expected_source_urls", + "rag_source_urls", + "answer", + "first_model_latency_seconds", + "tool_latency_seconds", + "final_model_latency_seconds", + "total_latency_seconds", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "error", + ] + + try: + with temp_path.open( + "w", + encoding="utf-8", + newline="", + ) as file: + writer = csv.DictWriter( + file, + fieldnames=fieldnames, + ) + + writer.writeheader() + + for result in results: + usage = ( + result.get("usage") + or {} + ) + + writer.writerow( + { + "id": result.get( + "id" + ), + "split": result.get( + "split" + ), + "category": result.get( + "category" + ), + "difficulty": result.get( + "difficulty" + ), + "question": result.get( + "question" + ), + "requested_model": ( + result.get( + "requested_model" + ) + ), + "response_model": ( + result.get( + "response_model" + ) + ), + "tool_called": ( + result.get( + "tool_called" + ) + ), + "tool_call_count": ( + result.get( + "tool_call_count" + ) + ), + "answer_contains_score": ( + result.get( + "answer_contains_score" + ) + ), + "source_url_score": ( + result.get( + "source_url_score" + ) + ), + "should_answer_score": ( + result.get( + "should_answer_score" + ) + ), + "overall_score": ( + result.get( + "overall_score" + ) + ), + "strict_pass": ( + result.get( + "strict_pass" + ) + ), + "expected_answer_contains": ( + json.dumps( + result.get( + "expected_answer_contains", + [], + ), + ensure_ascii=False, + ) + ), + "expected_source_urls": ( + json.dumps( + result.get( + "expected_source_urls", + [], + ), + ensure_ascii=False, + ) + ), + "rag_source_urls": ( + json.dumps( + result.get( + "rag_source_urls", + [], + ), + ensure_ascii=False, + ) + ), + "answer": result.get( + "answer", + "", + ), + "first_model_latency_seconds": ( + result.get( + "first_model_latency_seconds" + ) + ), + "tool_latency_seconds": ( + result.get( + "tool_latency_seconds" + ) + ), + "final_model_latency_seconds": ( + result.get( + "final_model_latency_seconds" + ) + ), + "total_latency_seconds": ( + result.get( + "total_latency_seconds" + ) + ), + "prompt_tokens": usage.get( + "prompt_tokens", + 0, + ), + "completion_tokens": usage.get( + "completion_tokens", + 0, + ), + "total_tokens": usage.get( + "total_tokens", + 0, + ), + "error": result.get( + "error", + "", + ), + } + ) + + file.flush() + os.fsync( + file.fileno() + ) + + os.replace( + temp_path, + path, + ) + + finally: + if temp_path.exists(): + temp_path.unlink() + + +def build_error_result( + base_result: dict[str, Any], + exc: Exception, +) -> dict[str, Any]: + return { + **base_result, + "answer": "", + "tool_called": False, + "tool_call_count": 0, + "tool_calls": [], + "rag_source_urls": [], + "first_model_latency_seconds": 0.0, + "tool_latency_seconds": 0.0, + "final_model_latency_seconds": 0.0, + "total_latency_seconds": 0.0, + "usage": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + }, + "response_model": None, + "answer_matches": [], + "answer_contains_score": 0.0, + "source_matches": [], + "source_url_score": 0.0, + "should_answer_ok": False, + "should_answer_score": 0.0, + "returned_no_answer": False, + "tool_score": 0.0, + "overall_score": 0.0, + "strict_pass": False, + "error": ( + f"{type(exc).__name__}: " + f"{exc}" + ), + } + + +def build_base_result( + *, + question: dict[str, Any], + question_id: str, + question_text: str, + selected_override_ids: list[str], + requested_model: str, +) -> dict[str, Any]: + return { + "id": question_id, + "split": question.get( + "split" + ), + "category": question.get( + "category", + "unknown", + ), + "difficulty": question.get( + "difficulty", + "unknown", + ), + "question": question_text, + "override_applied": ( + question_id + in selected_override_ids + ), + "requested_model": ( + requested_model + ), + "expected_documents": ( + question.get( + "expected_documents", + [], + ) + ), + "expected_source_urls": ( + question.get( + "expected_source_urls", + [], + ) + ), + "expected_answer_contains": ( + question.get( + "expected_answer_contains", + [], + ) + ), + "should_answer": question.get( + "should_answer", + True, + ), + "note": question.get( + "note" + ), + } + + +def build_run_configuration( + *, + args: argparse.Namespace, + questions: list[dict[str, Any]], + selected_override_ids: list[str], + operation_id: str, +) -> dict[str, Any]: + return { + "schema_version": ( + PARTIAL_SCHEMA_VERSION + ), + "questions_file": str( + args.questions.resolve() + ), + "overrides_file": str( + args.overrides.resolve() + ), + "questions_fingerprint": ( + questions_fingerprint( + questions + ) + ), + "selected_question_ids": [ + str(item["id"]) + for item in questions + ], + "applied_override_ids": list( + selected_override_ids + ), + "split": args.split, + "selected_question_count": len( + questions + ), + "requested_model": ( + args.model + ), + "operation_id": ( + operation_id + ), + "openwebui_url": ( + OPENWEBUI_URL + ), + "rag_url": ( + LOCAL_RAG_URL + ), + "timeout": args.timeout, + "max_attempts": ( + args.max_attempts + ), + "backoff_base": ( + args.backoff_base + ), + "backoff_max": ( + args.backoff_max + ), + "delay": args.delay, + } + + +def load_partial_payload( + path: Path, +) -> dict[str, Any]: + if not path.exists(): + raise FileNotFoundError( + "Partial výsledok " + f"neexistuje: {path}" + ) + + with path.open( + "r", + encoding="utf-8", + ) as file: + payload = json.load( + file + ) + + if not isinstance( + payload, + dict, + ): + raise ValueError( + "Partial výsledok musí " + "obsahovať JSON objekt." + ) + + configuration = payload.get( + "configuration" + ) + + if not isinstance( + configuration, + dict, + ): + raise ValueError( + "Partial výsledok nemá " + "kompatibilnú configuration " + "sekciu. Starý partial spusti " + "znova s --overwrite." + ) + + results = payload.get( + "results" + ) + + if not isinstance( + results, + list, + ): + raise ValueError( + "Partial výsledok nemá " + "platné results pole." + ) + + return payload + + +def validate_resume_compatibility( + partial_configuration: dict[ + str, + Any, + ], + current_configuration: dict[ + str, + Any, + ], +) -> None: + mismatches = [ + key + for key in ( + RESUME_COMPATIBILITY_KEYS + ) + if ( + partial_configuration.get( + key + ) + != current_configuration.get( + key + ) + ) + ] + + if mismatches: + raise ValueError( + "Partial výsledok nie je " + "kompatibilný s aktuálnym runom. " + "Nezhodné polia: " + + ", ".join( + mismatches + ) + ) + + +def extract_resume_state( + partial_payload: dict[str, Any], + *, + selected_question_ids: set[str], +) -> tuple[ + dict[str, dict[str, Any]], + list[str], +]: + successful: dict[ + str, + dict[str, Any], + ] = {} + + failed_ids: list[str] = [] + seen_ids: set[str] = set() + + for index, raw_result in enumerate( + partial_payload["results"], + start=1, + ): + if not isinstance( + raw_result, + dict, + ): + raise ValueError( + "Partial " + f"results[{index}] " + "musí byť JSON objekt." + ) + + question_id = raw_result.get( + "id" + ) + + if ( + not isinstance( + question_id, + str, + ) + or not question_id + ): + raise ValueError( + "Partial " + f"results[{index}] " + "nemá platné id." + ) + + if question_id in seen_ids: + raise ValueError( + "Partial obsahuje " + "duplicitné result ID: " + f"{question_id}" + ) + + seen_ids.add( + question_id + ) + + if ( + question_id + not in selected_question_ids + ): + raise ValueError( + "Partial obsahuje result " + "mimo aktuálneho výberu: " + f"{question_id}" + ) + + if raw_result.get( + "error" + ): + failed_ids.append( + question_id + ) + + continue + + successful[ + question_id + ] = raw_result + + return ( + successful, + failed_ids, + ) + + +def ordered_results( + questions: list[dict[str, Any]], + results_by_id: dict[ + str, + dict[str, Any], + ], +) -> list[dict[str, Any]]: + return [ + results_by_id[ + str(question["id"]) + ] + for question in questions + if str(question["id"]) + in results_by_id + ] + + +def determine_run_status( + results: list[dict[str, Any]], + *, + expected_total: int, +) -> str: + if len(results) < expected_total: + return "partial" + + if any( + result.get("error") + for result in results + ): + return ( + "complete_with_errors" + ) + + return "complete" + + +def build_partial_payload( + *, + status: str, + configuration: dict[str, Any], + results: list[dict[str, Any]], + expected_total: int, +) -> dict[str, Any]: + successful_count = sum( + 1 + for result in results + if not result.get( + "error" + ) + ) + + return { + "generated_at": ( + datetime.now( + timezone.utc + ).isoformat() + ), + "status": status, + "configuration": ( + configuration + ), + "attempted_so_far": len( + results + ), + "successful_so_far": ( + successful_count + ), + "remaining_for_clean_completion": ( + expected_total + - successful_count + ), + "expected_total": ( + expected_total + ), + "summary": ( + summarize_results( + results + ) + ), + "results": results, + } + + +def prepare_output_state( + *, + json_path: Path, + csv_path: Path, + partial_path: Path, + resume: bool, + overwrite: bool, +) -> None: + if resume: + if not partial_path.exists(): + raise FileNotFoundError( + "--resume bol zadaný, " + "ale partial neexistuje: " + f"{partial_path}" + ) + + return + + if overwrite: + for path in ( + partial_path, + json_path, + csv_path, + ): + if path.exists(): + path.unlink() + + return + + if partial_path.exists(): + raise FileExistsError( + "Existuje partial výsledok: " + f"{partial_path}. " + "Použi --resume alebo " + "--overwrite." + ) + + existing_final = [ + path + for path in ( + json_path, + csv_path, + ) + if path.exists() + ] + + if existing_final: + raise FileExistsError( + "Výsledný súbor už existuje: " + + ", ".join( + str(path) + for path in existing_final + ) + + ". Použi --overwrite." + ) + + +def print_summary( + summary: dict[str, Any], +) -> None: + print() + print( + "RAG answer evaluation" + ) + print( + "=" * 78 + ) + print( + f"Total: " + f"{summary['total']}" + ) + print( + f"Completed: " + f"{summary['completed']}" + ) + print( + f"Errors: " + f"{summary['errors']}" + ) + print( + "Tool call rate: " + f"{summary['tool_call_rate']:.3f}" + ) + print( + "Answer contains: " + f"{summary['answer_contains_score']:.3f}" + ) + print( + "Source URL: " + f"{summary['source_url_score']:.3f}" + ) + print( + "Should answer: " + f"{summary['should_answer_score']:.3f}" + ) + print( + "Overall: " + f"{summary['overall_score']:.3f}" + ) + print( + "Strict pass: " + f"{summary['strict_pass_count']}" + f"/{summary['total']} " + f"({summary['strict_pass_rate']:.3f})" + ) + print( + "Mean latency: " + f"{summary['mean_latency_seconds']:.3f} s" + ) + print( + "Total tokens: " + f"{summary['total_tokens']}" + ) + print( + "=" * 78 + ) + print() diff --git a/test/test_evaluate_rag_answers.py b/test/test_evaluate_rag_answers.py new file mode 100644 index 0000000..c6eefa3 --- /dev/null +++ b/test/test_evaluate_rag_answers.py @@ -0,0 +1,580 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +import evaluation.evaluate_rag_answers as evaluator + + +def sample_question( + question_id: str = "q0001", + *, + question: str = ( + "Aká je téma práce?" + ), +) -> dict[str, Any]: + return { + "id": question_id, + "split": "dev", + "category": "specific_fact", + "difficulty": "medium", + "question": question, + "expected_documents": [ + "pages/test/README.md" + ], + "expected_source_urls": [ + "https://example.test/source" + ], + "expected_answer_contains": [ + "test" + ], + "should_answer": True, + "note": None, + } + + +def sample_configuration() -> dict[ + str, + Any, +]: + return { + "schema_version": ( + evaluator.PARTIAL_SCHEMA_VERSION + ), + "questions_file": ( + "/tmp/questions.json" + ), + "overrides_file": ( + "/tmp/overrides.json" + ), + "questions_fingerprint": ( + "abc123" + ), + "selected_question_ids": [ + "q0001", + "q0002", + ], + "applied_override_ids": [], + "split": "dev", + "selected_question_count": 2, + "requested_model": ( + "model120-fast" + ), + "operation_id": ( + "retrieve_zpwiki_context" + ), + "openwebui_url": ( + "https://ui.example/api" + ), + "rag_url": ( + "http://localhost:8000/rag" + ), + "timeout": 180, + "max_attempts": 4, + "backoff_base": 1.0, + "backoff_max": 8.0, + "delay": 0.5, + } + + +def test_save_json_results_is_atomic_and_valid( + tmp_path: Path, +) -> None: + path = ( + tmp_path + / "result.partial.json" + ) + + payload = { + "status": "partial", + "results": [ + { + "id": "q0001" + } + ], + } + + evaluator.save_json_results( + path, + payload, + ) + + loaded = json.loads( + path.read_text( + encoding="utf-8" + ) + ) + + assert ( + loaded + == payload + ) + + assert ( + path.read_bytes() + .endswith( + b"\n" + ) + ) + + assert ( + list( + tmp_path.glob( + ".*.tmp" + ) + ) + == [] + ) + + +def test_questions_fingerprint_is_stable_and_detects_change() -> None: + first = [ + sample_question() + ] + + same = [ + sample_question() + ] + + changed = [ + sample_question( + question=( + "Iná otázka" + ) + ) + ] + + assert ( + evaluator.questions_fingerprint( + first + ) + == evaluator.questions_fingerprint( + same + ) + ) + + assert ( + evaluator.questions_fingerprint( + first + ) + != evaluator.questions_fingerprint( + changed + ) + ) + + +def test_validate_answer_questions_accepts_valid_question() -> None: + evaluator.validate_answer_questions( + [ + sample_question() + ] + ) + + +def test_validate_answer_questions_rejects_invalid_types() -> None: + question = ( + sample_question() + ) + + question[ + "should_answer" + ] = "yes" + + with pytest.raises( + ValueError, + match="should_answer", + ): + evaluator.validate_answer_questions( + [ + question + ] + ) + + question = ( + sample_question() + ) + + question[ + "expected_source_urls" + ] = ( + "https://example.test/source" + ) + + with pytest.raises( + ValueError, + match="expected_source_urls", + ): + evaluator.validate_answer_questions( + [ + question + ] + ) + + +def test_validate_resume_compatibility_accepts_matching_configuration() -> None: + configuration = ( + sample_configuration() + ) + + evaluator.validate_resume_compatibility( + configuration, + dict( + configuration + ), + ) + + +def test_validate_resume_compatibility_rejects_model_mismatch() -> None: + partial_configuration = ( + sample_configuration() + ) + + current_configuration = ( + sample_configuration() + ) + + current_configuration[ + "requested_model" + ] = "model2" + + with pytest.raises( + ValueError, + match="requested_model", + ): + evaluator.validate_resume_compatibility( + partial_configuration, + current_configuration, + ) + + +def test_extract_resume_state_skips_success_and_retries_errors() -> None: + partial_payload = { + "results": [ + { + "id": "q0001", + "answer": "ok", + "error": None, + }, + { + "id": "q0002", + "answer": "", + "error": "timeout", + }, + ] + } + + ( + successful, + failed_ids, + ) = evaluator.extract_resume_state( + partial_payload, + selected_question_ids={ + "q0001", + "q0002", + }, + ) + + assert ( + set( + successful + ) + == { + "q0001" + } + ) + + assert ( + failed_ids + == [ + "q0002" + ] + ) + + +def test_extract_resume_state_rejects_duplicate_ids() -> None: + partial_payload = { + "results": [ + { + "id": "q0001", + "error": None, + }, + { + "id": "q0001", + "error": None, + }, + ] + } + + with pytest.raises( + ValueError, + match="duplicitné", + ): + evaluator.extract_resume_state( + partial_payload, + selected_question_ids={ + "q0001" + }, + ) + + +def test_determine_run_status_distinguishes_states() -> None: + assert ( + evaluator.determine_run_status( + [], + expected_total=2, + ) + == "partial" + ) + + assert ( + evaluator.determine_run_status( + [ + { + "id": "q0001", + "error": None, + } + ], + expected_total=2, + ) + == "partial" + ) + + assert ( + evaluator.determine_run_status( + [ + { + "id": "q0001", + "error": None, + }, + { + "id": "q0002", + "error": "timeout", + }, + ], + expected_total=2, + ) + == "complete_with_errors" + ) + + assert ( + evaluator.determine_run_status( + [ + { + "id": "q0001", + "error": None, + }, + { + "id": "q0002", + "error": None, + }, + ], + expected_total=2, + ) + == "complete" + ) + + +def test_prepare_output_state_requires_resume_or_overwrite_for_partial( + tmp_path: Path, +) -> None: + json_path = ( + tmp_path + / "results.json" + ) + + csv_path = ( + tmp_path + / "results.csv" + ) + + partial_path = ( + tmp_path + / "results.partial.json" + ) + + partial_path.write_text( + "{}\n", + encoding="utf-8", + ) + + with pytest.raises( + FileExistsError, + match="--resume", + ): + evaluator.prepare_output_state( + json_path=( + json_path + ), + csv_path=( + csv_path + ), + partial_path=( + partial_path + ), + resume=False, + overwrite=False, + ) + + evaluator.prepare_output_state( + json_path=( + json_path + ), + csv_path=( + csv_path + ), + partial_path=( + partial_path + ), + resume=False, + overwrite=True, + ) + + assert ( + not partial_path.exists() + ) + + +def test_prepare_output_state_requires_existing_partial_for_resume( + tmp_path: Path, +) -> None: + with pytest.raises( + FileNotFoundError, + match="partial neexistuje", + ): + evaluator.prepare_output_state( + json_path=( + tmp_path + / "results.json" + ), + csv_path=( + tmp_path + / "results.csv" + ), + partial_path=( + tmp_path + / "results.partial.json" + ), + resume=True, + overwrite=False, + ) + + +def test_build_run_configuration_contains_fingerprint( + tmp_path: Path, +) -> None: + questions_path = ( + tmp_path + / "questions.json" + ) + + overrides_path = ( + tmp_path + / "overrides.json" + ) + + questions_path.write_text( + "[]\n", + encoding="utf-8", + ) + + overrides_path.write_text( + "{}\n", + encoding="utf-8", + ) + + args = SimpleNamespace( + questions=( + questions_path + ), + overrides=( + overrides_path + ), + split="dev", + model="model120-fast", + timeout=180, + max_attempts=4, + backoff_base=1.0, + backoff_max=8.0, + delay=0.5, + ) + + questions = [ + sample_question() + ] + + configuration = ( + evaluator.build_run_configuration( + args=args, + questions=( + questions + ), + selected_override_ids=[], + operation_id=( + "retrieve_zpwiki_context" + ), + ) + ) + + assert ( + configuration[ + "schema_version" + ] + == evaluator.PARTIAL_SCHEMA_VERSION + ) + + assert ( + configuration[ + "selected_question_ids" + ] + == [ + "q0001" + ] + ) + + assert ( + configuration[ + "questions_fingerprint" + ] + == evaluator.questions_fingerprint( + questions + ) + ) + + +def test_apply_question_overrides_rejects_empty_question( + tmp_path: Path, +) -> None: + questions = [ + sample_question() + ] + + overrides_path = ( + tmp_path + / "overrides.json" + ) + + overrides_path.write_text( + json.dumps( + { + "q0001": { + "question": " " + } + }, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + + with pytest.raises( + ValueError, + match="override question", + ): + evaluator.apply_question_overrides( + questions, + overrides_path, + )