606 lines
15 KiB
Python
606 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
|
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(
|
|
0,
|
|
str(PROJECT_ROOT),
|
|
)
|
|
|
|
|
|
from tools.evaluation.rag_delivery import finalize_rag_answer
|
|
from tools.rag.evaluation.rag_answer_data import (
|
|
DEFAULT_DELAY,
|
|
apply_question_overrides,
|
|
questions_fingerprint as questions_fingerprint,
|
|
result_prefix,
|
|
select_questions,
|
|
validate_answer_questions,
|
|
validate_string_list as validate_string_list,
|
|
)
|
|
from tools.rag.evaluation.rag_answer_state import (
|
|
PARTIAL_SCHEMA_VERSION as PARTIAL_SCHEMA_VERSION,
|
|
build_base_result,
|
|
build_error_result,
|
|
build_partial_payload,
|
|
build_run_configuration,
|
|
determine_run_status,
|
|
extract_resume_state,
|
|
load_partial_payload,
|
|
ordered_results,
|
|
prepare_output_state,
|
|
print_summary,
|
|
save_csv_results,
|
|
save_json_results,
|
|
validate_resume_compatibility,
|
|
)
|
|
|
|
# These three explicit aliases preserve the evaluator's public helper API.
|
|
# Existing replay callers import them through this module.
|
|
from tools.rag.evaluation.rag_metrics import (
|
|
evaluate_answer,
|
|
summarize_results,
|
|
)
|
|
from tools.rag.evaluation.rag_runner import (
|
|
DEFAULT_BACKOFF_BASE,
|
|
DEFAULT_BACKOFF_MAX,
|
|
DEFAULT_MAX_ATTEMPTS,
|
|
DEFAULT_MODEL,
|
|
DEFAULT_TIMEOUT,
|
|
LOCAL_OPENAPI_URL,
|
|
build_rag_tool,
|
|
load_env_value,
|
|
request_json,
|
|
run_question,
|
|
)
|
|
from tools.rag.evaluation.retrieval_runner import (
|
|
load_questions,
|
|
)
|
|
|
|
EVALUATION_DIR = PROJECT_ROOT / "experiments/rag"
|
|
|
|
JSON_FILES_DIR = EVALUATION_DIR / "datasets"
|
|
|
|
QUESTIONS_PATH = JSON_FILES_DIR / "questions.json"
|
|
|
|
RAG_ANSWER_OVERRIDES_PATH = JSON_FILES_DIR / "rag_answer_overrides.json"
|
|
|
|
RESULTS_DIR = EVALUATION_DIR / "results"
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description=("Answer-level RAG evaluácia " "cez OpenWebUI, Model120 " "a ZP Agent.")
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--split",
|
|
choices=(
|
|
"dev",
|
|
"test",
|
|
"all",
|
|
),
|
|
default="dev",
|
|
help=("Časť datasetu. " "Predvolené je dev."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=None,
|
|
help=("Maximálny počet otázok " "na vyhodnotenie."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--model",
|
|
default=DEFAULT_MODEL,
|
|
help=("OpenWebUI model ID. " f"Predvolené: {DEFAULT_MODEL}."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--questions",
|
|
type=Path,
|
|
default=QUESTIONS_PATH,
|
|
help=("Cesta k questions.json."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--overrides",
|
|
type=Path,
|
|
default=RAG_ANSWER_OVERRIDES_PATH,
|
|
help=("Cesta k answer-level " "override súboru."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--results-dir",
|
|
type=Path,
|
|
default=RESULTS_DIR,
|
|
help=("Adresár pre výsledky."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=int,
|
|
default=DEFAULT_TIMEOUT,
|
|
help=("HTTP timeout v sekundách. " f"Predvolené: {DEFAULT_TIMEOUT}."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-attempts",
|
|
type=int,
|
|
default=DEFAULT_MAX_ATTEMPTS,
|
|
help=(
|
|
"Maximálny počet HTTP pokusov "
|
|
"pre retryable chyby. "
|
|
f"Predvolené: "
|
|
f"{DEFAULT_MAX_ATTEMPTS}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--backoff-base",
|
|
type=float,
|
|
default=DEFAULT_BACKOFF_BASE,
|
|
help=(
|
|
"Počiatočný exponential "
|
|
"backoff v sekundách. "
|
|
f"Predvolené: "
|
|
f"{DEFAULT_BACKOFF_BASE}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--backoff-max",
|
|
type=float,
|
|
default=DEFAULT_BACKOFF_MAX,
|
|
help=(
|
|
"Maximálny exponential "
|
|
"backoff v sekundách. "
|
|
f"Predvolené: "
|
|
f"{DEFAULT_BACKOFF_MAX}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--delay",
|
|
type=float,
|
|
default=DEFAULT_DELAY,
|
|
help=("Pauza medzi otázkami " "v sekundách. " f"Predvolené: " f"{DEFAULT_DELAY}."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--question-id",
|
|
action="append",
|
|
default=[],
|
|
help=("Vyhodnoť iba konkrétne ID. " "Parameter možno použiť " "opakovane."),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--fail-fast",
|
|
action="store_true",
|
|
help=("Pri prvej chybe " "ukonči evaluáciu."),
|
|
)
|
|
|
|
output_group = parser.add_mutually_exclusive_group()
|
|
|
|
output_group.add_argument(
|
|
"--resume",
|
|
action="store_true",
|
|
help=(
|
|
"Pokračuj z kompatibilného "
|
|
".partial.json. Úspešné otázky "
|
|
"sa preskočia a chybové sa "
|
|
"vyhodnotia znova."
|
|
),
|
|
)
|
|
|
|
output_group.add_argument(
|
|
"--overwrite",
|
|
action="store_true",
|
|
help=("Začni nový run a povoľ " "prepísanie existujúcich " "výsledkov."),
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.limit is not None and args.limit <= 0:
|
|
parser.error("--limit musí byť > 0")
|
|
|
|
if args.timeout <= 0:
|
|
parser.error("--timeout musí byť > 0")
|
|
|
|
if args.max_attempts <= 0:
|
|
parser.error("--max-attempts musí byť > 0")
|
|
|
|
if args.backoff_base < 0:
|
|
parser.error("--backoff-base " "nesmie byť záporné")
|
|
|
|
if args.backoff_max < 0:
|
|
parser.error("--backoff-max " "nesmie byť záporné")
|
|
|
|
if args.delay < 0:
|
|
parser.error("--delay nesmie byť záporné")
|
|
|
|
return args
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
|
|
try:
|
|
all_questions = load_questions(args.questions)
|
|
|
|
(
|
|
all_questions,
|
|
applied_override_ids,
|
|
) = apply_question_overrides(
|
|
all_questions,
|
|
args.overrides,
|
|
)
|
|
|
|
questions = select_questions(
|
|
all_questions,
|
|
split=args.split,
|
|
question_ids=(args.question_id),
|
|
limit=args.limit,
|
|
)
|
|
|
|
if not questions:
|
|
raise ValueError("Žiadne otázky " "nezodpovedajú filtru.")
|
|
|
|
validate_answer_questions(questions)
|
|
|
|
except (
|
|
FileNotFoundError,
|
|
ValueError,
|
|
json.JSONDecodeError,
|
|
) as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
selected_ids = {str(question["id"]) for question in questions}
|
|
|
|
selected_override_ids = [
|
|
question_id for question_id in applied_override_ids if question_id in selected_ids
|
|
]
|
|
|
|
prefix = result_prefix(
|
|
split=args.split,
|
|
limit=args.limit,
|
|
question_ids=(args.question_id),
|
|
)
|
|
|
|
json_path = args.results_dir / f"{prefix}.json"
|
|
|
|
csv_path = args.results_dir / f"{prefix}.csv"
|
|
|
|
partial_path = args.results_dir / f"{prefix}.partial.json"
|
|
|
|
try:
|
|
prepare_output_state(
|
|
json_path=json_path,
|
|
csv_path=csv_path,
|
|
partial_path=(partial_path),
|
|
resume=args.resume,
|
|
overwrite=args.overwrite,
|
|
)
|
|
|
|
except (
|
|
FileNotFoundError,
|
|
FileExistsError,
|
|
) as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
try:
|
|
openwebui_api_key = load_env_value(
|
|
"OPENWEBUI_API_KEY",
|
|
allow_env_file=False,
|
|
)
|
|
|
|
search_api_key = load_env_value("SEARCH_API_KEY")
|
|
|
|
except RuntimeError as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
try:
|
|
openapi = request_json(
|
|
LOCAL_OPENAPI_URL,
|
|
timeout=args.timeout,
|
|
max_attempts=(args.max_attempts),
|
|
backoff_base=(args.backoff_base),
|
|
backoff_max=(args.backoff_max),
|
|
)
|
|
|
|
(
|
|
operation_id,
|
|
rag_tool,
|
|
) = build_rag_tool(openapi)
|
|
|
|
except RuntimeError as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
configuration = build_run_configuration(
|
|
args=args,
|
|
questions=questions,
|
|
selected_override_ids=(selected_override_ids),
|
|
operation_id=(operation_id),
|
|
)
|
|
|
|
results_by_id: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
previous_failed_ids: list[str] = []
|
|
|
|
if args.resume:
|
|
try:
|
|
partial_payload = load_partial_payload(partial_path)
|
|
|
|
validate_resume_compatibility(
|
|
partial_payload["configuration"],
|
|
configuration,
|
|
)
|
|
|
|
(
|
|
results_by_id,
|
|
previous_failed_ids,
|
|
) = extract_resume_state(
|
|
partial_payload,
|
|
selected_question_ids=(selected_ids),
|
|
)
|
|
|
|
except (
|
|
FileNotFoundError,
|
|
ValueError,
|
|
json.JSONDecodeError,
|
|
) as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
pending_questions = [
|
|
question for question in questions if str(question["id"]) not in results_by_id
|
|
]
|
|
|
|
print()
|
|
print("RAG answer evaluation")
|
|
print("=" * 60)
|
|
print(f"Dataset: " f"{args.questions}")
|
|
print(f"Overrides: " f"{args.overrides}")
|
|
print("Applied overrides: " f"{len(selected_override_ids)}")
|
|
|
|
if selected_override_ids:
|
|
print("Override IDs: " + ", ".join(selected_override_ids))
|
|
|
|
print(f"Model: " f"{args.model}")
|
|
print(f"Tool: " f"{operation_id}")
|
|
print(f"Split: " f"{args.split}")
|
|
print(f"Questions: " f"{len(questions)}")
|
|
print("Retry: " f"{args.max_attempts} " "pokusy")
|
|
print("Backoff: " f"{args.backoff_base}s " "→ max " f"{args.backoff_max}s")
|
|
print(f"Delay: " f"{args.delay}s")
|
|
|
|
if args.resume:
|
|
print("Resume: áno")
|
|
print(" úspešne obnovené: " f"{len(results_by_id)}")
|
|
print(" predchádzajúce chyby " "na retry: " f"{len(previous_failed_ids)}")
|
|
print(" zostáva spustiť: " f"{len(pending_questions)}")
|
|
|
|
else:
|
|
print("Resume: nie")
|
|
|
|
print("=" * 60)
|
|
print()
|
|
|
|
total = len(questions)
|
|
|
|
pending_total = len(pending_questions)
|
|
|
|
processed_pending = 0
|
|
stopped_early = False
|
|
|
|
for index, question in enumerate(
|
|
questions,
|
|
start=1,
|
|
):
|
|
question_id = str(question["id"])
|
|
|
|
question_text = str(question["question"])
|
|
|
|
if question_id in results_by_id:
|
|
print(f"[{index:04d}/" f"{total:04d}] " f"{question_id}: " "SKIP (resume)")
|
|
|
|
continue
|
|
|
|
print(f"[{index:04d}/" f"{total:04d}] " f"{question_id}: " f"{question_text}")
|
|
|
|
base_result = build_base_result(
|
|
question=question,
|
|
question_id=(question_id),
|
|
question_text=(question_text),
|
|
selected_override_ids=(selected_override_ids),
|
|
requested_model=(args.model),
|
|
)
|
|
|
|
try:
|
|
run_result = run_question(
|
|
question_text,
|
|
model=args.model,
|
|
operation_id=(operation_id),
|
|
rag_tool=rag_tool,
|
|
openwebui_api_key=(openwebui_api_key),
|
|
search_api_key=(search_api_key),
|
|
timeout=args.timeout,
|
|
max_attempts=(args.max_attempts),
|
|
backoff_base=(args.backoff_base),
|
|
backoff_max=(args.backoff_max),
|
|
)
|
|
|
|
run_result = finalize_rag_answer(run_result)
|
|
scores = evaluate_answer(
|
|
question,
|
|
run_result["answer"],
|
|
tool_called=bool(run_result["tool_called"]),
|
|
allowed_source_urls=run_result.get("rag_source_urls"),
|
|
)
|
|
|
|
result = {
|
|
**base_result,
|
|
**run_result,
|
|
**scores,
|
|
"strict_pass": bool(
|
|
scores["strict_pass"] and run_result.get("accepted_for_delivery", True)
|
|
),
|
|
"error": None,
|
|
}
|
|
|
|
print(
|
|
" tool:",
|
|
result["tool_called"],
|
|
"| answer:",
|
|
f"{result['answer_contains_score']:.2f}",
|
|
"| source:",
|
|
f"{result['source_url_score']:.2f}",
|
|
"| strict:",
|
|
result["strict_pass"],
|
|
"| latency:",
|
|
f"{result['total_latency_seconds']:.2f}s",
|
|
)
|
|
|
|
except Exception as exc:
|
|
result = build_error_result(
|
|
base_result,
|
|
exc,
|
|
)
|
|
|
|
print(
|
|
" ERROR:",
|
|
result["error"],
|
|
)
|
|
|
|
if args.fail_fast:
|
|
stopped_early = True
|
|
|
|
results_by_id[question_id] = result
|
|
|
|
processed_pending += 1
|
|
|
|
current_results = ordered_results(
|
|
questions,
|
|
results_by_id,
|
|
)
|
|
|
|
partial_payload = build_partial_payload(
|
|
status="partial",
|
|
configuration=(configuration),
|
|
results=(current_results),
|
|
expected_total=(total),
|
|
)
|
|
|
|
save_json_results(
|
|
partial_path,
|
|
partial_payload,
|
|
)
|
|
|
|
if stopped_early:
|
|
break
|
|
|
|
if args.delay > 0 and processed_pending < pending_total:
|
|
time.sleep(args.delay)
|
|
|
|
results = ordered_results(
|
|
questions,
|
|
results_by_id,
|
|
)
|
|
|
|
status = determine_run_status(
|
|
results,
|
|
expected_total=(total),
|
|
)
|
|
|
|
summary = summarize_results(results)
|
|
|
|
payload = {
|
|
"generated_at": (datetime.now(timezone.utc).isoformat()),
|
|
"status": status,
|
|
"configuration": (configuration),
|
|
"summary": summary,
|
|
"results": results,
|
|
}
|
|
|
|
save_json_results(
|
|
json_path,
|
|
payload,
|
|
)
|
|
|
|
save_csv_results(
|
|
csv_path,
|
|
results,
|
|
)
|
|
|
|
if status == "complete":
|
|
if partial_path.exists():
|
|
partial_path.unlink()
|
|
|
|
else:
|
|
save_json_results(
|
|
partial_path,
|
|
build_partial_payload(
|
|
status=status,
|
|
configuration=(configuration),
|
|
results=results,
|
|
expected_total=(total),
|
|
),
|
|
)
|
|
|
|
print_summary(summary)
|
|
|
|
print("Výsledky:")
|
|
print(f" JSON: " f"{json_path}")
|
|
print(f" CSV: " f"{csv_path}")
|
|
|
|
if partial_path.exists():
|
|
print(f" Partial: " f"{partial_path}")
|
|
|
|
print(f" Status: " f"{status}")
|
|
|
|
if summary["errors"]:
|
|
return 1
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|