911 lines
18 KiB
Python
911 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(
|
|
0,
|
|
str(PROJECT_ROOT),
|
|
)
|
|
|
|
|
|
from evaluation.rag_answer_data import (
|
|
DEFAULT_DELAY,
|
|
apply_question_overrides,
|
|
questions_fingerprint,
|
|
result_prefix,
|
|
select_questions,
|
|
validate_answer_questions,
|
|
validate_string_list,
|
|
)
|
|
from evaluation.rag_answer_state import (
|
|
PARTIAL_SCHEMA_VERSION,
|
|
build_base_result,
|
|
build_error_result,
|
|
build_partial_payload,
|
|
build_run_configuration,
|
|
determine_run_status,
|
|
extract_resume_state,
|
|
load_partial_payload,
|
|
ordered_results,
|
|
prepare_output_state,
|
|
print_summary,
|
|
save_csv_results,
|
|
save_json_results,
|
|
validate_resume_compatibility,
|
|
)
|
|
from evaluation.rag_metrics import (
|
|
evaluate_answer,
|
|
summarize_results,
|
|
)
|
|
from evaluation.rag_runner import (
|
|
DEFAULT_BACKOFF_BASE,
|
|
DEFAULT_BACKOFF_MAX,
|
|
DEFAULT_MAX_ATTEMPTS,
|
|
DEFAULT_MODEL,
|
|
DEFAULT_TIMEOUT,
|
|
LOCAL_OPENAPI_URL,
|
|
build_rag_tool,
|
|
load_env_value,
|
|
request_json,
|
|
run_question,
|
|
)
|
|
from evaluation.retrieval_runner import (
|
|
load_questions,
|
|
)
|
|
|
|
|
|
EVALUATION_DIR = (
|
|
PROJECT_ROOT
|
|
/ "evaluation"
|
|
)
|
|
|
|
JSON_FILES_DIR = (
|
|
EVALUATION_DIR
|
|
/ "json_files"
|
|
)
|
|
|
|
QUESTIONS_PATH = (
|
|
JSON_FILES_DIR
|
|
/ "questions.json"
|
|
)
|
|
|
|
RAG_ANSWER_OVERRIDES_PATH = (
|
|
JSON_FILES_DIR
|
|
/ "rag_answer_overrides.json"
|
|
)
|
|
|
|
RESULTS_DIR = (
|
|
EVALUATION_DIR
|
|
/ "results"
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description=(
|
|
"Answer-level RAG evaluácia "
|
|
"cez OpenWebUI, Model120 "
|
|
"a ZP Agent."
|
|
)
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--split",
|
|
choices=(
|
|
"dev",
|
|
"test",
|
|
"all",
|
|
),
|
|
default="dev",
|
|
help=(
|
|
"Časť datasetu. "
|
|
"Predvolené je dev."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=None,
|
|
help=(
|
|
"Maximálny počet otázok "
|
|
"na vyhodnotenie."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--model",
|
|
default=DEFAULT_MODEL,
|
|
help=(
|
|
"OpenWebUI model ID. "
|
|
f"Predvolené: {DEFAULT_MODEL}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--questions",
|
|
type=Path,
|
|
default=QUESTIONS_PATH,
|
|
help=(
|
|
"Cesta k questions.json."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--overrides",
|
|
type=Path,
|
|
default=RAG_ANSWER_OVERRIDES_PATH,
|
|
help=(
|
|
"Cesta k answer-level "
|
|
"override súboru."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--results-dir",
|
|
type=Path,
|
|
default=RESULTS_DIR,
|
|
help=(
|
|
"Adresár pre výsledky."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=int,
|
|
default=DEFAULT_TIMEOUT,
|
|
help=(
|
|
"HTTP timeout v sekundách. "
|
|
f"Predvolené: {DEFAULT_TIMEOUT}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-attempts",
|
|
type=int,
|
|
default=DEFAULT_MAX_ATTEMPTS,
|
|
help=(
|
|
"Maximálny počet HTTP pokusov "
|
|
"pre retryable chyby. "
|
|
f"Predvolené: "
|
|
f"{DEFAULT_MAX_ATTEMPTS}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--backoff-base",
|
|
type=float,
|
|
default=DEFAULT_BACKOFF_BASE,
|
|
help=(
|
|
"Počiatočný exponential "
|
|
"backoff v sekundách. "
|
|
f"Predvolené: "
|
|
f"{DEFAULT_BACKOFF_BASE}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--backoff-max",
|
|
type=float,
|
|
default=DEFAULT_BACKOFF_MAX,
|
|
help=(
|
|
"Maximálny exponential "
|
|
"backoff v sekundách. "
|
|
f"Predvolené: "
|
|
f"{DEFAULT_BACKOFF_MAX}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--delay",
|
|
type=float,
|
|
default=DEFAULT_DELAY,
|
|
help=(
|
|
"Pauza medzi otázkami "
|
|
"v sekundách. "
|
|
f"Predvolené: "
|
|
f"{DEFAULT_DELAY}."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--question-id",
|
|
action="append",
|
|
default=[],
|
|
help=(
|
|
"Vyhodnoť iba konkrétne ID. "
|
|
"Parameter možno použiť "
|
|
"opakovane."
|
|
),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--fail-fast",
|
|
action="store_true",
|
|
help=(
|
|
"Pri prvej chybe "
|
|
"ukonči evaluáciu."
|
|
),
|
|
)
|
|
|
|
output_group = (
|
|
parser.add_mutually_exclusive_group()
|
|
)
|
|
|
|
output_group.add_argument(
|
|
"--resume",
|
|
action="store_true",
|
|
help=(
|
|
"Pokračuj z kompatibilného "
|
|
".partial.json. Úspešné otázky "
|
|
"sa preskočia a chybové sa "
|
|
"vyhodnotia znova."
|
|
),
|
|
)
|
|
|
|
output_group.add_argument(
|
|
"--overwrite",
|
|
action="store_true",
|
|
help=(
|
|
"Začni nový run a povoľ "
|
|
"prepísanie existujúcich "
|
|
"výsledkov."
|
|
),
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if (
|
|
args.limit is not None
|
|
and args.limit <= 0
|
|
):
|
|
parser.error(
|
|
"--limit musí byť > 0"
|
|
)
|
|
|
|
if args.timeout <= 0:
|
|
parser.error(
|
|
"--timeout musí byť > 0"
|
|
)
|
|
|
|
if args.max_attempts <= 0:
|
|
parser.error(
|
|
"--max-attempts musí byť > 0"
|
|
)
|
|
|
|
if args.backoff_base < 0:
|
|
parser.error(
|
|
"--backoff-base "
|
|
"nesmie byť záporné"
|
|
)
|
|
|
|
if args.backoff_max < 0:
|
|
parser.error(
|
|
"--backoff-max "
|
|
"nesmie byť záporné"
|
|
)
|
|
|
|
if args.delay < 0:
|
|
parser.error(
|
|
"--delay nesmie byť záporné"
|
|
)
|
|
|
|
return args
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
|
|
try:
|
|
all_questions = (
|
|
load_questions(
|
|
args.questions
|
|
)
|
|
)
|
|
|
|
(
|
|
all_questions,
|
|
applied_override_ids,
|
|
) = apply_question_overrides(
|
|
all_questions,
|
|
args.overrides,
|
|
)
|
|
|
|
questions = select_questions(
|
|
all_questions,
|
|
split=args.split,
|
|
question_ids=(
|
|
args.question_id
|
|
),
|
|
limit=args.limit,
|
|
)
|
|
|
|
if not questions:
|
|
raise ValueError(
|
|
"Žiadne otázky "
|
|
"nezodpovedajú filtru."
|
|
)
|
|
|
|
validate_answer_questions(
|
|
questions
|
|
)
|
|
|
|
except (
|
|
FileNotFoundError,
|
|
ValueError,
|
|
json.JSONDecodeError,
|
|
) as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
selected_ids = {
|
|
str(
|
|
question["id"]
|
|
)
|
|
for question in questions
|
|
}
|
|
|
|
selected_override_ids = [
|
|
question_id
|
|
for question_id
|
|
in applied_override_ids
|
|
if question_id
|
|
in selected_ids
|
|
]
|
|
|
|
prefix = result_prefix(
|
|
split=args.split,
|
|
limit=args.limit,
|
|
question_ids=(
|
|
args.question_id
|
|
),
|
|
)
|
|
|
|
json_path = (
|
|
args.results_dir
|
|
/ f"{prefix}.json"
|
|
)
|
|
|
|
csv_path = (
|
|
args.results_dir
|
|
/ f"{prefix}.csv"
|
|
)
|
|
|
|
partial_path = (
|
|
args.results_dir
|
|
/ f"{prefix}.partial.json"
|
|
)
|
|
|
|
try:
|
|
prepare_output_state(
|
|
json_path=json_path,
|
|
csv_path=csv_path,
|
|
partial_path=(
|
|
partial_path
|
|
),
|
|
resume=args.resume,
|
|
overwrite=args.overwrite,
|
|
)
|
|
|
|
except (
|
|
FileNotFoundError,
|
|
FileExistsError,
|
|
) as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
try:
|
|
openwebui_api_key = (
|
|
load_env_value(
|
|
"OPENWEBUI_API_KEY",
|
|
allow_env_file=False,
|
|
)
|
|
)
|
|
|
|
search_api_key = (
|
|
load_env_value(
|
|
"SEARCH_API_KEY"
|
|
)
|
|
)
|
|
|
|
except RuntimeError as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
try:
|
|
openapi = request_json(
|
|
LOCAL_OPENAPI_URL,
|
|
timeout=args.timeout,
|
|
max_attempts=(
|
|
args.max_attempts
|
|
),
|
|
backoff_base=(
|
|
args.backoff_base
|
|
),
|
|
backoff_max=(
|
|
args.backoff_max
|
|
),
|
|
)
|
|
|
|
(
|
|
operation_id,
|
|
rag_tool,
|
|
) = build_rag_tool(
|
|
openapi
|
|
)
|
|
|
|
except RuntimeError as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
configuration = (
|
|
build_run_configuration(
|
|
args=args,
|
|
questions=questions,
|
|
selected_override_ids=(
|
|
selected_override_ids
|
|
),
|
|
operation_id=(
|
|
operation_id
|
|
),
|
|
)
|
|
)
|
|
|
|
results_by_id: dict[
|
|
str,
|
|
dict[str, Any],
|
|
] = {}
|
|
|
|
previous_failed_ids: list[
|
|
str
|
|
] = []
|
|
|
|
if args.resume:
|
|
try:
|
|
partial_payload = (
|
|
load_partial_payload(
|
|
partial_path
|
|
)
|
|
)
|
|
|
|
validate_resume_compatibility(
|
|
partial_payload[
|
|
"configuration"
|
|
],
|
|
configuration,
|
|
)
|
|
|
|
(
|
|
results_by_id,
|
|
previous_failed_ids,
|
|
) = extract_resume_state(
|
|
partial_payload,
|
|
selected_question_ids=(
|
|
selected_ids
|
|
),
|
|
)
|
|
|
|
except (
|
|
FileNotFoundError,
|
|
ValueError,
|
|
json.JSONDecodeError,
|
|
) as exc:
|
|
print(
|
|
f"ERROR: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
return 2
|
|
|
|
pending_questions = [
|
|
question
|
|
for question in questions
|
|
if str(
|
|
question["id"]
|
|
)
|
|
not in results_by_id
|
|
]
|
|
|
|
print()
|
|
print(
|
|
"RAG answer evaluation"
|
|
)
|
|
print(
|
|
"=" * 60
|
|
)
|
|
print(
|
|
f"Dataset: "
|
|
f"{args.questions}"
|
|
)
|
|
print(
|
|
f"Overrides: "
|
|
f"{args.overrides}"
|
|
)
|
|
print(
|
|
"Applied overrides: "
|
|
f"{len(selected_override_ids)}"
|
|
)
|
|
|
|
if selected_override_ids:
|
|
print(
|
|
"Override IDs: "
|
|
+ ", ".join(
|
|
selected_override_ids
|
|
)
|
|
)
|
|
|
|
print(
|
|
f"Model: "
|
|
f"{args.model}"
|
|
)
|
|
print(
|
|
f"Tool: "
|
|
f"{operation_id}"
|
|
)
|
|
print(
|
|
f"Split: "
|
|
f"{args.split}"
|
|
)
|
|
print(
|
|
f"Questions: "
|
|
f"{len(questions)}"
|
|
)
|
|
print(
|
|
"Retry: "
|
|
f"{args.max_attempts} "
|
|
"pokusy"
|
|
)
|
|
print(
|
|
"Backoff: "
|
|
f"{args.backoff_base}s "
|
|
"→ max "
|
|
f"{args.backoff_max}s"
|
|
)
|
|
print(
|
|
f"Delay: "
|
|
f"{args.delay}s"
|
|
)
|
|
|
|
if args.resume:
|
|
print(
|
|
"Resume: áno"
|
|
)
|
|
print(
|
|
" úspešne obnovené: "
|
|
f"{len(results_by_id)}"
|
|
)
|
|
print(
|
|
" predchádzajúce chyby "
|
|
"na retry: "
|
|
f"{len(previous_failed_ids)}"
|
|
)
|
|
print(
|
|
" zostáva spustiť: "
|
|
f"{len(pending_questions)}"
|
|
)
|
|
|
|
else:
|
|
print(
|
|
"Resume: nie"
|
|
)
|
|
|
|
print(
|
|
"=" * 60
|
|
)
|
|
print()
|
|
|
|
total = len(
|
|
questions
|
|
)
|
|
|
|
pending_total = len(
|
|
pending_questions
|
|
)
|
|
|
|
processed_pending = 0
|
|
stopped_early = False
|
|
|
|
for index, question in enumerate(
|
|
questions,
|
|
start=1,
|
|
):
|
|
question_id = str(
|
|
question["id"]
|
|
)
|
|
|
|
question_text = str(
|
|
question["question"]
|
|
)
|
|
|
|
if (
|
|
question_id
|
|
in results_by_id
|
|
):
|
|
print(
|
|
f"[{index:04d}/"
|
|
f"{total:04d}] "
|
|
f"{question_id}: "
|
|
"SKIP (resume)"
|
|
)
|
|
|
|
continue
|
|
|
|
print(
|
|
f"[{index:04d}/"
|
|
f"{total:04d}] "
|
|
f"{question_id}: "
|
|
f"{question_text}"
|
|
)
|
|
|
|
base_result = (
|
|
build_base_result(
|
|
question=question,
|
|
question_id=(
|
|
question_id
|
|
),
|
|
question_text=(
|
|
question_text
|
|
),
|
|
selected_override_ids=(
|
|
selected_override_ids
|
|
),
|
|
requested_model=(
|
|
args.model
|
|
),
|
|
)
|
|
)
|
|
|
|
try:
|
|
run_result = (
|
|
run_question(
|
|
question_text,
|
|
model=args.model,
|
|
operation_id=(
|
|
operation_id
|
|
),
|
|
rag_tool=rag_tool,
|
|
openwebui_api_key=(
|
|
openwebui_api_key
|
|
),
|
|
search_api_key=(
|
|
search_api_key
|
|
),
|
|
timeout=args.timeout,
|
|
max_attempts=(
|
|
args.max_attempts
|
|
),
|
|
backoff_base=(
|
|
args.backoff_base
|
|
),
|
|
backoff_max=(
|
|
args.backoff_max
|
|
),
|
|
)
|
|
)
|
|
|
|
scores = evaluate_answer(
|
|
question,
|
|
run_result[
|
|
"answer"
|
|
],
|
|
tool_called=bool(
|
|
run_result[
|
|
"tool_called"
|
|
]
|
|
),
|
|
)
|
|
|
|
result = {
|
|
**base_result,
|
|
**run_result,
|
|
**scores,
|
|
"error": None,
|
|
}
|
|
|
|
print(
|
|
" tool:",
|
|
result[
|
|
"tool_called"
|
|
],
|
|
"| answer:",
|
|
f"{result['answer_contains_score']:.2f}",
|
|
"| source:",
|
|
f"{result['source_url_score']:.2f}",
|
|
"| strict:",
|
|
result[
|
|
"strict_pass"
|
|
],
|
|
"| latency:",
|
|
f"{result['total_latency_seconds']:.2f}s",
|
|
)
|
|
|
|
except Exception as exc:
|
|
result = (
|
|
build_error_result(
|
|
base_result,
|
|
exc,
|
|
)
|
|
)
|
|
|
|
print(
|
|
" ERROR:",
|
|
result[
|
|
"error"
|
|
],
|
|
)
|
|
|
|
if args.fail_fast:
|
|
stopped_early = True
|
|
|
|
results_by_id[
|
|
question_id
|
|
] = result
|
|
|
|
processed_pending += 1
|
|
|
|
current_results = (
|
|
ordered_results(
|
|
questions,
|
|
results_by_id,
|
|
)
|
|
)
|
|
|
|
partial_payload = (
|
|
build_partial_payload(
|
|
status="partial",
|
|
configuration=(
|
|
configuration
|
|
),
|
|
results=(
|
|
current_results
|
|
),
|
|
expected_total=(
|
|
total
|
|
),
|
|
)
|
|
)
|
|
|
|
save_json_results(
|
|
partial_path,
|
|
partial_payload,
|
|
)
|
|
|
|
if stopped_early:
|
|
break
|
|
|
|
if (
|
|
args.delay > 0
|
|
and processed_pending
|
|
< pending_total
|
|
):
|
|
time.sleep(
|
|
args.delay
|
|
)
|
|
|
|
results = ordered_results(
|
|
questions,
|
|
results_by_id,
|
|
)
|
|
|
|
status = determine_run_status(
|
|
results,
|
|
expected_total=(
|
|
total
|
|
),
|
|
)
|
|
|
|
summary = (
|
|
summarize_results(
|
|
results
|
|
)
|
|
)
|
|
|
|
payload = {
|
|
"generated_at": (
|
|
datetime.now(
|
|
timezone.utc
|
|
).isoformat()
|
|
),
|
|
"status": status,
|
|
"configuration": (
|
|
configuration
|
|
),
|
|
"summary": summary,
|
|
"results": results,
|
|
}
|
|
|
|
save_json_results(
|
|
json_path,
|
|
payload,
|
|
)
|
|
|
|
save_csv_results(
|
|
csv_path,
|
|
results,
|
|
)
|
|
|
|
if status == "complete":
|
|
if partial_path.exists():
|
|
partial_path.unlink()
|
|
|
|
else:
|
|
save_json_results(
|
|
partial_path,
|
|
build_partial_payload(
|
|
status=status,
|
|
configuration=(
|
|
configuration
|
|
),
|
|
results=results,
|
|
expected_total=(
|
|
total
|
|
),
|
|
),
|
|
)
|
|
|
|
print_summary(
|
|
summary
|
|
)
|
|
|
|
print(
|
|
"Výsledky:"
|
|
)
|
|
print(
|
|
f" JSON: "
|
|
f"{json_path}"
|
|
)
|
|
print(
|
|
f" CSV: "
|
|
f"{csv_path}"
|
|
)
|
|
|
|
if partial_path.exists():
|
|
print(
|
|
f" Partial: "
|
|
f"{partial_path}"
|
|
)
|
|
|
|
print(
|
|
f" Status: "
|
|
f"{status}"
|
|
)
|
|
|
|
if summary[
|
|
"errors"
|
|
]:
|
|
return 1
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(
|
|
main()
|
|
)
|