139 lines
4.2 KiB
JSON
139 lines
4.2 KiB
JSON
{
|
|
"generated_at": "2026-08-15T22:25:13.203224+00:00",
|
|
"status": "complete",
|
|
"configuration": {
|
|
"questions_file": "/home/janko/DP/zp-agent/evaluation/json_files/questions.json",
|
|
"overrides_file": "/home/janko/DP/zp-agent/evaluation/json_files/rag_answer_overrides.json",
|
|
"applied_override_ids": [
|
|
"q0010"
|
|
],
|
|
"split": "dev",
|
|
"selected_question_count": 1,
|
|
"requested_model": "model120-fast",
|
|
"openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions",
|
|
"rag_url": "http://localhost:8000/rag",
|
|
"timeout": 180,
|
|
"delay": 0.0
|
|
},
|
|
"summary": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 5.838173,
|
|
"prompt_tokens": 3223,
|
|
"completion_tokens": 145,
|
|
"total_tokens": 3368,
|
|
"by_category": {
|
|
"adversarial_year": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 5.838173,
|
|
"prompt_tokens": 3223,
|
|
"completion_tokens": 145,
|
|
"total_tokens": 3368
|
|
}
|
|
},
|
|
"by_difficulty": {
|
|
"hard": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 5.838173,
|
|
"prompt_tokens": 3223,
|
|
"completion_tokens": 145,
|
|
"total_tokens": 3368
|
|
}
|
|
}
|
|
},
|
|
"results": [
|
|
{
|
|
"id": "q0010",
|
|
"split": "dev",
|
|
"category": "adversarial_year",
|
|
"difficulty": "hard",
|
|
"question": "Je rok 2016 rokom diplomovej práce Jána Holpa? Ak nie, čo rok 2016 označuje a v ktorom roku je uvedená jeho diplomová práca?",
|
|
"override_applied": true,
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016",
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"note": "Model má vysvetliť, že prvý rok je rok začiatku štúdia, ak to zdroj tak uvádza.",
|
|
"answer": "Rok 2016 nie je rokom diplomovej práce Jána Holpa, ale označuje rok začiatku jeho štúdia. Jeho diplomová práca s názvom „Systém získavania informácií v slovenskom jazyku\" je uvedená v roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Ján Holp diplomová práca rok",
|
|
"limit": 5
|
|
},
|
|
"latency_seconds": 0.039992,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.785037,
|
|
"tool_latency_seconds": 0.039992,
|
|
"final_model_latency_seconds": 4.012958,
|
|
"total_latency_seconds": 5.838173,
|
|
"usage": {
|
|
"prompt_tokens": 3223,
|
|
"completion_tokens": 145,
|
|
"total_tokens": 3368
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true,
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
}
|
|
]
|
|
}
|