jp170na/dp-zp-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
e26bbd096f
dp-zp-agent/evaluation/results/rag_answers_dev_selected.json
jp170na e26bbd096f csv a json
2026-08-16 00:08:59 +02:00

355 lines
12 KiB
JSON
Raw Blame History

{
"generated_at": "2026-08-15T22:03:41.030876+00:00",
"status": "complete",
"configuration": {
"questions_file": "/home/janko/DP/zp-agent/evaluation/json_files/questions.json",
"overrides_file": "/home/janko/DP/zp-agent/evaluation/json_files/rag_answer_overrides.json",
"applied_override_ids": [
"q0005",
"q0106",
"q0382",
"q0441"
],
"split": "dev",
"selected_question_count": 4,
"requested_model": "model120-fast",
"openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions",
"rag_url": "http://localhost:8000/rag",
"timeout": 180,
"delay": 0.0
},
"summary": {
"total": 4,
"completed": 4,
"errors": 0,
"tool_call_rate": 1.0,
"answer_contains_score": 0.75,
"source_url_score": 1.0,
"should_answer_score": 1.0,
"overall_score": 0.9375,
"strict_pass_count": 3,
"strict_pass_rate": 0.75,
"mean_latency_seconds": 7.128043,
"prompt_tokens": 23886,
"completion_tokens": 460,
"total_tokens": 24346,
"by_category": {
"paraphrase": {
"total": 4,
"completed": 4,
"errors": 0,
"tool_call_rate": 1.0,
"answer_contains_score": 0.75,
"source_url_score": 1.0,
"should_answer_score": 1.0,
"overall_score": 0.9375,
"strict_pass_count": 3,
"strict_pass_rate": 0.75,
"mean_latency_seconds": 7.128043,
"prompt_tokens": 23886,
"completion_tokens": 460,
"total_tokens": 24346
}
},
"by_difficulty": {
"medium": {
"total": 4,
"completed": 4,
"errors": 0,
"tool_call_rate": 1.0,
"answer_contains_score": 0.75,
"source_url_score": 1.0,
"should_answer_score": 1.0,
"overall_score": 0.9375,
"strict_pass_count": 3,
"strict_pass_rate": 0.75,
"mean_latency_seconds": 7.128043,
"prompt_tokens": 23886,
"completion_tokens": 460,
"total_tokens": 24346
}
}
},
"results": [
{
"id": "q0005",
"split": "dev",
"category": "paraphrase",
"difficulty": "medium",
"question": "Aká téma diplomovej práce je uvedená pri osobe „Ján Holp“?",
"override_applied": true,
"requested_model": "model120-fast",
"expected_documents": [
"pages/students/2016/jan_holp/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
],
"expected_answer_contains": [
"Systém získavania informácií v slovenskom jazyku"
],
"should_answer": true,
"note": null,
"answer": "Témou diplomovej práce Jána Holpa je „Systém získavania informácií v slovenskom jazyku\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
"tool_called": true,
"tool_call_count": 1,
"tool_calls": [
{
"name": "retrieve_zpwiki_context",
"arguments": {
"query": "téma diplomovej práce Ján Holp",
"limit": 5
},
"latency_seconds": 0.129755,
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
"https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus",
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
]
}
],
"rag_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
"https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus",
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
],
"first_model_latency_seconds": 1.828308,
"tool_latency_seconds": 0.129755,
"final_model_latency_seconds": 5.880594,
"total_latency_seconds": 7.839001,
"usage": {
"prompt_tokens": 6761,
"completion_tokens": 106,
"total_tokens": 6867
},
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
"answer_matches": [
true
],
"answer_contains_score": 1.0,
"source_matches": [
true
],
"source_url_score": 1.0,
"should_answer_ok": true,
"should_answer_score": 1.0,
"returned_no_answer": false,
"tool_score": 1.0,
"overall_score": 1.0,
"strict_pass": true,
"error": null
},
{
"id": "q0106",
"split": "dev",
"category": "paraphrase",
"difficulty": "medium",
"question": "Aká téma bakalárskej práce je uvedená pri osobe „Patrik Pavlišin“?",
"override_applied": true,
"requested_model": "model120-fast",
"expected_documents": [
"pages/students/2016/patrik_pavlisin/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
],
"expected_answer_contains": [
"Metódy automatického prekladu"
],
"should_answer": true,
"note": null,
"answer": "Témou bakalárskej práce Patrika Pavlišina je „Metódy automatického prekladu\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
"tool_called": true,
"tool_call_count": 1,
"tool_calls": [
{
"name": "retrieve_zpwiki_context",
"arguments": {
"query": "Patrik Pavlišin bakalárska práca téma"
},
"latency_seconds": 0.060836,
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
"https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
"https://zp.kemt.fei.tuke.sk/students/2019/david_stromp",
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
]
}
],
"rag_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
"https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
"https://zp.kemt.fei.tuke.sk/students/2019/david_stromp",
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
],
"first_model_latency_seconds": 1.348684,
"tool_latency_seconds": 0.060836,
"final_model_latency_seconds": 5.749333,
"total_latency_seconds": 7.159101,
"usage": {
"prompt_tokens": 6608,
"completion_tokens": 94,
"total_tokens": 6702
},
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
"answer_matches": [
true
],
"answer_contains_score": 1.0,
"source_matches": [
true
],
"source_url_score": 1.0,
"should_answer_ok": true,
"should_answer_score": 1.0,
"returned_no_answer": false,
"tool_score": 1.0,
"overall_score": 1.0,
"strict_pass": true,
"error": null
},
{
"id": "q0382",
"split": "dev",
"category": "paraphrase",
"difficulty": "medium",
"question": "Aká téma diplomového projektu je uvedená pri osobe „Ján Pták“?",
"override_applied": true,
"requested_model": "model120-fast",
"expected_documents": [
"pages/students/2022/jan_ptak/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
],
"expected_answer_contains": [
"Agent pre manažment záverečných prác"
],
"should_answer": true,
"note": null,
"answer": "Pri osobe Ján Pták je uvedená téma diplomového projektu Agent pre manažment záverečných prác.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",
"tool_called": true,
"tool_call_count": 1,
"tool_calls": [
{
"name": "retrieve_zpwiki_context",
"arguments": {
"query": "Ján Pták diplomový projekt téma",
"limit": 5
},
"latency_seconds": 0.039251,
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
]
}
],
"rag_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
],
"first_model_latency_seconds": 1.453879,
"tool_latency_seconds": 0.039251,
"final_model_latency_seconds": 3.338716,
"total_latency_seconds": 4.832013,
"usage": {
"prompt_tokens": 3161,
"completion_tokens": 101,
"total_tokens": 3262
},
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
"answer_matches": [
true
],
"answer_contains_score": 1.0,
"source_matches": [
true
],
"source_url_score": 1.0,
"should_answer_ok": true,
"should_answer_score": 1.0,
"returned_no_answer": false,
"tool_score": 1.0,
"overall_score": 1.0,
"strict_pass": true,
"error": null
},
{
"id": "q0441",
"split": "dev",
"category": "paraphrase",
"difficulty": "medium",
"question": "Aká téma bakalárskej práce je uvedená pri osobe „Yevhenii Medushivskyi“?",
"override_applied": true,
"requested_model": "model120-fast",
"expected_documents": [
"pages/students/2023/yevhenii_medushivskyi/README.md"
],
"expected_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi"
],
"expected_answer_contains": [
"Získavanie textu z webových stránok"
],
"should_answer": true,
"note": null,
"answer": "V dostupných dokumentoch ZP Wiki sa explicitne uvedená téma bakalárskej práce pre Yevhenii Medushivskyi nenachádza, dokument však obsahuje zoznam úloh a cieľov práce zameraných na vytvorenie crawlera, text mining a natrénoval prediktor získaného textu.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi",
"tool_called": true,
"tool_call_count": 1,
"tool_calls": [
{
"name": "retrieve_zpwiki_context",
"arguments": {
"query": "Yevhenii Medushivskyi téma bakalárskej práce",
"limit": 5
},
"latency_seconds": 0.063093,
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi",
"https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov",
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
"https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh",
"https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets"
]
}
],
"rag_source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi",
"https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov",
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
"https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh",
"https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets"
],
"first_model_latency_seconds": 1.570533,
"tool_latency_seconds": 0.063093,
"final_model_latency_seconds": 7.048205,
"total_latency_seconds": 8.682057,
"usage": {
"prompt_tokens": 7356,
"completion_tokens": 159,
"total_tokens": 7515
},
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
"answer_matches": [
false
],
"answer_contains_score": 0.0,
"source_matches": [
true
],
"source_url_score": 1.0,
"should_answer_ok": true,
"should_answer_score": 1.0,
"returned_no_answer": false,
"tool_score": 1.0,
"overall_score": 0.75,
"strict_pass": false,
"error": null
}
]
}
Reference in New Issue View Git Blame Copy Permalink