diff --git a/evaluation/results/rag_answers_dev_selected.csv b/evaluation/results/rag_answers_dev_selected.csv index 93d0a6f..d566988 100644 --- a/evaluation/results/rag_answers_dev_selected.csv +++ b/evaluation/results/rag_answers_dev_selected.csv @@ -1,4 +1,13 @@ id,split,category,difficulty,question,requested_model,response_model,tool_called,tool_call_count,answer_contains_score,source_url_score,should_answer_score,overall_score,strict_pass,expected_answer_contains,expected_source_urls,rag_source_urls,answer,first_model_latency_seconds,tool_latency_seconds,final_model_latency_seconds,total_latency_seconds,prompt_tokens,completion_tokens,total_tokens,error -q0005,dev,paraphrase,medium,Akej téme sa venuje diplomová práca Jána Holpa?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Systém získavania informácií v slovenskom jazyku""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny"", ""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"", ""https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura"", ""https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"", ""https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik""]","Diplomová práca Jána Holpa sa venuje téme Systém získavania informácií v slovenskom jazyku. +q0005,dev,paraphrase,medium,Aká téma diplomovej práce je uvedená pri osobe „Ján Holp“?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Systém získavania informácií v slovenskom jazyku""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"", ""https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura"", ""https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"", ""https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus"", ""https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik""]","Témou diplomovej práce Jána Holpa je „Systém získavania informácií v slovenskom jazyku"". -Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.951634,0.058957,5.793241,7.804296,6651,123,6774, +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.828308,0.129755,5.880594,7.839001,6761,106,6867, +q0106,dev,paraphrase,medium,Aká téma bakalárskej práce je uvedená pri osobe „Patrik Pavlišin“?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Metódy automatického prekladu""]","[""https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin""]","[""https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"", ""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"", ""https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan"", ""https://zp.kemt.fei.tuke.sk/students/2019/david_stromp"", ""https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko""]","Témou bakalárskej práce Patrika Pavlišina je „Metódy automatického prekladu"". + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",1.348684,0.060836,5.749333,7.159101,6608,94,6702, +q0382,dev,paraphrase,medium,Aká téma diplomového projektu je uvedená pri osobe „Ján Pták“?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Agent pre manažment záverečných prác""]","[""https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak""]","[""https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak""]","Pri osobe Ján Pták je uvedená téma diplomového projektu Agent pre manažment záverečných prác. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",1.453879,0.039251,3.338716,4.832013,3161,101,3262, +q0441,dev,paraphrase,medium,Aká téma bakalárskej práce je uvedená pri osobe „Yevhenii Medushivskyi“?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,0.0,1.0,1.0,0.75,False,"[""Získavanie textu z webových stránok""]","[""https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi""]","[""https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi"", ""https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov"", ""https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"", ""https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh"", ""https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets""]","V dostupných dokumentoch ZP Wiki sa explicitne uvedená téma bakalárskej práce pre Yevhenii Medushivskyi nenachádza, dokument však obsahuje zoznam úloh a cieľov práce zameraných na vytvorenie crawlera, text mining a natrénoval prediktor získaného textu. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi",1.570533,0.063093,7.048205,8.682057,7356,159,7515, diff --git a/evaluation/results/rag_answers_dev_selected.json b/evaluation/results/rag_answers_dev_selected.json index 12a6372..43b2994 100644 --- a/evaluation/results/rag_answers_dev_selected.json +++ b/evaluation/results/rag_answers_dev_selected.json @@ -1,60 +1,72 @@ { - "generated_at": "2026-08-15T21:23:09.401870+00:00", + "generated_at": "2026-08-15T22:03:41.030876+00:00", "status": "complete", - "split": "dev", - "requested_model": "model120-fast", - "questions_path": "/home/janko/DP/zp-agent/evaluation/questions.json", - "openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions", - "rag_url": "http://localhost:8000/rag", + "configuration": { + "questions_file": "/home/janko/DP/zp-agent/evaluation/json_files/questions.json", + "overrides_file": "/home/janko/DP/zp-agent/evaluation/json_files/rag_answer_overrides.json", + "applied_override_ids": [ + "q0005", + "q0106", + "q0382", + "q0441" + ], + "split": "dev", + "selected_question_count": 4, + "requested_model": "model120-fast", + "openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions", + "rag_url": "http://localhost:8000/rag", + "timeout": 180, + "delay": 0.0 + }, "summary": { - "total": 1, - "completed": 1, + "total": 4, + "completed": 4, "errors": 0, "tool_call_rate": 1.0, - "answer_contains_score": 1.0, + "answer_contains_score": 0.75, "source_url_score": 1.0, "should_answer_score": 1.0, - "overall_score": 1.0, - "strict_pass_count": 1, - "strict_pass_rate": 1.0, - "mean_latency_seconds": 7.804296, - "prompt_tokens": 6651, - "completion_tokens": 123, - "total_tokens": 6774, + "overall_score": 0.9375, + "strict_pass_count": 3, + "strict_pass_rate": 0.75, + "mean_latency_seconds": 7.128043, + "prompt_tokens": 23886, + "completion_tokens": 460, + "total_tokens": 24346, "by_category": { "paraphrase": { - "total": 1, - "completed": 1, + "total": 4, + "completed": 4, "errors": 0, "tool_call_rate": 1.0, - "answer_contains_score": 1.0, + "answer_contains_score": 0.75, "source_url_score": 1.0, "should_answer_score": 1.0, - "overall_score": 1.0, - "strict_pass_count": 1, - "strict_pass_rate": 1.0, - "mean_latency_seconds": 7.804296, - "prompt_tokens": 6651, - "completion_tokens": 123, - "total_tokens": 6774 + "overall_score": 0.9375, + "strict_pass_count": 3, + "strict_pass_rate": 0.75, + "mean_latency_seconds": 7.128043, + "prompt_tokens": 23886, + "completion_tokens": 460, + "total_tokens": 24346 } }, "by_difficulty": { "medium": { - "total": 1, - "completed": 1, + "total": 4, + "completed": 4, "errors": 0, "tool_call_rate": 1.0, - "answer_contains_score": 1.0, + "answer_contains_score": 0.75, "source_url_score": 1.0, "should_answer_score": 1.0, - "overall_score": 1.0, - "strict_pass_count": 1, - "strict_pass_rate": 1.0, - "mean_latency_seconds": 7.804296, - "prompt_tokens": 6651, - "completion_tokens": 123, - "total_tokens": 6774 + "overall_score": 0.9375, + "strict_pass_count": 3, + "strict_pass_rate": 0.75, + "mean_latency_seconds": 7.128043, + "prompt_tokens": 23886, + "completion_tokens": 460, + "total_tokens": 24346 } } }, @@ -64,7 +76,8 @@ "split": "dev", "category": "paraphrase", "difficulty": "medium", - "question": "Akej téme sa venuje diplomová práca Jána Holpa?", + "question": "Aká téma diplomovej práce je uvedená pri osobe „Ján Holp“?", + "override_applied": true, "requested_model": "model120-fast", "expected_documents": [ "pages/students/2016/jan_holp/README.md" @@ -77,42 +90,41 @@ ], "should_answer": true, "note": null, - "answer": "Diplomová práca Jána Holpa sa venuje téme Systém získavania informácií v slovenskom jazyku.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "answer": "Témou diplomovej práce Jána Holpa je „Systém získavania informácií v slovenskom jazyku\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", "tool_called": true, "tool_call_count": 1, "tool_calls": [ { "name": "retrieve_zpwiki_context", "arguments": { - "query": "diplomová práca Jána Holpa téma", - "limit": 5, - "published_only": true + "query": "téma diplomovej práce Ján Holp", + "limit": 5 }, - "latency_seconds": 0.058957, + "latency_seconds": 0.129755, "source_urls": [ - "https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny", "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura", "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky", + "https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus", "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik" ] } ], "rag_source_urls": [ - "https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny", "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura", "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky", + "https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus", "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik" ], - "first_model_latency_seconds": 1.951634, - "tool_latency_seconds": 0.058957, - "final_model_latency_seconds": 5.793241, - "total_latency_seconds": 7.804296, + "first_model_latency_seconds": 1.828308, + "tool_latency_seconds": 0.129755, + "final_model_latency_seconds": 5.880594, + "total_latency_seconds": 7.839001, "usage": { - "prompt_tokens": 6651, - "completion_tokens": 123, - "total_tokens": 6774 + "prompt_tokens": 6761, + "completion_tokens": 106, + "total_tokens": 6867 }, "response_model": "Qwen3.5-122B-A10B-Q4_K_M", "answer_matches": [ @@ -130,6 +142,213 @@ "overall_score": 1.0, "strict_pass": true, "error": null + }, + { + "id": "q0106", + "split": "dev", + "category": "paraphrase", + "difficulty": "medium", + "question": "Aká téma bakalárskej práce je uvedená pri osobe „Patrik Pavlišin“?", + "override_applied": true, + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/patrik_pavlisin/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin" + ], + "expected_answer_contains": [ + "Metódy automatického prekladu" + ], + "should_answer": true, + "note": null, + "answer": "Témou bakalárskej práce Patrika Pavlišina je „Metódy automatického prekladu\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Patrik Pavlišin bakalárska práca téma" + }, + "latency_seconds": 0.060836, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin", + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan", + "https://zp.kemt.fei.tuke.sk/students/2019/david_stromp", + "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin", + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan", + "https://zp.kemt.fei.tuke.sk/students/2019/david_stromp", + "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko" + ], + "first_model_latency_seconds": 1.348684, + "tool_latency_seconds": 0.060836, + "final_model_latency_seconds": 5.749333, + "total_latency_seconds": 7.159101, + "usage": { + "prompt_tokens": 6608, + "completion_tokens": 94, + "total_tokens": 6702 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0382", + "split": "dev", + "category": "paraphrase", + "difficulty": "medium", + "question": "Aká téma diplomového projektu je uvedená pri osobe „Ján Pták“?", + "override_applied": true, + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2022/jan_ptak/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak" + ], + "expected_answer_contains": [ + "Agent pre manažment záverečných prác" + ], + "should_answer": true, + "note": null, + "answer": "Pri osobe Ján Pták je uvedená téma diplomového projektu Agent pre manažment záverečných prác.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Ján Pták diplomový projekt téma", + "limit": 5 + }, + "latency_seconds": 0.039251, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak" + ], + "first_model_latency_seconds": 1.453879, + "tool_latency_seconds": 0.039251, + "final_model_latency_seconds": 3.338716, + "total_latency_seconds": 4.832013, + "usage": { + "prompt_tokens": 3161, + "completion_tokens": 101, + "total_tokens": 3262 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0441", + "split": "dev", + "category": "paraphrase", + "difficulty": "medium", + "question": "Aká téma bakalárskej práce je uvedená pri osobe „Yevhenii Medushivskyi“?", + "override_applied": true, + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2023/yevhenii_medushivskyi/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi" + ], + "expected_answer_contains": [ + "Získavanie textu z webových stránok" + ], + "should_answer": true, + "note": null, + "answer": "V dostupných dokumentoch ZP Wiki sa explicitne uvedená téma bakalárskej práce pre Yevhenii Medushivskyi nenachádza, dokument však obsahuje zoznam úloh a cieľov práce zameraných na vytvorenie crawlera, text mining a natrénoval prediktor získaného textu.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Yevhenii Medushivskyi téma bakalárskej práce", + "limit": 5 + }, + "latency_seconds": 0.063093, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi", + "https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov", + "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko", + "https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh", + "https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi", + "https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov", + "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko", + "https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh", + "https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets" + ], + "first_model_latency_seconds": 1.570533, + "tool_latency_seconds": 0.063093, + "final_model_latency_seconds": 7.048205, + "total_latency_seconds": 8.682057, + "usage": { + "prompt_tokens": 7356, + "completion_tokens": 159, + "total_tokens": 7515 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + false + ], + "answer_contains_score": 0.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 0.75, + "strict_pass": false, + "error": null } ] }