diff --git a/evaluation/results/rag_answers_dev_limit10.csv b/evaluation/results/rag_answers_dev_limit10.csv new file mode 100644 index 0000000..0ded9ed --- /dev/null +++ b/evaluation/results/rag_answers_dev_limit10.csv @@ -0,0 +1,31 @@ +id,split,category,difficulty,question,requested_model,response_model,tool_called,tool_call_count,answer_contains_score,source_url_score,should_answer_score,overall_score,strict_pass,expected_answer_contains,expected_source_urls,rag_source_urls,answer,first_model_latency_seconds,tool_latency_seconds,final_model_latency_seconds,total_latency_seconds,prompt_tokens,completion_tokens,total_tokens,error +q0001,dev,entity_lookup,easy,Nájdi dokument o osobe Ján Holp.,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Ján Holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","V dostupných dokumentoch ZP Wiki sa o osobe Ján Holp nachádzajú informácie o jeho štúdiu, ktoré začalo v roku 2016, a o jeho bakalárskej práci s názvom „Vyhľadávač na slovenskom internete"" z roku 2019, ako aj o diplomovej práci s názvom „Systém získavania informácií v slovenskom jazyku"" z roku 2021. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.855543,0.04037,6.536437,8.432721,5976,205,6181, +q0002,dev,fact_title,easy,Aký je názov diplomová práca osoby Ján Holp?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Systém získavania informácií v slovenskom jazyku""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","Názov diplomovej práce Ján Holpa je Systém získavania informácií v slovenskom jazyku. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.440291,0.034781,3.265832,4.741183,3035,103,3138, +q0003,dev,reverse_lookup,medium,Kto sa venuje práci s názvom „Systém získavania informácií v slovenskom jazyku“?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Ján Holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","Na práci s názvom „Systém získavania informácií v slovenskom jazyku"" sa v rámci svojej diplomovej práce z roku 2021 v roku začiatku štúdia 2016 podieľal Ján Holp. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.586781,0.032754,1.793497,3.413269,3077,144,3221, +q0004,dev,topic_lookup,medium,"Nájdi študenta alebo dokument, ktorý rieši tému „Systém získavania informácií v slovenskom jazyku“.",model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Ján Holp"", ""Systém získavania informácií v slovenskom jazyku""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","Študent Ján Holp vypracoval diplomovú prácu s názvom „Systém získavania informácií v slovenskom jazyku“ v roku 2021, pri ktorej začal štúdium v roku 2016. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.819523,0.034531,3.789378,5.643685,3098,157,3255, +q0005,dev,paraphrase,medium,Akej téme sa venuje diplomová práca Jána Holpa?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Systém získavania informácií v slovenskom jazyku""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"", ""https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura"", ""https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny"", ""https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"", ""https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky""]","Diplomová práca Jána Holpa sa zaoberá témou Systém získavania informácií v slovenskom jazyku. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.456083,0.042547,5.83287,7.331866,6435,110,6545, +q0006,dev,fact_start_year,easy,V ktorom roku začal študovať Ján Holp?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""2016""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","Ján Holp začal študovať v roku 2016. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.501519,0.036725,3.148166,4.686654,3039,96,3135, +q0007,dev,verification,medium,Je pri osobe Ján Holp uvedený rok začiatku štúdia 2016?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""2016""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","Áno, pri osobe Ján Holp je uvedený rok začiatku štúdia 2016. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.642486,0.035661,1.278389,2.956764,3067,110,3177, +q0008,dev,fact_work_year,easy,V ktorom roku je pri osobe Ján Holp uvedená diplomová práca?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""2021""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","Diplomová práca osobe Ján Holp je uvedená v roku 2021. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.292269,0.03313,3.205778,4.531426,3031,87,3118, +q0009,dev,disambiguation_year,hard,Rozlíš rok začiatku štúdia a rok práce osoby Ján Holp.,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""2016"", ""2021""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","Podľa dostupných informácií začal Ján Holp štúdium v roku 2016 a jeho diplomová práca bola vypracovaná v roku 2021. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.530212,0.034428,3.479579,5.044445,3056,123,3179, +q0010,dev,adversarial_year,hard,Je rok 2016 rokom záverečnej práce osoby Ján Holp?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""2016"", ""2021""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"", ""https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"", ""https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"", ""https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov"", ""https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko""]","Rok 2016 nie je rokom záverečnej práce Jána Holpa, ale rokom začiatku jeho štúdia. Podľa dostupných informácií vypracovával svoju diplomovú prácu v roku 2021. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.788465,0.041874,1.837584,3.668296,7025,153,7178, diff --git a/evaluation/results/rag_answers_dev_limit10.json b/evaluation/results/rag_answers_dev_limit10.json new file mode 100644 index 0000000..58a2dce --- /dev/null +++ b/evaluation/results/rag_answers_dev_limit10.json @@ -0,0 +1,894 @@ +{ + "generated_at": "2026-08-15T21:24:14.873861+00:00", + "status": "complete", + "split": "dev", + "requested_model": "model120-fast", + "questions_path": "/home/janko/DP/zp-agent/evaluation/questions.json", + "openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions", + "rag_url": "http://localhost:8000/rag", + "summary": { + "total": 10, + "completed": 10, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 10, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 5.045031, + "prompt_tokens": 40839, + "completion_tokens": 1288, + "total_tokens": 42127, + "by_category": { + "adversarial_year": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 3.668296, + "prompt_tokens": 7025, + "completion_tokens": 153, + "total_tokens": 7178 + }, + "disambiguation_year": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 5.044445, + "prompt_tokens": 3056, + "completion_tokens": 123, + "total_tokens": 3179 + }, + "entity_lookup": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 8.432721, + "prompt_tokens": 5976, + "completion_tokens": 205, + "total_tokens": 6181 + }, + "fact_start_year": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 4.686654, + "prompt_tokens": 3039, + "completion_tokens": 96, + "total_tokens": 3135 + }, + "fact_title": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 4.741183, + "prompt_tokens": 3035, + "completion_tokens": 103, + "total_tokens": 3138 + }, + "fact_work_year": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 4.531426, + "prompt_tokens": 3031, + "completion_tokens": 87, + "total_tokens": 3118 + }, + "paraphrase": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 7.331866, + "prompt_tokens": 6435, + "completion_tokens": 110, + "total_tokens": 6545 + }, + "reverse_lookup": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 3.413269, + "prompt_tokens": 3077, + "completion_tokens": 144, + "total_tokens": 3221 + }, + "topic_lookup": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 5.643685, + "prompt_tokens": 3098, + "completion_tokens": 157, + "total_tokens": 3255 + }, + "verification": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 2.956764, + "prompt_tokens": 3067, + "completion_tokens": 110, + "total_tokens": 3177 + } + }, + "by_difficulty": { + "easy": { + "total": 4, + "completed": 4, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 4, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 5.597996, + "prompt_tokens": 15081, + "completion_tokens": 491, + "total_tokens": 15572 + }, + "hard": { + "total": 2, + "completed": 2, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 2, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 4.35637, + "prompt_tokens": 10081, + "completion_tokens": 276, + "total_tokens": 10357 + }, + "medium": { + "total": 4, + "completed": 4, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 4, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 4.836396, + "prompt_tokens": 15677, + "completion_tokens": 521, + "total_tokens": 16198 + } + } + }, + "results": [ + { + "id": "q0001", + "split": "dev", + "category": "entity_lookup", + "difficulty": "easy", + "question": "Nájdi dokument o osobe Ján Holp.", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "Ján Holp" + ], + "should_answer": true, + "note": null, + "answer": "V dostupných dokumentoch ZP Wiki sa o osobe Ján Holp nachádzajú informácie o jeho štúdiu, ktoré začalo v roku 2016, a o jeho bakalárskej práci s názvom „Vyhľadávač na slovenskom internete\" z roku 2019, ako aj o diplomovej práci s názvom „Systém získavania informácií v slovenskom jazyku\" z roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Ján Holp", + "limit": 5, + "published_only": false, + "max_per_document": 10 + }, + "latency_seconds": 0.04037, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "first_model_latency_seconds": 1.855543, + "tool_latency_seconds": 0.04037, + "final_model_latency_seconds": 6.536437, + "total_latency_seconds": 8.432721, + "usage": { + "prompt_tokens": 5976, + "completion_tokens": 205, + "total_tokens": 6181 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0002", + "split": "dev", + "category": "fact_title", + "difficulty": "easy", + "question": "Aký je názov diplomová práca osoby Ján Holp?", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "Systém získavania informácií v slovenskom jazyku" + ], + "should_answer": true, + "note": null, + "answer": "Názov diplomovej práce Ján Holpa je Systém získavania informácií v slovenskom jazyku.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Ján Holp diplomová práca", + "limit": 5 + }, + "latency_seconds": 0.034781, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "first_model_latency_seconds": 1.440291, + "tool_latency_seconds": 0.034781, + "final_model_latency_seconds": 3.265832, + "total_latency_seconds": 4.741183, + "usage": { + "prompt_tokens": 3035, + "completion_tokens": 103, + "total_tokens": 3138 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0003", + "split": "dev", + "category": "reverse_lookup", + "difficulty": "medium", + "question": "Kto sa venuje práci s názvom „Systém získavania informácií v slovenskom jazyku“?", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "Ján Holp" + ], + "should_answer": true, + "note": null, + "answer": "Na práci s názvom „Systém získavania informácií v slovenskom jazyku\" sa v rámci svojej diplomovej práce z roku 2021 v roku začiatku štúdia 2016 podieľal Ján Holp.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Systém získavania informácií v slovenskom jazyku", + "limit": 5 + }, + "latency_seconds": 0.032754, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "first_model_latency_seconds": 1.586781, + "tool_latency_seconds": 0.032754, + "final_model_latency_seconds": 1.793497, + "total_latency_seconds": 3.413269, + "usage": { + "prompt_tokens": 3077, + "completion_tokens": 144, + "total_tokens": 3221 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0004", + "split": "dev", + "category": "topic_lookup", + "difficulty": "medium", + "question": "Nájdi študenta alebo dokument, ktorý rieši tému „Systém získavania informácií v slovenskom jazyku“.", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "Ján Holp", + "Systém získavania informácií v slovenskom jazyku" + ], + "should_answer": true, + "note": null, + "answer": "Študent Ján Holp vypracoval diplomovú prácu s názvom „Systém získavania informácií v slovenskom jazyku“ v roku 2021, pri ktorej začal štúdium v roku 2016.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Systém získavania informácií v slovenskom jazyku", + "limit": 5, + "published_only": true + }, + "latency_seconds": 0.034531, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "first_model_latency_seconds": 1.819523, + "tool_latency_seconds": 0.034531, + "final_model_latency_seconds": 3.789378, + "total_latency_seconds": 5.643685, + "usage": { + "prompt_tokens": 3098, + "completion_tokens": 157, + "total_tokens": 3255 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true, + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0005", + "split": "dev", + "category": "paraphrase", + "difficulty": "medium", + "question": "Akej téme sa venuje diplomová práca Jána Holpa?", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "Systém získavania informácií v slovenskom jazyku" + ], + "should_answer": true, + "note": null, + "answer": "Diplomová práca Jána Holpa sa zaoberá témou Systém získavania informácií v slovenskom jazyku.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "téma diplomovej práce Jána Holpa", + "limit": 5 + }, + "latency_seconds": 0.042547, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura", + "https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny", + "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura", + "https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny", + "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky" + ], + "first_model_latency_seconds": 1.456083, + "tool_latency_seconds": 0.042547, + "final_model_latency_seconds": 5.83287, + "total_latency_seconds": 7.331866, + "usage": { + "prompt_tokens": 6435, + "completion_tokens": 110, + "total_tokens": 6545 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0006", + "split": "dev", + "category": "fact_start_year", + "difficulty": "easy", + "question": "V ktorom roku začal študovať Ján Holp?", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "2016" + ], + "should_answer": true, + "note": null, + "answer": "Ján Holp začal študovať v roku 2016.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Ján Holp začiatok štúdia rok", + "limit": 5 + }, + "latency_seconds": 0.036725, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "first_model_latency_seconds": 1.501519, + "tool_latency_seconds": 0.036725, + "final_model_latency_seconds": 3.148166, + "total_latency_seconds": 4.686654, + "usage": { + "prompt_tokens": 3039, + "completion_tokens": 96, + "total_tokens": 3135 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0007", + "split": "dev", + "category": "verification", + "difficulty": "medium", + "question": "Je pri osobe Ján Holp uvedený rok začiatku štúdia 2016?", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "2016" + ], + "should_answer": true, + "note": null, + "answer": "Áno, pri osobe Ján Holp je uvedený rok začiatku štúdia 2016.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Ján Holp rok začiatku štúdia 2016", + "limit": 5 + }, + "latency_seconds": 0.035661, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "first_model_latency_seconds": 1.642486, + "tool_latency_seconds": 0.035661, + "final_model_latency_seconds": 1.278389, + "total_latency_seconds": 2.956764, + "usage": { + "prompt_tokens": 3067, + "completion_tokens": 110, + "total_tokens": 3177 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0008", + "split": "dev", + "category": "fact_work_year", + "difficulty": "easy", + "question": "V ktorom roku je pri osobe Ján Holp uvedená diplomová práca?", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "2021" + ], + "should_answer": true, + "note": null, + "answer": "Diplomová práca osobe Ján Holp je uvedená v roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Ján Holp diplomová práca" + }, + "latency_seconds": 0.03313, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "first_model_latency_seconds": 1.292269, + "tool_latency_seconds": 0.03313, + "final_model_latency_seconds": 3.205778, + "total_latency_seconds": 4.531426, + "usage": { + "prompt_tokens": 3031, + "completion_tokens": 87, + "total_tokens": 3118 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0009", + "split": "dev", + "category": "disambiguation_year", + "difficulty": "hard", + "question": "Rozlíš rok začiatku štúdia a rok práce osoby Ján Holp.", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "2016", + "2021" + ], + "should_answer": true, + "note": "Kontrola proti zámene roku začiatku štúdia s rokom záverečnej práce alebo projektu.", + "answer": "Podľa dostupných informácií začal Ján Holp štúdium v roku 2016 a jeho diplomová práca bola vypracovaná v roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Ján Holp rok začiatku štúdia rok práce", + "limit": 10 + }, + "latency_seconds": 0.034428, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "first_model_latency_seconds": 1.530212, + "tool_latency_seconds": 0.034428, + "final_model_latency_seconds": 3.479579, + "total_latency_seconds": 5.044445, + "usage": { + "prompt_tokens": 3056, + "completion_tokens": 123, + "total_tokens": 3179 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true, + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + }, + { + "id": "q0010", + "split": "dev", + "category": "adversarial_year", + "difficulty": "hard", + "question": "Je rok 2016 rokom záverečnej práce osoby Ján Holp?", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "2016", + "2021" + ], + "should_answer": true, + "note": "Model má vysvetliť, že prvý rok je rok začiatku štúdia, ak to zdroj tak uvádza.", + "answer": "Rok 2016 nie je rokom záverečnej práce Jána Holpa, ale rokom začiatku jeho štúdia. Podľa dostupných informácií vypracovával svoju diplomovú prácu v roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "Ján Holp záverečná práca rok 2016", + "limit": 5, + "published_only": true + }, + "latency_seconds": 0.041874, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky", + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov", + "https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky", + "https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov", + "https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko" + ], + "first_model_latency_seconds": 1.788465, + "tool_latency_seconds": 0.041874, + "final_model_latency_seconds": 1.837584, + "total_latency_seconds": 3.668296, + "usage": { + "prompt_tokens": 7025, + "completion_tokens": 153, + "total_tokens": 7178 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true, + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + } + ] +} diff --git a/evaluation/results/rag_answers_dev_selected.csv b/evaluation/results/rag_answers_dev_selected.csv new file mode 100644 index 0000000..93d0a6f --- /dev/null +++ b/evaluation/results/rag_answers_dev_selected.csv @@ -0,0 +1,4 @@ +id,split,category,difficulty,question,requested_model,response_model,tool_called,tool_call_count,answer_contains_score,source_url_score,should_answer_score,overall_score,strict_pass,expected_answer_contains,expected_source_urls,rag_source_urls,answer,first_model_latency_seconds,tool_latency_seconds,final_model_latency_seconds,total_latency_seconds,prompt_tokens,completion_tokens,total_tokens,error +q0005,dev,paraphrase,medium,Akej téme sa venuje diplomová práca Jána Holpa?,model120-fast,Qwen3.5-122B-A10B-Q4_K_M,True,1,1.0,1.0,1.0,1.0,True,"[""Systém získavania informácií v slovenskom jazyku""]","[""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp""]","[""https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny"", ""https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"", ""https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura"", ""https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"", ""https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik""]","Diplomová práca Jána Holpa sa venuje téme Systém získavania informácií v slovenskom jazyku. + +Zdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",1.951634,0.058957,5.793241,7.804296,6651,123,6774, diff --git a/evaluation/results/rag_answers_dev_selected.json b/evaluation/results/rag_answers_dev_selected.json new file mode 100644 index 0000000..12a6372 --- /dev/null +++ b/evaluation/results/rag_answers_dev_selected.json @@ -0,0 +1,135 @@ +{ + "generated_at": "2026-08-15T21:23:09.401870+00:00", + "status": "complete", + "split": "dev", + "requested_model": "model120-fast", + "questions_path": "/home/janko/DP/zp-agent/evaluation/questions.json", + "openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions", + "rag_url": "http://localhost:8000/rag", + "summary": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 7.804296, + "prompt_tokens": 6651, + "completion_tokens": 123, + "total_tokens": 6774, + "by_category": { + "paraphrase": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 7.804296, + "prompt_tokens": 6651, + "completion_tokens": 123, + "total_tokens": 6774 + } + }, + "by_difficulty": { + "medium": { + "total": 1, + "completed": 1, + "errors": 0, + "tool_call_rate": 1.0, + "answer_contains_score": 1.0, + "source_url_score": 1.0, + "should_answer_score": 1.0, + "overall_score": 1.0, + "strict_pass_count": 1, + "strict_pass_rate": 1.0, + "mean_latency_seconds": 7.804296, + "prompt_tokens": 6651, + "completion_tokens": 123, + "total_tokens": 6774 + } + } + }, + "results": [ + { + "id": "q0005", + "split": "dev", + "category": "paraphrase", + "difficulty": "medium", + "question": "Akej téme sa venuje diplomová práca Jána Holpa?", + "requested_model": "model120-fast", + "expected_documents": [ + "pages/students/2016/jan_holp/README.md" + ], + "expected_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp" + ], + "expected_answer_contains": [ + "Systém získavania informácií v slovenskom jazyku" + ], + "should_answer": true, + "note": null, + "answer": "Diplomová práca Jána Holpa sa venuje téme Systém získavania informácií v slovenskom jazyku.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "tool_called": true, + "tool_call_count": 1, + "tool_calls": [ + { + "name": "retrieve_zpwiki_context", + "arguments": { + "query": "diplomová práca Jána Holpa téma", + "limit": 5, + "published_only": true + }, + "latency_seconds": 0.058957, + "source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny", + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky", + "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik" + ] + } + ], + "rag_source_urls": [ + "https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny", + "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp", + "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura", + "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky", + "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik" + ], + "first_model_latency_seconds": 1.951634, + "tool_latency_seconds": 0.058957, + "final_model_latency_seconds": 5.793241, + "total_latency_seconds": 7.804296, + "usage": { + "prompt_tokens": 6651, + "completion_tokens": 123, + "total_tokens": 6774 + }, + "response_model": "Qwen3.5-122B-A10B-Q4_K_M", + "answer_matches": [ + true + ], + "answer_contains_score": 1.0, + "source_matches": [ + true + ], + "source_url_score": 1.0, + "should_answer_ok": true, + "should_answer_score": 1.0, + "returned_no_answer": false, + "tool_score": 1.0, + "overall_score": 1.0, + "strict_pass": true, + "error": null + } + ] +}