895 lines
28 KiB
JSON
895 lines
28 KiB
JSON
{
|
|
"generated_at": "2026-08-15T21:24:14.873861+00:00",
|
|
"status": "complete",
|
|
"split": "dev",
|
|
"requested_model": "model120-fast",
|
|
"questions_path": "/home/janko/DP/zp-agent/evaluation/questions.json",
|
|
"openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions",
|
|
"rag_url": "http://localhost:8000/rag",
|
|
"summary": {
|
|
"total": 10,
|
|
"completed": 10,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 10,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 5.045031,
|
|
"prompt_tokens": 40839,
|
|
"completion_tokens": 1288,
|
|
"total_tokens": 42127,
|
|
"by_category": {
|
|
"adversarial_year": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 3.668296,
|
|
"prompt_tokens": 7025,
|
|
"completion_tokens": 153,
|
|
"total_tokens": 7178
|
|
},
|
|
"disambiguation_year": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 5.044445,
|
|
"prompt_tokens": 3056,
|
|
"completion_tokens": 123,
|
|
"total_tokens": 3179
|
|
},
|
|
"entity_lookup": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 8.432721,
|
|
"prompt_tokens": 5976,
|
|
"completion_tokens": 205,
|
|
"total_tokens": 6181
|
|
},
|
|
"fact_start_year": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 4.686654,
|
|
"prompt_tokens": 3039,
|
|
"completion_tokens": 96,
|
|
"total_tokens": 3135
|
|
},
|
|
"fact_title": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 4.741183,
|
|
"prompt_tokens": 3035,
|
|
"completion_tokens": 103,
|
|
"total_tokens": 3138
|
|
},
|
|
"fact_work_year": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 4.531426,
|
|
"prompt_tokens": 3031,
|
|
"completion_tokens": 87,
|
|
"total_tokens": 3118
|
|
},
|
|
"paraphrase": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 7.331866,
|
|
"prompt_tokens": 6435,
|
|
"completion_tokens": 110,
|
|
"total_tokens": 6545
|
|
},
|
|
"reverse_lookup": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 3.413269,
|
|
"prompt_tokens": 3077,
|
|
"completion_tokens": 144,
|
|
"total_tokens": 3221
|
|
},
|
|
"topic_lookup": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 5.643685,
|
|
"prompt_tokens": 3098,
|
|
"completion_tokens": 157,
|
|
"total_tokens": 3255
|
|
},
|
|
"verification": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 2.956764,
|
|
"prompt_tokens": 3067,
|
|
"completion_tokens": 110,
|
|
"total_tokens": 3177
|
|
}
|
|
},
|
|
"by_difficulty": {
|
|
"easy": {
|
|
"total": 4,
|
|
"completed": 4,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 4,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 5.597996,
|
|
"prompt_tokens": 15081,
|
|
"completion_tokens": 491,
|
|
"total_tokens": 15572
|
|
},
|
|
"hard": {
|
|
"total": 2,
|
|
"completed": 2,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 2,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 4.35637,
|
|
"prompt_tokens": 10081,
|
|
"completion_tokens": 276,
|
|
"total_tokens": 10357
|
|
},
|
|
"medium": {
|
|
"total": 4,
|
|
"completed": 4,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 4,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 4.836396,
|
|
"prompt_tokens": 15677,
|
|
"completion_tokens": 521,
|
|
"total_tokens": 16198
|
|
}
|
|
}
|
|
},
|
|
"results": [
|
|
{
|
|
"id": "q0001",
|
|
"split": "dev",
|
|
"category": "entity_lookup",
|
|
"difficulty": "easy",
|
|
"question": "Nájdi dokument o osobe Ján Holp.",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ján Holp"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "V dostupných dokumentoch ZP Wiki sa o osobe Ján Holp nachádzajú informácie o jeho štúdiu, ktoré začalo v roku 2016, a o jeho bakalárskej práci s názvom „Vyhľadávač na slovenskom internete\" z roku 2019, ako aj o diplomovej práci s názvom „Systém získavania informácií v slovenskom jazyku\" z roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Ján Holp",
|
|
"limit": 5,
|
|
"published_only": false,
|
|
"max_per_document": 10
|
|
},
|
|
"latency_seconds": 0.04037,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.855543,
|
|
"tool_latency_seconds": 0.04037,
|
|
"final_model_latency_seconds": 6.536437,
|
|
"total_latency_seconds": 8.432721,
|
|
"usage": {
|
|
"prompt_tokens": 5976,
|
|
"completion_tokens": 205,
|
|
"total_tokens": 6181
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0002",
|
|
"split": "dev",
|
|
"category": "fact_title",
|
|
"difficulty": "easy",
|
|
"question": "Aký je názov diplomová práca osoby Ján Holp?",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Systém získavania informácií v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "Názov diplomovej práce Ján Holpa je Systém získavania informácií v slovenskom jazyku.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Ján Holp diplomová práca",
|
|
"limit": 5
|
|
},
|
|
"latency_seconds": 0.034781,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.440291,
|
|
"tool_latency_seconds": 0.034781,
|
|
"final_model_latency_seconds": 3.265832,
|
|
"total_latency_seconds": 4.741183,
|
|
"usage": {
|
|
"prompt_tokens": 3035,
|
|
"completion_tokens": 103,
|
|
"total_tokens": 3138
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0003",
|
|
"split": "dev",
|
|
"category": "reverse_lookup",
|
|
"difficulty": "medium",
|
|
"question": "Kto sa venuje práci s názvom „Systém získavania informácií v slovenskom jazyku“?",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ján Holp"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "Na práci s názvom „Systém získavania informácií v slovenskom jazyku\" sa v rámci svojej diplomovej práce z roku 2021 v roku začiatku štúdia 2016 podieľal Ján Holp.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Systém získavania informácií v slovenskom jazyku",
|
|
"limit": 5
|
|
},
|
|
"latency_seconds": 0.032754,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.586781,
|
|
"tool_latency_seconds": 0.032754,
|
|
"final_model_latency_seconds": 1.793497,
|
|
"total_latency_seconds": 3.413269,
|
|
"usage": {
|
|
"prompt_tokens": 3077,
|
|
"completion_tokens": 144,
|
|
"total_tokens": 3221
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0004",
|
|
"split": "dev",
|
|
"category": "topic_lookup",
|
|
"difficulty": "medium",
|
|
"question": "Nájdi študenta alebo dokument, ktorý rieši tému „Systém získavania informácií v slovenskom jazyku“.",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Ján Holp",
|
|
"Systém získavania informácií v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "Študent Ján Holp vypracoval diplomovú prácu s názvom „Systém získavania informácií v slovenskom jazyku“ v roku 2021, pri ktorej začal štúdium v roku 2016.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Systém získavania informácií v slovenskom jazyku",
|
|
"limit": 5,
|
|
"published_only": true
|
|
},
|
|
"latency_seconds": 0.034531,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.819523,
|
|
"tool_latency_seconds": 0.034531,
|
|
"final_model_latency_seconds": 3.789378,
|
|
"total_latency_seconds": 5.643685,
|
|
"usage": {
|
|
"prompt_tokens": 3098,
|
|
"completion_tokens": 157,
|
|
"total_tokens": 3255
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true,
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0005",
|
|
"split": "dev",
|
|
"category": "paraphrase",
|
|
"difficulty": "medium",
|
|
"question": "Akej téme sa venuje diplomová práca Jána Holpa?",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Systém získavania informácií v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "Diplomová práca Jána Holpa sa zaoberá témou Systém získavania informácií v slovenskom jazyku.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "téma diplomovej práce Jána Holpa",
|
|
"limit": 5
|
|
},
|
|
"latency_seconds": 0.042547,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky"
|
|
],
|
|
"first_model_latency_seconds": 1.456083,
|
|
"tool_latency_seconds": 0.042547,
|
|
"final_model_latency_seconds": 5.83287,
|
|
"total_latency_seconds": 7.331866,
|
|
"usage": {
|
|
"prompt_tokens": 6435,
|
|
"completion_tokens": 110,
|
|
"total_tokens": 6545
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0006",
|
|
"split": "dev",
|
|
"category": "fact_start_year",
|
|
"difficulty": "easy",
|
|
"question": "V ktorom roku začal študovať Ján Holp?",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "Ján Holp začal študovať v roku 2016.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Ján Holp začiatok štúdia rok",
|
|
"limit": 5
|
|
},
|
|
"latency_seconds": 0.036725,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.501519,
|
|
"tool_latency_seconds": 0.036725,
|
|
"final_model_latency_seconds": 3.148166,
|
|
"total_latency_seconds": 4.686654,
|
|
"usage": {
|
|
"prompt_tokens": 3039,
|
|
"completion_tokens": 96,
|
|
"total_tokens": 3135
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0007",
|
|
"split": "dev",
|
|
"category": "verification",
|
|
"difficulty": "medium",
|
|
"question": "Je pri osobe Ján Holp uvedený rok začiatku štúdia 2016?",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "Áno, pri osobe Ján Holp je uvedený rok začiatku štúdia 2016.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Ján Holp rok začiatku štúdia 2016",
|
|
"limit": 5
|
|
},
|
|
"latency_seconds": 0.035661,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.642486,
|
|
"tool_latency_seconds": 0.035661,
|
|
"final_model_latency_seconds": 1.278389,
|
|
"total_latency_seconds": 2.956764,
|
|
"usage": {
|
|
"prompt_tokens": 3067,
|
|
"completion_tokens": 110,
|
|
"total_tokens": 3177
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0008",
|
|
"split": "dev",
|
|
"category": "fact_work_year",
|
|
"difficulty": "easy",
|
|
"question": "V ktorom roku je pri osobe Ján Holp uvedená diplomová práca?",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "Diplomová práca osobe Ján Holp je uvedená v roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Ján Holp diplomová práca"
|
|
},
|
|
"latency_seconds": 0.03313,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.292269,
|
|
"tool_latency_seconds": 0.03313,
|
|
"final_model_latency_seconds": 3.205778,
|
|
"total_latency_seconds": 4.531426,
|
|
"usage": {
|
|
"prompt_tokens": 3031,
|
|
"completion_tokens": 87,
|
|
"total_tokens": 3118
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0009",
|
|
"split": "dev",
|
|
"category": "disambiguation_year",
|
|
"difficulty": "hard",
|
|
"question": "Rozlíš rok začiatku štúdia a rok práce osoby Ján Holp.",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016",
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"note": "Kontrola proti zámene roku začiatku štúdia s rokom záverečnej práce alebo projektu.",
|
|
"answer": "Podľa dostupných informácií začal Ján Holp štúdium v roku 2016 a jeho diplomová práca bola vypracovaná v roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Ján Holp rok začiatku štúdia rok práce",
|
|
"limit": 10
|
|
},
|
|
"latency_seconds": 0.034428,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"first_model_latency_seconds": 1.530212,
|
|
"tool_latency_seconds": 0.034428,
|
|
"final_model_latency_seconds": 3.479579,
|
|
"total_latency_seconds": 5.044445,
|
|
"usage": {
|
|
"prompt_tokens": 3056,
|
|
"completion_tokens": 123,
|
|
"total_tokens": 3179
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true,
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
},
|
|
{
|
|
"id": "q0010",
|
|
"split": "dev",
|
|
"category": "adversarial_year",
|
|
"difficulty": "hard",
|
|
"question": "Je rok 2016 rokom záverečnej práce osoby Ján Holp?",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"2016",
|
|
"2021"
|
|
],
|
|
"should_answer": true,
|
|
"note": "Model má vysvetliť, že prvý rok je rok začiatku štúdia, ak to zdroj tak uvádza.",
|
|
"answer": "Rok 2016 nie je rokom záverečnej práce Jána Holpa, ale rokom začiatku jeho štúdia. Podľa dostupných informácií vypracovával svoju diplomovú prácu v roku 2021.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Ján Holp záverečná práca rok 2016",
|
|
"limit": 5,
|
|
"published_only": true
|
|
},
|
|
"latency_seconds": 0.041874,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov",
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"https://zp.kemt.fei.tuke.sk/students/2023/oleksandr_dorybohov",
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/vladimir_ferko"
|
|
],
|
|
"first_model_latency_seconds": 1.788465,
|
|
"tool_latency_seconds": 0.041874,
|
|
"final_model_latency_seconds": 1.837584,
|
|
"total_latency_seconds": 3.668296,
|
|
"usage": {
|
|
"prompt_tokens": 7025,
|
|
"completion_tokens": 153,
|
|
"total_tokens": 7178
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true,
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
}
|
|
]
|
|
}
|