465 lines
18 KiB
JSON
465 lines
18 KiB
JSON
{
|
|
"configuration": {
|
|
"method": "Five preselected historical RAG smoke cases; not a new accuracy estimate.",
|
|
"selected_ids": [
|
|
"q0053",
|
|
"q0060",
|
|
"q0608",
|
|
"q1147",
|
|
"q0888"
|
|
],
|
|
"model": "model120-fast",
|
|
"source_file": "tools/rag/evaluation/results/test_classic_rag_frozen/rag_answers_test.json",
|
|
"source_sha256": "eba03639f8a3c32ee47a4e294d8592c1071fc6fd43e38967e39131b840fefa7e",
|
|
"generation_files": {
|
|
"/home/janko/DP/zp-agent/tools/answer_evaluation/check_rag_runtime.py": "37eef74ed8120109ed0c7f2d5a0edba497687a76ab5eaa154bd4626c9edb311c",
|
|
"tools/rag/evaluation/rag_runner.py": "031abf7d5b1a026eb199aa2a579bfb8baaa01175279d0018a364bfe50b76c3c1",
|
|
"tools/rag/evaluation/rag_metrics.py": "3e95922df90dd1b757a550040a2d9bd4f857e17d179f60170d7cc881a08f94b6",
|
|
"tools/answer_evaluation/context_policy.py": "e7b7c511e5e9cafdef04969eb4007a6a61ac4b99bcc60e9e722945a15bec1f26"
|
|
}
|
|
},
|
|
"results": [
|
|
{
|
|
"id": "q0053",
|
|
"question": "Aký je názov diplomová práca osoby Lukáš Pokrývka?",
|
|
"citations_ok": true,
|
|
"unsupported_urls": [],
|
|
"retrieval_coverage": [
|
|
{
|
|
"complete": false,
|
|
"truncated": true,
|
|
"truncation_reasons": [
|
|
"candidate_limit",
|
|
"per_document_limit",
|
|
"result_limit"
|
|
],
|
|
"index_version": {
|
|
"version": "10b78a8e70744e2792ffe4cf11d45571",
|
|
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
|
"created_at_utc": "2026-10-25T21:10:39.160542+00:00",
|
|
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
|
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
|
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
|
"embedding_model": "intfloat/multilingual-e5-small",
|
|
"publication_policy": "explicit-false-hidden-v1"
|
|
}
|
|
}
|
|
],
|
|
"context_issues": [
|
|
"missing_incompleteness_notice"
|
|
],
|
|
"accepted_for_delivery": false,
|
|
"answer": "Názov diplomovej práce Lukáša Pokrývku je „Paralelné trénovanie neurónových sietí\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Aký je názov diplomová práca osoby Lukáš Pokrývka?",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"latency_seconds": 8.6679,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/patrik_pokrivcak",
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak/dp2021"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/patrik_pokrivcak",
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak/dp2021"
|
|
],
|
|
"first_model_latency_seconds": 5.65362,
|
|
"tool_latency_seconds": 8.6679,
|
|
"final_model_latency_seconds": 23.053733,
|
|
"total_latency_seconds": 37.375855,
|
|
"usage": {
|
|
"prompt_tokens": 19045,
|
|
"completion_tokens": 137,
|
|
"total_tokens": 19182
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"citations_checked": true,
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_match_count": 1,
|
|
"source_required_count": 1,
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": false
|
|
},
|
|
{
|
|
"id": "q0060",
|
|
"question": "Rozlíš rok začiatku štúdia a rok práce osoby Lukáš Pokrývka.",
|
|
"citations_ok": true,
|
|
"unsupported_urls": [],
|
|
"retrieval_coverage": [
|
|
{
|
|
"complete": false,
|
|
"truncated": true,
|
|
"truncation_reasons": [
|
|
"candidate_limit",
|
|
"per_document_limit",
|
|
"result_limit"
|
|
],
|
|
"index_version": {
|
|
"version": "10b78a8e70744e2792ffe4cf11d45571",
|
|
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
|
"created_at_utc": "2026-10-25T21:10:39.160542+00:00",
|
|
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
|
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
|
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
|
"embedding_model": "intfloat/multilingual-e5-small",
|
|
"publication_policy": "explicit-false-hidden-v1"
|
|
}
|
|
}
|
|
],
|
|
"context_issues": [
|
|
"missing_incompleteness_notice"
|
|
],
|
|
"accepted_for_delivery": false,
|
|
"answer": "Rok začiatku štúdia osoby Lukáš Pokrývka je 2016, zatiaľ čo jeho diplomová práca bola realizovaná v roku 2021.\n\nZdroj:\nhttps://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Rozlíš rok začiatku štúdia a rok práce osoby Lukáš Pokrývka.",
|
|
"limit": 10,
|
|
"published_only": true,
|
|
"max_per_document": 5
|
|
},
|
|
"latency_seconds": 8.727633,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
|
|
"https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
|
|
"https://zp.kemt.fei.tuke.sk/students/2020/jakub_kristian_lukas",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
|
|
"https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
|
|
"https://zp.kemt.fei.tuke.sk/students/2020/jakub_kristian_lukas",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
|
|
],
|
|
"first_model_latency_seconds": 5.710692,
|
|
"tool_latency_seconds": 8.727633,
|
|
"final_model_latency_seconds": 40.777827,
|
|
"total_latency_seconds": 55.216681,
|
|
"usage": {
|
|
"prompt_tokens": 20160,
|
|
"completion_tokens": 155,
|
|
"total_tokens": 20315
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"citations_checked": true,
|
|
"answer_matches": [
|
|
true,
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_match_count": 1,
|
|
"source_required_count": 1,
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": false
|
|
},
|
|
{
|
|
"id": "q0608",
|
|
"question": "Aký source_url patrí k stránke „Podpora slovenčiny v knižnici Spacy“?",
|
|
"citations_ok": true,
|
|
"unsupported_urls": [],
|
|
"retrieval_coverage": [
|
|
{
|
|
"complete": false,
|
|
"truncated": true,
|
|
"truncation_reasons": [
|
|
"candidate_limit",
|
|
"per_document_limit",
|
|
"result_limit"
|
|
],
|
|
"index_version": {
|
|
"version": "10b78a8e70744e2792ffe4cf11d45571",
|
|
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
|
"created_at_utc": "2026-10-25T21:10:39.160542+00:00",
|
|
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
|
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
|
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
|
"embedding_model": "intfloat/multilingual-e5-small",
|
|
"publication_policy": "explicit-false-hidden-v1"
|
|
}
|
|
}
|
|
],
|
|
"context_issues": [
|
|
"missing_incompleteness_notice"
|
|
],
|
|
"accepted_for_delivery": false,
|
|
"answer": "Stránka s názvom „Podpora slovenčiny v knižnici Spacy\" má pridelený zdrojový URL adresu https://zp.kemt.fei.tuke.sk/topics/spacy.\n\nZdroj: https://zp.kemt.fei.tuke.sk/topics/spacy",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Aký source_url patrí k stránke „Podpora slovenčiny v knižnici Spacy“?",
|
|
"limit": 5,
|
|
"max_per_document": 1,
|
|
"published_only": true
|
|
},
|
|
"latency_seconds": 0.620349,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/spacy",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak",
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/than_trung_thanh",
|
|
"https://zp.kemt.fei.tuke.sk/topics/resources"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/topics/spacy",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak",
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/than_trung_thanh",
|
|
"https://zp.kemt.fei.tuke.sk/topics/resources"
|
|
],
|
|
"first_model_latency_seconds": 28.658617,
|
|
"tool_latency_seconds": 0.620349,
|
|
"final_model_latency_seconds": 35.815477,
|
|
"total_latency_seconds": 65.094973,
|
|
"usage": {
|
|
"prompt_tokens": 28510,
|
|
"completion_tokens": 141,
|
|
"total_tokens": 28651
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"citations_checked": true,
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_match_count": 1,
|
|
"source_required_count": 1,
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": false
|
|
},
|
|
{
|
|
"id": "q1147",
|
|
"question": "Porovnaj záznamy osôb Ján Holp a Ondrej Megela. Pri osobe Ján Holp použi záznam typu „diplomová práca“ a pri osobe Ondrej Megela záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.",
|
|
"citations_ok": true,
|
|
"unsupported_urls": [],
|
|
"retrieval_coverage": [
|
|
{
|
|
"complete": false,
|
|
"truncated": true,
|
|
"truncation_reasons": [
|
|
"candidate_limit",
|
|
"per_document_limit",
|
|
"result_limit"
|
|
],
|
|
"index_version": {
|
|
"version": "10b78a8e70744e2792ffe4cf11d45571",
|
|
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
|
"created_at_utc": "2026-10-25T21:10:39.160542+00:00",
|
|
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
|
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
|
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
|
"embedding_model": "intfloat/multilingual-e5-small",
|
|
"publication_policy": "explicit-false-hidden-v1"
|
|
}
|
|
}
|
|
],
|
|
"context_issues": [
|
|
"missing_incompleteness_notice"
|
|
],
|
|
"accepted_for_delivery": false,
|
|
"answer": "Ján Holp vypracoval diplomovú prácu s názvom „Systém získavania informácií v slovenskom jazyku\" v roku 2021 a Ondrej Megela vypracoval bakalársku prácu s názvom „Neurónové jazykové modelovanie typu BERT\" v roku 2020.\n\nZdroje:\nhttps://zp.kemt.fei.tuke.sk/students/2016/jan_holp\nhttps://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Porovnaj záznamy osôb Ján Holp a Ondrej Megela. Pri osobe Ján Holp použi záznam typu „diplomová práca“ a pri osobe Ondrej Megela záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.",
|
|
"limit": 5,
|
|
"published_only": true,
|
|
"max_per_document": 1
|
|
},
|
|
"latency_seconds": 1.001685,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
|
|
"https://zp.kemt.fei.tuke.sk/topics/faq",
|
|
"https://zp.kemt.fei.tuke.sk/topics/bert"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
|
|
"https://zp.kemt.fei.tuke.sk/topics/faq",
|
|
"https://zp.kemt.fei.tuke.sk/topics/bert"
|
|
],
|
|
"first_model_latency_seconds": 14.535465,
|
|
"tool_latency_seconds": 1.001685,
|
|
"final_model_latency_seconds": 56.647723,
|
|
"total_latency_seconds": 72.185893,
|
|
"usage": {
|
|
"prompt_tokens": 25893,
|
|
"completion_tokens": 208,
|
|
"total_tokens": 26101
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"citations_checked": true,
|
|
"answer_matches": [
|
|
true,
|
|
true,
|
|
true,
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true,
|
|
true
|
|
],
|
|
"source_match_count": 2,
|
|
"source_required_count": 2,
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": false
|
|
},
|
|
{
|
|
"id": "q0888",
|
|
"question": "Aký konkrétny finančný rozpočet bol uvedený pre projekt osoby Tomáš Kuchárik?",
|
|
"citations_ok": true,
|
|
"unsupported_urls": [],
|
|
"retrieval_coverage": [
|
|
{
|
|
"complete": false,
|
|
"truncated": true,
|
|
"truncation_reasons": [
|
|
"candidate_limit",
|
|
"per_document_limit",
|
|
"result_limit"
|
|
],
|
|
"index_version": {
|
|
"version": "10b78a8e70744e2792ffe4cf11d45571",
|
|
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
|
|
"created_at_utc": "2026-10-25T21:10:39.160542+00:00",
|
|
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
|
|
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
|
|
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
|
|
"embedding_model": "intfloat/multilingual-e5-small",
|
|
"publication_policy": "explicit-false-hidden-v1"
|
|
}
|
|
}
|
|
],
|
|
"context_issues": [
|
|
"missing_incompleteness_notice"
|
|
],
|
|
"accepted_for_delivery": false,
|
|
"answer": "V dostupných dokumentoch ZP Wiki sa informácia o konkrétnom finančnom rozpočte pre projekt osoby Tomáš Kuchárik nepodarilo spoľahlivo nájsť.\n\nZdroj:\nhttps://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "Aký konkrétny finančný rozpočet bol uvedený pre projekt osoby Tomáš Kuchárik?",
|
|
"limit": 5,
|
|
"published_only": true
|
|
},
|
|
"latency_seconds": 0.454629,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
|
|
"https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas",
|
|
"https://zp.kemt.fei.tuke.sk/students/2020/matus_suchanic",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
|
|
"https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas",
|
|
"https://zp.kemt.fei.tuke.sk/students/2020/matus_suchanic",
|
|
"https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
|
|
],
|
|
"first_model_latency_seconds": 129.609931,
|
|
"tool_latency_seconds": 0.454629,
|
|
"final_model_latency_seconds": 20.02263,
|
|
"total_latency_seconds": 150.087617,
|
|
"usage": {
|
|
"prompt_tokens": 14022,
|
|
"completion_tokens": 136,
|
|
"total_tokens": 14158
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"citations_checked": true,
|
|
"answer_matches": [],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_match_count": 1,
|
|
"source_required_count": 0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": true,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": false
|
|
}
|
|
],
|
|
"summary": {
|
|
"questions": 5,
|
|
"strict_pass": 0
|
|
}
|
|
}
|