136 lines
4.2 KiB
JSON
136 lines
4.2 KiB
JSON
{
|
|
"generated_at": "2026-08-15T21:23:09.401870+00:00",
|
|
"status": "complete",
|
|
"split": "dev",
|
|
"requested_model": "model120-fast",
|
|
"questions_path": "/home/janko/DP/zp-agent/evaluation/questions.json",
|
|
"openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions",
|
|
"rag_url": "http://localhost:8000/rag",
|
|
"summary": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 7.804296,
|
|
"prompt_tokens": 6651,
|
|
"completion_tokens": 123,
|
|
"total_tokens": 6774,
|
|
"by_category": {
|
|
"paraphrase": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 7.804296,
|
|
"prompt_tokens": 6651,
|
|
"completion_tokens": 123,
|
|
"total_tokens": 6774
|
|
}
|
|
},
|
|
"by_difficulty": {
|
|
"medium": {
|
|
"total": 1,
|
|
"completed": 1,
|
|
"errors": 0,
|
|
"tool_call_rate": 1.0,
|
|
"answer_contains_score": 1.0,
|
|
"source_url_score": 1.0,
|
|
"should_answer_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass_count": 1,
|
|
"strict_pass_rate": 1.0,
|
|
"mean_latency_seconds": 7.804296,
|
|
"prompt_tokens": 6651,
|
|
"completion_tokens": 123,
|
|
"total_tokens": 6774
|
|
}
|
|
}
|
|
},
|
|
"results": [
|
|
{
|
|
"id": "q0005",
|
|
"split": "dev",
|
|
"category": "paraphrase",
|
|
"difficulty": "medium",
|
|
"question": "Akej téme sa venuje diplomová práca Jána Holpa?",
|
|
"requested_model": "model120-fast",
|
|
"expected_documents": [
|
|
"pages/students/2016/jan_holp/README.md"
|
|
],
|
|
"expected_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
|
|
],
|
|
"expected_answer_contains": [
|
|
"Systém získavania informácií v slovenskom jazyku"
|
|
],
|
|
"should_answer": true,
|
|
"note": null,
|
|
"answer": "Diplomová práca Jána Holpa sa venuje téme Systém získavania informácií v slovenskom jazyku.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"tool_called": true,
|
|
"tool_call_count": 1,
|
|
"tool_calls": [
|
|
{
|
|
"name": "retrieve_zpwiki_context",
|
|
"arguments": {
|
|
"query": "diplomová práca Jána Holpa téma",
|
|
"limit": 5,
|
|
"published_only": true
|
|
},
|
|
"latency_seconds": 0.058957,
|
|
"source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
|
|
]
|
|
}
|
|
],
|
|
"rag_source_urls": [
|
|
"https://zp.kemt.fei.tuke.sk/students/2021/matej_novotny",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
|
|
"https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
|
|
"https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
|
|
"https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
|
|
],
|
|
"first_model_latency_seconds": 1.951634,
|
|
"tool_latency_seconds": 0.058957,
|
|
"final_model_latency_seconds": 5.793241,
|
|
"total_latency_seconds": 7.804296,
|
|
"usage": {
|
|
"prompt_tokens": 6651,
|
|
"completion_tokens": 123,
|
|
"total_tokens": 6774
|
|
},
|
|
"response_model": "Qwen3.5-122B-A10B-Q4_K_M",
|
|
"answer_matches": [
|
|
true
|
|
],
|
|
"answer_contains_score": 1.0,
|
|
"source_matches": [
|
|
true
|
|
],
|
|
"source_url_score": 1.0,
|
|
"should_answer_ok": true,
|
|
"should_answer_score": 1.0,
|
|
"returned_no_answer": false,
|
|
"tool_score": 1.0,
|
|
"overall_score": 1.0,
|
|
"strict_pass": true,
|
|
"error": null
|
|
}
|
|
]
|
|
}
|