{
  "generated_at": "2026-08-15T22:03:41.030876+00:00",
  "status": "complete",
  "configuration": {
    "questions_file": "/home/janko/DP/zp-agent/evaluation/json_files/questions.json",
    "overrides_file": "/home/janko/DP/zp-agent/evaluation/json_files/rag_answer_overrides.json",
    "applied_override_ids": [
      "q0005",
      "q0106",
      "q0382",
      "q0441"
    ],
    "split": "dev",
    "selected_question_count": 4,
    "requested_model": "model120-fast",
    "openwebui_url": "https://ui.tukekemt.xyz/api/chat/completions",
    "rag_url": "http://localhost:8000/rag",
    "timeout": 180,
    "delay": 0.0
  },
  "summary": {
    "total": 4,
    "completed": 4,
    "errors": 0,
    "tool_call_rate": 1.0,
    "answer_contains_score": 0.75,
    "source_url_score": 1.0,
    "should_answer_score": 1.0,
    "overall_score": 0.9375,
    "strict_pass_count": 3,
    "strict_pass_rate": 0.75,
    "mean_latency_seconds": 7.128043,
    "prompt_tokens": 23886,
    "completion_tokens": 460,
    "total_tokens": 24346,
    "by_category": {
      "paraphrase": {
        "total": 4,
        "completed": 4,
        "errors": 0,
        "tool_call_rate": 1.0,
        "answer_contains_score": 0.75,
        "source_url_score": 1.0,
        "should_answer_score": 1.0,
        "overall_score": 0.9375,
        "strict_pass_count": 3,
        "strict_pass_rate": 0.75,
        "mean_latency_seconds": 7.128043,
        "prompt_tokens": 23886,
        "completion_tokens": 460,
        "total_tokens": 24346
      }
    },
    "by_difficulty": {
      "medium": {
        "total": 4,
        "completed": 4,
        "errors": 0,
        "tool_call_rate": 1.0,
        "answer_contains_score": 0.75,
        "source_url_score": 1.0,
        "should_answer_score": 1.0,
        "overall_score": 0.9375,
        "strict_pass_count": 3,
        "strict_pass_rate": 0.75,
        "mean_latency_seconds": 7.128043,
        "prompt_tokens": 23886,
        "completion_tokens": 460,
        "total_tokens": 24346
      }
    }
  },
  "results": [
    {
      "id": "q0005",
      "split": "dev",
      "category": "paraphrase",
      "difficulty": "medium",
      "question": "Aká téma diplomovej práce je uvedená pri osobe „Ján Holp“?",
      "override_applied": true,
      "requested_model": "model120-fast",
      "expected_documents": [
        "pages/students/2016/jan_holp/README.md"
      ],
      "expected_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp"
      ],
      "expected_answer_contains": [
        "Systém získavania informácií v slovenskom jazyku"
      ],
      "should_answer": true,
      "note": null,
      "answer": "Témou diplomovej práce Jána Holpa je „Systém získavania informácií v slovenskom jazyku\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "téma diplomovej práce Ján Holp",
            "limit": 5
          },
          "latency_seconds": 0.129755,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
            "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
            "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
            "https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus",
            "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
        "https://zp.kemt.fei.tuke.sk/students/2017/martin_jancura",
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_malinovsky",
        "https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus",
        "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik"
      ],
      "first_model_latency_seconds": 1.828308,
      "tool_latency_seconds": 0.129755,
      "final_model_latency_seconds": 5.880594,
      "total_latency_seconds": 7.839001,
      "usage": {
        "prompt_tokens": 6761,
        "completion_tokens": 106,
        "total_tokens": 6867
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "answer_matches": [
        true
      ],
      "answer_contains_score": 1.0,
      "source_matches": [
        true
      ],
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": false,
      "tool_score": 1.0,
      "overall_score": 1.0,
      "strict_pass": true,
      "error": null
    },
    {
      "id": "q0106",
      "split": "dev",
      "category": "paraphrase",
      "difficulty": "medium",
      "question": "Aká téma bakalárskej práce je uvedená pri osobe „Patrik Pavlišin“?",
      "override_applied": true,
      "requested_model": "model120-fast",
      "expected_documents": [
        "pages/students/2016/patrik_pavlisin/README.md"
      ],
      "expected_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin"
      ],
      "expected_answer_contains": [
        "Metódy automatického prekladu"
      ],
      "should_answer": true,
      "note": null,
      "answer": "Témou bakalárskej práce Patrika Pavlišina je „Metódy automatického prekladu\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "Patrik Pavlišin bakalárska práca téma"
          },
          "latency_seconds": 0.060836,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
            "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
            "https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
            "https://zp.kemt.fei.tuke.sk/students/2019/david_stromp",
            "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2016/patrik_pavlisin",
        "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
        "https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
        "https://zp.kemt.fei.tuke.sk/students/2019/david_stromp",
        "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
      ],
      "first_model_latency_seconds": 1.348684,
      "tool_latency_seconds": 0.060836,
      "final_model_latency_seconds": 5.749333,
      "total_latency_seconds": 7.159101,
      "usage": {
        "prompt_tokens": 6608,
        "completion_tokens": 94,
        "total_tokens": 6702
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "answer_matches": [
        true
      ],
      "answer_contains_score": 1.0,
      "source_matches": [
        true
      ],
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": false,
      "tool_score": 1.0,
      "overall_score": 1.0,
      "strict_pass": true,
      "error": null
    },
    {
      "id": "q0382",
      "split": "dev",
      "category": "paraphrase",
      "difficulty": "medium",
      "question": "Aká téma diplomového projektu je uvedená pri osobe „Ján Pták“?",
      "override_applied": true,
      "requested_model": "model120-fast",
      "expected_documents": [
        "pages/students/2022/jan_ptak/README.md"
      ],
      "expected_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
      ],
      "expected_answer_contains": [
        "Agent pre manažment záverečných prác"
      ],
      "should_answer": true,
      "note": null,
      "answer": "Pri osobe Ján Pták je uvedená téma diplomového projektu Agent pre manažment záverečných prác.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "Ján Pták diplomový projekt téma",
            "limit": 5
          },
          "latency_seconds": 0.039251,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2022/jan_ptak"
      ],
      "first_model_latency_seconds": 1.453879,
      "tool_latency_seconds": 0.039251,
      "final_model_latency_seconds": 3.338716,
      "total_latency_seconds": 4.832013,
      "usage": {
        "prompt_tokens": 3161,
        "completion_tokens": 101,
        "total_tokens": 3262
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "answer_matches": [
        true
      ],
      "answer_contains_score": 1.0,
      "source_matches": [
        true
      ],
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": false,
      "tool_score": 1.0,
      "overall_score": 1.0,
      "strict_pass": true,
      "error": null
    },
    {
      "id": "q0441",
      "split": "dev",
      "category": "paraphrase",
      "difficulty": "medium",
      "question": "Aká téma bakalárskej práce je uvedená pri osobe „Yevhenii Medushivskyi“?",
      "override_applied": true,
      "requested_model": "model120-fast",
      "expected_documents": [
        "pages/students/2023/yevhenii_medushivskyi/README.md"
      ],
      "expected_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi"
      ],
      "expected_answer_contains": [
        "Získavanie textu z webových stránok"
      ],
      "should_answer": true,
      "note": null,
      "answer": "V dostupných dokumentoch ZP Wiki sa explicitne uvedená téma bakalárskej práce pre Yevhenii Medushivskyi nenachádza, dokument však obsahuje zoznam úloh a cieľov práce zameraných na vytvorenie crawlera, text mining a natrénoval prediktor získaného textu.\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "Yevhenii Medushivskyi téma bakalárskej práce",
            "limit": 5
          },
          "latency_seconds": 0.063093,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi",
            "https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov",
            "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
            "https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh",
            "https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2023/yevhenii_medushivskyi",
        "https://zp.kemt.fei.tuke.sk/students/2022/yevhenii_leonov",
        "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
        "https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh",
        "https://zp.kemt.fei.tuke.sk/students/2022/serhii_yemets"
      ],
      "first_model_latency_seconds": 1.570533,
      "tool_latency_seconds": 0.063093,
      "final_model_latency_seconds": 7.048205,
      "total_latency_seconds": 8.682057,
      "usage": {
        "prompt_tokens": 7356,
        "completion_tokens": 159,
        "total_tokens": 7515
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "answer_matches": [
        false
      ],
      "answer_contains_score": 0.0,
      "source_matches": [
        true
      ],
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": false,
      "tool_score": 1.0,
      "overall_score": 0.75,
      "strict_pass": false,
      "error": null
    }
  ]
}