{
  "configuration": {
    "method": "Deterministic delivery replay on the same saved model responses, not new generation.",
    "input": "tools/graphrag/evaluation/results/v6_preparation/rag_live_smoke.json",
    "input_sha256": "674272fd911a5ae04690150ac899531feae90b4d44a1735a530a2838a0d32ff3",
    "delivery_files": {
      "/home/janko/DP/zp-agent/tools/answer_evaluation/rag_delivery.py": "b83f93d446f44fdd7d41c12eeeaae69c0b02908076624f4eb3f4f72b9b88dbc8",
      "tools/rag/evaluation/evaluate_rag_answers.py": "945c2da655537fb37fe82a08e10dc3ed66a2d90980921f61f61efb171a4a26a2",
      "tools/rag/evaluation/rag_metrics.py": "3e95922df90dd1b757a550040a2d9bd4f857e17d179f60170d7cc881a08f94b6",
      "tools/rag/evaluation/rag_runner.py": "031abf7d5b1a026eb199aa2a579bfb8baaa01175279d0018a364bfe50b76c3c1",
      "tools/answer_evaluation/context_policy.py": "e7b7c511e5e9cafdef04969eb4007a6a61ac4b99bcc60e9e722945a15bec1f26"
    }
  },
  "results": [
    {
      "id": "q0053",
      "question": "Aký je názov diplomová práca osoby Lukáš Pokrývka?",
      "citations_ok": true,
      "unsupported_urls": [],
      "retrieval_coverage": [
        {
          "complete": false,
          "truncated": true,
          "truncation_reasons": [
            "candidate_limit",
            "per_document_limit",
            "result_limit"
          ],
          "index_version": {
            "version": "10b78a8e70744e2792ffe4cf11d45571",
            "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
            "created_at_utc": "2026-10-25T21:10:39.160542+00:00",
            "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
            "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
            "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
            "embedding_model": "intfloat/multilingual-e5-small",
            "publication_policy": "explicit-false-hidden-v1"
          }
        }
      ],
      "context_issues": [],
      "accepted_for_delivery": true,
      "answer": "Výpis je neúplný.\n\nNázov diplomovej práce Lukáša Pokrývku je „Paralelné trénovanie neurónových sietí\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "Aký je názov diplomová práca osoby Lukáš Pokrývka?",
            "limit": 5,
            "published_only": true,
            "max_per_document": 1
          },
          "latency_seconds": 8.6679,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
            "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
            "https://zp.kemt.fei.tuke.sk/students/2019/patrik_pokrivcak",
            "https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh",
            "https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak/dp2021"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
        "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
        "https://zp.kemt.fei.tuke.sk/students/2019/patrik_pokrivcak",
        "https://zp.kemt.fei.tuke.sk/students/2018/oleh_bilykh",
        "https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak/dp2021"
      ],
      "first_model_latency_seconds": 5.65362,
      "tool_latency_seconds": 8.6679,
      "final_model_latency_seconds": 23.053733,
      "total_latency_seconds": 37.375855,
      "usage": {
        "prompt_tokens": 19045,
        "completion_tokens": 137,
        "total_tokens": 19182
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "citations_checked": true,
      "answer_matches": [
        true
      ],
      "answer_contains_score": 1.0,
      "source_matches": [
        true
      ],
      "source_match_count": 1,
      "source_required_count": 1,
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": false,
      "tool_score": 1.0,
      "overall_score": 1.0,
      "strict_pass": true,
      "model_answer": "Názov diplomovej práce Lukáša Pokrývku je „Paralelné trénovanie neurónových sietí\".\n\nZdroj: https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
      "first_pass_accepted_for_delivery": false,
      "first_pass_context_issues": [
        "missing_incompleteness_notice"
      ],
      "display_notice_added": true
    },
    {
      "id": "q0060",
      "question": "Rozlíš rok začiatku štúdia a rok práce osoby Lukáš Pokrývka.",
      "citations_ok": true,
      "unsupported_urls": [],
      "retrieval_coverage": [
        {
          "complete": false,
          "truncated": true,
          "truncation_reasons": [
            "candidate_limit",
            "per_document_limit",
            "result_limit"
          ],
          "index_version": {
            "version": "10b78a8e70744e2792ffe4cf11d45571",
            "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
            "created_at_utc": "2026-10-25T21:10:39.160542+00:00",
            "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
            "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
            "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
            "embedding_model": "intfloat/multilingual-e5-small",
            "publication_policy": "explicit-false-hidden-v1"
          }
        }
      ],
      "context_issues": [],
      "accepted_for_delivery": true,
      "answer": "Výpis je neúplný.\n\nRok začiatku štúdia osoby Lukáš Pokrývka je 2016, zatiaľ čo jeho diplomová práca bola realizovaná v roku 2021.\n\nZdroj:\nhttps://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "Rozlíš rok začiatku štúdia a rok práce osoby Lukáš Pokrývka.",
            "limit": 10,
            "published_only": true,
            "max_per_document": 5
          },
          "latency_seconds": 8.727633,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
            "https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
            "https://zp.kemt.fei.tuke.sk/students/2020/jakub_kristian_lukas",
            "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
            "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
        "https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
        "https://zp.kemt.fei.tuke.sk/students/2020/jakub_kristian_lukas",
        "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
        "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela"
      ],
      "first_model_latency_seconds": 5.710692,
      "tool_latency_seconds": 8.727633,
      "final_model_latency_seconds": 40.777827,
      "total_latency_seconds": 55.216681,
      "usage": {
        "prompt_tokens": 20160,
        "completion_tokens": 155,
        "total_tokens": 20315
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "citations_checked": true,
      "answer_matches": [
        true,
        true
      ],
      "answer_contains_score": 1.0,
      "source_matches": [
        true
      ],
      "source_match_count": 1,
      "source_required_count": 1,
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": false,
      "tool_score": 1.0,
      "overall_score": 1.0,
      "strict_pass": true,
      "model_answer": "Rok začiatku štúdia osoby Lukáš Pokrývka je 2016, zatiaľ čo jeho diplomová práca bola realizovaná v roku 2021.\n\nZdroj:\nhttps://zp.kemt.fei.tuke.sk/students/2016/lukas_pokryvka",
      "first_pass_accepted_for_delivery": false,
      "first_pass_context_issues": [
        "missing_incompleteness_notice"
      ],
      "display_notice_added": true
    },
    {
      "id": "q0608",
      "question": "Aký source_url patrí k stránke „Podpora slovenčiny v knižnici Spacy“?",
      "citations_ok": true,
      "unsupported_urls": [],
      "retrieval_coverage": [
        {
          "complete": false,
          "truncated": true,
          "truncation_reasons": [
            "candidate_limit",
            "per_document_limit",
            "result_limit"
          ],
          "index_version": {
            "version": "10b78a8e70744e2792ffe4cf11d45571",
            "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
            "created_at_utc": "2026-10-25T21:10:39.160542+00:00",
            "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
            "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
            "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
            "embedding_model": "intfloat/multilingual-e5-small",
            "publication_policy": "explicit-false-hidden-v1"
          }
        }
      ],
      "context_issues": [],
      "accepted_for_delivery": true,
      "answer": "Výpis je neúplný.\n\nStránka s názvom „Podpora slovenčiny v knižnici Spacy\" má pridelený zdrojový URL adresu https://zp.kemt.fei.tuke.sk/topics/spacy.\n\nZdroj: https://zp.kemt.fei.tuke.sk/topics/spacy",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "Aký source_url patrí k stránke „Podpora slovenčiny v knižnici Spacy“?",
            "limit": 5,
            "max_per_document": 1,
            "published_only": true
          },
          "latency_seconds": 0.620349,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/topics/spacy",
            "https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus",
            "https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak",
            "https://zp.kemt.fei.tuke.sk/students/2017/than_trung_thanh",
            "https://zp.kemt.fei.tuke.sk/topics/resources"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/topics/spacy",
        "https://zp.kemt.fei.tuke.sk/students/2016/maros_harahus",
        "https://zp.kemt.fei.tuke.sk/students/2016/jakub_maruniak",
        "https://zp.kemt.fei.tuke.sk/students/2017/than_trung_thanh",
        "https://zp.kemt.fei.tuke.sk/topics/resources"
      ],
      "first_model_latency_seconds": 28.658617,
      "tool_latency_seconds": 0.620349,
      "final_model_latency_seconds": 35.815477,
      "total_latency_seconds": 65.094973,
      "usage": {
        "prompt_tokens": 28510,
        "completion_tokens": 141,
        "total_tokens": 28651
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "citations_checked": true,
      "answer_matches": [
        true
      ],
      "answer_contains_score": 1.0,
      "source_matches": [
        true
      ],
      "source_match_count": 1,
      "source_required_count": 1,
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": false,
      "tool_score": 1.0,
      "overall_score": 1.0,
      "strict_pass": true,
      "model_answer": "Stránka s názvom „Podpora slovenčiny v knižnici Spacy\" má pridelený zdrojový URL adresu https://zp.kemt.fei.tuke.sk/topics/spacy.\n\nZdroj: https://zp.kemt.fei.tuke.sk/topics/spacy",
      "first_pass_accepted_for_delivery": false,
      "first_pass_context_issues": [
        "missing_incompleteness_notice"
      ],
      "display_notice_added": true
    },
    {
      "id": "q1147",
      "question": "Porovnaj záznamy osôb Ján Holp a Ondrej Megela. Pri osobe Ján Holp použi záznam typu „diplomová práca“ a pri osobe Ondrej Megela záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.",
      "citations_ok": true,
      "unsupported_urls": [],
      "retrieval_coverage": [
        {
          "complete": false,
          "truncated": true,
          "truncation_reasons": [
            "candidate_limit",
            "per_document_limit",
            "result_limit"
          ],
          "index_version": {
            "version": "10b78a8e70744e2792ffe4cf11d45571",
            "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
            "created_at_utc": "2026-10-25T21:10:39.160542+00:00",
            "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
            "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
            "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
            "embedding_model": "intfloat/multilingual-e5-small",
            "publication_policy": "explicit-false-hidden-v1"
          }
        }
      ],
      "context_issues": [],
      "accepted_for_delivery": true,
      "answer": "Výpis je neúplný.\n\nJán Holp vypracoval diplomovú prácu s názvom „Systém získavania informácií v slovenskom jazyku\" v roku 2021 a Ondrej Megela vypracoval bakalársku prácu s názvom „Neurónové jazykové modelovanie typu BERT\" v roku 2020.\n\nZdroje:\nhttps://zp.kemt.fei.tuke.sk/students/2016/jan_holp\nhttps://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "Porovnaj záznamy osôb Ján Holp a Ondrej Megela. Pri osobe Ján Holp použi záznam typu „diplomová práca“ a pri osobe Ondrej Megela záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.",
            "limit": 5,
            "published_only": true,
            "max_per_document": 1
          },
          "latency_seconds": 1.001685,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
            "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
            "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
            "https://zp.kemt.fei.tuke.sk/topics/faq",
            "https://zp.kemt.fei.tuke.sk/topics/bert"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
        "https://zp.kemt.fei.tuke.sk/students/2016/jan_holp",
        "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko",
        "https://zp.kemt.fei.tuke.sk/topics/faq",
        "https://zp.kemt.fei.tuke.sk/topics/bert"
      ],
      "first_model_latency_seconds": 14.535465,
      "tool_latency_seconds": 1.001685,
      "final_model_latency_seconds": 56.647723,
      "total_latency_seconds": 72.185893,
      "usage": {
        "prompt_tokens": 25893,
        "completion_tokens": 208,
        "total_tokens": 26101
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "citations_checked": true,
      "answer_matches": [
        true,
        true,
        true,
        true
      ],
      "answer_contains_score": 1.0,
      "source_matches": [
        true,
        true
      ],
      "source_match_count": 2,
      "source_required_count": 2,
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": false,
      "tool_score": 1.0,
      "overall_score": 1.0,
      "strict_pass": true,
      "model_answer": "Ján Holp vypracoval diplomovú prácu s názvom „Systém získavania informácií v slovenskom jazyku\" v roku 2021 a Ondrej Megela vypracoval bakalársku prácu s názvom „Neurónové jazykové modelovanie typu BERT\" v roku 2020.\n\nZdroje:\nhttps://zp.kemt.fei.tuke.sk/students/2016/jan_holp\nhttps://zp.kemt.fei.tuke.sk/students/2018/ondrej_megela",
      "first_pass_accepted_for_delivery": false,
      "first_pass_context_issues": [
        "missing_incompleteness_notice"
      ],
      "display_notice_added": true
    },
    {
      "id": "q0888",
      "question": "Aký konkrétny finančný rozpočet bol uvedený pre projekt osoby Tomáš Kuchárik?",
      "citations_ok": true,
      "unsupported_urls": [],
      "retrieval_coverage": [
        {
          "complete": false,
          "truncated": true,
          "truncation_reasons": [
            "candidate_limit",
            "per_document_limit",
            "result_limit"
          ],
          "index_version": {
            "version": "10b78a8e70744e2792ffe4cf11d45571",
            "git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
            "created_at_utc": "2026-10-25T21:10:39.160542+00:00",
            "source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
            "documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
            "chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
            "embedding_model": "intfloat/multilingual-e5-small",
            "publication_policy": "explicit-false-hidden-v1"
          }
        }
      ],
      "context_issues": [],
      "accepted_for_delivery": true,
      "answer": "Výpis je neúplný.\n\nV dostupných dokumentoch ZP Wiki sa informácia o konkrétnom finančnom rozpočte pre projekt osoby Tomáš Kuchárik nepodarilo spoľahlivo nájsť.\n\nZdroj:\nhttps://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
      "tool_called": true,
      "tool_call_count": 1,
      "tool_calls": [
        {
          "name": "retrieve_zpwiki_context",
          "arguments": {
            "query": "Aký konkrétny finančný rozpočet bol uvedený pre projekt osoby Tomáš Kuchárik?",
            "limit": 5,
            "published_only": true
          },
          "latency_seconds": 0.454629,
          "source_urls": [
            "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
            "https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
            "https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas",
            "https://zp.kemt.fei.tuke.sk/students/2020/matus_suchanic",
            "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
          ]
        }
      ],
      "rag_source_urls": [
        "https://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
        "https://zp.kemt.fei.tuke.sk/students/2020/matej_kobylan",
        "https://zp.kemt.fei.tuke.sk/students/2019/filip_tomas",
        "https://zp.kemt.fei.tuke.sk/students/2020/matus_suchanic",
        "https://zp.kemt.fei.tuke.sk/students/2019/michal_stromko"
      ],
      "first_model_latency_seconds": 129.609931,
      "tool_latency_seconds": 0.454629,
      "final_model_latency_seconds": 20.02263,
      "total_latency_seconds": 150.087617,
      "usage": {
        "prompt_tokens": 14022,
        "completion_tokens": 136,
        "total_tokens": 14158
      },
      "response_model": "Qwen3.5-122B-A10B-Q4_K_M",
      "citations_checked": true,
      "answer_matches": [],
      "answer_contains_score": 1.0,
      "source_matches": [
        true
      ],
      "source_match_count": 1,
      "source_required_count": 0,
      "source_url_score": 1.0,
      "should_answer_ok": true,
      "should_answer_score": 1.0,
      "returned_no_answer": true,
      "tool_score": 1.0,
      "overall_score": 1.0,
      "strict_pass": true,
      "model_answer": "V dostupných dokumentoch ZP Wiki sa informácia o konkrétnom finančnom rozpočte pre projekt osoby Tomáš Kuchárik nepodarilo spoľahlivo nájsť.\n\nZdroj:\nhttps://zp.kemt.fei.tuke.sk/students/2016/tomas_kucharik",
      "first_pass_accepted_for_delivery": false,
      "first_pass_context_issues": [
        "missing_incompleteness_notice"
      ],
      "display_notice_added": true
    }
  ],
  "summary": {
    "questions": 5,
    "strict_pass": 5,
    "display_notice_added": 5
  }
}