jp170na/dp-zp-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
427bbc3222
dp-zp-agent/tools/graphrag/evaluation/results/answers_v7_regression_2026-10-26/failures.json

261 lines
10 KiB
JSON
Raw Blame History

{
"evaluation_kind": "historical_regression",
"questions": [
{
"id": "wide-285",
"category": "conjunctive_disambiguation",
"question": "Ktorá z osôb Dárius Lindvai a Dominik Nagy má kľúčové slovo pytorch a jej práca je z roku 2019?",
"graph_context": {
"status": "ok",
"reason": null,
"facts": [
{
"person": "Dárius Lindvai",
"year": 2019,
"document_path": "pages/students/2016/darius_lindvai/README.md",
"source_url": "https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai"
},
{
"person": "Dárius Lindvai",
"keyword": "pytorch",
"document_path": "pages/students/2016/darius_lindvai/timovy_projekt/tutorial/README.md",
"source_url": "https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial"
}
],
"sources": [
{
"source_id": "S1",
"document_path": "pages/students/2016/darius_lindvai/README.md",
"source_url": "https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai",
"title": "Dárius Lindvai"
},
{
"source_id": "S2",
"document_path": "pages/students/2016/darius_lindvai/timovy_projekt/tutorial/README.md",
"source_url": "https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial",
"title": "PYTORCH - LSTM TUTORIÁL"
},
{
"source_id": "S3",
"document_path": "pages/students/2016/dominik_nagy/README.md",
"source_url": "https://zp.kemt.fei.tuke.sk/students/2016/dominik_nagy",
"title": "Dominik Nagy"
}
],
"aggregation": null,
"complete": true,
"truncated": false,
"truncation_reasons": [],
"index_version": {
"chunks_sha256": "437dde589be46e0c30842dd330de22d96a0173d2dbaff9c5f129568cb2bca4c9",
"created_at_utc": "2026-10-26T00:10:27.444513+00:00",
"documents_sha256": "0542af2a373771cf7c4a95f93d14f4f687f6a0187846892c959aa6efc4bade52",
"embedding_model": "intfloat/multilingual-e5-small",
"git_revision": "3e892a1d49e9bce118cd03478ce15e01c2bebb4e",
"publication_policy": "explicit-false-hidden-v1",
"source_sha256": "007792c12f3467dd3203bb4951ccc197fdc0eef669580f1b1f893baf609ca363",
"version": "33ea0794472c416e97c84371a2a6cdf8"
},
"answer_candidates": [
{
"person": "Dárius Lindvai",
"year": 2019,
"keyword": "pytorch",
"evidence_documents": [
"pages/students/2016/darius_lindvai/README.md",
"pages/students/2016/darius_lindvai/timovy_projekt/tutorial/README.md"
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai",
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial"
]
}
]
},
"strict": false,
"attempts": [
{
"response": {
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"role": "assistant",
"content": "{\n \"answer\": \"\",\n \"status\": \"ok\",\n \"claims\": [\n {\n \"person\": \"Dárius Lindvai\",\n \"year\": 2019,\n \"keyword\": \"pytorch\",\n \"title\": \"PYTORCH - LSTM TUTORIÁL\",\n \"source_urls\": [\n \"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai\",\n \"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial\"\n ]\n }\n ],\n \"source_urls\": [\n \"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai\",\n \"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial\"\n ],\n \"count\": null\n}"
}
}
],
"created": 1792974833,
"model": "Qwen3.5-122B-A10B-Q4_K_M",
"system_fingerprint": "b8580-7c203670f",
"object": "chat.completion",
"usage": {
"completion_tokens": 223,
"prompt_tokens": 1772,
"total_tokens": 1995,
"prompt_tokens_details": {
"cached_tokens": 815
}
},
"id": "chatcmpl-PTuYHRUKhpKd7EhYb0cDozPNHqVaQYxm",
"timings": {
"cache_n": 815,
"prompt_n": 957,
"prompt_ms": 2525.225,
"prompt_per_token_ms": 2.6386886102403344,
"prompt_per_second": 378.97613083982617,
"predicted_n": 223,
"predicted_ms": 7961.962,
"predicted_per_token_ms": 35.70386547085202,
"predicted_per_second": 28.00817185512817
}
},
"format_error": "ValidationError",
"strict": false
},
{
"response": {
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"role": "assistant",
"content": "{\n \"answer\": \"\",\n \"status\": \"ok\",\n \"claims\": [\n {\n \"person\": \"Dárius Lindvai\",\n \"title\": \"PYTORCH - LSTM TUTORIÁL\",\n \"year\": 2019,\n \"keyword\": \"pytorch\"\n }\n ],\n \"source_urls\": [\n \"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai\",\n \"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial\"\n ],\n \"count\": null\n}"
}
}
],
"created": 1792974847,
"model": "Qwen3.5-122B-A10B-Q4_K_M",
"system_fingerprint": "b8580-7c203670f",
"object": "chat.completion",
"usage": {
"completion_tokens": 155,
"prompt_tokens": 2444,
"total_tokens": 2599,
"prompt_tokens_details": {
"cached_tokens": 1768
}
},
"id": "chatcmpl-EqJeUwzc7k5kwW0XpAbdx69edFgRW16H",
"timings": {
"cache_n": 1768,
"prompt_n": 676,
"prompt_ms": 946.839,
"prompt_per_token_ms": 1.4006494082840237,
"prompt_per_second": 713.9545371493991,
"predicted_n": 155,
"predicted_ms": 8280.544,
"predicted_per_token_ms": 53.42286451612903,
"predicted_per_second": 18.71857694373703
}
},
"answer": {
"answer": "",
"status": "ok",
"claims": [
{
"person": "Dárius Lindvai",
"title": "PYTORCH - LSTM TUTORIÁL",
"year": 2019,
"keyword": "pytorch"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai",
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial"
]
},
"format_normalized": false,
"context_issues": [
"unsupported_academic_binding"
],
"strict": false
}
],
"accepted_for_delivery": false,
"response": {
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"role": "assistant",
"content": "{\n \"answer\": \"\",\n \"status\": \"ok\",\n \"claims\": [\n {\n \"person\": \"Dárius Lindvai\",\n \"title\": \"PYTORCH - LSTM TUTORIÁL\",\n \"year\": 2019,\n \"keyword\": \"pytorch\"\n }\n ],\n \"source_urls\": [\n \"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai\",\n \"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial\"\n ],\n \"count\": null\n}"
}
}
],
"created": 1792974847,
"model": "Qwen3.5-122B-A10B-Q4_K_M",
"system_fingerprint": "b8580-7c203670f",
"object": "chat.completion",
"usage": {
"completion_tokens": 155,
"prompt_tokens": 2444,
"total_tokens": 2599,
"prompt_tokens_details": {
"cached_tokens": 1768
}
},
"id": "chatcmpl-EqJeUwzc7k5kwW0XpAbdx69edFgRW16H",
"timings": {
"cache_n": 1768,
"prompt_n": 676,
"prompt_ms": 946.839,
"prompt_per_token_ms": 1.4006494082840237,
"prompt_per_second": 713.9545371493991,
"predicted_n": 155,
"predicted_ms": 8280.544,
"predicted_per_token_ms": 53.42286451612903,
"predicted_per_second": 18.71857694373703
}
},
"answer": {
"answer": "",
"status": "ok",
"claims": [
{
"person": "Dárius Lindvai",
"title": "PYTORCH - LSTM TUTORIÁL",
"year": 2019,
"keyword": "pytorch"
}
],
"source_urls": [
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai",
"https://zp.kemt.fei.tuke.sk/students/2016/darius_lindvai/timovy_projekt/tutorial"
]
},
"context_rejected": true,
"bindings_ok": true,
"unsupported_claims": [
{
"person": "Dárius Lindvai",
"title": "PYTORCH - LSTM TUTORIÁL",
"year": 2019,
"keyword": "pytorch"
}
],
"sources_ok": true,
"status_ok": true,
"count_ok": true,
"set_ok": true,
"citations_ok": true,
"unsupported_urls": [],
"context_issues": [
"empty_answer_text",
"unsupported_academic_binding"
],
"text_values_ok": false,
"missing_text_values": [
"Dárius Lindvai",
"pytorch",
"2019"
],
"contradictory_thesis_label": false,
"latency_seconds": 28.01832078601001
}
]
}
Reference in New Issue View Git Blame Copy Permalink