417 lines
11 KiB
JSON
417 lines
11 KiB
JSON
{
|
|
"method": "Live synthetic functional checks; separate from corpus accuracy metrics.",
|
|
"model": "model120-fast",
|
|
"hashes": {
|
|
"/home/janko/DP/zp-agent/tools/answer_evaluation/check_context_scenarios.py": "cbb2edf8fdcba61b9e5cc50e28cb1a1dcc25b0e0ab990177b41f40031223ca63",
|
|
"/home/janko/DP/zp-agent/tools/answer_evaluation/graph_answer_prompt.txt": "8c8274a22bf5b44364d8fc8226a8f659b31c232f5a2aba5916d693b559e9f341",
|
|
"/home/janko/DP/zp-agent/tools/answer_evaluation/context_policy.py": "e7b7c511e5e9cafdef04969eb4007a6a61ac4b99bcc60e9e722945a15bec1f26",
|
|
"/home/janko/DP/zp-agent/tools/answer_evaluation/checked_generation.py": "7e017471e6281db62b79f270210db3ab14c7e31a4623bbfa3d9a5fd8911b1c20",
|
|
"/home/janko/DP/zp-agent/tools/answer_evaluation/answer_format.py": "86a27b4aebc6a5077e13794b81957a1cf063a1f3e070c583120ba53c1788d2f8"
|
|
},
|
|
"scenarios": [
|
|
{
|
|
"id": "complete",
|
|
"question": "Uveď názov práce osoby Eva Vzorová a rok práce.",
|
|
"context": {
|
|
"status": "ok",
|
|
"reason": null,
|
|
"complete": true,
|
|
"truncated": false,
|
|
"truncation_reasons": [],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": "S1",
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"facts": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"aggregation": null,
|
|
"answer_candidates": []
|
|
},
|
|
"expected": {
|
|
"status": "ok",
|
|
"claims": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "incomplete_without_truncated",
|
|
"question": "Uveď názov práce osoby Eva Vzorová a rok práce.",
|
|
"context": {
|
|
"status": "ok",
|
|
"reason": null,
|
|
"complete": false,
|
|
"truncated": false,
|
|
"truncation_reasons": [],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": "S1",
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"facts": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"aggregation": null,
|
|
"answer_candidates": []
|
|
},
|
|
"expected": {
|
|
"status": "ok",
|
|
"claims": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "truncated",
|
|
"question": "Uveď názov práce osoby Eva Vzorová a rok práce.",
|
|
"context": {
|
|
"status": "ok",
|
|
"reason": null,
|
|
"complete": false,
|
|
"truncated": true,
|
|
"truncation_reasons": [
|
|
"limit"
|
|
],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": "S1",
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"facts": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"aggregation": null,
|
|
"answer_candidates": []
|
|
},
|
|
"expected": {
|
|
"status": "ok",
|
|
"claims": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "unknown_completeness",
|
|
"question": "Uveď názov práce osoby Eva Vzorová a rok práce.",
|
|
"context": {
|
|
"status": "ok",
|
|
"reason": null,
|
|
"complete": null,
|
|
"truncated": false,
|
|
"truncation_reasons": [],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": "S1",
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"facts": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"aggregation": null,
|
|
"answer_candidates": []
|
|
},
|
|
"expected": {
|
|
"status": "ok",
|
|
"claims": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "exact_count_partial_evidence",
|
|
"question": "Aký je počet vyhovujúcich osôb?",
|
|
"context": {
|
|
"status": "ok",
|
|
"reason": null,
|
|
"complete": false,
|
|
"truncated": true,
|
|
"truncation_reasons": [
|
|
"evidence_limit"
|
|
],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": "S1",
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"facts": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"aggregation": {
|
|
"count": 2,
|
|
"complete": true
|
|
},
|
|
"answer_candidates": []
|
|
},
|
|
"expected": {
|
|
"status": "ok",
|
|
"count": 2
|
|
}
|
|
},
|
|
{
|
|
"id": "same_person_child_tag",
|
|
"question": "Uveď názov práce osoby Eva Vzorová v roku 2026 s kľúčovým slovom python.",
|
|
"context": {
|
|
"status": "ok",
|
|
"reason": null,
|
|
"complete": true,
|
|
"truncated": false,
|
|
"truncation_reasons": [],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": "S1",
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
},
|
|
{
|
|
"source_id": "S2",
|
|
"document_path": "child.md",
|
|
"source_url": "https://example.test/child"
|
|
}
|
|
],
|
|
"facts": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
},
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"keyword": "python",
|
|
"document_path": "child.md",
|
|
"source_url": "https://example.test/child"
|
|
}
|
|
],
|
|
"aggregation": null,
|
|
"answer_candidates": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"keyword": "python",
|
|
"evidence_documents": [
|
|
"a.md",
|
|
"child.md"
|
|
],
|
|
"source_urls": [
|
|
"https://example.test/a",
|
|
"https://example.test/child"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"expected": {
|
|
"status": "ok",
|
|
"claims": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"keyword": "python"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "same_person_year_child_tag",
|
|
"question": "Ktorá osoba spĺňa kľúčové slovo python a rok práce 2026?",
|
|
"context": {
|
|
"status": "ok",
|
|
"reason": null,
|
|
"complete": true,
|
|
"truncated": false,
|
|
"truncation_reasons": [],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": "S1",
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
},
|
|
{
|
|
"source_id": "S2",
|
|
"document_path": "child.md",
|
|
"source_url": "https://example.test/child"
|
|
}
|
|
],
|
|
"facts": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"year": 2026,
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
},
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"keyword": "python",
|
|
"document_path": "child.md",
|
|
"source_url": "https://example.test/child"
|
|
}
|
|
],
|
|
"aggregation": null,
|
|
"answer_candidates": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"year": 2026,
|
|
"keyword": "python",
|
|
"evidence_documents": [
|
|
"a.md",
|
|
"child.md"
|
|
],
|
|
"source_urls": [
|
|
"https://example.test/a",
|
|
"https://example.test/child"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"expected": {
|
|
"status": "ok",
|
|
"claims": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"year": 2026,
|
|
"keyword": "python"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "type_is_not_keyword",
|
|
"question": "Uveď názov diplomového projektu osoby Eva Vzorová a rok.",
|
|
"context": {
|
|
"status": "ok",
|
|
"reason": null,
|
|
"complete": true,
|
|
"truncated": false,
|
|
"truncation_reasons": [],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [
|
|
{
|
|
"source_id": "S1",
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"facts": [
|
|
{
|
|
"person": "Eva Vzorová",
|
|
"title": "Vyhľadávanie informácií",
|
|
"year": 2026,
|
|
"document_path": "a.md",
|
|
"source_url": "https://example.test/a"
|
|
}
|
|
],
|
|
"aggregation": null,
|
|
"answer_candidates": []
|
|
},
|
|
"expected": {
|
|
"status": "ok",
|
|
"no_keyword": true
|
|
}
|
|
},
|
|
{
|
|
"id": "no_answer",
|
|
"question": "Akú prácu má Neexistujúca Osoba?",
|
|
"context": {
|
|
"status": "no_answer",
|
|
"reason": "unknown_person",
|
|
"complete": true,
|
|
"truncated": false,
|
|
"truncation_reasons": [],
|
|
"index_version": {
|
|
"version": "synthetic-functional-fixture"
|
|
},
|
|
"sources": [],
|
|
"facts": [],
|
|
"aggregation": null,
|
|
"answer_candidates": []
|
|
},
|
|
"expected": {
|
|
"status": "no_answer"
|
|
}
|
|
}
|
|
]
|
|
}
|