Širšie overenie RAG

tools/rag/evaluation/results/verified_rag_2026-10-27/summary.json

@@ -4,7 +4,7 @@
   "status": {
     "local_wider_verification": "complete",
     "reports_and_logs": "complete",
-    "wider_80_model_answers": "awaiting_explicit_consent"
+    "wider_80_model_answers": "complete_with_findings"
   },
   "dataset": {
     "path": "tools/rag/evaluation/json_files/verified_rag_regression.json",
@@ -291,7 +291,82 @@
       }
     ]
   },
-  "broad_model_accuracy": null,
+  "broad_model_accuracy": {
+    "passed": 71,
+    "total": 80,
+    "success_percent": 88.75,
+    "meaning": "Strict end-to-end benchmark success, including blocked deliveries and no_answer failures; known regression questions, not an independent general accuracy estimate.",
+    "answerable": {
+      "passed": 61,
+      "total": 70
+    },
+    "unanswerable": {
+      "passed": 10,
+      "total": 10
+    },
+    "blocked_deliveries": 6,
+    "benchmark_no_answer_failures": 3,
+    "known_source_convention_cases": [
+      "q0074",
+      "q1168"
+    ],
+    "source_convention_cases_retained_in_primary_denominator": true,
+    "all_other_cases": {
+      "passed": 71,
+      "total": 78,
+      "success_percent": 91.03,
+      "note": "Secondary diagnostic only; excludes two source-convention cases identified before generation. Does not replace primary 71/80."
+    },
+    "model_attempts": 91,
+    "first_attempt_accepted": 69,
+    "repaired_then_passed": [
+      "q0066",
+      "q0421",
+      "q0476",
+      "q0567",
+      "q1063"
+    ],
+    "accepted_positive_benchmark_failures": 0,
+    "technical_request_errors": 0,
+    "categories": {
+      "disambiguation_year": {
+        "total": 8,
+        "passed": 8
+      },
+      "fact_start_year": {
+        "total": 8,
+        "passed": 8
+      },
+      "fact_title": {
+        "total": 10,
+        "passed": 10
+      },
+      "fact_work_year": {
+        "total": 10,
+        "passed": 9
+      },
+      "keyword_grounding": {
+        "total": 8,
+        "passed": 5
+      },
+      "multi_document": {
+        "total": 8,
+        "passed": 7
+      },
+      "single_document_synthesis": {
+        "total": 8,
+        "passed": 8
+      },
+      "specific_fact": {
+        "total": 10,
+        "passed": 6
+      },
+      "unanswerable_missing_fact": {
+        "total": 10,
+        "passed": 10
+      }
+    }
+  },
   "full_verification": {
     "passed": true,
     "checks": {
@@ -433,15 +508,156 @@
     "project_verification": {
       "path": "data/reports/verification/20261027T005852Z-ef38a940.json",
       "sha256": "eac6ef0caace933d42fd41c540a10c646699843aec6bc2bab28ef8be4727aa8f"
+    },
+    "live_80_raw": {
+      "path": "data/reports/rag_verified/20261027T014645Z-8ec7a6ce.json",
+      "sha256": "0f89ed617c30f4e20095a978540366f8c20d3f190da20415f3d463f704d56319"
+    },
+    "live_80_export": {
+      "path": "tools/rag/evaluation/results/verified_rag_2026-10-27/answers_80.json",
+      "sha256": "8f5f8bd7c031d90c96be70c45d81526d0740de6530d620a48e6d27c7f7180f43"
     }
   },
   "limitations": [
     "Known regression questions; not a new independent accuracy estimate.",
     "Dárius Lindvai: the diploma year 2021 is inferred from taxonomy category dp2021; no explicit diploma heading. Runtime checked RAG deliberately does not use category years as work-year proof.",
     "Source review is automated against original Markdown; independent human annotation has not been performed.",
-    "No 80-case model accuracy has been measured. Existing explicit permission covers five school-model questions; broader export approval is pending.",
     "360 guard checks are controlled fault injection, not 360 generated model responses.",
     "Finding the expected document or extracting a supported tuple does not by itself measure answer correctness.",
-    "Automatic quote relevance uses known expected-value coverage and literal support; independent human semantic review is still absent."
+    "Automatic quote relevance uses known expected-value coverage and literal support; independent human semantic review is still absent.",
-  ]
+    "80 actual model responses are now measured; 71/80 pass the strict known regression benchmark. This is not an independent estimate over unseen questions.",
+    "Nine failed questions remain unresolved in production. This commit records measurement and diagnostics; it does not claim those defects are repaired.",
+    "Automated expected-value coverage and context guards did not detect any accepted positive benchmark failure in this run; this is not proof that every delivered fact is semantically correct."
+  ],
+  "live_80_findings": [
+    {
+      "id": "q0074",
+      "question": "V ktorom roku je pri osobe Dárius Lindvai uvedená diplomová práca?",
+      "issues": [
+        "false_no_answer",
+        "missing_expected_answer_value",
+        "missing_expected_binding"
+      ],
+      "delivery_status": "no_answer",
+      "accepted_for_delivery": true,
+      "group": "source_convention",
+      "finding": "Expected diploma year 2021 is inferred from dp2021, without an explicit diploma heading/year. Runtime refused to infer this work year.",
+      "next_action": "Clarify and document source year policy or add verified explicit source evidence; do not silently change the benchmark after this run."
+    },
+    {
+      "id": "q0078",
+      "question": "Ktorý dokument alebo študent súvisí s témou BiLSTM a osobou Dárius Lindvai?",
+      "issues": [
+        "delivery_blocked",
+        "missing_expected_answer_value"
+      ],
+      "delivery_status": "error",
+      "accepted_for_delivery": false,
+      "group": "generation_schema_and_binding",
+      "finding": "Model added an unsupported diploma-project title/year binding and initially treated BiLSTM as a metadata keyword. Repair retained the unsupported academic binding, although literal topic evidence existed.",
+      "next_action": "For a general topic question prefer minimal literal evidence quotes; avoid unrelated academic claims and distinguish a body term from a metadata tag."
+    },
+    {
+      "id": "q0198",
+      "question": "Aké webové rozhranie mala Alina Vitko použiť pre demo aplikáciu?",
+      "issues": [
+        "delivery_blocked",
+        "missing_expected_answer_value"
+      ],
+      "delivery_status": "error",
+      "accepted_for_delivery": false,
+      "group": "overstrict_reference_guard",
+      "finding": "Literal supported Streamlit quotes were blocked because https://streamlit.io/ appears inside the quote but not in the wiki source_urls allowlist.",
+      "next_action": "Distinguish a verified link inside a literal source quote from its required wiki evidence citation; keep rejecting links absent from the corresponding delivered source."
+    },
+    {
+      "id": "q0544",
+      "question": "Ktorá stáž osoby Youssef Ressaissi súvisela s témou Slovak?",
+      "issues": [
+        "delivery_blocked",
+        "missing_expected_answer_value"
+      ],
+      "delivery_status": "error",
+      "accepted_for_delivery": false,
+      "group": "generation_schema_and_binding",
+      "finding": "Internship goal/year were emitted as academic title/year claims (initially research_project). Typed academic evidence does not support this classification, so both attempts were blocked.",
+      "next_action": "Use literal internship evidence for general questions or explicitly model internship fields; do not relabel internships as academic projects."
+    },
+    {
+      "id": "q0566",
+      "question": "Aký webový framework použila Sevval Bulburu pre aplikáciu?",
+      "issues": [
+        "false_no_answer",
+        "missing_expected_answer_value"
+      ],
+      "delivery_status": "no_answer",
+      "accepted_for_delivery": true,
+      "group": "missing_retrieved_fact",
+      "finding": "Original Sevval Bulburu Markdown explicitly mentions Django (lines 59 and 72), but Django is absent from the actual delivered context despite the correct profile being retrieved. No_answer follows the supplied evidence but fails corpus-level question coverage.",
+      "next_action": "Improve selection/coverage of relevant profile excerpts and verify framework queries without leaking expected answers into retrieval."
+    },
+    {
+      "id": "q0614",
+      "question": "Ktorí ľudia sú uvedení pri projekte podpory slovenčiny v Spacy?",
+      "issues": [
+        "delivery_blocked",
+        "missing_expected_answer_value"
+      ],
+      "delivery_status": "error",
+      "accepted_for_delivery": false,
+      "group": "generation_schema_and_quote",
+      "finding": "People-list answer added unnecessary academic claims without their specific sources and merged non-contiguous source sections into a purported literal quote. Relative links were additionally rejected by the quote reference policy.",
+      "next_action": "Use separate contiguous people-list quotes, cite their source, omit unrelated academic claims, and handle attested relative references."
+    },
+    {
+      "id": "q0631",
+      "question": "Aké tri prístupy alebo nástroje sú uvedené pri morfologickej analýze slovenského jazyka?",
+      "issues": [
+        "delivery_blocked",
+        "missing_expected_answer_value"
+      ],
+      "delivery_status": "error",
+      "accepted_for_delivery": false,
+      "group": "overstrict_reference_guard",
+      "finding": "Supported Hunspell/BPE/Morfessor quote was blocked for the explicitly supplied IEEE reference inside it.",
+      "next_action": "Handle attested quoted reference URLs without allowing invented citations."
+    },
+    {
+      "id": "q0816",
+      "question": "Ktorý záznam v ZP Wiki spája osobu Dávid Omasta s témou „generatívne modely“?",
+      "issues": [
+        "delivery_blocked",
+        "missing_expected_answer_value"
+      ],
+      "delivery_status": "error",
+      "accepted_for_delivery": false,
+      "group": "generation_schema_and_quote",
+      "finding": "Initial answer added unsupported study-start year. Repair removed it but still joined separated body passages into a purported contiguous quote, omitting an intervening line.",
+      "next_action": "Select each exact contiguous passage separately and omit unrequested uncertain metadata."
+    },
+    {
+      "id": "q1168",
+      "question": "Porovnaj záznamy osôb Dárius Lindvai a Tetiana Mohorian. Pri osobe Dárius Lindvai použi záznam typu „diplomová práca“ a pri osobe Tetiana Mohorian záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.",
+      "issues": [
+        "false_no_answer",
+        "missing_expected_answer_value",
+        "missing_expected_binding"
+      ],
+      "delivery_status": "no_answer",
+      "accepted_for_delivery": true,
+      "group": "source_convention",
+      "finding": "Comparison needs Darius Lindvai diploma title/year; its year has the same predeclared taxonomy-only ambiguity as q0074.",
+      "next_action": "Resolve source convention together with q0074."
+    }
+  ],
+  "live_80_verification": {
+    "consent": "Explicit user consent for all 80 questions and their retrieved wiki evidence to ui.tukekemt.xyz.",
+    "same_predeclared_dataset": true,
+    "unchanged_code_during_run": true,
+    "matching_deployed_python_modules": 125,
+    "single_index_version": "497fadb8f38c4af18d7d9e3b06cdb0a0",
+    "tools_ready_after_run": true,
+    "generation_or_benchmark_tuning_during_run": false,
+    "production_code_changes_for_this_measurement": false
+  }
 }