Širšie overenie RAG
This commit is contained in:
parent
966ef494c5
commit
084d04d867
16751
tools/rag/evaluation/results/verified_rag_2026-10-27/answers_80.json
Normal file
16751
tools/rag/evaluation/results/verified_rag_2026-10-27/answers_80.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -4,7 +4,7 @@
|
||||
"status": {
|
||||
"local_wider_verification": "complete",
|
||||
"reports_and_logs": "complete",
|
||||
"wider_80_model_answers": "awaiting_explicit_consent"
|
||||
"wider_80_model_answers": "complete_with_findings"
|
||||
},
|
||||
"dataset": {
|
||||
"path": "tools/rag/evaluation/json_files/verified_rag_regression.json",
|
||||
@ -291,7 +291,82 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"broad_model_accuracy": null,
|
||||
"broad_model_accuracy": {
|
||||
"passed": 71,
|
||||
"total": 80,
|
||||
"success_percent": 88.75,
|
||||
"meaning": "Strict end-to-end benchmark success, including blocked deliveries and no_answer failures; known regression questions, not an independent general accuracy estimate.",
|
||||
"answerable": {
|
||||
"passed": 61,
|
||||
"total": 70
|
||||
},
|
||||
"unanswerable": {
|
||||
"passed": 10,
|
||||
"total": 10
|
||||
},
|
||||
"blocked_deliveries": 6,
|
||||
"benchmark_no_answer_failures": 3,
|
||||
"known_source_convention_cases": [
|
||||
"q0074",
|
||||
"q1168"
|
||||
],
|
||||
"source_convention_cases_retained_in_primary_denominator": true,
|
||||
"all_other_cases": {
|
||||
"passed": 71,
|
||||
"total": 78,
|
||||
"success_percent": 91.03,
|
||||
"note": "Secondary diagnostic only; excludes two source-convention cases identified before generation. Does not replace primary 71/80."
|
||||
},
|
||||
"model_attempts": 91,
|
||||
"first_attempt_accepted": 69,
|
||||
"repaired_then_passed": [
|
||||
"q0066",
|
||||
"q0421",
|
||||
"q0476",
|
||||
"q0567",
|
||||
"q1063"
|
||||
],
|
||||
"accepted_positive_benchmark_failures": 0,
|
||||
"technical_request_errors": 0,
|
||||
"categories": {
|
||||
"disambiguation_year": {
|
||||
"total": 8,
|
||||
"passed": 8
|
||||
},
|
||||
"fact_start_year": {
|
||||
"total": 8,
|
||||
"passed": 8
|
||||
},
|
||||
"fact_title": {
|
||||
"total": 10,
|
||||
"passed": 10
|
||||
},
|
||||
"fact_work_year": {
|
||||
"total": 10,
|
||||
"passed": 9
|
||||
},
|
||||
"keyword_grounding": {
|
||||
"total": 8,
|
||||
"passed": 5
|
||||
},
|
||||
"multi_document": {
|
||||
"total": 8,
|
||||
"passed": 7
|
||||
},
|
||||
"single_document_synthesis": {
|
||||
"total": 8,
|
||||
"passed": 8
|
||||
},
|
||||
"specific_fact": {
|
||||
"total": 10,
|
||||
"passed": 6
|
||||
},
|
||||
"unanswerable_missing_fact": {
|
||||
"total": 10,
|
||||
"passed": 10
|
||||
}
|
||||
}
|
||||
},
|
||||
"full_verification": {
|
||||
"passed": true,
|
||||
"checks": {
|
||||
@ -433,15 +508,156 @@
|
||||
"project_verification": {
|
||||
"path": "data/reports/verification/20261027T005852Z-ef38a940.json",
|
||||
"sha256": "eac6ef0caace933d42fd41c540a10c646699843aec6bc2bab28ef8be4727aa8f"
|
||||
},
|
||||
"live_80_raw": {
|
||||
"path": "data/reports/rag_verified/20261027T014645Z-8ec7a6ce.json",
|
||||
"sha256": "0f89ed617c30f4e20095a978540366f8c20d3f190da20415f3d463f704d56319"
|
||||
},
|
||||
"live_80_export": {
|
||||
"path": "tools/rag/evaluation/results/verified_rag_2026-10-27/answers_80.json",
|
||||
"sha256": "8f5f8bd7c031d90c96be70c45d81526d0740de6530d620a48e6d27c7f7180f43"
|
||||
}
|
||||
},
|
||||
"limitations": [
|
||||
"Known regression questions; not a new independent accuracy estimate.",
|
||||
"Dárius Lindvai: the diploma year 2021 is inferred from taxonomy category dp2021; no explicit diploma heading. Runtime checked RAG deliberately does not use category years as work-year proof.",
|
||||
"Source review is automated against original Markdown; independent human annotation has not been performed.",
|
||||
"No 80-case model accuracy has been measured. Existing explicit permission covers five school-model questions; broader export approval is pending.",
|
||||
"360 guard checks are controlled fault injection, not 360 generated model responses.",
|
||||
"Finding the expected document or extracting a supported tuple does not by itself measure answer correctness.",
|
||||
"Automatic quote relevance uses known expected-value coverage and literal support; independent human semantic review is still absent."
|
||||
]
|
||||
"Automatic quote relevance uses known expected-value coverage and literal support; independent human semantic review is still absent.",
|
||||
"80 actual model responses are now measured; 71/80 pass the strict known regression benchmark. This is not an independent estimate over unseen questions.",
|
||||
"Nine failed questions remain unresolved in production. This commit records measurement and diagnostics; it does not claim those defects are repaired.",
|
||||
"Automated expected-value coverage and context guards did not detect any accepted positive benchmark failure in this run; this is not proof that every delivered fact is semantically correct."
|
||||
],
|
||||
"live_80_findings": [
|
||||
{
|
||||
"id": "q0074",
|
||||
"question": "V ktorom roku je pri osobe Dárius Lindvai uvedená diplomová práca?",
|
||||
"issues": [
|
||||
"false_no_answer",
|
||||
"missing_expected_answer_value",
|
||||
"missing_expected_binding"
|
||||
],
|
||||
"delivery_status": "no_answer",
|
||||
"accepted_for_delivery": true,
|
||||
"group": "source_convention",
|
||||
"finding": "Expected diploma year 2021 is inferred from dp2021, without an explicit diploma heading/year. Runtime refused to infer this work year.",
|
||||
"next_action": "Clarify and document source year policy or add verified explicit source evidence; do not silently change the benchmark after this run."
|
||||
},
|
||||
{
|
||||
"id": "q0078",
|
||||
"question": "Ktorý dokument alebo študent súvisí s témou BiLSTM a osobou Dárius Lindvai?",
|
||||
"issues": [
|
||||
"delivery_blocked",
|
||||
"missing_expected_answer_value"
|
||||
],
|
||||
"delivery_status": "error",
|
||||
"accepted_for_delivery": false,
|
||||
"group": "generation_schema_and_binding",
|
||||
"finding": "Model added an unsupported diploma-project title/year binding and initially treated BiLSTM as a metadata keyword. Repair retained the unsupported academic binding, although literal topic evidence existed.",
|
||||
"next_action": "For a general topic question prefer minimal literal evidence quotes; avoid unrelated academic claims and distinguish a body term from a metadata tag."
|
||||
},
|
||||
{
|
||||
"id": "q0198",
|
||||
"question": "Aké webové rozhranie mala Alina Vitko použiť pre demo aplikáciu?",
|
||||
"issues": [
|
||||
"delivery_blocked",
|
||||
"missing_expected_answer_value"
|
||||
],
|
||||
"delivery_status": "error",
|
||||
"accepted_for_delivery": false,
|
||||
"group": "overstrict_reference_guard",
|
||||
"finding": "Literal supported Streamlit quotes were blocked because https://streamlit.io/ appears inside the quote but not in the wiki source_urls allowlist.",
|
||||
"next_action": "Distinguish a verified link inside a literal source quote from its required wiki evidence citation; keep rejecting links absent from the corresponding delivered source."
|
||||
},
|
||||
{
|
||||
"id": "q0544",
|
||||
"question": "Ktorá stáž osoby Youssef Ressaissi súvisela s témou Slovak?",
|
||||
"issues": [
|
||||
"delivery_blocked",
|
||||
"missing_expected_answer_value"
|
||||
],
|
||||
"delivery_status": "error",
|
||||
"accepted_for_delivery": false,
|
||||
"group": "generation_schema_and_binding",
|
||||
"finding": "Internship goal/year were emitted as academic title/year claims (initially research_project). Typed academic evidence does not support this classification, so both attempts were blocked.",
|
||||
"next_action": "Use literal internship evidence for general questions or explicitly model internship fields; do not relabel internships as academic projects."
|
||||
},
|
||||
{
|
||||
"id": "q0566",
|
||||
"question": "Aký webový framework použila Sevval Bulburu pre aplikáciu?",
|
||||
"issues": [
|
||||
"false_no_answer",
|
||||
"missing_expected_answer_value"
|
||||
],
|
||||
"delivery_status": "no_answer",
|
||||
"accepted_for_delivery": true,
|
||||
"group": "missing_retrieved_fact",
|
||||
"finding": "Original Sevval Bulburu Markdown explicitly mentions Django (lines 59 and 72), but Django is absent from the actual delivered context despite the correct profile being retrieved. No_answer follows the supplied evidence but fails corpus-level question coverage.",
|
||||
"next_action": "Improve selection/coverage of relevant profile excerpts and verify framework queries without leaking expected answers into retrieval."
|
||||
},
|
||||
{
|
||||
"id": "q0614",
|
||||
"question": "Ktorí ľudia sú uvedení pri projekte podpory slovenčiny v Spacy?",
|
||||
"issues": [
|
||||
"delivery_blocked",
|
||||
"missing_expected_answer_value"
|
||||
],
|
||||
"delivery_status": "error",
|
||||
"accepted_for_delivery": false,
|
||||
"group": "generation_schema_and_quote",
|
||||
"finding": "People-list answer added unnecessary academic claims without their specific sources and merged non-contiguous source sections into a purported literal quote. Relative links were additionally rejected by the quote reference policy.",
|
||||
"next_action": "Use separate contiguous people-list quotes, cite their source, omit unrelated academic claims, and handle attested relative references."
|
||||
},
|
||||
{
|
||||
"id": "q0631",
|
||||
"question": "Aké tri prístupy alebo nástroje sú uvedené pri morfologickej analýze slovenského jazyka?",
|
||||
"issues": [
|
||||
"delivery_blocked",
|
||||
"missing_expected_answer_value"
|
||||
],
|
||||
"delivery_status": "error",
|
||||
"accepted_for_delivery": false,
|
||||
"group": "overstrict_reference_guard",
|
||||
"finding": "Supported Hunspell/BPE/Morfessor quote was blocked for the explicitly supplied IEEE reference inside it.",
|
||||
"next_action": "Handle attested quoted reference URLs without allowing invented citations."
|
||||
},
|
||||
{
|
||||
"id": "q0816",
|
||||
"question": "Ktorý záznam v ZP Wiki spája osobu Dávid Omasta s témou „generatívne modely“?",
|
||||
"issues": [
|
||||
"delivery_blocked",
|
||||
"missing_expected_answer_value"
|
||||
],
|
||||
"delivery_status": "error",
|
||||
"accepted_for_delivery": false,
|
||||
"group": "generation_schema_and_quote",
|
||||
"finding": "Initial answer added unsupported study-start year. Repair removed it but still joined separated body passages into a purported contiguous quote, omitting an intervening line.",
|
||||
"next_action": "Select each exact contiguous passage separately and omit unrequested uncertain metadata."
|
||||
},
|
||||
{
|
||||
"id": "q1168",
|
||||
"question": "Porovnaj záznamy osôb Dárius Lindvai a Tetiana Mohorian. Pri osobe Dárius Lindvai použi záznam typu „diplomová práca“ a pri osobe Tetiana Mohorian záznam typu „bakalárska práca“. Uveď názov každej práce a rok, pri ktorom je na ZP Wiki uvedená.",
|
||||
"issues": [
|
||||
"false_no_answer",
|
||||
"missing_expected_answer_value",
|
||||
"missing_expected_binding"
|
||||
],
|
||||
"delivery_status": "no_answer",
|
||||
"accepted_for_delivery": true,
|
||||
"group": "source_convention",
|
||||
"finding": "Comparison needs Darius Lindvai diploma title/year; its year has the same predeclared taxonomy-only ambiguity as q0074.",
|
||||
"next_action": "Resolve source convention together with q0074."
|
||||
}
|
||||
],
|
||||
"live_80_verification": {
|
||||
"consent": "Explicit user consent for all 80 questions and their retrieved wiki evidence to ui.tukekemt.xyz.",
|
||||
"same_predeclared_dataset": true,
|
||||
"unchanged_code_during_run": true,
|
||||
"matching_deployed_python_modules": 125,
|
||||
"single_index_version": "497fadb8f38c4af18d7d9e3b06cdb0a0",
|
||||
"tools_ready_after_run": true,
|
||||
"generation_or_benchmark_tuning_during_run": false,
|
||||
"production_code_changes_for_this_measurement": false
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user