diff --git a/README.md b/README.md index 1c4a883..8281a51 100644 --- a/README.md +++ b/README.md @@ -1160,7 +1160,7 @@ upozornenie na neúplnosť, nezamieňať autora dokumentu s autorom práce a pri sa nevysvetľujú odhadom; verzia dát neoznačuje verziu programu. Tieto pokyny ovplyvňujú správanie Claude, jeho finálny text však náš backend nemôže programovo vynútiť. Po aktualizácii už nahratých Skills nahraď ich -novými ZIPmi z `packages/claude/dist/` (verzia 1.1.0) a over správanie v novom chate. +novými ZIPmi z `packages/claude/dist/` (verzia 1.2.0) a over správanie v novom chate. ### Nahratie Skills do webového Claude @@ -1475,6 +1475,23 @@ staré ploché cesty `data/nazov.json` sa presmerujú do príslušnej oblasti. ### Širšie overenie kontrolovaného RAG +`complete`, `truncated` a `truncation_reasons` zostávajú technickým opisom +vyhľadávania. Overená odpoveď navyše obsahuje `answer_coverage` so scope, status, +warnings a limitations. Status `supported` znamená doložené tvrdenia/výňatky, +nie úplné prehľadanie korpusu. Bežný výber kandidátov a zdrojov preto nepridáva +automatické upozornenie k názvu, roku alebo krátkemu opisu. Obmedzený zoznam +záznamov a orezané citované zadanie dostanú konkrétne upozornenie. Pri nenájdenej +informácii sa zachová opatrné tvrdenie o dostupných podkladoch. Žiadny limit sa +kvôli tomu nezvyšuje ani sa neprepisuje na `complete=true`. + +Otázka na stručný obsah pri zobrazení vyberá najviac dva krátke overené výňatky +na osobu (do 600 znakov a dvoch bodov na osobu), bez voľnej modelovej parafrázy. +Skrátenie robí renderer z už overených citátov; dlhší platný citát nie je dôvod +na odmietnutie faktov ani ďalšie modelové volanie. Pri neoddeliteľnom technickom +reťazci má zachovanie dôkazu prednosť pred limitom stručnosti. Otázka na zadanie +môže zobraziť všetky doložené body zadania. Citácie a kontroly ich doslovnosti +sa uplatňujú v oboch režimoch. + Otázky na obsah, zadanie, ciele alebo metódy práce používajú textové výňatky z príslušnej akademickej sekcie, nie iba metadáta o názve a roku. Výňatok musí patriť požadovanej osobe, typu práce a prípadne roku. Názov, rok ani samotný diff --git a/packages/claude/client.py b/packages/claude/client.py index f334712..bf659af 100644 --- a/packages/claude/client.py +++ b/packages/claude/client.py @@ -163,7 +163,14 @@ def query_backend(query: str, engine: str, *, mode='answer', limit=5, config=Non or not all(isinstance(x, str) for x in value['source_urls']) or not isinstance(value.get('claims'), list)): raise ClientError('invalid_backend_response') - fields |= {'answer', 'source_urls', 'claims', 'count', 'accepted_for_delivery'} + if 'answer_coverage' in value: + coverage = value['answer_coverage'] + if (not isinstance(coverage, dict) or coverage.get('scope') not in ('fact','overview','assignment','list','count') + or coverage.get('status') not in ('supported','partial','unknown','not_found','unavailable') + or any(not isinstance(coverage.get(key), list) or + any(not isinstance(item, str) for item in coverage[key]) for key in ['warnings','limitations'])): + raise ClientError('invalid_backend_response') + fields |= {'answer', 'source_urls', 'claims', 'count', 'accepted_for_delivery', 'answer_coverage'} else: if not isinstance(value.get('sources'), list): raise ClientError('invalid_backend_response') diff --git a/packages/claude/dist/packages.json b/packages/claude/dist/packages.json index a60a0b7..bd17241 100644 --- a/packages/claude/dist/packages.json +++ b/packages/claude/dist/packages.json @@ -3,12 +3,12 @@ { "archive": "zpwiki-rag.zip", "name": "rag", - "sha256": "3382847549d88c6b3c25dc29cc95c9d8ceb8df1b3558680e772374a06b1a04c8" + "sha256": "8f0c51e27dc68a1e3ee845f2cfe5987e51714d4408f98c7d275861dc1e22953f" }, { "archive": "zpwiki-graphrag.zip", "name": "graphrag", - "sha256": "f1090fa4e22fb1cd4ebe3cc1a5c9b0c7d8fb7a57a8057713223007060df7f59a" + "sha256": "aa65a97211f48fa0352a98e50c6ca6a243e1d23105c0ff144c7f360ad192c376" } ], "schema_version": 1 diff --git a/packages/claude/dist/zpwiki-graphrag.zip b/packages/claude/dist/zpwiki-graphrag.zip index 4d1fb91..4dbee2f 100644 Binary files a/packages/claude/dist/zpwiki-graphrag.zip and b/packages/claude/dist/zpwiki-graphrag.zip differ diff --git a/packages/claude/dist/zpwiki-rag.zip b/packages/claude/dist/zpwiki-rag.zip index 447ebd2..67de16a 100644 Binary files a/packages/claude/dist/zpwiki-rag.zip and b/packages/claude/dist/zpwiki-rag.zip differ diff --git a/packages/claude/mcp_server.py b/packages/claude/mcp_server.py index 804d088..6aec4d2 100644 --- a/packages/claude/mcp_server.py +++ b/packages/claude/mcp_server.py @@ -25,6 +25,10 @@ ANSWER_PRESENTATION_POLICY = ( 'document_author means author of the source document; it does not establish thesis authorship, ' 'supervision or a maintainer role. Do not guess those roles. ' 'complete and truncated describe retrieval coverage, not the truth of each returned fact. ' + 'For RAG, answer_coverage separately describes the delivered answer scope. supported means ' + 'evidence-backed, not exhaustive. Do not append a generic incompleteness warning merely ' + 'because retrieval selected top candidates/results. Preserve the specific notices already ' + 'included in answer for limited lists or clipped source text. ' 'An incomplete listing can contain supported facts, but cannot establish uniqueness or absence ' 'of other records. Retain its caveat; do not dismiss it using earlier answers. ' 'For no_answer, return only the supplied absence-of-evidence answer and any supplied citations. ' diff --git a/packages/claude/rag/SKILL.md b/packages/claude/rag/SKILL.md index 8aa5674..c5e541f 100644 --- a/packages/claude/rag/SKILL.md +++ b/packages/claude/rag/SKILL.md @@ -22,13 +22,17 @@ Use this text-search tool rather than GraphRAG when the user explicitly selects never reproduce rejected text or invent a replacement. For `ambiguous`, ask the user to clarify. For `no_answer`, report the absence of evidence without claiming the fact is false. 5. Respect `complete`, `truncated`, `truncation_reasons` and `index_version`. + These describe retrieval, not whether the selected answer is supported. `answer_coverage` + separately describes the answer scope: `supported` means evidence-backed, never exhaustive. + Present the backend's specific notices unchanged. Do not append a generic incompleteness + warning to a supported fact or short description just because retrieval selected top results. Never describe an incomplete or unknown-completeness result as an exhaustive list. Do not combine different index versions as if they represented one snapshot. Presentation boundaries: - Use the current tool response for the current question. Do not append factual material or reassurances from previous chat answers. -- An incomplete listing can contain supported facts. Preserve its caveat without implying +- An incomplete listing can contain supported facts. Preserve any supplied caveat without implying the returned title/year is wrong or that it is the only matching record. Earlier answers cannot waive the current response's coverage limits. - `document_author` identifies the source document's author. Do not infer thesis authorship, diff --git a/scripts/build_claude_packages.py b/scripts/build_claude_packages.py index feae622..8744b12 100644 --- a/scripts/build_claude_packages.py +++ b/scripts/build_claude_packages.py @@ -11,7 +11,7 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] BASE = ROOT / 'packages/claude' ENGINES = ('rag', 'graphrag') -PACKAGE_VERSION = '1.1.0' +PACKAGE_VERSION = '1.2.0' def json_bytes(value): diff --git a/test/test_claude_packages.py b/test/test_claude_packages.py index 32403bb..fabcf16 100644 --- a/test/test_claude_packages.py +++ b/test/test_claude_packages.py @@ -132,6 +132,25 @@ def test_presentation_instructions_never_escape_and_evidence_is_preserved(backen assert backend['checked'] == original +def test_answer_coverage_is_forwarded_without_overwriting_retrieval_limits(backend): + coverage = {'scope': 'fact', 'status': 'supported', 'warnings': [], 'limitations': ['result_limit']} + backend['checked']['answer_coverage'] = coverage + result = client.query_backend('Názov práce?', 'rag') + assert result['answer_coverage'] == coverage + assert result['complete'] is False and result['truncated'] is True + assert 'presentation_instructions' not in result + + +@pytest.mark.parametrize('coverage', [None, [], {'status': 'complete'}, + {'scope': 'fact', 'status': 'supported', 'warnings': 'none', 'limitations': []}, + {'scope': [], 'status': 'supported', 'warnings': [], 'limitations': []}, + {'scope': 'fact', 'status': {}, 'warnings': [], 'limitations': []}]) +def test_invalid_answer_coverage_is_not_delivered(backend, coverage): + backend['checked']['answer_coverage'] = coverage + result = client.query_backend('Názov práce?', 'rag') + assert result['reason'] == 'invalid_backend_response' and not result['accepted_for_delivery'] + + @pytest.mark.parametrize('code,reason', [(401, 'unauthorized'), (403, 'forbidden'), (503, 'backend_unavailable'), (302, 'redirect_not_allowed')]) def test_transport_errors_and_redirects_do_not_disclose_payload(backend, code, reason): diff --git a/test/test_rag_answer_coverage.py b/test/test_rag_answer_coverage.py new file mode 100644 index 0000000..219d181 --- /dev/null +++ b/test/test_rag_answer_coverage.py @@ -0,0 +1,198 @@ +"""Technical retrieval limits must not become false claims about answer quality.""" +import copy +import json + +import pytest + +from tools.answer_evaluation.answer_coverage import answer_coverage +from tools.answer_evaluation.rag_evidence import rag_evidence_context +from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues, render_rag_answer +from tools.rag.rag_utils import build_context_text + + +URL = 'https://example.test/eva' +GOALS = '1. Preskúmajte metódy vyhľadávania.\n2. Implementujte vyhľadávač.\n3. Vyhodnoťte presnosť.\n4. Navrhnite zlepšenia.' + + +@pytest.fixture +def payload(): + sources = [{'source_id': 'S1', 'title': 'Eva Nová', 'document_path': 'pages/students/2020/eva_nova/README.md', + 'source_url': URL, 'text': 'Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\nZadanie:\n' + GOALS}] + return {'status': 'ok', 'reason': None, 'complete': False, 'truncated': True, + 'truncation_reasons': ['candidate_limit', 'per_document_limit', 'result_limit'], + 'index_version': {'version': 'fixture'}, 'sources': sources, 'context': build_context_text(sources)} + + +def answer(): + return {'status': 'ok', 'answer': '', 'claims': [{'person': 'Eva Nová', 'work_type': 'diploma_thesis', + 'title': 'Vyhľadávanie informácií', 'year': 2026}], 'evidence_quotes': [], 'source_urls': [URL]} + + +def quote(text): + return {'status': 'ok', 'answer': '', 'claims': [], 'source_urls': [URL], + 'evidence_quotes': [{'source_id': 'S1', 'quote': text}]} + + +@pytest.mark.parametrize('complete,truncated,reasons', [ + (False, True, ['candidate_limit', 'per_document_limit', 'result_limit']), + (None, False, []), (False, True, ['llm_context_budget'])]) +def test_supported_fact_is_not_an_incomplete_answer(payload, complete, truncated, reasons): + payload.update(complete=complete, truncated=truncated, truncation_reasons=reasons) + c = rag_evidence_context('Aký je názov a rok diplomovej práce Evy Novej?', payload) + before = copy.deepcopy(c) + coverage = answer_coverage(answer(), c) + assert coverage['status'] == 'supported' and coverage['warnings'] == [] + text = render_rag_answer(answer(), c) + assert '2026' in text and URL in text and 'neúpln' not in text + assert c == before # Never fake complete=true or remove diagnostic limits. + + +@pytest.mark.parametrize('question', ['Vypíš všetky práce Evy Novej.', 'Aké práce má Eva Nová?', + 'Uveď kompletný zoznam prác Evy Novej.', + 'Vypíš všetky bakalárske a diplomové práce Evy Novej.']) +def test_limited_record_list_still_warns(payload, question): + c = rag_evidence_context(question, payload) + coverage = answer_coverage(answer(), c) + assert coverage['scope'] == 'list' and coverage['status'] == 'partial' + assert 'Uvedený zoznam nemusí obsahovať všetky záznamy.' in render_rag_answer(answer(), c) + + +def test_explicitly_complete_retrieval_does_not_add_a_list_warning(payload): + payload.update(complete=True, truncated=False, truncation_reasons=[]) + c = rag_evidence_context('Vypíš všetky práce Evy Novej.', payload) + assert answer_coverage(answer(), c)['warnings'] == [] + + +def test_quoted_title_is_not_an_exhaustive_list_request(payload): + c = rag_evidence_context('Aký je rok diplomovej práce „Všetky práce v katalógu“?', payload) + assert answer_coverage(answer(), c)['scope'] == 'fact' + + +def test_unknown_list_coverage_is_not_silently_exhaustive(payload): + payload.update(complete=None, truncated=False, truncation_reasons=[]) + c = rag_evidence_context('Aké práce má Eva Nová?', payload) + assert answer_coverage(answer(), c)['warnings'] == ['list_may_be_incomplete'] + + +def test_legacy_context_without_limit_metadata_remains_usable(payload): + payload.pop('truncation_reasons'); payload.pop('complete'); payload.pop('truncated') + c = rag_evidence_context('Aký je názov diplomovej práce Evy Novej?', payload) + assert answer_coverage(answer(), c)['warnings'] == [] + assert 'Vyhľadávanie informácií' in render_rag_answer(answer(), c) + + +def test_regular_assignment_can_show_all_supplied_goals_without_global_warning(payload): + c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload) + a = quote(GOALS) + assert rag_answer_issues(a, c) == [] + text = render_rag_answer(a, c) + assert GOALS in text and 'neúpln' not in text and 'Zo zdroja' not in text + assert answer_coverage(a, c)['scope'] == 'assignment' + + +def test_clipped_assignment_gets_specific_warning(payload): + payload['sources'][0]['text'] += '\n[OBSAH ZDROJA OREZANÝ]' + payload['context'] = build_context_text(payload['sources']) + c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload) + a = quote('1. Preskúmajte metódy vyhľadávania.') + coverage = answer_coverage(a, c) + assert coverage['status'] == 'partial' and coverage['warnings'] == ['assignment_text_clipped'] + assert 'uvedené body nemusia tvoriť celé zadanie' in render_rag_answer(a, c) + + +def test_request_for_all_assignment_items_preserves_uncertainty(payload): + c = rag_evidence_context('Uveď všetky body zadania diplomovej práce Evy Novej.', payload) + assert answer_coverage(quote(GOALS), c)['warnings'] == ['assignment_completeness_unknown'] + + +@pytest.mark.parametrize('budget', [False, True]) +def test_no_answer_is_bounded_and_only_reports_actual_context_cut(payload, budget): + payload['truncation_reasons'] = ['llm_context_budget'] if budget else ['result_limit'] + c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej v roku 1999?', payload) + a = {'status': 'no_answer', 'claims': [], 'evidence_quotes': [], 'source_urls': []} + text = render_rag_answer(a, c) + assert 'nepodarilo spoľahlivo nájsť' in text and 'Výpis je neúplný' not in text + assert ('Časť podkladov bola pri príprave kontextu skrátená.' in text) == budget + assert answer_coverage(a, c)['status'] == 'not_found' + + +def test_irrelevant_dropped_source_does_not_make_cited_assignment_partial(payload): + payload['sources'].append({'source_id': 'S2', 'title': 'Iný dokument', 'source_url': 'https://example.test/other', + 'document_path': 'pages/topics/other/README.md', 'text': 'Iný obsah\n[OBSAH ZDROJA OREZANÝ]'}) + payload['context'] = build_context_text(payload['sources']) + payload['truncation_reasons'].append('llm_context_budget') + c = rag_evidence_context('Aké je zadanie diplomovej práce Evy Novej?', payload) + assert answer_coverage(quote(GOALS), c)['warnings'] == [] + + +def test_exact_aggregate_can_be_supported_despite_limited_evidence(payload): + c = rag_evidence_context('Koľko prác je evidovaných?', payload) + c['aggregation'] = {'count': 2, 'complete': True} + a = {'status': 'ok', 'claims': [], 'evidence_quotes': [], 'source_urls': [], 'count': 2} + assert answer_coverage(a, c)['status'] == 'supported' + c['aggregation']['complete'] = False + assert 'count_not_established' in answer_coverage(a, c)['warnings'] + + +def test_long_valid_overview_is_shortened_without_another_model_request(payload): + question = 'Čomu sa venuje diplomová práca Evy Novej?' + short = '1. Preskúmajte metódy vyhľadávania.\n2. Implementujte vyhľadávač.' + c = rag_evidence_context(question, payload) + assert rag_answer_issues(quote(GOALS), c) == [] + calls = [] + def complete(messages): + calls.append(messages) + return {'choices': [{'message': {'content': json.dumps(quote(GOALS))}}]} + result = generate_verified_rag(question, payload, complete) + assert result['accepted_for_delivery'] and len(result['attempts']) == len(calls) == 1 + assert short in result['answer'] and '3.' not in result['answer'] + assert result['attempts'][0]['parsed_answer']['evidence_quotes'][0]['quote'] == GOALS + assert result['complete'] is False and result['truncation_reasons'] == payload['truncation_reasons'] + assert result['answer_coverage']['status'] == 'supported' and result['answer_coverage']['warnings'] == [] + + +def test_short_comparison_can_cover_three_people_instead_of_dropping_one(payload): + sources = [] + quotes = [] + for index, (name, slug) in enumerate([('Eva Nová', 'eva_nova'), ('Boris Nový', 'boris_novy'), + ('Cyril Malý', 'cyril_maly')], 1): + source = {**payload['sources'][0], 'source_id': f'S{index}', 'title': name, + 'source_url': f'https://example.test/{slug}', + 'document_path': f'pages/students/2020/{slug}/README.md'} + sources.append(source) + quotes.append({'source_id': source['source_id'], 'quote': '2. Implementujte vyhľadávač.'}) + payload['sources'] = sources; payload['context'] = build_context_text(sources) + c = rag_evidence_context('Porovnaj obsah diplomových prác osôb Eva Nová, Boris Nový a Cyril Malý.', payload) + assert len(c['requirements']['people']) == 3 + a = {'status': 'ok', 'claims': [], 'evidence_quotes': quotes, 'source_urls': [s['source_url'] for s in sources]} + assert rag_answer_issues(a, c) == [] + a['evidence_quotes'].pop() + assert 'missing_bound_content_evidence' in rag_answer_issues(a, c) + + +def test_overview_renderer_only_selects_literal_units_and_preserves_full_evidence(payload): + from tools.answer_evaluation.verified_rag import overview_excerpts + extra = 'Stav:\n\n- Implementované indexovanie dokumentov.' + payload['sources'][0]['text'] += '\n' + extra + payload['context'] = build_context_text(payload['sources']) + c = rag_evidence_context('Čomu sa venuje diplomová práca Evy Novej?', payload) + a = quote(GOALS) + a['evidence_quotes'].append({'source_id': 'S1', 'quote': extra}) + before = copy.deepcopy(a) + assert rag_answer_issues(a, c) == [] + units = overview_excerpts(a, c) + assert len(units) == 2 and sum(map(len, units)) <= 600 + assert all(any(unit in evidence['quote'] for evidence in a['evidence_quotes']) for unit in units) + assert a == before + + +def test_long_prose_excerpt_is_selected_without_inventing_a_summary(payload): + from tools.answer_evaluation.verified_rag import overview_excerpts + prose = 'Práca skúma vyhľadávanie v slovenských dokumentoch. ' * 18 + payload['sources'][0]['text'] += '\n' + prose + payload['context'] = build_context_text(payload['sources']) + c = rag_evidence_context('Čomu sa venuje diplomová práca Evy Novej?', payload) + a = quote(prose) + assert rag_answer_issues(a, c) == [] + units = overview_excerpts(a, c) + assert units and sum(map(len, units)) <= 600 and all(unit in prose for unit in units) diff --git a/test/test_rag_work_content.py b/test/test_rag_work_content.py index 8318d57..6fe7880 100644 --- a/test/test_rag_work_content.py +++ b/test/test_rag_work_content.py @@ -153,7 +153,8 @@ def test_correction_sends_real_description_but_no_unrelated_sections(payload): text = json.dumps(sent['content_passages'], ensure_ascii=False) assert 'ohodnotenia dokumentov' in text and 'Cassandra' not in text and 'medicínskych' not in text assert 'missing_content_evidence' in result['attempts'][0]['issues'] - assert ASSIGNMENT in result['answer'] and 'Výpis je neúplný.' in result['answer'] + assert ASSIGNMENT in result['answer'] and 'Výpis je neúplný.' not in result['answer'] + assert result['complete'] is False and result['answer_coverage']['status'] == 'supported' assert 'expected_bindings' not in json.dumps(calls) diff --git a/test/test_verified_rag.py b/test/test_verified_rag.py index f4447cf..0733eb1 100644 --- a/test/test_verified_rag.py +++ b/test/test_verified_rag.py @@ -104,7 +104,8 @@ def test_bad_answer_is_repaired_once_and_only_verified_fields_are_rendered(paylo assert result['accepted_for_delivery'] and len(calls) == 2 assert 'Ondrej Megela — bakalárska práca' in result['answer'] assert 'rok práce: 2020' in result['answer'] and '2099' not in result['answer'] - assert 'Výpis je neúplný.' in result['answer'] + assert 'Výpis je neúplný.' not in result['answer'] + assert result['complete'] is False and result['answer_coverage']['status'] == 'supported' assert 'unsupported_person_work_type_year_binding' in result['attempts'][0]['issues'] diff --git a/tools/answer_evaluation/answer_coverage.py b/tools/answer_evaluation/answer_coverage.py new file mode 100644 index 0000000..8e98def --- /dev/null +++ b/tools/answer_evaluation/answer_coverage.py @@ -0,0 +1,77 @@ +"""Separate supported answer scope from the search's technical selection limits. + +Supported means the delivered facts/excerpts have evidence; it never promises +exhaustive retrieval, a complete source document or absence of other records. +""" +import re + +from .context_policy import normalized + + +def response_scope(question: str, *, content: bool = False) -> str: + query = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question)) + # Record lists differ from the numbered assignment of one selected work. + records = r'(?:prac\w*|projekt\w*|student\w*|osob\w*|autor\w*|tag\w*|kategori\w*|zdroj\w*)' + modifiers = r'(?:(?:vsetk\w*|diplomov\w*|bakalarsk\w*|zaverecn\w*|timov\w*|vyskumn\w*|a|aj)\s+)*' + if re.search(r'\b(?:kolko|pocet|spocitaj)\b', query): + return 'count' + if re.search(rf'\b(?:vsetk\w*|upln\w*|kompletn\w*)\s+{modifiers}{records}\b' + rf'|\b(?:ake|ktore|vypis|zoznam|prehlad)\s+{modifiers}{records}\b', query): + return 'list' + if content: + if re.search(r'\b(?:zadani\w*|ciel\w*)\b', query) and not re.search(r'\b(?:strucn\w*|zhrn\w*)\b', query): + return 'assignment' + if re.search(r'\b(?:comu|o com|opis\w*|popis\w*|obsah\w*|zhrn\w*|strucn\w*)\b', query): + return 'overview' + return 'fact' + + +def answer_coverage(answer: dict, context: dict) -> dict: + """Describe only the scope of an already validated RAG selection.""" + requirements = context.get('requirements', {}) + scope = response_scope(context.get('question', ''), content=requirements.get('content', False)) + reasons = list(context.get('truncation_reasons') or []) + selected = {row.get('source_id') for row in answer.get('evidence_quotes', [])} + cited = set(answer.get('source_urls', [])) + clipped = any('[OBSAH ZDROJA OREZANÝ]' in row.get('body', '') + for row in context.get('source_blocks', []) + if row.get('source_id') in selected or (not selected and + (row.get('source_url') in cited or answer.get('status') == 'no_answer'))) + # A context budget may drop only unrelated sources. A clipping marker in + # the actual cited block is stronger evidence than this global reason. + context_limited = 'llm_context_budget' in reasons + limited = context.get('complete') is not True or context.get('truncated') is True + limitations = [*reasons] + if clipped: + limitations.append('cited_source_text_clipped') + warnings = [] + status = answer.get('status') + if status == 'ok': + coverage = 'supported' + if scope == 'list' and (limited or clipped): + coverage = 'partial'; warnings.append('list_may_be_incomplete') + elif scope == 'count' and (context.get('aggregation') or {}).get('complete') is not True: + coverage = 'unknown'; warnings.append('count_not_established') + elif scope == 'assignment' and clipped: + coverage = 'partial'; warnings.append('assignment_text_clipped') + elif scope == 'assignment' and limited and re.search(r'\b(?:vsetk\w*|upln\w*|kompletn\w*|cele)\b', normalized(context.get('question', ''))): + coverage = 'partial'; warnings.append('assignment_completeness_unknown') + elif status == 'no_answer': + coverage = 'not_found' + if clipped or context_limited: + warnings.append('missing_information_with_clipped_context') + else: + coverage = 'unavailable' + return {'scope': scope, 'status': coverage, 'warnings': warnings, + 'limitations': list(dict.fromkeys(limitations))} + + +def coverage_notices(coverage: dict) -> list[str]: + messages = { + 'list_may_be_incomplete': 'Uvedený zoznam nemusí obsahovať všetky záznamy.', + 'assignment_text_clipped': 'Časť podkladov k zadaniu bola skrátená; uvedené body nemusia tvoriť celé zadanie.', + 'assignment_completeness_unknown': 'Z dostupných podkladov nemožno zaručiť všetky body zadania.', + 'missing_information_with_clipped_context': 'Časť podkladov bola pri príprave kontextu skrátená.', + 'count_not_established': 'Celkový počet sa z dostupných podkladov nedá spoľahlivo určiť.', + } + return [messages[reason] for reason in coverage['warnings']] diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index fbeee5d..4db2e63 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -313,6 +313,8 @@ def rag_evidence_context(question: str, payload: dict) -> dict: 'start_year': start_year, 'response_mode': 'claims' if academic or start_year else 'quotes'} if content: + from .answer_coverage import response_scope + requirements['content_style'] = response_scope(question, content=True) requirements['response_mode'] = 'mixed' if any(requirements[key] for key in ['title', 'year', 'start_year']) else 'quotes' stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else [] requirements['years'] = ([int(value) for value in stated_years if len(value) == 4] diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index f3af2d7..6dde0b7 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field from .answer_format import Claim from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized, text_urls from .rag_evidence import WORK_LABELS, descriptive_text, rag_evidence_context +from .answer_coverage import answer_coverage, coverage_notices class RagClaim(Claim): @@ -50,6 +51,10 @@ Vyber opisujúce evidence_quotes z content_passages zodpovedajúcej osoby, typu Samotný názov, rok, meno vedúceho ani nadpis zadania nenahrádzajú opis. Pri response_mode=mixed uveď požadované claims aj opisujúce evidence_quotes. Použi iba dodané content_passages; opis inej práce alebo iného roku nie je odpoveď. +Pri requirements.content_style=overview vyber najviac DVA krátke doslovné výňatky +na požadovanú osobu, najviac 600 znakov a dve odrážky/body na osobu. Odpovedaj stručne na obsah; +neprepisuj celé zadanie a históriu stretnutí. Pri content_style=assignment zachovaj +požadované body zadania; tento stručný limit sa na zadanie nevzťahuje. Každý výňatok musí byť JEDEN súvislý úsek z konkrétneho zdroja vrátane Markdownu. Nespájaj oddelené pasáže cez vynechané riadky; použi viac samostatných kratších quotes. Vyber iba minimum potrebné na zodpovedanie otázky. Výňatok nesmieš parafrázovať ani meniť čísla. @@ -218,6 +223,49 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: return list(dict.fromkeys(issues)) +def overview_excerpts(answer: dict, context: dict) -> list[str]: + """Shorten validated quotes by selection, never by factual paraphrase. + + Presentation length is not a reason to reject otherwise supported evidence + or to spend another model request rewriting the same facts. + """ + selected = []; counts = {}; sizes = {}; seen = set() + for quote in answer.get('evidence_quotes', []): + sid = quote['source_id'] + owner = next((p.get('person') for p in context.get('content_passages', []) + if p['source_id'] == sid and p.get('person')), sid) + identities = [row.get(key) for row in context.get('facts', []) if row.get('source_id') == sid + for key in ('person', 'title', 'document_author')] + blocks = re.split(r'\n\s*\n|(?=^[ \t]*(?:\d+[.)]|[-+*])\s+)', quote['quote'], flags=re.M) + for block in blocks: + if counts.get(owner, 0) >= 2: + break + block = block.strip() + if not descriptive_text(block, identities): + continue + # Prefer complete sentences when a prose paragraph is very long. + units = re.split(r'(?<=[.!?])\s+(?=[A-ZÁČĎÉÍĽŇÓŠŤÚÝŽ])', block) if len(block) > 600 else [block] + for unit in units: + remaining = 600 - sizes.get(owner, 0) + if counts.get(owner, 0) >= 2 or remaining < 30: + break + unit = unit.strip() + if not descriptive_text(unit, identities) or (owner, unit) in seen: + continue + if len(unit) > remaining: + # An explicit ellipsis makes a shortened literal prefix + # visible; never cut a word/URL and pretend it is complete. + prefix = unit[:remaining - 2].rsplit(' ', 1)[0].rstrip() + if not prefix or ' ' not in prefix or not descriptive_text(prefix, identities): + continue + unit = prefix + ' …' + selected.append(unit); seen.add((owner, unit)) + counts[owner] = counts.get(owner, 0) + 1 + sizes[owner] = sizes.get(owner, 0) + len(unit) + # An indivisible technical string is better retained than silently omitted. + return selected or [q['quote'] for q in answer.get('evidence_quotes', [])] + + def render_rag_answer(answer: dict, context: dict) -> str: lines = [] if answer['status'] == 'ok': @@ -227,9 +275,18 @@ def render_rag_answer(answer: dict, context: dict) -> str: values = [WORK_LABELS[claim['work_type']]] if claim.get('work_type') else [] values += [f'{label}: {claim[key]}' for key, label in labels.items() if claim.get(key) is not None] lines.append(claim['person'] + ' — ' + '; '.join(values) + '.') - sources = {row['source_id']: row for row in context['source_blocks']} - lines += ['Zo zdroja ' + str(sources[quote['source_id']].get('title') or quote['source_id']) + - ': „' + quote['quote'] + '“' for quote in answer.get('evidence_quotes', [])] + quotes = answer.get('evidence_quotes', []) + style = context.get('requirements', {}).get('content_style') + if quotes and style in {'overview', 'assignment'}: + if style == 'overview': + lines.append('Stručne z dostupných podkladov:') + lines += overview_excerpts(answer, context) + else: + lines += [quote['quote'] for quote in quotes] + else: + sources = {row['source_id']: row for row in context['source_blocks']} + lines += ['Zo zdroja ' + str(sources[quote['source_id']].get('title') or quote['source_id']) + + ': „' + quote['quote'] + '“' for quote in quotes] if answer.get('count') is not None: lines.append(f'Počet: {answer["count"]}.') elif answer['status'] == 'no_answer': @@ -238,10 +295,7 @@ def render_rag_answer(answer: dict, context: dict) -> str: lines.append('Údaje sú nejednoznačné; požiadavku treba spresniť.') else: lines.append('Overenú odpoveď sa nepodarilo pripraviť.') - if context.get('truncated') or context.get('complete') is False: - lines.append('Výpis je neúplný.') - elif context.get('complete') is None: - lines.append('Úplnosť vyhľadania nemožno zaručiť.') + lines.extend(coverage_notices(answer_coverage(answer, context))) if answer.get('source_urls'): lines.append('Zdroje: ' + ', '.join(answer['source_urls'])) return '\n'.join(lines) @@ -254,6 +308,7 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: empty = {'status': status, 'claims': [], 'evidence_quotes': [], 'source_urls': []} return {'status': status, 'reason': payload.get('reason'), 'accepted_for_delivery': status != 'error', 'answer': render_rag_answer(empty, context), 'source_urls': [], 'claims': [], + 'answer_coverage': answer_coverage(empty, context), 'attempts': [], 'index_version': context['index_version'], 'complete': context['complete'], 'truncated': context['truncated'], 'truncation_reasons': context['truncation_reasons']} sent = {key: context[key] for key in ['status', 'reason', 'complete', 'truncated', @@ -287,6 +342,7 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: text = render_rag_answer(answer, context) return {'status': answer['status'], 'reason': None, 'accepted_for_delivery': True, 'answer': text, 'source_urls': answer['source_urls'], 'claims': answer['claims'], + 'answer_coverage': answer_coverage(answer, context), 'attempts': attempts, 'index_version': context['index_version'], 'complete': context['complete'], 'truncated': context['truncated'], 'truncation_reasons': context['truncation_reasons']} @@ -299,7 +355,9 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: 'Nepotrebné citáty a metadáta vynechaj.' if context['requirements']['response_mode'] == 'quotes' else '') + (' Opis práce musí obsahovať doslovné evidence_quotes z content_passages. ' 'Samotný názov a rok nestačia; pri mixed zachovaj aj požadované claims.' - if context['requirements'].get('content') else '')}] + if context['requirements'].get('content') else '') + + (' Pri overview vyber najviac dva krátke výňatky, najviac 600 znakov a dva body na požadovanú osobu.' + if context['requirements'].get('content_style') == 'overview' else '')}] return {'status': 'error', 'reason': 'answer_rejected', 'accepted_for_delivery': False, 'answer': 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.', 'attempts': attempts, 'index_version': context['index_version']} diff --git a/tools/rag/rag_utils.py b/tools/rag/rag_utils.py index bc96356..929a110 100644 --- a/tools/rag/rag_utils.py +++ b/tools/rag/rag_utils.py @@ -29,9 +29,11 @@ NO_ANSWER_TEXT = ( RAG_INSTRUCTIONS = [ - 'Rešpektuj complete, truncated, truncation_reasons a index_version. Pri truncated=true alebo ' - 'complete=false v odpovedi jasne uveď, že výpis je neúplný. Pri complete=null nemožno zaručiť ' - 'úplnosť; neoznačuj výber za všetky práce alebo úplný zoznam. Cituj iba URL zo sources.', + 'Rešpektuj complete, truncated, truncation_reasons a index_version ako rozsah vyhľadania. ' + 'Výber najlepších kandidátov a zdrojov sám osebe nezneplatňuje doložený fakt ani opis. ' + 'Pri požiadavke na zoznam upozorni, ak nemožno zaručiť všetky záznamy. Pri orezanom ' + 'texte a chýbajúcej informácii pomenuj konkrétne obmedzenie. Nikdy neoznačuj neúplný ' + 'alebo neistý výber za úplný zoznam. Cituj iba URL zo sources.', ( "Odpovedaj výhradne podľa informácií vrátených " "v poli context a sources zo ZP Wiki."