From e0f5713da4b5a1474d4874055b76ee8b96cf267b Mon Sep 17 00:00:00 2001 From: jp170na Date: Tue, 3 Nov 2026 02:02:05 +0100 Subject: [PATCH] =?UTF-8?q?Prirodzenej=C5=A1ie=20odpovede?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 + test/test_answer_presentation.py | 120 +++++++++++++++ test/test_graph_work_listing.py | 3 +- test/test_rag_answer_coverage.py | 6 +- test/test_rag_work_content.py | 5 +- test/test_verified_rag.py | 6 +- tools/answer_evaluation/context_policy.py | 11 ++ tools/answer_evaluation/graph_context.py | 3 + tools/answer_evaluation/presentation.py | 145 +++++++++++++++++++ tools/answer_evaluation/render_answer.py | 43 ++++-- tools/answer_evaluation/verified_rag.py | 52 +++++-- tools/graphrag/evaluation/reproducibility.py | 2 + 12 files changed, 369 insertions(+), 35 deletions(-) create mode 100644 test/test_answer_presentation.py create mode 100644 tools/answer_evaluation/presentation.py diff --git a/README.md b/README.md index 8281a51..bccc360 100644 --- a/README.md +++ b/README.md @@ -1492,6 +1492,14 @@ reťazci má zachovanie dôkazu prednosť pred limitom stručnosti. Otázka na z môže zobraziť všetky doložené body zadania. Citácie a kontroly ich doslovnosti sa uplatňujú v oboch režimoch. +Finálne formulovanie je oddelené v `tools/answer_evaluation/presentation.py`. +Overené názvy a roky sa zobrazujú vo vetách, práce v číslovanom zozname. +Rok práce sa nevydáva za dátum jej začatia alebo dokončenia. Pri opise sa +známe rozkazovacie formulácie zadania menia na pomenovania cieľov, napríklad +„Vypracujte prehľad“ na „Vypracovanie prehľadu“. Ide o obmedzené gramatické +pravidlá po kontrole doslovného dôkazu, nie o voľné dopĺňanie faktov modelom. +Neznámy opis, podmienky, negácie a údaje o dosiahnutom stave zostávajú zachované. + Otázky na obsah, zadanie, ciele alebo metódy práce používajú textové výňatky z príslušnej akademickej sekcie, nie iba metadáta o názve a roku. Výňatok musí patriť požadovanej osobe, typu práce a prípadne roku. Názov, rok ani samotný diff --git a/test/test_answer_presentation.py b/test/test_answer_presentation.py new file mode 100644 index 0000000..3a97ee3 --- /dev/null +++ b/test/test_answer_presentation.py @@ -0,0 +1,120 @@ +"""Natural presentation must preserve evidence, modality, binding and guards.""" +import copy + +import pytest + +from tools.answer_evaluation.context_policy import answer_context_issues, prose_issues +from tools.answer_evaluation.graph_context import graph_context +from tools.answer_evaluation.presentation import content_lines, fact_sentences, goal_text, owner_in_question +from tools.answer_evaluation.render_answer import render_graph_answer +from tools.answer_evaluation.verified_rag import generate_verified_rag +from test_graph_work_listing import retrieved, listing_answer +from test_rag_work_content import payload, description, completion, ASSIGNMENT, QUESTION, URL + + +def test_identity_is_a_sentence_but_year_is_not_a_start_or_completion_date(): + claim = {'person': 'Ján Holp', 'work_type': 'diploma_thesis', + 'title': 'Systém získavania informácií.', 'year': 2021} + before = copy.deepcopy(claim) + text = fact_sentences(claim, 'Aký je názov diplomovej práce Jána Holpa a z ktorého roku je?') + assert text == ('Diplomová práca Jána Holpa má názov **Systém získavania informácií**. ' + 'Rok práce je **2021**.') + assert not any(word in text for word in ('začal', 'dokonč', 'vypracovaná')) + assert claim == before + + +@pytest.mark.parametrize('person,question,expected', [ + ('Ján Holp', 'Čomu sa venuje diplomová práca Jána Holpa?', 'Jána Holpa'), + ('Eva Nová', 'Aké je zadanie diplomovej práce Evy Novej?', None), + ('Ján Holp', 'Aké práce má Ján Holp?', None), + ('Ján Holp', 'Aké práce má Martin Jančura?', None), + ('Ján Holp', 'Aké je zadanie práce Jána Holpáka?', None), +]) +def test_display_name_is_bound_to_the_whole_name(person, question, expected): + assert owner_in_question(person, question) == expected + + +def test_a_name_without_a_verified_genitive_remains_in_its_original_form(): + text = fact_sentences({'person': 'Alex Kim', 'work_type': 'diploma_thesis', + 'title': 'Vyhľadávanie', 'year': 2020}) + assert text == 'Alex Kim má diplomovú prácu s názvom **Vyhľadávanie** (rok práce: **2020**).' + + +def test_content_changes_only_grammar_after_literal_evidence_passes(payload): + before = copy.deepcopy(payload) + result = generate_verified_rag(QUESTION, payload, lambda _: completion(description())) + assert result['accepted_for_delivery'] + assert 'Diplomová práca Jána Holpa má podľa dostupného zadania tieto ciele:' in result['answer'] + assert '1. Vypracovanie prehľadu metód ohodnotenia dokumentov' in result['answer'] + assert '2. Implementácia systému pre získavanie informácií v slovenskom jazyku.' in result['answer'] + assert 'Vypracujte' not in result['answer'] and 'Implementujte' not in result['answer'] + assert result['attempts'][0]['parsed_answer']['evidence_quotes'][0]['quote'] == ASSIGNMENT + assert result['source_urls'] == [URL] and result['complete'] is False + assert 'neúplný' not in result['answer'] and payload == before + + +@pytest.mark.parametrize('original,expected', [ + ('1. Vypracujte prehľad metód iba pre slovenské texty.', + 'Vypracovanie prehľadu metód iba pre slovenské texty.'), + ('2. Implementujte jednoduchý systém, v ktorom využijete ohodnotenie dokumentov.', + 'Implementácia jednoduchého systému, v ktorom sa má využiť ohodnotenie dokumentov.'), + ('3. Navrhnite a vypracujte experimenty, v ktorých vyhodnotíte vybrané metódy.', + 'Návrh a vypracovanie experimentov s cieľom vyhodnotiť vybrané metódy.'), + ('4. Navrhnite možné zlepšenia presnosti vyhľadávania.', + 'Návrh možných zlepšení presnosti vyhľadávania.'), + ('Navrhnite a implementujte API bez zmeny databázy.', + 'Navrhnúť a implementovať API bez zmeny databázy.'), +]) +def test_goals_keep_constraints_and_do_not_claim_completion(original, expected): + assert goal_text(original) == (expected, True) + + +@pytest.mark.parametrize('text', [ + 'Implementovaný PageRank a indexovanie webových stránok.', + 'Práca neimplementuje neurónové siete.', + 'Ak bude dostupný korpus, otestuje sa presnosť.', + 'Neimplementujte externé API.', +]) +def test_progress_negations_and_unknown_prose_are_not_reinterpreted(text): + assert goal_text(text) == (text, False) + rendered = '\n'.join(content_lines([text], 'Ján Holp', 'diploma_thesis')) + assert text in rendered and 'tieto ciele' not in rendered + + +def test_assignment_heading_does_not_keep_imperative_style(): + lines = content_lines(['*Zadanie diplomovej práce*:', '1. Vypracujte prehľad metód.', + '2. Implementujte systém.'], 'Ján Holp', 'diploma_thesis') + assert lines[-2:] == ['1. Vypracovanie prehľadu metód.', '2. Implementácia systému.'] + + +def test_graph_list_keeps_all_records_and_missing_values_without_technical_labels(retrieved): + context = graph_context(retrieved); answer = listing_answer(context) + before = copy.deepcopy(answer) + rendered = render_graph_answer(answer, context) + assert answer_context_issues(rendered, context) == [] + text = rendered['answer'] + assert text.startswith('Ján Holp má v dostupných podkladoch tieto práce a projekty:') + assert all(f'\n{i}. **' in text for i in range(1, 6)) + assert text.count('názov v zdroji nie je uvedený') == 3 + assert 'Diplomový projekt 2 2020' in text + assert 'typ práce:' not in text and 'záznam:' not in text + assert answer == before and rendered['claims'] == answer['claims'] + + +def test_ordinal_numbers_do_not_license_fabricated_years_or_totals(retrieved): + c = graph_context(retrieved); a = render_graph_answer(listing_answer(c), c) + assert prose_issues(a) == [] + for suffix in ['\nPráca bola dokončená v roku 2099.', '\nPočet: 98765.', '\n2099. Ďalšia práca.']: + assert 'unsupported_prose_number' in prose_issues({**a, 'answer': a['answer'] + suffix}) + a['count'] = 98765 + assert 'unsupported_total_count' in answer_context_issues(a, c) + + +def test_negative_is_a_bounded_absence_not_a_claim_about_real_existence(): + c = {'status': 'no_answer', 'requested_people': ['Ján Holp'], 'facts': [], 'sources': [], + 'complete': True, 'question': 'Akú prácu mal Ján Holp v roku 1999?'} + a = {'status': 'no_answer', 'claims': [], 'source_urls': []} + text = render_graph_answer(a, c)['answer'] + assert 'Ján Holp' in text and 'zodpovedajúce tejto otázke' in text + assert 'neexistuje' not in text and '2021' not in text + assert answer_context_issues(render_graph_answer(a, c), c) == [] diff --git a/test/test_graph_work_listing.py b/test/test_graph_work_listing.py index 627db76..fe5df84 100644 --- a/test/test_graph_work_listing.py +++ b/test/test_graph_work_listing.py @@ -155,7 +155,8 @@ def test_missing_title_is_reported_without_invention_and_ids_stay_private(retrie assert rendered['answer'].count('názov v zdroji nie je uvedený') == 3 assert 'Diplomový projekt 2 2020' in rendered['answer'] assert 'Systém získavania informácií' in rendered['answer'] - assert 'typ práce: Bakalárska práca' in rendered['answer'] + assert '**Bakalárska práca 2019**' in rendered['answer'] + assert 'typ práce:' not in rendered['answer'] assert all(claim['record_id'] not in rendered['answer'] for claim in answer['claims']) answer['claims'][2]['title'] = 'Vymyslený názov projektu' assert 'unsupported_academic_binding' in answer_context_issues(answer, context, check_text=False) diff --git a/test/test_rag_answer_coverage.py b/test/test_rag_answer_coverage.py index 219d181..47c1bf3 100644 --- a/test/test_rag_answer_coverage.py +++ b/test/test_rag_answer_coverage.py @@ -86,7 +86,9 @@ def test_regular_assignment_can_show_all_supplied_goals_without_global_warning(p a = quote(GOALS) assert rag_answer_issues(a, c) == [] text = render_rag_answer(a, c) - assert GOALS in text and 'neúpln' not in text and 'Zo zdroja' not in text + assert '1. Preskúmanie metód vyhľadávania.' in text + assert '4. Návrh zlepšení.' in text + assert 'neúpln' not in text and 'Zo zdroja' not in text assert answer_coverage(a, c)['scope'] == 'assignment' @@ -136,7 +138,7 @@ def test_exact_aggregate_can_be_supported_despite_limited_evidence(payload): def test_long_valid_overview_is_shortened_without_another_model_request(payload): question = 'Čomu sa venuje diplomová práca Evy Novej?' - short = '1. Preskúmajte metódy vyhľadávania.\n2. Implementujte vyhľadávač.' + short = '1. Preskúmanie metód vyhľadávania.\n2. Implementácia vyhľadávača.' c = rag_evidence_context(question, payload) assert rag_answer_issues(quote(GOALS), c) == [] calls = [] diff --git a/test/test_rag_work_content.py b/test/test_rag_work_content.py index 6fe7880..3545cf1 100644 --- a/test/test_rag_work_content.py +++ b/test/test_rag_work_content.py @@ -153,7 +153,10 @@ def test_correction_sends_real_description_but_no_unrelated_sections(payload): text = json.dumps(sent['content_passages'], ensure_ascii=False) assert 'ohodnotenia dokumentov' in text and 'Cassandra' not in text and 'medicínskych' not in text assert 'missing_content_evidence' in result['attempts'][0]['issues'] - assert ASSIGNMENT in result['answer'] and 'Výpis je neúplný.' not in result['answer'] + assert 'Vypracovanie prehľadu metód ohodnotenia dokumentov' in result['answer'] + assert 'Implementácia systému pre získavanie informácií v slovenskom jazyku.' in result['answer'] + assert result['attempts'][-1]['parsed_answer']['evidence_quotes'][0]['quote'] == ASSIGNMENT + assert 'Výpis je neúplný.' not in result['answer'] assert result['complete'] is False and result['answer_coverage']['status'] == 'supported' assert 'expected_bindings' not in json.dumps(calls) diff --git a/test/test_verified_rag.py b/test/test_verified_rag.py index 0733eb1..6a2b16c 100644 --- a/test/test_verified_rag.py +++ b/test/test_verified_rag.py @@ -102,8 +102,8 @@ def test_bad_answer_is_repaired_once_and_only_verified_fields_are_rendered(paylo return completion(outputs[len(calls) - 1]) result = generate_verified_rag(QUESTION, payload, complete) assert result['accepted_for_delivery'] and len(calls) == 2 - assert 'Ondrej Megela — bakalárska práca' in result['answer'] - assert 'rok práce: 2020' in result['answer'] and '2099' not in result['answer'] + assert 'Ondrej Megela má bakalársku prácu' in result['answer'] + assert 'rok práce: **2020**' in result['answer'] and '2099' not in result['answer'] assert 'Výpis je neúplný.' not in result['answer'] assert result['complete'] is False and result['answer_coverage']['status'] == 'supported' assert 'unsupported_person_work_type_year_binding' in result['attempts'][0]['issues'] @@ -182,7 +182,7 @@ def test_graph_production_delivery_checks_renders_and_propagates_metadata(monkey answer = {'status': 'ok', 'answer': '', 'claims': [{'person': 'Eva Nová', 'year': 2026}], 'source_urls': ['https://example.test/eva']} result = delivery.deliver_answer('Práca Evy Novej?', engine='graphrag', complete=lambda messages: completion(answer)) - assert result['accepted_for_delivery'] and 'rok práce: 2026' in result['answer'] + assert result['accepted_for_delivery'] and 'rok práce: **2026**' in result['answer'] assert result['complete'] is False and result['index_version'] == {'version': 'fixture'} assert 'Výpis je neúplný.' in result['answer'] answer['claims'][0]['year'] = 2099 diff --git a/tools/answer_evaluation/context_policy.py b/tools/answer_evaluation/context_policy.py index 39bc7bd..98fa3c3 100644 --- a/tools/answer_evaluation/context_policy.py +++ b/tools/answer_evaluation/context_policy.py @@ -111,6 +111,17 @@ def prose_issues(answer: dict) -> list[str]: text = str(answer.get('answer', '')) for url in sorted(text_urls(text), key=len, reverse=True): text = text.replace(url, '') + # A bounded ordered-list marker is presentation, not a factual count/year. + # Accept only a contiguous sequence starting at one, with at most one item + # per structured claim. Other numbers still need explicit factual support. + expected = 1 + def list_marker(match): + nonlocal expected + if int(match[1]) == expected and expected <= len(answer.get('claims', [])): + expected += 1 + return '' + return match[0] + text = re.sub(r'^\s*(\d+)[.)]\s+', list_marker, text, flags=re.M) allowed_numbers = set() for claim in answer.get('claims', []): for key, value in claim.items(): diff --git a/tools/answer_evaluation/graph_context.py b/tools/answer_evaluation/graph_context.py index faf0972..cf48cad 100644 --- a/tools/answer_evaluation/graph_context.py +++ b/tools/answer_evaluation/graph_context.py @@ -52,6 +52,9 @@ def graph_context(result: dict) -> dict: 'truncation_reasons':result.get('truncation_reasons',[]), 'index_version':result.get('index_version')} context['answer_candidates']=joined_candidates(facts,intent) + context['question'] = result.get('query', '') + context['requested_people'] = list(dict.fromkeys(row['name'] + for row in result.get('resolved_entities', []) if row.get('kind') == 'Person' and row.get('name'))) if work_listing: context['required_academic_records'] = academic_records return context diff --git a/tools/answer_evaluation/presentation.py b/tools/answer_evaluation/presentation.py new file mode 100644 index 0000000..7e96c96 --- /dev/null +++ b/tools/answer_evaluation/presentation.py @@ -0,0 +1,145 @@ +"""Slovak presentation templates for validated facts and literal source excerpts. + +No model, gold answers or extra retrieval is used here. Task wording can be +converted to a goal, never to a claim that the task has already been completed. +""" +from __future__ import annotations + +import re + +from .context_policy import normalized +from .rag_evidence import WORK_LABELS + + +ACCUSATIVE = { + 'bachelor_thesis': 'bakalársku prácu', 'diploma_thesis': 'diplomovú prácu', + 'doctoral_thesis': 'dizertačnú prácu', 'bachelor_project': 'bakalársky projekt', + 'diploma_project': 'diplomový projekt', 'team_project': 'tímový projekt', + 'research_project': 'výskumný projekt', +} + + +def owner_in_question(person: str, question: str) -> str | None: + """Reuse a bound genitive from the question; never guess a name's declension.""" + words = list(re.finditer(r"[^\W\d_]+(?:[-'][^\W\d_]+)*", question)) + tokens = person.split() + for index in range(len(words) - len(tokens) + 1): + found = words[index:index + len(tokens)] + if not re.search(r'\b(?:práca|práce|prácu|prác|projekt|projektu|projektov)\s*$', + question[:found[0].start()], re.I): + continue + valid = True + for token, match in zip(tokens, found): + base = normalized(token) + variants = {base, base + 'a', base + 'u'} + valid &= normalized(match[0]) in variants + if valid: + return question[found[0].start():found[-1].end()] + return None + + +def work_subject(person: str, kind: str | None, question: str = '') -> str: + label = WORK_LABELS.get(kind, 'práca') + owner = owner_in_question(person, question) + return label.capitalize() + (' ' + owner if owner else f' ({person})') + + +def fact_sentences(claim: dict, question: str = '') -> str: + """Keep each person's academic tuple together, with explicit year semantics.""" + person = claim['person']; kind = claim.get('work_type') + title = claim.get('title'); year = claim.get('year') + displayed_title = title.rstrip('.') if title is not None else None + parts = [] + if kind or title is not None or year is not None: + owner = owner_in_question(person, question) + if owner: + subject = work_subject(person, kind, question) + if title is not None: + parts.append(f'{subject} má názov **{displayed_title}**.') + elif year is not None: + parts.append(f'{subject} je v podkladoch evidovaná s rokom **{year}**.' + if not (kind or '').endswith('_project') else + f'{subject} je v podkladoch evidovaný s rokom **{year}**.') + else: + parts.append(f'V podkladoch je uvedená {subject[0].lower() + subject[1:]}.' + if not (kind or '').endswith('_project') else + f'V podkladoch je uvedený {subject[0].lower() + subject[1:]}.') + else: + sentence = f'{person} má {ACCUSATIVE.get(kind, "prácu")}' + if title is not None: + sentence += f' s názvom **{displayed_title}**' + if year is not None: + sentence += f' (rok práce: **{year}**)' + parts.append(sentence + '.') + if owner and title is not None and year is not None: + parts.append(f'Rok práce je **{year}**.') + if claim.get('start_year') is not None: + parts.append(f'{person} má uvedený začiatok štúdia v roku **{claim["start_year"]}**.') + for key, label in [('keyword', 'kľúčové slovo'), ('category', 'kategória'), + ('document_author', 'autor dokumentu')]: + if claim.get(key) is not None: + parts.append(f'Pri osobe {person} je uvedené: {label} **{claim[key]}**.') + return ' '.join(parts) or person + '.' + + +GOAL_NOUNS = ( + ('Vypracujte prehľad', 'Vypracovanie prehľadu'), + ('Vypracujte rešerš', 'Vypracovanie rešerše'), + ('Implementujte jednoduchý systém', 'Implementácia jednoduchého systému'), + ('Implementujte systém', 'Implementácia systému'), + ('Implementujte vyhľadávač', 'Implementácia vyhľadávača'), + ('Navrhnite a vypracujte experimenty', 'Návrh a vypracovanie experimentov'), + ('Navrhnite možné zlepšenia', 'Návrh možných zlepšení'), + ('Navrhnite zlepšenia', 'Návrh zlepšení'), + ('Preskúmajte metódy', 'Preskúmanie metód'), + ('Vyhodnoťte presnosť', 'Vyhodnotenie presnosti'), +) +GOAL_VERBS = { + 'vypracujte': 'Vypracovať', 'implementujte': 'Implementovať', + 'navrhnite': 'Navrhnúť', 'preskúmajte': 'Preskúmať', + 'vyhodnoťte': 'Vyhodnotiť', 'porovnajte': 'Porovnať', + 'vytvorte': 'Vytvoriť', 'analyzujte': 'Analyzovať', + 'otestujte': 'Otestovať', 'overte': 'Overiť', +} + + +def goal_text(unit: str) -> tuple[str, bool]: + """Only bounded grammatical substitutions; unknown prose stays verbatim.""" + text = re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+', '', unit.strip()) + for original, replacement in GOAL_NOUNS: + if re.match(re.escape(original) + r'\b', text, re.I): + result = replacement + text[len(original):] + result = result.replace('v ktorom využijete', 'v ktorom sa má využiť') + result = result.replace(', v ktorých vyhodnotíte', ' s cieľom vyhodnotiť') + result = result.replace('v ktorých vyhodnotíte', 's cieľom vyhodnotiť') + return result, True + first = text.split(maxsplit=1)[0].casefold() if text else '' + if first in GOAL_VERBS: + result = re.sub(r'\b(' + '|'.join(map(re.escape, GOAL_VERBS)) + r')\b', + lambda match: GOAL_VERBS[match[0].casefold()].lower(), text, flags=re.I) + return result[0].upper() + result[1:], True + return unit.strip(), False + + +def content_lines(units: list[str], person: str | None, kind: str | None, + question: str = '') -> list[str]: + """Distinguish documented goals from factual descriptions or progress notes.""" + # A task heading has no factual content of its own and otherwise prevents + # a whole assignment quote from being presented as individual goals. + units = [unit for unit in units if normalized(unit) not in { + 'zadanie', 'zadanie diplomovej prace', 'zadanie bakalarskej prace', 'ciele prace'}] + converted = [goal_text(unit) for unit in units] + subject = work_subject(person, kind, question) if person else 'Práca' + if converted and all(is_goal for _, is_goal in converted): + return [subject + ' má podľa dostupného zadania tieto ciele:', '', + *[f'{index}. {text}' for index, (text, _) in enumerate(converted, 1)]] + return [subject + ' je v dostupných podkladoch opísaná takto:' if not (kind or '').endswith('_project') + else subject + ' je v dostupných podkladoch opísaný takto:', '', *units] + + +def no_answer_text(context: dict) -> str: + people = context.get('requested_people') or context.get('requirements', {}).get('people', []) + if len(people) == 1: + return (f'Pre osobu {people[0]} sa v dostupných podkladoch nepodarilo spoľahlivo nájsť ' + 'údaje zodpovedajúce tejto otázke.') + return 'K tejto otázke sa v dostupných podkladoch nepodarilo spoľahlivo nájsť zodpovedajúce údaje.' diff --git a/tools/answer_evaluation/render_answer.py b/tools/answer_evaluation/render_answer.py index 044be6b..a670302 100644 --- a/tools/answer_evaluation/render_answer.py +++ b/tools/answer_evaluation/render_answer.py @@ -1,24 +1,43 @@ """Render only validated structured facts; never infer facts or read gold.""" from copy import deepcopy +import re from tools.graphrag.graphrag import WORK_TYPE_LABELS +from .presentation import fact_sentences, no_answer_text def render_graph_answer(answer: dict, context: dict) -> dict: out = deepcopy(answer) lines = [] if out['status'] == 'ok': - labels = {'work_label': 'záznam', 'work_type': 'typ práce', - 'title': 'názov práce', 'year': 'rok práce', 'start_year': 'začiatok štúdia', - 'keyword': 'kľúčové slovo', 'category': 'kategória', 'document_author': 'autor dokumentu'} + records = {} for claim in out['claims']: - values = [f'{label}: {WORK_TYPE_LABELS.get(claim[key], claim[key]) if key == "work_type" else claim[key]}' - for key, label in labels.items() if claim.get(key) is not None] - if claim.get('record_id') and not claim.get('title'): - values.append('názov v zdroji nie je uvedený') - if claim.get('record_id') and claim.get('year') is None: - values.append('rok v zdroji nie je uvedený') - lines.append(claim['person'] + (' — ' + '; '.join(values) if values else '') + '.') + if claim.get('record_id'): + records.setdefault(claim['person'], []).append(claim) + else: + lines.append(fact_sentences(claim, context.get('question', ''))) + index = 0 + for person, claims in records.items(): + lines.extend([f'{person} má v dostupných podkladoch tieto práce a projekty:', '']) + # Stable sorting retains the source order of same-year projects. + for claim in sorted(claims, key=lambda row: (row.get('year') is None, row.get('year') or 0)): + index += 1 + kind = WORK_TYPE_LABELS.get(claim.get('work_type'), 'Práca') + label = claim.get('work_label') or kind + details = [] + if kind.casefold() not in label.casefold(): + details.append(kind) + if claim.get('year') is not None and not re.search(r'\b' + str(claim['year']) + r'\b', label): + details.append(f'rok práce: {claim["year"]}') + if claim.get('year') is None: + details.append('rok v zdroji nie je uvedený') + title = '**' + claim['title'].rstrip('.') + '**' if claim.get('title') else 'názov v zdroji nie je uvedený' + lines.append(f'{index}. **{label}**' + (' (' + '; '.join(details) + ')' if details else '') + f' — {title}.') + extra = {key: claim[key] for key in ('person', 'keyword', 'category', 'document_author', 'start_year') + if claim.get(key) is not None} + if len(extra) > 1: + lines.append(fact_sentences(extra)) + lines.append('') if out.get('count') is not None: lines.append(f'Počet vyhovujúcich osôb: {out["count"]}.') if context.get('complete') is False or context.get('truncated'): @@ -30,8 +49,8 @@ def render_graph_answer(answer: dict, context: dict) -> dict: elif out['status'] == 'error': lines.append('Vyhľadanie zlyhalo; odpoveď nemožno pripraviť.') else: - lines.append('Požadované údaje sa v dostupných podkladoch nenašli.') + lines.append(no_answer_text(context)) if out['source_urls']: - lines.append('Zdroje: ' + ', '.join(out['source_urls'])) + lines.extend(['', ('Zdroj: ' if len(out['source_urls']) == 1 else 'Zdroje: ') + ', '.join(out['source_urls'])]) out['answer'] = '\n'.join(lines) return out diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index 6dde0b7..ebcff43 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -9,8 +9,9 @@ from pydantic import BaseModel, ConfigDict, Field from .answer_format import Claim from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized, text_urls -from .rag_evidence import WORK_LABELS, descriptive_text, rag_evidence_context +from .rag_evidence import descriptive_text, rag_evidence_context from .answer_coverage import answer_coverage, coverage_notices +from .presentation import content_lines, fact_sentences, no_answer_text class RagClaim(Claim): @@ -223,7 +224,7 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: return list(dict.fromkeys(issues)) -def overview_excerpts(answer: dict, context: dict) -> list[str]: +def _overview_selections(answer: dict, context: dict) -> list[dict]: """Shorten validated quotes by selection, never by factual paraphrase. Presentation length is not a reason to reject otherwise supported evidence @@ -259,30 +260,49 @@ def overview_excerpts(answer: dict, context: dict) -> list[str]: if not prefix or ' ' not in prefix or not descriptive_text(prefix, identities): continue unit = prefix + ' …' - selected.append(unit); seen.add((owner, unit)) + selected.append({'evidence': quote, 'text': unit}); seen.add((owner, unit)) counts[owner] = counts.get(owner, 0) + 1 sizes[owner] = sizes.get(owner, 0) + len(unit) # An indivisible technical string is better retained than silently omitted. - return selected or [q['quote'] for q in answer.get('evidence_quotes', [])] + return selected or [{'evidence': q, 'text': q['quote']} for q in answer.get('evidence_quotes', [])] + + +def overview_excerpts(answer: dict, context: dict) -> list[str]: + return [selection['text'] for selection in _overview_selections(answer, context)] def render_rag_answer(answer: dict, context: dict) -> str: lines = [] if answer['status'] == 'ok': - labels = {'title': 'názov práce', 'year': 'rok práce', 'start_year': 'začiatok štúdia', - 'keyword': 'kľúčové slovo', 'category': 'kategória', 'document_author': 'autor dokumentu'} for claim in answer.get('claims', []): - values = [WORK_LABELS[claim['work_type']]] if claim.get('work_type') else [] - values += [f'{label}: {claim[key]}' for key, label in labels.items() if claim.get(key) is not None] - lines.append(claim['person'] + ' — ' + '; '.join(values) + '.') + lines.append(fact_sentences(claim, context.get('question', ''))) quotes = answer.get('evidence_quotes', []) style = context.get('requirements', {}).get('content_style') if quotes and style in {'overview', 'assignment'}: - if style == 'overview': - lines.append('Stručne z dostupných podkladov:') - lines += overview_excerpts(answer, context) - else: - lines += [quote['quote'] for quote in quotes] + # Bind each paragraph to the same source/person/work as the checked + # quotes. Never summarize several people's excerpts under one name. + groups = {} + display = (_overview_selections(answer, context) if style == 'overview' else + [{'evidence': quote, 'text': quote['quote']} for quote in quotes]) + for selection in display: + quote = selection['evidence'] + passage = next((p for p in context.get('content_passages', []) + if p['source_id'] == quote['source_id'] and + ' '.join(quote['quote'].split()) in ' '.join(p['text'].split())), {}) + identity = (passage.get('person'), passage.get('work_type'), passage.get('year')) + groups.setdefault(identity, []).append(selection['text']) + for (person, kind, _year), selections in groups.items(): + if style == 'overview': + units = selections + else: + units = [] + for text in selections: + # Split only real list boundaries, retaining the source + # wording and every supplied assignment item. + units.extend(block.strip() for block in re.split( + r'(?=^[ \t]*(?:\d+[.)]|[-+*])\s+)', text, flags=re.M) + if block.strip()) + lines.extend(content_lines(units, person, kind, context.get('question', ''))) else: sources = {row['source_id']: row for row in context['source_blocks']} lines += ['Zo zdroja ' + str(sources[quote['source_id']].get('title') or quote['source_id']) + @@ -290,14 +310,14 @@ def render_rag_answer(answer: dict, context: dict) -> str: if answer.get('count') is not None: lines.append(f'Počet: {answer["count"]}.') elif answer['status'] == 'no_answer': - lines.append('V dostupných podkladoch sa požadovanú informáciu nepodarilo spoľahlivo nájsť.') + lines.append(no_answer_text(context)) elif answer['status'] == 'ambiguous': lines.append('Údaje sú nejednoznačné; požiadavku treba spresniť.') else: lines.append('Overenú odpoveď sa nepodarilo pripraviť.') lines.extend(coverage_notices(answer_coverage(answer, context))) if answer.get('source_urls'): - lines.append('Zdroje: ' + ', '.join(answer['source_urls'])) + lines.extend(['', ('Zdroj: ' if len(answer['source_urls']) == 1 else 'Zdroje: ') + ', '.join(answer['source_urls'])]) return '\n'.join(lines) diff --git a/tools/graphrag/evaluation/reproducibility.py b/tools/graphrag/evaluation/reproducibility.py index 219a900..5dc3ec6 100644 --- a/tools/graphrag/evaluation/reproducibility.py +++ b/tools/graphrag/evaluation/reproducibility.py @@ -44,6 +44,8 @@ def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]: 'tools/rag/rag_utils.py', 'tools/answer_evaluation/context_policy.py', 'tools/answer_evaluation/checked_generation.py', 'tools/answer_evaluation/answer_format.py', 'tools/answer_evaluation/render_answer.py', 'tools/answer_evaluation/graph_context.py', + 'tools/answer_evaluation/presentation.py', + 'tools/answer_evaluation/rag_evidence.py', 'tools/answer_evaluation/graph_answer_prompt.txt', 'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py']) files = sorted(set(files))