From a5dd917e6dd69c6000e380dc9e3c7cc96fa80d4e Mon Sep 17 00:00:00 2001 From: jp170na Date: Tue, 3 Nov 2026 20:56:36 +0100 Subject: [PATCH] =?UTF-8?q?Zrozumite=C4=BEnej=C5=A1ie=20odpovede?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 + test/test_answer_clarity.py | 158 +++++++++++++++++++ test/test_answer_presentation.py | 6 +- test/test_comparison_answers.py | 8 +- test/test_quality_block.py | 2 +- test/test_rag_answer_coverage.py | 3 +- tools/answer_evaluation/answer_coverage.py | 6 +- tools/answer_evaluation/content_selection.py | 70 ++++---- tools/answer_evaluation/context_policy.py | 10 +- tools/answer_evaluation/graph_context.py | 2 + tools/answer_evaluation/presentation.py | 76 ++++++++- tools/answer_evaluation/rag_evidence.py | 8 + tools/answer_evaluation/render_answer.py | 10 +- tools/answer_evaluation/verified_rag.py | 31 ++-- 14 files changed, 334 insertions(+), 63 deletions(-) create mode 100644 test/test_answer_clarity.py diff --git a/README.md b/README.md index 166dcf6..087123d 100644 --- a/README.md +++ b/README.md @@ -1527,6 +1527,13 @@ aby dlhý prvý profil nevytlačil ostatné. Skrátenie rešpektuje celé odseky zoznamu vrátane ich výhrad; pri stručnom opise má zachovanie podmienky prednosť pred orientačným limitom dĺžky. +Krátke porovnanie zobrazuje rovnaké oblasti vedľa seba v tabuľke; dlhé výňatky, +kód a odkazy zostávajú v samostatných sekciách bez úpravy ich obsahu. Prázdne +pomocné sekcie sa nevypisujú a chýbajúce oblasti sa konkrétne pomenujú pri danej +osobe. Spoločné vlastnosti sa uvádzajú najprv súhrnne a potom s doloženými +osobami a zdrojmi. Stručnejšie formulácie nemenia rok práce na dátum jej +začatia alebo dokončenia a ciele neprezentujú ako dosiahnuté výsledky. + Rozpoznané, ale nepodporované operátory (napríklad interval rokov, vylučovanie alebo minimálny počet) vedú k `ambiguous` a konkrétnemu spresneniu. Rok práce sa nepoužije ako dôkaz dátumu jej začatia či obhajoby. Chybové odpovede odlišujú diff --git a/test/test_answer_clarity.py b/test/test_answer_clarity.py new file mode 100644 index 0000000..44c63ec --- /dev/null +++ b/test/test_answer_clarity.py @@ -0,0 +1,158 @@ +"""Concise presentation must keep factual bindings, qualifications and gaps.""" +import copy +import sqlite3 + +import pytest + +from test_quality_block import corpus, prepare, complete +from test_evidence_selection import update_eva, selection +from tools.answer_evaluation.context_policy import answer_context_issues +from tools.answer_evaluation.content_selection import facet_lines +from tools.answer_evaluation.graph_context import graph_context +from tools.answer_evaluation.presentation import comparison_table +from tools.answer_evaluation.rag_evidence import rag_evidence_context +from tools.answer_evaluation.render_answer import render_graph_answer +from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues + + +def test_missing_aspects_share_one_specific_notice_without_empty_sections(corpus): + update_eva(corpus,'Stav:\n- Implementovaný parser.') + q='Aké výsledky, problémy a plánované experimenty má diplomová práca Evy Novej?' + p=prepare(corpus,q); c=rag_evidence_context(q,p) + result=generate_verified_rag(q,p,complete(selection(c))) + assert result['accepted_for_delivery'] and result['answer_coverage']['status']=='partial' + text=result['answer'] + assert 'Implementovaný parser.' in text + assert text.count('V poskytnutých podkladoch chýbajú údaje')==1 + assert 'riešených problémoch, plánovaných úlohách a plánovaných experimentoch' in text + assert '**Riešené problémy:**' not in text and '**Plánované experimenty' not in text + assert 'Táto časť odpovede' not in text and '\n\n\n' not in text + + +def test_short_comparison_aligns_people_and_different_results(corpus): + q='Porovnaj výsledky Evy Novej a Borisa Nového.' + p=prepare(corpus,q); c=rag_evidence_context(q,p); a=selection(c) + before=copy.deepcopy((a,c)) + lines=facet_lines(a,c); text='\n'.join(lines) + assert text.startswith('| Oblasť | Eva Nová — diplomová práca (2024) | Boris Nový — bakalárska práca (2023) |') + row=next(line for line in lines if line.startswith('| Zaznamenané výsledky |')) + assert 'BERT' in row.split('|')[2] and 'Python' in row.split('|')[3] + assert 'Konzultácia' not in text and 'Pripravte experiment' not in text + assert 'lepš' not in text and (a,c)==before + r=generate_verified_rag(q,p,complete(a)) + assert r['accepted_for_delivery'] and len(r['source_urls'])==2 + + +@pytest.mark.parametrize('literal',[ + 'Implementovaný parser. Výsledok platí iba pre malý korpus. '+('Technický opis. '*70), + 'Implementovaný parser pre `a | b` iba na testovacích dátach.', + 'Implementovaný parser podľa https://example.test/method iba na testovacích dátach.', +]) +def test_complex_comparisons_fall_back_without_losing_qualifiers_or_literals(corpus,literal): + update_eva(corpus,'Výsledky:\n- '+literal) + q='Porovnaj výsledky Evy Novej a Borisa Nového.' + p=prepare(corpus,q); c=rag_evidence_context(q,p); a=selection(c) + text='\n'.join(facet_lines(a,c)) + assert not text.startswith('| Oblasť |') and literal.strip() in text + assert 'Boris Nový' in text and 'Python' in text + assert all('Ďalšie doložené' not in line for line in text.splitlines()) + + +def test_shared_excerpt_is_not_repeated_under_each_matching_facet(corpus): + update_eva(corpus,'Výsledky:\n- Vyhodnotená presnosť modelu v experimente.') + q='Aké experimenty a výsledky má diplomová práca Evy Novej?' + p=prepare(corpus,q); c=rag_evidence_context(q,p) + result=generate_verified_rag(q,p,complete(selection(c))) + assert result['accepted_for_delivery'] + assert result['answer'].count('Vyhodnotená presnosť modelu v experimente.')==1 + assert 'chýbajú údaje' not in result['answer'] and 'Doložené vyššie' not in result['answer'] + assert 'Zaznamenaný výsledok:' in result['answer'] or '**Zaznamenané výsledky:**' in result['answer'] + + +def test_multiline_table_is_bounded_and_does_not_rewrite_source_html(): + table=comparison_table(['Oblasť','Eva','Boris'],[('Výsledky',['Prvý výsledok.\nDruhý výsledok.','Parser.'])]) + assert table and 'Prvý výsledok.
Druhý výsledok.' in table[-1] + assert comparison_table(['Oblasť','Eva'],[('Výsledky',['Prvý výsledok.'])]) is None + + +def test_shared_result_precedes_people_and_citations_without_repeating_tags(): + people=['Ján Pták','Ján Malinovský']; urls=['https://example.test/ptak','https://example.test/malinovsky'] + claims=[{'person':person,'keyword':tag} for person in people for tag in ['NLP','RAG']] + c={'status':'ok','complete':True,'truncated':False,'facts':[ + {**claim,'source_url':urls[people.index(claim['person'])]} for claim in claims], + 'sources':[{'source_url':u} for u in urls], 'requested_people':people, + 'shared_properties':[{'field':'keyword','value':tag,'people':people} for tag in ['NLP','RAG']]} + a={'status':'ok','claims':claims,'source_urls':urls} + r=render_graph_answer(a,c); text=r['answer'] + assert text.startswith('Spoločné kľúčové slová: **NLP**, **RAG**.') + assert text.index('RAG') str: if content: if re.search(r'\b(?:zadani\w*|ciel\w*)\b', query) and not re.search(r'\b(?:strucn\w*|zhrn\w*)\b', query): return 'assignment' - if re.search(r'\b(?:comu|o com|opis\w*|popis\w*|obsah\w*|zhrn\w*|strucn\w*)\b', query): + if re.search(r'\b(?:comu|o com|opis\w*|popis\w*|obsah\w*|zamer\w*|zhrn\w*|strucn\w*)\b', query): return 'overview' return 'fact' @@ -55,6 +55,10 @@ def answer_coverage(answer: dict, context: dict) -> dict: from .content_selection import missing_facets if missing_facets(answer,context): coverage='partial'; warnings.append('partial_requested_aspects') + elif requirements.get('content'): + from .rag_evidence import missing_content_people + if missing_content_people(answer,context): + coverage='partial'; warnings.append('partial_requested_aspects') if scope == 'list' and (limited or clipped): coverage = 'partial'; warnings.append('list_may_be_incomplete') elif scope == 'count' and (context.get('aggregation') or {}).get('complete') is not True: diff --git a/tools/answer_evaluation/content_selection.py b/tools/answer_evaluation/content_selection.py index 5ccb003..c753a66 100644 --- a/tools/answer_evaluation/content_selection.py +++ b/tools/answer_evaluation/content_selection.py @@ -218,34 +218,22 @@ def fit_candidates(sent: dict, context: dict, limit: int, encoding: str) -> None def facet_lines(answer: dict, context: dict) -> list[str]: - from .presentation import goal_text, work_subject, owner_in_question - from .rag_evidence import WORK_LABELS + from .presentation import goal_text, evidence_subject, missing_aspects, comparison_table req=context['requirements']; units=evidence_units(context,answer.get('evidence_quotes',[])) - lines=[] + facets=req.get('facets',[]);groups=[];labels={f:FACET_LABELS[f] for f in facets} + if 'experiments' in facets and planned_experiments(context.get('question','')): + labels['experiments']='Plánované experimenty a vyhodnotenie' for person in req['people']: - scopes={(p['work_type'],p.get('year')) for p in context.get('content_passages',[]) if p.get('person')==person} - kind=req['work_types'].get(person) or (next(iter(scopes))[0] if len(scopes)==1 else None) - subject=work_subject(person,kind,context.get('question','')) - if not owner_in_question(person,context.get('question','')): - subject=person+(' — '+WORK_LABELS[kind] if kind in WORK_LABELS else '') - if len(scopes)==1 and next(iter(scopes))[1] is not None: - subject+=' ('+str(next(iter(scopes))[1])+')' - labels={p.get('work_label') for p in context.get('content_passages',[]) - if p.get('person')==person and p.get('work_label')} - if req.get('project_number') is not None and len(labels)==1: - subject=person+' — '+' '.join(next(iter(labels)).split()) - lines.extend(['**'+subject+'**', '']) - seen=set() - for facet in req.get('facets', []): - label=FACET_LABELS[facet] - if facet=='experiments' and planned_experiments(context.get('question','')): - label='Plánované experimenty a vyhodnotenie' + group={'subject':evidence_subject(person,context),'items':{},'missing':[]};seen=set() + for facet in facets: rows=[u for u in units if u['person']==person and facet in u['facets']] + if facet=='progress' and 'results' in facets and any(u['phase']=='progress' for u in rows): + rows=[u for u in rows if u['phase']=='progress'] rows.sort(key=lambda u:-unit_score(u['text'],u['kind'],facet)) - lines.append('**'+label+':**') - shown=0 + if not rows:group['missing'].append(facet) + rendered=[] for row in rows: - key=' '.join(re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text']).split()) + key=(' '.join(re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text']).split()),row['phase']) if key in seen: continue text=row['text'] @@ -253,14 +241,30 @@ def facet_lines(answer: dict, context: dict) -> list[str]: text,_=goal_text(text) text=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',text) phase=' (plán)' if row['kind'] in {'assignment','task'} and facet not in {'assignment','plans'} else '' - state=({'result':'Zaznamenaný výsledok: ','progress':'Priebežný stav: '}.get(row['phase'],'') - if facet=='results' else '') - lines.append('- '+state+text+phase); seen.add(key); shown+=1 - if shown==2: break - if not shown: - if rows: - lines.append('Je doložené rovnakým výňatkom uvedeným vyššie.') - else: - lines.append('Táto časť odpovede nie je v poskytnutých podkladoch doložená.') - lines.append('') + state='Zaznamenaný výsledok: ' if row['phase']=='result' and 'results' in facets and facet!='results' else '' + rendered.append(state+text+phase);seen.add(key) + if len(rendered)==2:break + group['items'][facet]=rendered + groups.append(group) + if len(groups)>1 and re.search(r'\bporovnaj\w*\b',normalized(context.get('question',''))): + rows=[] + for facet in facets: + if not any(g['items'][facet] for g in groups): + if all(facet not in g['missing'] for g in groups):continue + values=['\n'.join(g['items'][facet]) or ('V podkladoch chýbajú údaje.' if facet in g['missing'] else 'Doložené vyššie.') for g in groups] + rows.append((labels[facet],values)) + if any(any(g['items'][f] for f in facets) for g in groups): + table=comparison_table(['Oblasť',*(g['subject'] for g in groups)],rows) + if table is not None:return table + lines=[] + for group in groups: + title=group['subject'] + if len(facets)==1 and group['items'][facets[0]]:title+=' — '+labels[facets[0]].lower() + lines.extend(['**'+title+'**','']) + for facet in facets: + items=group['items'][facet] + if not items:continue + if len(facets)>1:lines.append('**'+labels[facet]+':**') + lines.extend('- '+text for text in items);lines.append('') + if group['missing']:lines.extend([missing_aspects(group['missing'],context.get('question','')),'']) return lines diff --git a/tools/answer_evaluation/context_policy.py b/tools/answer_evaluation/context_policy.py index 54b98f9..b9f3afd 100644 --- a/tools/answer_evaluation/context_policy.py +++ b/tools/answer_evaluation/context_policy.py @@ -108,9 +108,15 @@ def coverage_issues(answer: dict, context: dict) -> list[str]: return list(dict.fromkeys(issues)) -def prose_issues(answer: dict) -> list[str]: +def prose_issues(answer: dict, context: dict | None = None) -> list[str]: """Conservative numeric consistency check; final prose is rendered from claims.""" text = str(answer.get('answer', '')) + if context is not None and answer.get('status')=='no_answer': + from .presentation import no_answer_text + expected=no_answer_text(context) + # These years describe the failed query, never a positive factual claim. + # Strip only the exact trusted template; appended prose stays checked. + if text.startswith(expected):text=text[len(expected):] for url in sorted(text_urls(text), key=len, reverse=True): text = text.replace(url, '') # A bounded ordered-list marker is presentation, not a factual count/year. @@ -236,7 +242,7 @@ def answer_context_issues(answer: dict, context: dict, *, check_text: bool = Tru issues.append('split_or_missing_person_join') if context.get('status') in {'no_answer','ambiguous','error'} and answer.get('status')=='ok': issues.append('unsupported_positive_status') - issues.extend(prose_issues(answer)) + issues.extend(prose_issues(answer,context)) if check_text or str(answer.get('answer', '')).strip(): issues.extend(coverage_issues(answer,context)) return list(dict.fromkeys(issues)) diff --git a/tools/answer_evaluation/graph_context.py b/tools/answer_evaluation/graph_context.py index 3b64fe9..b4b3e3a 100644 --- a/tools/answer_evaluation/graph_context.py +++ b/tools/answer_evaluation/graph_context.py @@ -61,6 +61,8 @@ def graph_context(result: dict) -> dict: 'index_version':result.get('index_version')} context['answer_candidates']=joined_candidates(facts,intent) context['question'] = result.get('query', '') + context['requested_years']=intent.get('years',[]) if not intent.get('mentions_start_year') else [] + context['requested_work_types']=[*intent.get('thesis_types',[]),*intent.get('work_types',[])] context['clarification'] = result.get('clarification') context['requested_people'] = list(dict.fromkeys(row['name'] for row in result.get('resolved_entities', []) if row.get('kind') == 'Person' and row.get('name'))) diff --git a/tools/answer_evaluation/presentation.py b/tools/answer_evaluation/presentation.py index e7d8161..9501e14 100644 --- a/tools/answer_evaluation/presentation.py +++ b/tools/answer_evaluation/presentation.py @@ -53,7 +53,8 @@ def fact_sentences(claim: dict, question: str = '') -> str: if owner: subject = work_subject(person, kind, question) if title is not None: - parts.append(f'{subject} má názov **{displayed_title}**.') + parts.append(f'{subject} má názov **{displayed_title}**'+ + (f' (rok práce: **{year}**).' if year is not None else '.')) elif year is not None: parts.append(f'{subject} je v podkladoch evidovaná s rokom **{year}**.' if not (kind or '').endswith('_project') else @@ -69,14 +70,17 @@ def fact_sentences(claim: dict, question: str = '') -> str: if year is not None: sentence += f' (rok práce: **{year}**)' parts.append(sentence + '.') - if owner and title is not None and year is not None: - parts.append(f'Rok práce je **{year}**.') if claim.get('start_year') is not None: parts.append(f'{person} má uvedený začiatok štúdia v roku **{claim["start_year"]}**.') for key, label in [('keyword', 'kľúčové slovo'), ('category', 'kategória'), ('document_author', 'autor dokumentu')]: if claim.get(key) is not None: - parts.append(f'Pri osobe {person} je uvedené: {label} **{claim[key]}**.') + if key=='keyword': + if parts and (kind or title is not None or year is not None): + parts[-1]=parts[-1].rstrip('.')+f' a tag **{claim[key]}**.' + else:parts.append(f'{person} má tag **{claim[key]}**.') + else: + parts.append(f'{person}: '+('autor dokumentu je' if key=='document_author' else 'uvedená kategória je')+f' **{claim[key]}**.') return ' '.join(parts) or person + '.' @@ -158,7 +162,7 @@ def content_lines(units: list[str], person: str | None, kind: str | None, lines = [subject + ' má podľa dostupných podkladov tieto ciele:', '', *[f'{index}. {text}' for index, text in enumerate(goals, 1)]] if other: - lines.extend(['', 'Ďalšie podklady:', *other]) + lines.extend(['', *other]) return lines return [subject + ' je v dostupných podkladoch opísaná takto:' if not (kind or '').endswith('_project') else subject + ' je v dostupných podkladoch opísaný takto:', '', *units] @@ -167,6 +171,62 @@ def content_lines(units: list[str], person: str | None, kind: str | None, def no_answer_text(context: dict) -> str: people = context.get('requested_people') or context.get('requirements', {}).get('people', []) if len(people) == 1: - return (f'Pre osobu {people[0]} sa v dostupných podkladoch nepodarilo spoľahlivo nájsť ' - 'údaje zodpovedajúce tejto otázke.') - return 'K tejto otázke sa v dostupných podkladoch nepodarilo spoľahlivo nájsť zodpovedajúce údaje.' + req=context.get('requirements',{}) + years=context.get('requested_years') or req.get('years') or [] + kinds=context.get('requested_work_types') or [req.get('work_types',{}).get(people[0])] + genitive={'bachelor_thesis':'bakalárskej práce','diploma_thesis':'diplomovej práce', + 'doctoral_thesis':'dizertačnej práce','bachelor_project':'bakalárskeho projektu', + 'diploma_project':'diplomového projektu','team_project':'tímového projektu', + 'research_project':'výskumného projektu'} + kind=genitive.get(kinds[0],'práce alebo projektu') if len(kinds)==1 else 'práce alebo projektu' + if years: + date=(' z roku **'+str(years[0])+'**' if len(years)==1 else + ' z požadovaných rokov **'+', '.join(map(str,years))+'**') + return f'{people[0]}: v dostupných podkladoch sa nenašiel záznam {kind}{date}.' + return f'{people[0]}: požadované údaje sa v dostupných podkladoch nenašli.' + return 'Požadované údaje sa v dostupných podkladoch nenašli.' + + +def missing_aspects(facets: list[str], question: str = '') -> str: + """Name only the missing requested fields, without empty helper sections.""" + from .question_intent import planned_experiments + labels={'assignment':'zadaní a cieľoch','plans':'plánovaných úlohách','progress':'priebežnom stave', + 'results':'výsledkoch','problems':'riešených problémoch','methods':'metódach a technológiách', + 'experiments':('plánovaných experimentoch' if planned_experiments(question) else 'experimentoch a ich vyhodnotení')} + values=list(dict.fromkeys(labels.get(f,f) for f in facets)) + if not values:return '' + joined=values[0] if len(values)==1 else ', '.join(values[:-1])+' a '+values[-1] + return 'V poskytnutých podkladoch chýbajú údaje o '+joined+'.' + + +def clean_answer(lines: list[str]) -> str: + text='\n'.join(lines).strip() + return re.sub(r'\n{3,}','\n\n',text) + + +def evidence_subject(person: str, context: dict) -> str: + """Use only work identity actually present in the supplied passages.""" + req=context.get('requirements',{}) + passages=[p for p in context.get('content_passages',[]) if p.get('person')==person] + scopes={(p.get('work_type'),p.get('year')) for p in passages} + if not scopes:return person + kind=req.get('work_types',{}).get(person) or (next(iter(scopes))[0] if len(scopes)==1 else None) + subject=work_subject(person,kind,context.get('question','')) + if not owner_in_question(person,context.get('question','')): + subject=person+(' — '+WORK_LABELS[kind] if kind in WORK_LABELS else '') + if len(scopes)==1 and next(iter(scopes))[1] is not None: + subject+=' ('+str(next(iter(scopes))[1])+')' + labels={p.get('work_label') for p in passages if p.get('work_label')} + if req.get('project_number') is not None and len(labels)==1: + subject=person+' — '+' '.join(next(iter(labels)).split()) + return subject + + +def comparison_table(headers: list[str], rows: list[tuple[str,list[str]]]) -> list[str] | None: + """Lay out short literal evidence; keep complex text outside Markdown tables.""" + cells=[*headers,*(cell for label,values in rows for cell in [label,*values])] + if len(headers)>4 or any(len(cell)>900 or '|' in cell or '`' in cell or + re.search(r'<[^>]+>',cell) or text_urls(cell) for cell in cells): + return None + def row(values):return '| '+' | '.join(v.replace('\n','
') for v in values)+' |' + return [row(headers),row(['---']*len(headers)),*(row([label,*values]) for label,values in rows)] diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index 433bfea..411233e 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -280,6 +280,14 @@ def quote_matches_passage(quote: dict, passage: dict) -> bool: ' '.join(quote['quote'].split()) in ' '.join(passage['text'].split())) +def missing_content_people(answer: dict, context: dict) -> list[str]: + """Requested people without a selected, work-bound descriptive excerpt.""" + return [person for person in context.get('requirements',{}).get('people',[]) + if not any(p.get('person')==person and quote_matches_passage(q,p) + for p in context.get('content_passages',[]) + for q in answer.get('evidence_quotes',[]))] + + def quote_sections(quote: dict, context: dict) -> list[dict]: """Classify only literal excerpts bound to the delivered academic passage.""" full = [section for passage in context.get('content_passages', []) diff --git a/tools/answer_evaluation/render_answer.py b/tools/answer_evaluation/render_answer.py index b724015..e2dc034 100644 --- a/tools/answer_evaluation/render_answer.py +++ b/tools/answer_evaluation/render_answer.py @@ -3,7 +3,7 @@ from copy import deepcopy import re from tools.graphrag.graphrag import WORK_TYPE_LABELS -from .presentation import fact_sentences, no_answer_text +from .presentation import fact_sentences, no_answer_text, clean_answer def render_graph_answer(answer: dict, context: dict) -> dict: @@ -11,6 +11,7 @@ def render_graph_answer(answer: dict, context: dict) -> dict: lines = [] if out['status'] == 'ok': shared_fields=set() + shared_people=[] labels={'keyword':'kľúčové slová','category':'kategórie','document_author':'autori dokumentov'} for field,label in labels.items(): values=sorted({row['value'] for row in context.get('shared_properties',[]) if row['field']==field @@ -20,6 +21,11 @@ def render_graph_answer(answer: dict, context: dict) -> dict: shared_fields.add(field) prefix='Spoločné ' if context.get('complete') is True and not context.get('truncated') else 'Doložené spoločné ' lines.append(prefix+label+': '+', '.join('**'+v+'**' for v in values)+'.') + for row in context.get('shared_properties',[]): + if row['field']==field and row['value'] in values: + shared_people.extend(row['people']) + if shared_people: + lines.extend(['Doložené pri: '+', '.join('**'+p+'**' for p in dict.fromkeys(shared_people))+'.','']) comparison = context.get('academic_year_comparison') if comparison: people = comparison.get('people', []) @@ -81,5 +87,5 @@ def render_graph_answer(answer: dict, context: dict) -> dict: lines.append(no_answer_text(context)) if out['source_urls']: lines.extend(['', ('Zdroj: ' if len(out['source_urls']) == 1 else 'Zdroje: ') + ', '.join(out['source_urls'])]) - out['answer'] = '\n'.join(lines) + out['answer'] = clean_answer(lines) return out diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index bd20de2..28ff089 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -12,7 +12,8 @@ from .answer_format import Claim from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized, text_urls from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections, quote_matches_passage from .answer_coverage import answer_coverage, coverage_notices -from .presentation import content_lines, fact_sentences, no_answer_text, work_subject +from .presentation import (content_lines, fact_sentences, no_answer_text, work_subject, + clean_answer, evidence_subject, comparison_table) from .content_selection import facet_lines, evidence_units, missing_facets, fit_candidates, has_qualification @@ -200,7 +201,7 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: issues.append('rejected_supported_partial_fact') if answer['status'] == 'no_answer' and requirements.get('content'): query = normalized(context['question']) - overview = re.search(r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|opis\w*|popis\w*|obsah\w*|zhrn\w*)\b', query) + overview = re.search(r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|opis\w*|popis\w*|obsah\w*|zamer\w*|zhrn\w*)\b', query) # Only a broad overview is directly licensed by a documented assignment. # Do not reject a refusal about specific results/budgets merely because # some unrelated narrative was retrieved from the requested work. @@ -260,7 +261,8 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: all(claim.get(key) is None or claim[key] == p.get(key) for key in ['work_type', 'year']) for claim in claims)] eligible = any(c.get('person')==person for c in context.get('content_candidates', [])) - if (not requirements.get('facets') or eligible) and not any(quote_matches_passage(quote,p) + required = eligible if requirements.get('facets') else bool(passages) + if required and not any(quote_matches_passage(quote,p) for p in passages for quote in quotes): issues.append('missing_bound_content_evidence') if requirements.get('distinguish_progress') and not requirements.get('facets'): @@ -347,8 +349,8 @@ def comparison_lines(answer: dict, context: dict) -> list[str]: key = section['kind'] if section['kind'] in roles else 'other' roles[key].append({'source_id': quote['source_id'], 'quote': section['text']}) for role, label in [('assignment', 'Zadanie a ciele'), ('progress', 'Zaznamenaný stav a výsledky')]: - lines.append('**' + label + ':**') if roles[role]: + lines.append('**' + label + ':**') units = overview_excerpts({'evidence_quotes': roles[role]}, context) if role == 'assignment': lines.extend(content_lines(units, person, kind, context['question'])[1:]) @@ -359,13 +361,12 @@ def comparison_lines(answer: dict, context: dict) -> list[str]: ('nie je zadanie doložené.' if role == 'assignment' else 'nie sú dosiahnuté výsledky doložené.')) lines.append('') if roles['task']: - lines.append('**Ďalšie doložené úlohy (plán):**') + lines.append('**Plánované úlohy:**') units = overview_excerpts({'evidence_quotes': roles['task']}, context) lines.extend(content_lines(units, person, kind, context['question'])[1:]) lines.append('') if roles['other']: - lines.extend(['**Ďalšie doložené podklady:**', - *overview_excerpts({'evidence_quotes': roles['other']}, context), '']) + lines.extend([*overview_excerpts({'evidence_quotes': roles['other']}, context), '']) return lines @@ -408,7 +409,15 @@ def render_rag_answer(answer: dict, context: dict) -> str: if quote_matches_passage(quote,p)), {}) identity = (passage.get('person'), passage.get('work_type'), passage.get('year')) groups.setdefault(identity, []).append(selection['text']) - for (person, kind, _year), selections in groups.items(): + table = None + if style == 'overview' and len(groups)>1 and re.search(r'\bporovnaj\w*\b',normalized(context.get('question',''))): + from .presentation import goal_text + table=comparison_table(['Oblasť',*(evidence_subject(person,context) for person,_,_ in groups)], + [('Zameranie podľa podkladov',['\n'.join(goal_text(text)[0] for text in units) + for units in groups.values()])]) + if table is not None: + lines.extend(table) + for (person, kind, _year), selections in ([] if table is not None else groups.items()): if style == 'overview': units = selections else: @@ -426,6 +435,10 @@ def render_rag_answer(answer: dict, context: dict) -> str: ': „' + quote['quote'] + '“' for quote in quotes] if answer.get('count') is not None: lines.append(f'Počet: {answer["count"]}.') + if req.get('content') and not req.get('facets'): + from .rag_evidence import missing_content_people + for person in missing_content_people(answer,context): + lines.extend(['',f'{person}: v poskytnutých podkladoch chýba opis požadovanej práce alebo projektu.']) elif answer['status'] == 'no_answer': if context.get('requirements', {}).get('facets') and context['requirements']['people']: lines.extend(facet_lines(answer,context)) @@ -438,7 +451,7 @@ def render_rag_answer(answer: dict, context: dict) -> str: lines.extend(coverage_notices(answer_coverage(answer, context))) if answer.get('source_urls'): lines.extend(['', ('Zdroj: ' if len(answer['source_urls']) == 1 else 'Zdroje: ') + ', '.join(answer['source_urls'])]) - return '\n'.join(lines) + return clean_answer(lines) def generate_verified_rag(question: str, payload: dict, complete) -> dict: