From 151eca35715bddc556e445b600f58720e90debce Mon Sep 17 00:00:00 2001 From: jp170na Date: Wed, 4 Nov 2026 00:55:54 +0100 Subject: [PATCH] =?UTF-8?q?=C3=9Aplnej=C5=A1ie=20odpovede?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 17 +- test/test_content_selection_ids.py | 199 ++++++++++++++++++ test/test_evidence_selection.py | 2 +- test/test_quality_block.py | 2 +- test/test_rag_answer_coverage.py | 11 +- test/test_rag_work_content.py | 4 +- tools/answer_evaluation/answer_coverage.py | 3 + tools/answer_evaluation/content_selection.py | 14 +- tools/answer_evaluation/evidence_selection.py | 85 ++++++++ tools/answer_evaluation/question_intent.py | 15 ++ tools/answer_evaluation/rag_evidence.py | 10 +- tools/answer_evaluation/verified_rag.py | 70 ++++-- tools/answers/answer_delivery.py | 7 + tools/graphrag/evaluation/reproducibility.py | 1 + 14 files changed, 403 insertions(+), 37 deletions(-) create mode 100644 test/test_content_selection_ids.py create mode 100644 tools/answer_evaluation/evidence_selection.py diff --git a/README.md b/README.md index 087123d..940cd37 100644 --- a/README.md +++ b/README.md @@ -1484,13 +1484,16 @@ záznamov a orezané citované zadanie dostanú konkrétne upozornenie. Pri nen informácii sa zachová opatrné tvrdenie o dostupných podkladoch. Žiadny limit sa kvôli tomu nezvyšuje ani sa neprepisuje na `complete=true`. -Otázka na stručný obsah pri zobrazení vyberá najviac dva krátke overené výňatky -na osobu (do 600 znakov a dvoch bodov na osobu), bez voľnej modelovej parafrázy. -Skrátenie robí renderer z už overených citátov; dlhší platný citát nie je dôvod -na odmietnutie faktov ani ďalšie modelové volanie. Pri neoddeliteľnom technickom -reťazci má zachovanie dôkazu prednosť pred limitom stručnosti. Otázka na zadanie -môže zobraziť všetky doložené body zadania. Citácie a kontroly ich doslovnosti -sa uplatňujú v oboch režimoch. +Stručný opis zachová všetky dostupné hlavné body zadania; pevný limit dvoch bodov +sa na ne nevzťahuje. Zásobník úloh a história stretnutí nenahrádzajú hlavné zadanie. +Bez explicitného zadania sa použije obmedzený výber opisných podkladov. Model +vyberá `evidence_ids` z dodaných kandidátov a aplikácia dosadí ich presné texty +a zdroje. Neznámy identifikátor, chýbajúci hlavný bod alebo nesprávne citácie +neprejdú kontrolou. Všetky podklady podliehajú spoločnému tokenovému rozpočtu; +jeho skutočné obmedzenie dostane konkrétne upozornenie aj pri stručnom opise. +Rok v obsahovej otázke môže byť iba filtrom; výpis roku sa vyžaduje osobitne. +Grafový nástroj pri otázkach na textové zadanie, problémy alebo experimenty +odporučí `/rag`, pretože samotné uzly a vzťahy tieto textové dôkazy neobsahujú. Finálne formulovanie je oddelené v `tools/answer_evaluation/presentation.py`. Overené názvy a roky sa zobrazujú vo vetách, práce v číslovanom zozname. diff --git a/test/test_content_selection_ids.py b/test/test_content_selection_ids.py new file mode 100644 index 0000000..d58becc --- /dev/null +++ b/test/test_content_selection_ids.py @@ -0,0 +1,199 @@ +"""Cover the real comparison failures without relaxing factual evidence checks.""" +import copy +import json +import sqlite3 + +import pytest + +from test_rag_work_content import payload, QUESTION, URL +from test_quality_block import corpus, prepare, complete +from tools.answer_evaluation.evidence_selection import resolve_selection, EvidenceSelectionError +from tools.answer_evaluation.rag_evidence import rag_evidence_context +from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues +from tools.answer_evaluation.content_selection import fit_candidates +from tools.answer_evaluation.question_intent import graph_needs_text +from tools.answers import answer_delivery +from tools.rag.rag_utils import build_context_text + + +GOALS=['1. Vypracujte prehľad metód ohodnotenia dokumentov.', + '2. Implementujte systém získavania informácií v slovenskom jazyku.', + '3. Navrhnite a vypracujte experimenty, v ktorých vyhodnotíte vybrané metódy.', + '4. Navrhnite možné zlepšenia presnosti vyhľadávania.'] + + +@pytest.fixture +def four_goals(payload): + payload['sources']=payload['sources'][:1] + payload['sources'][0]['text']='Sekcia: Diplomová práca 2021\nNázov: Vyhľadávanie informácií.\nZadanie:\n'+'\n'.join(GOALS) + payload['context']=build_context_text(payload['sources']) + return payload + + +def id_answer(context): + return {'status':'ok','answer':'','claims':[],'evidence_quotes':[], + 'evidence_ids':[row['evidence_id'] for row in context['content_candidates']], 'source_urls':[]} + + +@pytest.mark.parametrize('question',[QUESTION,'Aké je celé zadanie diplomovej práce Jána Holpa?']) +def test_all_four_goals_are_delivered_with_exact_evidence_and_automatic_citation(four_goals,question): + c=rag_evidence_context(question,four_goals);a=id_answer(c) + assert len(a['evidence_ids'])==4 + before=copy.deepcopy((a,c));expanded=resolve_selection(a,c) + assert (a,c)==before and expanded['source_urls']==[URL] + assert [q['quote'] for q in expanded['evidence_quotes']]==GOALS + r=generate_verified_rag(question,four_goals,complete(a)) + assert r['accepted_for_delivery'] and len(r['attempts'])==1 + assert '3. Návrh a vypracovanie experimentov' in r['answer'] + assert '4. Návrh možných zlepšení presnosti vyhľadávania.' in r['answer'] + if r['answer_coverage']['scope']=='assignment': + assert r['answer_coverage']['status']=='partial' + assert 'assignment_completeness_unknown' in r['answer_coverage']['warnings'] + else:assert r['answer_coverage']['status']=='supported' + assert 'Výpis je neúplný' not in r['answer'] + assert r['attempts'][0]['selected_answer']['evidence_ids']==a['evidence_ids'] + + +def test_two_of_four_goals_cannot_be_silently_accepted_as_an_overview(four_goals): + c=rag_evidence_context(QUESTION,four_goals);a=id_answer(c);a['evidence_ids']=a['evidence_ids'][:2] + r=generate_verified_rag(QUESTION,four_goals,complete(a)) + assert not r['accepted_for_delivery'] and len(r['attempts'])==2 + assert all('missing_assignment_evidence' in attempt['issues'] for attempt in r['attempts']) + + +def test_unknown_id_is_rejected_in_both_attempts(four_goals): + a=id_answer(rag_evidence_context(QUESTION,four_goals));a['evidence_ids']=['Eunknown'] + r=generate_verified_rag(QUESTION,four_goals,complete(a)) + assert not r['accepted_for_delivery'] + assert all(attempt['issues']==['unknown_evidence_id'] for attempt in r['attempts']) + + +def test_ids_cannot_smuggle_additional_quotes_or_urls(four_goals): + c=rag_evidence_context(QUESTION,four_goals);a=id_answer(c) + both={**a,'evidence_quotes':[{'source_id':'S1','quote':GOALS[0]}]} + with pytest.raises(EvidenceSelectionError,match='mixed_evidence_selection'): + resolve_selection(both,c) + a['source_urls']=['https://invented.test/result'] + r=generate_verified_rag(QUESTION,four_goals,complete(a)) + assert not r['accepted_for_delivery'] and 'unsupported_citation' in r['attempts'][-1]['issues'] + + +def test_duplicate_ids_do_not_duplicate_facts(four_goals): + c=rag_evidence_context(QUESTION,four_goals);a=id_answer(c);a['evidence_ids']*=2 + resolved=resolve_selection(a,c) + assert len(resolved['evidence_quotes'])==4 and resolved['source_urls']==[URL] + a['evidence_ids'].reverse() + assert [q['quote'] for q in resolve_selection(a,c)['evidence_quotes']]==GOALS + + +def test_model_does_not_receive_duplicate_full_sources_or_need_to_copy_markdown(four_goals): + def select(messages): + c=json.loads(messages[1]['content'])['context'] + assert 'content_passages' not in c and 'context' not in c + assert all('text' not in row for row in c['sources']) + assert all(row['required'] for row in c['content_candidates']) + return complete(id_answer(c))(messages) + assert generate_verified_rag(QUESTION,four_goals,select)['accepted_for_delivery'] + + +def test_primary_goals_are_not_extended_with_the_task_backlog(four_goals): + four_goals['sources'][0]['text']+='\nZásobník úloh:\n- Implementujte neskorší doplnkový modul.' + four_goals['context']=build_context_text(four_goals['sources']) + c=rag_evidence_context(QUESTION,four_goals) + assert len(c['content_candidates'])==4 + r=generate_verified_rag(QUESTION,four_goals,complete(id_answer(c))) + assert r['accepted_for_delivery'] and 'doplnkový modul' not in r['answer'] + + +@pytest.mark.parametrize('heading',['Napr.','Príklady:']) +def test_illustrative_examples_after_assignment_are_not_extra_mandatory_goals(four_goals,heading): + four_goals['sources'][0]['text']+='\n\n'+heading+'\n\n- osnovy pre výuku na základnej škole.' + four_goals['context']=build_context_text(four_goals['sources']) + c=rag_evidence_context(QUESTION,four_goals) + assert len(c['content_candidates'])==4 + r=generate_verified_rag(QUESTION,four_goals,complete(id_answer(c))) + assert r['accepted_for_delivery'] and 'osnovy pre výuku' not in r['answer'] + + +def test_qualified_goal_is_kept_whole(four_goals): + text='- Implementujte parser. Výsledok sa smie použiť iba na testovacom korpuse; nejde o overenie produkcie.' + four_goals['sources'][0]['text']+='\n'+text + four_goals['context']=build_context_text(four_goals['sources']) + c=rag_evidence_context(QUESTION,four_goals);r=generate_verified_rag(QUESTION,four_goals,complete(id_answer(c))) + assert r['accepted_for_delivery'] + assert 'iba na testovacom korpuse; nejde o overenie produkcie' in r['answer'] + + +def test_budgeted_out_id_cannot_be_resolved_and_limited_overview_has_a_specific_notice(four_goals): + c=rag_evidence_context(QUESTION,four_goals);original=c['content_candidates'][:] + sent={'requirements':c['requirements'],'content_candidates':original} + fit_candidates(sent,c,450,'cl100k_base') + dropped=[row for row in original if row not in c['content_candidates']] + assert dropped and c['truncated'] and 'answer_candidate_budget' in c['truncation_reasons'] + with pytest.raises(EvidenceSelectionError,match='unknown_evidence_id'): + resolve_selection({**id_answer(c),'evidence_ids':[dropped[0]['evidence_id']]},c) + from tools.answer_evaluation.answer_coverage import answer_coverage,coverage_notices + a=resolve_selection(id_answer(c),c) + coverage=answer_coverage(a,c) + assert coverage['status']=='partial' and 'overview_aspects_clipped' in coverage['warnings'] + assert any('prehľad preto nemusí pokrývať všetky oblasti' in n for n in coverage_notices(coverage)) + + +def test_an_indivisible_overlimit_goal_does_not_force_an_answer_the_model_never_received(four_goals): + four_goals['sources'][0]['text']='Sekcia: Diplomová práca 2021\nZadanie:\n- Implementujte parser. '+('Platí iba pre testovací korpus. '*100) + four_goals['context']=build_context_text(four_goals['sources']) + c=rag_evidence_context(QUESTION,four_goals) + assert not c['content_candidates'] and c['truncated'] + a={'status':'no_answer','claims':[],'evidence_quotes':[],'source_urls':[]} + r=generate_verified_rag(QUESTION,four_goals,complete(a)) + assert r['accepted_for_delivery'] and r['status']=='no_answer' + assert 'Časť podkladov bola pri príprave kontextu skrátená.' in r['answer'] + + +@pytest.mark.parametrize('tail,want_year',[('',False),(' Uveď aj rok práce.',True)]) +def test_year_filter_is_preserved_without_forcing_a_year_claim(corpus,tail,want_year): + q='Aké problémy riešila Eva Nová pri diplomovej práci v roku 2024?'+tail + p=prepare(corpus,q);c=rag_evidence_context(q,p) + assert c['requirements']['years']==[2024] and c['requirements']['year']==want_year + assert all(row['year']==2024 for row in c['content_candidates']) + if not want_year: + r=generate_verified_rag(q,p,complete(id_answer(c))) + assert r['accepted_for_delivery'] and r['claims']==[] + + +def test_comparison_cannot_replace_one_persons_evidence_with_another(corpus): + q='Porovnaj výsledky Evy Novej a Borisa Nového.' + p=prepare(corpus,q);c=rag_evidence_context(q,p);a=id_answer(c) + a['evidence_ids']=[row['evidence_id'] for row in c['content_candidates'] if row['person']=='Eva Nová'] + r=generate_verified_rag(q,p,complete(a)) + assert not r['accepted_for_delivery'] and 'missing_progress_evidence' in r['attempts'][-1]['issues'] + + +def test_partial_refusal_gets_corrected_without_inventing_the_missing_persons_evidence(corpus): + db,_=corpus + with sqlite3.connect(db) as conn: + conn.execute("UPDATE chunks SET text='Zadanie:\n- Implementujte parser.' WHERE document_path LIKE '%boris_novy%'") + q='Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?' + p=prepare(corpus,q);c=rag_evidence_context(q,p);calls=[] + def select(messages): + calls.append(copy.deepcopy(messages)) + assert 'Doložené časti po osobách' in messages[0]['content'] + assert 'Eva Nová' not in messages[0]['content'] and 'Boris Nový' not in messages[0]['content'] + return complete({'status':'no_answer','claims':[],'evidence_quotes':[],'source_urls':[]} + if len(calls)==1 else id_answer(c))(messages) + r=generate_verified_rag(q,p,select) + assert r['accepted_for_delivery'] and r['answer_coverage']['status']=='partial' and len(calls)==2 + assert 'rejected_supported_content_comparison' in r['attempts'][0]['issues'] + assert 'Boris Nový' in r['answer'] and 'chýbajú údaje o riešených problémoch' in r['answer'] + assert len(r['source_urls'])==2 and r['claims']==[] + + +def test_graph_narrative_request_gets_specific_rag_guidance_without_a_model(monkeypatch): + def forbidden(*args,**kwargs):pytest.fail('Graph/text mismatch must not invoke retrieval or a model') + monkeypatch.setattr(answer_delivery,'search_graph',forbidden) + q='Aké problémy riešili Ján Pták a Ján Malinovský pri diplomovom projekte v roku 2026 a aké experimenty mali pripraviť?' + r=answer_delivery.deliver_answer(q,engine='graphrag',complete=forbidden) + assert r['status']=='ambiguous' and r['reason']=='text_evidence_required' + assert r['accepted_for_delivery'] and '/rag' in r['answer'] and r['claims']==[] + assert not graph_needs_text('Ktorí študenti majú tag RAG a zároveň diplomový projekt z roku 2026?') + assert not graph_needs_text('Aké spoločné kľúčové slová majú Ján Pták a Ján Malinovský?') diff --git a/test/test_evidence_selection.py b/test/test_evidence_selection.py index 37c2724..4280f50 100644 --- a/test/test_evidence_selection.py +++ b/test/test_evidence_selection.py @@ -166,7 +166,7 @@ def test_explicit_project_number_binds_the_excerpt_to_that_project(corpus): p=prepare(corpus,q);c=rag_evidence_context(q,p) assert p['status']=='ok' and 'Cassandra' in p['context'] and 'BERT' not in p['context'] assert all('projekt 2' in u['work_label'] for u in evidence_units(c)) - a=selection(c);a['claims']=[{'person':'Eva Nová','work_type':'diploma_project','year':2022}] + a=selection(c) result=generate_verified_rag(q,p,complete(a)) assert result['accepted_for_delivery'] and 'projekt 2' in result['answer'] diff --git a/test/test_quality_block.py b/test/test_quality_block.py index 950ad57..d24b9f9 100644 --- a/test/test_quality_block.py +++ b/test/test_quality_block.py @@ -214,7 +214,7 @@ def test_plans_cannot_answer_actual_use_or_completed_experiments(): def test_method_answer_does_not_add_valid_but_unrequested_profile_metadata(corpus): _,raw=corpus raw['sources'][0]['author']='Daniel Hladek' - q='Aké metódy používala Eva Nová pri diplomovej práci v roku 2024?' + q='Aké metódy používala Eva Nová pri diplomovej práci v roku 2024? Uveď aj rok práce.' p=prepare(corpus,q); a=model_selection(q,p) a['claims']=[{'person':'Eva Nová','work_type':'diploma_thesis','year':2024, 'document_author':'Daniel Hladek'}] diff --git a/test/test_rag_answer_coverage.py b/test/test_rag_answer_coverage.py index fc269cd..fe1aa70 100644 --- a/test/test_rag_answer_coverage.py +++ b/test/test_rag_answer_coverage.py @@ -137,7 +137,7 @@ def test_exact_aggregate_can_be_supported_despite_limited_evidence(payload): assert 'count_not_established' in answer_coverage(a, c)['warnings'] -def test_long_valid_overview_is_shortened_without_another_model_request(payload): +def test_overview_preserves_all_main_assignment_items_without_another_model_request(payload): question = 'Čomu sa venuje diplomová práca Evy Novej?' short = '1. Preskúmanie metód vyhľadávania.\n2. Implementácia vyhľadávača.' c = rag_evidence_context(question, payload) @@ -148,7 +148,8 @@ def test_long_valid_overview_is_shortened_without_another_model_request(payload) return {'choices': [{'message': {'content': json.dumps(quote(GOALS))}}]} result = generate_verified_rag(question, payload, complete) assert result['accepted_for_delivery'] and len(result['attempts']) == len(calls) == 1 - assert short in result['answer'] and '3.' not in result['answer'] + assert short in result['answer'] and '3. Vyhodnotenie presnosti.' in result['answer'] + assert '4. Návrh zlepšení.' in result['answer'] assert result['attempts'][0]['parsed_answer']['evidence_quotes'][0]['quote'] == GOALS assert result['complete'] is False and result['truncation_reasons'] == payload['truncation_reasons'] assert result['answer_coverage']['status'] == 'supported' and result['answer_coverage']['warnings'] == [] @@ -163,7 +164,7 @@ def test_short_comparison_can_cover_three_people_instead_of_dropping_one(payload 'source_url': f'https://example.test/{slug}', 'document_path': f'pages/students/2020/{slug}/README.md'} sources.append(source) - quotes.append({'source_id': source['source_id'], 'quote': '2. Implementujte vyhľadávač.'}) + quotes.append({'source_id': source['source_id'], 'quote': GOALS}) payload['sources'] = sources; payload['context'] = build_context_text(sources) c = rag_evidence_context('Porovnaj obsah diplomových prác osôb Eva Nová, Boris Nový a Cyril Malý.', payload) assert len(c['requirements']['people']) == 3 @@ -184,7 +185,7 @@ def test_overview_renderer_only_selects_literal_units_and_preserves_full_evidenc before = copy.deepcopy(a) assert rag_answer_issues(a, c) == [] units = overview_excerpts(a, c) - assert len(units) == 2 and sum(map(len, units)) <= 600 + assert len(units) == 4 and all(line in units for line in GOALS.splitlines()) assert all(any(unit in evidence['quote'] for evidence in a['evidence_quotes']) for unit in units) assert a == before @@ -192,7 +193,7 @@ def test_overview_renderer_only_selects_literal_units_and_preserves_full_evidenc def test_long_prose_excerpt_is_selected_without_inventing_a_summary(payload): from tools.answer_evaluation.verified_rag import overview_excerpts prose = 'Práca skúma vyhľadávanie v slovenských dokumentoch. ' * 18 - payload['sources'][0]['text'] += '\n' + prose + payload['sources'][0]['text'] = 'Sekcia: Diplomová práca 2026\nNázov: Vyhľadávanie informácií.\n'+prose payload['context'] = build_context_text(payload['sources']) c = rag_evidence_context('Čomu sa venuje diplomová práca Evy Novej?', payload) a = quote(prose) diff --git a/test/test_rag_work_content.py b/test/test_rag_work_content.py index 3545cf1..9f81c97 100644 --- a/test/test_rag_work_content.py +++ b/test/test_rag_work_content.py @@ -150,7 +150,9 @@ def test_correction_sends_real_description_but_no_unrelated_sections(payload): assert result['accepted_for_delivery'] and len(calls) == 2 sent = json.loads(calls[0][1]['content'])['context'] assert 'facts' not in sent and 'context' not in sent - text = json.dumps(sent['content_passages'], ensure_ascii=False) + text = json.dumps(sent['content_candidates'], ensure_ascii=False) + assert all(c.get('evidence_id') for c in sent['content_candidates']) + assert all('text' not in s for s in sent['sources']) assert 'ohodnotenia dokumentov' in text and 'Cassandra' not in text and 'medicínskych' not in text assert 'missing_content_evidence' in result['attempts'][0]['issues'] assert 'Vypracovanie prehľadu metód ohodnotenia dokumentov' in result['answer'] diff --git a/tools/answer_evaluation/answer_coverage.py b/tools/answer_evaluation/answer_coverage.py index 025beec..60b32a5 100644 --- a/tools/answer_evaluation/answer_coverage.py +++ b/tools/answer_evaluation/answer_coverage.py @@ -59,6 +59,8 @@ def answer_coverage(answer: dict, context: dict) -> dict: from .rag_evidence import missing_content_people if missing_content_people(answer,context): coverage='partial'; warnings.append('partial_requested_aspects') + if scope=='overview' and (clipped or context_limited): + coverage='partial';warnings.append('overview_aspects_clipped') if scope == 'list' and (limited or clipped): coverage = 'partial'; warnings.append('list_may_be_incomplete') elif scope == 'count' and (context.get('aggregation') or {}).get('complete') is not True: @@ -79,6 +81,7 @@ def answer_coverage(answer: dict, context: dict) -> dict: def coverage_notices(coverage: dict) -> list[str]: messages = { + 'overview_aspects_clipped':'Časť podkladov bola skrátená; prehľad preto nemusí pokrývať všetky oblasti práce.', 'list_may_be_incomplete': 'Uvedený zoznam nemusí obsahovať všetky záznamy.', 'assignment_text_clipped': 'Časť podkladov k zadaniu bola skrátená; uvedené body nemusia tvoriť celé zadanie.', 'assignment_completeness_unknown': 'Z dostupných podkladov nemožno zaručiť všetky body zadania.', diff --git a/tools/answer_evaluation/content_selection.py b/tools/answer_evaluation/content_selection.py index c753a66..ee636cb 100644 --- a/tools/answer_evaluation/content_selection.py +++ b/tools/answer_evaluation/content_selection.py @@ -160,6 +160,7 @@ def evidence_units(context: dict, quotes: list[dict] | None = None) -> list[dict def candidates(context: dict) -> list[dict]: + from .evidence_selection import identify facets = context['requirements'].get('facets', []) selected = []; seen = set(); units = evidence_units(context) for person in context['requirements']['people']: @@ -167,7 +168,7 @@ def candidates(context: dict) -> list[dict]: relevant = [u for u in units if u['person']==person and facet in u['facets']] if facet=='progress' and 'results' in facets and any(u['phase']=='progress' for u in relevant): relevant=[u for u in relevant if u['phase']=='progress'] - relevant.sort(key=lambda u: -unit_score(u['text'],u['kind'],facet)) + if facet!='assignment':relevant.sort(key=lambda u: -unit_score(u['text'],u['kind'],facet)) accepted=0 for unit in relevant: text = unit['text'] @@ -175,11 +176,12 @@ def candidates(context: dict) -> list[dict]: context['complete']=False; context['truncated']=True context['truncation_reasons']=list(dict.fromkeys([*(context.get('truncation_reasons') or []),'evidence_unit_limit'])) continue - key=(unit['source_id'],text) + key=(unit.get('passage_id'),unit['source_id'],text,unit['phase']) if key not in seen: - seen.add(key); selected.append({**unit,'text':text}) + seen.add(key); selected.append(identify({**unit,'text':text}, + required=unit['kind']=='assignment' and 'assignment' in facets)) accepted+=1 - if accepted==2: break + if accepted==2 and facet!='assignment': break return selected @@ -229,7 +231,7 @@ def facet_lines(answer: dict, context: dict) -> list[str]: rows=[u for u in units if u['person']==person and facet in u['facets']] if facet=='progress' and 'results' in facets and any(u['phase']=='progress' for u in rows): rows=[u for u in rows if u['phase']=='progress'] - rows.sort(key=lambda u:-unit_score(u['text'],u['kind'],facet)) + if facet!='assignment':rows.sort(key=lambda u:-unit_score(u['text'],u['kind'],facet)) if not rows:group['missing'].append(facet) rendered=[] for row in rows: @@ -243,7 +245,7 @@ def facet_lines(answer: dict, context: dict) -> list[str]: phase=' (plán)' if row['kind'] in {'assignment','task'} and facet not in {'assignment','plans'} else '' state='Zaznamenaný výsledok: ' if row['phase']=='result' and 'results' in facets and facet!='results' else '' rendered.append(state+text+phase);seen.add(key) - if len(rendered)==2:break + if len(rendered)==2 and facet!='assignment':break group['items'][facet]=rendered groups.append(group) if len(groups)>1 and re.search(r'\bporovnaj\w*\b',normalized(context.get('question',''))): diff --git a/tools/answer_evaluation/evidence_selection.py b/tools/answer_evaluation/evidence_selection.py new file mode 100644 index 0000000..5edc2ea --- /dev/null +++ b/tools/answer_evaluation/evidence_selection.py @@ -0,0 +1,85 @@ +"""Select literal evidence by opaque IDs; never repair a model's factual prose.""" +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +import re + + +class EvidenceSelectionError(ValueError): + pass + + +def identify(row: dict, *, required: bool = False) -> dict: + payload={k:row.get(k) for k in ['source_id','passage_id','person','work_type','year', + 'work_label','text','kind','phase']} + digest=hashlib.sha256(json.dumps(payload,sort_keys=True,ensure_ascii=False).encode()).hexdigest()[:24] + return {**row,'evidence_id':'E'+digest,'required':required} + + +def overview_candidates(context: dict) -> list[dict]: + """Keep every main assignment item; do not confuse a backlog with goals.""" + from .content_selection import evidence_units, administrative_only, unit_score + rows=evidence_units(context); selected=[] + for person in context['requirements']['people']: + own=[row for row in rows if row['person']==person] + main=[row for row in own if row['kind']=='assignment'] + # Without an explicit assignment, retain a bounded descriptive sample. + # A plan or meeting history cannot become the main assignment. + fallback=[row for row in own if row['kind'] in {'other','progress'} and not administrative_only(row['text'])] + fallback.sort(key=lambda row:-unit_score(row['text'],row['kind'])) + for row in main or fallback[:2]: + if len(row['text'])>2500: + context['complete']=False;context['truncated']=True + context['truncation_reasons']=list(dict.fromkeys([*(context.get('truncation_reasons') or []),'evidence_unit_limit'])) + continue + selected.append(identify(row,required=bool(main))) + return selected + + +def resolve_selection(answer: dict, context: dict) -> dict: + """Expand only IDs from the actual budgeted model input, keeping raw data intact.""" + out=deepcopy(answer); ids=out.pop('evidence_ids',[]) + if not ids:return out + if out.get('evidence_quotes'): + raise EvidenceSelectionError('mixed_evidence_selection') + choices={row['evidence_id']:row for row in context.get('content_candidates',[]) if row.get('evidence_id')} + if any(identifier not in choices for identifier in ids): + raise EvidenceSelectionError('unknown_evidence_id') + wanted=set(ids) + chosen=[row for row in context.get('content_candidates',[]) if row.get('evidence_id') in wanted] + out['evidence_quotes']=[{k:row[k] for k in ['source_id','passage_id']}|{'quote':row['text']} for row in chosen] + sources={row['source_id']:row for row in context.get('sources',[])} + urls=[sources[row['source_id']]['source_url'] for row in chosen + if sources.get(row['source_id'],{}).get('source_url')] + # Extra model-supplied URLs remain visible to the citation guard. + out['source_urls']=list(dict.fromkeys([*out.get('source_urls',[]),*urls])) + return out + + +def missing_required_evidence(answer: dict, context: dict) -> list[str]: + """A short overview must still cover the supplied main assignment items.""" + def plain(text): + return ' '.join(re.sub(r'^[ \t]*(?:\d+[.)]|[-+*])\s+','',text,flags=re.M).split()) + quotes=answer.get('evidence_quotes',[]) + return [row['evidence_id'] for row in context.get('content_candidates',[]) if row.get('required') + and not any(q['source_id']==row['source_id'] and + (not q.get('passage_id') or q['passage_id']==row['passage_id']) and + plain(row['text']) in plain(q['quote']) for q in quotes)] + + +def overview_units(answer: dict, context: dict) -> list[dict]: + """Render all checked assignment items and at most two fallback units per person.""" + from .content_selection import evidence_units + rows=evidence_units(context,answer.get('evidence_quotes',[])) + selected=[] + for person in context.get('requirements',{}).get('people',[]): + own=[row for row in rows if row['person']==person] + main=[row for row in own if row['kind']=='assignment'] + for row in main or own[:2]: + quote=next(q for q in answer['evidence_quotes'] if q['source_id']==row['source_id'] and + ' '.join(row['text'].split()) in ' '.join(q['quote'].split()) and + (not q.get('passage_id') or q['passage_id']==row['passage_id'])) + selected.append({'evidence':quote,'text':row['text']}) + return selected diff --git a/tools/answer_evaluation/question_intent.py b/tools/answer_evaluation/question_intent.py index 624f806..5c62fdf 100644 --- a/tools/answer_evaluation/question_intent.py +++ b/tools/answer_evaluation/question_intent.py @@ -34,6 +34,21 @@ def facet_mode(question: str) -> bool: return any(facet in facets for facet in ['plans','progress','problems','methods','experiments','results']) +def output_year_requested(question: str) -> bool: + # A supplied year scopes the evidence; it does not itself ask for a year claim. + text=normalized(question) + without_filters=re.sub(r'\b(?:v|z|pre|na)\s+rok(?:u)?\s+\d+\b',' ',text) + return bool(re.search(r'\brok\w*\b',without_filters)) + + +def graph_needs_text(question: str) -> bool: + text=normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]',' ',question)) + # Explicit graph properties stay on their existing path. + if re.search(r'\b(?:tag\w*|klucov\w*|kategor\w*|tem\w*|nazov\w*|autor\w*)\b',text): + return False + return bool(set(requested_facets(question)) & {'assignment','plans','problems','experiments','progress','results'}) + + def planned_experiments(question: str) -> bool: text=normalized(question) if re.search(r'\b(?:vykonal\w*|vykonan\w*|uskutocnil\w*|uskutocnen\w*|realizoval\w*|realizovan\w*|otestoval\w*|vyhodnotil\w*)\b',text): diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index 411233e..46e1da8 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -11,7 +11,7 @@ import sqlite3 import hashlib from contextlib import closing from .context_policy import normalized, text_contains_value, person_pattern -from .question_intent import requested_facets, content_question, facet_mode +from .question_intent import requested_facets, content_question, facet_mode, output_year_requested from tools.retrieval.question_constraints import requested_years WORK_TYPES = { @@ -262,6 +262,8 @@ def content_sections(text: str) -> list[dict]: next_kind = 'progress' elif heading and role_value in {'ulohy', 'zasobnik uloh', 'dalsie ulohy', 'plan', 'planovane ulohy'}: next_kind = 'task' + elif heading and role_value in {'napr','napriklad','priklady','priklad'} and not line[:1].isspace(): + next_kind = 'other' elif value.startswith(('stretnutie ', 'virtualne stretnutie ', 'sekcia ')): next_kind = 'other' if next_kind is not None: @@ -458,7 +460,8 @@ def rag_evidence_context(question: str, payload: dict) -> dict: requirements = {'people': requested, 'work_types': expected_types, 'academic': academic, 'content': content, 'title': bool(re.search(r'\bnaz(?:ov|v)\w*\b', query)) or ('tema' in query and not content), - 'year': academic and ('rok' in query or ('porovnaj' in query and not content)), + 'year': academic and (output_year_requested(question) if content else + ('rok' in query or 'porovnaj' in query)), 'start_year': start_year, 'response_mode': 'claims' if academic or start_year else 'quotes'} if content: @@ -489,6 +492,9 @@ def rag_evidence_context(question: str, payload: dict) -> dict: if requirements.get('facets'): from .content_selection import candidates context['content_candidates'] = candidates(context) + elif requirements['content_style'] in {'overview','assignment'}: + from .evidence_selection import overview_candidates + context['content_candidates'] = overview_candidates(context) if requirements['distinguish_progress']: requirements['content_roles'] = {person: sorted({section['kind'] for p in context['content_passages'] if p['person'] == person diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index 28ff089..b945307 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -15,6 +15,8 @@ from .answer_coverage import answer_coverage, coverage_notices from .presentation import (content_lines, fact_sentences, no_answer_text, work_subject, clean_answer, evidence_subject, comparison_table) from .content_selection import facet_lines, evidence_units, missing_facets, fit_candidates, has_qualification +from .evidence_selection import (resolve_selection, EvidenceSelectionError, + missing_required_evidence, overview_units) class RagClaim(Claim): @@ -35,6 +37,7 @@ class RagAnswer(BaseModel): status: Literal['ok', 'no_answer', 'ambiguous', 'error'] claims: list[RagClaim] = Field(default_factory=list) evidence_quotes: list[EvidenceQuote] = Field(default_factory=list) + evidence_ids: list[str] = Field(default_factory=list,max_length=256) source_urls: list[str] = Field(default_factory=list) count: int | None = None @@ -58,10 +61,10 @@ Vyber opisujúce evidence_quotes z content_passages zodpovedajúcej osoby, typu Samotný názov, rok, meno vedúceho ani nadpis zadania nenahrádzajú opis. Pri response_mode=mixed uveď požadované claims aj opisujúce evidence_quotes. Použi iba dodané content_passages; opis inej práce alebo iného roku nie je odpoveď. -Pri requirements.content_style=overview vyber najviac DVA krátke doslovné výňatky -na požadovanú osobu, najviac 600 znakov a dve odrážky/body na osobu. Odpovedaj stručne na obsah; -neprepisuj celé zadanie a históriu stretnutí. Pri content_style=assignment zachovaj -požadované body zadania; tento stručný limit sa na zadanie nevzťahuje. +Pri requirements.content_style=overview zachovaj všetky hlavné oblasti zadania každej osoby. +Stručnosť nesmie vynechať jeho ďalšie body. Neprepisuj celú históriu stretnutí. +Pri content_style=assignment zachovaj všetky dostupné body hlavného zadania; +zásobník ďalších úloh nie je hlavné zadanie. Pri requirements.distinguish_progress=true porovnaj dostupné podklady každej osoby: vyber samostatne výňatok zadania/cieľov a samostatne výňatok zaznamenaného stavu. requirements.content_roles uvádza, ktoré z týchto podkladov sú naozaj dostupné pre každú osobu. @@ -71,7 +74,11 @@ aplikácia uvedie, čo chýba. no_answer znamená žiadnu odpoveď a musí mať Pri zaznamenanom stave uprednostni konkrétnu implementáciu, natrénovanie modelu a vykonané vyhodnotenie pred konzultáciami alebo samotným písaním textu. Ani priebežný stav nevydávaj za splnenie všetkých cieľov alebo za konečné výsledky práce. -Ak sú dodané content_candidates, vyber quotes iba z ich textu. Každý kandidát uvádza +Ak sú dodané content_candidates, vyplň evidence_ids identifikátormi evidence_id vybraných +kandidátov a evidence_quotes nechaj prázdne. Texty neprepisuj: aplikácia ich presne dosadí +aj s citáciami. Pri response_mode=quotes nechaj claims=[] a source_urls=[]. +Vždy vyber každý kandidát s required=true. Neznámy identifikátor ani nový citát nevytváraj. +Každý kandidát uvádza osobu, typ práce, rok, work_label, passage_id, kind, phase a facets, ktoré dokladá. Pri evidence_quotes zachovaj passage_id kandidáta alebo dodanej content_passage. Rovnaký text pod inou prácou, rokom alebo sekciou nie je zameniteľný dôkaz. @@ -99,7 +106,9 @@ def parse_rag_answer(text: str) -> dict: if not matches: raise ValueError('Invalid JSON fence') text = matches[1] - return RagAnswer.model_validate(json.loads(text)).model_dump(exclude_none=True) + parsed=RagAnswer.model_validate(json.loads(text)).model_dump(exclude_none=True) + if not parsed.get('evidence_ids'):parsed.pop('evidence_ids',None) + return parsed def quoted_reference_urls(answer: dict, context: dict) -> set[str]: @@ -206,8 +215,9 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: # Do not reject a refusal about specific results/budgets merely because # some unrelated narrative was retrieved from the requested work. specific = re.search(r'\b(?:vysledk\w*|dosiahn\w*|rozpoc\w*|financ\w*)\b', query) + delivered_descriptions=context.get('content_candidates',context.get('content_passages',[])) if overview and not specific and any(re.search(r'\b(?:zadanie|ciel\w*|venuj\w*|ries\w*|implement\w*|vypracuj\w*)\b', - normalized(passage['text'])) for passage in context.get('content_passages', [])): + normalized(passage['text'])) for passage in delivered_descriptions): issues.append('rejected_supported_work_description') cited = set(answer.get('source_urls', [])) blocks = {row['source_id']: row for row in context['source_blocks']} @@ -254,6 +264,8 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: if not citation_check({'answer': quote['quote'], 'source_urls': []}, allowed_urls(context) | references)['citations_ok']: issues.append('unsupported_quote_citation') if requirements.get('content') and answer['status'] == 'ok': + if missing_required_evidence(answer,context): + issues.append('missing_assignment_evidence') for person in requested: passages = [p for p in context.get('content_passages', []) if p.get('person') == person] if requirements['response_mode'] == 'mixed' and any(claim.get('person')==person for claim in claims): @@ -297,6 +309,9 @@ def _overview_selections(answer: dict, context: dict) -> list[dict]: Presentation length is not a reason to reject otherwise supported evidence or to spend another model request rewriting the same facts. """ + if context.get('requirements',{}).get('content_style')=='overview': + selected=overview_units(answer,context) + if selected:return selected selected = []; counts = {}; sizes = {}; seen = set() for quote in answer.get('evidence_quotes', []): sid = quote['source_id'] @@ -479,19 +494,41 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: sent = {key: context[key] for key in ['status', 'reason', 'complete', 'truncated', 'truncation_reasons', 'index_version', 'sources', 'facts', 'requirements']} if context['requirements'].get('content'): - if context['requirements'].get('facets'): + sent['sources']=[{key:source[key] for key in ['source_id','source_url','title','document_path'] if key in source} + for source in context['sources']] + if 'content_candidates' in context: sent['content_candidates']=context['content_candidates'] - context['candidate_only']=True + if context['requirements'].get('facets'):context['candidate_only']=True else: sent['content_passages'] = context['content_passages'] elif not context['requirements']['academic']: sent['context'] = payload.get('context', '') if context['requirements']['response_mode'] == 'quotes': sent.pop('facts') - if context['requirements'].get('facets'): + if 'content_candidates' in context: fit_candidates(sent,context,payload.get('context_budget',{}).get('max_tokens',12000), os.getenv('RAG_LLM_TOKEN_ENCODING','cl100k_base')) - messages = [{'role': 'system', 'content': PROMPT + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)}, + selection_policy='' + if 'content_candidates' in sent: + selection_policy=('\nPre tento dotaz vyber iba evidence_ids. evidence_quotes=[] a answer="". ' + 'Každý required=true musí byť vybraný. Text ani Markdown nekopíruj. ' + 'Rok kandidáta už spĺňa časový filter; nie je to automatická požiadavka na ďalší claim. ' + + ('claims=[] a source_urls=[]; citácie dosadí aplikácia podľa IDs.' + if context['requirements']['response_mode']=='quotes' else + 'Pri mixed vyplň iba výslovne požadované claims a cituj aj ich zdroje.')) + facets=context['requirements'].get('facets',[]) + if facets: + support={f'osoba_{i}':[f for f in facets if any(row.get('person')==person and f in row.get('facets',[]) + for row in sent['content_candidates'])] for i,person in enumerate(context['requirements']['people'],1)} + # Only trusted ordinals/facet labels enter the system instruction; + # names and literal wiki text remain data in the user context. + if any(support.values()): + selection_policy+='\nDoložené časti po osobách v poradí requirements.people: '+json.dumps(support,ensure_ascii=False)+'. ' + selection_policy+=('Vráť status=ok s IDs pre doložené časti aj vtedy, keď ostatné chýbajú. ' + 'Aplikácia konkrétne pomenuje chýbajúce informácie. no_answer tu nesmie odmietnuť ' + 'aj dostupnú časť. Plánovaný experiment alebo vyhodnotenie označené phase=planned ' + 'odpovedá na otázku, čo mali pripraviť; nie je to dosiahnutý výsledok.') + messages = [{'role': 'system', 'content': PROMPT + selection_policy + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)}, {'role': 'user', 'content': json.dumps({'question': question, 'context': sent}, ensure_ascii=False)}] attempts = [] for number in range(2): @@ -501,9 +538,13 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: response = complete(messages) raw = response['choices'][0]['message']['content'] record['raw_response'] = raw - answer = parse_rag_answer(raw) + selected = parse_rag_answer(raw) + record['selected_answer'] = selected + answer = resolve_selection(selected,context) record['parsed_answer'] = answer issues = rag_answer_issues(answer, context) + except EvidenceSelectionError as error: + issues=[str(error)] except (ValueError, TypeError, KeyError, IndexError): issues = ['invalid_answer_format'] except Exception: @@ -534,8 +575,9 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: + (' Opis práce musí obsahovať doslovné evidence_quotes z content_passages. ' 'Samotný názov a rok nestačia; pri mixed zachovaj aj požadované claims.' if context['requirements'].get('content') else '') - + (' Pri overview vyber najviac dva krátke výňatky, najviac 600 znakov a dva body na požadovanú osobu.' - if context['requirements'].get('content_style') == 'overview' else '')}] + + (' Použi evidence_ids z content_candidates a evidence_quotes=[]. Vyber všetky required=true; ' + 'doslovné texty aj ich citácie dosadí aplikácia. Pri response_mode=quotes musí byť claims=[].' + if 'content_candidates' in context else '')}] return {'status': 'error', 'reason': 'answer_rejected', 'accepted_for_delivery': False, 'answer': 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.', 'attempts': attempts, 'index_version': context['index_version']} diff --git a/tools/answers/answer_delivery.py b/tools/answers/answer_delivery.py index efb244b..d2eb048 100644 --- a/tools/answers/answer_delivery.py +++ b/tools/answers/answer_delivery.py @@ -50,6 +50,13 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, result = generate_verified_rag(question, retrieved, model) context = retrieved elif engine == 'graphrag': + from tools.answer_evaluation.question_intent import graph_needs_text + if graph_needs_text(question): + return {'status':'ambiguous','reason':'text_evidence_required','accepted_for_delivery':True, + 'answer':'Táto otázka vyžaduje textové podklady o zadaní, priebehu alebo výsledkoch práce. ' + 'Použi /rag s rovnakou otázkou; graf samotný tieto výňatky neposkytuje.', + 'claims':[],'source_urls':[],'engine':engine,'index_version':None, + 'complete':False,'truncated':False,'truncation_reasons':[]} try: context = graph_context(search_graph(question)) except Exception: diff --git a/tools/graphrag/evaluation/reproducibility.py b/tools/graphrag/evaluation/reproducibility.py index 679fd3b..d7372c7 100644 --- a/tools/graphrag/evaluation/reproducibility.py +++ b/tools/graphrag/evaluation/reproducibility.py @@ -47,6 +47,7 @@ def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]: 'tools/answer_evaluation/presentation.py', 'tools/answer_evaluation/rag_evidence.py', 'tools/answer_evaluation/verified_rag.py', 'tools/rag/llm_context.py', 'tools/answer_evaluation/question_intent.py', 'tools/answer_evaluation/content_selection.py', + 'tools/answer_evaluation/evidence_selection.py', 'tools/answer_evaluation/rag_preparation.py', 'tools/answer_evaluation/answer_coverage.py', 'tools/answer_evaluation/graph_answer_prompt.txt', 'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py'])