diff --git a/README.md b/README.md index 551c9b6..166dcf6 100644 --- a/README.md +++ b/README.md @@ -1517,6 +1517,16 @@ len tieto kandidátske výňatky; ich skutočný JSON podlieha tokenovému rozpo Chýbajúce časti sa označia jednotlivo, aj keď sú ostatné časti zodpovedateľné. Pri limite kontextu sa nestratí informácia, ktoré osoby používateľ požadoval. +Výňatky nesú interný `passage_id`, osobu, pôvodné označenie práce a rok. Kontrola +odmietne zamenený identifikátor a pri rovnakom texte pod rôznymi prácami vyžaduje +jednoznačné priradenie. Číslo projektu odlišuje aj dva projekty rovnakého typu +a roku. Zadanie, plánované úlohy, priebežný stav a zaznamenané výsledky sa +zobrazujú oddelene; negatívny výsledok zostáva výsledkom. Administratívna poznámka +nenahrádza technický výsledok. Porovnávané osoby si delia kontextový rozpočet, +aby dlhý prvý profil nevytlačil ostatné. Skrátenie rešpektuje celé odseky a položky +zoznamu vrátane ich výhrad; pri stručnom opise má zachovanie podmienky prednosť +pred orientačným limitom dĺžky. + Rozpoznané, ale nepodporované operátory (napríklad interval rokov, vylučovanie alebo minimálny počet) vedú k `ambiguous` a konkrétnemu spresneniu. Rok práce sa nepoužije ako dôkaz dátumu jej začatia či obhajoby. Chybové odpovede odlišujú diff --git a/test/test_comparison_answers.py b/test/test_comparison_answers.py index 1e93189..f8c6763 100644 --- a/test/test_comparison_answers.py +++ b/test/test_comparison_answers.py @@ -150,7 +150,7 @@ def test_comparison_separates_goals_and_results(comparison_payload): result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]}) assert result['accepted_for_delivery'] text=result['answer'] - assert text.count('**Zadanie a ciele:**')==text.count('**Zaznamenaný stav a výsledky:**')==2 + assert text.count('**Zadanie a ciele:**')==text.count('**Zaznamenané výsledky:**')==2 assert 'Natrénovať jazykový model metódou BERT.' in text assert 'Natrénovaný model wiki103' in text and 'Implementovaný PageRank' in text assert 'Cassandra' not in text and len(result['source_urls'])==2 diff --git a/test/test_evidence_selection.py b/test/test_evidence_selection.py new file mode 100644 index 0000000..37c2724 --- /dev/null +++ b/test/test_evidence_selection.py @@ -0,0 +1,238 @@ +"""Bound excerpts to works, preserve their phase and share comparison space.""" +import copy +import json +import sqlite3 +import pytest + +from test_quality_block import corpus, prepare, complete +from tools.answer_evaluation.content_selection import administrative_only, evidence_units +from tools.answer_evaluation.rag_evidence import rag_evidence_context, content_sections +from tools.answer_evaluation.rag_preparation import prepare_named_content +from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues +from tools.answer_evaluation.verified_rag import overview_excerpts +from tools.rag.llm_context import compact_rag_payload +from tools.rag.rag_utils import build_context_text + + +def update_eva(corpus,text,headings=None): + db,raw=corpus + with sqlite3.connect(db) as conn: + conn.execute('UPDATE chunks SET text=?,heading_paths_json=? WHERE document_path=?', + (text,json.dumps(headings or [['Eva Nová','Diplomová práca 2024']]),raw['sources'][0]['document_path'])) + + +def selection(context): + return {'status':'ok','answer':'','claims':[], + 'evidence_quotes':[{key:c[key] for key in ['source_id','passage_id'] if c.get(key)}|{'quote':c['text']} + for c in context.get('content_candidates',[])], + 'source_urls':[s['source_url'] for s in context['sources']]} + + +def test_assignment_plan_progress_and_recorded_result_are_separate(corpus): + update_eva(corpus,'Zadanie:\n- Implementujte vyhľadávač.\nPriebežný stav:\n' + '- Rozpracovaná integrácia vyhľadávania.\n- Bude implementovaný klasifikátor.\n' + 'Výsledky:\n- Natrénovaný model BERT.\nPlánované úlohy:\n- Pripravte experiment s presnosťou.') + q='Zhrň zadanie, plánované úlohy, priebežný stav a výsledky diplomovej práce Evy Novej.' + payload=prepare(corpus,q);context=rag_evidence_context(q,payload) + units=evidence_units(context) + assert {u['phase'] for u in units}>={'assignment','planned','progress','result'} + future=next(u for u in units if 'Bude implementovaný' in u['text']) + assert future['phase']=='planned' and 'results' not in future['facets'] + result=generate_verified_rag(q,payload,complete(selection(context))) + assert result['accepted_for_delivery'] + assert all(label in result['answer'] for label in ['Zadanie a ciele','Plánované úlohy','Priebežný stav','Zaznamenané výsledky']) + assert 'Bude implementovaný' not in result['answer'].split('**Zaznamenané výsledky:**')[1] + + +@pytest.mark.parametrize('heading',['Zadanie','Zadanie diplomovej práce']) +def test_phase_from_chunk_heading_ancestry_is_not_lost(corpus,heading): + update_eva(corpus,'- Pripravte vyhodnotenie modelu.', + [['Eva Nová','Diplomová práca 2024',heading]]) + q='Aké je zadanie a plánované úlohy diplomovej práce Evy Novej?' + c=rag_evidence_context(q,prepare(corpus,q)) + assert any(u['kind']=='assignment' and u['phase']=='assignment' for u in evidence_units(c)) + assert not any(u['kind']=='task' for u in evidence_units(c)) + assert all(u['year']==2024 and u['work_label']=='Diplomová práca 2024' for u in evidence_units(c)) + + +def test_negated_completion_is_progress_not_a_completed_result(corpus): + update_eva(corpus,'Stav:\n- Parser zatiaľ nie je implementovaný.') + q='Aký je priebežný stav diplomovej práce Evy Novej?' + c=rag_evidence_context(q,prepare(corpus,q)) + row=next(u for u in evidence_units(c) if 'Parser' in u['text']) + assert row['phase']=='progress' and 'results' not in row['facets'] + q='Aké výsledky dosiahla Eva Nová pri diplomovej práci?' + r=generate_verified_rag(q,prepare(corpus,q),lambda _: (_ for _ in ()).throw(AssertionError('No results model call'))) + assert r['status']=='no_answer' and r['accepted_for_delivery'] + + +def test_administration_alone_is_not_a_technical_result(corpus): + update_eva(corpus,'Stav:\n- Konzultácia štruktúry práce.\n- Dohodnutý termín odovzdania.') + q='Aké výsledky dosiahla Eva Nová pri diplomovej práci?' + c=rag_evidence_context(q,prepare(corpus,q)) + assert not c['content_candidates'] + assert administrative_only('Konzultácia štruktúry práce.') + assert not administrative_only('Vyriešený problém s GPU po konzultácii.') + assert not administrative_only('Odovzdaný zdrojový kód indexovania do SQLite.') + + +def test_recorded_working_system_is_a_result_not_just_administration(corpus): + update_eva(corpus,'Stav:\n- Funguje RAG systém, zostáva overiť presnosť.') + q='Aké výsledky má diplomová práca Evy Novej?' + c=rag_evidence_context(q,prepare(corpus,q)) + assert any(u['phase']=='result' and 'zostáva' in u['text'] for u in c['content_candidates']) + + +def test_completed_experiments_are_not_confused_with_planned_tasks(corpus): + update_eva(corpus,'Výsledky:\n- Vyhodnotená presnosť modelu v experimente.\n' + 'Úlohy:\n- Pripravte experiment s novým tokenizérom.') + q='Aké vykonané experimenty má Eva Nová a aké plánované úlohy jej zostali?' + c=rag_evidence_context(q,prepare(corpus,q)) + planned=next(u for u in evidence_units(c) if 'Pripravte' in u['text']) + assert 'plans' in planned['facets'] and 'experiments' not in planned['facets'] + + +def test_a_planned_method_does_not_prove_an_actually_used_method(corpus): + update_eva(corpus,'Úlohy:\n- Použiť metódu BERT.') + q='Aké použité metódy má diplomová práca Evy Novej?' + c=rag_evidence_context(q,prepare(corpus,q)) + assert not c['content_candidates'] + + +def test_explicit_negative_finding_remains_a_result(corpus): + update_eva(corpus,'Výsledky:\n- Model nebol úspešný pri teste presnosti.') + q='Aké výsledky má diplomová práca Evy Novej?' + p=prepare(corpus,q);c=rag_evidence_context(q,p) + assert c['content_candidates'][0]['phase']=='result' + r=generate_verified_rag(q,p,complete(selection(c))) + assert r['accepted_for_delivery'] and 'nebol úspešný' in r['answer'] + + +def test_identical_text_in_two_work_years_needs_passage_identity(corpus): + _,raw=corpus;source=raw['sources'][0] + source['text']='Sekcia: Diplomová práca 2024\nStav:\nImplementovaný parser.\n\n' + source['text']+='Sekcia: Diplomová práca 2025\nStav:\nImplementovaný parser.' + raw['sources']=[source];raw['context']=build_context_text(raw['sources']) + q='Aké výsledky má Eva Nová pri diplomovej práci?';c=rag_evidence_context(q,raw) + a={'status':'ok','answer':'','claims':[],'source_urls':[source['source_url']], + 'evidence_quotes':[{'source_id':'S1','quote':'Implementovaný parser.'}]} + assert 'ambiguous_evidence_scope' in rag_answer_issues(a,c) + p=c['content_passages'][0];a['evidence_quotes'][0]['passage_id']=p['passage_id'] + assert 'ambiguous_evidence_scope' not in rag_answer_issues(a,c) + assert len({p['passage_id'] for p in c['content_passages']})==2 + + +def test_identical_text_cannot_reuse_another_persons_passage_id(corpus): + for person_name in ['eva_nova','boris_novy']: + db,_=corpus + with sqlite3.connect(db) as conn: + conn.execute('UPDATE chunks SET text=? WHERE document_path LIKE ?', + ('Stav:\nImplementovaný parser.',f'%{person_name}%')) + q='Porovnaj výsledky Evy Novej a Borisa Nového.';p=prepare(corpus,q);c=rag_evidence_context(q,p) + a=selection(c) + assert not rag_answer_issues(a,c) + wrong=copy.deepcopy(a) + wrong['evidence_quotes'][0]['passage_id']=wrong['evidence_quotes'][1]['passage_id'] + assert 'unsupported_evidence_passage' in rag_answer_issues(wrong,c) + + +def test_overlapping_excerpt_preserves_qualification_without_repetition(corpus): + update_eva(corpus,'Stav:\n- Natrénovaný model BERT.\n' + '- Natrénovaný model BERT. Výsledok platí iba pre malý korpus.') + q='Aké výsledky dosiahla Eva Nová?';p=prepare(corpus,q);c=rag_evidence_context(q,p) + r=generate_verified_rag(q,p,complete(selection(c))) + assert r['accepted_for_delivery'] + assert r['answer'].count('Natrénovaný model BERT')==1 and 'iba pre malý korpus' in r['answer'] + + +def add_same_year_projects(corpus): + db,raw=corpus + with sqlite3.connect(db) as conn: + for i,method in [(1,'BERT'),(2,'Cassandra')]: + conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)', + (raw['sources'][0]['document_path'],f'Stav:\nImplementovaná metóda {method}.', + json.dumps([['Eva Nová',f'Diplomový projekt {i} 2022']]),1,i)) + + +def test_same_year_project_records_are_not_merged(corpus): + add_same_year_projects(corpus) + p=prepare(corpus,'Aké metódy používala Eva Nová pri diplomovom projekte v roku 2022?') + assert p['status']=='ambiguous' and 'projekt 1' in p['clarification'] and 'projekt 2' in p['clarification'] + + +def test_explicit_project_number_binds_the_excerpt_to_that_project(corpus): + add_same_year_projects(corpus) + q='Aké metódy používala Eva Nová pri diplomovom projekte 2 v roku 2022?' + p=prepare(corpus,q);c=rag_evidence_context(q,p) + assert p['status']=='ok' and 'Cassandra' in p['context'] and 'BERT' not in p['context'] + assert all('projekt 2' in u['work_label'] for u in evidence_units(c)) + a=selection(c);a['claims']=[{'person':'Eva Nová','work_type':'diploma_project','year':2022}] + result=generate_verified_rag(q,p,complete(a)) + assert result['accepted_for_delivery'] and 'projekt 2' in result['answer'] + + +def test_large_first_profile_cannot_starve_the_second_person(corpus): + db,raw=corpus;q='Porovnaj výsledky Evy Novej a Borisa Nového.' + p=prepare_named_content(db,q,raw) + p['sources'][0]['text']+='\n'+('Dlhé administratívne poznámky. '*3000) + compact=compact_rag_payload(p,max_tokens=2200) + assert compact['source_count']==2 and compact['context_budget']['used_tokens']<=2200 + assert compact['truncated'] and 'llm_context_budget' in compact['truncation_reasons'] + c=rag_evidence_context(q,compact) + assert {row['person'] for row in c['content_candidates']}=={'Eva Nová','Boris Nový'} + assert all(row['passage_id'] and row['work_label'] for row in c['content_candidates']) + + +def test_clipping_cannot_drop_a_wrapped_qualification_from_an_excerpt(corpus): + db,raw=corpus;q='Porovnaj výsledky Evy Novej a Borisa Nového.' + p=prepare_named_content(db,q,raw) + p['sources'][0]['text']='Sekcia: Diplomová práca 2024\nStav:\n- Natrénovaný model BERT.\n ' + p['sources'][0]['text']+='Výsledok platí iba pre malý korpus. '*2000 + compact=compact_rag_payload(p,max_tokens=1700) + assert compact['truncated'] and 'BERT' not in compact['context'] + c=rag_evidence_context(q,compact) + assert any(row['person']=='Boris Nový' for row in c['content_candidates']) + assert not any(row['person']=='Eva Nová' for row in c['content_candidates']) + + +def test_single_profile_clipping_preserves_the_same_qualification_rule(corpus): + _,raw=corpus;raw['sources']=raw['sources'][:1] + raw['sources'][0]['text']='Sekcia: Diplomová práca 2024\nStav:\n- Natrénovaný model BERT.\n ' + raw['sources'][0]['text']+='Výsledok platí iba pre malý testovací korpus. '*2000 + compact=compact_rag_payload(raw,max_tokens=1700) + assert compact['truncated'] and 'BERT' not in compact['context'] + + +@pytest.mark.parametrize('with_title',[False,True]) +def test_relevant_results_are_prioritized_before_context_is_clipped(corpus,with_title): + update_eva(corpus,('Názov: Skutočný názov práce.\n' if with_title else '')+'Zadanie:\n- Implementujte parser.') + db,raw=corpus;path=raw['sources'][0]['document_path'] + with sqlite3.connect(db) as conn: + for i in range(1,12): + conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path, + 'Stav:\n- Konzultácia štruktúry práce. '+('Administratívne poznámky. '*100), + json.dumps([['Eva Nová','Diplomová práca 2024']]),1,i)) + conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path, + 'Výsledky:\n- Vyhodnotená presnosť parsera.', + json.dumps([['Eva Nová','Diplomová práca 2024']]),1,12)) + q='Porovnaj '+('názvy a ' if with_title else '')+'výsledky Evy Novej a Borisa Nového.' + compact=compact_rag_payload(prepare_named_content(db,q,raw),max_tokens=1700) + assert compact['truncated'] and 'Vyhodnotená presnosť parsera.' in compact['context'] + c=rag_evidence_context(q,compact) + assert {u['person'] for u in c['content_candidates']}=={'Eva Nová','Boris Nový'} + if with_title: + assert 'Skutočný názov práce' in compact['context'] and 'title' in c['requirements']['available_fields']['Eva Nová'] + + +def test_overview_cannot_cut_off_a_qualification_for_brevity(): + text='Systém vyhľadáva dokumenty. '+('Opis implementácie a dát. '*30)+'Platí iba pre malý testovací korpus.' + quote={'source_id':'S1','quote':text} + c={'content_passages':[{'source_id':'S1','person':'Eva Nová','text':text}],'facts':[]} + assert overview_excerpts({'evidence_quotes':[quote]},c)==[text] + + +def test_markdown_role_headings_preserve_explicit_source_phase(): + sections=content_sections('## Zadanie\nImplementujte systém.\n## Plánované úlohy\nPripravte dáta.\n' + '## Priebežný stav\nRozpracovaná integrácia.\n## Výsledky\nVyhodnotená presnosť.') + assert [(s['kind'],s['phase']) for s in sections]==[ + ('assignment','assignment'),('task','planned'),('progress','progress'),('progress','result')] diff --git a/tools/answer_evaluation/content_selection.py b/tools/answer_evaluation/content_selection.py index 8dd2106..5ccb003 100644 --- a/tools/answer_evaluation/content_selection.py +++ b/tools/answer_evaluation/content_selection.py @@ -3,11 +3,23 @@ from __future__ import annotations import re from .context_policy import normalized -from .question_intent import FACET_LABELS, planned_experiments +from .question_intent import FACET_LABELS, planned_experiments, requested_facets -COMPLETED = r'\b(?:implementovan\w*|natrenovan\w*|vytvoren\w*|extrahovan\w*|vykonan\w*|urobene vyhodnotenie|vyhodnoteny\w*|nameran\w*)\b' -PLANNED = r'\b(?:bude|budu|planuj\w*|cielom|mal by|ma sa|treba|pripravte|navrhnite|otestujte|vyhodnotte)\b' -ADMIN = r'\b(?:konzultac\w*|struktura prace|pisomn\w*|pisat|odovzda\w*|wip|termin\w*)\b' +COMPLETED = (r'\b(?:(?:implementovan|natrenovan|vytvoren|extrahovan|vykonan|vyhodnoten|nameran|' + r'vyriesen|odstranen|otestovan|porovnan|dosiahnut|dokoncen)(?:y|a|e|u|ou|ych|ym|ymi|om)|' + r'urobene vyhodnotenie|implementoval\w*|natrenoval\w*|vyhodnotil\w*|funguje|funguju)\b') +PLANNED = r'\b(?:bude|budu|planuj\w*|planovan\w*|cielom|mal by|ma sa|treba|pripravte|navrhnite|otestujte|vyhodnotte)\b' +ADMIN = r'\b(?:konzultac\w*|struktura prace|pisomn\w*|pisat|odovzda\w*\s+(?:text\w*|pisomn\w*|prac\w*)|wip|termin\w*)\b' +NOT_COMPLETED = r'\b(?:nie je|nie su|nebol\w*|este nie|zatial nie|nebude|nebudu)\b' + + +def has_qualification(text: str) -> bool: + return bool(re.search(r'\b(?:iba|len|avsak|ale|zatial|podmien\w*|obmedzen\w*|nebol\w*|nie|nepotvrd\w*)\b',normalized(text))) + + +def administrative_only(text: str) -> bool: + n=normalized(text) + return bool(re.search(ADMIN,n) and not re.search(COMPLETED,n)) def split_units(text: str) -> list[str]: @@ -19,7 +31,7 @@ def split_units(text: str) -> list[str]: if not descriptive_text(part): continue # A source paragraph is kept literal, including its negations and qualifications. - for unit in ([part] if len(part) <= 900 else re.split(r'(?<=[.!?])\s+(?=[A-ZÁČĎÉÍĽŇÓŠŤÚÝŽ])', part)): + for unit in ([part] if len(part) <= 900 or has_qualification(part) else re.split(r'(?<=[.!?])\s+(?=[A-ZÁČĎÉÍĽŇÓŠŤÚÝŽ])', part)): if descriptive_text(unit): units.append(unit.strip()) return units @@ -27,28 +39,45 @@ def split_units(text: str) -> list[str]: def effective_role(text: str, role: str) -> str: n = normalized(text) - if role=='other': + if role in {'other','progress'}: from .presentation import GOAL_VERBS first=n.split()[0] if n else '' - if first in {normalized(word) for word in GOAL_VERBS} or n.startswith('cielom je'): + if (first in {normalized(word) for word in [*GOAL_VERBS,*GOAL_VERBS.values()]} or n.startswith('cielom je') or + re.search(PLANNED,n)): return 'task' if role == 'other' and re.search(COMPLETED, n) and not re.search(PLANNED, n): return 'progress' return role -def facets_for(text: str, role: str, question: str) -> list[str]: +def evidence_phase(text: str, role: str, source_phase: str = 'unknown') -> str: + if role=='assignment':return 'assignment' + if role=='task':return 'planned' + if administrative_only(text):return 'administrative' + if role=='progress': + if source_phase=='result' or (not re.search(NOT_COMPLETED,normalized(text)) and + re.search(COMPLETED,normalized(text))): + return 'result' + return 'progress' + return 'unknown' + + +def facets_for(text: str, role: str, question: str, phase: str | None = None) -> list[str]: n = normalized(text); facets = [] if role == 'assignment': facets.append('assignment') - if role == 'progress' and (not re.search(PLANNED,n) or re.search(COMPLETED,n)): - facets.append('results') + if role == 'task':facets.append('plans') + if role == 'progress' and not re.search(PLANNED,n) and not administrative_only(text): + facets.append('progress') + if (phase or evidence_phase(text,role))=='result':facets.append('results') if re.search(r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*|zlyha\w*|nepodar\w*|nevyries\w*)\b', n) and role not in {'assignment','task'}: facets.append('problems') if re.search(r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*)\b', n): - if role not in {'assignment','task'} or not re.search(r'\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*)\b',normalized(question)): + actual_use=(re.search(r'\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*)\b',normalized(question)) or + re.search(r'\b(?:pouzit\w*|aplikovan\w*|vyuzit\w*)\s+(?:metod\w*|technolog\w*|nastroj\w*)\b',normalized(question))) + if role not in {'assignment','task'} or not actual_use: facets.append('methods') if re.search(r'\b(?:experiment\w*|meran\w*|test\w*|otest\w*|vyhodnot\w*|benchmark\w*|f1|presnost\w*)\b', n): - wants_done=re.search(r'\b(?:vykonal\w*|otestoval\w*|uskutocnil\w*|vyhodnotil\w*)\b',normalized(question)) + wants_done=re.search(r'\b(?:vykonal\w*|vykonan\w*|otestoval\w*|uskutocnil\w*|uskutocnen\w*|realizovan\w*|vyhodnotil\w*)\b',normalized(question)) if wants_done and role in {'assignment','task'}: pass elif not planned_experiments(question) or role in {'assignment','task'} or re.search(PLANNED, n): @@ -61,35 +90,73 @@ def unit_score(text: str, role: str, facet: str = '') -> int: score = 25 if role == 'progress' else 10 if re.search(COMPLETED, n): score += 45 if re.search(r'\b(?:model|system|algoritm\w*|vyhodnot\w*|experiment\w*|presnost\w*|pagerank|bert)\b', n): score += 15 - if re.search(ADMIN, n): score -= 70 + if administrative_only(text): score -= 70 if facet == 'problems' and re.search(r'\b(?:problem\w*|chyb\w*|zlyha\w*)\b', n): score += 35 if role in {'assignment','task'} and facet == 'results': score -= 100 return score +def prioritize_frames(frames: list[str], question: str) -> list[str]: + """Order whole source windows before budgeting; do not rewrite their text.""" + from .rag_evidence import content_sections, TITLE_LABELS + rankings=[] + for facet in requested_facets(question): + scores=[] + for index,frame in enumerate(frames): + relevant=[] + for section in content_sections(frame): + for text in split_units(section['text']): + role=effective_role(text,section['kind']) + phase=evidence_phase(text,role,section.get('phase','unknown')) + if len(text)<=2500 and facet in facets_for(text,role,question,phase): + relevant.append(unit_score(text,role,facet)) + if relevant:scores.append((index,max(relevant))) + rankings.append([i for i,_ in sorted(scores,key=lambda item:-item[1])]) + order=[] + if re.search(r'\bnaz(?:ov|v)\w*\b',normalized(question)) and frames: + order=[i for i,frame in enumerate(frames) if any(':' in line and + normalized(line.split(':',1)[0]) in TITLE_LABELS for line in frame.splitlines())] or [0] + seen=set(order) + for pos in range(max(map(len,rankings),default=0)): + for ranking in rankings: + if pos list[dict]: - from .rag_evidence import content_sections, quote_sections + from .rag_evidence import content_sections, quote_sections, quote_matches_passage rows = [] for passage in context.get('content_passages', []): pieces = content_sections(passage['text']) if quotes is None else [ - piece for quote in quotes if quote['source_id'] == passage['source_id'] and - ' '.join(quote['quote'].split()) in ' '.join(passage['text'].split()) - for piece in quote_sections(quote, context)] + piece for quote in quotes if quote_matches_passage(quote,passage) + for piece in quote_sections(quote, {'content_passages':[passage]})] for piece in pieces: for unit in split_units(piece['text']): role = effective_role(unit, piece['kind']) - rows.append({k: passage.get(k) for k in ['person','work_type','year','source_id']} | - {'text':unit,'kind':role,'facets':facets_for(unit,role,context.get('question',''))}) + phase=evidence_phase(unit,role,piece.get('phase','unknown')) + rows.append({k: passage.get(k) for k in ['person','work_type','year','work_label','source_id','passage_id']} | + {'text':unit,'kind':role,'phase':phase, + 'facets':facets_for(unit,role,context.get('question',''),phase)}) # Overlapping chunks/quotes can repeat the same fact with different roles. # An explicit role wins over an unlabelled duplicate, within the same work. unique = {} for row in rows: clean=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text']) - key = tuple(row.get(k) for k in ['person','work_type','year']) + (' '.join(clean.split()),) + key = tuple(row.get(k) for k in ['person','work_type','year']) + (normalized(row.get('work_label')), + row['phase'],' '.join(clean.split())) old = unique.get(key) if old is None or (old['kind']=='other' and row['kind']!='other'): unique[key]=row - return list(unique.values()) + units=list(unique.values()) + # Prefer a whole longer excerpt over its overlapping prefix, within one + # work and phase. Never merge people, years or plans with recorded results. + def plain(row):return ' '.join(re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text']).split()) + return [row for row in units if not any( + all(row.get(k)==other.get(k) for k in ['person','work_type','year','phase']) and + normalized(row.get('work_label'))==normalized(other.get('work_label')) and + len(plain(row)) list[dict]: @@ -98,6 +165,8 @@ def candidates(context: dict) -> list[dict]: for person in context['requirements']['people']: for facet in facets: relevant = [u for u in units if u['person']==person and facet in u['facets']] + if facet=='progress' and 'results' in facets and any(u['phase']=='progress' for u in relevant): + relevant=[u for u in relevant if u['phase']=='progress'] relevant.sort(key=lambda u: -unit_score(u['text'],u['kind'],facet)) accepted=0 for unit in relevant: @@ -161,6 +230,10 @@ def facet_lines(answer: dict, context: dict) -> list[str]: subject=person+(' — '+WORK_LABELS[kind] if kind in WORK_LABELS else '') if len(scopes)==1 and next(iter(scopes))[1] is not None: subject+=' ('+str(next(iter(scopes))[1])+')' + labels={p.get('work_label') for p in context.get('content_passages',[]) + if p.get('person')==person and p.get('work_label')} + if req.get('project_number') is not None and len(labels)==1: + subject=person+' — '+' '.join(next(iter(labels)).split()) lines.extend(['**'+subject+'**', '']) seen=set() for facet in req.get('facets', []): @@ -179,8 +252,10 @@ def facet_lines(answer: dict, context: dict) -> list[str]: if row['kind'] in {'assignment','task'}: text,_=goal_text(text) text=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',text) - phase=' (plán)' if row['kind'] in {'assignment','task'} and facet not in {'assignment'} else '' - lines.append('- '+text+phase); seen.add(key); shown+=1 + phase=' (plán)' if row['kind'] in {'assignment','task'} and facet not in {'assignment','plans'} else '' + state=({'result':'Zaznamenaný výsledok: ','progress':'Priebežný stav: '}.get(row['phase'],'') + if facet=='results' else '') + lines.append('- '+state+text+phase); seen.add(key); shown+=1 if shown==2: break if not shown: if rows: diff --git a/tools/answer_evaluation/question_intent.py b/tools/answer_evaluation/question_intent.py index e7716c5..624f806 100644 --- a/tools/answer_evaluation/question_intent.py +++ b/tools/answer_evaluation/question_intent.py @@ -4,14 +4,16 @@ from .context_policy import normalized FACET_PATTERNS = { 'assignment': r'\b(?:zadani\w*|ciel\w*)\b', + 'plans': r'\b(?:planovan\w*|dalsi\w*\s+uloh\w*|uloh\w*)\b', 'problems': r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*)\b', 'methods': r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*)\b', 'experiments': r'\b(?:experiment\w*|merani\w*|testovani\w*|vyhodnocov\w*)\b', - 'results': r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b', + 'progress':r'\b(?:priebezn\w*\s+)?stav\w*\b', + 'results': r'\b(?:vysledk\w*|dosiahn\w*)\b', } -FACET_LABELS = {'assignment': 'Zadanie a ciele', 'problems': 'Riešené problémy', +FACET_LABELS = {'assignment': 'Zadanie a ciele', 'plans':'Plánované úlohy', 'problems': 'Riešené problémy', 'methods': 'Metódy a technológie', 'experiments': 'Experimenty a vyhodnotenie', - 'results': 'Zaznamenaný stav a výsledky'} + 'progress':'Priebežný stav','results': 'Zaznamenané výsledky'} def requested_facets(question: str) -> list[str]: @@ -29,10 +31,12 @@ def content_question(question: str) -> bool: def facet_mode(question: str) -> bool: facets = requested_facets(question) - return any(facet in facets for facet in ['problems','methods','experiments','results']) + return any(facet in facets for facet in ['plans','progress','problems','methods','experiments','results']) def planned_experiments(question: str) -> bool: text=normalized(question) + if re.search(r'\b(?:vykonal\w*|vykonan\w*|uskutocnil\w*|uskutocnen\w*|realizoval\w*|realizovan\w*|otestoval\w*|vyhodnotil\w*)\b',text): + return False return bool(re.search(r'\b(?:plan\w*|pripravit|navrhnut)|' r'\b(?:mal|mali|mala|ma|maju)\s+(?:priprav\w*|navrhn\w*|otest\w*|vyhodnot\w*)',text)) diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index 22c2e0f..433bfea 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -8,6 +8,7 @@ from __future__ import annotations import re import json import sqlite3 +import hashlib from contextlib import closing from .context_policy import normalized, text_contains_value, person_pattern from .question_intent import requested_facets, content_question, facet_mode @@ -50,6 +51,8 @@ def academic_scope(path: str) -> tuple[str, int | None] | None: for heading in reversed(path.split('>')): if normalized(heading).startswith(('priprava na ', 'priprava k ')): return None + if not TYPE_PATTERN.match(normalized(heading)): + continue kind = work_type(heading) if kind: years = [int(year) for year in re.findall(r'(? dict[str, str]: return result -def academic_windows(body: str) -> list[tuple[tuple[str, int | None] | None, str]]: +def project_number(text: str) -> int | None: + match=re.search(r'\bprojekt\w*\s+([1-9]\d?)(?!\d)\b',normalized(text)) + return int(match[1]) if match else None + + +def academic_windows_metadata(body: str) -> list[tuple]: """Keep literal passages bound to the academic section actually delivered.""" - windows = []; scope = None; heading_stack = []; lines = [] + windows = []; scope = None; label = None; heading_stack = []; lines = [] def flush(): if lines: - windows.append((scope, '\n'.join(lines))) + windows.append((scope, label, '\n'.join(lines))) lines.clear() for line in body.splitlines(): heading = re.match(r'^(?:Sekcia(?: dôkazu)?:|#{1,6}\s)(.*)', line) if line.startswith(('RELEVANTNÁ SEKCIA ', 'DÔKAZ Z ', 'ĎALŠÍ DÔKAZ ', 'ĎALŠÍ RELEVANTNÝ ', 'NAJRELEVANTNEJŠÍ ', 'ZAČIATOK RELEVANTNEJ ')): - flush(); scope = None; heading_stack.clear() + flush(); scope = None; label = None; heading_stack.clear() elif line.startswith('Dokument:'): - flush(); scope = None; heading_stack.clear() + flush(); scope = None; label = None; heading_stack.clear() elif heading: flush() if line.startswith('Sekcia'): - scope = academic_scope(heading[1]); heading_stack = [(0, scope)] + scope = academic_scope(heading[1]); label = None + parts=[p.strip() for p in heading[1].split('>')] + academic=[i for i,p in enumerate(parts) if TYPE_PATTERN.match(normalized(p))] + if scope and academic: + pos=academic[-1];label=parts[pos] + # Chunk ancestry can carry the only explicit role heading. + lines.extend('# '+part for part in parts[pos+1:] if part) + heading_stack = [(0, scope, label)] else: level = len(line) - len(line.lstrip('#')) while heading_stack and heading_stack[-1][0] >= level: @@ -149,7 +164,8 @@ def academic_windows(body: str) -> list[tuple[tuple[str, int | None] | None, str detected = academic_scope(heading[1]) scope = (None if normalized(heading[1]).startswith(('priprava na ', 'priprava k ')) else detected or (heading_stack[-1][1] if heading_stack else None)) - heading_stack.append((level, scope)) + label = heading[1].strip() if detected else (heading_stack[-1][2] if heading_stack else None) + heading_stack.append((level, scope, label)) if scope is not None and detected is None: # Keep semantic child headings (Stav, Úlohy, Metódy) as # literal source text instead of losing their meaning. @@ -160,6 +176,10 @@ def academic_windows(body: str) -> list[tuple[tuple[str, int | None] | None, str return windows +def academic_windows(body: str) -> list[tuple[tuple[str, int | None] | None, str]]: + return [(scope,text) for scope,_label,text in academic_windows_metadata(body)] + + def descriptive_text(text: str, titles=()) -> str: """Exclude labels/identity lines; a title or year alone is not a description.""" metadata = TITLE_LABELS | {'meno veduceho', 'veduci', 'veduci prace', 'autor', 'autor dokumentu', @@ -202,16 +222,19 @@ def academic_content_passages(blocks: list[dict], facts: list[dict], requirement types = {requirements['work_types'][person] for person in people if person in requirements['work_types']} or query_types titles = [row.get(key) for row in facts if row.get('source_id') == source['source_id'] for key in ('title', 'person', 'document_author')] - for scope, text in academic_windows(source['body']): + for scope, label, text in academic_windows_metadata(source['body']): if not scope or (types and scope[0] not in types): continue if requirements['unsupported_year'] or (requirements['years'] and scope[1] not in requirements['years']): continue + if requirements.get('project_number') is not None and project_number(label or '')!=requirements['project_number']: + continue if not descriptive_text(text, titles): continue owner = (source.get('title') if PROFILE.fullmatch(path) else people[0] if len(people) == 1 else None) passage = {'source_id': source['source_id'], 'person': owner, - 'work_type': scope[0], 'year': scope[1], 'text': text} + 'document_path':path, 'work_type': scope[0], 'year': scope[1], 'work_label':label, 'text': text} + passage['passage_id']='P'+hashlib.sha256(json.dumps(passage,ensure_ascii=False,sort_keys=True).encode()).hexdigest()[:20] if passage not in result: result.append(passage) return result @@ -219,48 +242,56 @@ def academic_content_passages(blocks: list[dict], facts: list[dict], requirement def content_sections(text: str) -> list[dict]: """Separate explicit source goals from recorded progress, without inference.""" - sections = []; lines = []; kind = 'other' + sections = []; lines = []; kind = 'other'; phase = 'unknown' def flush(): if lines and '\n'.join(lines).strip(): - sections.append({'kind': kind, 'text': '\n'.join(lines).strip()}) + sections.append({'kind': kind, 'phase':phase, 'text': '\n'.join(lines).strip()}) lines.clear() for line in text.splitlines(): value = normalized(line) label = line.strip().strip('*_# ').rstrip(':').strip() heading = value == normalized(label) and len(value.split()) <= 8 + role_value=re.sub(r'\s+(?:diplomov\w*|bakalarsk\w*|zaverecn\w*)\s+(?:prac\w*|projekt\w*)(?:\s+\d{4})?$','',value) next_kind = None - if heading and value in {'ciel', 'ciele', 'ciele na semester', 'ciele prace', 'zadanie', + if heading and role_value in {'ciel', 'ciele', 'ciele na semester', 'ciele prace', 'zadanie', 'zadanie diplomovej prace', 'zadanie bakalarskej prace', 'zadanie diplomoveho projektu', 'zadanie bakalarskeho projektu', 'navrh na zadanie', 'navrh zadania'}: next_kind = 'assignment' - elif heading and value in {'stav', 'vysledky', 'dosiahnute vysledky', 'dosiahnuty stav'}: + elif heading and role_value in {'stav', 'priebezny stav', 'vysledky', 'dosiahnute vysledky', 'dosiahnuty stav'}: next_kind = 'progress' - elif heading and value in {'ulohy', 'zasobnik uloh', 'dalsie ulohy'}: + elif heading and role_value in {'ulohy', 'zasobnik uloh', 'dalsie ulohy', 'plan', 'planovane ulohy'}: next_kind = 'task' elif value.startswith(('stretnutie ', 'virtualne stretnutie ', 'sekcia ')): next_kind = 'other' if next_kind is not None: flush(); kind = next_kind + phase=({'assignment':'assignment','task':'planned','other':'unknown'}.get(kind) or + ('result' if role_value in {'vysledky','dosiahnute vysledky'} else 'progress')) else: lines.append(line) flush() return sections +def quote_matches_passage(quote: dict, passage: dict) -> bool: + return (quote['source_id']==passage['source_id'] and + (not quote.get('passage_id') or quote['passage_id']==passage.get('passage_id')) and + ' '.join(quote['quote'].split()) in ' '.join(passage['text'].split())) + + def quote_sections(quote: dict, context: dict) -> list[dict]: """Classify only literal excerpts bound to the delivered academic passage.""" full = [section for passage in context.get('content_passages', []) - if passage['source_id'] == quote['source_id'] and - ' '.join(quote['quote'].split()) in ' '.join(passage['text'].split()) + if quote_matches_passage(quote,passage) for section in content_sections(passage['text'])] pieces = content_sections(quote['quote']) for piece in pieces: if piece['kind'] == 'other': - kinds = {section['kind'] for section in full + roles = {(section['kind'],section['phase']) for section in full if ' '.join(piece['text'].split()) in ' '.join(section['text'].split())} - if len(kinds) == 1: - piece['kind'] = kinds.pop() + if len(roles) == 1: + piece['kind'],piece['phase'] = roles.pop() return pieces @@ -298,7 +329,7 @@ def expand_comparison_sources(db_file, question: str, payload: dict) -> dict: paths = json.loads(row['heading_paths_json'] or '[]') headings = [' > '.join(path) if isinstance(path, list) else str(path) for path in paths] first = headings[0] if headings else '' - for scope, text in academic_windows('Sekcia: ' + first + '\n' + row['text']): + for scope, label, text in academic_windows_metadata('Sekcia: ' + first + '\n' + row['text']): if not scope or (req['work_types'].get(person) and scope[0] != req['work_types'][person]): continue if req['unsupported_year'] or (req['years'] and scope[1] not in req['years']): @@ -306,8 +337,7 @@ def expand_comparison_sources(db_file, question: str, payload: dict) -> dict: if not descriptive_text(text) or (scope, text) in seen: continue seen.add((scope, text)) - frames.append('Sekcia: ' + WORK_LABELS[scope[0]] + - (f' {scope[1]}' if scope[1] is not None else '') + '\n' + text) + frames.append('Sekcia: ' + (label or WORK_LABELS[scope[0]]) + '\n' + text) selected.append({**source, **({'text': '\n\n'.join(frames)} if frames else {})}) changed |= bool(frames) if not changed: @@ -328,7 +358,9 @@ def rag_evidence_context(question: str, payload: dict) -> dict: people.append(person) base = {'person': person, 'source_id': source['source_id'], 'source_url': source.get('source_url'), 'document_path': source['document_path']} - windows = academic_windows(source['body']) + windows = [(scope,text) for scope,label,text in academic_windows_metadata(source['body']) + if project_number(question) is None or + (scope is not None and project_number(label or '')==project_number(question))] unscoped = [] for section, text in windows: titles = [] @@ -432,6 +464,7 @@ def rag_evidence_context(question: str, payload: dict) -> dict: stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else [] requirements['years'] = requested_years(question) if academic and not requirements['start_year'] else [] requirements['unsupported_year'] = any(len(value) != 4 for value in stated_years) + requirements['project_number'] = project_number(question) if requested: facts = [row for row in facts if row['person'] in requested] requirements['available_fields'] = {person:[key for key in ['title','year','start_year'] diff --git a/tools/answer_evaluation/rag_preparation.py b/tools/answer_evaluation/rag_preparation.py index 37070d0..b629f3e 100644 --- a/tools/answer_evaluation/rag_preparation.py +++ b/tools/answer_evaluation/rag_preparation.py @@ -8,7 +8,7 @@ from contextlib import closing from .context_policy import normalized, person_pattern, text_contains_value from .question_intent import content_question -from .rag_evidence import PROFILE, academic_scope, academic_windows, requested_types, WORK_LABELS +from .rag_evidence import PROFILE, academic_windows_metadata, requested_types, WORK_LABELS, project_number from tools.retrieval.question_constraints import requested_years @@ -64,17 +64,21 @@ def prepare_named_content(db_file, question: str, payload: dict) -> dict: paths=json.loads(row['heading_paths_json'] or '[]') headings=[' > '.join(p) if isinstance(p,list) else str(p) for p in paths] first=headings[0] if headings else '' - for scope,text in academic_windows('Sekcia: '+first+'\n'+str(row['text'] or '')): + for scope,label,text in academic_windows_metadata('Sekcia: '+first+'\n'+str(row['text'] or '')): if not scope or (types.get(person['title']) and scope[0]!=types[person['title']]) or (years and scope[1] not in years): continue - label=WORK_LABELS[scope[0]]+(f' {scope[1]}' if scope[1] is not None else '') - records[scope]=label - if (scope,text) not in seen: - seen.add((scope,text));frames.append('Sekcia: '+label+'\n'+text) + label=label or WORK_LABELS[scope[0]]+(f' {scope[1]}' if scope[1] is not None else '') + if project_number(question) is not None and project_number(label)!=project_number(question): + continue + records[(scope,normalized(label))]=label + if (scope,normalized(label),text) not in seen: + seen.add((scope,normalized(label),text));frames.append('Sekcia: '+label+'\n'+text) if len(records)>1: options='; '.join(dict.fromkeys(records.values())) return clarify(payload,'ambiguous_work',f'Ktorú prácu osoby {person["title"]} máš na mysli? ' - f'V indexe sú tieto možnosti: {options}. Uveď typ práce a rok.') + f'V indexe sú tieto možnosti: {options}. Uveď typ práce a rok alebo presné označenie projektu.') + from .content_selection import prioritize_frames + frames=prioritize_frames(frames,question) source=next((s for s in payload.get('sources',[]) if s.get('document_path')==person['path']),None) if source is None: from tools.rag.search_core import make_source_url diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index 8360254..bd20de2 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -10,10 +10,10 @@ from pydantic import BaseModel, ConfigDict, Field from .answer_format import Claim from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized, text_urls -from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections +from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections, quote_matches_passage from .answer_coverage import answer_coverage, coverage_notices from .presentation import content_lines, fact_sentences, no_answer_text, work_subject -from .content_selection import facet_lines, evidence_units, missing_facets, fit_candidates +from .content_selection import facet_lines, evidence_units, missing_facets, fit_candidates, has_qualification class RagClaim(Claim): @@ -25,6 +25,7 @@ class EvidenceQuote(BaseModel): model_config = ConfigDict(extra='forbid', strict=True) source_id: str quote: str = Field(min_length=1, max_length=2500) + passage_id: str | None = Field(default=None,min_length=1,max_length=64) class RagAnswer(BaseModel): @@ -70,7 +71,11 @@ Pri zaznamenanom stave uprednostni konkrétnu implementáciu, natrénovanie mode vyhodnotenie pred konzultáciami alebo samotným písaním textu. Ani priebežný stav nevydávaj za splnenie všetkých cieľov alebo za konečné výsledky práce. Ak sú dodané content_candidates, vyber quotes iba z ich textu. Každý kandidát uvádza -osobu, typ práce, rok, kind (assignment/task/progress/other) a facets, ktoré dokladá. +osobu, typ práce, rok, work_label, passage_id, kind, phase a facets, ktoré dokladá. +Pri evidence_quotes zachovaj passage_id kandidáta alebo dodanej content_passage. +Rovnaký text pod inou prácou, rokom alebo sekciou nie je zameniteľný dôkaz. +Phase rozlišuje assignment (zadanie), planned (plán), progress (priebežný stav) +a result (zaznamenaný výsledok). Zaznamenaný výsledok nie je dôkaz dokončenia celej práce. Pokry každú dostupnú požadovanú časť requirements.facets za každú osobu. Pre chýbajúcu časť nevymýšľaj dôkaz: aplikácia ju výslovne označí. Čiastočná doložená odpoveď má status=ok. Pri výsledkoch použi skutočný zaznamenaný stav; úlohy a plánované experimenty nie sú výsledky. @@ -212,7 +217,18 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: elif source.get('source_url') not in cited: issues.append('missing_quote_source') if source: + matches=[p for p in context.get('content_passages',[]) if quote_matches_passage(quote,p)] + if requirements.get('content'): + if quote.get('passage_id') and not matches: + issues.append('unsupported_evidence_passage') + elif not quote.get('passage_id') and len(matches)>1: + identities={(p.get('person'),p.get('work_type'),p.get('year'),normalized(p.get('work_label')), + tuple(sorted({(u['kind'],u['phase']) for u in evidence_units( + {'content_passages':[p],'question':context.get('question','')},[quote])}))) for p in matches} + if len(identities)>1: + issues.append('ambiguous_evidence_scope') if context.get('candidate_only') and not any(c['source_id']==quote['source_id'] and + (not quote.get('passage_id') or quote['passage_id']==c.get('passage_id')) and ' '.join(quote['quote'].split()) in ' '.join(c['text'].split()) for c in context.get('content_candidates', [])): issues.append('quote_outside_delivered_candidates') @@ -221,9 +237,7 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: for key in ('title', 'person', 'document_author')] if not descriptive_text(quote['quote'], titles): issues.append('metadata_quote_does_not_describe_work') - if not any(passage['source_id'] == quote['source_id'] and - ' '.join(quote['quote'].split()) in ' '.join(passage['text'].split()) - for passage in context.get('content_passages', [])): + if not matches: issues.append('quote_outside_requested_work') if (requested and source.get('document_path', '').startswith(('pages/students/', 'pages/interns/')) and not any(source.get('document_path', '').startswith(row['document_path'].removesuffix('README.md')) @@ -246,14 +260,12 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: all(claim.get(key) is None or claim[key] == p.get(key) for key in ['work_type', 'year']) for claim in claims)] eligible = any(c.get('person')==person for c in context.get('content_candidates', [])) - if (not requirements.get('facets') or eligible) and not any(quote['source_id'] == p['source_id'] and - ' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) + if (not requirements.get('facets') or eligible) and not any(quote_matches_passage(quote,p) for p in passages for quote in quotes): issues.append('missing_bound_content_evidence') if requirements.get('distinguish_progress') and not requirements.get('facets'): bound_quotes = [quote for quote in quotes if any( - quote['source_id'] == p['source_id'] and - ' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) for p in passages)] + quote_matches_passage(quote,p) for p in passages)] kinds = {section['kind'] for quote in bound_quotes for section in quote_sections(quote, context) if descriptive_text(section['text'])} for kind in requirements.get('content_roles', {}).get(person, []): @@ -287,7 +299,7 @@ def _overview_selections(answer: dict, context: dict) -> list[dict]: for quote in answer.get('evidence_quotes', []): sid = quote['source_id'] owner = next((p.get('person') for p in context.get('content_passages', []) - if p['source_id'] == sid and p.get('person')), sid) + if quote_matches_passage(quote,p) and p.get('person')), sid) identities = [row.get(key) for row in context.get('facts', []) if row.get('source_id') == sid for key in ('person', 'title', 'document_author')] blocks = re.split(r'\n\s*\n|(?=^[ \t]*(?:\d+[.)]|[-+*])\s+)', quote['quote'], flags=re.M) @@ -298,21 +310,17 @@ def _overview_selections(answer: dict, context: dict) -> list[dict]: if not descriptive_text(block, identities): continue # Prefer complete sentences when a prose paragraph is very long. - units = re.split(r'(?<=[.!?])\s+(?=[A-ZÁČĎÉÍĽŇÓŠŤÚÝŽ])', block) if len(block) > 600 else [block] + qualified=has_qualification(block) + units = re.split(r'(?<=[.!?])\s+(?=[A-ZÁČĎÉÍĽŇÓŠŤÚÝŽ])', block) if len(block) > 600 and not qualified else [block] for unit in units: remaining = 600 - sizes.get(owner, 0) - if counts.get(owner, 0) >= 2 or remaining < 30: + if counts.get(owner, 0) >= 2 or (remaining < 30 and not qualified): break unit = unit.strip() if not descriptive_text(unit, identities) or (owner, unit) in seen: continue - if len(unit) > remaining: - # An explicit ellipsis makes a shortened literal prefix - # visible; never cut a word/URL and pretend it is complete. - prefix = unit[:remaining - 2].rsplit(' ', 1)[0].rstrip() - if not prefix or ' ' not in prefix or not descriptive_text(prefix, identities): - continue - unit = prefix + ' …' + if len(unit) > remaining and not qualified and counts.get(owner,0): + continue selected.append({'evidence': quote, 'text': unit}); seen.add((owner, unit)) counts[owner] = counts.get(owner, 0) + 1 sizes[owner] = sizes.get(owner, 0) + len(unit) @@ -332,8 +340,7 @@ def comparison_lines(answer: dict, context: dict) -> list[str]: lines.extend([work_subject(person, kind, context['question']), '']) roles = {'assignment': [], 'progress': [], 'task': [], 'other': []} for quote in answer.get('evidence_quotes', []): - if not any(p.get('person') == person and p['source_id'] == quote['source_id'] and - ' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) + if not any(p.get('person') == person and quote_matches_passage(quote,p) for p in context.get('content_passages', [])): continue for section in quote_sections(quote, context): @@ -398,8 +405,7 @@ def render_rag_answer(answer: dict, context: dict) -> str: for selection in display: quote = selection['evidence'] passage = next((p for p in context.get('content_passages', []) - if p['source_id'] == quote['source_id'] and - ' '.join(quote['quote'].split()) in ' '.join(p['text'].split())), {}) + if quote_matches_passage(quote,p)), {}) identity = (passage.get('person'), passage.get('work_type'), passage.get('year')) groups.setdefault(identity, []).append(selection['text']) for (person, kind, _year), selections in groups.items(): diff --git a/tools/graphrag/evaluation/reproducibility.py b/tools/graphrag/evaluation/reproducibility.py index fde51c6..679fd3b 100644 --- a/tools/graphrag/evaluation/reproducibility.py +++ b/tools/graphrag/evaluation/reproducibility.py @@ -45,7 +45,7 @@ def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]: 'tools/answer_evaluation/checked_generation.py', 'tools/answer_evaluation/answer_format.py', 'tools/answer_evaluation/render_answer.py', 'tools/answer_evaluation/graph_context.py', 'tools/answer_evaluation/presentation.py', - 'tools/answer_evaluation/rag_evidence.py', + 'tools/answer_evaluation/rag_evidence.py', 'tools/answer_evaluation/verified_rag.py', 'tools/rag/llm_context.py', 'tools/answer_evaluation/question_intent.py', 'tools/answer_evaluation/content_selection.py', 'tools/answer_evaluation/rag_preparation.py', 'tools/answer_evaluation/answer_coverage.py', 'tools/answer_evaluation/graph_answer_prompt.txt', diff --git a/tools/rag/llm_context.py b/tools/rag/llm_context.py index 4e3dc14..fe5b2a1 100644 --- a/tools/rag/llm_context.py +++ b/tools/rag/llm_context.py @@ -4,6 +4,7 @@ from __future__ import annotations import copy import json import os +import re from tools.rag.build_chunks import TokenCounter from tools.retrieval.text_budget import split_to_fit @@ -62,6 +63,60 @@ def compact_rag_payload(data: dict, *, max_tokens: int | None = None, def fits(payload): return payload['context_budget']['used_tokens'] <= limit + def whole_prefix(body,end): + boundaries=[m.end() for m in re.finditer( + r'\n\s*\n|(?=^[ \t]*(?:\d+[.)]|[-+*])\s+|^#{1,6}\s|^Sekcia:)',body,re.M) + if m.end()<=end] + return body[:max(boundaries,default=0)] + + def balanced_sources(sources): + """Reserve evidence space for every requested person before expanding.""" + suffix='\n[OBSAH ZDROJA OREZANÝ]' + selected=[{**s,'text':suffix if s.get('text') else ''} for s in sources] + if not fits(render(selected,truncated=True)): + raise ValueError('Rozpočet nestačí na metadáta všetkých porovnávaných osôb') + families={s.get('document_path','').removesuffix('README.md'):s['title'] for s in sources + if s.get('title') in base['named_subjects'] and s.get('document_path')} + groups={} + for i,s in enumerate(sources): + owner=next((p for path,p in families.items() if s.get('document_path','').startswith(path)),f'other:{i}') + groups.setdefault(owner,[]).append(i) + + def used():return render(selected,truncated=True)['context_budget']['used_tokens'] + + def expand(index,budget): + body=sources[index].get('text') or '' + if not body:return + ceiling=min(limit,used()+max(0,budget)) + def candidate(text): + replacement={**selected[index],'text':text+(suffix if len(text)1: + result=balanced_sources(sources) + elif not fits(result): selected = [] if not fits(render([], truncated=True)): raise ValueError('Tokenový rozpočet je menší ako povinné pokyny a metadáta') @@ -93,7 +150,7 @@ def compact_rag_payload(data: dict, *, max_tokens: int | None = None, middle = (low + high + 1) // 2 if body_fits(body[:middle]): low = middle else: high = middle - 1 - selected.append({**source, 'text': body[:low] + suffix}) + selected.append({**source, 'text': whole_prefix(body,low) + suffix}) break result = render(selected, truncated=True) if not selected: