diff --git a/README.md b/README.md index bccc360..e5e4271 100644 --- a/README.md +++ b/README.md @@ -1500,6 +1500,13 @@ známe rozkazovacie formulácie zadania menia na pomenovania cieľov, napríklad pravidlá po kontrole doslovného dôkazu, nie o voľné dopĺňanie faktov modelom. Neznámy opis, podmienky, negácie a údaje o dosiahnutom stave zostávajú zachované. +Pri obsahovom porovnaní viacerých študentov sa pre každého načíta požadovaný typ +práce z rovnakého SQLite snapshotu; všetky podklady naďalej podliehajú spoločnému +tokenovému rozpočtu. Zadanie a zaznamenaný stav sa kontrolujú a zobrazujú oddelene. +Chýbajúce výsledky sa oznámia, neodvodzujú sa z cieľov. Grafová otázka na roky, +v ktorých majú práce „obaja“, používa prienik rokov a prikladá doložené typy a názvy +prác z týchto rokov; nezobrazuje namiesto odpovede všetky individuálne roky. + Otázky na obsah, zadanie, ciele alebo metódy práce používajú textové výňatky z príslušnej akademickej sekcie, nie iba metadáta o názve a roku. Výňatok musí patriť požadovanej osobe, typu práce a prípadne roku. Názov, rok ani samotný diff --git a/test/test_answer_presentation.py b/test/test_answer_presentation.py index 3a97ee3..b3025dd 100644 --- a/test/test_answer_presentation.py +++ b/test/test_answer_presentation.py @@ -25,7 +25,7 @@ def test_identity_is_a_sentence_but_year_is_not_a_start_or_completion_date(): @pytest.mark.parametrize('person,question,expected', [ ('Ján Holp', 'Čomu sa venuje diplomová práca Jána Holpa?', 'Jána Holpa'), - ('Eva Nová', 'Aké je zadanie diplomovej práce Evy Novej?', None), + ('Eva Nová', 'Aké je zadanie diplomovej práce Evy Novej?', 'Evy Novej'), ('Ján Holp', 'Aké práce má Ján Holp?', None), ('Ján Holp', 'Aké práce má Martin Jančura?', None), ('Ján Holp', 'Aké je zadanie práce Jána Holpáka?', None), @@ -44,7 +44,7 @@ def test_content_changes_only_grammar_after_literal_evidence_passes(payload): before = copy.deepcopy(payload) result = generate_verified_rag(QUESTION, payload, lambda _: completion(description())) assert result['accepted_for_delivery'] - assert 'Diplomová práca Jána Holpa má podľa dostupného zadania tieto ciele:' in result['answer'] + assert 'Diplomová práca Jána Holpa má podľa dostupných podkladov tieto ciele:' in result['answer'] assert '1. Vypracovanie prehľadu metód ohodnotenia dokumentov' in result['answer'] assert '2. Implementácia systému pre získavanie informácií v slovenskom jazyku.' in result['answer'] assert 'Vypracujte' not in result['answer'] and 'Implementujte' not in result['answer'] diff --git a/test/test_comparison_answers.py b/test/test_comparison_answers.py new file mode 100644 index 0000000..5e22c2d --- /dev/null +++ b/test/test_comparison_answers.py @@ -0,0 +1,235 @@ +"""Real multi-person query regressions, including deliberately wrong evidence.""" +import copy +import json +import sqlite3 + +import pytest + +from tools.answer_evaluation.context_policy import answer_context_issues, text_contains_value +from tools.answer_evaluation.graph_context import graph_context +from tools.answer_evaluation.presentation import content_lines, goal_text +from tools.answer_evaluation.rag_evidence import (content_sections, expand_comparison_sources, + rag_evidence_context, requested_types) +from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues +from tools.answer_evaluation.render_answer import render_graph_answer +from tools.graphrag import graph_search as gs +from tools.graphrag.tests.test_graph_search import FakeSession +from tools.rag.llm_context import compact_rag_payload +from tools.rag.rag_utils import build_context_text + +COMPARE = ('Porovnaj zameranie bakalárskej práce Ondreja Megelu a diplomovej práce Jána Holpa. ' + 'Rozlišuj zadanie od dosiahnutých výsledkov a uveď zdroje.') +SHARED = 'V ktorých rokoch majú Ján Holp a Ondrej Megela obaja evidovanú prácu alebo projekt?' + + +@pytest.mark.parametrize('canonical,wording', [ + ('Ondrej Megela', 'Ondreja Megelu'), ('Martin Šarišský', 'Martina Šarišského'), + ('Ján Malinovský', 'Jána Malinovského'), ('Eva Nová', 'Evy Novej')]) +def test_complete_case_endings(canonical, wording): + assert text_contains_value(wording, canonical, person=True) + + +@pytest.mark.parametrize('canonical,other', [ + ('Martin Šarišský', 'Martin Jančura'), ('Ján Holp', 'Ján Holpák'), + ('Ondrej Megela', 'Ondrej Megelovec'), ('Martin NaN1', 'Martin NaN2')]) +def test_no_short_prefix_matching(canonical, other): + assert not text_contains_value(other, canonical, person=True) + + +def test_each_person_gets_the_requested_work_type(): + assert requested_types(COMPARE, ['Ján Holp', 'Ondrej Megela']) == { + 'Ján Holp': 'diploma_thesis', 'Ondrej Megela': 'bachelor_thesis'} + + +def test_goal_conversion_is_not_all_or_nothing(): + text = '\n'.join(content_lines(['- Zistite, čo je to RAG.', '- Naučte sa čo je to GraphRAG.', + '- Zostavte malý znalostný graf.', '- Využite graf pri vyhľadávaní.', + '- Použite databázu Neo4j.', 'Práca neimplementuje vlastný model.'], + 'Ján Malinovský', 'diploma_project', 'Aké sú ciele diplomového projektu Jána Malinovského?')) + assert 'Diplomový projekt Jána Malinovského' in text + assert 'Zistiť, čo je to RAG.' in text and 'Použiť databázu Neo4j.' in text + assert 'Ďalšie podklady:' in text and 'Práca neimplementuje vlastný model.' in text + assert not any(word in text for word in ['Zistite', 'Naučte', 'Zostavte', 'Využite', 'Použite']) + + +def test_prefixed_goal_keeps_other_converted_items(): + text = '\n'.join(content_lines(['1. Vypracujte prehľad systémov generovania prirodzeného jazyka.', + '2. Na základe tohto prehľadu navrhnite a implementujte agentový systém.'], + 'Martin Šarišský', 'diploma_thesis', 'Čomu sa venuje diplomová práca Martina Šarišského?')) + assert 'Diplomová práca Martina Šarišského' in text + assert 'Vypracovanie prehľadu' in text and 'navrhnúť a implementovať' in text + assert 'Vypracujte' not in text and 'navrhnite' not in text + + +def test_coordinated_commands_and_objects_keep_grammatical_infinitives(): + text='\n'.join(content_lines(['Vypracujte prehľad aplikácií BERT a spôsoby ich vyhodnotenia.', + 'Vypracujte prehľad metód a implementujte systém.'],'Ondrej Megela','bachelor_thesis')) + assert 'Vypracovať prehľad aplikácií BERT a spôsoby ich vyhodnotenia.' in text + assert 'Vypracovať prehľad metód a implementovať systém.' in text + + +def test_goal_headings_are_not_extra_evidence_and_sentences_keep_capitals(): + text='\n'.join(content_lines(['Ciele na semester.', 'Zlepšite proces. Zlepšite extrakciu.'], + 'Ján Malinovský','diploma_project')) + assert 'Ďalšie podklady' not in text and 'Zlepšiť proces. Zlepšiť extrakciu.' in text + + +@pytest.mark.parametrize('literal', ['Použite https://example.test/pouzite/vypracujte.', + 'Implementujte parser pre `navrhnite a implementujte`.']) +def test_grammar_does_not_change_urls_or_code_literals(literal): + assert goal_text(literal)==(literal,False) + + +@pytest.fixture +def comparison_payload(tmp_path): + sources = []; rows = [] + for i, (person,slug,kind,year,goal,progress) in enumerate([ + ('Ján Holp','jan_holp','Diplomová práca',2021,'Implementujte systém vyhľadávania.', + 'Implementovaný PageRank a indexovanie webových stránok.'), + ('Ondrej Megela','ondrej_megela','Bakalárska práca',2020,'Natrénujte jazykový model metódou BERT.', + 'Natrénovaný model wiki103 a vykonané vyhodnotenie.')],1): + path=f'pages/students/2018/{slug}/README.md' + sources.append({'source_id':f'S{i}','title':person,'document_path':path, + 'source_url':f'https://example.test/{slug}','text':f'Sekcia: {kind} {year}\nNázov: Ukážkový názov práce.'}) + rows.append((path,json.dumps([[person,f'{kind} {year}']]),f'Zadanie:\n- {goal}\nStav:\n- {progress}',1,0)) + rows.append((path,json.dumps([[person,'Diplomový projekt 2022']]),'Stav:\nCudzí projekt využíva Cassandra.',1,1)) + db=tmp_path/'index.sqlite' + with sqlite3.connect(db) as conn: + conn.execute('CREATE TABLE chunks(document_path,heading_paths_json,text,published,chunk_index)') + conn.executemany('INSERT INTO chunks VALUES(?,?,?,?,?)',rows) + raw={'status':'ok','reason':None,'sources':sources,'context':build_context_text(sources), + 'complete':False,'truncated':True,'truncation_reasons':['result_limit'],'index_version':{'version':'fixture'}} + return db,raw + + +def selected_comparison(context): + quotes=[{'source_id':p['source_id'],'quote':s['text']} for p in context['content_passages'] + for s in content_sections(p['text']) if s['kind'] in {'assignment','progress'}] + return {'status':'ok','answer':'','claims':[],'evidence_quotes':quotes, + 'source_urls':[s['source_url'] for s in context['sources']]} + + +def test_expansion_preserves_binding_versions_and_budget(comparison_payload): + db,raw=comparison_payload; before=copy.deepcopy(raw) + payload=compact_rag_payload(expand_comparison_sources(db,COMPARE,raw)) + c=rag_evidence_context(COMPARE,payload) + assert c['requirements']['work_types']=={'Ján Holp':'diploma_thesis','Ondrej Megela':'bachelor_thesis'} + assert c['requirements']['content_roles']=={p:['assignment','progress'] for p in c['requirements']['people']} + assert 'Cassandra' not in payload['context'] + assert payload['index_version']==raw['index_version'] and payload['complete'] is False + assert payload['context_budget']['used_tokens']<=payload['context_budget']['max_tokens'] + assert raw==before + + +def test_hidden_chunks_are_not_added_to_comparison_evidence(comparison_payload): + db,raw=comparison_payload; path=raw['sources'][0]['document_path'] + with sqlite3.connect(db) as conn: + conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path, + json.dumps([['Ján Holp','Diplomová práca 2021']]),'Stav:\nNezverejnený tajný výsledok.',0,2)) + expanded=expand_comparison_sources(db,COMPARE,raw) + assert 'tajný výsledok' not in expanded['context'] + + +def test_relevant_child_document_is_preserved(comparison_payload): + db,raw=comparison_payload + child={**raw['sources'][0],'source_id':'S3','title':'Meranie projektu', + 'document_path':raw['sources'][0]['document_path'].replace('README.md','meranie/README.md'), + 'text':'Sekcia: Diplomová práca 2021\nStav:\nVyhodnotená presnosť vyhľadávania.'} + raw['sources'].append(child); raw['context']=build_context_text(raw['sources']) + expanded=expand_comparison_sources(db,COMPARE,raw) + assert 'Vyhodnotená presnosť vyhľadávania.' in expanded['context'] + + +def test_comparison_separates_goals_and_results(comparison_payload): + db,raw=comparison_payload; payload=compact_rag_payload(expand_comparison_sources(db,COMPARE,raw)) + c=rag_evidence_context(COMPARE,payload); selected=selected_comparison(c) + assert rag_answer_issues(selected,c)==[] + result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]}) + assert result['accepted_for_delivery'] + text=result['answer'] + assert text.count('**Zadanie a ciele:**')==text.count('**Zaznamenaný stav a výsledky:**')==2 + assert 'Natrénovať jazykový model metódou BERT.' in text + assert 'Natrénovaný model wiki103' in text and 'Implementovaný PageRank' in text + assert 'Cassandra' not in text and len(result['source_urls'])==2 + + +def test_goals_cannot_replace_available_results(comparison_payload): + db,raw=comparison_payload; c=rag_evidence_context(COMPARE,expand_comparison_sources(db,COMPARE,raw)) + a=selected_comparison(c) + a['evidence_quotes']=[q for q in a['evidence_quotes'] if not any( + word in q['quote'] for word in ['Implementovaný','Natrénovaný'])] + assert 'missing_progress_evidence' in rag_answer_issues(a,c) + + +def test_missing_results_allow_an_explicit_partial_comparison(comparison_payload): + db,raw=comparison_payload + with sqlite3.connect(db) as conn: + conn.execute("UPDATE chunks SET text='Zadanie:\n- Implementujte systém vyhľadávania.' WHERE document_path LIKE '%jan_holp%' AND chunk_index=0") + payload=expand_comparison_sources(db,COMPARE,raw); c=rag_evidence_context(COMPARE,payload) + selected=selected_comparison(c) + result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]}) + assert result['accepted_for_delivery'] + assert 'nie sú dosiahnuté výsledky doložené' in result['answer'] + assert 'Implementácia systému vyhľadávania' in result['answer'] + + +def test_future_tasks_are_not_presented_as_achieved_results(comparison_payload): + db,raw=comparison_payload + with sqlite3.connect(db) as conn: + conn.execute("UPDATE chunks SET text=text || '\nÚlohy:\n- Pripravte experiment s PageRank.' WHERE document_path LIKE '%jan_holp%' AND chunk_index=0") + payload=expand_comparison_sources(db,COMPARE,raw); c=rag_evidence_context(COMPARE,payload) + selected=selected_comparison(c) + p=next(p for p in c['content_passages'] if p['person']=='Ján Holp') + selected['evidence_quotes'].append({'source_id':p['source_id'],'quote':'- Pripravte experiment s PageRank.'}) + result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]}) + assert result['accepted_for_delivery'] + assert '**Ďalšie doložené úlohy (plán):**' in result['answer'] + assert 'Pripraviť experiment s PageRank.' in result['answer'] + + +@pytest.fixture +def shared_session(): + catalog=[]; theses=[]; works=[] + for pid,person,slug in [('p1','Ján Holp','jan'),('p2','Ondrej Megela','ondrej')]: + path=f'pages/students/2018/{slug}/README.md'; url=f'https://example.test/{slug}' + catalog.append({'element_id':pid,'kind':'Person','name':person,'source_document':path}) + base={'person_element_id':pid,'person':person,'document_path':path,'source_url':url} + theses.append({**base,'thesis_element_id':pid+'t','thesis_id':pid+'t','title':'Ukážkový názov', + 'year':2019 if pid=='p1' else 2020,'heading':'Bakalárska práca', 'thesis_type':'bachelor_thesis'}) + works.append({**base,'work_element_id':pid+'w','work_id':pid+'w','year':2020, + 'heading':'Diplomový projekt','work_type':'diploma_project'}) + return FakeSession({gs.CATALOG_QUERY:catalog,gs.THESIS_FACTS_QUERY:theses,gs.WORK_FACTS_QUERY:works}) + + +@pytest.mark.parametrize('word',['obaja','obidvaja']) +def test_both_means_shared_years(word): + c=gs.extract_query_constraints(SHARED.replace('obaja',word)) + assert c['mentions_shared'] and c['mentions_academic_year'] + + +def test_shared_year_has_only_intersection_and_bound_work_details(shared_session): + result=gs.search_graph_with_session(shared_session,SHARED) + assert result['academic_year_comparison']['shared_years']==[2020] + c=graph_context(result); records=c['required_academic_records'] + assert len(records)==3 and all(r['year']==2020 for r in records) + a={'status':'ok','claims':records,'source_urls':[s['source_url'] for s in c['sources']]} + rendered=render_graph_answer(a,c) + assert answer_context_issues(rendered,c)==[] + assert 'Spoločný evidovaný rok je **2020**.' in rendered['answer'] + assert '2019' not in rendered['answer'] and 'Ukážkový názov' in rendered['answer'] + + +def test_no_common_year_is_not_a_union(shared_session): + for row in shared_session.responses[gs.WORK_FACTS_QUERY]: + if row['person_element_id']=='p1': row['year']=2021 + result=gs.search_graph_with_session(shared_session,SHARED) + assert result['status']=='no_answer' and result['academic_year_comparison']['shared_years']==[] + c=graph_context(result) + assert 'spoločný rok nenašiel' in render_graph_answer({'status':'no_answer','claims':[],'source_urls':[]},c)['answer'] + + +def test_partial_records_do_not_claim_an_unwitnessed_common_year(shared_session): + result=gs.search_graph_with_session(shared_session,SHARED,limit=1); c=graph_context(result) + a={'status':'ok','claims':c['required_academic_records'],'source_urls':[s['source_url'] for s in c['sources']]} + text=render_graph_answer(a,c)['answer'] + assert 'Výpis je neúplný.' in text and 'Spoločný evidovaný rok je' not in text diff --git a/tools/answer_evaluation/context_policy.py b/tools/answer_evaluation/context_policy.py index 98fa3c3..c92c06c 100644 --- a/tools/answer_evaluation/context_policy.py +++ b/tools/answer_evaluation/context_policy.py @@ -133,13 +133,26 @@ def prose_issues(answer: dict) -> list[str]: return ['unsupported_prose_number'] if unsupported else [] -def text_contains_value(text: str, value: str, *, person: bool = False) -> bool: - value = normalized(value); text = normalized(text) - if not person: return value in text +def person_token_pattern(token: str) -> str: + """Bounded case endings of a complete name token, never fuzzy prefixes.""" + token = normalized(token) suffix = r'(?:a|u|ovi|om|e|ou|y|ovej|oveho|ym|emu)?' - pattern = r'\b' + r'\s+'.join(re.escape(token) + (suffix if token.isalpha() else '') - for token in value.split()) + r'\b' - return bool(re.search(pattern, text)) + variants = [re.escape(token) + (suffix if token.isalpha() else '')] + if token.isalpha() and len(token) >= 3: + if token.endswith('a'): + variants.append(re.escape(token[:-1]) + r'(?:u|y|e|ou|ovi|om|ej)') + elif token.endswith('y'): + variants.append(re.escape(token[:-1]) + r'(?:eho|emu|om|ym)') + return '(?:' + '|'.join(variants) + ')' + + +def person_pattern(person: str) -> str: + return r'\b' + r'\s+'.join(person_token_pattern(token) for token in normalized(person).split()) + r'\b' + + +def text_contains_value(text: str, value: str, *, person: bool = False) -> bool: + if not person: return normalized(value) in normalized(text) + return bool(re.search(person_pattern(value), normalized(text))) def _same_claim_value(key: str, actual, expected) -> bool: diff --git a/tools/answer_evaluation/graph_answer_prompt.txt b/tools/answer_evaluation/graph_answer_prompt.txt index f593d5a..b7709e3 100644 --- a/tools/answer_evaluation/graph_answer_prompt.txt +++ b/tools/answer_evaluation/graph_answer_prompt.txt @@ -6,6 +6,7 @@ Title je názov akademickej práce/projektu, nikdy názov profilovej stránky al Všetky podmienky jednej claim patria tej istej osobe. Title a year patria rovnakému akademickému záznamu. Category a document_author musia byť doložené tým istým document_path. Tag patrí osobe cez jej profil aj dcérske dokumenty. Pre kombináciu názvu/roku a tagu nevyžaduj rovnaký dokument. Answer_candidates sú už doložené spojenia cez tú istú osobu; zachovaj ich person, year, keyword a prípadne title spolu v jednej claim. Neodmietaj toto spojenie len pre odlišné document_path. Claims musia pokryť všetky relevantné výsledky dotazu a požadované väzby. Pri porovnaní ľudí uveď každého v samostatnej claim. Pri výbere kandidátov vynechaj nevyhovujúce osoby. Pri názve práce uveď aj rok, ak je vo faktoch známy. +Academic_year_comparison opisuje spoločné roky všetkých požadovaných osôb, nie zjednotenie ich rokov. Pri dodaných required_academic_records zachovaj aj typy, názvy a identity prác z týchto rokov. Nepridávaj práce z iných rokov. Source_urls musí obsahovať konkrétne relevantné source_url zo sources pre všetky tvrdenia. Nevymýšľaj ani neupravuj URL. Ak názov/rok dokladá profil a tag dcérsky dokument, cituj oba potrebné zdroje. Nepoužívaj URL iných osôb ako náhradnú citáciu. Count môže byť číslo iba pri aggregation.complete=true a musí sa rovnať aggregation.count. Inak count=null, aj pri no_answer. Počet nie je počet zobrazených zdrojov. Presný agregovaný počet môže byť úplný aj pri orezanom výpise dôkazov. Pri status kontextu no_answer vráť status=no_answer, claims=[], count=null. Pri ambiguous alebo error zachovaj tento status a prázdne claims. Nevymýšľaj pozitívnu odpoveď. Nulová úplná agregácia so status=ok je platný výsledok s count=0. diff --git a/tools/answer_evaluation/graph_context.py b/tools/answer_evaluation/graph_context.py index cf48cad..be0190e 100644 --- a/tools/answer_evaluation/graph_context.py +++ b/tools/answer_evaluation/graph_context.py @@ -9,7 +9,8 @@ def graph_context(result: dict) -> dict: Values/subjects/source paths are copied without inventing or rewriting facts. """ facts = []; intent = result['constraints'] - work_listing = intent.get('mentions_work_listing', False) + shared_year_query = intent.get('mentions_shared') and intent.get('mentions_academic_year') + work_listing = intent.get('mentions_work_listing', False) or shared_year_query academic_records = [] for row in result['facts']: kind = row.get('fact_type', '') @@ -55,6 +56,10 @@ def graph_context(result: dict) -> dict: context['question'] = result.get('query', '') context['requested_people'] = list(dict.fromkeys(row['name'] for row in result.get('resolved_entities', []) if row.get('kind') == 'Person' and row.get('name'))) + comparison = result.get('academic_year_comparison') or next((row for row in result['facts'] + if row.get('fact_type') == 'academic_year_comparison'), None) + if comparison is not None: + context['academic_year_comparison'] = comparison if work_listing: context['required_academic_records'] = academic_records return context diff --git a/tools/answer_evaluation/presentation.py b/tools/answer_evaluation/presentation.py index 7e96c96..e7d8161 100644 --- a/tools/answer_evaluation/presentation.py +++ b/tools/answer_evaluation/presentation.py @@ -7,7 +7,7 @@ from __future__ import annotations import re -from .context_policy import normalized +from .context_policy import normalized, person_token_pattern, text_urls from .rag_evidence import WORK_LABELS @@ -30,9 +30,7 @@ def owner_in_question(person: str, question: str) -> str | None: continue valid = True for token, match in zip(tokens, found): - base = normalized(token) - variants = {base, base + 'a', base + 'u'} - valid &= normalized(match[0]) in variants + valid &= bool(re.fullmatch(person_token_pattern(token), normalized(match[0]))) if valid: return question[found[0].start():found[-1].end()] return None @@ -100,24 +98,47 @@ GOAL_VERBS = { 'vyhodnoťte': 'Vyhodnotiť', 'porovnajte': 'Porovnať', 'vytvorte': 'Vytvoriť', 'analyzujte': 'Analyzovať', 'otestujte': 'Otestovať', 'overte': 'Overiť', + 'zistite': 'Zistiť', 'naučte': 'Naučiť', 'zostavte': 'Zostaviť', + 'využite': 'Využiť', 'zlepšite': 'Zlepšiť', 'použite': 'Použiť', + 'natrénujte': 'Natrénovať', 'optimalizujte': 'Optimalizovať', + 'identifikujte': 'Identifikovať', 'preštudujte': 'Preštudovať', + 'pripravte': 'Pripraviť', } def goal_text(unit: str) -> tuple[str, bool]: """Only bounded grammatical substitutions; unknown prose stays verbatim.""" + if text_urls(unit) or '`' in unit: + # A word inside a URL/code literal must never be conjugated. Preserve + # technical excerpts exactly rather than changing their references. + return unit.strip(), False text = re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+', '', unit.strip()) for original, replacement in GOAL_NOUNS: if re.match(re.escape(original) + r'\b', text, re.I): + tail = text[len(original):] + # Infinitives retain the original object cases when a second + # command or coordinated object makes a noun substitution unsafe. + if (re.search(r'\b(?:' + '|'.join(map(re.escape, GOAL_VERBS)) + r')\b', tail, re.I) + or re.search(r'\ba\s+(?:spôsoby|metódy|aplikácie|systémy|experimenty|možnosti)\b', tail, re.I)): + break result = replacement + text[len(original):] result = result.replace('v ktorom využijete', 'v ktorom sa má využiť') result = result.replace(', v ktorých vyhodnotíte', ' s cieľom vyhodnotiť') result = result.replace('v ktorých vyhodnotíte', 's cieľom vyhodnotiť') return result, True - first = text.split(maxsplit=1)[0].casefold() if text else '' - if first in GOAL_VERBS: + match = re.match(r'[^\W\d_]+', text) + first = match[0].casefold() if match else '' + if first in GOAL_VERBS or (first == 'na' and text.casefold().startswith('na základe ') + and re.search(r'\b(?:' + '|'.join(map(re.escape, GOAL_VERBS)) + r')\b', text, re.I)): + def infinitive(match): + word = GOAL_VERBS[match[0].casefold()] + prefix = text[:match.start()].rstrip() + return word if not prefix or prefix[-1] in '.!?' else word.lower() result = re.sub(r'\b(' + '|'.join(map(re.escape, GOAL_VERBS)) + r')\b', - lambda match: GOAL_VERBS[match[0].casefold()].lower(), text, flags=re.I) + infinitive, text, flags=re.I) return result[0].upper() + result[1:], True + if first in {value.casefold() for value in GOAL_VERBS.values()}: + return text, True return unit.strip(), False @@ -127,12 +148,18 @@ def content_lines(units: list[str], person: str | None, kind: str | None, # A task heading has no factual content of its own and otherwise prevents # a whole assignment quote from being presented as individual goals. units = [unit for unit in units if normalized(unit) not in { - 'zadanie', 'zadanie diplomovej prace', 'zadanie bakalarskej prace', 'ciele prace'}] + 'zadanie', 'zadanie diplomovej prace', 'zadanie bakalarskej prace', + 'ciele prace', 'ciele na semester', 'navrh na zadanie', 'navrh zadania'}] converted = [goal_text(unit) for unit in units] subject = work_subject(person, kind, question) if person else 'Práca' - if converted and all(is_goal for _, is_goal in converted): - return [subject + ' má podľa dostupného zadania tieto ciele:', '', - *[f'{index}. {text}' for index, (text, _) in enumerate(converted, 1)]] + goals = [text for text, is_goal in converted if is_goal] + other = [text for text, is_goal in converted if not is_goal] + if goals: + lines = [subject + ' má podľa dostupných podkladov tieto ciele:', '', + *[f'{index}. {text}' for index, text in enumerate(goals, 1)]] + if other: + lines.extend(['', 'Ďalšie podklady:', *other]) + return lines return [subject + ' je v dostupných podkladoch opísaná takto:' if not (kind or '').endswith('_project') else subject + ' je v dostupných podkladoch opísaný takto:', '', *units] diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index 4db2e63..2e42ee5 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -6,7 +6,10 @@ Taxonomy years are deliberately not treated as years of academic works. from __future__ import annotations import re -from .context_policy import normalized, text_contains_value +import json +import sqlite3 +from contextlib import closing +from .context_policy import normalized, text_contains_value, person_pattern WORK_TYPES = { 'bakalarska praca': 'bachelor_thesis', 'diplomova praca': 'diploma_thesis', @@ -102,10 +105,9 @@ def requested_types(question: str, people: list[str]) -> dict[str, str]: result[names[0]] = next(iter(kinds)) if len(result) == len(people): return result - suffix = r'(?:a|u|ovi|om|e|ou|y|ovej|oveho|ym|emu)?' positions = [] for person in people: - pattern = r'\b' + r'\s+'.join(re.escape(token) + suffix for token in normalized(person).split()) + r'\b' + pattern = person_pattern(person) positions.extend((m.start(), m.end(), person) for m in re.finditer(pattern, text)) for start, end, kind in types: distances = [(max(start - right, left - end, 0), person) for left, right, person in positions] @@ -209,6 +211,105 @@ def academic_content_passages(blocks: list[dict], facts: list[dict], requirement return result +def content_sections(text: str) -> list[dict]: + """Separate explicit source goals from recorded progress, without inference.""" + sections = []; lines = []; kind = 'other' + def flush(): + if lines and '\n'.join(lines).strip(): + sections.append({'kind': kind, 'text': '\n'.join(lines).strip()}) + lines.clear() + for line in text.splitlines(): + value = normalized(line) + label = line.strip().strip('*_# ').rstrip(':').strip() + heading = value == normalized(label) and len(value.split()) <= 8 + next_kind = None + if heading and value in {'ciel', 'ciele', 'ciele na semester', 'ciele prace', 'zadanie', + 'zadanie diplomovej prace', 'zadanie bakalarskej prace', + 'zadanie diplomoveho projektu', 'zadanie bakalarskeho projektu', + 'navrh na zadanie', 'navrh zadania'}: + next_kind = 'assignment' + elif heading and value in {'stav', 'vysledky', 'dosiahnute vysledky', 'dosiahnuty stav'}: + next_kind = 'progress' + elif heading and value in {'ulohy', 'zasobnik uloh', 'dalsie ulohy'}: + next_kind = 'task' + elif value.startswith(('stretnutie ', 'virtualne stretnutie ', 'sekcia ')): + next_kind = 'other' + if next_kind is not None: + flush(); kind = next_kind + else: + lines.append(line) + flush() + return sections + + +def quote_sections(quote: dict, context: dict) -> list[dict]: + """Classify only literal excerpts bound to the delivered academic passage.""" + full = [section for passage in context.get('content_passages', []) + if passage['source_id'] == quote['source_id'] and + ' '.join(quote['quote'].split()) in ' '.join(passage['text'].split()) + for section in content_sections(passage['text'])] + pieces = content_sections(quote['quote']) + for piece in pieces: + if piece['kind'] == 'other': + kinds = {section['kind'] for section in full + if ' '.join(piece['text'].split()) in ' '.join(section['text'].split())} + if len(kinds) == 1: + piece['kind'] = kinds.pop() + return pieces + + +def expand_comparison_sources(db_file, question: str, payload: dict) -> dict: + """Read requested academic sections from the same SQLite snapshot. + + Multi-person comparisons need each person's requested work, not the single + global section intent used by ordinary retrieval expansion. The caller + holds index_snapshot; the normal shared LLM budget is applied afterwards. + """ + context = rag_evidence_context(question, payload) + req = context['requirements'] + if not req.get('content') or len(req['people']) < 2: + return payload + from tools.rag.rag_utils import build_context_text + selected = []; changed = False + families = [source['document_path'].removesuffix('README.md') for source in payload.get('sources', []) + if source.get('title') in req['people'] and PROFILE.fullmatch(source.get('document_path', ''))] + with closing(sqlite3.connect(db_file.resolve().as_uri() + '?mode=ro', uri=True)) as conn: + conn.row_factory = sqlite3.Row + columns = {row[1] for row in conn.execute('PRAGMA table_info(chunks)')} + if not {'text', 'heading_paths_json', 'document_path', 'published', 'chunk_index'} <= columns: + return payload + for source in payload.get('sources', []): + person = source.get('title') + if person not in req['people'] or not PROFILE.fullmatch(source.get('document_path', '')): + if any(source.get('document_path', '').startswith(prefix) for prefix in families): + selected.append(source) + continue + frames = []; seen = set() + rows = conn.execute('''SELECT text, heading_paths_json FROM chunks + WHERE document_path=? AND COALESCE(published,1)=1 ORDER BY chunk_index''', + (source['document_path'],)) + for row in rows: + paths = json.loads(row['heading_paths_json'] or '[]') + headings = [' > '.join(path) if isinstance(path, list) else str(path) for path in paths] + first = headings[0] if headings else '' + for scope, text in academic_windows('Sekcia: ' + first + '\n' + row['text']): + if not scope or (req['work_types'].get(person) and scope[0] != req['work_types'][person]): + continue + if req['unsupported_year'] or (req['years'] and scope[1] not in req['years']): + continue + if not descriptive_text(text) or (scope, text) in seen: + continue + seen.add((scope, text)) + frames.append('Sekcia: ' + WORK_LABELS[scope[0]] + + (f' {scope[1]}' if scope[1] is not None else '') + '\n' + text) + selected.append({**source, **({'text': '\n\n'.join(frames)} if frames else {})}) + changed |= bool(frames) + if not changed: + return payload + selected = [{**source, 'source_id': f'S{index}'} for index, source in enumerate(selected, 1)] + return {**payload, 'sources': selected, 'source_count': len(selected), 'context': build_context_text(selected)} + + def rag_evidence_context(question: str, payload: dict) -> dict: blocks = source_blocks(payload) if payload.get('sources') else [] facts = [] @@ -316,6 +417,8 @@ def rag_evidence_context(question: str, payload: dict) -> dict: from .answer_coverage import response_scope requirements['content_style'] = response_scope(question, content=True) requirements['response_mode'] = 'mixed' if any(requirements[key] for key in ['title', 'year', 'start_year']) else 'quotes' + requirements['distinguish_progress'] = bool(re.search(r'\b(?:zadani\w*|ciel\w*)\b', query) + and re.search(r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b', query)) stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else [] requirements['years'] = ([int(value) for value in stated_years if len(value) == 4] if not requirements['start_year'] else []) @@ -328,6 +431,12 @@ def rag_evidence_context(question: str, payload: dict) -> dict: 'aggregation': None, 'answer_candidates': []} if content: context['content_passages'] = academic_content_passages(blocks, facts, requirements, question) + if requirements['distinguish_progress']: + requirements['content_roles'] = {person: sorted({section['kind'] + for p in context['content_passages'] if p['person'] == person + for section in content_sections(p['text']) + if section['kind'] in {'assignment', 'progress'} and descriptive_text(section['text'])}) + for person in requested} return context diff --git a/tools/answer_evaluation/render_answer.py b/tools/answer_evaluation/render_answer.py index a670302..ebe5675 100644 --- a/tools/answer_evaluation/render_answer.py +++ b/tools/answer_evaluation/render_answer.py @@ -10,6 +10,18 @@ def render_graph_answer(answer: dict, context: dict) -> dict: out = deepcopy(answer) lines = [] if out['status'] == 'ok': + comparison = context.get('academic_year_comparison') + if comparison: + people = comparison.get('people', []) + witnessed = [{claim['year'] for claim in out['claims'] + if claim.get('person') == person and claim.get('year') is not None} for person in people] + visible = set.intersection(*witnessed) if witnessed else set() + years = sorted(visible & set(comparison.get('shared_years', []))) + if years: + label = ('Spoločný evidovaný rok je' if len(years) == 1 else 'Spoločné evidované roky sú') + if context.get('complete') is not True or context.get('truncated'): + label = 'Medzi doložené spoločné roky patria' + lines.extend([label + ' **' + ', '.join(map(str, years)) + '**.', '']) records = {} for claim in out['claims']: if claim.get('record_id'): @@ -49,7 +61,10 @@ def render_graph_answer(answer: dict, context: dict) -> dict: elif out['status'] == 'error': lines.append('Vyhľadanie zlyhalo; odpoveď nemožno pripraviť.') else: - lines.append(no_answer_text(context)) + if context.get('academic_year_comparison') is not None: + lines.append('V dostupných podkladoch sa pre zadané podmienky spoločný rok nenašiel.') + else: + lines.append(no_answer_text(context)) if out['source_urls']: lines.extend(['', ('Zdroj: ' if len(out['source_urls']) == 1 else 'Zdroje: ') + ', '.join(out['source_urls'])]) out['answer'] = '\n'.join(lines) diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index ebcff43..8784f0c 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -9,9 +9,9 @@ from pydantic import BaseModel, ConfigDict, Field from .answer_format import Claim from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized, text_urls -from .rag_evidence import descriptive_text, rag_evidence_context +from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections from .answer_coverage import answer_coverage, coverage_notices -from .presentation import content_lines, fact_sentences, no_answer_text +from .presentation import content_lines, fact_sentences, no_answer_text, work_subject class RagClaim(Claim): @@ -56,6 +56,15 @@ Pri requirements.content_style=overview vyber najviac DVA krátke doslovné vý na požadovanú osobu, najviac 600 znakov a dve odrážky/body na osobu. Odpovedaj stručne na obsah; neprepisuj celé zadanie a históriu stretnutí. Pri content_style=assignment zachovaj požadované body zadania; tento stručný limit sa na zadanie nevzťahuje. +Pri requirements.distinguish_progress=true porovnaj dostupné podklady každej osoby: +vyber samostatne výňatok zadania/cieľov a samostatne výňatok zaznamenaného stavu. +requirements.content_roles uvádza, ktoré z týchto podkladov sú naozaj dostupné pre každú osobu. +Zadanie nie je dosiahnutý výsledok. Nepíš vlastný opis ani nedoplň chýbajúci výsledok. +Aj keď výsledky pre osobu chýbajú, môžeš vrátiť status=ok s doloženým zadaním; +aplikácia uvedie, čo chýba. no_answer znamená žiadnu odpoveď a musí mať prázdne evidence_quotes. +Pri zaznamenanom stave uprednostni konkrétnu implementáciu, natrénovanie modelu a vykonané +vyhodnotenie pred konzultáciami alebo samotným písaním textu. Ani priebežný stav nevydávaj +za splnenie všetkých cieľov alebo za konečné výsledky práce. Každý výňatok musí byť JEDEN súvislý úsek z konkrétneho zdroja vrátane Markdownu. Nespájaj oddelené pasáže cez vynechané riadky; použi viac samostatných kratších quotes. Vyber iba minimum potrebné na zodpovedanie otázky. Výňatok nesmieš parafrázovať ani meniť čísla. @@ -217,6 +226,18 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: ' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) for p in passages for quote in quotes): issues.append('missing_bound_content_evidence') + if requirements.get('distinguish_progress'): + bound_quotes = [quote for quote in quotes if any( + quote['source_id'] == p['source_id'] and + ' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) for p in passages)] + kinds = {section['kind'] for quote in bound_quotes for section in quote_sections(quote, context) + if descriptive_text(section['text'])} + for kind in requirements.get('content_roles', {}).get(person, []): + if kind not in kinds: + issues.append('missing_' + kind + '_evidence') + if (requirements.get('distinguish_progress') and answer['status'] == 'no_answer' and requested + and all(any(p.get('person') == person for p in context.get('content_passages', [])) for person in requested)): + issues.append('rejected_supported_content_comparison') if not citation_check(answer, allowed_urls(context))['citations_ok']: issues.append('unsupported_citation') if (claims or quotes) and not cited: @@ -271,6 +292,44 @@ def overview_excerpts(answer: dict, context: dict) -> list[str]: return [selection['text'] for selection in _overview_selections(answer, context)] +def comparison_lines(answer: dict, context: dict) -> list[str]: + """Present each person's goals and recorded state separately, including gaps.""" + lines = [] + for person in context['requirements']['people']: + kind = context['requirements']['work_types'].get(person) + lines.extend([work_subject(person, kind, context['question']), '']) + roles = {'assignment': [], 'progress': [], 'task': [], 'other': []} + for quote in answer.get('evidence_quotes', []): + if not any(p.get('person') == person and p['source_id'] == quote['source_id'] and + ' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) + for p in context.get('content_passages', [])): + continue + for section in quote_sections(quote, context): + key = section['kind'] if section['kind'] in roles else 'other' + roles[key].append({'source_id': quote['source_id'], 'quote': section['text']}) + for role, label in [('assignment', 'Zadanie a ciele'), ('progress', 'Zaznamenaný stav a výsledky')]: + lines.append('**' + label + ':**') + if roles[role]: + units = overview_excerpts({'evidence_quotes': roles[role]}, context) + if role == 'assignment': + lines.extend(content_lines(units, person, kind, context['question'])[1:]) + else: + lines.extend('- ' + re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+', '', unit) for unit in units) + else: + lines.append('V poskytnutých podkladoch ' + + ('nie je zadanie doložené.' if role == 'assignment' else 'nie sú dosiahnuté výsledky doložené.')) + lines.append('') + if roles['task']: + lines.append('**Ďalšie doložené úlohy (plán):**') + units = overview_excerpts({'evidence_quotes': roles['task']}, context) + lines.extend(content_lines(units, person, kind, context['question'])[1:]) + lines.append('') + if roles['other']: + lines.extend(['**Ďalšie doložené podklady:**', + *overview_excerpts({'evidence_quotes': roles['other']}, context), '']) + return lines + + def render_rag_answer(answer: dict, context: dict) -> str: lines = [] if answer['status'] == 'ok': @@ -278,7 +337,9 @@ def render_rag_answer(answer: dict, context: dict) -> str: lines.append(fact_sentences(claim, context.get('question', ''))) quotes = answer.get('evidence_quotes', []) style = context.get('requirements', {}).get('content_style') - if quotes and style in {'overview', 'assignment'}: + if quotes and context.get('requirements', {}).get('distinguish_progress'): + lines.extend(comparison_lines(answer, context)) + elif quotes and style in {'overview', 'assignment'}: # Bind each paragraph to the same source/person/work as the checked # quotes. Never summarize several people's excerpts under one name. groups = {} diff --git a/tools/answers/answer_delivery.py b/tools/answers/answer_delivery.py index d33e981..4ff9643 100644 --- a/tools/answers/answer_delivery.py +++ b/tools/answers/answer_delivery.py @@ -11,6 +11,8 @@ from tools.graphrag.graph_search import search_graph from tools.answer_evaluation.graph_context import graph_context from tools.answer_evaluation.checked_generation import generate_checked_answer from tools.answer_evaluation.verified_rag import generate_verified_rag +from tools.answer_evaluation.rag_evidence import expand_comparison_sources +from tools.sync.index_state import index_snapshot from tools.answers.model_client import complete_with_configured_model _capacity = threading.BoundedSemaphore(2) @@ -24,7 +26,10 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, try: model = complete or complete_with_configured_model if engine == 'rag': - retrieved = compact_rag_payload(build_rag_context(db_file, question, limit=limit)) + with index_snapshot(db_file.parent, db_file=db_file): + retrieved = build_rag_context(db_file, question, limit=limit) + retrieved = expand_comparison_sources(db_file, question, retrieved) + retrieved = compact_rag_payload(retrieved) result = generate_verified_rag(question, retrieved, model) context = retrieved elif engine == 'graphrag': diff --git a/tools/graphrag/graph_search.py b/tools/graphrag/graph_search.py index 1311910..1f6d749 100644 --- a/tools/graphrag/graph_search.py +++ b/tools/graphrag/graph_search.py @@ -947,6 +947,7 @@ def extract_query_constraints( "rovnak", ) or (mentions_academic_year and has_stem(tokens, "spaj")) + or (mentions_academic_year and bool(token_set & {"obaja", "obidvaja", "obe", "obidve", "vsetci"})) ) mentions_person_answer = ( @@ -2638,6 +2639,11 @@ def search_graph_with_session( "query": clean_query, "strategies": strategies, "constraints": constraints, "resolved_entities": resolved_entities, "paths": paths[:limit], "facts": facts[:limit], "documents": documents, "aggregation": aggregation, + "academic_year_comparison": ({ + "shared_years": shared_years, + "people": [row["name"] for row in resolved_entities if row.get("kind") == "Person"], + "complete": not (entity_limit_reached or fetch_capped), + } if shared_years is not None else None), "checked_entities": resolved_entities, "rejection_evidence": [row for row in facts if row.get("fact_type") in { "academic_year_comparison", "start_year_comparison"