diff --git a/README.md b/README.md index e5e4271..9253089 100644 --- a/README.md +++ b/README.md @@ -1160,7 +1160,7 @@ upozornenie na neúplnosť, nezamieňať autora dokumentu s autorom práce a pri sa nevysvetľujú odhadom; verzia dát neoznačuje verziu programu. Tieto pokyny ovplyvňujú správanie Claude, jeho finálny text však náš backend nemôže programovo vynútiť. Po aktualizácii už nahratých Skills nahraď ich -novými ZIPmi z `packages/claude/dist/` (verzia 1.2.0) a over správanie v novom chate. +novými ZIPmi z `packages/claude/dist/` (verzia 1.3.0) a over správanie v novom chate. ### Nahratie Skills do webového Claude @@ -1507,6 +1507,22 @@ Chýbajúce výsledky sa oznámia, neodvodzujú sa z cieľov. Grafová otázka n v ktorých majú práce „obaja“, používa prienik rokov a prikladá doložené typy a názvy prác z týchto rokov; nezobrazuje namiesto odpovede všetky individuálne roky. +`rag_preparation.py` vie pomenované profily dohľadať v SQLite aj mimo prvých +výsledkov a pri viacerých možných prácach vyžiada typ alebo rok. Obsahové +porovnanie je obmedzené na štyri osoby naraz. `question_intent.py` rozlišuje +požadované problémy, metódy, experimenty, ciele a výsledky. `content_selection.py` +vyberá doslovné, typovo a osobne viazané podklady, odstraňuje opakovania a +uprednostňuje vecné výsledky pred administratívnymi poznámkami. Model dostane +len tieto kandidátske výňatky; ich skutočný JSON podlieha tokenovému rozpočtu. +Chýbajúce časti sa označia jednotlivo, aj keď sú ostatné časti zodpovedateľné. +Pri limite kontextu sa nestratí informácia, ktoré osoby používateľ požadoval. + +Rozpoznané, ale nepodporované operátory (napríklad interval rokov, vylučovanie +alebo minimálny počet) vedú k `ambiguous` a konkrétnemu spresneniu. Rok práce +sa nepoužije ako dôkaz dátumu jej začatia či obhajoby. Chybové odpovede odlišujú +nedostupné vyhľadávanie, nedostupný model, vyťaženie a zamietnutý neoverený text; +raw výstupy odmietnutého modelu sa štandardnému klientovi neposkytujú. + Otázky na obsah, zadanie, ciele alebo metódy práce používajú textové výňatky z príslušnej akademickej sekcie, nie iba metadáta o názve a roku. Výňatok musí patriť požadovanej osobe, typu práce a prípadne roku. Názov, rok ani samotný diff --git a/app/routes.py b/app/routes.py index 2ad9df6..9d98c3c 100644 --- a/app/routes.py +++ b/app/routes.py @@ -49,6 +49,7 @@ from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY from tools.runtime.tool_health import tools_readiness from tools.retrieval.retrieval_contract import RetrievalResponse, RetrievalSource, RetrievalErrorResponse, error_reason from tools.answers.answer_delivery import deliver_answer +from tools.answers.answer_errors import public_failure logger = logging.getLogger(__name__) @@ -192,9 +193,7 @@ def verified_answer(request: AnswerRequest) -> dict: 'answer': 'Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.'} if (result.get('accepted_for_delivery') is not True or result.get('status') not in {'ok', 'no_answer', 'ambiguous'}): - result = {**result, 'status': 'error', 'accepted_for_delivery': False, - 'answer': 'Odpoveď neprešla overením podkladov. Neoverené údaje sa nezobrazia.', - 'claims': [], 'source_urls': []} + result = public_failure(result) if not request.include_diagnostics: result.pop('attempts', None) result.pop('retrieval_context', None) diff --git a/packages/claude/client.py b/packages/claude/client.py index bf659af..539b398 100644 --- a/packages/claude/client.py +++ b/packages/claude/client.py @@ -81,8 +81,22 @@ class Config: def error_result(reason): + messages={ + 'unauthorized':'Prístup nebol autorizovaný. Skontroluj prihlásenie konektora a jeho konfiguráciu.', + 'forbidden':'Pre tento nástroj nemáš potrebné oprávnenie.', + 'invalid_query':'Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.', + 'model_unavailable':'Jazykový model momentálne nie je dostupný. Skús otázku neskôr.', + 'answer_rejected':'Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. ' + 'Spresni osobu, typ práce alebo rok; neoverený text sa nezobrazí.', + 'answer_capacity_exceeded':'Agent práve spracúva iné otázky. Skús to o chvíľu.', + 'backend_busy':'Agent práve spracúva iné otázky. Skús to o chvíľu.', + 'backend_timeout':'Vyhľadávanie neodpovedalo včas. Skús otázku zopakovať neskôr.', + 'backend_unavailable':'Spojenie s backendom momentálne nie je dostupné. Skontroluj, či bežia služby a tunel.', + 'retrieval_unavailable':'Vyhľadávací index momentálne nie je dostupný. Skús otázku neskôr.', + } return {'status': 'error', 'reason': reason, 'accepted_for_delivery': False, - 'answer': '', 'sources': [], 'index_version': None, 'complete': False, + 'answer': messages.get(reason,'Overenú odpoveď sa momentálne nepodarilo získať.'), + 'sources': [], 'index_version': None, 'complete': False, 'truncated': False, 'truncation_reasons': []} diff --git a/packages/claude/dist/packages.json b/packages/claude/dist/packages.json index bd17241..bcbfe9a 100644 --- a/packages/claude/dist/packages.json +++ b/packages/claude/dist/packages.json @@ -3,12 +3,12 @@ { "archive": "zpwiki-rag.zip", "name": "rag", - "sha256": "8f0c51e27dc68a1e3ee845f2cfe5987e51714d4408f98c7d275861dc1e22953f" + "sha256": "09d3e6b196b61f6c025eb86ccd1773b913b37660163a995ad1a2acfa6d947a0c" }, { "archive": "zpwiki-graphrag.zip", "name": "graphrag", - "sha256": "aa65a97211f48fa0352a98e50c6ca6a243e1d23105c0ff144c7f360ad192c376" + "sha256": "cfcb60213e103d40c2adbc6c7d4133751120148415836cfffa7935c020167e95" } ], "schema_version": 1 diff --git a/packages/claude/dist/zpwiki-graphrag.zip b/packages/claude/dist/zpwiki-graphrag.zip index 4dbee2f..01336d6 100644 Binary files a/packages/claude/dist/zpwiki-graphrag.zip and b/packages/claude/dist/zpwiki-graphrag.zip differ diff --git a/packages/claude/dist/zpwiki-rag.zip b/packages/claude/dist/zpwiki-rag.zip index 67de16a..ec09e2d 100644 Binary files a/packages/claude/dist/zpwiki-rag.zip and b/packages/claude/dist/zpwiki-rag.zip differ diff --git a/scripts/build_claude_packages.py b/scripts/build_claude_packages.py index 8744b12..0dd4f7b 100644 --- a/scripts/build_claude_packages.py +++ b/scripts/build_claude_packages.py @@ -11,7 +11,7 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] BASE = ROOT / 'packages/claude' ENGINES = ('rag', 'graphrag') -PACKAGE_VERSION = '1.2.0' +PACKAGE_VERSION = '1.3.0' def json_bytes(value): diff --git a/test/test_claude_packages.py b/test/test_claude_packages.py index fabcf16..2114a87 100644 --- a/test/test_claude_packages.py +++ b/test/test_claude_packages.py @@ -156,7 +156,7 @@ def test_invalid_answer_coverage_is_not_delivered(backend, coverage): def test_transport_errors_and_redirects_do_not_disclose_payload(backend, code, reason): backend.update(raw=b'SECRET ERROR', code=code) value = client.query_backend('Práca?', 'rag') - assert value['reason'] == reason and value['answer'] == '' + assert value['reason'] == reason and value['answer'] assert len(backend['requests']) == 1 and 'SECRET' not in json.dumps(value) @@ -168,7 +168,7 @@ def test_rejected_or_malformed_answer_is_never_delivered(backend, change): backend.update(raw=json.dumps({**backend['checked'], **change}).encode()) value = client.query_backend('Práca?', 'rag') assert value['status'] == 'error' and not value['accepted_for_delivery'] - assert value['answer'] == '' and 'Eva' not in json.dumps(value) + assert value['answer'] and 'Eva' not in json.dumps(value) def test_invalid_json_and_oversized_response(backend): @@ -279,7 +279,7 @@ def test_mcp_stdio_initialization_discovery_call_and_rejection(backend): assert all('presentation_instructions' not in json.loads(block.text) for block in result.content if block.type == 'text') rejected = await session.call_tool('zpwiki_rag', {'query': ''}) - assert rejected.isError and rejected.structuredContent['answer'] == '' + assert rejected.isError and rejected.structuredContent['answer'] asyncio.run(run()) diff --git a/test/test_comparison_answers.py b/test/test_comparison_answers.py index 5e22c2d..1e93189 100644 --- a/test/test_comparison_answers.py +++ b/test/test_comparison_answers.py @@ -103,8 +103,11 @@ def comparison_payload(tmp_path): def selected_comparison(context): - quotes=[{'source_id':p['source_id'],'quote':s['text']} for p in context['content_passages'] + quotes=([{'source_id':c['source_id'],'quote':c['text']} for c in context['content_candidates']] + if 'content_candidates' in context else + [{'source_id':p['source_id'],'quote':s['text']} for p in context['content_passages'] for s in content_sections(p['text']) if s['kind'] in {'assignment','progress'}] + ) return {'status':'ok','answer':'','claims':[],'evidence_quotes':quotes, 'source_urls':[s['source_url'] for s in context['sources']]} @@ -169,7 +172,7 @@ def test_missing_results_allow_an_explicit_partial_comparison(comparison_payload selected=selected_comparison(c) result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]}) assert result['accepted_for_delivery'] - assert 'nie sú dosiahnuté výsledky doložené' in result['answer'] + assert 'Táto časť odpovede nie je v poskytnutých podkladoch doložená.' in result['answer'] assert 'Implementácia systému vyhľadávania' in result['answer'] @@ -182,9 +185,9 @@ def test_future_tasks_are_not_presented_as_achieved_results(comparison_payload): p=next(p for p in c['content_passages'] if p['person']=='Ján Holp') selected['evidence_quotes'].append({'source_id':p['source_id'],'quote':'- Pripravte experiment s PageRank.'}) result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]}) - assert result['accepted_for_delivery'] - assert '**Ďalšie doložené úlohy (plán):**' in result['answer'] - assert 'Pripraviť experiment s PageRank.' in result['answer'] + # The model may not quote a future task omitted from its actual candidates. + assert not result['accepted_for_delivery'] + assert 'quote_outside_delivered_candidates' in result['attempts'][-1]['issues'] @pytest.fixture diff --git a/test/test_quality_block.py b/test/test_quality_block.py new file mode 100644 index 0000000..7d4d821 --- /dev/null +++ b/test/test_quality_block.py @@ -0,0 +1,262 @@ +"""Behavioral checks for scope, relevant evidence, partial answers and failures.""" +import copy +import json +import sqlite3 + +import pytest + +from tools.answer_evaluation.content_selection import candidates, facets_for +from tools.answer_evaluation.rag_evidence import rag_evidence_context +from tools.answer_evaluation.rag_preparation import prepare_named_content +from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues +from tools.answers.answer_errors import public_failure +from tools.graphrag import graph_search as gs +from tools.graphrag.tests.test_graph_search import FakeSession +from tools.rag.llm_context import compact_rag_payload +from tools.rag.rag_utils import build_context_text +from tools.retrieval.question_constraints import unsupported_constraint, requested_years + + +@pytest.fixture +def corpus(tmp_path): + db=tmp_path/'index.sqlite'; sources=[] + with sqlite3.connect(db) as conn: + conn.execute('CREATE TABLE documents(path,title,author,published)') + conn.execute('CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)') + for i,(person,slug,kind,year,text) in enumerate([ + ('Eva Nová','eva_nova','Diplomová práca',2024, + 'Zadanie:\n1. Implementujte systém vyhľadávania.\nStav:\n- Konzultácia štruktúry práce.\n' + '- Natrénovaný model BERT a vykonané vyhodnotenie presnosti.\n' + '- Vznikol problém s tokenizérom.\nÚlohy:\n- Pripravte experiment s rôznymi tokenizérmi.'), + ('Boris Nový','boris_novy','Bakalárska práca',2023, + 'Zadanie:\n- Implementujte parser.\nStav:\n- Implementovaný parser pomocou Python.\n' + '- Vznikla chyba pri načítaní PDF.\nÚlohy:\n- Pripravte experiment s presnosťou extrakcie.')],1): + path=f'pages/students/2020/{slug}/README.md' + conn.execute('INSERT INTO documents VALUES(?,?,?,?)',(path,person,'Daniel Hladek',1)) + conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path,text,json.dumps([[person,f'{kind} {year}']]),1,0)) + sources.append({'source_id':f'S{i}','title':person,'document_path':path,'published':True, + 'source_url':f'https://example.test/{slug}', + 'text':f'Sekcia: {kind} {year}\nNázov: Ukážkový názov.'}) + raw={'status':'ok','reason':None,'sources':sources,'context':build_context_text(sources), + 'complete':False,'truncated':True,'truncation_reasons':['result_limit'],'index_version':{'version':'fixed'}} + return db,raw + + +def prepare(corpus,question): + db,raw=corpus + return compact_rag_payload(prepare_named_content(db,question,raw)) + + +def model_selection(question,payload): + c=rag_evidence_context(question,payload) + return {'status':'ok','answer':'','claims':[], + 'evidence_quotes':[{'source_id':x['source_id'],'quote':x['text']} for x in c.get('content_candidates',[])], + 'source_urls':[s['source_url'] for s in c['sources']]} + + +def complete(answer): + return lambda _: {'choices':[{'message':{'content':json.dumps(answer)}}]} + + +@pytest.mark.parametrize('question,facets',[ + ('Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?',{'problems','experiments'}), + ('Aké metódy používa Eva Nová?',{'methods'}), + ('Aké sú dosiahnuté výsledky Evy Novej?',{'results'}), +]) +def test_content_without_explicit_work_type(corpus,question,facets): + payload=prepare(corpus,question); c=rag_evidence_context(question,payload) + assert c['requirements']['content'] and set(c['requirements']['facets'])==facets + assert c['content_candidates'] + + +def test_multiple_people_and_aspects_remain_bound(corpus): + q='Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?' + payload=prepare(corpus,q); a=model_selection(q,payload) + r=generate_verified_rag(q,payload,complete(a)) + assert r['accepted_for_delivery'] and r['status']=='ok' + text=r['answer']; eva,boris=text.split('Boris Nový — bakalárska práca') + assert 'tokenizérom' in eva and 'PDF' not in eva + assert 'PDF' in boris and 'tokenizérmi' not in boris + assert text.count('**Riešené problémy:**')==2 + assert text.count('**Plánované experimenty a vyhodnotenie:**')==2 + + +def test_ambiguous_work_is_a_clarification_without_model(corpus): + db,raw=corpus; source=raw['sources'][0] + with sqlite3.connect(db) as conn: + conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(source['document_path'], + 'Stav:\nVznikol iný problém.',json.dumps([['Eva Nová','Bakalárska práca 2022']]),1,1)) + q='Aké problémy mala Eva Nová počas práce?' + p=prepare(corpus,q) + r=generate_verified_rag(q,p,lambda _:pytest.fail('No model call for a clarification')) + assert r['status']=='ambiguous' and r['reason']=='ambiguous_work' + assert '2022' in r['answer'] and '2024' in r['answer'] + + +def test_explicit_type_and_year_resolve_scope(corpus): + db,raw=corpus + with sqlite3.connect(db) as conn: + conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(raw['sources'][0]['document_path'], + 'Stav:\nChyba v inom projekte Cassandra.',json.dumps([['Eva Nová','Diplomový projekt 2022']]),1,1)) + q='Aké problémy mala Eva Nová pri diplomovej práci v roku 2024?' + p=prepare(corpus,q) + assert p['status']=='ok' and 'Cassandra' not in p['context'] + + +def test_year_condition_without_the_words_v_roku_is_preserved(corpus): + q='Aké problémy mala Eva Nová pri diplomovej práci 1999?' + p=prepare(corpus,q); c=rag_evidence_context(q,p) + assert c['requirements']['years']==[1999] and not c.get('content_candidates') + r=generate_verified_rag(q,p,lambda _:pytest.fail('No model for missing scoped evidence')) + assert r['accepted_for_delivery'] and r['status']=='no_answer' and 'tokenizérom' not in r['answer'] + assert requested_years('Práca „Model 2020“ z roku 2024')==[2024] + + +def test_named_source_can_be_loaded_when_absent_from_top_results(corpus): + db,raw=corpus; raw['sources']=raw['sources'][:1];raw['context']=build_context_text(raw['sources']) + p=prepare(corpus,'Aké problémy riešili Eva Nová a Boris Nový?') + assert {s['title'] for s in p['sources']}=={'Eva Nová','Boris Nový'} + + +def test_preparation_does_not_change_the_original_retrieval(corpus): + db,raw=corpus; before=copy.deepcopy(raw) + prepare_named_content(db,'Aké problémy riešili Eva Nová a Boris Nový?',raw) + assert raw==before + + +def test_source_budget_cannot_silently_remove_a_requested_person(corpus): + db,raw=corpus; q='Aké problémy riešili Eva Nová a Boris Nový?' + p=prepare_named_content(db,q,raw) + p['sources'][0]['text']+='\n'+'Veľmi dlhý zdrojový text. '*1000 + compact=compact_rag_payload(p,max_tokens=1100) + c=rag_evidence_context(q,compact) + assert compact['truncated'] + assert set(c['requirements']['people'])=={'Eva Nová','Boris Nový'} + + +def test_candidates_are_actual_bound_literal_evidence(corpus): + q='Aké problémy riešili Eva Nová a Boris Nový?'; c=rag_evidence_context(q,prepare(corpus,q)) + for candidate in candidates(c): + assert any(all(candidate[k]==p[k] for k in ['person','work_type','year','source_id']) + and candidate['text'] in p['text'] for p in c['content_passages']) + + +def test_unknown_second_subject_is_not_silently_ignored(corpus): + p=prepare(corpus,'Aké problémy riešili študenti Eva Nová a Boris Noyy?') + assert p['status']=='ambiguous' and p['reason']=='unresolved_subject' + + +@pytest.mark.parametrize('question',[ + 'Aké problémy riešili Eva Nová a Xaver Neexistujúci?', + 'Aké problémy riešil Boris Noyy?', +]) +def test_unresolved_subject_cannot_be_replaced_by_available_profiles(corpus,question): + p=prepare(corpus,question) + assert p['status']=='ambiguous' and p['reason']=='unresolved_subject' + + +def test_partial_answer_names_the_missing_aspect(corpus): + db,raw=corpus + with sqlite3.connect(db) as conn: + conn.execute("UPDATE chunks SET text='Stav:\nVznikol problém s parserom.' WHERE document_path LIKE '%boris_novy%'") + q='Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?' + p=prepare(corpus,q); r=generate_verified_rag(q,p,complete(model_selection(q,p))) + assert r['accepted_for_delivery'] and r['answer_coverage']['status']=='partial' + assert 'Táto časť odpovede nie je v poskytnutých podkladoch doložená.' in r['answer'] + assert 'parserom' in r['answer'] + + +def test_all_aspects_absent_is_no_answer_not_rejection(corpus): + db,raw=corpus + with sqlite3.connect(db) as conn: + conn.execute("UPDATE chunks SET text='Zadanie:\n- Implementujte parser.'") + q='Aké problémy riešili Eva Nová a Boris Nový?' + r=generate_verified_rag(q,prepare(corpus,q),lambda _:pytest.fail('No model for a bounded empty result')) + assert r['status']=='no_answer' and r['accepted_for_delivery'] and not r['claims'] + + +def test_known_year_is_returned_when_requested_title_is_missing(corpus): + _,raw=corpus; source=raw['sources'][0] + source['text']='Sekcia: Diplomová práca 2024\nStav:\nImplementovaný parser.' + raw['sources']=[source];raw['context']=build_context_text(raw['sources']) + q='Uveď názov a rok diplomovej práce Evy Novej.' + a={'status':'ok','answer':'','claims':[{'person':'Eva Nová','work_type':'diploma_thesis','year':2024}], + 'evidence_quotes':[],'source_urls':[source['source_url']]} + r=generate_verified_rag(q,raw,complete(a)) + assert r['accepted_for_delivery'] and '2024' in r['answer'] + assert 'názov práce nie je v poskytnutých podkladoch doložený' in r['answer'] + assert r['answer_coverage']['status']=='partial' and not r['claims'][0].get('title') + + +def test_concrete_results_win_over_administrative_notes(corpus): + q='Aké výsledky dosiahla Eva Nová?'; p=prepare(corpus,q); c=rag_evidence_context(q,p) + text='\n'.join(x['text'] for x in c['content_candidates']) + assert 'Natrénovaný model BERT' in text + r=generate_verified_rag(q,p,complete(model_selection(q,p))) + assert r['accepted_for_delivery'] and 'Natrénovaný model BERT' in r['answer'] + assert 'Konzultácia' not in r['answer'] + + +def test_duplicate_quotes_and_overlapping_chunks_do_not_duplicate_output(corpus): + q='Aké výsledky dosiahla Eva Nová?'; p=prepare(corpus,q); a=model_selection(q,p) + a['evidence_quotes']*=2 + r=generate_verified_rag(q,p,complete(a)) + assert r['accepted_for_delivery'] and r['answer'].count('Natrénovaný model BERT')==1 + assert 'Ďalšie podklady:' not in r['answer'] and 'Ďalšie doložené podklady' not in r['answer'] + + +def test_plans_cannot_answer_actual_use_or_completed_experiments(): + assert 'methods' not in facets_for('Použite metódu BERT.','task','Aké metódy skutočne používal?') + assert 'experiments' not in facets_for('Pripravte experiment.','task','Aké experimenty vykonal?') + assert 'results' not in facets_for('Cieľom je implementovať systém.','progress','Aké dosiahol výsledky?') + + +def test_method_answer_does_not_add_valid_but_unrequested_profile_metadata(corpus): + _,raw=corpus + raw['sources'][0]['author']='Daniel Hladek' + q='Aké metódy používala Eva Nová pri diplomovej práci v roku 2024?' + p=prepare(corpus,q); a=model_selection(q,p) + a['claims']=[{'person':'Eva Nová','work_type':'diploma_thesis','year':2024, + 'document_author':'Daniel Hladek'}] + r=generate_verified_rag(q,p,complete(a)) + assert r['accepted_for_delivery'] and 'BERT' in r['answer'] + assert 'Daniel Hladek' not in r['answer'] and 'autor dokumentu' not in r['answer'] + assert r['answer'].count('2024')==1 + + +@pytest.mark.parametrize('question',[ + 'Práce od roku 2020 do roku 2024?', 'Práce pred rokom 2020?', + 'Študenti s aspoň 3 prácami?', 'Práce okrem diplomových?', 'Študenti bez tagu RAG?', + 'V ktorom roku Ján Holp začal pracovať na diplomovej práci?', + 'Kedy Ján Holp obhájil diplomovú prácu?', +]) +def test_unsupported_operators_are_not_ignored(question): + assert unsupported_constraint(question) + result=gs.search_graph_with_session(FakeSession({}),question) + assert result['status']=='ambiguous' and result['reason']=='unsupported_constraint' and not result['facts'] + + +@pytest.mark.parametrize('question',[ + 'Práca v roku 1999?', 'Aký je názov práce „Od 2020 do 2024“?', + 'Aké práce alebo projekty má Eva Nová?', +]) +def test_supported_year_and_work_alternative_are_not_blocked(question): + assert unsupported_constraint(question) is None + + +@pytest.mark.parametrize('reason',['model_unavailable','retrieval_unavailable','answer_rejected','answer_capacity_exceeded']) +def test_public_errors_are_distinct_and_do_not_leak_rejected_text(reason): + r=public_failure({'reason':reason,'answer':'SECRET YEAR 2099','claims':[{'person':'Eva'}],'source_urls':['SECRET']}) + assert r['reason']==reason and not r['accepted_for_delivery'] + assert 'SECRET' not in r['answer'] and '2099' not in r['answer'] + assert not r['claims'] and not r['source_urls'] + + +def test_retrieval_failure_is_not_misreported_as_bad_model_answer(monkeypatch): + import tools.answers.answer_delivery as delivery + def unavailable(_): raise RuntimeError('SECRET backend address') + monkeypatch.setattr(delivery,'search_graph',unavailable) + r=delivery.deliver_answer('Aké práce má Eva Nová?',engine='graphrag', + complete=lambda _:pytest.fail('No model call after failed retrieval')) + assert r['status']=='error' and r['reason']=='retrieval_unavailable' + assert 'SECRET' not in r['answer'] diff --git a/tools/answer_evaluation/answer_coverage.py b/tools/answer_evaluation/answer_coverage.py index 8e98def..c6b9eda 100644 --- a/tools/answer_evaluation/answer_coverage.py +++ b/tools/answer_evaluation/answer_coverage.py @@ -39,7 +39,7 @@ def answer_coverage(answer: dict, context: dict) -> dict: (row.get('source_url') in cited or answer.get('status') == 'no_answer'))) # A context budget may drop only unrelated sources. A clipping marker in # the actual cited block is stronger evidence than this global reason. - context_limited = 'llm_context_budget' in reasons + context_limited = any(reason in reasons for reason in ['llm_context_budget','answer_candidate_budget','evidence_unit_limit']) limited = context.get('complete') is not True or context.get('truncated') is True limitations = [*reasons] if clipped: @@ -48,6 +48,13 @@ def answer_coverage(answer: dict, context: dict) -> dict: status = answer.get('status') if status == 'ok': coverage = 'supported' + if any(requirements.get(key) and key not in fields for fields in requirements.get('available_fields',{}).values() + for key in ['title','year','start_year']): + coverage='partial'; warnings.append('partial_requested_fields') + if requirements.get('facets'): + from .content_selection import missing_facets + if missing_facets(answer,context): + coverage='partial'; warnings.append('partial_requested_aspects') if scope == 'list' and (limited or clipped): coverage = 'partial'; warnings.append('list_may_be_incomplete') elif scope == 'count' and (context.get('aggregation') or {}).get('complete') is not True: @@ -74,4 +81,7 @@ def coverage_notices(coverage: dict) -> list[str]: 'missing_information_with_clipped_context': 'Časť podkladov bola pri príprave kontextu skrátená.', 'count_not_established': 'Celkový počet sa z dostupných podkladov nedá spoľahlivo určiť.', } - return [messages[reason] for reason in coverage['warnings']] + notices=[messages[reason] for reason in coverage['warnings'] if reason in messages] + if 'list_may_be_incomplete' in coverage['warnings']: + notices.append('Výpis môžeš zúžiť uvedením osoby, typu práce alebo konkrétneho roku.') + return notices diff --git a/tools/answer_evaluation/checked_generation.py b/tools/answer_evaluation/checked_generation.py index f5e9e29..305e535 100644 --- a/tools/answer_evaluation/checked_generation.py +++ b/tools/answer_evaluation/checked_generation.py @@ -16,7 +16,8 @@ def generate_checked_answer(question: str, context: dict, prompt: str, complete) result['response']=response;record['response']=response text=response['choices'][0]['message']['content'].strip() except Exception as error: - result['error']=f'{type(error).__name__}: {str(error)[:500]}' + result['error']=type(error).__name__ + result['reason']='model_unavailable' result['attempts'].append(record) return result result['attempts'].append(record) diff --git a/tools/answer_evaluation/content_selection.py b/tools/answer_evaluation/content_selection.py new file mode 100644 index 0000000..8dd2106 --- /dev/null +++ b/tools/answer_evaluation/content_selection.py @@ -0,0 +1,191 @@ +"""Role-aware, bounded excerpt selection and presentation from actual evidence.""" +from __future__ import annotations + +import re +from .context_policy import normalized +from .question_intent import FACET_LABELS, planned_experiments + +COMPLETED = r'\b(?:implementovan\w*|natrenovan\w*|vytvoren\w*|extrahovan\w*|vykonan\w*|urobene vyhodnotenie|vyhodnoteny\w*|nameran\w*)\b' +PLANNED = r'\b(?:bude|budu|planuj\w*|cielom|mal by|ma sa|treba|pripravte|navrhnite|otestujte|vyhodnotte)\b' +ADMIN = r'\b(?:konzultac\w*|struktura prace|pisomn\w*|pisat|odovzda\w*|wip|termin\w*)\b' + + +def split_units(text: str) -> list[str]: + from .rag_evidence import descriptive_text + parts = re.split(r'\n\s*\n|(?=^[ \t]*(?:\d+[.)]|[-+*])\s+)', text, flags=re.M) + units = [] + for part in parts: + part = part.strip() + if not descriptive_text(part): + continue + # A source paragraph is kept literal, including its negations and qualifications. + for unit in ([part] if len(part) <= 900 else re.split(r'(?<=[.!?])\s+(?=[A-ZÁČĎÉÍĽŇÓŠŤÚÝŽ])', part)): + if descriptive_text(unit): + units.append(unit.strip()) + return units + + +def effective_role(text: str, role: str) -> str: + n = normalized(text) + if role=='other': + from .presentation import GOAL_VERBS + first=n.split()[0] if n else '' + if first in {normalized(word) for word in GOAL_VERBS} or n.startswith('cielom je'): + return 'task' + if role == 'other' and re.search(COMPLETED, n) and not re.search(PLANNED, n): + return 'progress' + return role + + +def facets_for(text: str, role: str, question: str) -> list[str]: + n = normalized(text); facets = [] + if role == 'assignment': facets.append('assignment') + if role == 'progress' and (not re.search(PLANNED,n) or re.search(COMPLETED,n)): + facets.append('results') + if re.search(r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*|zlyha\w*|nepodar\w*|nevyries\w*)\b', n) and role not in {'assignment','task'}: + facets.append('problems') + if re.search(r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*)\b', n): + if role not in {'assignment','task'} or not re.search(r'\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*)\b',normalized(question)): + facets.append('methods') + if re.search(r'\b(?:experiment\w*|meran\w*|test\w*|otest\w*|vyhodnot\w*|benchmark\w*|f1|presnost\w*)\b', n): + wants_done=re.search(r'\b(?:vykonal\w*|otestoval\w*|uskutocnil\w*|vyhodnotil\w*)\b',normalized(question)) + if wants_done and role in {'assignment','task'}: + pass + elif not planned_experiments(question) or role in {'assignment','task'} or re.search(PLANNED, n): + facets.append('experiments') + return facets + + +def unit_score(text: str, role: str, facet: str = '') -> int: + n = normalized(text) + score = 25 if role == 'progress' else 10 + if re.search(COMPLETED, n): score += 45 + if re.search(r'\b(?:model|system|algoritm\w*|vyhodnot\w*|experiment\w*|presnost\w*|pagerank|bert)\b', n): score += 15 + if re.search(ADMIN, n): score -= 70 + if facet == 'problems' and re.search(r'\b(?:problem\w*|chyb\w*|zlyha\w*)\b', n): score += 35 + if role in {'assignment','task'} and facet == 'results': score -= 100 + return score + + +def evidence_units(context: dict, quotes: list[dict] | None = None) -> list[dict]: + from .rag_evidence import content_sections, quote_sections + rows = [] + for passage in context.get('content_passages', []): + pieces = content_sections(passage['text']) if quotes is None else [ + piece for quote in quotes if quote['source_id'] == passage['source_id'] and + ' '.join(quote['quote'].split()) in ' '.join(passage['text'].split()) + for piece in quote_sections(quote, context)] + for piece in pieces: + for unit in split_units(piece['text']): + role = effective_role(unit, piece['kind']) + rows.append({k: passage.get(k) for k in ['person','work_type','year','source_id']} | + {'text':unit,'kind':role,'facets':facets_for(unit,role,context.get('question',''))}) + # Overlapping chunks/quotes can repeat the same fact with different roles. + # An explicit role wins over an unlabelled duplicate, within the same work. + unique = {} + for row in rows: + clean=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text']) + key = tuple(row.get(k) for k in ['person','work_type','year']) + (' '.join(clean.split()),) + old = unique.get(key) + if old is None or (old['kind']=='other' and row['kind']!='other'): + unique[key]=row + return list(unique.values()) + + +def candidates(context: dict) -> list[dict]: + facets = context['requirements'].get('facets', []) + selected = []; seen = set(); units = evidence_units(context) + for person in context['requirements']['people']: + for facet in facets: + relevant = [u for u in units if u['person']==person and facet in u['facets']] + relevant.sort(key=lambda u: -unit_score(u['text'],u['kind'],facet)) + accepted=0 + for unit in relevant: + text = unit['text'] + if len(text)>2500: + context['complete']=False; context['truncated']=True + context['truncation_reasons']=list(dict.fromkeys([*(context.get('truncation_reasons') or []),'evidence_unit_limit'])) + continue + key=(unit['source_id'],text) + if key not in seen: + seen.add(key); selected.append({**unit,'text':text}) + accepted+=1 + if accepted==2: break + return selected + + +def missing_facets(answer: dict, context: dict) -> list[tuple[str,str]]: + found = evidence_units(context, answer.get('evidence_quotes', [])) + return [(person,facet) for person in context['requirements']['people'] + for facet in context['requirements'].get('facets', []) + if not any(row['person']==person and facet in row['facets'] for row in found)] + + +def fit_candidates(sent: dict, context: dict, limit: int, encoding: str) -> None: + """Budget the actual candidate JSON, sharing space across named people.""" + import json + from tools.rag.build_chunks import TokenCounter + counter=TokenCounter(encoding) + if counter.encoding is None: + raise RuntimeError('Missing context tokenizer') + original=list(context.get('content_candidates',[])) + groups=[[row for row in original if row.get('person')==person] + for person in context['requirements']['people']] + ordered=[group[i] for i in range(max(map(len,groups),default=0)) for group in groups if ilimit: + selected.pop() + if counter.count(json.dumps(sent,ensure_ascii=False))>limit: + raise ValueError('Mandatory answer metadata exceeds context budget') + + +def facet_lines(answer: dict, context: dict) -> list[str]: + from .presentation import goal_text, work_subject, owner_in_question + from .rag_evidence import WORK_LABELS + req=context['requirements']; units=evidence_units(context,answer.get('evidence_quotes',[])) + lines=[] + for person in req['people']: + scopes={(p['work_type'],p.get('year')) for p in context.get('content_passages',[]) if p.get('person')==person} + kind=req['work_types'].get(person) or (next(iter(scopes))[0] if len(scopes)==1 else None) + subject=work_subject(person,kind,context.get('question','')) + if not owner_in_question(person,context.get('question','')): + subject=person+(' — '+WORK_LABELS[kind] if kind in WORK_LABELS else '') + if len(scopes)==1 and next(iter(scopes))[1] is not None: + subject+=' ('+str(next(iter(scopes))[1])+')' + lines.extend(['**'+subject+'**', '']) + seen=set() + for facet in req.get('facets', []): + label=FACET_LABELS[facet] + if facet=='experiments' and planned_experiments(context.get('question','')): + label='Plánované experimenty a vyhodnotenie' + rows=[u for u in units if u['person']==person and facet in u['facets']] + rows.sort(key=lambda u:-unit_score(u['text'],u['kind'],facet)) + lines.append('**'+label+':**') + shown=0 + for row in rows: + key=' '.join(re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text']).split()) + if key in seen: + continue + text=row['text'] + if row['kind'] in {'assignment','task'}: + text,_=goal_text(text) + text=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',text) + phase=' (plán)' if row['kind'] in {'assignment','task'} and facet not in {'assignment'} else '' + lines.append('- '+text+phase); seen.add(key); shown+=1 + if shown==2: break + if not shown: + if rows: + lines.append('Je doložené rovnakým výňatkom uvedeným vyššie.') + else: + lines.append('Táto časť odpovede nie je v poskytnutých podkladoch doložená.') + lines.append('') + return lines diff --git a/tools/answer_evaluation/graph_context.py b/tools/answer_evaluation/graph_context.py index be0190e..51e8ced 100644 --- a/tools/answer_evaluation/graph_context.py +++ b/tools/answer_evaluation/graph_context.py @@ -54,6 +54,7 @@ def graph_context(result: dict) -> dict: 'index_version':result.get('index_version')} context['answer_candidates']=joined_candidates(facts,intent) context['question'] = result.get('query', '') + context['clarification'] = result.get('clarification') context['requested_people'] = list(dict.fromkeys(row['name'] for row in result.get('resolved_entities', []) if row.get('kind') == 'Person' and row.get('name'))) comparison = result.get('academic_year_comparison') or next((row for row in result['facts'] diff --git a/tools/answer_evaluation/question_intent.py b/tools/answer_evaluation/question_intent.py new file mode 100644 index 0000000..e7716c5 --- /dev/null +++ b/tools/answer_evaluation/question_intent.py @@ -0,0 +1,38 @@ +"""Explicit answer aspects; these select evidence, never supply facts.""" +import re +from .context_policy import normalized + +FACET_PATTERNS = { + 'assignment': r'\b(?:zadani\w*|ciel\w*)\b', + 'problems': r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*)\b', + 'methods': r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*)\b', + 'experiments': r'\b(?:experiment\w*|merani\w*|testovani\w*|vyhodnocov\w*)\b', + 'results': r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b', +} +FACET_LABELS = {'assignment': 'Zadanie a ciele', 'problems': 'Riešené problémy', + 'methods': 'Metódy a technológie', 'experiments': 'Experimenty a vyhodnotenie', + 'results': 'Zaznamenaný stav a výsledky'} + + +def requested_facets(question: str) -> list[str]: + text = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question)) + matches = [(m.start(), key) for key, pattern in FACET_PATTERNS.items() + if (m := re.search(pattern, text))] + return [key for _, key in sorted(matches)] + + +def content_question(question: str) -> bool: + text = normalized(question) + return bool(requested_facets(question) or re.search( + r'\b(?:comu|o com|obsah\w*|opis\w*|popis\w*|zamer\w*|zhrn\w*|implement\w*)\b', text)) + + +def facet_mode(question: str) -> bool: + facets = requested_facets(question) + return any(facet in facets for facet in ['problems','methods','experiments','results']) + + +def planned_experiments(question: str) -> bool: + text=normalized(question) + return bool(re.search(r'\b(?:plan\w*|pripravit|navrhnut)|' + r'\b(?:mal|mali|mala|ma|maju)\s+(?:priprav\w*|navrhn\w*|otest\w*|vyhodnot\w*)',text)) diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index 2e42ee5..22c2e0f 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -10,6 +10,8 @@ import json import sqlite3 from contextlib import closing from .context_policy import normalized, text_contains_value, person_pattern +from .question_intent import requested_facets, content_question, facet_mode +from tools.retrieval.question_constraints import requested_years WORK_TYPES = { 'bakalarska praca': 'bachelor_thesis', 'diplomova praca': 'diploma_thesis', @@ -148,6 +150,10 @@ def academic_windows(body: str) -> list[tuple[tuple[str, int | None] | None, str scope = (None if normalized(heading[1]).startswith(('priprava na ', 'priprava k ')) else detected or (heading_stack[-1][1] if heading_stack else None)) heading_stack.append((level, scope)) + if scope is not None and detected is None: + # Keep semantic child headings (Stav, Úlohy, Metódy) as + # literal source text instead of losing their meaning. + lines.append(line) else: lines.append(line) flush() @@ -392,14 +398,16 @@ def rag_evidence_context(question: str, payload: dict) -> dict: facts.extend({**base, field: value} for value in values) facts = list({tuple(sorted(row.items())): row for row in facts}.values()) requested = [person for person in people if text_contains_value(question, person, person=True)] + requested=list(dict.fromkeys([*payload.get('named_subjects',[]),*requested])) expected_types = requested_types(question, requested) query = normalized(question) intent_query = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question)) academic = bool(TYPE_PATTERN.search(query) or re.search(r'\b(?:rok\w* prac\w*|nazov\w* prac\w*)\b', query)) - content = bool(academic or (requested and re.search(r'\b(?:prac\w*|projekt\w*)\b', query))) and bool(re.search( + content = bool(academic or requested) and bool(re.search( r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|co\s+ries\w*|obsah\w*|opis\w*|popis\w*|' r'zhrn\w*|zhrnut\w*|vysvetl\w*|zamer\w*|zadani\w*|ciel\w*|metod\w*|technolog\w*|' - r'pouziv\w*|implement\w*|vysledk\w*|dosiahn\w*)\b', intent_query)) + r'pouziv\w*|implement\w*|vysledk\w*|dosiahn\w*|problem\w*|tazkost\w*|prekazk\w*|experiment\w*|merani\w*)\b', intent_query) + or content_question(question)) identity_first = re.search(r'^(?:aky|aka|ake|ktory|ktora|ktore|v ktorom|uved)' r'(?:\s+(?:je|su|bol|bola|boli|presny|presne)){0,3}\s+(?:nazov\w*|rok\w*)\b', intent_query) explicit_description = re.search(r'\b(?:comu|o com|opis\w*|popis\w*|zhrn\w*|obsah\w*|zadani\w*|ciel\w*)\b', intent_query) @@ -419,18 +427,27 @@ def rag_evidence_context(question: str, payload: dict) -> dict: requirements['response_mode'] = 'mixed' if any(requirements[key] for key in ['title', 'year', 'start_year']) else 'quotes' requirements['distinguish_progress'] = bool(re.search(r'\b(?:zadani\w*|ciel\w*)\b', query) and re.search(r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b', query)) + if facet_mode(question): + requirements['facets'] = requested_facets(question) stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else [] - requirements['years'] = ([int(value) for value in stated_years if len(value) == 4] - if not requirements['start_year'] else []) + requirements['years'] = requested_years(question) if academic and not requirements['start_year'] else [] requirements['unsupported_year'] = any(len(value) != 4 for value in stated_years) if requested: facts = [row for row in facts if row['person'] in requested] - context = {'question': question, **{key: payload.get(key) for key in ['status', 'reason', 'complete', 'truncated', + requirements['available_fields'] = {person:[key for key in ['title','year','start_year'] + if any(row.get('person')==person and row.get(key) is not None and + (key=='start_year' or ((not expected_types.get(person) or row.get('work_type')==expected_types[person]) + and (not requirements['years'] or row.get('year') in requirements['years']))) for row in facts)] + for person in requested} + context = {'question': question, 'clarification':payload.get('clarification'), **{key: payload.get(key) for key in ['status', 'reason', 'complete', 'truncated', 'truncation_reasons', 'index_version']}, 'sources': payload.get('sources', []), 'facts': facts, 'source_blocks': blocks, 'requirements': requirements, 'aggregation': None, 'answer_candidates': []} if content: context['content_passages'] = academic_content_passages(blocks, facts, requirements, question) + if requirements.get('facets'): + from .content_selection import candidates + context['content_candidates'] = candidates(context) if requirements['distinguish_progress']: requirements['content_roles'] = {person: sorted({section['kind'] for p in context['content_passages'] if p['person'] == person diff --git a/tools/answer_evaluation/rag_preparation.py b/tools/answer_evaluation/rag_preparation.py new file mode 100644 index 0000000..37070d0 --- /dev/null +++ b/tools/answer_evaluation/rag_preparation.py @@ -0,0 +1,91 @@ +"""Resolve named source profiles and scope content before the shared token budget.""" +from __future__ import annotations + +import json +import re +import sqlite3 +from contextlib import closing + +from .context_policy import normalized, person_pattern, text_contains_value +from .question_intent import content_question +from .rag_evidence import PROFILE, academic_scope, academic_windows, requested_types, WORK_LABELS +from tools.retrieval.question_constraints import requested_years + + +def clarify(payload: dict, reason: str, message: str) -> dict: + return {**payload,'status':'ambiguous','reason':reason,'complete':False, + 'clarification':message} + + +def prepare_named_content(db_file, question: str, payload: dict) -> dict: + if not content_question(question): + return payload + if re.search(r'\b(?:s tagom|s klucovym slovom|z kategorie|s autorom dokumentu)\b',normalized(question)): + return clarify(payload,'content_filter_requires_graph', + 'Najprv over podmienku tagu, kategórie alebo autora cez /graphrag. ' + 'Potom môžem opísať obsah konkrétnej vyhovujúcej práce; túto podmienku nevynechám.') + from tools.rag.rag_utils import build_context_text + with closing(sqlite3.connect(db_file.resolve().as_uri()+'?mode=ro',uri=True)) as conn: + conn.row_factory=sqlite3.Row + if not {'documents','chunks'} <= {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}: + return payload + catalog=[dict(r) for r in conn.execute('SELECT path,title,author,published FROM documents WHERE COALESCE(published,1)=1') + if PROFILE.fullmatch(r['path']) and r['title']] + named=[r for r in catalog if text_contains_value(question,r['title'],person=True)] + # Preserve question order rather than the original retrieval ranking. + named.sort(key=lambda r:re.search(person_pattern(r['title']),normalized(question)).start()) + if len(named)>4: + return clarify(payload,'too_many_subjects','Pre podrobné obsahové porovnanie vyber najviac štyri osoby naraz.') + from tools.graphrag.graph_search import requested_name_phrases + first_names={normalized(r['title']).split()[0] for r in catalog} + capitalized={normalized(match[0]) for match in re.finditer( + r'\b[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][^\W\d_]+(?:\s+[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][^\W\d_]+){1,3}\b',question)} + unknown=[] + for phrase in requested_name_phrases(question): + words=phrase.split() + named_companion=(named and phrase in capitalized and + re.search(r'\ba\s+'+re.escape(phrase)+r'\b',normalized(question))) + if (2<=len(words)<=4 and (words[0] in first_names or named_companion) and + not any(text_contains_value(phrase,r['title'],person=True) for r in catalog)): + unknown.append(phrase) + if unknown: + return clarify(payload,'unresolved_subject','Meno „'+unknown[0]+'“ sa nepodarilo spoľahlivo priradiť. ' + 'Uveď jeho presné celé meno; ostatné osoby nebudem používať ako náhradu.') + if not named: + return payload + types=requested_types(question,[r['title'] for r in named]) + years=requested_years(question) + selected=[] + for person in named: + rows=list(conn.execute('SELECT text,heading_paths_json FROM chunks WHERE document_path=? ' + 'AND COALESCE(published,1)=1 ORDER BY chunk_index',(person['path'],))) + frames=[]; records={};seen=set() + for row in rows: + paths=json.loads(row['heading_paths_json'] or '[]') + headings=[' > '.join(p) if isinstance(p,list) else str(p) for p in paths] + first=headings[0] if headings else '' + for scope,text in academic_windows('Sekcia: '+first+'\n'+str(row['text'] or '')): + if not scope or (types.get(person['title']) and scope[0]!=types[person['title']]) or (years and scope[1] not in years): + continue + label=WORK_LABELS[scope[0]]+(f' {scope[1]}' if scope[1] is not None else '') + records[scope]=label + if (scope,text) not in seen: + seen.add((scope,text));frames.append('Sekcia: '+label+'\n'+text) + if len(records)>1: + options='; '.join(dict.fromkeys(records.values())) + return clarify(payload,'ambiguous_work',f'Ktorú prácu osoby {person["title"]} máš na mysli? ' + f'V indexe sú tieto možnosti: {options}. Uveď typ práce a rok.') + source=next((s for s in payload.get('sources',[]) if s.get('document_path')==person['path']),None) + if source is None: + from tools.rag.search_core import make_source_url + source={'title':person['title'],'document_path':person['path'],'source_url':make_source_url(person['path']), + 'author':person['author'],'published':person['published']} + selected.append({**source,'text':'\n\n'.join(frames)}) + # Keep already-retrieved child documents of the requested profiles. + for source in payload.get('sources',[]): + if any(source.get('document_path','').startswith(p['path'].removesuffix('README.md')) + and source.get('document_path')!=p['path'] for p in named): + selected.append(source) + selected=[{**source,'source_id':f'S{i}'} for i,source in enumerate(selected,1)] + return {**payload,'sources':selected,'source_count':len(selected),'context':build_context_text(selected), + 'named_subjects':[p['title'] for p in named]} diff --git a/tools/answer_evaluation/render_answer.py b/tools/answer_evaluation/render_answer.py index ebe5675..dfbedd6 100644 --- a/tools/answer_evaluation/render_answer.py +++ b/tools/answer_evaluation/render_answer.py @@ -54,6 +54,8 @@ def render_graph_answer(answer: dict, context: dict) -> dict: lines.append(f'Počet vyhovujúcich osôb: {out["count"]}.') if context.get('complete') is False or context.get('truncated'): lines.append('Výpis je neúplný.') + if records: + lines.append('Výpis môžeš zúžiť uvedením osoby, typu práce alebo konkrétneho roku.') elif context.get('complete') is None: lines.append('Úplnosť vyhľadania nemožno zaručiť.') elif out['status'] == 'ambiguous': diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index 8784f0c..8360254 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import os import re from typing import Literal @@ -12,6 +13,7 @@ from .context_policy import answer_context_issues, citation_check, allowed_urls, from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections from .answer_coverage import answer_coverage, coverage_notices from .presentation import content_lines, fact_sentences, no_answer_text, work_subject +from .content_selection import facet_lines, evidence_units, missing_facets, fit_candidates class RagClaim(Claim): @@ -46,6 +48,8 @@ uveď start_year aj year a work_type. Citácia ani nesúvisiaci výňatok údaj Rešpektuj requirements.response_mode. Pri response_mode=quotes MUSÍ byť claims=[]: pri všeobecnej otázke, téme, technológii, zozname ľudí alebo stáži vyber iba evidence_quotes: source_id a presný doslovný výňatok, ktorý odpovedá na otázku. +Pri výbere zachovaj negácie, podmienky a výhrady zdroja. Nevytrhni tvrdenie z vety, +ktorá ho obmedzuje alebo spochybňuje. Nevypĺňaj pri nich názvy prác, roky, tagy ani iné nežiadané metadáta do claims. Pri requirements.content=true otázka žiada obsah práce, nie iba jej identifikáciu. Vyber opisujúce evidence_quotes z content_passages zodpovedajúcej osoby, typu a roku. @@ -65,6 +69,11 @@ aplikácia uvedie, čo chýba. no_answer znamená žiadnu odpoveď a musí mať Pri zaznamenanom stave uprednostni konkrétnu implementáciu, natrénovanie modelu a vykonané vyhodnotenie pred konzultáciami alebo samotným písaním textu. Ani priebežný stav nevydávaj za splnenie všetkých cieľov alebo za konečné výsledky práce. +Ak sú dodané content_candidates, vyber quotes iba z ich textu. Každý kandidát uvádza +osobu, typ práce, rok, kind (assignment/task/progress/other) a facets, ktoré dokladá. +Pokry každú dostupnú požadovanú časť requirements.facets za každú osobu. Pre chýbajúcu časť +nevymýšľaj dôkaz: aplikácia ju výslovne označí. Čiastočná doložená odpoveď má status=ok. +Pri výsledkoch použi skutočný zaznamenaný stav; úlohy a plánované experimenty nie sú výsledky. Každý výňatok musí byť JEDEN súvislý úsek z konkrétneho zdroja vrátane Markdownu. Nespájaj oddelené pasáže cez vynechané riadky; použi viac samostatných kratších quotes. Vyber iba minimum potrebné na zodpovedanie otázky. Výňatok nesmieš parafrázovať ani meniť čísla. @@ -127,7 +136,8 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: issues.append('missing_required_evidence') if requirements.get('response_mode') == 'quotes' and claims: issues.append('unrequested_structured_claims') - if requirements.get('content') and answer['status'] == 'ok' and not quotes: + if (requirements.get('content') and answer['status'] == 'ok' and not quotes + and (not requirements.get('facets') or context.get('content_candidates'))): issues.append('missing_content_evidence') requested = requirements['people'] types = requirements['work_types'] @@ -159,7 +169,12 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: matching = [claim for claim in claims if claim['person'] == person] for key in ['title', 'year', 'start_year']: if requirements[key] and not any(claim.get(key) is not None for claim in matching): - issues.append('missing_requested_' + key) + if key in requirements.get('available_fields',{}).get(person,[key]): + issues.append('missing_requested_' + key) + wanted=[key for key in ['title','year','start_year'] if requirements[key]] + if (wanted and not quotes and not any(key in fields for fields in requirements.get('available_fields',{}).values() + for key in wanted)): + issues.append('no_requested_fact_supported') if answer['status'] == 'no_answer' and (requirements['academic'] or requirements['start_year']) and requested: # Reject an unsupported refusal only if every requested academic tuple # is explicitly present. Never invent a value to fill an absent field. @@ -174,6 +189,10 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: and (not requirements['start_year'] or start_supported) and not requirements['unsupported_year'] and (not requirements.get('content') or context.get('content_passages'))): issues.append('rejected_supported_academic_answer') + if (not requirements.get('content') and not requirements['unsupported_year'] and any( + requirements[key] and key in fields for fields in requirements.get('available_fields',{}).values() + for key in ['title','year','start_year'])): + issues.append('rejected_supported_partial_fact') if answer['status'] == 'no_answer' and requirements.get('content'): query = normalized(context['question']) overview = re.search(r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|opis\w*|popis\w*|obsah\w*|zhrn\w*)\b', query) @@ -193,6 +212,10 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: elif source.get('source_url') not in cited: issues.append('missing_quote_source') if source: + if context.get('candidate_only') and not any(c['source_id']==quote['source_id'] and + ' '.join(quote['quote'].split()) in ' '.join(c['text'].split()) + for c in context.get('content_candidates', [])): + issues.append('quote_outside_delivered_candidates') if requirements.get('content'): titles = [row.get(key) for row in facts if row.get('source_id') == quote['source_id'] for key in ('title', 'person', 'document_author')] @@ -218,15 +241,16 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: if requirements.get('content') and answer['status'] == 'ok': for person in requested: passages = [p for p in context.get('content_passages', []) if p.get('person') == person] - if requirements['response_mode'] == 'mixed': + if requirements['response_mode'] == 'mixed' and any(claim.get('person')==person for claim in claims): passages = [p for p in passages if any(claim.get('person') == person and all(claim.get(key) is None or claim[key] == p.get(key) for key in ['work_type', 'year']) for claim in claims)] - if not any(quote['source_id'] == p['source_id'] and + eligible = any(c.get('person')==person for c in context.get('content_candidates', [])) + if (not requirements.get('facets') or eligible) and not any(quote['source_id'] == p['source_id'] and ' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) for p in passages for quote in quotes): issues.append('missing_bound_content_evidence') - if requirements.get('distinguish_progress'): + if requirements.get('distinguish_progress') and not requirements.get('facets'): bound_quotes = [quote for quote in quotes if any( quote['source_id'] == p['source_id'] and ' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) for p in passages)] @@ -235,7 +259,15 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: for kind in requirements.get('content_roles', {}).get(person, []): if kind not in kinds: issues.append('missing_' + kind + '_evidence') - if (requirements.get('distinguish_progress') and answer['status'] == 'no_answer' and requested + if requirements.get('facets'): + available={(c['person'],facet) for c in context.get('content_candidates',[]) for facet in c['facets']} + if answer['status']=='ok': + for person,facet in missing_facets(answer,context): + if (person,facet) in available: + issues.append('missing_'+('progress' if facet=='results' else facet)+'_evidence') + elif answer['status']=='no_answer' and available: + issues.append('rejected_supported_content_comparison') + if (requirements.get('distinguish_progress') and not requirements.get('facets') and answer['status'] == 'no_answer' and requested and all(any(p.get('person') == person for p in context.get('content_passages', [])) for person in requested)): issues.append('rejected_supported_content_comparison') if not citation_check(answer, allowed_urls(context))['citations_ok']: @@ -333,11 +365,29 @@ def comparison_lines(answer: dict, context: dict) -> list[str]: def render_rag_answer(answer: dict, context: dict) -> str: lines = [] if answer['status'] == 'ok': + req=context.get('requirements',{}) for claim in answer.get('claims', []): - lines.append(fact_sentences(claim, context.get('question', ''))) + if req.get('facets'): + # Facet headings already identify the scoped work/year. Extra + # valid metadata selected by the model is not part of the answer. + fields=[key for key in ['title','year','start_year'] if req.get(key)] + if not req.get('title') and not req.get('start_year') and req.get('years'): + continue + displayed={key:value for key,value in claim.items() if key in ['person','work_type',*fields]} + if not any(displayed.get(key) is not None for key in fields): + continue + lines.append(fact_sentences(displayed,context.get('question',''))) + else: + lines.append(fact_sentences(claim, context.get('question', ''))) + for person in req.get('people',[]): + for key,label in [('title','Názov práce'),('year','Rok práce'),('start_year','Začiatok štúdia')]: + if req.get(key) and key not in req.get('available_fields',{}).get(person,[key]): + lines.append(f'{person} — {label.lower()} nie je v poskytnutých podkladoch doložený.') quotes = answer.get('evidence_quotes', []) style = context.get('requirements', {}).get('content_style') - if quotes and context.get('requirements', {}).get('distinguish_progress'): + if context.get('requirements', {}).get('facets'): + lines.extend(facet_lines(answer,context)) + elif quotes and context.get('requirements', {}).get('distinguish_progress'): lines.extend(comparison_lines(answer, context)) elif quotes and style in {'overview', 'assignment'}: # Bind each paragraph to the same source/person/work as the checked @@ -371,7 +421,10 @@ def render_rag_answer(answer: dict, context: dict) -> str: if answer.get('count') is not None: lines.append(f'Počet: {answer["count"]}.') elif answer['status'] == 'no_answer': - lines.append(no_answer_text(context)) + if context.get('requirements', {}).get('facets') and context['requirements']['people']: + lines.extend(facet_lines(answer,context)) + else: + lines.append(no_answer_text(context)) elif answer['status'] == 'ambiguous': lines.append('Údaje sú nejednoznačné; požiadavku treba spresniť.') else: @@ -384,6 +437,18 @@ def render_rag_answer(answer: dict, context: dict) -> str: def generate_verified_rag(question: str, payload: dict, complete) -> dict: context = rag_evidence_context(question, payload) + if payload.get('status')=='ambiguous' or context.get('clarification'): + return {'status':'ambiguous','reason':payload.get('reason') or 'ambiguous_request', + 'accepted_for_delivery':True,'answer':context.get('clarification') or 'Spresni osobu, typ práce alebo rok.', + 'claims':[],'source_urls':[],'attempts':[],'complete':False,'truncated':False, + 'truncation_reasons':[],'index_version':context.get('index_version')} + if (context['requirements'].get('facets') and context['requirements']['response_mode']=='quotes' + and not context.get('content_candidates') and context['requirements']['people']): + empty={'status':'no_answer','claims':[],'evidence_quotes':[], + 'source_urls':[s['source_url'] for s in context['sources'] if s.get('source_url')]} + return {**empty,'reason':'requested_aspects_not_found','answer':render_rag_answer(empty,context), + 'accepted_for_delivery':True,'attempts':[],'answer_coverage':answer_coverage(empty,context), + **{k:context.get(k) for k in ['index_version','complete','truncated','truncation_reasons']}} if not payload.get('sources'): status = 'error' if payload.get('status') == 'error' else 'no_answer' empty = {'status': status, 'claims': [], 'evidence_quotes': [], 'source_urls': []} @@ -395,11 +460,18 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: sent = {key: context[key] for key in ['status', 'reason', 'complete', 'truncated', 'truncation_reasons', 'index_version', 'sources', 'facts', 'requirements']} if context['requirements'].get('content'): - sent['content_passages'] = context['content_passages'] + if context['requirements'].get('facets'): + sent['content_candidates']=context['content_candidates'] + context['candidate_only']=True + else: + sent['content_passages'] = context['content_passages'] elif not context['requirements']['academic']: sent['context'] = payload.get('context', '') if context['requirements']['response_mode'] == 'quotes': sent.pop('facts') + if context['requirements'].get('facets'): + fit_candidates(sent,context,payload.get('context_budget',{}).get('max_tokens',12000), + os.getenv('RAG_LLM_TOKEN_ENCODING','cl100k_base')) messages = [{'role': 'system', 'content': PROMPT + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)}, {'role': 'user', 'content': json.dumps({'question': question, 'context': sent}, ensure_ascii=False)}] attempts = [] @@ -420,6 +492,12 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: 'answer': 'Jazykový model nie je dostupný; overenú odpoveď sa nepodarilo pripraviť.', 'attempts': attempts} record['issues'] = issues if not issues: + if context['requirements'].get('facets') and missing_facets(answer,context): + # Sources inspected for a missing requested aspect remain + # traceable, even when only the other person's quotes exist. + extra=[s['source_url'] for s in context['sources'] if s.get('title') in + {person for person,_ in missing_facets(answer,context)} and s.get('source_url')] + answer={**answer,'source_urls':list(dict.fromkeys([*answer['source_urls'],*extra]))} text = render_rag_answer(answer, context) return {'status': answer['status'], 'reason': None, 'accepted_for_delivery': True, 'answer': text, 'source_urls': answer['source_urls'], 'claims': answer['claims'], diff --git a/tools/answers/answer_delivery.py b/tools/answers/answer_delivery.py index 4ff9643..efb244b 100644 --- a/tools/answers/answer_delivery.py +++ b/tools/answers/answer_delivery.py @@ -12,12 +12,20 @@ from tools.answer_evaluation.graph_context import graph_context from tools.answer_evaluation.checked_generation import generate_checked_answer from tools.answer_evaluation.verified_rag import generate_verified_rag from tools.answer_evaluation.rag_evidence import expand_comparison_sources +from tools.answer_evaluation.rag_preparation import prepare_named_content +from tools.retrieval.question_constraints import unsupported_constraint, constraint_message from tools.sync.index_state import index_snapshot from tools.answers.model_client import complete_with_configured_model +from tools.answers.answer_errors import public_failure _capacity = threading.BoundedSemaphore(2) +def retrieval_failure(engine: str) -> dict: + return public_failure({'reason':'retrieval_unavailable','engine':engine,'index_version':None, + 'complete':False,'truncated':False,'truncation_reasons':[]}) + + def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, complete=None, db_file: Path = DB_FILE, include_diagnostics: bool = False) -> dict: if not _capacity.acquire(blocking=False): @@ -25,15 +33,32 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, 'accepted_for_delivery': False, 'answer': 'Agent práve spracúva iné odpovede. Skúste to o chvíľu.'} try: model = complete or complete_with_configured_model + unsupported=unsupported_constraint(question) + if unsupported: + return {'status':'ambiguous','reason':'unsupported_constraint','accepted_for_delivery':True, + 'answer':constraint_message(unsupported),'claims':[],'source_urls':[], + 'index_version':None,'complete':False,'truncated':False,'truncation_reasons':[], 'engine':engine} if engine == 'rag': - with index_snapshot(db_file.parent, db_file=db_file): - retrieved = build_rag_context(db_file, question, limit=limit) - retrieved = expand_comparison_sources(db_file, question, retrieved) - retrieved = compact_rag_payload(retrieved) + try: + with index_snapshot(db_file.parent, db_file=db_file): + retrieved = build_rag_context(db_file, question, limit=limit) + retrieved = expand_comparison_sources(db_file, question, retrieved) + retrieved = prepare_named_content(db_file, question, retrieved) + retrieved = compact_rag_payload(retrieved) + except Exception: + return retrieval_failure(engine) result = generate_verified_rag(question, retrieved, model) context = retrieved elif engine == 'graphrag': - context = graph_context(search_graph(question)) + try: + context = graph_context(search_graph(question)) + except Exception: + return retrieval_failure(engine) + if context.get('clarification'): + return {'status':'ambiguous','reason':context.get('reason'),'accepted_for_delivery':True, + 'answer':context['clarification'],'claims':[],'source_urls':[], 'engine':engine, + 'complete':False,'truncated':False,'truncation_reasons':[], + 'index_version':context.get('index_version')} prompt = (Path(__file__).resolve().parents[1] / 'answer_evaluation/graph_answer_prompt.txt').read_text() generated = generate_checked_answer(question, context, prompt, model) if generated.get('accepted_for_delivery'): @@ -42,8 +67,10 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, 'truncated': context.get('truncated'), 'truncation_reasons': context.get('truncation_reasons'), 'attempts': generated['attempts']} else: - result = {'status': 'error', 'reason': 'answer_rejected', 'accepted_for_delivery': False, - 'answer': 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.', + reason=generated.get('reason','answer_rejected') + result = {'status': 'error', 'reason': reason, 'accepted_for_delivery': False, + 'answer': ('Jazykový model nie je dostupný. Skús otázku neskôr.' if reason=='model_unavailable' else + 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.'), 'index_version': context.get('index_version'), 'attempts': generated['attempts']} else: raise ValueError('Unsupported answer engine') diff --git a/tools/answers/answer_errors.py b/tools/answers/answer_errors.py new file mode 100644 index 0000000..e7afcf3 --- /dev/null +++ b/tools/answers/answer_errors.py @@ -0,0 +1,18 @@ +"""Public failure wording; transport/model exception text is never a UI message.""" +MESSAGES={ + 'model_unavailable':'Jazykový model momentálne nie je dostupný. Skús otázku neskôr.', + 'retrieval_unavailable':'Vyhľadávací index momentálne nie je dostupný. Skús otázku neskôr.', + 'answer_capacity_exceeded':'Agent práve spracúva iné otázky. Skús to o chvíľu.', + 'answer_rejected':'Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. ' + 'Skús spresniť osobu, typ práce alebo rok; neoverené údaje sa nezobrazia.', + 'answer_unavailable':'Overenú odpoveď sa momentálne nepodarilo pripraviť. Skús to neskôr.', +} + + +def public_failure(result: dict) -> dict: + reason=result.get('reason') + if not isinstance(reason,str) or reason not in MESSAGES: + reason='answer_unavailable' + return {'index_version':None,'complete':False,'truncated':False,'truncation_reasons':[], + **result,'status':'error','reason':reason,'accepted_for_delivery':False, + 'answer':MESSAGES[reason],'claims':[],'source_urls':[]} diff --git a/tools/graphrag/evaluation/reproducibility.py b/tools/graphrag/evaluation/reproducibility.py index 5dc3ec6..fde51c6 100644 --- a/tools/graphrag/evaluation/reproducibility.py +++ b/tools/graphrag/evaluation/reproducibility.py @@ -46,6 +46,8 @@ def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]: 'tools/answer_evaluation/render_answer.py', 'tools/answer_evaluation/graph_context.py', 'tools/answer_evaluation/presentation.py', 'tools/answer_evaluation/rag_evidence.py', + 'tools/answer_evaluation/question_intent.py', 'tools/answer_evaluation/content_selection.py', + 'tools/answer_evaluation/rag_preparation.py', 'tools/answer_evaluation/answer_coverage.py', 'tools/answer_evaluation/graph_answer_prompt.txt', 'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py']) files = sorted(set(files)) diff --git a/tools/graphrag/graph_search.py b/tools/graphrag/graph_search.py index 1f6d749..0bc47d3 100644 --- a/tools/graphrag/graph_search.py +++ b/tools/graphrag/graph_search.py @@ -8,6 +8,7 @@ from typing import Any, Iterable from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION from .publication import public_query from tools.retrieval.retrieval_contract import result_fields, source_references, error_result +from tools.retrieval.question_constraints import unsupported_constraint, constraint_message GRAPH_SCOPE = "zpwiki" @@ -2334,6 +2335,12 @@ def search_graph_with_session( raise ValueError("max_entities musí byť aspoň 1") constraints = extract_query_constraints(clean_query) + unsupported = unsupported_constraint(clean_query) + if unsupported: + return {**result_fields(status='ambiguous',sources=[],reason='unsupported_constraint',complete=False), + 'query':clean_query,'constraints':constraints,'clarification':constraint_message(unsupported), + 'facts':[],'paths':[],'documents':[],'aggregation':None,'resolved_entities':[], + 'checked_entities':[],'rejection_evidence':[],'strategies':[],'reasons':['unsupported_constraint']} catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE, published_only=published_only) constraints['published_only'] = published_only # Explicit tag filters are resolved separately. Incidental names such as the diff --git a/tools/rag/llm_context.py b/tools/rag/llm_context.py index 05cd5b6..4e3dc14 100644 --- a/tools/rag/llm_context.py +++ b/tools/rag/llm_context.py @@ -11,7 +11,7 @@ from tools.retrieval.text_budget import split_to_fit SOURCE_FIELDS = ('source_id', 'title', 'author', 'document_path', 'source_url', 'published', 'categories', 'tags', 'section') FIELDS = ('status', 'reason', 'query', 'engine', 'strategies', 'instructions', 'answer_format', - 'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count') + 'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count', 'clarification', 'named_subjects') def compact_rag_payload(data: dict, *, max_tokens: int | None = None, diff --git a/tools/rag/rag_utils.py b/tools/rag/rag_utils.py index 929a110..0431c4f 100644 --- a/tools/rag/rag_utils.py +++ b/tools/rag/rag_utils.py @@ -709,6 +709,13 @@ def build_rag_context( published_only: bool = DEFAULT_PUBLISHED_ONLY, max_per_document: int = 1, ) -> dict[str, Any]: + from tools.retrieval.question_constraints import unsupported_constraint, constraint_message + unsupported=unsupported_constraint(query) + if unsupported: + return {**result_fields(status='ambiguous',sources=[],reason='unsupported_constraint',complete=False), + 'query':query,'engine':'hybrid','strategies':[],'source_count':0,'sources':[], + 'context':'','clarification':constraint_message(unsupported), + 'instructions':RAG_INSTRUCTIONS,'answer_format':ANSWER_FORMAT} response = search_database( db_path, query, diff --git a/tools/retrieval/question_constraints.py b/tools/retrieval/question_constraints.py new file mode 100644 index 0000000..03f05d8 --- /dev/null +++ b/tools/retrieval/question_constraints.py @@ -0,0 +1,42 @@ +"""Fail closed for recognized query operators without an implemented meaning.""" +import re +import unicodedata + + +def requested_years(question: str) -> list[int]: + """An unquoted four-digit year remains a condition without 'v roku'.""" + text=re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question) + return list(dict.fromkeys(int(value) for value in re.findall(r'\b\d{4}\b',text))) + + +def unsupported_constraint(question: str) -> str | None: + text = ''.join(c for c in unicodedata.normalize('NFKD', question).casefold() + if not unicodedata.combining(c)) + # Do not treat words inside quoted academic titles as query operators. + text = re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', text) + if (re.search(r'\b(?:rok\w*|datum\w*|kedy)\b',text) and + re.search(r'\b(?:obhajob\w*|obhajil\w*|dokonc\w*|ukoncil\w*|zacal\w* pracovat)\b',text) and + re.search(r'\b(?:prac\w*|projekt\w*)\b',text)): + return 'work_event_date' + if re.search(r'\b(?:od|do|pred|po|medzi)\s+(?:rok(?:om|mi|u)?\s+)?\d{4}\b|\b\d{4}\s*[-–]\s*\d{4}\b', text): + return 'year_range' + if re.search(r'\b(?:aspon|najviac|viac ako|menej ako|minimalne|maximalne)\s+\d+', text): + return 'quantity_condition' + if re.search(r'\b(?:okrem|s vynimkou)\b|\b(?:nema|nemaju)\s+(?:tag|klucov|kategori|prac|projekt)|' + r'\bbez\s+(?:tagu|klucoveho slova|kategorie)\b', text): + return 'exclusion' + if re.search(r'\b(?:tag\w*|klucov\w*\s+slov\w*)\s+\w+\s+alebo\s+\w+', text): + return 'alternative_tags' + return None + + +def constraint_message(code: str) -> str: + return { + 'work_event_date':'Rok uvedený pri práci nepreukazuje dátum jej začatia, dokončenia alebo obhajoby. ' + 'Chceš zistiť rok evidovaný pri práci?', + 'year_range': 'Časovú podmienku neviem spoľahlivo vyhodnotiť ako interval. Uveď konkrétny rok práce.', + 'quantity_condition': 'Podmienku na minimálny alebo maximálny počet neviem spoľahlivo vyhodnotiť. ' + 'Požiadaj najprv o konkrétny zoznam alebo počet.', + 'exclusion': 'Vylučovaciu podmienku neviem spoľahlivo vyhodnotiť. Uveď, ktoré vlastnosti majú výsledky spĺňať.', + 'alternative_tags': 'Uveď, či požaduješ oba tagy zároveň, alebo vyhľadávanie pre každý tag samostatne.', + }[code]