Kvalitnejšie odpovede
This commit is contained in:
parent
59f208b571
commit
16af0f8b4f
18
README.md
18
README.md
@ -1160,7 +1160,7 @@ upozornenie na neúplnosť, nezamieňať autora dokumentu s autorom práce a pri
|
||||
sa nevysvetľujú odhadom; verzia dát neoznačuje verziu programu.
|
||||
Tieto pokyny ovplyvňujú správanie Claude, jeho finálny text však náš backend
|
||||
nemôže programovo vynútiť. Po aktualizácii už nahratých Skills nahraď ich
|
||||
novými ZIPmi z `packages/claude/dist/` (verzia 1.2.0) a over správanie v novom chate.
|
||||
novými ZIPmi z `packages/claude/dist/` (verzia 1.3.0) a over správanie v novom chate.
|
||||
|
||||
### Nahratie Skills do webového Claude
|
||||
|
||||
@ -1507,6 +1507,22 @@ Chýbajúce výsledky sa oznámia, neodvodzujú sa z cieľov. Grafová otázka n
|
||||
v ktorých majú práce „obaja“, používa prienik rokov a prikladá doložené typy a názvy
|
||||
prác z týchto rokov; nezobrazuje namiesto odpovede všetky individuálne roky.
|
||||
|
||||
`rag_preparation.py` vie pomenované profily dohľadať v SQLite aj mimo prvých
|
||||
výsledkov a pri viacerých možných prácach vyžiada typ alebo rok. Obsahové
|
||||
porovnanie je obmedzené na štyri osoby naraz. `question_intent.py` rozlišuje
|
||||
požadované problémy, metódy, experimenty, ciele a výsledky. `content_selection.py`
|
||||
vyberá doslovné, typovo a osobne viazané podklady, odstraňuje opakovania a
|
||||
uprednostňuje vecné výsledky pred administratívnymi poznámkami. Model dostane
|
||||
len tieto kandidátske výňatky; ich skutočný JSON podlieha tokenovému rozpočtu.
|
||||
Chýbajúce časti sa označia jednotlivo, aj keď sú ostatné časti zodpovedateľné.
|
||||
Pri limite kontextu sa nestratí informácia, ktoré osoby používateľ požadoval.
|
||||
|
||||
Rozpoznané, ale nepodporované operátory (napríklad interval rokov, vylučovanie
|
||||
alebo minimálny počet) vedú k `ambiguous` a konkrétnemu spresneniu. Rok práce
|
||||
sa nepoužije ako dôkaz dátumu jej začatia či obhajoby. Chybové odpovede odlišujú
|
||||
nedostupné vyhľadávanie, nedostupný model, vyťaženie a zamietnutý neoverený text;
|
||||
raw výstupy odmietnutého modelu sa štandardnému klientovi neposkytujú.
|
||||
|
||||
Otázky na obsah, zadanie, ciele alebo metódy práce používajú textové výňatky
|
||||
z príslušnej akademickej sekcie, nie iba metadáta o názve a roku. Výňatok musí
|
||||
patriť požadovanej osobe, typu práce a prípadne roku. Názov, rok ani samotný
|
||||
|
||||
@ -49,6 +49,7 @@ from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY
|
||||
from tools.runtime.tool_health import tools_readiness
|
||||
from tools.retrieval.retrieval_contract import RetrievalResponse, RetrievalSource, RetrievalErrorResponse, error_reason
|
||||
from tools.answers.answer_delivery import deliver_answer
|
||||
from tools.answers.answer_errors import public_failure
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -192,9 +193,7 @@ def verified_answer(request: AnswerRequest) -> dict:
|
||||
'answer': 'Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.'}
|
||||
if (result.get('accepted_for_delivery') is not True or
|
||||
result.get('status') not in {'ok', 'no_answer', 'ambiguous'}):
|
||||
result = {**result, 'status': 'error', 'accepted_for_delivery': False,
|
||||
'answer': 'Odpoveď neprešla overením podkladov. Neoverené údaje sa nezobrazia.',
|
||||
'claims': [], 'source_urls': []}
|
||||
result = public_failure(result)
|
||||
if not request.include_diagnostics:
|
||||
result.pop('attempts', None)
|
||||
result.pop('retrieval_context', None)
|
||||
|
||||
@ -81,8 +81,22 @@ class Config:
|
||||
|
||||
|
||||
def error_result(reason):
|
||||
messages={
|
||||
'unauthorized':'Prístup nebol autorizovaný. Skontroluj prihlásenie konektora a jeho konfiguráciu.',
|
||||
'forbidden':'Pre tento nástroj nemáš potrebné oprávnenie.',
|
||||
'invalid_query':'Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.',
|
||||
'model_unavailable':'Jazykový model momentálne nie je dostupný. Skús otázku neskôr.',
|
||||
'answer_rejected':'Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. '
|
||||
'Spresni osobu, typ práce alebo rok; neoverený text sa nezobrazí.',
|
||||
'answer_capacity_exceeded':'Agent práve spracúva iné otázky. Skús to o chvíľu.',
|
||||
'backend_busy':'Agent práve spracúva iné otázky. Skús to o chvíľu.',
|
||||
'backend_timeout':'Vyhľadávanie neodpovedalo včas. Skús otázku zopakovať neskôr.',
|
||||
'backend_unavailable':'Spojenie s backendom momentálne nie je dostupné. Skontroluj, či bežia služby a tunel.',
|
||||
'retrieval_unavailable':'Vyhľadávací index momentálne nie je dostupný. Skús otázku neskôr.',
|
||||
}
|
||||
return {'status': 'error', 'reason': reason, 'accepted_for_delivery': False,
|
||||
'answer': '', 'sources': [], 'index_version': None, 'complete': False,
|
||||
'answer': messages.get(reason,'Overenú odpoveď sa momentálne nepodarilo získať.'),
|
||||
'sources': [], 'index_version': None, 'complete': False,
|
||||
'truncated': False, 'truncation_reasons': []}
|
||||
|
||||
|
||||
|
||||
4
packages/claude/dist/packages.json
vendored
4
packages/claude/dist/packages.json
vendored
@ -3,12 +3,12 @@
|
||||
{
|
||||
"archive": "zpwiki-rag.zip",
|
||||
"name": "rag",
|
||||
"sha256": "8f0c51e27dc68a1e3ee845f2cfe5987e51714d4408f98c7d275861dc1e22953f"
|
||||
"sha256": "09d3e6b196b61f6c025eb86ccd1773b913b37660163a995ad1a2acfa6d947a0c"
|
||||
},
|
||||
{
|
||||
"archive": "zpwiki-graphrag.zip",
|
||||
"name": "graphrag",
|
||||
"sha256": "aa65a97211f48fa0352a98e50c6ca6a243e1d23105c0ff144c7f360ad192c376"
|
||||
"sha256": "cfcb60213e103d40c2adbc6c7d4133751120148415836cfffa7935c020167e95"
|
||||
}
|
||||
],
|
||||
"schema_version": 1
|
||||
|
||||
BIN
packages/claude/dist/zpwiki-graphrag.zip
vendored
BIN
packages/claude/dist/zpwiki-graphrag.zip
vendored
Binary file not shown.
BIN
packages/claude/dist/zpwiki-rag.zip
vendored
BIN
packages/claude/dist/zpwiki-rag.zip
vendored
Binary file not shown.
@ -11,7 +11,7 @@ import zipfile
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = ROOT / 'packages/claude'
|
||||
ENGINES = ('rag', 'graphrag')
|
||||
PACKAGE_VERSION = '1.2.0'
|
||||
PACKAGE_VERSION = '1.3.0'
|
||||
|
||||
|
||||
def json_bytes(value):
|
||||
|
||||
@ -156,7 +156,7 @@ def test_invalid_answer_coverage_is_not_delivered(backend, coverage):
|
||||
def test_transport_errors_and_redirects_do_not_disclose_payload(backend, code, reason):
|
||||
backend.update(raw=b'SECRET ERROR', code=code)
|
||||
value = client.query_backend('Práca?', 'rag')
|
||||
assert value['reason'] == reason and value['answer'] == ''
|
||||
assert value['reason'] == reason and value['answer']
|
||||
assert len(backend['requests']) == 1 and 'SECRET' not in json.dumps(value)
|
||||
|
||||
|
||||
@ -168,7 +168,7 @@ def test_rejected_or_malformed_answer_is_never_delivered(backend, change):
|
||||
backend.update(raw=json.dumps({**backend['checked'], **change}).encode())
|
||||
value = client.query_backend('Práca?', 'rag')
|
||||
assert value['status'] == 'error' and not value['accepted_for_delivery']
|
||||
assert value['answer'] == '' and 'Eva' not in json.dumps(value)
|
||||
assert value['answer'] and 'Eva' not in json.dumps(value)
|
||||
|
||||
|
||||
def test_invalid_json_and_oversized_response(backend):
|
||||
@ -279,7 +279,7 @@ def test_mcp_stdio_initialization_discovery_call_and_rejection(backend):
|
||||
assert all('presentation_instructions' not in json.loads(block.text)
|
||||
for block in result.content if block.type == 'text')
|
||||
rejected = await session.call_tool('zpwiki_rag', {'query': ''})
|
||||
assert rejected.isError and rejected.structuredContent['answer'] == ''
|
||||
assert rejected.isError and rejected.structuredContent['answer']
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
|
||||
@ -103,8 +103,11 @@ def comparison_payload(tmp_path):
|
||||
|
||||
|
||||
def selected_comparison(context):
|
||||
quotes=[{'source_id':p['source_id'],'quote':s['text']} for p in context['content_passages']
|
||||
quotes=([{'source_id':c['source_id'],'quote':c['text']} for c in context['content_candidates']]
|
||||
if 'content_candidates' in context else
|
||||
[{'source_id':p['source_id'],'quote':s['text']} for p in context['content_passages']
|
||||
for s in content_sections(p['text']) if s['kind'] in {'assignment','progress'}]
|
||||
)
|
||||
return {'status':'ok','answer':'','claims':[],'evidence_quotes':quotes,
|
||||
'source_urls':[s['source_url'] for s in context['sources']]}
|
||||
|
||||
@ -169,7 +172,7 @@ def test_missing_results_allow_an_explicit_partial_comparison(comparison_payload
|
||||
selected=selected_comparison(c)
|
||||
result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]})
|
||||
assert result['accepted_for_delivery']
|
||||
assert 'nie sú dosiahnuté výsledky doložené' in result['answer']
|
||||
assert 'Táto časť odpovede nie je v poskytnutých podkladoch doložená.' in result['answer']
|
||||
assert 'Implementácia systému vyhľadávania' in result['answer']
|
||||
|
||||
|
||||
@ -182,9 +185,9 @@ def test_future_tasks_are_not_presented_as_achieved_results(comparison_payload):
|
||||
p=next(p for p in c['content_passages'] if p['person']=='Ján Holp')
|
||||
selected['evidence_quotes'].append({'source_id':p['source_id'],'quote':'- Pripravte experiment s PageRank.'})
|
||||
result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]})
|
||||
assert result['accepted_for_delivery']
|
||||
assert '**Ďalšie doložené úlohy (plán):**' in result['answer']
|
||||
assert 'Pripraviť experiment s PageRank.' in result['answer']
|
||||
# The model may not quote a future task omitted from its actual candidates.
|
||||
assert not result['accepted_for_delivery']
|
||||
assert 'quote_outside_delivered_candidates' in result['attempts'][-1]['issues']
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
262
test/test_quality_block.py
Normal file
262
test/test_quality_block.py
Normal file
@ -0,0 +1,262 @@
|
||||
"""Behavioral checks for scope, relevant evidence, partial answers and failures."""
|
||||
import copy
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.answer_evaluation.content_selection import candidates, facets_for
|
||||
from tools.answer_evaluation.rag_evidence import rag_evidence_context
|
||||
from tools.answer_evaluation.rag_preparation import prepare_named_content
|
||||
from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues
|
||||
from tools.answers.answer_errors import public_failure
|
||||
from tools.graphrag import graph_search as gs
|
||||
from tools.graphrag.tests.test_graph_search import FakeSession
|
||||
from tools.rag.llm_context import compact_rag_payload
|
||||
from tools.rag.rag_utils import build_context_text
|
||||
from tools.retrieval.question_constraints import unsupported_constraint, requested_years
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def corpus(tmp_path):
|
||||
db=tmp_path/'index.sqlite'; sources=[]
|
||||
with sqlite3.connect(db) as conn:
|
||||
conn.execute('CREATE TABLE documents(path,title,author,published)')
|
||||
conn.execute('CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)')
|
||||
for i,(person,slug,kind,year,text) in enumerate([
|
||||
('Eva Nová','eva_nova','Diplomová práca',2024,
|
||||
'Zadanie:\n1. Implementujte systém vyhľadávania.\nStav:\n- Konzultácia štruktúry práce.\n'
|
||||
'- Natrénovaný model BERT a vykonané vyhodnotenie presnosti.\n'
|
||||
'- Vznikol problém s tokenizérom.\nÚlohy:\n- Pripravte experiment s rôznymi tokenizérmi.'),
|
||||
('Boris Nový','boris_novy','Bakalárska práca',2023,
|
||||
'Zadanie:\n- Implementujte parser.\nStav:\n- Implementovaný parser pomocou Python.\n'
|
||||
'- Vznikla chyba pri načítaní PDF.\nÚlohy:\n- Pripravte experiment s presnosťou extrakcie.')],1):
|
||||
path=f'pages/students/2020/{slug}/README.md'
|
||||
conn.execute('INSERT INTO documents VALUES(?,?,?,?)',(path,person,'Daniel Hladek',1))
|
||||
conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path,text,json.dumps([[person,f'{kind} {year}']]),1,0))
|
||||
sources.append({'source_id':f'S{i}','title':person,'document_path':path,'published':True,
|
||||
'source_url':f'https://example.test/{slug}',
|
||||
'text':f'Sekcia: {kind} {year}\nNázov: Ukážkový názov.'})
|
||||
raw={'status':'ok','reason':None,'sources':sources,'context':build_context_text(sources),
|
||||
'complete':False,'truncated':True,'truncation_reasons':['result_limit'],'index_version':{'version':'fixed'}}
|
||||
return db,raw
|
||||
|
||||
|
||||
def prepare(corpus,question):
|
||||
db,raw=corpus
|
||||
return compact_rag_payload(prepare_named_content(db,question,raw))
|
||||
|
||||
|
||||
def model_selection(question,payload):
|
||||
c=rag_evidence_context(question,payload)
|
||||
return {'status':'ok','answer':'','claims':[],
|
||||
'evidence_quotes':[{'source_id':x['source_id'],'quote':x['text']} for x in c.get('content_candidates',[])],
|
||||
'source_urls':[s['source_url'] for s in c['sources']]}
|
||||
|
||||
|
||||
def complete(answer):
|
||||
return lambda _: {'choices':[{'message':{'content':json.dumps(answer)}}]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize('question,facets',[
|
||||
('Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?',{'problems','experiments'}),
|
||||
('Aké metódy používa Eva Nová?',{'methods'}),
|
||||
('Aké sú dosiahnuté výsledky Evy Novej?',{'results'}),
|
||||
])
|
||||
def test_content_without_explicit_work_type(corpus,question,facets):
|
||||
payload=prepare(corpus,question); c=rag_evidence_context(question,payload)
|
||||
assert c['requirements']['content'] and set(c['requirements']['facets'])==facets
|
||||
assert c['content_candidates']
|
||||
|
||||
|
||||
def test_multiple_people_and_aspects_remain_bound(corpus):
|
||||
q='Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?'
|
||||
payload=prepare(corpus,q); a=model_selection(q,payload)
|
||||
r=generate_verified_rag(q,payload,complete(a))
|
||||
assert r['accepted_for_delivery'] and r['status']=='ok'
|
||||
text=r['answer']; eva,boris=text.split('Boris Nový — bakalárska práca')
|
||||
assert 'tokenizérom' in eva and 'PDF' not in eva
|
||||
assert 'PDF' in boris and 'tokenizérmi' not in boris
|
||||
assert text.count('**Riešené problémy:**')==2
|
||||
assert text.count('**Plánované experimenty a vyhodnotenie:**')==2
|
||||
|
||||
|
||||
def test_ambiguous_work_is_a_clarification_without_model(corpus):
|
||||
db,raw=corpus; source=raw['sources'][0]
|
||||
with sqlite3.connect(db) as conn:
|
||||
conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(source['document_path'],
|
||||
'Stav:\nVznikol iný problém.',json.dumps([['Eva Nová','Bakalárska práca 2022']]),1,1))
|
||||
q='Aké problémy mala Eva Nová počas práce?'
|
||||
p=prepare(corpus,q)
|
||||
r=generate_verified_rag(q,p,lambda _:pytest.fail('No model call for a clarification'))
|
||||
assert r['status']=='ambiguous' and r['reason']=='ambiguous_work'
|
||||
assert '2022' in r['answer'] and '2024' in r['answer']
|
||||
|
||||
|
||||
def test_explicit_type_and_year_resolve_scope(corpus):
|
||||
db,raw=corpus
|
||||
with sqlite3.connect(db) as conn:
|
||||
conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(raw['sources'][0]['document_path'],
|
||||
'Stav:\nChyba v inom projekte Cassandra.',json.dumps([['Eva Nová','Diplomový projekt 2022']]),1,1))
|
||||
q='Aké problémy mala Eva Nová pri diplomovej práci v roku 2024?'
|
||||
p=prepare(corpus,q)
|
||||
assert p['status']=='ok' and 'Cassandra' not in p['context']
|
||||
|
||||
|
||||
def test_year_condition_without_the_words_v_roku_is_preserved(corpus):
|
||||
q='Aké problémy mala Eva Nová pri diplomovej práci 1999?'
|
||||
p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
||||
assert c['requirements']['years']==[1999] and not c.get('content_candidates')
|
||||
r=generate_verified_rag(q,p,lambda _:pytest.fail('No model for missing scoped evidence'))
|
||||
assert r['accepted_for_delivery'] and r['status']=='no_answer' and 'tokenizérom' not in r['answer']
|
||||
assert requested_years('Práca „Model 2020“ z roku 2024')==[2024]
|
||||
|
||||
|
||||
def test_named_source_can_be_loaded_when_absent_from_top_results(corpus):
|
||||
db,raw=corpus; raw['sources']=raw['sources'][:1];raw['context']=build_context_text(raw['sources'])
|
||||
p=prepare(corpus,'Aké problémy riešili Eva Nová a Boris Nový?')
|
||||
assert {s['title'] for s in p['sources']}=={'Eva Nová','Boris Nový'}
|
||||
|
||||
|
||||
def test_preparation_does_not_change_the_original_retrieval(corpus):
|
||||
db,raw=corpus; before=copy.deepcopy(raw)
|
||||
prepare_named_content(db,'Aké problémy riešili Eva Nová a Boris Nový?',raw)
|
||||
assert raw==before
|
||||
|
||||
|
||||
def test_source_budget_cannot_silently_remove_a_requested_person(corpus):
|
||||
db,raw=corpus; q='Aké problémy riešili Eva Nová a Boris Nový?'
|
||||
p=prepare_named_content(db,q,raw)
|
||||
p['sources'][0]['text']+='\n'+'Veľmi dlhý zdrojový text. '*1000
|
||||
compact=compact_rag_payload(p,max_tokens=1100)
|
||||
c=rag_evidence_context(q,compact)
|
||||
assert compact['truncated']
|
||||
assert set(c['requirements']['people'])=={'Eva Nová','Boris Nový'}
|
||||
|
||||
|
||||
def test_candidates_are_actual_bound_literal_evidence(corpus):
|
||||
q='Aké problémy riešili Eva Nová a Boris Nový?'; c=rag_evidence_context(q,prepare(corpus,q))
|
||||
for candidate in candidates(c):
|
||||
assert any(all(candidate[k]==p[k] for k in ['person','work_type','year','source_id'])
|
||||
and candidate['text'] in p['text'] for p in c['content_passages'])
|
||||
|
||||
|
||||
def test_unknown_second_subject_is_not_silently_ignored(corpus):
|
||||
p=prepare(corpus,'Aké problémy riešili študenti Eva Nová a Boris Noyy?')
|
||||
assert p['status']=='ambiguous' and p['reason']=='unresolved_subject'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('question',[
|
||||
'Aké problémy riešili Eva Nová a Xaver Neexistujúci?',
|
||||
'Aké problémy riešil Boris Noyy?',
|
||||
])
|
||||
def test_unresolved_subject_cannot_be_replaced_by_available_profiles(corpus,question):
|
||||
p=prepare(corpus,question)
|
||||
assert p['status']=='ambiguous' and p['reason']=='unresolved_subject'
|
||||
|
||||
|
||||
def test_partial_answer_names_the_missing_aspect(corpus):
|
||||
db,raw=corpus
|
||||
with sqlite3.connect(db) as conn:
|
||||
conn.execute("UPDATE chunks SET text='Stav:\nVznikol problém s parserom.' WHERE document_path LIKE '%boris_novy%'")
|
||||
q='Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?'
|
||||
p=prepare(corpus,q); r=generate_verified_rag(q,p,complete(model_selection(q,p)))
|
||||
assert r['accepted_for_delivery'] and r['answer_coverage']['status']=='partial'
|
||||
assert 'Táto časť odpovede nie je v poskytnutých podkladoch doložená.' in r['answer']
|
||||
assert 'parserom' in r['answer']
|
||||
|
||||
|
||||
def test_all_aspects_absent_is_no_answer_not_rejection(corpus):
|
||||
db,raw=corpus
|
||||
with sqlite3.connect(db) as conn:
|
||||
conn.execute("UPDATE chunks SET text='Zadanie:\n- Implementujte parser.'")
|
||||
q='Aké problémy riešili Eva Nová a Boris Nový?'
|
||||
r=generate_verified_rag(q,prepare(corpus,q),lambda _:pytest.fail('No model for a bounded empty result'))
|
||||
assert r['status']=='no_answer' and r['accepted_for_delivery'] and not r['claims']
|
||||
|
||||
|
||||
def test_known_year_is_returned_when_requested_title_is_missing(corpus):
|
||||
_,raw=corpus; source=raw['sources'][0]
|
||||
source['text']='Sekcia: Diplomová práca 2024\nStav:\nImplementovaný parser.'
|
||||
raw['sources']=[source];raw['context']=build_context_text(raw['sources'])
|
||||
q='Uveď názov a rok diplomovej práce Evy Novej.'
|
||||
a={'status':'ok','answer':'','claims':[{'person':'Eva Nová','work_type':'diploma_thesis','year':2024}],
|
||||
'evidence_quotes':[],'source_urls':[source['source_url']]}
|
||||
r=generate_verified_rag(q,raw,complete(a))
|
||||
assert r['accepted_for_delivery'] and '2024' in r['answer']
|
||||
assert 'názov práce nie je v poskytnutých podkladoch doložený' in r['answer']
|
||||
assert r['answer_coverage']['status']=='partial' and not r['claims'][0].get('title')
|
||||
|
||||
|
||||
def test_concrete_results_win_over_administrative_notes(corpus):
|
||||
q='Aké výsledky dosiahla Eva Nová?'; p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
||||
text='\n'.join(x['text'] for x in c['content_candidates'])
|
||||
assert 'Natrénovaný model BERT' in text
|
||||
r=generate_verified_rag(q,p,complete(model_selection(q,p)))
|
||||
assert r['accepted_for_delivery'] and 'Natrénovaný model BERT' in r['answer']
|
||||
assert 'Konzultácia' not in r['answer']
|
||||
|
||||
|
||||
def test_duplicate_quotes_and_overlapping_chunks_do_not_duplicate_output(corpus):
|
||||
q='Aké výsledky dosiahla Eva Nová?'; p=prepare(corpus,q); a=model_selection(q,p)
|
||||
a['evidence_quotes']*=2
|
||||
r=generate_verified_rag(q,p,complete(a))
|
||||
assert r['accepted_for_delivery'] and r['answer'].count('Natrénovaný model BERT')==1
|
||||
assert 'Ďalšie podklady:' not in r['answer'] and 'Ďalšie doložené podklady' not in r['answer']
|
||||
|
||||
|
||||
def test_plans_cannot_answer_actual_use_or_completed_experiments():
|
||||
assert 'methods' not in facets_for('Použite metódu BERT.','task','Aké metódy skutočne používal?')
|
||||
assert 'experiments' not in facets_for('Pripravte experiment.','task','Aké experimenty vykonal?')
|
||||
assert 'results' not in facets_for('Cieľom je implementovať systém.','progress','Aké dosiahol výsledky?')
|
||||
|
||||
|
||||
def test_method_answer_does_not_add_valid_but_unrequested_profile_metadata(corpus):
|
||||
_,raw=corpus
|
||||
raw['sources'][0]['author']='Daniel Hladek'
|
||||
q='Aké metódy používala Eva Nová pri diplomovej práci v roku 2024?'
|
||||
p=prepare(corpus,q); a=model_selection(q,p)
|
||||
a['claims']=[{'person':'Eva Nová','work_type':'diploma_thesis','year':2024,
|
||||
'document_author':'Daniel Hladek'}]
|
||||
r=generate_verified_rag(q,p,complete(a))
|
||||
assert r['accepted_for_delivery'] and 'BERT' in r['answer']
|
||||
assert 'Daniel Hladek' not in r['answer'] and 'autor dokumentu' not in r['answer']
|
||||
assert r['answer'].count('2024')==1
|
||||
|
||||
|
||||
@pytest.mark.parametrize('question',[
|
||||
'Práce od roku 2020 do roku 2024?', 'Práce pred rokom 2020?',
|
||||
'Študenti s aspoň 3 prácami?', 'Práce okrem diplomových?', 'Študenti bez tagu RAG?',
|
||||
'V ktorom roku Ján Holp začal pracovať na diplomovej práci?',
|
||||
'Kedy Ján Holp obhájil diplomovú prácu?',
|
||||
])
|
||||
def test_unsupported_operators_are_not_ignored(question):
|
||||
assert unsupported_constraint(question)
|
||||
result=gs.search_graph_with_session(FakeSession({}),question)
|
||||
assert result['status']=='ambiguous' and result['reason']=='unsupported_constraint' and not result['facts']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('question',[
|
||||
'Práca v roku 1999?', 'Aký je názov práce „Od 2020 do 2024“?',
|
||||
'Aké práce alebo projekty má Eva Nová?',
|
||||
])
|
||||
def test_supported_year_and_work_alternative_are_not_blocked(question):
|
||||
assert unsupported_constraint(question) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('reason',['model_unavailable','retrieval_unavailable','answer_rejected','answer_capacity_exceeded'])
|
||||
def test_public_errors_are_distinct_and_do_not_leak_rejected_text(reason):
|
||||
r=public_failure({'reason':reason,'answer':'SECRET YEAR 2099','claims':[{'person':'Eva'}],'source_urls':['SECRET']})
|
||||
assert r['reason']==reason and not r['accepted_for_delivery']
|
||||
assert 'SECRET' not in r['answer'] and '2099' not in r['answer']
|
||||
assert not r['claims'] and not r['source_urls']
|
||||
|
||||
|
||||
def test_retrieval_failure_is_not_misreported_as_bad_model_answer(monkeypatch):
|
||||
import tools.answers.answer_delivery as delivery
|
||||
def unavailable(_): raise RuntimeError('SECRET backend address')
|
||||
monkeypatch.setattr(delivery,'search_graph',unavailable)
|
||||
r=delivery.deliver_answer('Aké práce má Eva Nová?',engine='graphrag',
|
||||
complete=lambda _:pytest.fail('No model call after failed retrieval'))
|
||||
assert r['status']=='error' and r['reason']=='retrieval_unavailable'
|
||||
assert 'SECRET' not in r['answer']
|
||||
@ -39,7 +39,7 @@ def answer_coverage(answer: dict, context: dict) -> dict:
|
||||
(row.get('source_url') in cited or answer.get('status') == 'no_answer')))
|
||||
# A context budget may drop only unrelated sources. A clipping marker in
|
||||
# the actual cited block is stronger evidence than this global reason.
|
||||
context_limited = 'llm_context_budget' in reasons
|
||||
context_limited = any(reason in reasons for reason in ['llm_context_budget','answer_candidate_budget','evidence_unit_limit'])
|
||||
limited = context.get('complete') is not True or context.get('truncated') is True
|
||||
limitations = [*reasons]
|
||||
if clipped:
|
||||
@ -48,6 +48,13 @@ def answer_coverage(answer: dict, context: dict) -> dict:
|
||||
status = answer.get('status')
|
||||
if status == 'ok':
|
||||
coverage = 'supported'
|
||||
if any(requirements.get(key) and key not in fields for fields in requirements.get('available_fields',{}).values()
|
||||
for key in ['title','year','start_year']):
|
||||
coverage='partial'; warnings.append('partial_requested_fields')
|
||||
if requirements.get('facets'):
|
||||
from .content_selection import missing_facets
|
||||
if missing_facets(answer,context):
|
||||
coverage='partial'; warnings.append('partial_requested_aspects')
|
||||
if scope == 'list' and (limited or clipped):
|
||||
coverage = 'partial'; warnings.append('list_may_be_incomplete')
|
||||
elif scope == 'count' and (context.get('aggregation') or {}).get('complete') is not True:
|
||||
@ -74,4 +81,7 @@ def coverage_notices(coverage: dict) -> list[str]:
|
||||
'missing_information_with_clipped_context': 'Časť podkladov bola pri príprave kontextu skrátená.',
|
||||
'count_not_established': 'Celkový počet sa z dostupných podkladov nedá spoľahlivo určiť.',
|
||||
}
|
||||
return [messages[reason] for reason in coverage['warnings']]
|
||||
notices=[messages[reason] for reason in coverage['warnings'] if reason in messages]
|
||||
if 'list_may_be_incomplete' in coverage['warnings']:
|
||||
notices.append('Výpis môžeš zúžiť uvedením osoby, typu práce alebo konkrétneho roku.')
|
||||
return notices
|
||||
|
||||
@ -16,7 +16,8 @@ def generate_checked_answer(question: str, context: dict, prompt: str, complete)
|
||||
result['response']=response;record['response']=response
|
||||
text=response['choices'][0]['message']['content'].strip()
|
||||
except Exception as error:
|
||||
result['error']=f'{type(error).__name__}: {str(error)[:500]}'
|
||||
result['error']=type(error).__name__
|
||||
result['reason']='model_unavailable'
|
||||
result['attempts'].append(record)
|
||||
return result
|
||||
result['attempts'].append(record)
|
||||
|
||||
191
tools/answer_evaluation/content_selection.py
Normal file
191
tools/answer_evaluation/content_selection.py
Normal file
@ -0,0 +1,191 @@
|
||||
"""Role-aware, bounded excerpt selection and presentation from actual evidence."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from .context_policy import normalized
|
||||
from .question_intent import FACET_LABELS, planned_experiments
|
||||
|
||||
COMPLETED = r'\b(?:implementovan\w*|natrenovan\w*|vytvoren\w*|extrahovan\w*|vykonan\w*|urobene vyhodnotenie|vyhodnoteny\w*|nameran\w*)\b'
|
||||
PLANNED = r'\b(?:bude|budu|planuj\w*|cielom|mal by|ma sa|treba|pripravte|navrhnite|otestujte|vyhodnotte)\b'
|
||||
ADMIN = r'\b(?:konzultac\w*|struktura prace|pisomn\w*|pisat|odovzda\w*|wip|termin\w*)\b'
|
||||
|
||||
|
||||
def split_units(text: str) -> list[str]:
|
||||
from .rag_evidence import descriptive_text
|
||||
parts = re.split(r'\n\s*\n|(?=^[ \t]*(?:\d+[.)]|[-+*])\s+)', text, flags=re.M)
|
||||
units = []
|
||||
for part in parts:
|
||||
part = part.strip()
|
||||
if not descriptive_text(part):
|
||||
continue
|
||||
# A source paragraph is kept literal, including its negations and qualifications.
|
||||
for unit in ([part] if len(part) <= 900 else re.split(r'(?<=[.!?])\s+(?=[A-ZÁČĎÉÍĽŇÓŠŤÚÝŽ])', part)):
|
||||
if descriptive_text(unit):
|
||||
units.append(unit.strip())
|
||||
return units
|
||||
|
||||
|
||||
def effective_role(text: str, role: str) -> str:
|
||||
n = normalized(text)
|
||||
if role=='other':
|
||||
from .presentation import GOAL_VERBS
|
||||
first=n.split()[0] if n else ''
|
||||
if first in {normalized(word) for word in GOAL_VERBS} or n.startswith('cielom je'):
|
||||
return 'task'
|
||||
if role == 'other' and re.search(COMPLETED, n) and not re.search(PLANNED, n):
|
||||
return 'progress'
|
||||
return role
|
||||
|
||||
|
||||
def facets_for(text: str, role: str, question: str) -> list[str]:
|
||||
n = normalized(text); facets = []
|
||||
if role == 'assignment': facets.append('assignment')
|
||||
if role == 'progress' and (not re.search(PLANNED,n) or re.search(COMPLETED,n)):
|
||||
facets.append('results')
|
||||
if re.search(r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*|zlyha\w*|nepodar\w*|nevyries\w*)\b', n) and role not in {'assignment','task'}:
|
||||
facets.append('problems')
|
||||
if re.search(r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*)\b', n):
|
||||
if role not in {'assignment','task'} or not re.search(r'\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*)\b',normalized(question)):
|
||||
facets.append('methods')
|
||||
if re.search(r'\b(?:experiment\w*|meran\w*|test\w*|otest\w*|vyhodnot\w*|benchmark\w*|f1|presnost\w*)\b', n):
|
||||
wants_done=re.search(r'\b(?:vykonal\w*|otestoval\w*|uskutocnil\w*|vyhodnotil\w*)\b',normalized(question))
|
||||
if wants_done and role in {'assignment','task'}:
|
||||
pass
|
||||
elif not planned_experiments(question) or role in {'assignment','task'} or re.search(PLANNED, n):
|
||||
facets.append('experiments')
|
||||
return facets
|
||||
|
||||
|
||||
def unit_score(text: str, role: str, facet: str = '') -> int:
|
||||
n = normalized(text)
|
||||
score = 25 if role == 'progress' else 10
|
||||
if re.search(COMPLETED, n): score += 45
|
||||
if re.search(r'\b(?:model|system|algoritm\w*|vyhodnot\w*|experiment\w*|presnost\w*|pagerank|bert)\b', n): score += 15
|
||||
if re.search(ADMIN, n): score -= 70
|
||||
if facet == 'problems' and re.search(r'\b(?:problem\w*|chyb\w*|zlyha\w*)\b', n): score += 35
|
||||
if role in {'assignment','task'} and facet == 'results': score -= 100
|
||||
return score
|
||||
|
||||
|
||||
def evidence_units(context: dict, quotes: list[dict] | None = None) -> list[dict]:
|
||||
from .rag_evidence import content_sections, quote_sections
|
||||
rows = []
|
||||
for passage in context.get('content_passages', []):
|
||||
pieces = content_sections(passage['text']) if quotes is None else [
|
||||
piece for quote in quotes if quote['source_id'] == passage['source_id'] and
|
||||
' '.join(quote['quote'].split()) in ' '.join(passage['text'].split())
|
||||
for piece in quote_sections(quote, context)]
|
||||
for piece in pieces:
|
||||
for unit in split_units(piece['text']):
|
||||
role = effective_role(unit, piece['kind'])
|
||||
rows.append({k: passage.get(k) for k in ['person','work_type','year','source_id']} |
|
||||
{'text':unit,'kind':role,'facets':facets_for(unit,role,context.get('question',''))})
|
||||
# Overlapping chunks/quotes can repeat the same fact with different roles.
|
||||
# An explicit role wins over an unlabelled duplicate, within the same work.
|
||||
unique = {}
|
||||
for row in rows:
|
||||
clean=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text'])
|
||||
key = tuple(row.get(k) for k in ['person','work_type','year']) + (' '.join(clean.split()),)
|
||||
old = unique.get(key)
|
||||
if old is None or (old['kind']=='other' and row['kind']!='other'):
|
||||
unique[key]=row
|
||||
return list(unique.values())
|
||||
|
||||
|
||||
def candidates(context: dict) -> list[dict]:
|
||||
facets = context['requirements'].get('facets', [])
|
||||
selected = []; seen = set(); units = evidence_units(context)
|
||||
for person in context['requirements']['people']:
|
||||
for facet in facets:
|
||||
relevant = [u for u in units if u['person']==person and facet in u['facets']]
|
||||
relevant.sort(key=lambda u: -unit_score(u['text'],u['kind'],facet))
|
||||
accepted=0
|
||||
for unit in relevant:
|
||||
text = unit['text']
|
||||
if len(text)>2500:
|
||||
context['complete']=False; context['truncated']=True
|
||||
context['truncation_reasons']=list(dict.fromkeys([*(context.get('truncation_reasons') or []),'evidence_unit_limit']))
|
||||
continue
|
||||
key=(unit['source_id'],text)
|
||||
if key not in seen:
|
||||
seen.add(key); selected.append({**unit,'text':text})
|
||||
accepted+=1
|
||||
if accepted==2: break
|
||||
return selected
|
||||
|
||||
|
||||
def missing_facets(answer: dict, context: dict) -> list[tuple[str,str]]:
|
||||
found = evidence_units(context, answer.get('evidence_quotes', []))
|
||||
return [(person,facet) for person in context['requirements']['people']
|
||||
for facet in context['requirements'].get('facets', [])
|
||||
if not any(row['person']==person and facet in row['facets'] for row in found)]
|
||||
|
||||
|
||||
def fit_candidates(sent: dict, context: dict, limit: int, encoding: str) -> None:
|
||||
"""Budget the actual candidate JSON, sharing space across named people."""
|
||||
import json
|
||||
from tools.rag.build_chunks import TokenCounter
|
||||
counter=TokenCounter(encoding)
|
||||
if counter.encoding is None:
|
||||
raise RuntimeError('Missing context tokenizer')
|
||||
original=list(context.get('content_candidates',[]))
|
||||
groups=[[row for row in original if row.get('person')==person]
|
||||
for person in context['requirements']['people']]
|
||||
ordered=[group[i] for i in range(max(map(len,groups),default=0)) for group in groups if i<len(group)]
|
||||
selected=[]
|
||||
for row in ordered:
|
||||
if counter.count(json.dumps({**sent,'content_candidates':[*selected,row]},ensure_ascii=False))<=limit:
|
||||
selected.append(row)
|
||||
if len(selected)!=len(original):
|
||||
context['complete']=False;context['truncated']=True
|
||||
context['truncation_reasons']=list(dict.fromkeys([*(context.get('truncation_reasons') or []),'answer_candidate_budget']))
|
||||
sent.update({key:context[key] for key in ['complete','truncated','truncation_reasons']})
|
||||
context['content_candidates']=selected;sent['content_candidates']=selected
|
||||
# Account for the added limitation metadata as well.
|
||||
while selected and counter.count(json.dumps(sent,ensure_ascii=False))>limit:
|
||||
selected.pop()
|
||||
if counter.count(json.dumps(sent,ensure_ascii=False))>limit:
|
||||
raise ValueError('Mandatory answer metadata exceeds context budget')
|
||||
|
||||
|
||||
def facet_lines(answer: dict, context: dict) -> list[str]:
|
||||
from .presentation import goal_text, work_subject, owner_in_question
|
||||
from .rag_evidence import WORK_LABELS
|
||||
req=context['requirements']; units=evidence_units(context,answer.get('evidence_quotes',[]))
|
||||
lines=[]
|
||||
for person in req['people']:
|
||||
scopes={(p['work_type'],p.get('year')) for p in context.get('content_passages',[]) if p.get('person')==person}
|
||||
kind=req['work_types'].get(person) or (next(iter(scopes))[0] if len(scopes)==1 else None)
|
||||
subject=work_subject(person,kind,context.get('question',''))
|
||||
if not owner_in_question(person,context.get('question','')):
|
||||
subject=person+(' — '+WORK_LABELS[kind] if kind in WORK_LABELS else '')
|
||||
if len(scopes)==1 and next(iter(scopes))[1] is not None:
|
||||
subject+=' ('+str(next(iter(scopes))[1])+')'
|
||||
lines.extend(['**'+subject+'**', ''])
|
||||
seen=set()
|
||||
for facet in req.get('facets', []):
|
||||
label=FACET_LABELS[facet]
|
||||
if facet=='experiments' and planned_experiments(context.get('question','')):
|
||||
label='Plánované experimenty a vyhodnotenie'
|
||||
rows=[u for u in units if u['person']==person and facet in u['facets']]
|
||||
rows.sort(key=lambda u:-unit_score(u['text'],u['kind'],facet))
|
||||
lines.append('**'+label+':**')
|
||||
shown=0
|
||||
for row in rows:
|
||||
key=' '.join(re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text']).split())
|
||||
if key in seen:
|
||||
continue
|
||||
text=row['text']
|
||||
if row['kind'] in {'assignment','task'}:
|
||||
text,_=goal_text(text)
|
||||
text=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',text)
|
||||
phase=' (plán)' if row['kind'] in {'assignment','task'} and facet not in {'assignment'} else ''
|
||||
lines.append('- '+text+phase); seen.add(key); shown+=1
|
||||
if shown==2: break
|
||||
if not shown:
|
||||
if rows:
|
||||
lines.append('Je doložené rovnakým výňatkom uvedeným vyššie.')
|
||||
else:
|
||||
lines.append('Táto časť odpovede nie je v poskytnutých podkladoch doložená.')
|
||||
lines.append('')
|
||||
return lines
|
||||
@ -54,6 +54,7 @@ def graph_context(result: dict) -> dict:
|
||||
'index_version':result.get('index_version')}
|
||||
context['answer_candidates']=joined_candidates(facts,intent)
|
||||
context['question'] = result.get('query', '')
|
||||
context['clarification'] = result.get('clarification')
|
||||
context['requested_people'] = list(dict.fromkeys(row['name']
|
||||
for row in result.get('resolved_entities', []) if row.get('kind') == 'Person' and row.get('name')))
|
||||
comparison = result.get('academic_year_comparison') or next((row for row in result['facts']
|
||||
|
||||
38
tools/answer_evaluation/question_intent.py
Normal file
38
tools/answer_evaluation/question_intent.py
Normal file
@ -0,0 +1,38 @@
|
||||
"""Explicit answer aspects; these select evidence, never supply facts."""
|
||||
import re
|
||||
from .context_policy import normalized
|
||||
|
||||
FACET_PATTERNS = {
|
||||
'assignment': r'\b(?:zadani\w*|ciel\w*)\b',
|
||||
'problems': r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*)\b',
|
||||
'methods': r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*)\b',
|
||||
'experiments': r'\b(?:experiment\w*|merani\w*|testovani\w*|vyhodnocov\w*)\b',
|
||||
'results': r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b',
|
||||
}
|
||||
FACET_LABELS = {'assignment': 'Zadanie a ciele', 'problems': 'Riešené problémy',
|
||||
'methods': 'Metódy a technológie', 'experiments': 'Experimenty a vyhodnotenie',
|
||||
'results': 'Zaznamenaný stav a výsledky'}
|
||||
|
||||
|
||||
def requested_facets(question: str) -> list[str]:
|
||||
text = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question))
|
||||
matches = [(m.start(), key) for key, pattern in FACET_PATTERNS.items()
|
||||
if (m := re.search(pattern, text))]
|
||||
return [key for _, key in sorted(matches)]
|
||||
|
||||
|
||||
def content_question(question: str) -> bool:
|
||||
text = normalized(question)
|
||||
return bool(requested_facets(question) or re.search(
|
||||
r'\b(?:comu|o com|obsah\w*|opis\w*|popis\w*|zamer\w*|zhrn\w*|implement\w*)\b', text))
|
||||
|
||||
|
||||
def facet_mode(question: str) -> bool:
|
||||
facets = requested_facets(question)
|
||||
return any(facet in facets for facet in ['problems','methods','experiments','results'])
|
||||
|
||||
|
||||
def planned_experiments(question: str) -> bool:
|
||||
text=normalized(question)
|
||||
return bool(re.search(r'\b(?:plan\w*|pripravit|navrhnut)|'
|
||||
r'\b(?:mal|mali|mala|ma|maju)\s+(?:priprav\w*|navrhn\w*|otest\w*|vyhodnot\w*)',text))
|
||||
@ -10,6 +10,8 @@ import json
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from .context_policy import normalized, text_contains_value, person_pattern
|
||||
from .question_intent import requested_facets, content_question, facet_mode
|
||||
from tools.retrieval.question_constraints import requested_years
|
||||
|
||||
WORK_TYPES = {
|
||||
'bakalarska praca': 'bachelor_thesis', 'diplomova praca': 'diploma_thesis',
|
||||
@ -148,6 +150,10 @@ def academic_windows(body: str) -> list[tuple[tuple[str, int | None] | None, str
|
||||
scope = (None if normalized(heading[1]).startswith(('priprava na ', 'priprava k ')) else
|
||||
detected or (heading_stack[-1][1] if heading_stack else None))
|
||||
heading_stack.append((level, scope))
|
||||
if scope is not None and detected is None:
|
||||
# Keep semantic child headings (Stav, Úlohy, Metódy) as
|
||||
# literal source text instead of losing their meaning.
|
||||
lines.append(line)
|
||||
else:
|
||||
lines.append(line)
|
||||
flush()
|
||||
@ -392,14 +398,16 @@ def rag_evidence_context(question: str, payload: dict) -> dict:
|
||||
facts.extend({**base, field: value} for value in values)
|
||||
facts = list({tuple(sorted(row.items())): row for row in facts}.values())
|
||||
requested = [person for person in people if text_contains_value(question, person, person=True)]
|
||||
requested=list(dict.fromkeys([*payload.get('named_subjects',[]),*requested]))
|
||||
expected_types = requested_types(question, requested)
|
||||
query = normalized(question)
|
||||
intent_query = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question))
|
||||
academic = bool(TYPE_PATTERN.search(query) or re.search(r'\b(?:rok\w* prac\w*|nazov\w* prac\w*)\b', query))
|
||||
content = bool(academic or (requested and re.search(r'\b(?:prac\w*|projekt\w*)\b', query))) and bool(re.search(
|
||||
content = bool(academic or requested) and bool(re.search(
|
||||
r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|co\s+ries\w*|obsah\w*|opis\w*|popis\w*|'
|
||||
r'zhrn\w*|zhrnut\w*|vysvetl\w*|zamer\w*|zadani\w*|ciel\w*|metod\w*|technolog\w*|'
|
||||
r'pouziv\w*|implement\w*|vysledk\w*|dosiahn\w*)\b', intent_query))
|
||||
r'pouziv\w*|implement\w*|vysledk\w*|dosiahn\w*|problem\w*|tazkost\w*|prekazk\w*|experiment\w*|merani\w*)\b', intent_query)
|
||||
or content_question(question))
|
||||
identity_first = re.search(r'^(?:aky|aka|ake|ktory|ktora|ktore|v ktorom|uved)'
|
||||
r'(?:\s+(?:je|su|bol|bola|boli|presny|presne)){0,3}\s+(?:nazov\w*|rok\w*)\b', intent_query)
|
||||
explicit_description = re.search(r'\b(?:comu|o com|opis\w*|popis\w*|zhrn\w*|obsah\w*|zadani\w*|ciel\w*)\b', intent_query)
|
||||
@ -419,18 +427,27 @@ def rag_evidence_context(question: str, payload: dict) -> dict:
|
||||
requirements['response_mode'] = 'mixed' if any(requirements[key] for key in ['title', 'year', 'start_year']) else 'quotes'
|
||||
requirements['distinguish_progress'] = bool(re.search(r'\b(?:zadani\w*|ciel\w*)\b', query)
|
||||
and re.search(r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b', query))
|
||||
if facet_mode(question):
|
||||
requirements['facets'] = requested_facets(question)
|
||||
stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else []
|
||||
requirements['years'] = ([int(value) for value in stated_years if len(value) == 4]
|
||||
if not requirements['start_year'] else [])
|
||||
requirements['years'] = requested_years(question) if academic and not requirements['start_year'] else []
|
||||
requirements['unsupported_year'] = any(len(value) != 4 for value in stated_years)
|
||||
if requested:
|
||||
facts = [row for row in facts if row['person'] in requested]
|
||||
context = {'question': question, **{key: payload.get(key) for key in ['status', 'reason', 'complete', 'truncated',
|
||||
requirements['available_fields'] = {person:[key for key in ['title','year','start_year']
|
||||
if any(row.get('person')==person and row.get(key) is not None and
|
||||
(key=='start_year' or ((not expected_types.get(person) or row.get('work_type')==expected_types[person])
|
||||
and (not requirements['years'] or row.get('year') in requirements['years']))) for row in facts)]
|
||||
for person in requested}
|
||||
context = {'question': question, 'clarification':payload.get('clarification'), **{key: payload.get(key) for key in ['status', 'reason', 'complete', 'truncated',
|
||||
'truncation_reasons', 'index_version']}, 'sources': payload.get('sources', []),
|
||||
'facts': facts, 'source_blocks': blocks, 'requirements': requirements,
|
||||
'aggregation': None, 'answer_candidates': []}
|
||||
if content:
|
||||
context['content_passages'] = academic_content_passages(blocks, facts, requirements, question)
|
||||
if requirements.get('facets'):
|
||||
from .content_selection import candidates
|
||||
context['content_candidates'] = candidates(context)
|
||||
if requirements['distinguish_progress']:
|
||||
requirements['content_roles'] = {person: sorted({section['kind']
|
||||
for p in context['content_passages'] if p['person'] == person
|
||||
|
||||
91
tools/answer_evaluation/rag_preparation.py
Normal file
91
tools/answer_evaluation/rag_preparation.py
Normal file
@ -0,0 +1,91 @@
|
||||
"""Resolve named source profiles and scope content before the shared token budget."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
|
||||
from .context_policy import normalized, person_pattern, text_contains_value
|
||||
from .question_intent import content_question
|
||||
from .rag_evidence import PROFILE, academic_scope, academic_windows, requested_types, WORK_LABELS
|
||||
from tools.retrieval.question_constraints import requested_years
|
||||
|
||||
|
||||
def clarify(payload: dict, reason: str, message: str) -> dict:
|
||||
return {**payload,'status':'ambiguous','reason':reason,'complete':False,
|
||||
'clarification':message}
|
||||
|
||||
|
||||
def prepare_named_content(db_file, question: str, payload: dict) -> dict:
|
||||
if not content_question(question):
|
||||
return payload
|
||||
if re.search(r'\b(?:s tagom|s klucovym slovom|z kategorie|s autorom dokumentu)\b',normalized(question)):
|
||||
return clarify(payload,'content_filter_requires_graph',
|
||||
'Najprv over podmienku tagu, kategórie alebo autora cez /graphrag. '
|
||||
'Potom môžem opísať obsah konkrétnej vyhovujúcej práce; túto podmienku nevynechám.')
|
||||
from tools.rag.rag_utils import build_context_text
|
||||
with closing(sqlite3.connect(db_file.resolve().as_uri()+'?mode=ro',uri=True)) as conn:
|
||||
conn.row_factory=sqlite3.Row
|
||||
if not {'documents','chunks'} <= {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}:
|
||||
return payload
|
||||
catalog=[dict(r) for r in conn.execute('SELECT path,title,author,published FROM documents WHERE COALESCE(published,1)=1')
|
||||
if PROFILE.fullmatch(r['path']) and r['title']]
|
||||
named=[r for r in catalog if text_contains_value(question,r['title'],person=True)]
|
||||
# Preserve question order rather than the original retrieval ranking.
|
||||
named.sort(key=lambda r:re.search(person_pattern(r['title']),normalized(question)).start())
|
||||
if len(named)>4:
|
||||
return clarify(payload,'too_many_subjects','Pre podrobné obsahové porovnanie vyber najviac štyri osoby naraz.')
|
||||
from tools.graphrag.graph_search import requested_name_phrases
|
||||
first_names={normalized(r['title']).split()[0] for r in catalog}
|
||||
capitalized={normalized(match[0]) for match in re.finditer(
|
||||
r'\b[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][^\W\d_]+(?:\s+[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][^\W\d_]+){1,3}\b',question)}
|
||||
unknown=[]
|
||||
for phrase in requested_name_phrases(question):
|
||||
words=phrase.split()
|
||||
named_companion=(named and phrase in capitalized and
|
||||
re.search(r'\ba\s+'+re.escape(phrase)+r'\b',normalized(question)))
|
||||
if (2<=len(words)<=4 and (words[0] in first_names or named_companion) and
|
||||
not any(text_contains_value(phrase,r['title'],person=True) for r in catalog)):
|
||||
unknown.append(phrase)
|
||||
if unknown:
|
||||
return clarify(payload,'unresolved_subject','Meno „'+unknown[0]+'“ sa nepodarilo spoľahlivo priradiť. '
|
||||
'Uveď jeho presné celé meno; ostatné osoby nebudem používať ako náhradu.')
|
||||
if not named:
|
||||
return payload
|
||||
types=requested_types(question,[r['title'] for r in named])
|
||||
years=requested_years(question)
|
||||
selected=[]
|
||||
for person in named:
|
||||
rows=list(conn.execute('SELECT text,heading_paths_json FROM chunks WHERE document_path=? '
|
||||
'AND COALESCE(published,1)=1 ORDER BY chunk_index',(person['path'],)))
|
||||
frames=[]; records={};seen=set()
|
||||
for row in rows:
|
||||
paths=json.loads(row['heading_paths_json'] or '[]')
|
||||
headings=[' > '.join(p) if isinstance(p,list) else str(p) for p in paths]
|
||||
first=headings[0] if headings else ''
|
||||
for scope,text in academic_windows('Sekcia: '+first+'\n'+str(row['text'] or '')):
|
||||
if not scope or (types.get(person['title']) and scope[0]!=types[person['title']]) or (years and scope[1] not in years):
|
||||
continue
|
||||
label=WORK_LABELS[scope[0]]+(f' {scope[1]}' if scope[1] is not None else '')
|
||||
records[scope]=label
|
||||
if (scope,text) not in seen:
|
||||
seen.add((scope,text));frames.append('Sekcia: '+label+'\n'+text)
|
||||
if len(records)>1:
|
||||
options='; '.join(dict.fromkeys(records.values()))
|
||||
return clarify(payload,'ambiguous_work',f'Ktorú prácu osoby {person["title"]} máš na mysli? '
|
||||
f'V indexe sú tieto možnosti: {options}. Uveď typ práce a rok.')
|
||||
source=next((s for s in payload.get('sources',[]) if s.get('document_path')==person['path']),None)
|
||||
if source is None:
|
||||
from tools.rag.search_core import make_source_url
|
||||
source={'title':person['title'],'document_path':person['path'],'source_url':make_source_url(person['path']),
|
||||
'author':person['author'],'published':person['published']}
|
||||
selected.append({**source,'text':'\n\n'.join(frames)})
|
||||
# Keep already-retrieved child documents of the requested profiles.
|
||||
for source in payload.get('sources',[]):
|
||||
if any(source.get('document_path','').startswith(p['path'].removesuffix('README.md'))
|
||||
and source.get('document_path')!=p['path'] for p in named):
|
||||
selected.append(source)
|
||||
selected=[{**source,'source_id':f'S{i}'} for i,source in enumerate(selected,1)]
|
||||
return {**payload,'sources':selected,'source_count':len(selected),'context':build_context_text(selected),
|
||||
'named_subjects':[p['title'] for p in named]}
|
||||
@ -54,6 +54,8 @@ def render_graph_answer(answer: dict, context: dict) -> dict:
|
||||
lines.append(f'Počet vyhovujúcich osôb: {out["count"]}.')
|
||||
if context.get('complete') is False or context.get('truncated'):
|
||||
lines.append('Výpis je neúplný.')
|
||||
if records:
|
||||
lines.append('Výpis môžeš zúžiť uvedením osoby, typu práce alebo konkrétneho roku.')
|
||||
elif context.get('complete') is None:
|
||||
lines.append('Úplnosť vyhľadania nemožno zaručiť.')
|
||||
elif out['status'] == 'ambiguous':
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Literal
|
||||
|
||||
@ -12,6 +13,7 @@ from .context_policy import answer_context_issues, citation_check, allowed_urls,
|
||||
from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections
|
||||
from .answer_coverage import answer_coverage, coverage_notices
|
||||
from .presentation import content_lines, fact_sentences, no_answer_text, work_subject
|
||||
from .content_selection import facet_lines, evidence_units, missing_facets, fit_candidates
|
||||
|
||||
|
||||
class RagClaim(Claim):
|
||||
@ -46,6 +48,8 @@ uveď start_year aj year a work_type. Citácia ani nesúvisiaci výňatok údaj
|
||||
Rešpektuj requirements.response_mode. Pri response_mode=quotes MUSÍ byť claims=[]:
|
||||
pri všeobecnej otázke, téme, technológii, zozname ľudí alebo stáži vyber iba
|
||||
evidence_quotes: source_id a presný doslovný výňatok, ktorý odpovedá na otázku.
|
||||
Pri výbere zachovaj negácie, podmienky a výhrady zdroja. Nevytrhni tvrdenie z vety,
|
||||
ktorá ho obmedzuje alebo spochybňuje.
|
||||
Nevypĺňaj pri nich názvy prác, roky, tagy ani iné nežiadané metadáta do claims.
|
||||
Pri requirements.content=true otázka žiada obsah práce, nie iba jej identifikáciu.
|
||||
Vyber opisujúce evidence_quotes z content_passages zodpovedajúcej osoby, typu a roku.
|
||||
@ -65,6 +69,11 @@ aplikácia uvedie, čo chýba. no_answer znamená žiadnu odpoveď a musí mať
|
||||
Pri zaznamenanom stave uprednostni konkrétnu implementáciu, natrénovanie modelu a vykonané
|
||||
vyhodnotenie pred konzultáciami alebo samotným písaním textu. Ani priebežný stav nevydávaj
|
||||
za splnenie všetkých cieľov alebo za konečné výsledky práce.
|
||||
Ak sú dodané content_candidates, vyber quotes iba z ich textu. Každý kandidát uvádza
|
||||
osobu, typ práce, rok, kind (assignment/task/progress/other) a facets, ktoré dokladá.
|
||||
Pokry každú dostupnú požadovanú časť requirements.facets za každú osobu. Pre chýbajúcu časť
|
||||
nevymýšľaj dôkaz: aplikácia ju výslovne označí. Čiastočná doložená odpoveď má status=ok.
|
||||
Pri výsledkoch použi skutočný zaznamenaný stav; úlohy a plánované experimenty nie sú výsledky.
|
||||
Každý výňatok musí byť JEDEN súvislý úsek z konkrétneho zdroja vrátane Markdownu.
|
||||
Nespájaj oddelené pasáže cez vynechané riadky; použi viac samostatných kratších quotes.
|
||||
Vyber iba minimum potrebné na zodpovedanie otázky. Výňatok nesmieš parafrázovať ani meniť čísla.
|
||||
@ -127,7 +136,8 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
||||
issues.append('missing_required_evidence')
|
||||
if requirements.get('response_mode') == 'quotes' and claims:
|
||||
issues.append('unrequested_structured_claims')
|
||||
if requirements.get('content') and answer['status'] == 'ok' and not quotes:
|
||||
if (requirements.get('content') and answer['status'] == 'ok' and not quotes
|
||||
and (not requirements.get('facets') or context.get('content_candidates'))):
|
||||
issues.append('missing_content_evidence')
|
||||
requested = requirements['people']
|
||||
types = requirements['work_types']
|
||||
@ -159,7 +169,12 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
||||
matching = [claim for claim in claims if claim['person'] == person]
|
||||
for key in ['title', 'year', 'start_year']:
|
||||
if requirements[key] and not any(claim.get(key) is not None for claim in matching):
|
||||
issues.append('missing_requested_' + key)
|
||||
if key in requirements.get('available_fields',{}).get(person,[key]):
|
||||
issues.append('missing_requested_' + key)
|
||||
wanted=[key for key in ['title','year','start_year'] if requirements[key]]
|
||||
if (wanted and not quotes and not any(key in fields for fields in requirements.get('available_fields',{}).values()
|
||||
for key in wanted)):
|
||||
issues.append('no_requested_fact_supported')
|
||||
if answer['status'] == 'no_answer' and (requirements['academic'] or requirements['start_year']) and requested:
|
||||
# Reject an unsupported refusal only if every requested academic tuple
|
||||
# is explicitly present. Never invent a value to fill an absent field.
|
||||
@ -174,6 +189,10 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
||||
and (not requirements['start_year'] or start_supported) and not requirements['unsupported_year']
|
||||
and (not requirements.get('content') or context.get('content_passages'))):
|
||||
issues.append('rejected_supported_academic_answer')
|
||||
if (not requirements.get('content') and not requirements['unsupported_year'] and any(
|
||||
requirements[key] and key in fields for fields in requirements.get('available_fields',{}).values()
|
||||
for key in ['title','year','start_year'])):
|
||||
issues.append('rejected_supported_partial_fact')
|
||||
if answer['status'] == 'no_answer' and requirements.get('content'):
|
||||
query = normalized(context['question'])
|
||||
overview = re.search(r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|opis\w*|popis\w*|obsah\w*|zhrn\w*)\b', query)
|
||||
@ -193,6 +212,10 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
||||
elif source.get('source_url') not in cited:
|
||||
issues.append('missing_quote_source')
|
||||
if source:
|
||||
if context.get('candidate_only') and not any(c['source_id']==quote['source_id'] and
|
||||
' '.join(quote['quote'].split()) in ' '.join(c['text'].split())
|
||||
for c in context.get('content_candidates', [])):
|
||||
issues.append('quote_outside_delivered_candidates')
|
||||
if requirements.get('content'):
|
||||
titles = [row.get(key) for row in facts if row.get('source_id') == quote['source_id']
|
||||
for key in ('title', 'person', 'document_author')]
|
||||
@ -218,15 +241,16 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
||||
if requirements.get('content') and answer['status'] == 'ok':
|
||||
for person in requested:
|
||||
passages = [p for p in context.get('content_passages', []) if p.get('person') == person]
|
||||
if requirements['response_mode'] == 'mixed':
|
||||
if requirements['response_mode'] == 'mixed' and any(claim.get('person')==person for claim in claims):
|
||||
passages = [p for p in passages if any(claim.get('person') == person and
|
||||
all(claim.get(key) is None or claim[key] == p.get(key) for key in ['work_type', 'year'])
|
||||
for claim in claims)]
|
||||
if not any(quote['source_id'] == p['source_id'] and
|
||||
eligible = any(c.get('person')==person for c in context.get('content_candidates', []))
|
||||
if (not requirements.get('facets') or eligible) and not any(quote['source_id'] == p['source_id'] and
|
||||
' '.join(quote['quote'].split()) in ' '.join(p['text'].split())
|
||||
for p in passages for quote in quotes):
|
||||
issues.append('missing_bound_content_evidence')
|
||||
if requirements.get('distinguish_progress'):
|
||||
if requirements.get('distinguish_progress') and not requirements.get('facets'):
|
||||
bound_quotes = [quote for quote in quotes if any(
|
||||
quote['source_id'] == p['source_id'] and
|
||||
' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) for p in passages)]
|
||||
@ -235,7 +259,15 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
||||
for kind in requirements.get('content_roles', {}).get(person, []):
|
||||
if kind not in kinds:
|
||||
issues.append('missing_' + kind + '_evidence')
|
||||
if (requirements.get('distinguish_progress') and answer['status'] == 'no_answer' and requested
|
||||
if requirements.get('facets'):
|
||||
available={(c['person'],facet) for c in context.get('content_candidates',[]) for facet in c['facets']}
|
||||
if answer['status']=='ok':
|
||||
for person,facet in missing_facets(answer,context):
|
||||
if (person,facet) in available:
|
||||
issues.append('missing_'+('progress' if facet=='results' else facet)+'_evidence')
|
||||
elif answer['status']=='no_answer' and available:
|
||||
issues.append('rejected_supported_content_comparison')
|
||||
if (requirements.get('distinguish_progress') and not requirements.get('facets') and answer['status'] == 'no_answer' and requested
|
||||
and all(any(p.get('person') == person for p in context.get('content_passages', [])) for person in requested)):
|
||||
issues.append('rejected_supported_content_comparison')
|
||||
if not citation_check(answer, allowed_urls(context))['citations_ok']:
|
||||
@ -333,11 +365,29 @@ def comparison_lines(answer: dict, context: dict) -> list[str]:
|
||||
def render_rag_answer(answer: dict, context: dict) -> str:
|
||||
lines = []
|
||||
if answer['status'] == 'ok':
|
||||
req=context.get('requirements',{})
|
||||
for claim in answer.get('claims', []):
|
||||
lines.append(fact_sentences(claim, context.get('question', '')))
|
||||
if req.get('facets'):
|
||||
# Facet headings already identify the scoped work/year. Extra
|
||||
# valid metadata selected by the model is not part of the answer.
|
||||
fields=[key for key in ['title','year','start_year'] if req.get(key)]
|
||||
if not req.get('title') and not req.get('start_year') and req.get('years'):
|
||||
continue
|
||||
displayed={key:value for key,value in claim.items() if key in ['person','work_type',*fields]}
|
||||
if not any(displayed.get(key) is not None for key in fields):
|
||||
continue
|
||||
lines.append(fact_sentences(displayed,context.get('question','')))
|
||||
else:
|
||||
lines.append(fact_sentences(claim, context.get('question', '')))
|
||||
for person in req.get('people',[]):
|
||||
for key,label in [('title','Názov práce'),('year','Rok práce'),('start_year','Začiatok štúdia')]:
|
||||
if req.get(key) and key not in req.get('available_fields',{}).get(person,[key]):
|
||||
lines.append(f'{person} — {label.lower()} nie je v poskytnutých podkladoch doložený.')
|
||||
quotes = answer.get('evidence_quotes', [])
|
||||
style = context.get('requirements', {}).get('content_style')
|
||||
if quotes and context.get('requirements', {}).get('distinguish_progress'):
|
||||
if context.get('requirements', {}).get('facets'):
|
||||
lines.extend(facet_lines(answer,context))
|
||||
elif quotes and context.get('requirements', {}).get('distinguish_progress'):
|
||||
lines.extend(comparison_lines(answer, context))
|
||||
elif quotes and style in {'overview', 'assignment'}:
|
||||
# Bind each paragraph to the same source/person/work as the checked
|
||||
@ -371,7 +421,10 @@ def render_rag_answer(answer: dict, context: dict) -> str:
|
||||
if answer.get('count') is not None:
|
||||
lines.append(f'Počet: {answer["count"]}.')
|
||||
elif answer['status'] == 'no_answer':
|
||||
lines.append(no_answer_text(context))
|
||||
if context.get('requirements', {}).get('facets') and context['requirements']['people']:
|
||||
lines.extend(facet_lines(answer,context))
|
||||
else:
|
||||
lines.append(no_answer_text(context))
|
||||
elif answer['status'] == 'ambiguous':
|
||||
lines.append('Údaje sú nejednoznačné; požiadavku treba spresniť.')
|
||||
else:
|
||||
@ -384,6 +437,18 @@ def render_rag_answer(answer: dict, context: dict) -> str:
|
||||
|
||||
def generate_verified_rag(question: str, payload: dict, complete) -> dict:
|
||||
context = rag_evidence_context(question, payload)
|
||||
if payload.get('status')=='ambiguous' or context.get('clarification'):
|
||||
return {'status':'ambiguous','reason':payload.get('reason') or 'ambiguous_request',
|
||||
'accepted_for_delivery':True,'answer':context.get('clarification') or 'Spresni osobu, typ práce alebo rok.',
|
||||
'claims':[],'source_urls':[],'attempts':[],'complete':False,'truncated':False,
|
||||
'truncation_reasons':[],'index_version':context.get('index_version')}
|
||||
if (context['requirements'].get('facets') and context['requirements']['response_mode']=='quotes'
|
||||
and not context.get('content_candidates') and context['requirements']['people']):
|
||||
empty={'status':'no_answer','claims':[],'evidence_quotes':[],
|
||||
'source_urls':[s['source_url'] for s in context['sources'] if s.get('source_url')]}
|
||||
return {**empty,'reason':'requested_aspects_not_found','answer':render_rag_answer(empty,context),
|
||||
'accepted_for_delivery':True,'attempts':[],'answer_coverage':answer_coverage(empty,context),
|
||||
**{k:context.get(k) for k in ['index_version','complete','truncated','truncation_reasons']}}
|
||||
if not payload.get('sources'):
|
||||
status = 'error' if payload.get('status') == 'error' else 'no_answer'
|
||||
empty = {'status': status, 'claims': [], 'evidence_quotes': [], 'source_urls': []}
|
||||
@ -395,11 +460,18 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict:
|
||||
sent = {key: context[key] for key in ['status', 'reason', 'complete', 'truncated',
|
||||
'truncation_reasons', 'index_version', 'sources', 'facts', 'requirements']}
|
||||
if context['requirements'].get('content'):
|
||||
sent['content_passages'] = context['content_passages']
|
||||
if context['requirements'].get('facets'):
|
||||
sent['content_candidates']=context['content_candidates']
|
||||
context['candidate_only']=True
|
||||
else:
|
||||
sent['content_passages'] = context['content_passages']
|
||||
elif not context['requirements']['academic']:
|
||||
sent['context'] = payload.get('context', '')
|
||||
if context['requirements']['response_mode'] == 'quotes':
|
||||
sent.pop('facts')
|
||||
if context['requirements'].get('facets'):
|
||||
fit_candidates(sent,context,payload.get('context_budget',{}).get('max_tokens',12000),
|
||||
os.getenv('RAG_LLM_TOKEN_ENCODING','cl100k_base'))
|
||||
messages = [{'role': 'system', 'content': PROMPT + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)},
|
||||
{'role': 'user', 'content': json.dumps({'question': question, 'context': sent}, ensure_ascii=False)}]
|
||||
attempts = []
|
||||
@ -420,6 +492,12 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict:
|
||||
'answer': 'Jazykový model nie je dostupný; overenú odpoveď sa nepodarilo pripraviť.', 'attempts': attempts}
|
||||
record['issues'] = issues
|
||||
if not issues:
|
||||
if context['requirements'].get('facets') and missing_facets(answer,context):
|
||||
# Sources inspected for a missing requested aspect remain
|
||||
# traceable, even when only the other person's quotes exist.
|
||||
extra=[s['source_url'] for s in context['sources'] if s.get('title') in
|
||||
{person for person,_ in missing_facets(answer,context)} and s.get('source_url')]
|
||||
answer={**answer,'source_urls':list(dict.fromkeys([*answer['source_urls'],*extra]))}
|
||||
text = render_rag_answer(answer, context)
|
||||
return {'status': answer['status'], 'reason': None, 'accepted_for_delivery': True,
|
||||
'answer': text, 'source_urls': answer['source_urls'], 'claims': answer['claims'],
|
||||
|
||||
@ -12,12 +12,20 @@ from tools.answer_evaluation.graph_context import graph_context
|
||||
from tools.answer_evaluation.checked_generation import generate_checked_answer
|
||||
from tools.answer_evaluation.verified_rag import generate_verified_rag
|
||||
from tools.answer_evaluation.rag_evidence import expand_comparison_sources
|
||||
from tools.answer_evaluation.rag_preparation import prepare_named_content
|
||||
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
|
||||
from tools.sync.index_state import index_snapshot
|
||||
from tools.answers.model_client import complete_with_configured_model
|
||||
from tools.answers.answer_errors import public_failure
|
||||
|
||||
_capacity = threading.BoundedSemaphore(2)
|
||||
|
||||
|
||||
def retrieval_failure(engine: str) -> dict:
|
||||
return public_failure({'reason':'retrieval_unavailable','engine':engine,'index_version':None,
|
||||
'complete':False,'truncated':False,'truncation_reasons':[]})
|
||||
|
||||
|
||||
def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
|
||||
complete=None, db_file: Path = DB_FILE, include_diagnostics: bool = False) -> dict:
|
||||
if not _capacity.acquire(blocking=False):
|
||||
@ -25,15 +33,32 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
|
||||
'accepted_for_delivery': False, 'answer': 'Agent práve spracúva iné odpovede. Skúste to o chvíľu.'}
|
||||
try:
|
||||
model = complete or complete_with_configured_model
|
||||
unsupported=unsupported_constraint(question)
|
||||
if unsupported:
|
||||
return {'status':'ambiguous','reason':'unsupported_constraint','accepted_for_delivery':True,
|
||||
'answer':constraint_message(unsupported),'claims':[],'source_urls':[],
|
||||
'index_version':None,'complete':False,'truncated':False,'truncation_reasons':[], 'engine':engine}
|
||||
if engine == 'rag':
|
||||
with index_snapshot(db_file.parent, db_file=db_file):
|
||||
retrieved = build_rag_context(db_file, question, limit=limit)
|
||||
retrieved = expand_comparison_sources(db_file, question, retrieved)
|
||||
retrieved = compact_rag_payload(retrieved)
|
||||
try:
|
||||
with index_snapshot(db_file.parent, db_file=db_file):
|
||||
retrieved = build_rag_context(db_file, question, limit=limit)
|
||||
retrieved = expand_comparison_sources(db_file, question, retrieved)
|
||||
retrieved = prepare_named_content(db_file, question, retrieved)
|
||||
retrieved = compact_rag_payload(retrieved)
|
||||
except Exception:
|
||||
return retrieval_failure(engine)
|
||||
result = generate_verified_rag(question, retrieved, model)
|
||||
context = retrieved
|
||||
elif engine == 'graphrag':
|
||||
context = graph_context(search_graph(question))
|
||||
try:
|
||||
context = graph_context(search_graph(question))
|
||||
except Exception:
|
||||
return retrieval_failure(engine)
|
||||
if context.get('clarification'):
|
||||
return {'status':'ambiguous','reason':context.get('reason'),'accepted_for_delivery':True,
|
||||
'answer':context['clarification'],'claims':[],'source_urls':[], 'engine':engine,
|
||||
'complete':False,'truncated':False,'truncation_reasons':[],
|
||||
'index_version':context.get('index_version')}
|
||||
prompt = (Path(__file__).resolve().parents[1] / 'answer_evaluation/graph_answer_prompt.txt').read_text()
|
||||
generated = generate_checked_answer(question, context, prompt, model)
|
||||
if generated.get('accepted_for_delivery'):
|
||||
@ -42,8 +67,10 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
|
||||
'truncated': context.get('truncated'), 'truncation_reasons': context.get('truncation_reasons'),
|
||||
'attempts': generated['attempts']}
|
||||
else:
|
||||
result = {'status': 'error', 'reason': 'answer_rejected', 'accepted_for_delivery': False,
|
||||
'answer': 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.',
|
||||
reason=generated.get('reason','answer_rejected')
|
||||
result = {'status': 'error', 'reason': reason, 'accepted_for_delivery': False,
|
||||
'answer': ('Jazykový model nie je dostupný. Skús otázku neskôr.' if reason=='model_unavailable' else
|
||||
'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.'),
|
||||
'index_version': context.get('index_version'), 'attempts': generated['attempts']}
|
||||
else:
|
||||
raise ValueError('Unsupported answer engine')
|
||||
|
||||
18
tools/answers/answer_errors.py
Normal file
18
tools/answers/answer_errors.py
Normal file
@ -0,0 +1,18 @@
|
||||
"""Public failure wording; transport/model exception text is never a UI message."""
|
||||
MESSAGES={
|
||||
'model_unavailable':'Jazykový model momentálne nie je dostupný. Skús otázku neskôr.',
|
||||
'retrieval_unavailable':'Vyhľadávací index momentálne nie je dostupný. Skús otázku neskôr.',
|
||||
'answer_capacity_exceeded':'Agent práve spracúva iné otázky. Skús to o chvíľu.',
|
||||
'answer_rejected':'Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. '
|
||||
'Skús spresniť osobu, typ práce alebo rok; neoverené údaje sa nezobrazia.',
|
||||
'answer_unavailable':'Overenú odpoveď sa momentálne nepodarilo pripraviť. Skús to neskôr.',
|
||||
}
|
||||
|
||||
|
||||
def public_failure(result: dict) -> dict:
|
||||
reason=result.get('reason')
|
||||
if not isinstance(reason,str) or reason not in MESSAGES:
|
||||
reason='answer_unavailable'
|
||||
return {'index_version':None,'complete':False,'truncated':False,'truncation_reasons':[],
|
||||
**result,'status':'error','reason':reason,'accepted_for_delivery':False,
|
||||
'answer':MESSAGES[reason],'claims':[],'source_urls':[]}
|
||||
@ -46,6 +46,8 @@ def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]:
|
||||
'tools/answer_evaluation/render_answer.py', 'tools/answer_evaluation/graph_context.py',
|
||||
'tools/answer_evaluation/presentation.py',
|
||||
'tools/answer_evaluation/rag_evidence.py',
|
||||
'tools/answer_evaluation/question_intent.py', 'tools/answer_evaluation/content_selection.py',
|
||||
'tools/answer_evaluation/rag_preparation.py', 'tools/answer_evaluation/answer_coverage.py',
|
||||
'tools/answer_evaluation/graph_answer_prompt.txt',
|
||||
'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py'])
|
||||
files = sorted(set(files))
|
||||
|
||||
@ -8,6 +8,7 @@ from typing import Any, Iterable
|
||||
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION
|
||||
from .publication import public_query
|
||||
from tools.retrieval.retrieval_contract import result_fields, source_references, error_result
|
||||
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
|
||||
|
||||
|
||||
GRAPH_SCOPE = "zpwiki"
|
||||
@ -2334,6 +2335,12 @@ def search_graph_with_session(
|
||||
raise ValueError("max_entities musí byť aspoň 1")
|
||||
|
||||
constraints = extract_query_constraints(clean_query)
|
||||
unsupported = unsupported_constraint(clean_query)
|
||||
if unsupported:
|
||||
return {**result_fields(status='ambiguous',sources=[],reason='unsupported_constraint',complete=False),
|
||||
'query':clean_query,'constraints':constraints,'clarification':constraint_message(unsupported),
|
||||
'facts':[],'paths':[],'documents':[],'aggregation':None,'resolved_entities':[],
|
||||
'checked_entities':[],'rejection_evidence':[],'strategies':[],'reasons':['unsupported_constraint']}
|
||||
catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE, published_only=published_only)
|
||||
constraints['published_only'] = published_only
|
||||
# Explicit tag filters are resolved separately. Incidental names such as the
|
||||
|
||||
@ -11,7 +11,7 @@ from tools.retrieval.text_budget import split_to_fit
|
||||
SOURCE_FIELDS = ('source_id', 'title', 'author', 'document_path', 'source_url', 'published',
|
||||
'categories', 'tags', 'section')
|
||||
FIELDS = ('status', 'reason', 'query', 'engine', 'strategies', 'instructions', 'answer_format',
|
||||
'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count')
|
||||
'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count', 'clarification', 'named_subjects')
|
||||
|
||||
|
||||
def compact_rag_payload(data: dict, *, max_tokens: int | None = None,
|
||||
|
||||
@ -709,6 +709,13 @@ def build_rag_context(
|
||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||
max_per_document: int = 1,
|
||||
) -> dict[str, Any]:
|
||||
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
|
||||
unsupported=unsupported_constraint(query)
|
||||
if unsupported:
|
||||
return {**result_fields(status='ambiguous',sources=[],reason='unsupported_constraint',complete=False),
|
||||
'query':query,'engine':'hybrid','strategies':[],'source_count':0,'sources':[],
|
||||
'context':'','clarification':constraint_message(unsupported),
|
||||
'instructions':RAG_INSTRUCTIONS,'answer_format':ANSWER_FORMAT}
|
||||
response = search_database(
|
||||
db_path,
|
||||
query,
|
||||
|
||||
42
tools/retrieval/question_constraints.py
Normal file
42
tools/retrieval/question_constraints.py
Normal file
@ -0,0 +1,42 @@
|
||||
"""Fail closed for recognized query operators without an implemented meaning."""
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
|
||||
def requested_years(question: str) -> list[int]:
|
||||
"""An unquoted four-digit year remains a condition without 'v roku'."""
|
||||
text=re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question)
|
||||
return list(dict.fromkeys(int(value) for value in re.findall(r'\b\d{4}\b',text)))
|
||||
|
||||
|
||||
def unsupported_constraint(question: str) -> str | None:
|
||||
text = ''.join(c for c in unicodedata.normalize('NFKD', question).casefold()
|
||||
if not unicodedata.combining(c))
|
||||
# Do not treat words inside quoted academic titles as query operators.
|
||||
text = re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', text)
|
||||
if (re.search(r'\b(?:rok\w*|datum\w*|kedy)\b',text) and
|
||||
re.search(r'\b(?:obhajob\w*|obhajil\w*|dokonc\w*|ukoncil\w*|zacal\w* pracovat)\b',text) and
|
||||
re.search(r'\b(?:prac\w*|projekt\w*)\b',text)):
|
||||
return 'work_event_date'
|
||||
if re.search(r'\b(?:od|do|pred|po|medzi)\s+(?:rok(?:om|mi|u)?\s+)?\d{4}\b|\b\d{4}\s*[-–]\s*\d{4}\b', text):
|
||||
return 'year_range'
|
||||
if re.search(r'\b(?:aspon|najviac|viac ako|menej ako|minimalne|maximalne)\s+\d+', text):
|
||||
return 'quantity_condition'
|
||||
if re.search(r'\b(?:okrem|s vynimkou)\b|\b(?:nema|nemaju)\s+(?:tag|klucov|kategori|prac|projekt)|'
|
||||
r'\bbez\s+(?:tagu|klucoveho slova|kategorie)\b', text):
|
||||
return 'exclusion'
|
||||
if re.search(r'\b(?:tag\w*|klucov\w*\s+slov\w*)\s+\w+\s+alebo\s+\w+', text):
|
||||
return 'alternative_tags'
|
||||
return None
|
||||
|
||||
|
||||
def constraint_message(code: str) -> str:
|
||||
return {
|
||||
'work_event_date':'Rok uvedený pri práci nepreukazuje dátum jej začatia, dokončenia alebo obhajoby. '
|
||||
'Chceš zistiť rok evidovaný pri práci?',
|
||||
'year_range': 'Časovú podmienku neviem spoľahlivo vyhodnotiť ako interval. Uveď konkrétny rok práce.',
|
||||
'quantity_condition': 'Podmienku na minimálny alebo maximálny počet neviem spoľahlivo vyhodnotiť. '
|
||||
'Požiadaj najprv o konkrétny zoznam alebo počet.',
|
||||
'exclusion': 'Vylučovaciu podmienku neviem spoľahlivo vyhodnotiť. Uveď, ktoré vlastnosti majú výsledky spĺňať.',
|
||||
'alternative_tags': 'Uveď, či požaduješ oba tagy zároveň, alebo vyhľadávanie pre každý tag samostatne.',
|
||||
}[code]
|
||||
Loading…
Reference in New Issue
Block a user