Kvalitnejšie odpovede
This commit is contained in:
parent
59f208b571
commit
16af0f8b4f
18
README.md
18
README.md
@ -1160,7 +1160,7 @@ upozornenie na neúplnosť, nezamieňať autora dokumentu s autorom práce a pri
|
|||||||
sa nevysvetľujú odhadom; verzia dát neoznačuje verziu programu.
|
sa nevysvetľujú odhadom; verzia dát neoznačuje verziu programu.
|
||||||
Tieto pokyny ovplyvňujú správanie Claude, jeho finálny text však náš backend
|
Tieto pokyny ovplyvňujú správanie Claude, jeho finálny text však náš backend
|
||||||
nemôže programovo vynútiť. Po aktualizácii už nahratých Skills nahraď ich
|
nemôže programovo vynútiť. Po aktualizácii už nahratých Skills nahraď ich
|
||||||
novými ZIPmi z `packages/claude/dist/` (verzia 1.2.0) a over správanie v novom chate.
|
novými ZIPmi z `packages/claude/dist/` (verzia 1.3.0) a over správanie v novom chate.
|
||||||
|
|
||||||
### Nahratie Skills do webového Claude
|
### Nahratie Skills do webového Claude
|
||||||
|
|
||||||
@ -1507,6 +1507,22 @@ Chýbajúce výsledky sa oznámia, neodvodzujú sa z cieľov. Grafová otázka n
|
|||||||
v ktorých majú práce „obaja“, používa prienik rokov a prikladá doložené typy a názvy
|
v ktorých majú práce „obaja“, používa prienik rokov a prikladá doložené typy a názvy
|
||||||
prác z týchto rokov; nezobrazuje namiesto odpovede všetky individuálne roky.
|
prác z týchto rokov; nezobrazuje namiesto odpovede všetky individuálne roky.
|
||||||
|
|
||||||
|
`rag_preparation.py` vie pomenované profily dohľadať v SQLite aj mimo prvých
|
||||||
|
výsledkov a pri viacerých možných prácach vyžiada typ alebo rok. Obsahové
|
||||||
|
porovnanie je obmedzené na štyri osoby naraz. `question_intent.py` rozlišuje
|
||||||
|
požadované problémy, metódy, experimenty, ciele a výsledky. `content_selection.py`
|
||||||
|
vyberá doslovné, typovo a osobne viazané podklady, odstraňuje opakovania a
|
||||||
|
uprednostňuje vecné výsledky pred administratívnymi poznámkami. Model dostane
|
||||||
|
len tieto kandidátske výňatky; ich skutočný JSON podlieha tokenovému rozpočtu.
|
||||||
|
Chýbajúce časti sa označia jednotlivo, aj keď sú ostatné časti zodpovedateľné.
|
||||||
|
Pri limite kontextu sa nestratí informácia, ktoré osoby používateľ požadoval.
|
||||||
|
|
||||||
|
Rozpoznané, ale nepodporované operátory (napríklad interval rokov, vylučovanie
|
||||||
|
alebo minimálny počet) vedú k `ambiguous` a konkrétnemu spresneniu. Rok práce
|
||||||
|
sa nepoužije ako dôkaz dátumu jej začatia či obhajoby. Chybové odpovede odlišujú
|
||||||
|
nedostupné vyhľadávanie, nedostupný model, vyťaženie a zamietnutý neoverený text;
|
||||||
|
raw výstupy odmietnutého modelu sa štandardnému klientovi neposkytujú.
|
||||||
|
|
||||||
Otázky na obsah, zadanie, ciele alebo metódy práce používajú textové výňatky
|
Otázky na obsah, zadanie, ciele alebo metódy práce používajú textové výňatky
|
||||||
z príslušnej akademickej sekcie, nie iba metadáta o názve a roku. Výňatok musí
|
z príslušnej akademickej sekcie, nie iba metadáta o názve a roku. Výňatok musí
|
||||||
patriť požadovanej osobe, typu práce a prípadne roku. Názov, rok ani samotný
|
patriť požadovanej osobe, typu práce a prípadne roku. Názov, rok ani samotný
|
||||||
|
|||||||
@ -49,6 +49,7 @@ from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY
|
|||||||
from tools.runtime.tool_health import tools_readiness
|
from tools.runtime.tool_health import tools_readiness
|
||||||
from tools.retrieval.retrieval_contract import RetrievalResponse, RetrievalSource, RetrievalErrorResponse, error_reason
|
from tools.retrieval.retrieval_contract import RetrievalResponse, RetrievalSource, RetrievalErrorResponse, error_reason
|
||||||
from tools.answers.answer_delivery import deliver_answer
|
from tools.answers.answer_delivery import deliver_answer
|
||||||
|
from tools.answers.answer_errors import public_failure
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@ -192,9 +193,7 @@ def verified_answer(request: AnswerRequest) -> dict:
|
|||||||
'answer': 'Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.'}
|
'answer': 'Overenú odpoveď sa nepodarilo pripraviť. Skúste to neskôr.'}
|
||||||
if (result.get('accepted_for_delivery') is not True or
|
if (result.get('accepted_for_delivery') is not True or
|
||||||
result.get('status') not in {'ok', 'no_answer', 'ambiguous'}):
|
result.get('status') not in {'ok', 'no_answer', 'ambiguous'}):
|
||||||
result = {**result, 'status': 'error', 'accepted_for_delivery': False,
|
result = public_failure(result)
|
||||||
'answer': 'Odpoveď neprešla overením podkladov. Neoverené údaje sa nezobrazia.',
|
|
||||||
'claims': [], 'source_urls': []}
|
|
||||||
if not request.include_diagnostics:
|
if not request.include_diagnostics:
|
||||||
result.pop('attempts', None)
|
result.pop('attempts', None)
|
||||||
result.pop('retrieval_context', None)
|
result.pop('retrieval_context', None)
|
||||||
|
|||||||
@ -81,8 +81,22 @@ class Config:
|
|||||||
|
|
||||||
|
|
||||||
def error_result(reason):
|
def error_result(reason):
|
||||||
|
messages={
|
||||||
|
'unauthorized':'Prístup nebol autorizovaný. Skontroluj prihlásenie konektora a jeho konfiguráciu.',
|
||||||
|
'forbidden':'Pre tento nástroj nemáš potrebné oprávnenie.',
|
||||||
|
'invalid_query':'Otázku sa nepodarilo spracovať. Zadaj konkrétnu otázku a primeraný počet výsledkov.',
|
||||||
|
'model_unavailable':'Jazykový model momentálne nie je dostupný. Skús otázku neskôr.',
|
||||||
|
'answer_rejected':'Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. '
|
||||||
|
'Spresni osobu, typ práce alebo rok; neoverený text sa nezobrazí.',
|
||||||
|
'answer_capacity_exceeded':'Agent práve spracúva iné otázky. Skús to o chvíľu.',
|
||||||
|
'backend_busy':'Agent práve spracúva iné otázky. Skús to o chvíľu.',
|
||||||
|
'backend_timeout':'Vyhľadávanie neodpovedalo včas. Skús otázku zopakovať neskôr.',
|
||||||
|
'backend_unavailable':'Spojenie s backendom momentálne nie je dostupné. Skontroluj, či bežia služby a tunel.',
|
||||||
|
'retrieval_unavailable':'Vyhľadávací index momentálne nie je dostupný. Skús otázku neskôr.',
|
||||||
|
}
|
||||||
return {'status': 'error', 'reason': reason, 'accepted_for_delivery': False,
|
return {'status': 'error', 'reason': reason, 'accepted_for_delivery': False,
|
||||||
'answer': '', 'sources': [], 'index_version': None, 'complete': False,
|
'answer': messages.get(reason,'Overenú odpoveď sa momentálne nepodarilo získať.'),
|
||||||
|
'sources': [], 'index_version': None, 'complete': False,
|
||||||
'truncated': False, 'truncation_reasons': []}
|
'truncated': False, 'truncation_reasons': []}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
4
packages/claude/dist/packages.json
vendored
4
packages/claude/dist/packages.json
vendored
@ -3,12 +3,12 @@
|
|||||||
{
|
{
|
||||||
"archive": "zpwiki-rag.zip",
|
"archive": "zpwiki-rag.zip",
|
||||||
"name": "rag",
|
"name": "rag",
|
||||||
"sha256": "8f0c51e27dc68a1e3ee845f2cfe5987e51714d4408f98c7d275861dc1e22953f"
|
"sha256": "09d3e6b196b61f6c025eb86ccd1773b913b37660163a995ad1a2acfa6d947a0c"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"archive": "zpwiki-graphrag.zip",
|
"archive": "zpwiki-graphrag.zip",
|
||||||
"name": "graphrag",
|
"name": "graphrag",
|
||||||
"sha256": "aa65a97211f48fa0352a98e50c6ca6a243e1d23105c0ff144c7f360ad192c376"
|
"sha256": "cfcb60213e103d40c2adbc6c7d4133751120148415836cfffa7935c020167e95"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"schema_version": 1
|
"schema_version": 1
|
||||||
|
|||||||
BIN
packages/claude/dist/zpwiki-graphrag.zip
vendored
BIN
packages/claude/dist/zpwiki-graphrag.zip
vendored
Binary file not shown.
BIN
packages/claude/dist/zpwiki-rag.zip
vendored
BIN
packages/claude/dist/zpwiki-rag.zip
vendored
Binary file not shown.
@ -11,7 +11,7 @@ import zipfile
|
|||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
BASE = ROOT / 'packages/claude'
|
BASE = ROOT / 'packages/claude'
|
||||||
ENGINES = ('rag', 'graphrag')
|
ENGINES = ('rag', 'graphrag')
|
||||||
PACKAGE_VERSION = '1.2.0'
|
PACKAGE_VERSION = '1.3.0'
|
||||||
|
|
||||||
|
|
||||||
def json_bytes(value):
|
def json_bytes(value):
|
||||||
|
|||||||
@ -156,7 +156,7 @@ def test_invalid_answer_coverage_is_not_delivered(backend, coverage):
|
|||||||
def test_transport_errors_and_redirects_do_not_disclose_payload(backend, code, reason):
|
def test_transport_errors_and_redirects_do_not_disclose_payload(backend, code, reason):
|
||||||
backend.update(raw=b'SECRET ERROR', code=code)
|
backend.update(raw=b'SECRET ERROR', code=code)
|
||||||
value = client.query_backend('Práca?', 'rag')
|
value = client.query_backend('Práca?', 'rag')
|
||||||
assert value['reason'] == reason and value['answer'] == ''
|
assert value['reason'] == reason and value['answer']
|
||||||
assert len(backend['requests']) == 1 and 'SECRET' not in json.dumps(value)
|
assert len(backend['requests']) == 1 and 'SECRET' not in json.dumps(value)
|
||||||
|
|
||||||
|
|
||||||
@ -168,7 +168,7 @@ def test_rejected_or_malformed_answer_is_never_delivered(backend, change):
|
|||||||
backend.update(raw=json.dumps({**backend['checked'], **change}).encode())
|
backend.update(raw=json.dumps({**backend['checked'], **change}).encode())
|
||||||
value = client.query_backend('Práca?', 'rag')
|
value = client.query_backend('Práca?', 'rag')
|
||||||
assert value['status'] == 'error' and not value['accepted_for_delivery']
|
assert value['status'] == 'error' and not value['accepted_for_delivery']
|
||||||
assert value['answer'] == '' and 'Eva' not in json.dumps(value)
|
assert value['answer'] and 'Eva' not in json.dumps(value)
|
||||||
|
|
||||||
|
|
||||||
def test_invalid_json_and_oversized_response(backend):
|
def test_invalid_json_and_oversized_response(backend):
|
||||||
@ -279,7 +279,7 @@ def test_mcp_stdio_initialization_discovery_call_and_rejection(backend):
|
|||||||
assert all('presentation_instructions' not in json.loads(block.text)
|
assert all('presentation_instructions' not in json.loads(block.text)
|
||||||
for block in result.content if block.type == 'text')
|
for block in result.content if block.type == 'text')
|
||||||
rejected = await session.call_tool('zpwiki_rag', {'query': ''})
|
rejected = await session.call_tool('zpwiki_rag', {'query': ''})
|
||||||
assert rejected.isError and rejected.structuredContent['answer'] == ''
|
assert rejected.isError and rejected.structuredContent['answer']
|
||||||
asyncio.run(run())
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -103,8 +103,11 @@ def comparison_payload(tmp_path):
|
|||||||
|
|
||||||
|
|
||||||
def selected_comparison(context):
|
def selected_comparison(context):
|
||||||
quotes=[{'source_id':p['source_id'],'quote':s['text']} for p in context['content_passages']
|
quotes=([{'source_id':c['source_id'],'quote':c['text']} for c in context['content_candidates']]
|
||||||
|
if 'content_candidates' in context else
|
||||||
|
[{'source_id':p['source_id'],'quote':s['text']} for p in context['content_passages']
|
||||||
for s in content_sections(p['text']) if s['kind'] in {'assignment','progress'}]
|
for s in content_sections(p['text']) if s['kind'] in {'assignment','progress'}]
|
||||||
|
)
|
||||||
return {'status':'ok','answer':'','claims':[],'evidence_quotes':quotes,
|
return {'status':'ok','answer':'','claims':[],'evidence_quotes':quotes,
|
||||||
'source_urls':[s['source_url'] for s in context['sources']]}
|
'source_urls':[s['source_url'] for s in context['sources']]}
|
||||||
|
|
||||||
@ -169,7 +172,7 @@ def test_missing_results_allow_an_explicit_partial_comparison(comparison_payload
|
|||||||
selected=selected_comparison(c)
|
selected=selected_comparison(c)
|
||||||
result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]})
|
result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]})
|
||||||
assert result['accepted_for_delivery']
|
assert result['accepted_for_delivery']
|
||||||
assert 'nie sú dosiahnuté výsledky doložené' in result['answer']
|
assert 'Táto časť odpovede nie je v poskytnutých podkladoch doložená.' in result['answer']
|
||||||
assert 'Implementácia systému vyhľadávania' in result['answer']
|
assert 'Implementácia systému vyhľadávania' in result['answer']
|
||||||
|
|
||||||
|
|
||||||
@ -182,9 +185,9 @@ def test_future_tasks_are_not_presented_as_achieved_results(comparison_payload):
|
|||||||
p=next(p for p in c['content_passages'] if p['person']=='Ján Holp')
|
p=next(p for p in c['content_passages'] if p['person']=='Ján Holp')
|
||||||
selected['evidence_quotes'].append({'source_id':p['source_id'],'quote':'- Pripravte experiment s PageRank.'})
|
selected['evidence_quotes'].append({'source_id':p['source_id'],'quote':'- Pripravte experiment s PageRank.'})
|
||||||
result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]})
|
result=generate_verified_rag(COMPARE,payload,lambda _: {'choices':[{'message':{'content':json.dumps(selected)}}]})
|
||||||
assert result['accepted_for_delivery']
|
# The model may not quote a future task omitted from its actual candidates.
|
||||||
assert '**Ďalšie doložené úlohy (plán):**' in result['answer']
|
assert not result['accepted_for_delivery']
|
||||||
assert 'Pripraviť experiment s PageRank.' in result['answer']
|
assert 'quote_outside_delivered_candidates' in result['attempts'][-1]['issues']
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|||||||
262
test/test_quality_block.py
Normal file
262
test/test_quality_block.py
Normal file
@ -0,0 +1,262 @@
|
|||||||
|
"""Behavioral checks for scope, relevant evidence, partial answers and failures."""
|
||||||
|
import copy
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tools.answer_evaluation.content_selection import candidates, facets_for
|
||||||
|
from tools.answer_evaluation.rag_evidence import rag_evidence_context
|
||||||
|
from tools.answer_evaluation.rag_preparation import prepare_named_content
|
||||||
|
from tools.answer_evaluation.verified_rag import generate_verified_rag, rag_answer_issues
|
||||||
|
from tools.answers.answer_errors import public_failure
|
||||||
|
from tools.graphrag import graph_search as gs
|
||||||
|
from tools.graphrag.tests.test_graph_search import FakeSession
|
||||||
|
from tools.rag.llm_context import compact_rag_payload
|
||||||
|
from tools.rag.rag_utils import build_context_text
|
||||||
|
from tools.retrieval.question_constraints import unsupported_constraint, requested_years
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def corpus(tmp_path):
|
||||||
|
db=tmp_path/'index.sqlite'; sources=[]
|
||||||
|
with sqlite3.connect(db) as conn:
|
||||||
|
conn.execute('CREATE TABLE documents(path,title,author,published)')
|
||||||
|
conn.execute('CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)')
|
||||||
|
for i,(person,slug,kind,year,text) in enumerate([
|
||||||
|
('Eva Nová','eva_nova','Diplomová práca',2024,
|
||||||
|
'Zadanie:\n1. Implementujte systém vyhľadávania.\nStav:\n- Konzultácia štruktúry práce.\n'
|
||||||
|
'- Natrénovaný model BERT a vykonané vyhodnotenie presnosti.\n'
|
||||||
|
'- Vznikol problém s tokenizérom.\nÚlohy:\n- Pripravte experiment s rôznymi tokenizérmi.'),
|
||||||
|
('Boris Nový','boris_novy','Bakalárska práca',2023,
|
||||||
|
'Zadanie:\n- Implementujte parser.\nStav:\n- Implementovaný parser pomocou Python.\n'
|
||||||
|
'- Vznikla chyba pri načítaní PDF.\nÚlohy:\n- Pripravte experiment s presnosťou extrakcie.')],1):
|
||||||
|
path=f'pages/students/2020/{slug}/README.md'
|
||||||
|
conn.execute('INSERT INTO documents VALUES(?,?,?,?)',(path,person,'Daniel Hladek',1))
|
||||||
|
conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path,text,json.dumps([[person,f'{kind} {year}']]),1,0))
|
||||||
|
sources.append({'source_id':f'S{i}','title':person,'document_path':path,'published':True,
|
||||||
|
'source_url':f'https://example.test/{slug}',
|
||||||
|
'text':f'Sekcia: {kind} {year}\nNázov: Ukážkový názov.'})
|
||||||
|
raw={'status':'ok','reason':None,'sources':sources,'context':build_context_text(sources),
|
||||||
|
'complete':False,'truncated':True,'truncation_reasons':['result_limit'],'index_version':{'version':'fixed'}}
|
||||||
|
return db,raw
|
||||||
|
|
||||||
|
|
||||||
|
def prepare(corpus,question):
|
||||||
|
db,raw=corpus
|
||||||
|
return compact_rag_payload(prepare_named_content(db,question,raw))
|
||||||
|
|
||||||
|
|
||||||
|
def model_selection(question,payload):
|
||||||
|
c=rag_evidence_context(question,payload)
|
||||||
|
return {'status':'ok','answer':'','claims':[],
|
||||||
|
'evidence_quotes':[{'source_id':x['source_id'],'quote':x['text']} for x in c.get('content_candidates',[])],
|
||||||
|
'source_urls':[s['source_url'] for s in c['sources']]}
|
||||||
|
|
||||||
|
|
||||||
|
def complete(answer):
|
||||||
|
return lambda _: {'choices':[{'message':{'content':json.dumps(answer)}}]}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('question,facets',[
|
||||||
|
('Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?',{'problems','experiments'}),
|
||||||
|
('Aké metódy používa Eva Nová?',{'methods'}),
|
||||||
|
('Aké sú dosiahnuté výsledky Evy Novej?',{'results'}),
|
||||||
|
])
|
||||||
|
def test_content_without_explicit_work_type(corpus,question,facets):
|
||||||
|
payload=prepare(corpus,question); c=rag_evidence_context(question,payload)
|
||||||
|
assert c['requirements']['content'] and set(c['requirements']['facets'])==facets
|
||||||
|
assert c['content_candidates']
|
||||||
|
|
||||||
|
|
||||||
|
def test_multiple_people_and_aspects_remain_bound(corpus):
|
||||||
|
q='Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?'
|
||||||
|
payload=prepare(corpus,q); a=model_selection(q,payload)
|
||||||
|
r=generate_verified_rag(q,payload,complete(a))
|
||||||
|
assert r['accepted_for_delivery'] and r['status']=='ok'
|
||||||
|
text=r['answer']; eva,boris=text.split('Boris Nový — bakalárska práca')
|
||||||
|
assert 'tokenizérom' in eva and 'PDF' not in eva
|
||||||
|
assert 'PDF' in boris and 'tokenizérmi' not in boris
|
||||||
|
assert text.count('**Riešené problémy:**')==2
|
||||||
|
assert text.count('**Plánované experimenty a vyhodnotenie:**')==2
|
||||||
|
|
||||||
|
|
||||||
|
def test_ambiguous_work_is_a_clarification_without_model(corpus):
|
||||||
|
db,raw=corpus; source=raw['sources'][0]
|
||||||
|
with sqlite3.connect(db) as conn:
|
||||||
|
conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(source['document_path'],
|
||||||
|
'Stav:\nVznikol iný problém.',json.dumps([['Eva Nová','Bakalárska práca 2022']]),1,1))
|
||||||
|
q='Aké problémy mala Eva Nová počas práce?'
|
||||||
|
p=prepare(corpus,q)
|
||||||
|
r=generate_verified_rag(q,p,lambda _:pytest.fail('No model call for a clarification'))
|
||||||
|
assert r['status']=='ambiguous' and r['reason']=='ambiguous_work'
|
||||||
|
assert '2022' in r['answer'] and '2024' in r['answer']
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_type_and_year_resolve_scope(corpus):
|
||||||
|
db,raw=corpus
|
||||||
|
with sqlite3.connect(db) as conn:
|
||||||
|
conn.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(raw['sources'][0]['document_path'],
|
||||||
|
'Stav:\nChyba v inom projekte Cassandra.',json.dumps([['Eva Nová','Diplomový projekt 2022']]),1,1))
|
||||||
|
q='Aké problémy mala Eva Nová pri diplomovej práci v roku 2024?'
|
||||||
|
p=prepare(corpus,q)
|
||||||
|
assert p['status']=='ok' and 'Cassandra' not in p['context']
|
||||||
|
|
||||||
|
|
||||||
|
def test_year_condition_without_the_words_v_roku_is_preserved(corpus):
|
||||||
|
q='Aké problémy mala Eva Nová pri diplomovej práci 1999?'
|
||||||
|
p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
||||||
|
assert c['requirements']['years']==[1999] and not c.get('content_candidates')
|
||||||
|
r=generate_verified_rag(q,p,lambda _:pytest.fail('No model for missing scoped evidence'))
|
||||||
|
assert r['accepted_for_delivery'] and r['status']=='no_answer' and 'tokenizérom' not in r['answer']
|
||||||
|
assert requested_years('Práca „Model 2020“ z roku 2024')==[2024]
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_source_can_be_loaded_when_absent_from_top_results(corpus):
|
||||||
|
db,raw=corpus; raw['sources']=raw['sources'][:1];raw['context']=build_context_text(raw['sources'])
|
||||||
|
p=prepare(corpus,'Aké problémy riešili Eva Nová a Boris Nový?')
|
||||||
|
assert {s['title'] for s in p['sources']}=={'Eva Nová','Boris Nový'}
|
||||||
|
|
||||||
|
|
||||||
|
def test_preparation_does_not_change_the_original_retrieval(corpus):
|
||||||
|
db,raw=corpus; before=copy.deepcopy(raw)
|
||||||
|
prepare_named_content(db,'Aké problémy riešili Eva Nová a Boris Nový?',raw)
|
||||||
|
assert raw==before
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_budget_cannot_silently_remove_a_requested_person(corpus):
|
||||||
|
db,raw=corpus; q='Aké problémy riešili Eva Nová a Boris Nový?'
|
||||||
|
p=prepare_named_content(db,q,raw)
|
||||||
|
p['sources'][0]['text']+='\n'+'Veľmi dlhý zdrojový text. '*1000
|
||||||
|
compact=compact_rag_payload(p,max_tokens=1100)
|
||||||
|
c=rag_evidence_context(q,compact)
|
||||||
|
assert compact['truncated']
|
||||||
|
assert set(c['requirements']['people'])=={'Eva Nová','Boris Nový'}
|
||||||
|
|
||||||
|
|
||||||
|
def test_candidates_are_actual_bound_literal_evidence(corpus):
|
||||||
|
q='Aké problémy riešili Eva Nová a Boris Nový?'; c=rag_evidence_context(q,prepare(corpus,q))
|
||||||
|
for candidate in candidates(c):
|
||||||
|
assert any(all(candidate[k]==p[k] for k in ['person','work_type','year','source_id'])
|
||||||
|
and candidate['text'] in p['text'] for p in c['content_passages'])
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_second_subject_is_not_silently_ignored(corpus):
|
||||||
|
p=prepare(corpus,'Aké problémy riešili študenti Eva Nová a Boris Noyy?')
|
||||||
|
assert p['status']=='ambiguous' and p['reason']=='unresolved_subject'
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('question',[
|
||||||
|
'Aké problémy riešili Eva Nová a Xaver Neexistujúci?',
|
||||||
|
'Aké problémy riešil Boris Noyy?',
|
||||||
|
])
|
||||||
|
def test_unresolved_subject_cannot_be_replaced_by_available_profiles(corpus,question):
|
||||||
|
p=prepare(corpus,question)
|
||||||
|
assert p['status']=='ambiguous' and p['reason']=='unresolved_subject'
|
||||||
|
|
||||||
|
|
||||||
|
def test_partial_answer_names_the_missing_aspect(corpus):
|
||||||
|
db,raw=corpus
|
||||||
|
with sqlite3.connect(db) as conn:
|
||||||
|
conn.execute("UPDATE chunks SET text='Stav:\nVznikol problém s parserom.' WHERE document_path LIKE '%boris_novy%'")
|
||||||
|
q='Aké problémy riešili Eva Nová a Boris Nový a aké experimenty mali pripraviť?'
|
||||||
|
p=prepare(corpus,q); r=generate_verified_rag(q,p,complete(model_selection(q,p)))
|
||||||
|
assert r['accepted_for_delivery'] and r['answer_coverage']['status']=='partial'
|
||||||
|
assert 'Táto časť odpovede nie je v poskytnutých podkladoch doložená.' in r['answer']
|
||||||
|
assert 'parserom' in r['answer']
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_aspects_absent_is_no_answer_not_rejection(corpus):
|
||||||
|
db,raw=corpus
|
||||||
|
with sqlite3.connect(db) as conn:
|
||||||
|
conn.execute("UPDATE chunks SET text='Zadanie:\n- Implementujte parser.'")
|
||||||
|
q='Aké problémy riešili Eva Nová a Boris Nový?'
|
||||||
|
r=generate_verified_rag(q,prepare(corpus,q),lambda _:pytest.fail('No model for a bounded empty result'))
|
||||||
|
assert r['status']=='no_answer' and r['accepted_for_delivery'] and not r['claims']
|
||||||
|
|
||||||
|
|
||||||
|
def test_known_year_is_returned_when_requested_title_is_missing(corpus):
|
||||||
|
_,raw=corpus; source=raw['sources'][0]
|
||||||
|
source['text']='Sekcia: Diplomová práca 2024\nStav:\nImplementovaný parser.'
|
||||||
|
raw['sources']=[source];raw['context']=build_context_text(raw['sources'])
|
||||||
|
q='Uveď názov a rok diplomovej práce Evy Novej.'
|
||||||
|
a={'status':'ok','answer':'','claims':[{'person':'Eva Nová','work_type':'diploma_thesis','year':2024}],
|
||||||
|
'evidence_quotes':[],'source_urls':[source['source_url']]}
|
||||||
|
r=generate_verified_rag(q,raw,complete(a))
|
||||||
|
assert r['accepted_for_delivery'] and '2024' in r['answer']
|
||||||
|
assert 'názov práce nie je v poskytnutých podkladoch doložený' in r['answer']
|
||||||
|
assert r['answer_coverage']['status']=='partial' and not r['claims'][0].get('title')
|
||||||
|
|
||||||
|
|
||||||
|
def test_concrete_results_win_over_administrative_notes(corpus):
|
||||||
|
q='Aké výsledky dosiahla Eva Nová?'; p=prepare(corpus,q); c=rag_evidence_context(q,p)
|
||||||
|
text='\n'.join(x['text'] for x in c['content_candidates'])
|
||||||
|
assert 'Natrénovaný model BERT' in text
|
||||||
|
r=generate_verified_rag(q,p,complete(model_selection(q,p)))
|
||||||
|
assert r['accepted_for_delivery'] and 'Natrénovaný model BERT' in r['answer']
|
||||||
|
assert 'Konzultácia' not in r['answer']
|
||||||
|
|
||||||
|
|
||||||
|
def test_duplicate_quotes_and_overlapping_chunks_do_not_duplicate_output(corpus):
|
||||||
|
q='Aké výsledky dosiahla Eva Nová?'; p=prepare(corpus,q); a=model_selection(q,p)
|
||||||
|
a['evidence_quotes']*=2
|
||||||
|
r=generate_verified_rag(q,p,complete(a))
|
||||||
|
assert r['accepted_for_delivery'] and r['answer'].count('Natrénovaný model BERT')==1
|
||||||
|
assert 'Ďalšie podklady:' not in r['answer'] and 'Ďalšie doložené podklady' not in r['answer']
|
||||||
|
|
||||||
|
|
||||||
|
def test_plans_cannot_answer_actual_use_or_completed_experiments():
|
||||||
|
assert 'methods' not in facets_for('Použite metódu BERT.','task','Aké metódy skutočne používal?')
|
||||||
|
assert 'experiments' not in facets_for('Pripravte experiment.','task','Aké experimenty vykonal?')
|
||||||
|
assert 'results' not in facets_for('Cieľom je implementovať systém.','progress','Aké dosiahol výsledky?')
|
||||||
|
|
||||||
|
|
||||||
|
def test_method_answer_does_not_add_valid_but_unrequested_profile_metadata(corpus):
|
||||||
|
_,raw=corpus
|
||||||
|
raw['sources'][0]['author']='Daniel Hladek'
|
||||||
|
q='Aké metódy používala Eva Nová pri diplomovej práci v roku 2024?'
|
||||||
|
p=prepare(corpus,q); a=model_selection(q,p)
|
||||||
|
a['claims']=[{'person':'Eva Nová','work_type':'diploma_thesis','year':2024,
|
||||||
|
'document_author':'Daniel Hladek'}]
|
||||||
|
r=generate_verified_rag(q,p,complete(a))
|
||||||
|
assert r['accepted_for_delivery'] and 'BERT' in r['answer']
|
||||||
|
assert 'Daniel Hladek' not in r['answer'] and 'autor dokumentu' not in r['answer']
|
||||||
|
assert r['answer'].count('2024')==1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('question',[
|
||||||
|
'Práce od roku 2020 do roku 2024?', 'Práce pred rokom 2020?',
|
||||||
|
'Študenti s aspoň 3 prácami?', 'Práce okrem diplomových?', 'Študenti bez tagu RAG?',
|
||||||
|
'V ktorom roku Ján Holp začal pracovať na diplomovej práci?',
|
||||||
|
'Kedy Ján Holp obhájil diplomovú prácu?',
|
||||||
|
])
|
||||||
|
def test_unsupported_operators_are_not_ignored(question):
|
||||||
|
assert unsupported_constraint(question)
|
||||||
|
result=gs.search_graph_with_session(FakeSession({}),question)
|
||||||
|
assert result['status']=='ambiguous' and result['reason']=='unsupported_constraint' and not result['facts']
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('question',[
|
||||||
|
'Práca v roku 1999?', 'Aký je názov práce „Od 2020 do 2024“?',
|
||||||
|
'Aké práce alebo projekty má Eva Nová?',
|
||||||
|
])
|
||||||
|
def test_supported_year_and_work_alternative_are_not_blocked(question):
|
||||||
|
assert unsupported_constraint(question) is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('reason',['model_unavailable','retrieval_unavailable','answer_rejected','answer_capacity_exceeded'])
|
||||||
|
def test_public_errors_are_distinct_and_do_not_leak_rejected_text(reason):
|
||||||
|
r=public_failure({'reason':reason,'answer':'SECRET YEAR 2099','claims':[{'person':'Eva'}],'source_urls':['SECRET']})
|
||||||
|
assert r['reason']==reason and not r['accepted_for_delivery']
|
||||||
|
assert 'SECRET' not in r['answer'] and '2099' not in r['answer']
|
||||||
|
assert not r['claims'] and not r['source_urls']
|
||||||
|
|
||||||
|
|
||||||
|
def test_retrieval_failure_is_not_misreported_as_bad_model_answer(monkeypatch):
|
||||||
|
import tools.answers.answer_delivery as delivery
|
||||||
|
def unavailable(_): raise RuntimeError('SECRET backend address')
|
||||||
|
monkeypatch.setattr(delivery,'search_graph',unavailable)
|
||||||
|
r=delivery.deliver_answer('Aké práce má Eva Nová?',engine='graphrag',
|
||||||
|
complete=lambda _:pytest.fail('No model call after failed retrieval'))
|
||||||
|
assert r['status']=='error' and r['reason']=='retrieval_unavailable'
|
||||||
|
assert 'SECRET' not in r['answer']
|
||||||
@ -39,7 +39,7 @@ def answer_coverage(answer: dict, context: dict) -> dict:
|
|||||||
(row.get('source_url') in cited or answer.get('status') == 'no_answer')))
|
(row.get('source_url') in cited or answer.get('status') == 'no_answer')))
|
||||||
# A context budget may drop only unrelated sources. A clipping marker in
|
# A context budget may drop only unrelated sources. A clipping marker in
|
||||||
# the actual cited block is stronger evidence than this global reason.
|
# the actual cited block is stronger evidence than this global reason.
|
||||||
context_limited = 'llm_context_budget' in reasons
|
context_limited = any(reason in reasons for reason in ['llm_context_budget','answer_candidate_budget','evidence_unit_limit'])
|
||||||
limited = context.get('complete') is not True or context.get('truncated') is True
|
limited = context.get('complete') is not True or context.get('truncated') is True
|
||||||
limitations = [*reasons]
|
limitations = [*reasons]
|
||||||
if clipped:
|
if clipped:
|
||||||
@ -48,6 +48,13 @@ def answer_coverage(answer: dict, context: dict) -> dict:
|
|||||||
status = answer.get('status')
|
status = answer.get('status')
|
||||||
if status == 'ok':
|
if status == 'ok':
|
||||||
coverage = 'supported'
|
coverage = 'supported'
|
||||||
|
if any(requirements.get(key) and key not in fields for fields in requirements.get('available_fields',{}).values()
|
||||||
|
for key in ['title','year','start_year']):
|
||||||
|
coverage='partial'; warnings.append('partial_requested_fields')
|
||||||
|
if requirements.get('facets'):
|
||||||
|
from .content_selection import missing_facets
|
||||||
|
if missing_facets(answer,context):
|
||||||
|
coverage='partial'; warnings.append('partial_requested_aspects')
|
||||||
if scope == 'list' and (limited or clipped):
|
if scope == 'list' and (limited or clipped):
|
||||||
coverage = 'partial'; warnings.append('list_may_be_incomplete')
|
coverage = 'partial'; warnings.append('list_may_be_incomplete')
|
||||||
elif scope == 'count' and (context.get('aggregation') or {}).get('complete') is not True:
|
elif scope == 'count' and (context.get('aggregation') or {}).get('complete') is not True:
|
||||||
@ -74,4 +81,7 @@ def coverage_notices(coverage: dict) -> list[str]:
|
|||||||
'missing_information_with_clipped_context': 'Časť podkladov bola pri príprave kontextu skrátená.',
|
'missing_information_with_clipped_context': 'Časť podkladov bola pri príprave kontextu skrátená.',
|
||||||
'count_not_established': 'Celkový počet sa z dostupných podkladov nedá spoľahlivo určiť.',
|
'count_not_established': 'Celkový počet sa z dostupných podkladov nedá spoľahlivo určiť.',
|
||||||
}
|
}
|
||||||
return [messages[reason] for reason in coverage['warnings']]
|
notices=[messages[reason] for reason in coverage['warnings'] if reason in messages]
|
||||||
|
if 'list_may_be_incomplete' in coverage['warnings']:
|
||||||
|
notices.append('Výpis môžeš zúžiť uvedením osoby, typu práce alebo konkrétneho roku.')
|
||||||
|
return notices
|
||||||
|
|||||||
@ -16,7 +16,8 @@ def generate_checked_answer(question: str, context: dict, prompt: str, complete)
|
|||||||
result['response']=response;record['response']=response
|
result['response']=response;record['response']=response
|
||||||
text=response['choices'][0]['message']['content'].strip()
|
text=response['choices'][0]['message']['content'].strip()
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
result['error']=f'{type(error).__name__}: {str(error)[:500]}'
|
result['error']=type(error).__name__
|
||||||
|
result['reason']='model_unavailable'
|
||||||
result['attempts'].append(record)
|
result['attempts'].append(record)
|
||||||
return result
|
return result
|
||||||
result['attempts'].append(record)
|
result['attempts'].append(record)
|
||||||
|
|||||||
191
tools/answer_evaluation/content_selection.py
Normal file
191
tools/answer_evaluation/content_selection.py
Normal file
@ -0,0 +1,191 @@
|
|||||||
|
"""Role-aware, bounded excerpt selection and presentation from actual evidence."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from .context_policy import normalized
|
||||||
|
from .question_intent import FACET_LABELS, planned_experiments
|
||||||
|
|
||||||
|
COMPLETED = r'\b(?:implementovan\w*|natrenovan\w*|vytvoren\w*|extrahovan\w*|vykonan\w*|urobene vyhodnotenie|vyhodnoteny\w*|nameran\w*)\b'
|
||||||
|
PLANNED = r'\b(?:bude|budu|planuj\w*|cielom|mal by|ma sa|treba|pripravte|navrhnite|otestujte|vyhodnotte)\b'
|
||||||
|
ADMIN = r'\b(?:konzultac\w*|struktura prace|pisomn\w*|pisat|odovzda\w*|wip|termin\w*)\b'
|
||||||
|
|
||||||
|
|
||||||
|
def split_units(text: str) -> list[str]:
|
||||||
|
from .rag_evidence import descriptive_text
|
||||||
|
parts = re.split(r'\n\s*\n|(?=^[ \t]*(?:\d+[.)]|[-+*])\s+)', text, flags=re.M)
|
||||||
|
units = []
|
||||||
|
for part in parts:
|
||||||
|
part = part.strip()
|
||||||
|
if not descriptive_text(part):
|
||||||
|
continue
|
||||||
|
# A source paragraph is kept literal, including its negations and qualifications.
|
||||||
|
for unit in ([part] if len(part) <= 900 else re.split(r'(?<=[.!?])\s+(?=[A-ZÁČĎÉÍĽŇÓŠŤÚÝŽ])', part)):
|
||||||
|
if descriptive_text(unit):
|
||||||
|
units.append(unit.strip())
|
||||||
|
return units
|
||||||
|
|
||||||
|
|
||||||
|
def effective_role(text: str, role: str) -> str:
|
||||||
|
n = normalized(text)
|
||||||
|
if role=='other':
|
||||||
|
from .presentation import GOAL_VERBS
|
||||||
|
first=n.split()[0] if n else ''
|
||||||
|
if first in {normalized(word) for word in GOAL_VERBS} or n.startswith('cielom je'):
|
||||||
|
return 'task'
|
||||||
|
if role == 'other' and re.search(COMPLETED, n) and not re.search(PLANNED, n):
|
||||||
|
return 'progress'
|
||||||
|
return role
|
||||||
|
|
||||||
|
|
||||||
|
def facets_for(text: str, role: str, question: str) -> list[str]:
|
||||||
|
n = normalized(text); facets = []
|
||||||
|
if role == 'assignment': facets.append('assignment')
|
||||||
|
if role == 'progress' and (not re.search(PLANNED,n) or re.search(COMPLETED,n)):
|
||||||
|
facets.append('results')
|
||||||
|
if re.search(r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*|zlyha\w*|nepodar\w*|nevyries\w*)\b', n) and role not in {'assignment','task'}:
|
||||||
|
facets.append('problems')
|
||||||
|
if re.search(r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*)\b', n):
|
||||||
|
if role not in {'assignment','task'} or not re.search(r'\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*)\b',normalized(question)):
|
||||||
|
facets.append('methods')
|
||||||
|
if re.search(r'\b(?:experiment\w*|meran\w*|test\w*|otest\w*|vyhodnot\w*|benchmark\w*|f1|presnost\w*)\b', n):
|
||||||
|
wants_done=re.search(r'\b(?:vykonal\w*|otestoval\w*|uskutocnil\w*|vyhodnotil\w*)\b',normalized(question))
|
||||||
|
if wants_done and role in {'assignment','task'}:
|
||||||
|
pass
|
||||||
|
elif not planned_experiments(question) or role in {'assignment','task'} or re.search(PLANNED, n):
|
||||||
|
facets.append('experiments')
|
||||||
|
return facets
|
||||||
|
|
||||||
|
|
||||||
|
def unit_score(text: str, role: str, facet: str = '') -> int:
|
||||||
|
n = normalized(text)
|
||||||
|
score = 25 if role == 'progress' else 10
|
||||||
|
if re.search(COMPLETED, n): score += 45
|
||||||
|
if re.search(r'\b(?:model|system|algoritm\w*|vyhodnot\w*|experiment\w*|presnost\w*|pagerank|bert)\b', n): score += 15
|
||||||
|
if re.search(ADMIN, n): score -= 70
|
||||||
|
if facet == 'problems' and re.search(r'\b(?:problem\w*|chyb\w*|zlyha\w*)\b', n): score += 35
|
||||||
|
if role in {'assignment','task'} and facet == 'results': score -= 100
|
||||||
|
return score
|
||||||
|
|
||||||
|
|
||||||
|
def evidence_units(context: dict, quotes: list[dict] | None = None) -> list[dict]:
|
||||||
|
from .rag_evidence import content_sections, quote_sections
|
||||||
|
rows = []
|
||||||
|
for passage in context.get('content_passages', []):
|
||||||
|
pieces = content_sections(passage['text']) if quotes is None else [
|
||||||
|
piece for quote in quotes if quote['source_id'] == passage['source_id'] and
|
||||||
|
' '.join(quote['quote'].split()) in ' '.join(passage['text'].split())
|
||||||
|
for piece in quote_sections(quote, context)]
|
||||||
|
for piece in pieces:
|
||||||
|
for unit in split_units(piece['text']):
|
||||||
|
role = effective_role(unit, piece['kind'])
|
||||||
|
rows.append({k: passage.get(k) for k in ['person','work_type','year','source_id']} |
|
||||||
|
{'text':unit,'kind':role,'facets':facets_for(unit,role,context.get('question',''))})
|
||||||
|
# Overlapping chunks/quotes can repeat the same fact with different roles.
|
||||||
|
# An explicit role wins over an unlabelled duplicate, within the same work.
|
||||||
|
unique = {}
|
||||||
|
for row in rows:
|
||||||
|
clean=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text'])
|
||||||
|
key = tuple(row.get(k) for k in ['person','work_type','year']) + (' '.join(clean.split()),)
|
||||||
|
old = unique.get(key)
|
||||||
|
if old is None or (old['kind']=='other' and row['kind']!='other'):
|
||||||
|
unique[key]=row
|
||||||
|
return list(unique.values())
|
||||||
|
|
||||||
|
|
||||||
|
def candidates(context: dict) -> list[dict]:
|
||||||
|
facets = context['requirements'].get('facets', [])
|
||||||
|
selected = []; seen = set(); units = evidence_units(context)
|
||||||
|
for person in context['requirements']['people']:
|
||||||
|
for facet in facets:
|
||||||
|
relevant = [u for u in units if u['person']==person and facet in u['facets']]
|
||||||
|
relevant.sort(key=lambda u: -unit_score(u['text'],u['kind'],facet))
|
||||||
|
accepted=0
|
||||||
|
for unit in relevant:
|
||||||
|
text = unit['text']
|
||||||
|
if len(text)>2500:
|
||||||
|
context['complete']=False; context['truncated']=True
|
||||||
|
context['truncation_reasons']=list(dict.fromkeys([*(context.get('truncation_reasons') or []),'evidence_unit_limit']))
|
||||||
|
continue
|
||||||
|
key=(unit['source_id'],text)
|
||||||
|
if key not in seen:
|
||||||
|
seen.add(key); selected.append({**unit,'text':text})
|
||||||
|
accepted+=1
|
||||||
|
if accepted==2: break
|
||||||
|
return selected
|
||||||
|
|
||||||
|
|
||||||
|
def missing_facets(answer: dict, context: dict) -> list[tuple[str,str]]:
|
||||||
|
found = evidence_units(context, answer.get('evidence_quotes', []))
|
||||||
|
return [(person,facet) for person in context['requirements']['people']
|
||||||
|
for facet in context['requirements'].get('facets', [])
|
||||||
|
if not any(row['person']==person and facet in row['facets'] for row in found)]
|
||||||
|
|
||||||
|
|
||||||
|
def fit_candidates(sent: dict, context: dict, limit: int, encoding: str) -> None:
|
||||||
|
"""Budget the actual candidate JSON, sharing space across named people."""
|
||||||
|
import json
|
||||||
|
from tools.rag.build_chunks import TokenCounter
|
||||||
|
counter=TokenCounter(encoding)
|
||||||
|
if counter.encoding is None:
|
||||||
|
raise RuntimeError('Missing context tokenizer')
|
||||||
|
original=list(context.get('content_candidates',[]))
|
||||||
|
groups=[[row for row in original if row.get('person')==person]
|
||||||
|
for person in context['requirements']['people']]
|
||||||
|
ordered=[group[i] for i in range(max(map(len,groups),default=0)) for group in groups if i<len(group)]
|
||||||
|
selected=[]
|
||||||
|
for row in ordered:
|
||||||
|
if counter.count(json.dumps({**sent,'content_candidates':[*selected,row]},ensure_ascii=False))<=limit:
|
||||||
|
selected.append(row)
|
||||||
|
if len(selected)!=len(original):
|
||||||
|
context['complete']=False;context['truncated']=True
|
||||||
|
context['truncation_reasons']=list(dict.fromkeys([*(context.get('truncation_reasons') or []),'answer_candidate_budget']))
|
||||||
|
sent.update({key:context[key] for key in ['complete','truncated','truncation_reasons']})
|
||||||
|
context['content_candidates']=selected;sent['content_candidates']=selected
|
||||||
|
# Account for the added limitation metadata as well.
|
||||||
|
while selected and counter.count(json.dumps(sent,ensure_ascii=False))>limit:
|
||||||
|
selected.pop()
|
||||||
|
if counter.count(json.dumps(sent,ensure_ascii=False))>limit:
|
||||||
|
raise ValueError('Mandatory answer metadata exceeds context budget')
|
||||||
|
|
||||||
|
|
||||||
|
def facet_lines(answer: dict, context: dict) -> list[str]:
|
||||||
|
from .presentation import goal_text, work_subject, owner_in_question
|
||||||
|
from .rag_evidence import WORK_LABELS
|
||||||
|
req=context['requirements']; units=evidence_units(context,answer.get('evidence_quotes',[]))
|
||||||
|
lines=[]
|
||||||
|
for person in req['people']:
|
||||||
|
scopes={(p['work_type'],p.get('year')) for p in context.get('content_passages',[]) if p.get('person')==person}
|
||||||
|
kind=req['work_types'].get(person) or (next(iter(scopes))[0] if len(scopes)==1 else None)
|
||||||
|
subject=work_subject(person,kind,context.get('question',''))
|
||||||
|
if not owner_in_question(person,context.get('question','')):
|
||||||
|
subject=person+(' — '+WORK_LABELS[kind] if kind in WORK_LABELS else '')
|
||||||
|
if len(scopes)==1 and next(iter(scopes))[1] is not None:
|
||||||
|
subject+=' ('+str(next(iter(scopes))[1])+')'
|
||||||
|
lines.extend(['**'+subject+'**', ''])
|
||||||
|
seen=set()
|
||||||
|
for facet in req.get('facets', []):
|
||||||
|
label=FACET_LABELS[facet]
|
||||||
|
if facet=='experiments' and planned_experiments(context.get('question','')):
|
||||||
|
label='Plánované experimenty a vyhodnotenie'
|
||||||
|
rows=[u for u in units if u['person']==person and facet in u['facets']]
|
||||||
|
rows.sort(key=lambda u:-unit_score(u['text'],u['kind'],facet))
|
||||||
|
lines.append('**'+label+':**')
|
||||||
|
shown=0
|
||||||
|
for row in rows:
|
||||||
|
key=' '.join(re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',row['text']).split())
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
text=row['text']
|
||||||
|
if row['kind'] in {'assignment','task'}:
|
||||||
|
text,_=goal_text(text)
|
||||||
|
text=re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',text)
|
||||||
|
phase=' (plán)' if row['kind'] in {'assignment','task'} and facet not in {'assignment'} else ''
|
||||||
|
lines.append('- '+text+phase); seen.add(key); shown+=1
|
||||||
|
if shown==2: break
|
||||||
|
if not shown:
|
||||||
|
if rows:
|
||||||
|
lines.append('Je doložené rovnakým výňatkom uvedeným vyššie.')
|
||||||
|
else:
|
||||||
|
lines.append('Táto časť odpovede nie je v poskytnutých podkladoch doložená.')
|
||||||
|
lines.append('')
|
||||||
|
return lines
|
||||||
@ -54,6 +54,7 @@ def graph_context(result: dict) -> dict:
|
|||||||
'index_version':result.get('index_version')}
|
'index_version':result.get('index_version')}
|
||||||
context['answer_candidates']=joined_candidates(facts,intent)
|
context['answer_candidates']=joined_candidates(facts,intent)
|
||||||
context['question'] = result.get('query', '')
|
context['question'] = result.get('query', '')
|
||||||
|
context['clarification'] = result.get('clarification')
|
||||||
context['requested_people'] = list(dict.fromkeys(row['name']
|
context['requested_people'] = list(dict.fromkeys(row['name']
|
||||||
for row in result.get('resolved_entities', []) if row.get('kind') == 'Person' and row.get('name')))
|
for row in result.get('resolved_entities', []) if row.get('kind') == 'Person' and row.get('name')))
|
||||||
comparison = result.get('academic_year_comparison') or next((row for row in result['facts']
|
comparison = result.get('academic_year_comparison') or next((row for row in result['facts']
|
||||||
|
|||||||
38
tools/answer_evaluation/question_intent.py
Normal file
38
tools/answer_evaluation/question_intent.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
"""Explicit answer aspects; these select evidence, never supply facts."""
|
||||||
|
import re
|
||||||
|
from .context_policy import normalized
|
||||||
|
|
||||||
|
FACET_PATTERNS = {
|
||||||
|
'assignment': r'\b(?:zadani\w*|ciel\w*)\b',
|
||||||
|
'problems': r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*)\b',
|
||||||
|
'methods': r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouziv\w*|pouzil\w*)\b',
|
||||||
|
'experiments': r'\b(?:experiment\w*|merani\w*|testovani\w*|vyhodnocov\w*)\b',
|
||||||
|
'results': r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b',
|
||||||
|
}
|
||||||
|
FACET_LABELS = {'assignment': 'Zadanie a ciele', 'problems': 'Riešené problémy',
|
||||||
|
'methods': 'Metódy a technológie', 'experiments': 'Experimenty a vyhodnotenie',
|
||||||
|
'results': 'Zaznamenaný stav a výsledky'}
|
||||||
|
|
||||||
|
|
||||||
|
def requested_facets(question: str) -> list[str]:
|
||||||
|
text = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question))
|
||||||
|
matches = [(m.start(), key) for key, pattern in FACET_PATTERNS.items()
|
||||||
|
if (m := re.search(pattern, text))]
|
||||||
|
return [key for _, key in sorted(matches)]
|
||||||
|
|
||||||
|
|
||||||
|
def content_question(question: str) -> bool:
|
||||||
|
text = normalized(question)
|
||||||
|
return bool(requested_facets(question) or re.search(
|
||||||
|
r'\b(?:comu|o com|obsah\w*|opis\w*|popis\w*|zamer\w*|zhrn\w*|implement\w*)\b', text))
|
||||||
|
|
||||||
|
|
||||||
|
def facet_mode(question: str) -> bool:
|
||||||
|
facets = requested_facets(question)
|
||||||
|
return any(facet in facets for facet in ['problems','methods','experiments','results'])
|
||||||
|
|
||||||
|
|
||||||
|
def planned_experiments(question: str) -> bool:
|
||||||
|
text=normalized(question)
|
||||||
|
return bool(re.search(r'\b(?:plan\w*|pripravit|navrhnut)|'
|
||||||
|
r'\b(?:mal|mali|mala|ma|maju)\s+(?:priprav\w*|navrhn\w*|otest\w*|vyhodnot\w*)',text))
|
||||||
@ -10,6 +10,8 @@ import json
|
|||||||
import sqlite3
|
import sqlite3
|
||||||
from contextlib import closing
|
from contextlib import closing
|
||||||
from .context_policy import normalized, text_contains_value, person_pattern
|
from .context_policy import normalized, text_contains_value, person_pattern
|
||||||
|
from .question_intent import requested_facets, content_question, facet_mode
|
||||||
|
from tools.retrieval.question_constraints import requested_years
|
||||||
|
|
||||||
WORK_TYPES = {
|
WORK_TYPES = {
|
||||||
'bakalarska praca': 'bachelor_thesis', 'diplomova praca': 'diploma_thesis',
|
'bakalarska praca': 'bachelor_thesis', 'diplomova praca': 'diploma_thesis',
|
||||||
@ -148,6 +150,10 @@ def academic_windows(body: str) -> list[tuple[tuple[str, int | None] | None, str
|
|||||||
scope = (None if normalized(heading[1]).startswith(('priprava na ', 'priprava k ')) else
|
scope = (None if normalized(heading[1]).startswith(('priprava na ', 'priprava k ')) else
|
||||||
detected or (heading_stack[-1][1] if heading_stack else None))
|
detected or (heading_stack[-1][1] if heading_stack else None))
|
||||||
heading_stack.append((level, scope))
|
heading_stack.append((level, scope))
|
||||||
|
if scope is not None and detected is None:
|
||||||
|
# Keep semantic child headings (Stav, Úlohy, Metódy) as
|
||||||
|
# literal source text instead of losing their meaning.
|
||||||
|
lines.append(line)
|
||||||
else:
|
else:
|
||||||
lines.append(line)
|
lines.append(line)
|
||||||
flush()
|
flush()
|
||||||
@ -392,14 +398,16 @@ def rag_evidence_context(question: str, payload: dict) -> dict:
|
|||||||
facts.extend({**base, field: value} for value in values)
|
facts.extend({**base, field: value} for value in values)
|
||||||
facts = list({tuple(sorted(row.items())): row for row in facts}.values())
|
facts = list({tuple(sorted(row.items())): row for row in facts}.values())
|
||||||
requested = [person for person in people if text_contains_value(question, person, person=True)]
|
requested = [person for person in people if text_contains_value(question, person, person=True)]
|
||||||
|
requested=list(dict.fromkeys([*payload.get('named_subjects',[]),*requested]))
|
||||||
expected_types = requested_types(question, requested)
|
expected_types = requested_types(question, requested)
|
||||||
query = normalized(question)
|
query = normalized(question)
|
||||||
intent_query = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question))
|
intent_query = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question))
|
||||||
academic = bool(TYPE_PATTERN.search(query) or re.search(r'\b(?:rok\w* prac\w*|nazov\w* prac\w*)\b', query))
|
academic = bool(TYPE_PATTERN.search(query) or re.search(r'\b(?:rok\w* prac\w*|nazov\w* prac\w*)\b', query))
|
||||||
content = bool(academic or (requested and re.search(r'\b(?:prac\w*|projekt\w*)\b', query))) and bool(re.search(
|
content = bool(academic or requested) and bool(re.search(
|
||||||
r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|co\s+ries\w*|obsah\w*|opis\w*|popis\w*|'
|
r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|co\s+ries\w*|obsah\w*|opis\w*|popis\w*|'
|
||||||
r'zhrn\w*|zhrnut\w*|vysvetl\w*|zamer\w*|zadani\w*|ciel\w*|metod\w*|technolog\w*|'
|
r'zhrn\w*|zhrnut\w*|vysvetl\w*|zamer\w*|zadani\w*|ciel\w*|metod\w*|technolog\w*|'
|
||||||
r'pouziv\w*|implement\w*|vysledk\w*|dosiahn\w*)\b', intent_query))
|
r'pouziv\w*|implement\w*|vysledk\w*|dosiahn\w*|problem\w*|tazkost\w*|prekazk\w*|experiment\w*|merani\w*)\b', intent_query)
|
||||||
|
or content_question(question))
|
||||||
identity_first = re.search(r'^(?:aky|aka|ake|ktory|ktora|ktore|v ktorom|uved)'
|
identity_first = re.search(r'^(?:aky|aka|ake|ktory|ktora|ktore|v ktorom|uved)'
|
||||||
r'(?:\s+(?:je|su|bol|bola|boli|presny|presne)){0,3}\s+(?:nazov\w*|rok\w*)\b', intent_query)
|
r'(?:\s+(?:je|su|bol|bola|boli|presny|presne)){0,3}\s+(?:nazov\w*|rok\w*)\b', intent_query)
|
||||||
explicit_description = re.search(r'\b(?:comu|o com|opis\w*|popis\w*|zhrn\w*|obsah\w*|zadani\w*|ciel\w*)\b', intent_query)
|
explicit_description = re.search(r'\b(?:comu|o com|opis\w*|popis\w*|zhrn\w*|obsah\w*|zadani\w*|ciel\w*)\b', intent_query)
|
||||||
@ -419,18 +427,27 @@ def rag_evidence_context(question: str, payload: dict) -> dict:
|
|||||||
requirements['response_mode'] = 'mixed' if any(requirements[key] for key in ['title', 'year', 'start_year']) else 'quotes'
|
requirements['response_mode'] = 'mixed' if any(requirements[key] for key in ['title', 'year', 'start_year']) else 'quotes'
|
||||||
requirements['distinguish_progress'] = bool(re.search(r'\b(?:zadani\w*|ciel\w*)\b', query)
|
requirements['distinguish_progress'] = bool(re.search(r'\b(?:zadani\w*|ciel\w*)\b', query)
|
||||||
and re.search(r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b', query))
|
and re.search(r'\b(?:vysledk\w*|dosiahn\w*|stav\w*)\b', query))
|
||||||
|
if facet_mode(question):
|
||||||
|
requirements['facets'] = requested_facets(question)
|
||||||
stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else []
|
stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else []
|
||||||
requirements['years'] = ([int(value) for value in stated_years if len(value) == 4]
|
requirements['years'] = requested_years(question) if academic and not requirements['start_year'] else []
|
||||||
if not requirements['start_year'] else [])
|
|
||||||
requirements['unsupported_year'] = any(len(value) != 4 for value in stated_years)
|
requirements['unsupported_year'] = any(len(value) != 4 for value in stated_years)
|
||||||
if requested:
|
if requested:
|
||||||
facts = [row for row in facts if row['person'] in requested]
|
facts = [row for row in facts if row['person'] in requested]
|
||||||
context = {'question': question, **{key: payload.get(key) for key in ['status', 'reason', 'complete', 'truncated',
|
requirements['available_fields'] = {person:[key for key in ['title','year','start_year']
|
||||||
|
if any(row.get('person')==person and row.get(key) is not None and
|
||||||
|
(key=='start_year' or ((not expected_types.get(person) or row.get('work_type')==expected_types[person])
|
||||||
|
and (not requirements['years'] or row.get('year') in requirements['years']))) for row in facts)]
|
||||||
|
for person in requested}
|
||||||
|
context = {'question': question, 'clarification':payload.get('clarification'), **{key: payload.get(key) for key in ['status', 'reason', 'complete', 'truncated',
|
||||||
'truncation_reasons', 'index_version']}, 'sources': payload.get('sources', []),
|
'truncation_reasons', 'index_version']}, 'sources': payload.get('sources', []),
|
||||||
'facts': facts, 'source_blocks': blocks, 'requirements': requirements,
|
'facts': facts, 'source_blocks': blocks, 'requirements': requirements,
|
||||||
'aggregation': None, 'answer_candidates': []}
|
'aggregation': None, 'answer_candidates': []}
|
||||||
if content:
|
if content:
|
||||||
context['content_passages'] = academic_content_passages(blocks, facts, requirements, question)
|
context['content_passages'] = academic_content_passages(blocks, facts, requirements, question)
|
||||||
|
if requirements.get('facets'):
|
||||||
|
from .content_selection import candidates
|
||||||
|
context['content_candidates'] = candidates(context)
|
||||||
if requirements['distinguish_progress']:
|
if requirements['distinguish_progress']:
|
||||||
requirements['content_roles'] = {person: sorted({section['kind']
|
requirements['content_roles'] = {person: sorted({section['kind']
|
||||||
for p in context['content_passages'] if p['person'] == person
|
for p in context['content_passages'] if p['person'] == person
|
||||||
|
|||||||
91
tools/answer_evaluation/rag_preparation.py
Normal file
91
tools/answer_evaluation/rag_preparation.py
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
"""Resolve named source profiles and scope content before the shared token budget."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
from contextlib import closing
|
||||||
|
|
||||||
|
from .context_policy import normalized, person_pattern, text_contains_value
|
||||||
|
from .question_intent import content_question
|
||||||
|
from .rag_evidence import PROFILE, academic_scope, academic_windows, requested_types, WORK_LABELS
|
||||||
|
from tools.retrieval.question_constraints import requested_years
|
||||||
|
|
||||||
|
|
||||||
|
def clarify(payload: dict, reason: str, message: str) -> dict:
|
||||||
|
return {**payload,'status':'ambiguous','reason':reason,'complete':False,
|
||||||
|
'clarification':message}
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_named_content(db_file, question: str, payload: dict) -> dict:
|
||||||
|
if not content_question(question):
|
||||||
|
return payload
|
||||||
|
if re.search(r'\b(?:s tagom|s klucovym slovom|z kategorie|s autorom dokumentu)\b',normalized(question)):
|
||||||
|
return clarify(payload,'content_filter_requires_graph',
|
||||||
|
'Najprv over podmienku tagu, kategórie alebo autora cez /graphrag. '
|
||||||
|
'Potom môžem opísať obsah konkrétnej vyhovujúcej práce; túto podmienku nevynechám.')
|
||||||
|
from tools.rag.rag_utils import build_context_text
|
||||||
|
with closing(sqlite3.connect(db_file.resolve().as_uri()+'?mode=ro',uri=True)) as conn:
|
||||||
|
conn.row_factory=sqlite3.Row
|
||||||
|
if not {'documents','chunks'} <= {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}:
|
||||||
|
return payload
|
||||||
|
catalog=[dict(r) for r in conn.execute('SELECT path,title,author,published FROM documents WHERE COALESCE(published,1)=1')
|
||||||
|
if PROFILE.fullmatch(r['path']) and r['title']]
|
||||||
|
named=[r for r in catalog if text_contains_value(question,r['title'],person=True)]
|
||||||
|
# Preserve question order rather than the original retrieval ranking.
|
||||||
|
named.sort(key=lambda r:re.search(person_pattern(r['title']),normalized(question)).start())
|
||||||
|
if len(named)>4:
|
||||||
|
return clarify(payload,'too_many_subjects','Pre podrobné obsahové porovnanie vyber najviac štyri osoby naraz.')
|
||||||
|
from tools.graphrag.graph_search import requested_name_phrases
|
||||||
|
first_names={normalized(r['title']).split()[0] for r in catalog}
|
||||||
|
capitalized={normalized(match[0]) for match in re.finditer(
|
||||||
|
r'\b[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][^\W\d_]+(?:\s+[A-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ][^\W\d_]+){1,3}\b',question)}
|
||||||
|
unknown=[]
|
||||||
|
for phrase in requested_name_phrases(question):
|
||||||
|
words=phrase.split()
|
||||||
|
named_companion=(named and phrase in capitalized and
|
||||||
|
re.search(r'\ba\s+'+re.escape(phrase)+r'\b',normalized(question)))
|
||||||
|
if (2<=len(words)<=4 and (words[0] in first_names or named_companion) and
|
||||||
|
not any(text_contains_value(phrase,r['title'],person=True) for r in catalog)):
|
||||||
|
unknown.append(phrase)
|
||||||
|
if unknown:
|
||||||
|
return clarify(payload,'unresolved_subject','Meno „'+unknown[0]+'“ sa nepodarilo spoľahlivo priradiť. '
|
||||||
|
'Uveď jeho presné celé meno; ostatné osoby nebudem používať ako náhradu.')
|
||||||
|
if not named:
|
||||||
|
return payload
|
||||||
|
types=requested_types(question,[r['title'] for r in named])
|
||||||
|
years=requested_years(question)
|
||||||
|
selected=[]
|
||||||
|
for person in named:
|
||||||
|
rows=list(conn.execute('SELECT text,heading_paths_json FROM chunks WHERE document_path=? '
|
||||||
|
'AND COALESCE(published,1)=1 ORDER BY chunk_index',(person['path'],)))
|
||||||
|
frames=[]; records={};seen=set()
|
||||||
|
for row in rows:
|
||||||
|
paths=json.loads(row['heading_paths_json'] or '[]')
|
||||||
|
headings=[' > '.join(p) if isinstance(p,list) else str(p) for p in paths]
|
||||||
|
first=headings[0] if headings else ''
|
||||||
|
for scope,text in academic_windows('Sekcia: '+first+'\n'+str(row['text'] or '')):
|
||||||
|
if not scope or (types.get(person['title']) and scope[0]!=types[person['title']]) or (years and scope[1] not in years):
|
||||||
|
continue
|
||||||
|
label=WORK_LABELS[scope[0]]+(f' {scope[1]}' if scope[1] is not None else '')
|
||||||
|
records[scope]=label
|
||||||
|
if (scope,text) not in seen:
|
||||||
|
seen.add((scope,text));frames.append('Sekcia: '+label+'\n'+text)
|
||||||
|
if len(records)>1:
|
||||||
|
options='; '.join(dict.fromkeys(records.values()))
|
||||||
|
return clarify(payload,'ambiguous_work',f'Ktorú prácu osoby {person["title"]} máš na mysli? '
|
||||||
|
f'V indexe sú tieto možnosti: {options}. Uveď typ práce a rok.')
|
||||||
|
source=next((s for s in payload.get('sources',[]) if s.get('document_path')==person['path']),None)
|
||||||
|
if source is None:
|
||||||
|
from tools.rag.search_core import make_source_url
|
||||||
|
source={'title':person['title'],'document_path':person['path'],'source_url':make_source_url(person['path']),
|
||||||
|
'author':person['author'],'published':person['published']}
|
||||||
|
selected.append({**source,'text':'\n\n'.join(frames)})
|
||||||
|
# Keep already-retrieved child documents of the requested profiles.
|
||||||
|
for source in payload.get('sources',[]):
|
||||||
|
if any(source.get('document_path','').startswith(p['path'].removesuffix('README.md'))
|
||||||
|
and source.get('document_path')!=p['path'] for p in named):
|
||||||
|
selected.append(source)
|
||||||
|
selected=[{**source,'source_id':f'S{i}'} for i,source in enumerate(selected,1)]
|
||||||
|
return {**payload,'sources':selected,'source_count':len(selected),'context':build_context_text(selected),
|
||||||
|
'named_subjects':[p['title'] for p in named]}
|
||||||
@ -54,6 +54,8 @@ def render_graph_answer(answer: dict, context: dict) -> dict:
|
|||||||
lines.append(f'Počet vyhovujúcich osôb: {out["count"]}.')
|
lines.append(f'Počet vyhovujúcich osôb: {out["count"]}.')
|
||||||
if context.get('complete') is False or context.get('truncated'):
|
if context.get('complete') is False or context.get('truncated'):
|
||||||
lines.append('Výpis je neúplný.')
|
lines.append('Výpis je neúplný.')
|
||||||
|
if records:
|
||||||
|
lines.append('Výpis môžeš zúžiť uvedením osoby, typu práce alebo konkrétneho roku.')
|
||||||
elif context.get('complete') is None:
|
elif context.get('complete') is None:
|
||||||
lines.append('Úplnosť vyhľadania nemožno zaručiť.')
|
lines.append('Úplnosť vyhľadania nemožno zaručiť.')
|
||||||
elif out['status'] == 'ambiguous':
|
elif out['status'] == 'ambiguous':
|
||||||
|
|||||||
@ -2,6 +2,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
@ -12,6 +13,7 @@ from .context_policy import answer_context_issues, citation_check, allowed_urls,
|
|||||||
from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections
|
from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections
|
||||||
from .answer_coverage import answer_coverage, coverage_notices
|
from .answer_coverage import answer_coverage, coverage_notices
|
||||||
from .presentation import content_lines, fact_sentences, no_answer_text, work_subject
|
from .presentation import content_lines, fact_sentences, no_answer_text, work_subject
|
||||||
|
from .content_selection import facet_lines, evidence_units, missing_facets, fit_candidates
|
||||||
|
|
||||||
|
|
||||||
class RagClaim(Claim):
|
class RagClaim(Claim):
|
||||||
@ -46,6 +48,8 @@ uveď start_year aj year a work_type. Citácia ani nesúvisiaci výňatok údaj
|
|||||||
Rešpektuj requirements.response_mode. Pri response_mode=quotes MUSÍ byť claims=[]:
|
Rešpektuj requirements.response_mode. Pri response_mode=quotes MUSÍ byť claims=[]:
|
||||||
pri všeobecnej otázke, téme, technológii, zozname ľudí alebo stáži vyber iba
|
pri všeobecnej otázke, téme, technológii, zozname ľudí alebo stáži vyber iba
|
||||||
evidence_quotes: source_id a presný doslovný výňatok, ktorý odpovedá na otázku.
|
evidence_quotes: source_id a presný doslovný výňatok, ktorý odpovedá na otázku.
|
||||||
|
Pri výbere zachovaj negácie, podmienky a výhrady zdroja. Nevytrhni tvrdenie z vety,
|
||||||
|
ktorá ho obmedzuje alebo spochybňuje.
|
||||||
Nevypĺňaj pri nich názvy prác, roky, tagy ani iné nežiadané metadáta do claims.
|
Nevypĺňaj pri nich názvy prác, roky, tagy ani iné nežiadané metadáta do claims.
|
||||||
Pri requirements.content=true otázka žiada obsah práce, nie iba jej identifikáciu.
|
Pri requirements.content=true otázka žiada obsah práce, nie iba jej identifikáciu.
|
||||||
Vyber opisujúce evidence_quotes z content_passages zodpovedajúcej osoby, typu a roku.
|
Vyber opisujúce evidence_quotes z content_passages zodpovedajúcej osoby, typu a roku.
|
||||||
@ -65,6 +69,11 @@ aplikácia uvedie, čo chýba. no_answer znamená žiadnu odpoveď a musí mať
|
|||||||
Pri zaznamenanom stave uprednostni konkrétnu implementáciu, natrénovanie modelu a vykonané
|
Pri zaznamenanom stave uprednostni konkrétnu implementáciu, natrénovanie modelu a vykonané
|
||||||
vyhodnotenie pred konzultáciami alebo samotným písaním textu. Ani priebežný stav nevydávaj
|
vyhodnotenie pred konzultáciami alebo samotným písaním textu. Ani priebežný stav nevydávaj
|
||||||
za splnenie všetkých cieľov alebo za konečné výsledky práce.
|
za splnenie všetkých cieľov alebo za konečné výsledky práce.
|
||||||
|
Ak sú dodané content_candidates, vyber quotes iba z ich textu. Každý kandidát uvádza
|
||||||
|
osobu, typ práce, rok, kind (assignment/task/progress/other) a facets, ktoré dokladá.
|
||||||
|
Pokry každú dostupnú požadovanú časť requirements.facets za každú osobu. Pre chýbajúcu časť
|
||||||
|
nevymýšľaj dôkaz: aplikácia ju výslovne označí. Čiastočná doložená odpoveď má status=ok.
|
||||||
|
Pri výsledkoch použi skutočný zaznamenaný stav; úlohy a plánované experimenty nie sú výsledky.
|
||||||
Každý výňatok musí byť JEDEN súvislý úsek z konkrétneho zdroja vrátane Markdownu.
|
Každý výňatok musí byť JEDEN súvislý úsek z konkrétneho zdroja vrátane Markdownu.
|
||||||
Nespájaj oddelené pasáže cez vynechané riadky; použi viac samostatných kratších quotes.
|
Nespájaj oddelené pasáže cez vynechané riadky; použi viac samostatných kratších quotes.
|
||||||
Vyber iba minimum potrebné na zodpovedanie otázky. Výňatok nesmieš parafrázovať ani meniť čísla.
|
Vyber iba minimum potrebné na zodpovedanie otázky. Výňatok nesmieš parafrázovať ani meniť čísla.
|
||||||
@ -127,7 +136,8 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
|||||||
issues.append('missing_required_evidence')
|
issues.append('missing_required_evidence')
|
||||||
if requirements.get('response_mode') == 'quotes' and claims:
|
if requirements.get('response_mode') == 'quotes' and claims:
|
||||||
issues.append('unrequested_structured_claims')
|
issues.append('unrequested_structured_claims')
|
||||||
if requirements.get('content') and answer['status'] == 'ok' and not quotes:
|
if (requirements.get('content') and answer['status'] == 'ok' and not quotes
|
||||||
|
and (not requirements.get('facets') or context.get('content_candidates'))):
|
||||||
issues.append('missing_content_evidence')
|
issues.append('missing_content_evidence')
|
||||||
requested = requirements['people']
|
requested = requirements['people']
|
||||||
types = requirements['work_types']
|
types = requirements['work_types']
|
||||||
@ -159,7 +169,12 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
|||||||
matching = [claim for claim in claims if claim['person'] == person]
|
matching = [claim for claim in claims if claim['person'] == person]
|
||||||
for key in ['title', 'year', 'start_year']:
|
for key in ['title', 'year', 'start_year']:
|
||||||
if requirements[key] and not any(claim.get(key) is not None for claim in matching):
|
if requirements[key] and not any(claim.get(key) is not None for claim in matching):
|
||||||
issues.append('missing_requested_' + key)
|
if key in requirements.get('available_fields',{}).get(person,[key]):
|
||||||
|
issues.append('missing_requested_' + key)
|
||||||
|
wanted=[key for key in ['title','year','start_year'] if requirements[key]]
|
||||||
|
if (wanted and not quotes and not any(key in fields for fields in requirements.get('available_fields',{}).values()
|
||||||
|
for key in wanted)):
|
||||||
|
issues.append('no_requested_fact_supported')
|
||||||
if answer['status'] == 'no_answer' and (requirements['academic'] or requirements['start_year']) and requested:
|
if answer['status'] == 'no_answer' and (requirements['academic'] or requirements['start_year']) and requested:
|
||||||
# Reject an unsupported refusal only if every requested academic tuple
|
# Reject an unsupported refusal only if every requested academic tuple
|
||||||
# is explicitly present. Never invent a value to fill an absent field.
|
# is explicitly present. Never invent a value to fill an absent field.
|
||||||
@ -174,6 +189,10 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
|||||||
and (not requirements['start_year'] or start_supported) and not requirements['unsupported_year']
|
and (not requirements['start_year'] or start_supported) and not requirements['unsupported_year']
|
||||||
and (not requirements.get('content') or context.get('content_passages'))):
|
and (not requirements.get('content') or context.get('content_passages'))):
|
||||||
issues.append('rejected_supported_academic_answer')
|
issues.append('rejected_supported_academic_answer')
|
||||||
|
if (not requirements.get('content') and not requirements['unsupported_year'] and any(
|
||||||
|
requirements[key] and key in fields for fields in requirements.get('available_fields',{}).values()
|
||||||
|
for key in ['title','year','start_year'])):
|
||||||
|
issues.append('rejected_supported_partial_fact')
|
||||||
if answer['status'] == 'no_answer' and requirements.get('content'):
|
if answer['status'] == 'no_answer' and requirements.get('content'):
|
||||||
query = normalized(context['question'])
|
query = normalized(context['question'])
|
||||||
overview = re.search(r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|opis\w*|popis\w*|obsah\w*|zhrn\w*)\b', query)
|
overview = re.search(r'\b(?:comu(?:\s+sa)?\s+ven(?:uj|ov)\w*|o\s+com|opis\w*|popis\w*|obsah\w*|zhrn\w*)\b', query)
|
||||||
@ -193,6 +212,10 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
|||||||
elif source.get('source_url') not in cited:
|
elif source.get('source_url') not in cited:
|
||||||
issues.append('missing_quote_source')
|
issues.append('missing_quote_source')
|
||||||
if source:
|
if source:
|
||||||
|
if context.get('candidate_only') and not any(c['source_id']==quote['source_id'] and
|
||||||
|
' '.join(quote['quote'].split()) in ' '.join(c['text'].split())
|
||||||
|
for c in context.get('content_candidates', [])):
|
||||||
|
issues.append('quote_outside_delivered_candidates')
|
||||||
if requirements.get('content'):
|
if requirements.get('content'):
|
||||||
titles = [row.get(key) for row in facts if row.get('source_id') == quote['source_id']
|
titles = [row.get(key) for row in facts if row.get('source_id') == quote['source_id']
|
||||||
for key in ('title', 'person', 'document_author')]
|
for key in ('title', 'person', 'document_author')]
|
||||||
@ -218,15 +241,16 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
|||||||
if requirements.get('content') and answer['status'] == 'ok':
|
if requirements.get('content') and answer['status'] == 'ok':
|
||||||
for person in requested:
|
for person in requested:
|
||||||
passages = [p for p in context.get('content_passages', []) if p.get('person') == person]
|
passages = [p for p in context.get('content_passages', []) if p.get('person') == person]
|
||||||
if requirements['response_mode'] == 'mixed':
|
if requirements['response_mode'] == 'mixed' and any(claim.get('person')==person for claim in claims):
|
||||||
passages = [p for p in passages if any(claim.get('person') == person and
|
passages = [p for p in passages if any(claim.get('person') == person and
|
||||||
all(claim.get(key) is None or claim[key] == p.get(key) for key in ['work_type', 'year'])
|
all(claim.get(key) is None or claim[key] == p.get(key) for key in ['work_type', 'year'])
|
||||||
for claim in claims)]
|
for claim in claims)]
|
||||||
if not any(quote['source_id'] == p['source_id'] and
|
eligible = any(c.get('person')==person for c in context.get('content_candidates', []))
|
||||||
|
if (not requirements.get('facets') or eligible) and not any(quote['source_id'] == p['source_id'] and
|
||||||
' '.join(quote['quote'].split()) in ' '.join(p['text'].split())
|
' '.join(quote['quote'].split()) in ' '.join(p['text'].split())
|
||||||
for p in passages for quote in quotes):
|
for p in passages for quote in quotes):
|
||||||
issues.append('missing_bound_content_evidence')
|
issues.append('missing_bound_content_evidence')
|
||||||
if requirements.get('distinguish_progress'):
|
if requirements.get('distinguish_progress') and not requirements.get('facets'):
|
||||||
bound_quotes = [quote for quote in quotes if any(
|
bound_quotes = [quote for quote in quotes if any(
|
||||||
quote['source_id'] == p['source_id'] and
|
quote['source_id'] == p['source_id'] and
|
||||||
' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) for p in passages)]
|
' '.join(quote['quote'].split()) in ' '.join(p['text'].split()) for p in passages)]
|
||||||
@ -235,7 +259,15 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
|
|||||||
for kind in requirements.get('content_roles', {}).get(person, []):
|
for kind in requirements.get('content_roles', {}).get(person, []):
|
||||||
if kind not in kinds:
|
if kind not in kinds:
|
||||||
issues.append('missing_' + kind + '_evidence')
|
issues.append('missing_' + kind + '_evidence')
|
||||||
if (requirements.get('distinguish_progress') and answer['status'] == 'no_answer' and requested
|
if requirements.get('facets'):
|
||||||
|
available={(c['person'],facet) for c in context.get('content_candidates',[]) for facet in c['facets']}
|
||||||
|
if answer['status']=='ok':
|
||||||
|
for person,facet in missing_facets(answer,context):
|
||||||
|
if (person,facet) in available:
|
||||||
|
issues.append('missing_'+('progress' if facet=='results' else facet)+'_evidence')
|
||||||
|
elif answer['status']=='no_answer' and available:
|
||||||
|
issues.append('rejected_supported_content_comparison')
|
||||||
|
if (requirements.get('distinguish_progress') and not requirements.get('facets') and answer['status'] == 'no_answer' and requested
|
||||||
and all(any(p.get('person') == person for p in context.get('content_passages', [])) for person in requested)):
|
and all(any(p.get('person') == person for p in context.get('content_passages', [])) for person in requested)):
|
||||||
issues.append('rejected_supported_content_comparison')
|
issues.append('rejected_supported_content_comparison')
|
||||||
if not citation_check(answer, allowed_urls(context))['citations_ok']:
|
if not citation_check(answer, allowed_urls(context))['citations_ok']:
|
||||||
@ -333,11 +365,29 @@ def comparison_lines(answer: dict, context: dict) -> list[str]:
|
|||||||
def render_rag_answer(answer: dict, context: dict) -> str:
|
def render_rag_answer(answer: dict, context: dict) -> str:
|
||||||
lines = []
|
lines = []
|
||||||
if answer['status'] == 'ok':
|
if answer['status'] == 'ok':
|
||||||
|
req=context.get('requirements',{})
|
||||||
for claim in answer.get('claims', []):
|
for claim in answer.get('claims', []):
|
||||||
lines.append(fact_sentences(claim, context.get('question', '')))
|
if req.get('facets'):
|
||||||
|
# Facet headings already identify the scoped work/year. Extra
|
||||||
|
# valid metadata selected by the model is not part of the answer.
|
||||||
|
fields=[key for key in ['title','year','start_year'] if req.get(key)]
|
||||||
|
if not req.get('title') and not req.get('start_year') and req.get('years'):
|
||||||
|
continue
|
||||||
|
displayed={key:value for key,value in claim.items() if key in ['person','work_type',*fields]}
|
||||||
|
if not any(displayed.get(key) is not None for key in fields):
|
||||||
|
continue
|
||||||
|
lines.append(fact_sentences(displayed,context.get('question','')))
|
||||||
|
else:
|
||||||
|
lines.append(fact_sentences(claim, context.get('question', '')))
|
||||||
|
for person in req.get('people',[]):
|
||||||
|
for key,label in [('title','Názov práce'),('year','Rok práce'),('start_year','Začiatok štúdia')]:
|
||||||
|
if req.get(key) and key not in req.get('available_fields',{}).get(person,[key]):
|
||||||
|
lines.append(f'{person} — {label.lower()} nie je v poskytnutých podkladoch doložený.')
|
||||||
quotes = answer.get('evidence_quotes', [])
|
quotes = answer.get('evidence_quotes', [])
|
||||||
style = context.get('requirements', {}).get('content_style')
|
style = context.get('requirements', {}).get('content_style')
|
||||||
if quotes and context.get('requirements', {}).get('distinguish_progress'):
|
if context.get('requirements', {}).get('facets'):
|
||||||
|
lines.extend(facet_lines(answer,context))
|
||||||
|
elif quotes and context.get('requirements', {}).get('distinguish_progress'):
|
||||||
lines.extend(comparison_lines(answer, context))
|
lines.extend(comparison_lines(answer, context))
|
||||||
elif quotes and style in {'overview', 'assignment'}:
|
elif quotes and style in {'overview', 'assignment'}:
|
||||||
# Bind each paragraph to the same source/person/work as the checked
|
# Bind each paragraph to the same source/person/work as the checked
|
||||||
@ -371,7 +421,10 @@ def render_rag_answer(answer: dict, context: dict) -> str:
|
|||||||
if answer.get('count') is not None:
|
if answer.get('count') is not None:
|
||||||
lines.append(f'Počet: {answer["count"]}.')
|
lines.append(f'Počet: {answer["count"]}.')
|
||||||
elif answer['status'] == 'no_answer':
|
elif answer['status'] == 'no_answer':
|
||||||
lines.append(no_answer_text(context))
|
if context.get('requirements', {}).get('facets') and context['requirements']['people']:
|
||||||
|
lines.extend(facet_lines(answer,context))
|
||||||
|
else:
|
||||||
|
lines.append(no_answer_text(context))
|
||||||
elif answer['status'] == 'ambiguous':
|
elif answer['status'] == 'ambiguous':
|
||||||
lines.append('Údaje sú nejednoznačné; požiadavku treba spresniť.')
|
lines.append('Údaje sú nejednoznačné; požiadavku treba spresniť.')
|
||||||
else:
|
else:
|
||||||
@ -384,6 +437,18 @@ def render_rag_answer(answer: dict, context: dict) -> str:
|
|||||||
|
|
||||||
def generate_verified_rag(question: str, payload: dict, complete) -> dict:
|
def generate_verified_rag(question: str, payload: dict, complete) -> dict:
|
||||||
context = rag_evidence_context(question, payload)
|
context = rag_evidence_context(question, payload)
|
||||||
|
if payload.get('status')=='ambiguous' or context.get('clarification'):
|
||||||
|
return {'status':'ambiguous','reason':payload.get('reason') or 'ambiguous_request',
|
||||||
|
'accepted_for_delivery':True,'answer':context.get('clarification') or 'Spresni osobu, typ práce alebo rok.',
|
||||||
|
'claims':[],'source_urls':[],'attempts':[],'complete':False,'truncated':False,
|
||||||
|
'truncation_reasons':[],'index_version':context.get('index_version')}
|
||||||
|
if (context['requirements'].get('facets') and context['requirements']['response_mode']=='quotes'
|
||||||
|
and not context.get('content_candidates') and context['requirements']['people']):
|
||||||
|
empty={'status':'no_answer','claims':[],'evidence_quotes':[],
|
||||||
|
'source_urls':[s['source_url'] for s in context['sources'] if s.get('source_url')]}
|
||||||
|
return {**empty,'reason':'requested_aspects_not_found','answer':render_rag_answer(empty,context),
|
||||||
|
'accepted_for_delivery':True,'attempts':[],'answer_coverage':answer_coverage(empty,context),
|
||||||
|
**{k:context.get(k) for k in ['index_version','complete','truncated','truncation_reasons']}}
|
||||||
if not payload.get('sources'):
|
if not payload.get('sources'):
|
||||||
status = 'error' if payload.get('status') == 'error' else 'no_answer'
|
status = 'error' if payload.get('status') == 'error' else 'no_answer'
|
||||||
empty = {'status': status, 'claims': [], 'evidence_quotes': [], 'source_urls': []}
|
empty = {'status': status, 'claims': [], 'evidence_quotes': [], 'source_urls': []}
|
||||||
@ -395,11 +460,18 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict:
|
|||||||
sent = {key: context[key] for key in ['status', 'reason', 'complete', 'truncated',
|
sent = {key: context[key] for key in ['status', 'reason', 'complete', 'truncated',
|
||||||
'truncation_reasons', 'index_version', 'sources', 'facts', 'requirements']}
|
'truncation_reasons', 'index_version', 'sources', 'facts', 'requirements']}
|
||||||
if context['requirements'].get('content'):
|
if context['requirements'].get('content'):
|
||||||
sent['content_passages'] = context['content_passages']
|
if context['requirements'].get('facets'):
|
||||||
|
sent['content_candidates']=context['content_candidates']
|
||||||
|
context['candidate_only']=True
|
||||||
|
else:
|
||||||
|
sent['content_passages'] = context['content_passages']
|
||||||
elif not context['requirements']['academic']:
|
elif not context['requirements']['academic']:
|
||||||
sent['context'] = payload.get('context', '')
|
sent['context'] = payload.get('context', '')
|
||||||
if context['requirements']['response_mode'] == 'quotes':
|
if context['requirements']['response_mode'] == 'quotes':
|
||||||
sent.pop('facts')
|
sent.pop('facts')
|
||||||
|
if context['requirements'].get('facets'):
|
||||||
|
fit_candidates(sent,context,payload.get('context_budget',{}).get('max_tokens',12000),
|
||||||
|
os.getenv('RAG_LLM_TOKEN_ENCODING','cl100k_base'))
|
||||||
messages = [{'role': 'system', 'content': PROMPT + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)},
|
messages = [{'role': 'system', 'content': PROMPT + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)},
|
||||||
{'role': 'user', 'content': json.dumps({'question': question, 'context': sent}, ensure_ascii=False)}]
|
{'role': 'user', 'content': json.dumps({'question': question, 'context': sent}, ensure_ascii=False)}]
|
||||||
attempts = []
|
attempts = []
|
||||||
@ -420,6 +492,12 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict:
|
|||||||
'answer': 'Jazykový model nie je dostupný; overenú odpoveď sa nepodarilo pripraviť.', 'attempts': attempts}
|
'answer': 'Jazykový model nie je dostupný; overenú odpoveď sa nepodarilo pripraviť.', 'attempts': attempts}
|
||||||
record['issues'] = issues
|
record['issues'] = issues
|
||||||
if not issues:
|
if not issues:
|
||||||
|
if context['requirements'].get('facets') and missing_facets(answer,context):
|
||||||
|
# Sources inspected for a missing requested aspect remain
|
||||||
|
# traceable, even when only the other person's quotes exist.
|
||||||
|
extra=[s['source_url'] for s in context['sources'] if s.get('title') in
|
||||||
|
{person for person,_ in missing_facets(answer,context)} and s.get('source_url')]
|
||||||
|
answer={**answer,'source_urls':list(dict.fromkeys([*answer['source_urls'],*extra]))}
|
||||||
text = render_rag_answer(answer, context)
|
text = render_rag_answer(answer, context)
|
||||||
return {'status': answer['status'], 'reason': None, 'accepted_for_delivery': True,
|
return {'status': answer['status'], 'reason': None, 'accepted_for_delivery': True,
|
||||||
'answer': text, 'source_urls': answer['source_urls'], 'claims': answer['claims'],
|
'answer': text, 'source_urls': answer['source_urls'], 'claims': answer['claims'],
|
||||||
|
|||||||
@ -12,12 +12,20 @@ from tools.answer_evaluation.graph_context import graph_context
|
|||||||
from tools.answer_evaluation.checked_generation import generate_checked_answer
|
from tools.answer_evaluation.checked_generation import generate_checked_answer
|
||||||
from tools.answer_evaluation.verified_rag import generate_verified_rag
|
from tools.answer_evaluation.verified_rag import generate_verified_rag
|
||||||
from tools.answer_evaluation.rag_evidence import expand_comparison_sources
|
from tools.answer_evaluation.rag_evidence import expand_comparison_sources
|
||||||
|
from tools.answer_evaluation.rag_preparation import prepare_named_content
|
||||||
|
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
|
||||||
from tools.sync.index_state import index_snapshot
|
from tools.sync.index_state import index_snapshot
|
||||||
from tools.answers.model_client import complete_with_configured_model
|
from tools.answers.model_client import complete_with_configured_model
|
||||||
|
from tools.answers.answer_errors import public_failure
|
||||||
|
|
||||||
_capacity = threading.BoundedSemaphore(2)
|
_capacity = threading.BoundedSemaphore(2)
|
||||||
|
|
||||||
|
|
||||||
|
def retrieval_failure(engine: str) -> dict:
|
||||||
|
return public_failure({'reason':'retrieval_unavailable','engine':engine,'index_version':None,
|
||||||
|
'complete':False,'truncated':False,'truncation_reasons':[]})
|
||||||
|
|
||||||
|
|
||||||
def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
|
def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
|
||||||
complete=None, db_file: Path = DB_FILE, include_diagnostics: bool = False) -> dict:
|
complete=None, db_file: Path = DB_FILE, include_diagnostics: bool = False) -> dict:
|
||||||
if not _capacity.acquire(blocking=False):
|
if not _capacity.acquire(blocking=False):
|
||||||
@ -25,15 +33,32 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
|
|||||||
'accepted_for_delivery': False, 'answer': 'Agent práve spracúva iné odpovede. Skúste to o chvíľu.'}
|
'accepted_for_delivery': False, 'answer': 'Agent práve spracúva iné odpovede. Skúste to o chvíľu.'}
|
||||||
try:
|
try:
|
||||||
model = complete or complete_with_configured_model
|
model = complete or complete_with_configured_model
|
||||||
|
unsupported=unsupported_constraint(question)
|
||||||
|
if unsupported:
|
||||||
|
return {'status':'ambiguous','reason':'unsupported_constraint','accepted_for_delivery':True,
|
||||||
|
'answer':constraint_message(unsupported),'claims':[],'source_urls':[],
|
||||||
|
'index_version':None,'complete':False,'truncated':False,'truncation_reasons':[], 'engine':engine}
|
||||||
if engine == 'rag':
|
if engine == 'rag':
|
||||||
with index_snapshot(db_file.parent, db_file=db_file):
|
try:
|
||||||
retrieved = build_rag_context(db_file, question, limit=limit)
|
with index_snapshot(db_file.parent, db_file=db_file):
|
||||||
retrieved = expand_comparison_sources(db_file, question, retrieved)
|
retrieved = build_rag_context(db_file, question, limit=limit)
|
||||||
retrieved = compact_rag_payload(retrieved)
|
retrieved = expand_comparison_sources(db_file, question, retrieved)
|
||||||
|
retrieved = prepare_named_content(db_file, question, retrieved)
|
||||||
|
retrieved = compact_rag_payload(retrieved)
|
||||||
|
except Exception:
|
||||||
|
return retrieval_failure(engine)
|
||||||
result = generate_verified_rag(question, retrieved, model)
|
result = generate_verified_rag(question, retrieved, model)
|
||||||
context = retrieved
|
context = retrieved
|
||||||
elif engine == 'graphrag':
|
elif engine == 'graphrag':
|
||||||
context = graph_context(search_graph(question))
|
try:
|
||||||
|
context = graph_context(search_graph(question))
|
||||||
|
except Exception:
|
||||||
|
return retrieval_failure(engine)
|
||||||
|
if context.get('clarification'):
|
||||||
|
return {'status':'ambiguous','reason':context.get('reason'),'accepted_for_delivery':True,
|
||||||
|
'answer':context['clarification'],'claims':[],'source_urls':[], 'engine':engine,
|
||||||
|
'complete':False,'truncated':False,'truncation_reasons':[],
|
||||||
|
'index_version':context.get('index_version')}
|
||||||
prompt = (Path(__file__).resolve().parents[1] / 'answer_evaluation/graph_answer_prompt.txt').read_text()
|
prompt = (Path(__file__).resolve().parents[1] / 'answer_evaluation/graph_answer_prompt.txt').read_text()
|
||||||
generated = generate_checked_answer(question, context, prompt, model)
|
generated = generate_checked_answer(question, context, prompt, model)
|
||||||
if generated.get('accepted_for_delivery'):
|
if generated.get('accepted_for_delivery'):
|
||||||
@ -42,8 +67,10 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5,
|
|||||||
'truncated': context.get('truncated'), 'truncation_reasons': context.get('truncation_reasons'),
|
'truncated': context.get('truncated'), 'truncation_reasons': context.get('truncation_reasons'),
|
||||||
'attempts': generated['attempts']}
|
'attempts': generated['attempts']}
|
||||||
else:
|
else:
|
||||||
result = {'status': 'error', 'reason': 'answer_rejected', 'accepted_for_delivery': False,
|
reason=generated.get('reason','answer_rejected')
|
||||||
'answer': 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.',
|
result = {'status': 'error', 'reason': reason, 'accepted_for_delivery': False,
|
||||||
|
'answer': ('Jazykový model nie je dostupný. Skús otázku neskôr.' if reason=='model_unavailable' else
|
||||||
|
'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.'),
|
||||||
'index_version': context.get('index_version'), 'attempts': generated['attempts']}
|
'index_version': context.get('index_version'), 'attempts': generated['attempts']}
|
||||||
else:
|
else:
|
||||||
raise ValueError('Unsupported answer engine')
|
raise ValueError('Unsupported answer engine')
|
||||||
|
|||||||
18
tools/answers/answer_errors.py
Normal file
18
tools/answers/answer_errors.py
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
"""Public failure wording; transport/model exception text is never a UI message."""
|
||||||
|
MESSAGES={
|
||||||
|
'model_unavailable':'Jazykový model momentálne nie je dostupný. Skús otázku neskôr.',
|
||||||
|
'retrieval_unavailable':'Vyhľadávací index momentálne nie je dostupný. Skús otázku neskôr.',
|
||||||
|
'answer_capacity_exceeded':'Agent práve spracúva iné otázky. Skús to o chvíľu.',
|
||||||
|
'answer_rejected':'Odpoveď sa nepodarilo spoľahlivo overiť podľa podkladov. '
|
||||||
|
'Skús spresniť osobu, typ práce alebo rok; neoverené údaje sa nezobrazia.',
|
||||||
|
'answer_unavailable':'Overenú odpoveď sa momentálne nepodarilo pripraviť. Skús to neskôr.',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def public_failure(result: dict) -> dict:
|
||||||
|
reason=result.get('reason')
|
||||||
|
if not isinstance(reason,str) or reason not in MESSAGES:
|
||||||
|
reason='answer_unavailable'
|
||||||
|
return {'index_version':None,'complete':False,'truncated':False,'truncation_reasons':[],
|
||||||
|
**result,'status':'error','reason':reason,'accepted_for_delivery':False,
|
||||||
|
'answer':MESSAGES[reason],'claims':[],'source_urls':[]}
|
||||||
@ -46,6 +46,8 @@ def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]:
|
|||||||
'tools/answer_evaluation/render_answer.py', 'tools/answer_evaluation/graph_context.py',
|
'tools/answer_evaluation/render_answer.py', 'tools/answer_evaluation/graph_context.py',
|
||||||
'tools/answer_evaluation/presentation.py',
|
'tools/answer_evaluation/presentation.py',
|
||||||
'tools/answer_evaluation/rag_evidence.py',
|
'tools/answer_evaluation/rag_evidence.py',
|
||||||
|
'tools/answer_evaluation/question_intent.py', 'tools/answer_evaluation/content_selection.py',
|
||||||
|
'tools/answer_evaluation/rag_preparation.py', 'tools/answer_evaluation/answer_coverage.py',
|
||||||
'tools/answer_evaluation/graph_answer_prompt.txt',
|
'tools/answer_evaluation/graph_answer_prompt.txt',
|
||||||
'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py'])
|
'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py'])
|
||||||
files = sorted(set(files))
|
files = sorted(set(files))
|
||||||
|
|||||||
@ -8,6 +8,7 @@ from typing import Any, Iterable
|
|||||||
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION
|
from tools.retrieval.publication import DEFAULT_PUBLISHED_ONLY, PUBLICATION_POLICY_VERSION
|
||||||
from .publication import public_query
|
from .publication import public_query
|
||||||
from tools.retrieval.retrieval_contract import result_fields, source_references, error_result
|
from tools.retrieval.retrieval_contract import result_fields, source_references, error_result
|
||||||
|
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
|
||||||
|
|
||||||
|
|
||||||
GRAPH_SCOPE = "zpwiki"
|
GRAPH_SCOPE = "zpwiki"
|
||||||
@ -2334,6 +2335,12 @@ def search_graph_with_session(
|
|||||||
raise ValueError("max_entities musí byť aspoň 1")
|
raise ValueError("max_entities musí byť aspoň 1")
|
||||||
|
|
||||||
constraints = extract_query_constraints(clean_query)
|
constraints = extract_query_constraints(clean_query)
|
||||||
|
unsupported = unsupported_constraint(clean_query)
|
||||||
|
if unsupported:
|
||||||
|
return {**result_fields(status='ambiguous',sources=[],reason='unsupported_constraint',complete=False),
|
||||||
|
'query':clean_query,'constraints':constraints,'clarification':constraint_message(unsupported),
|
||||||
|
'facts':[],'paths':[],'documents':[],'aggregation':None,'resolved_entities':[],
|
||||||
|
'checked_entities':[],'rejection_evidence':[],'strategies':[],'reasons':['unsupported_constraint']}
|
||||||
catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE, published_only=published_only)
|
catalog = run_rows(session, CATALOG_QUERY, scope=GRAPH_SCOPE, published_only=published_only)
|
||||||
constraints['published_only'] = published_only
|
constraints['published_only'] = published_only
|
||||||
# Explicit tag filters are resolved separately. Incidental names such as the
|
# Explicit tag filters are resolved separately. Incidental names such as the
|
||||||
|
|||||||
@ -11,7 +11,7 @@ from tools.retrieval.text_budget import split_to_fit
|
|||||||
SOURCE_FIELDS = ('source_id', 'title', 'author', 'document_path', 'source_url', 'published',
|
SOURCE_FIELDS = ('source_id', 'title', 'author', 'document_path', 'source_url', 'published',
|
||||||
'categories', 'tags', 'section')
|
'categories', 'tags', 'section')
|
||||||
FIELDS = ('status', 'reason', 'query', 'engine', 'strategies', 'instructions', 'answer_format',
|
FIELDS = ('status', 'reason', 'query', 'engine', 'strategies', 'instructions', 'answer_format',
|
||||||
'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count')
|
'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count', 'clarification', 'named_subjects')
|
||||||
|
|
||||||
|
|
||||||
def compact_rag_payload(data: dict, *, max_tokens: int | None = None,
|
def compact_rag_payload(data: dict, *, max_tokens: int | None = None,
|
||||||
|
|||||||
@ -709,6 +709,13 @@ def build_rag_context(
|
|||||||
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
published_only: bool = DEFAULT_PUBLISHED_ONLY,
|
||||||
max_per_document: int = 1,
|
max_per_document: int = 1,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
from tools.retrieval.question_constraints import unsupported_constraint, constraint_message
|
||||||
|
unsupported=unsupported_constraint(query)
|
||||||
|
if unsupported:
|
||||||
|
return {**result_fields(status='ambiguous',sources=[],reason='unsupported_constraint',complete=False),
|
||||||
|
'query':query,'engine':'hybrid','strategies':[],'source_count':0,'sources':[],
|
||||||
|
'context':'','clarification':constraint_message(unsupported),
|
||||||
|
'instructions':RAG_INSTRUCTIONS,'answer_format':ANSWER_FORMAT}
|
||||||
response = search_database(
|
response = search_database(
|
||||||
db_path,
|
db_path,
|
||||||
query,
|
query,
|
||||||
|
|||||||
42
tools/retrieval/question_constraints.py
Normal file
42
tools/retrieval/question_constraints.py
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
"""Fail closed for recognized query operators without an implemented meaning."""
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
|
||||||
|
def requested_years(question: str) -> list[int]:
|
||||||
|
"""An unquoted four-digit year remains a condition without 'v roku'."""
|
||||||
|
text=re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question)
|
||||||
|
return list(dict.fromkeys(int(value) for value in re.findall(r'\b\d{4}\b',text)))
|
||||||
|
|
||||||
|
|
||||||
|
def unsupported_constraint(question: str) -> str | None:
|
||||||
|
text = ''.join(c for c in unicodedata.normalize('NFKD', question).casefold()
|
||||||
|
if not unicodedata.combining(c))
|
||||||
|
# Do not treat words inside quoted academic titles as query operators.
|
||||||
|
text = re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', text)
|
||||||
|
if (re.search(r'\b(?:rok\w*|datum\w*|kedy)\b',text) and
|
||||||
|
re.search(r'\b(?:obhajob\w*|obhajil\w*|dokonc\w*|ukoncil\w*|zacal\w* pracovat)\b',text) and
|
||||||
|
re.search(r'\b(?:prac\w*|projekt\w*)\b',text)):
|
||||||
|
return 'work_event_date'
|
||||||
|
if re.search(r'\b(?:od|do|pred|po|medzi)\s+(?:rok(?:om|mi|u)?\s+)?\d{4}\b|\b\d{4}\s*[-–]\s*\d{4}\b', text):
|
||||||
|
return 'year_range'
|
||||||
|
if re.search(r'\b(?:aspon|najviac|viac ako|menej ako|minimalne|maximalne)\s+\d+', text):
|
||||||
|
return 'quantity_condition'
|
||||||
|
if re.search(r'\b(?:okrem|s vynimkou)\b|\b(?:nema|nemaju)\s+(?:tag|klucov|kategori|prac|projekt)|'
|
||||||
|
r'\bbez\s+(?:tagu|klucoveho slova|kategorie)\b', text):
|
||||||
|
return 'exclusion'
|
||||||
|
if re.search(r'\b(?:tag\w*|klucov\w*\s+slov\w*)\s+\w+\s+alebo\s+\w+', text):
|
||||||
|
return 'alternative_tags'
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def constraint_message(code: str) -> str:
|
||||||
|
return {
|
||||||
|
'work_event_date':'Rok uvedený pri práci nepreukazuje dátum jej začatia, dokončenia alebo obhajoby. '
|
||||||
|
'Chceš zistiť rok evidovaný pri práci?',
|
||||||
|
'year_range': 'Časovú podmienku neviem spoľahlivo vyhodnotiť ako interval. Uveď konkrétny rok práce.',
|
||||||
|
'quantity_condition': 'Podmienku na minimálny alebo maximálny počet neviem spoľahlivo vyhodnotiť. '
|
||||||
|
'Požiadaj najprv o konkrétny zoznam alebo počet.',
|
||||||
|
'exclusion': 'Vylučovaciu podmienku neviem spoľahlivo vyhodnotiť. Uveď, ktoré vlastnosti majú výsledky spĺňať.',
|
||||||
|
'alternative_tags': 'Uveď, či požaduješ oba tagy zároveň, alebo vyhľadávanie pre každý tag samostatne.',
|
||||||
|
}[code]
|
||||||
Loading…
Reference in New Issue
Block a user