jp170na/dp-zp-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Opravy RAG odpovedí

Browse Source
This commit is contained in:
Ján Pták 2026-10-27 03:42:44 +01:00
parent 084d04d867
commit fbf4266e77
8 changed files with 251 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@ -1156,6 +1156,11 @@ Lokálne testy:
.venv/bin/python scripts/verify_project.py --git-network
```
Bez vytvárania reportov a logových súborov pridajte `--no-save`.
Výsledky kontrol sa vtedy zobrazia iba v termináli. Overenie zahŕňa aj
skutočné tokenové limity všetkých chunkov a embeddingových segmentov,
zachovanie obsahu pri segmentácii a úplnosť uložených vektorov.
Testy používajú skutočný dočasný HTTP server na loopback rozhraní a
FastAPI s riadenými dátami. Overujú autorizáciu, zachovanie otázky,
zdrojov a úplnosti, chyby a timeouty, zákaz presmerovaní aj import JSON/ZIP
@ -1436,6 +1441,13 @@ explicitný dôkaz roku práce. Táto neistota je uvedená v datasete.
.venv/bin/python -m tools.answer_evaluation.evaluate_verified_rag --mode replay --input <report.json>
```
Aj evaluačný runner podporuje `--no-save`; v tomto režime otázky a odpovede
spracuje v pamäti a vypíše skóre bez zápisu reportu či deklarácie.
Kontrolovaná cesta používa pri všeobecných otázkach iba doslovné výňatky
bez nežiadaných akademických tvrdení. Referenčný odkaz v citovanom texte
sa prijme iba vtedy, keď je doložený tým istým dodaným zdrojom; položky
`source_urls` naďalej označujú pôvodné wiki zdroje.
Runner posiela na backend iba otázku a parametre; očakávané odpovede model
nedostane. Vyhodnocuje finálny stav, prijatie odpovede, citácie, úplné
väzby osoba–práca–typ–rok a pokrytie očakávaných údajov vo výňatkoch.

79
scripts/verify_project.py
View File

@ -16,6 +16,7 @@ import sqlite3
import subprocess
import sys
import time
import tempfile
import types
import urllib.request
import zipfile
@ -66,7 +67,7 @@ def local_test_worker() -> int:
socket.socket.connect, socket.socket.connect_ex = guarded_connect, guarded_connect_ex
socket.getaddrinfo = guarded_dns
import pytest
return pytest.main(['-q', 'test', 'tools/rag/tests', 'tools/graphrag/tests'])
return pytest.main(['-q', '-p', 'no:cacheprovider', 'test', 'tools/rag/tests', 'tools/graphrag/tests'])
def verify_code() -> dict:
@ -99,6 +100,44 @@ def verify_database() -> dict:
return counts
def verify_chunk_limits() -> dict:
import numpy as np
from tools.rag.build_chunks import TokenCounter
from tools.rag.embedding_utils import build_embedding_text, split_passage, passage_token_count, get_embedding_model, embedding_model_name
chunks = json.loads((ROOT/'data/chunks.json').read_text())
if isinstance(chunks, dict):
chunks = chunks['chunks']
counter = TokenCounter(os.getenv('CHUNK_TOKEN_ENCODING', 'cl100k_base'))
limit = int(os.getenv('CHUNK_MAX_TOKENS', '450'))
model = get_embedding_model(embedding_model_name())
maximum = 0
expected_segments = set()
with sqlite3.connect((ROOT/'data/zp_index.sqlite').resolve().as_uri()+'?mode=ro',uri=True) as conn:
stored = dict(conn.execute('SELECT chunk_id,text FROM chunks'))
assert len(stored) == len(chunks)
for chunk in chunks:
assert stored[chunk['chunk_id']] == chunk['text']
size = counter.count(chunk['text'])
maximum = max(maximum, size)
assert size <= limit, 'Oversized source chunk'
original = build_embedding_text(chunk)
parts = split_passage(original)
assert ''.join(parts) == original, 'Embedding segmentation loses content'
for index, part in enumerate(parts):
assert passage_token_count(part) <= model.max_seq_length, 'Oversized embedding input'
expected_segments.add((chunk['chunk_id'],index))
actual_segments = set()
for cid,index,dimensions,blob in conn.execute('SELECT chunk_id,segment_index,dimensions,embedding FROM chunk_embeddings'):
vector = np.frombuffer(blob,dtype=np.float32)
assert len(vector)==dimensions and np.isfinite(vector).all()
assert np.isclose(np.linalg.norm(vector),1.0,atol=0.001), 'Invalid embedding vector'
actual_segments.add((cid,index))
assert actual_segments == expected_segments, 'Embedding coverage mismatch'
return {'chunks':len(chunks),'max_chunk_tokens':maximum,'chunk_limit':limit,
'embedding_segments':len(actual_segments),'embedding_limit':model.max_seq_length,
'source_content_preserved':True}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--docker-context', default='default')
@ -106,6 +145,7 @@ def main() -> int:
parser.add_argument('--wait-ready-seconds', type=int, default=90,
help='Bounded wait for API/model warm-up after Docker startup')
parser.add_argument('--output', type=Path)
parser.add_argument('--no-save', action='store_true', help='Check in memory without writing reports or logs')
parser.add_argument('--test-worker', action='store_true', help=argparse.SUPPRESS)
args = parser.parse_args()
os.chdir(ROOT)
@ -113,8 +153,11 @@ def main() -> int:
if args.test_worker:
return local_test_worker()
from tools.runtime.report_files import report_output, report_log, retention_plan, apply_moves
args.output = report_output(args.output, 'verification')
if args.no_save and args.output:
parser.error('--no-save cannot be combined with --output')
args.output = None if args.no_save else report_output(args.output, 'verification')
report = {'checks': {}, 'external_llm_calls': False}
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
def check(name, action):
@ -126,6 +169,7 @@ def main() -> int:
record = {'passed': False, 'error_type': type(error).__name__}
record['seconds'] = round(time.monotonic() - started, 2)
report['checks'][name] = record
if args.output:
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
print(f'{name}: {"OK" if record["passed"] else "FAILED"}', flush=True)
@ -163,6 +207,7 @@ def main() -> int:
check('api_and_both_indexes', health)
check('sqlite_integrity', verify_database)
check('chunk_and_embedding_limits', verify_chunk_limits)
check('container_readiness', lambda: command(*compose, 'exec', '-T', 'zp-agent-api',
'python', '-m', 'tools.runtime.runtime_health'))
if args.git_network:
@ -183,11 +228,11 @@ def main() -> int:
assert key, 'Search API key is missing'
built = []
async def run():
async def run(package_dir):
graph_tool = None
versions = []
for engine in ('rag', 'graphrag'):
package = build_package(engine, ROOT / 'data/tool_packages')
package = build_package(engine, package_dir)
with zipfile.ZipFile(package['archive']) as archive:
form = json.loads(archive.read('openwebui-tool.json'))[0]
module = types.ModuleType('verify_packaged_' + engine)
@ -213,27 +258,41 @@ def main() -> int:
assert negative['status'] == 'no_answer' and not negative['facts']
graph_tool.valves.BACKEND_API_KEY = 'invalid-test-key'
assert json.loads(await graph_tool.retrieve_zpwiki_graph('Práca?'))['reason'] == 'unauthorized'
asyncio.run(run())
if args.no_save:
with tempfile.TemporaryDirectory(prefix='zp-verify-packages-') as folder:
asyncio.run(run(Path(folder)))
else:
asyncio.run(run(ROOT / 'data/tool_packages'))
return {'packages': built, 'live_retrieval': True, 'same_data_version': True,
'negative_query': True, 'authorization': True, 'external_llm_calls': False}
check('openwebui_tool_packages', tool_packages)
def tests():
log = report_log(args.output)
environment = {**os.environ}
environment.pop('OPENWEBUI_API_KEY', None)
environment['PYTHONDONTWRITEBYTECODE'] = '1'
command = [sys.executable, str(Path(__file__).resolve()), '--test-worker']
if args.no_save:
result = subprocess.run(command, env=environment, capture_output=True, text=True, timeout=900)
text = result.stdout + result.stderr
if result.returncode:
print(text[-16000:], flush=True)
log = None
else:
log = report_log(args.output)
with log.open('w') as output:
result = subprocess.run([sys.executable, str(Path(__file__).resolve()), '--test-worker'],
env=environment, stdout=output, stderr=subprocess.STDOUT, timeout=900)
assert result.returncode == 0, 'Regression failed; see local pytest log'
result = subprocess.run(command, env=environment, stdout=output, stderr=subprocess.STDOUT, timeout=900)
text = log.read_text()
assert result.returncode == 0, 'Regression failed; see console or selected pytest log'
counts = {name: int(value) for value, name in re.findall(r'(\d+) (passed|failed|skipped)', text)}
assert counts.get('passed', 0) > 0 and counts.get('skipped', 0) == 0
return {**counts, 'log': str(log), 'external_network_blocked': True}
print('Regression: ' + str(counts), flush=True)
return {**counts, 'log': str(log) if log else None, 'external_network_blocked': True}
check('full_local_regression', tests)
report['passed'] = all(item['passed'] for item in report['checks'].values())
if args.output:
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
moves = retention_plan()
if moves:

75
test/test_verified_rag_extended.py
View File

@ -130,3 +130,78 @@ def test_evaluator_accepts_extra_supported_project_but_rejects_wrong_rendered_ye
assert evaluate_case(case,p,delivered)['passed']
delivered['answer'] += '\nPráca bola ukončená v roku 2099.'
assert 'delivered_text_differs_from_verified_renderer' in evaluate_case(case,p,delivered)['issues']
@pytest.mark.parametrize('reference', ['https://library.example/docs', '/topics/library',
'../manual', '#installation', '[guide]: /guide'])
def test_literal_source_references_are_allowed_without_replacing_wiki_citation(reference):
from tools.answer_evaluation.verified_rag import quoted_reference_urls, render_rag_answer
from tools.answer_evaluation.context_policy import citation_check, allowed_urls
quote = ('Dokumentácia: '+reference if not reference.startswith('[guide]') else reference)
if reference.startswith(('/', '.', '#')):
quote = '[Dokumentácia]('+reference+')'
c = rag_evidence_context('Aká dokumentácia je uvedená?',payload(quote))
a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'],
'evidence_quotes':[{'source_id':'S1','quote':quote}]}
assert rag_answer_issues(a,c) == []
rendered = {**a,'answer':render_rag_answer(a,c)}
assert citation_check(rendered,allowed_urls(c)|quoted_reference_urls(a,c))['citations_ok']
refs = quoted_reference_urls(a,c)
assert refs
a['source_urls'].extend(refs)
assert 'unsupported_citation' in rag_answer_issues(a,c)
@pytest.mark.parametrize('quote', ['Pozri https://unknown.example/doc',
'[manuál](/unknown)', '<a href="/unknown">manuál</a>'])
def test_quote_cannot_borrow_an_unknown_or_other_source_reference(quote):
p = payload('Presný obsah bez odkazu.')
p['sources'].append({**p['sources'][0],'source_id':'S2','title':'Iný dokument',
'source_url':'https://example.test/other','text':quote})
p['context'] = build_context_text(p['sources'])
c = rag_evidence_context('Aký je obsah dokumentu?',p)
a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'],
'evidence_quotes':[{'source_id':'S1','quote':quote}]}
assert 'unsupported_quote_citation' in rag_answer_issues(a,c)
def test_active_url_schemes_and_noncontiguous_quotes_are_rejected():
c = rag_evidence_context('Aký je obsah dokumentu?',payload('[odkaz](javascript:alert(1))\nPrvý fakt.\nMedziľahlý text.\nDruhý fakt.'))
a = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'],
'evidence_quotes':[{'source_id':'S1','quote':'[odkaz](javascript:alert(1))'}]}
assert 'unsupported_quote_citation' in rag_answer_issues(a,c)
a['evidence_quotes']=[{'source_id':'S1','quote':'Prvý fakt.\nDruhý fakt.'}]
assert 'unsupported_evidence_quote' in rag_answer_issues(a,c)
def test_general_question_repairs_unrequested_claims_and_sends_only_quote_evidence():
p = payload('Sekcia: Diplomová práca 2025\nNázov: Overené modelovanie textu\nPoužíva Python.')
bad = {'status':'ok','answer':'','claims':[{'person':'Alena Nová','work_type':'diploma_thesis','year':2025}],
'evidence_quotes':[{'source_id':'S1','quote':'Používa Python.'}],
'source_urls':['https://example.test/alena']}
good = {**bad,'claims':[]}
outputs = [bad, good]; calls=[]
def complete(messages):
calls.append(json.loads(messages[1]['content']))
return {'choices':[{'message':{'content':json.dumps(outputs[len(calls)-1])}}]}
delivered = generate_verified_rag('Aký jazyk používa Alena Nová?',p,complete)
assert delivered['accepted_for_delivery'] and len(calls)==2
assert 'unrequested_structured_claims' in delivered['attempts'][0]['issues']
assert all('facts' not in call['context'] for call in calls)
assert delivered['claims']==[] and 'Používa Python.' in delivered['answer']
def test_evaluator_accepts_exact_quoted_reference_but_rejects_invented_rendered_link():
from tools.answer_evaluation.verified_rag import render_rag_answer
p = payload('Knižnica Library: https://library.example/docs')
question = 'Aká knižnica je uvedená?'
c = rag_evidence_context(question,p)
parsed = {'status':'ok','claims':[],'source_urls':['https://example.test/alena'],
'evidence_quotes':[{'source_id':'S1','quote':'Knižnica Library: https://library.example/docs'}]}
delivered = {**parsed,'accepted_for_delivery':True,'answer':render_rag_answer(parsed,c),
'attempts':[{'parsed_answer':parsed,'issues':[]}]}
case = {'id':'reference','category':'specific_fact','question':question,'should_answer':True,
'expected_bindings':[],'expected_source_urls':parsed['source_urls'],'expected_answer_contains':['Library']}
assert evaluate_case(case,p,delivered)['passed']
delivered['answer']+=' https://invented.example/'
assert 'delivered_unsupported_citation' in evaluate_case(case,p,delivered)['issues']

17
tools/answer_evaluation/evaluate_verified_rag.py
View File

@ -19,7 +19,7 @@ from dotenv import load_dotenv
from tools.answer_evaluation.context_policy import normalized, citation_check, allowed_urls
from tools.answer_evaluation.rag_evidence import rag_evidence_context
from tools.answer_evaluation.verified_rag import rag_answer_issues, render_rag_answer
from tools.answer_evaluation.verified_rag import rag_answer_issues, render_rag_answer, quoted_reference_urls
from tools.graphrag.evaluation.source_oracle import DEFAULT_WIKI, load_source_oracle
from tools.rag.evaluation.rag_metrics import expected_phrase_matches
from tools.runtime.report_files import report_output, resolve_report_input
@ -86,7 +86,8 @@ def evaluate_case(case: dict, payload: dict, delivered: dict | None = None) -> d
issues.append('delivered_text_differs_from_verified_renderer')
if delivered.get('claims', []) != parsed.get('claims', []):
issues.append('delivered_claims_differ_from_verified_selection')
if accepted and not citation_check(delivered, source_urls)['citations_ok']:
references = quoted_reference_urls(parsed or {}, context)
if accepted and not citation_check(delivered, source_urls | references)['citations_ok']:
issues.append('delivered_unsupported_citation')
claims = delivered.get('claims', [])
if accepted:
@ -150,6 +151,7 @@ def main():
parser.add_argument('--dataset', type=Path, default=DATASET)
parser.add_argument('--backend', default='http://127.0.0.1:8000')
parser.add_argument('--output', type=Path)
parser.add_argument('--no-save', action='store_true', help='Print results only; create no reports or logs')
args = parser.parse_args()
load_dotenv(ROOT / '.env', override=False)
dataset = json.loads(args.dataset.read_text())
@ -157,7 +159,9 @@ def main():
oracle = load_source_oracle()
audit = {c['id']: source_audit(c, oracle) for c in dataset['cases']}
assert not any(audit.values()), 'Expected answer/source audit failed'
output = report_output(args.output, 'rag_verified')
if args.no_save and args.output:
parser.error('--no-save cannot be combined with --output')
output = None if args.no_save else report_output(args.output, 'rag_verified')
files = [Path(__file__), ROOT/'tools/answers/answer_delivery.py', ROOT/'app/routes.py',
*[(ROOT/'tools/answer_evaluation'/name) for name in ['verified_rag.py','rag_evidence.py','context_policy.py']],
ROOT/'tools/rag/rag_utils.py', ROOT/'tools/rag/llm_context.py']
@ -167,6 +171,7 @@ def main():
'code_hashes': {str(p.relative_to(ROOT)): digest(p) for p in files},
'model_calls': args.mode == 'deliver', 'backend': args.backend,
'model': os.getenv('ANSWER_MODEL', 'model120-fast') if args.mode == 'deliver' else None}
if output:
output.with_suffix('.declaration.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2)+'\n')
key = os.getenv('SEARCH_API_KEY', '')
saved = None
@ -207,11 +212,17 @@ def main():
for future in as_completed(futures):
rows.append(future.result())
result = {'configuration': metadata, 'summary': summarize(rows), 'results': sorted(rows, key=lambda r: r['id'])}
if output:
output.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
print(f'{len(rows)}/{len(dataset["cases"])} {rows[-1]["id"]}: {rows[-1].get("issues", "retrieved")}', flush=True)
assert metadata['code_hashes'] == {name: digest(ROOT/name) for name in metadata['code_hashes']}, 'Code changed during measurement'
print(json.dumps(result['summary'], ensure_ascii=False))
if output:
print('Report: '+str(output))
if args.no_save:
print(json.dumps({'failures':[{'id':r['id'],'issues':r['issues'],
'attempt_issues':[a.get('issues') for a in r.get('delivered',{}).get('attempts',[])]}
for r in result['results'] if not r['passed']]}, ensure_ascii=False))
if __name__ == '__main__':

3
tools/answer_evaluation/rag_evidence.py
View File

@ -239,7 +239,8 @@ def rag_evidence_context(question: str, payload: dict) -> dict:
requirements = {'people': requested, 'work_types': expected_types,
'academic': academic, 'title': 'nazov' in query or 'tema' in query,
'year': academic and ('rok' in query or 'porovnaj' in query),
'start_year': start_year}
'start_year': start_year,
'response_mode': 'claims' if academic or start_year else 'quotes'}
stated_years = re.findall(r'\b(?:v roku|z roku|pre rok|na rok|roku|rok)\s+(\d+)\b', query) if academic else []
requirements['years'] = ([int(value) for value in stated_years if len(value) == 4]
if not requirements['start_year'] else [])

55
tools/answer_evaluation/verified_rag.py
View File

@ -8,7 +8,7 @@ from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from .answer_format import Claim
from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized
from .context_policy import answer_context_issues, citation_check, allowed_urls, normalized, text_urls
from .rag_evidence import WORK_LABELS, rag_evidence_context
@ -41,9 +41,17 @@ Pri porovnávaní odpovedz za každú požadovanú osobu; rešpektuj requirement
start_year je len explicitný začiatok štúdia, nikdy rok z adresára.
Pri otázke na začiatok štúdia vyplň claims so start_year; pri rozlíšení rokov
uveď start_year aj year a work_type. Citácia ani nesúvisiaci výňatok údaj nenahrádzajú.
Pri všeobecnej otázke môžeš namiesto claims vybrať evidence_quotes: source_id a presný
doslovný výňatok, ktorý odpovedá na otázku. Výňatok nesmieš parafrázovať ani meniť čísla.
Rešpektuj requirements.response_mode. Pri response_mode=quotes MUSÍ byť claims=[]:
pri všeobecnej otázke, téme, technológii, zozname ľudí alebo stáži vyber iba
evidence_quotes: source_id a presný doslovný výňatok, ktorý odpovedá na otázku.
Nevypĺňaj pri nich názvy prác, roky, tagy ani iné nežiadané metadáta do claims.
Každý výňatok musí byť JEDEN súvislý úsek z konkrétneho zdroja vrátane Markdownu.
Nespájaj oddelené pasáže cez vynechané riadky; použi viac samostatných kratších quotes.
Vyber iba minimum potrebné na zodpovedanie otázky. Výňatok nesmieš parafrázovať ani meniť čísla.
Pri otázke na skutočne použitý nástroj odlišuj dosiahnutý stav od návrhu alebo úlohy.
Cituj iba source_urls zo sources, ktoré dokladajú vybrané údaje. Názov profilu nie je názov práce.
Odkaz, ktorý je priamo v doslovnom výňatku, môže zostať v ňom; nenahrádza však wiki
citáciu a nesmieš ho pridávať do source_urls. Odkazy ani obsah si nedomýšľaj.
Nedomýšľaj chýbajúce údaje. Ak podklady neobsahujú požadovaný fakt (napríklad finančný
rozpočet), vráť no_answer a prázdne claims/evidence_quotes. count nechaj null, ak nie je
dodaná úplná agregácia. Nepoužívaj vedomosti ani fakty, ktoré nie sú v podkladoch.'''
@ -59,6 +67,30 @@ def parse_rag_answer(text: str) -> dict:
return RagAnswer.model_validate(json.loads(text)).model_dump(exclude_none=True)
def quoted_reference_urls(answer: dict, context: dict) -> set[str]:
"""License references only inside an exact, cited quote of that source.
The structured source_urls still must contain the original wiki citations.
A link elsewhere in the corpus or in another source never licenses a quote.
"""
blocks = {row['source_id']: row for row in context['source_blocks']}
cited = set(answer.get('source_urls', []))
references = set()
from urllib.parse import urlsplit
for quote in answer.get('evidence_quotes', []):
source = blocks.get(quote['source_id'])
if (not source or source.get('source_url') not in cited or
' '.join(quote['quote'].split()) not in ' '.join(source['body'].split())):
continue
for url in text_urls(quote['quote']) & text_urls(source['body']):
try:
if urlsplit(url).scheme.lower() in {'', 'http', 'https'}:
references.add(url)
except ValueError:
continue
return references
def rag_answer_issues(answer: dict, context: dict) -> list[str]:
requirements = context['requirements']
claims = answer.get('claims', [])
@ -73,6 +105,8 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
issues.append('evidence_on_nonpositive_answer')
if answer['status'] == 'ok' and not claims and not quotes and answer.get('count') is None:
issues.append('missing_required_evidence')
if requirements.get('response_mode') == 'quotes' and claims:
issues.append('unrequested_structured_claims')
requested = requirements['people']
types = requirements['work_types']
for claim in claims:
@ -130,12 +164,14 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]:
not any(source.get('document_path', '').startswith(row['document_path'].removesuffix('README.md'))
for row in context['source_blocks'] if row.get('title') in requested)):
issues.append('unrequested_quote_person')
identity_question = bool(re.search(r'\b(?:url|odkaz\w*|dokument\w*)\b', normalized(context.get('question', ''))))
identity_question = bool(re.search(r'\b(?:url|odkaz\w*|dokument\w*|zaznam\w*|tag\w*|kategor\w*|autor\w*|klucov\w*)\b', normalized(context.get('question', ''))))
if (not identity_question and
' '.join(quote['quote'].split()) not in ' '.join(source['body'].split())):
issues.append('metadata_quote_does_not_answer_fact')
# A quoted embedded URL is not automatically an allowed citation.
if not citation_check({'answer': quote['quote'], 'source_urls': []}, allowed_urls(context))['citations_ok']:
# Source references may occur inside a verified literal quote, while
# evidence citations themselves remain restricted to the wiki sources.
references = quoted_reference_urls({**answer, 'evidence_quotes': [quote]}, context)
if not citation_check({'answer': quote['quote'], 'source_urls': []}, allowed_urls(context) | references)['citations_ok']:
issues.append('unsupported_quote_citation')
if not citation_check(answer, allowed_urls(context))['citations_ok']:
issues.append('unsupported_citation')
@ -186,6 +222,8 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict:
'truncation_reasons', 'index_version', 'sources', 'facts', 'requirements']}
if not context['requirements']['academic']:
sent['context'] = payload.get('context', '')
if context['requirements']['response_mode'] == 'quotes':
sent.pop('facts')
messages = [{'role': 'system', 'content': PROMPT + '\nSchéma: ' + json.dumps(RagAnswer.model_json_schema(), ensure_ascii=False)},
{'role': 'user', 'content': json.dumps({'question': question, 'context': sent}, ensure_ascii=False)}]
attempts = []
@ -215,7 +253,10 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict:
if number == 0:
messages += [{'role': 'assistant', 'content': record.get('raw_response', '')},
{'role': 'user', 'content': 'Oprav jediný JSON podľa pôvodných podkladov. Porušenia: ' + ', '.join(issues)
+ '. Odpoveď answer nechaj prázdnu. Nevymýšľaj údaje; osoba, typ, názov a rok musia patriť jednému záznamu.'}]
+ '. Odpoveď answer nechaj prázdnu. Nevymýšľaj údaje; osoba, typ, názov a rok musia patriť jednému záznamu.'
+ (' Pri tejto všeobecnej otázke nastav claims=[] a vyber iba krátke doslovné evidence_quotes. '
'Každý quote musí byť samostatný súvislý úsek, nesmie preskočiť riadky ani meniť Markdown. '
'Nepotrebné citáty a metadáta vynechaj.' if context['requirements']['response_mode'] == 'quotes' else '')}]
return {'status': 'error', 'reason': 'answer_rejected', 'accepted_for_delivery': False,
'answer': 'Odpoveď neprešla kontrolou podkladov; neoverené údaje sa nezobrazia.',
'attempts': attempts, 'index_version': context['index_version']}

14
tools/rag/rag_query_evidence.py
View File

@ -32,6 +32,8 @@ FOCUS_LINE_RADIUS = 7
MAX_FOCUS_CHARACTERS = 2200
CANONICAL_TOKEN_ALIASES = {
"aplikacia": "application", "aplikaciu": "application", "aplikacie": "application",
"aplikacii": "application", "application": "application", "applications": "application",
"anotacia": "annotation",
"anotacii": "annotation",
"anotacny": "annotation",
@ -540,6 +542,8 @@ def evidence_score(
def _line_score(
query: str,
line: str,
*,
ignored_tokens: frozenset[str] = frozenset(),
) -> float:
normalized_line = (
normalize_for_compare(
@ -550,9 +554,7 @@ def _line_score(
score = (
5.0
* count_token_matches(
query_content_tokens(
query
),
[token for token in query_content_tokens(query) if token not in ignored_tokens],
line,
)
)
@ -579,12 +581,18 @@ def build_focus_excerpt(
return ""
lines = clean_text.splitlines()
# The generated document header is repeated in every chunk. Its owner's
# name identifies the document, but must not hide a later factual passage
# when choosing an excerpt within that already selected document.
owner = re.search(r'^Dokument:\s*(.+)$', clean_text, re.M)
ignored = frozenset(normalized_tokens(owner[1])) if owner else frozenset()
scored = [
(
_line_score(
query,
line,
ignored_tokens=ignored,
),
index,
)

8
tools/rag/tests/test_rag_query_evidence.py
View File

@ -10,6 +10,14 @@ from tools.rag.rag_query_evidence import (
)
def test_profile_header_does_not_hide_later_application_evidence():
text = ('Dokument: Anna Nová\n\nAnna Nová\n' + '\n'.join(['Všeobecná poznámka.']*25) +
'\nState:\n- Proposed own Flask application\n- Created a FastAPI application with a data model.\n')
excerpt = build_focus_excerpt('Aký webový framework použila Anna Nová pre aplikáciu?',text)
assert 'Created a FastAPI application' in excerpt
assert 'Proposed own Flask application' in excerpt
def make_db(
tmp_path: Path,
) -> Path: